diff --git a/.claude/rules/data-sync-storage.md b/.claude/rules/data-sync-storage.md index 76e7269ff1..9bb200a6ab 100644 --- a/.claude/rules/data-sync-storage.md +++ b/.claude/rules/data-sync-storage.md @@ -77,7 +77,7 @@ paths: - "apple/TablePro/Core/RoutineDebugging/RoutineDebugOptionStore.swift" - "apple/TablePro/Core/RowGeneration/RowGenerationModel.swift" - "apple/TablePro/Core/SSH/HostKeyStore.swift" - - "apple/TablePro/Core/Services/Export/ImportColumnMatcher.swift" + - "apple/TablePro/Core/Transfer/Shared/ImportColumnMatcher.swift" - "apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift" - "apple/TablePro/Core/Services/TeamLibrary/TeamLibraryState.swift" - "apple/TablePro/Core/Services/Account/*Store.swift" diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index e32362d417..f1cf998972 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -51,6 +51,7 @@ let package = Package( .library(name: "TableProR2SQLCore", targets: ["TableProR2SQLCore"]), .library(name: "TableProConnectionLibrary", targets: ["TableProConnectionLibrary"]), .library(name: "TableProSQLGrammar", targets: ["TableProSQLGrammar"]), + .library(name: "TableProSQLFile", targets: ["TableProSQLFile"]), .library(name: "TableProSQLQuoting", targets: ["TableProSQLQuoting"]), .library(name: "TableProProcess", targets: ["TableProProcess"]), .library(name: "TableProSQLVocabulary", targets: ["TableProSQLVocabulary"]), @@ -268,6 +269,13 @@ let package = Package( dependencies: ["TableProSQLQuoting"], path: "Sources/TableProSQLGrammar" ), + // Reads a .sql file for both apps' imports, so never TableProPluginKit. + .target( + name: "TableProSQLFile", + dependencies: ["TableProSQLGrammar", "TableProTabularIO"], + path: "Sources/TableProSQLFile", + swiftSettings: pluginCoreSettings + ), // Linked by PluginKit, plugin bundles and both apps, so it depends on nothing. .target( name: "TableProSQLQuoting", @@ -527,6 +535,12 @@ let package = Package( dependencies: ["TableProSQLGrammar"], path: "Tests/TableProSQLGrammarTests" ), + .testTarget( + name: "TableProSQLFileTests", + dependencies: ["TableProSQLFile", "TableProSQLGrammar", "TableProTabularIO"], + path: "Tests/TableProSQLFileTests", + swiftSettings: pluginCoreSettings + ), .testTarget( name: "TableProSQLQuotingTests", dependencies: ["TableProSQLQuoting", "TableProSQLGrammar", "CSQLite"], diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraObjectQueries.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraObjectQueries.swift index 1b0525285a..1d795aaf39 100644 --- a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraObjectQueries.swift +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraObjectQueries.swift @@ -154,4 +154,9 @@ public enum CassandraObjectQueries { USING '\(escapeLiteral(className))'; """ } + + // A CQL trigger is named per table, so the drop needs its ON clause. + public static func dropTriggerStatement(name: String, keyspace: String, table: String) -> String { + "DROP TRIGGER IF EXISTS \(quote(name)) ON \(quote(keyspace)).\(quote(table))" + } } diff --git a/apple/Packages/TableProCore/Sources/TableProDatabase/ConnectionManager.swift b/apple/Packages/TableProCore/Sources/TableProDatabase/ConnectionManager.swift index 905b25e6cf..04a36a844a 100644 --- a/apple/Packages/TableProCore/Sources/TableProDatabase/ConnectionManager.swift +++ b/apple/Packages/TableProCore/Sources/TableProDatabase/ConnectionManager.swift @@ -21,6 +21,9 @@ public final class ConnectionManager: @unchecked Sendable { private var dials: [UUID: SessionDial] = [:] private var catalogs: [UUID: CatalogConnection] = [:] private var catalogHolds: [UUID: Int] = [:] + private var transferDrivers: [UUID: [ObjectIdentifier: any DatabaseDriver]] = [:] + private var transferHolds: [UUID: Int] = [:] + private var transferHoldObservers: [@Sendable (UUID) -> Void] = [] private let now: @Sendable () -> Date // The Mac's metadata pool values: an idle server connection is not held for good, and a failure is tried again. @@ -235,13 +238,139 @@ public final class ConnectionManager: @unchecked Sendable { return ids } + // A connection a transfer holds keeps its file: the transfer's own background task keeps the process alive. private func suspensionBlockingIds() -> [UUID] { lock.lock() defer { lock.unlock() } - let connected = sessions.filter { $0.value.driver.holdsSuspensionBlockingResource }.map(\.key) + let connected = sessions.filter { id, session in + session.driver.holdsSuspensionBlockingResource && transferHolds[id] == nil + }.map(\.key) return Array(blockingTeardowns.union(connected)) } + // MARK: - Transfers + + public func holdForTransfer(_ connectionId: UUID) -> TransferHold { + lock.lock() + transferHolds[connectionId, default: 0] += 1 + lock.unlock() + return TransferHold { [weak self] in self?.releaseTransferHold(connectionId) } + } + + // Called once the last hold on a connection ends, so a pass that skipped it can release it now. + public func observeTransferHoldsEnded(_ observer: @escaping @Sendable (UUID) -> Void) { + lock.lock() + defer { lock.unlock() } + transferHoldObservers.append(observer) + } + + private func releaseTransferHold(_ connectionId: UUID) { + lock.lock() + let left = transferHolds[connectionId, default: 1] - 1 + transferHolds[connectionId] = left > 0 ? left : nil + let observers = left > 0 ? [] : transferHoldObservers + lock.unlock() + for observer in observers { + observer(connectionId) + } + } + + // A driver of the transfer's own, so a long read never shares the session the grid streams on. It is the session's + // driver itself when the engine has no second connection to give now (a SQLite file with a transaction open, an + // in-memory database). `closeDedicatedDriver` ends it; a disconnect or a catalog hold ends it first. + public func openDedicatedDriver(for connectionId: UUID, database: String?) async throws -> any DatabaseDriver { + guard let (session, dial) = sessionAndDial(for: connectionId) else { throw ConnectionError.notConnected } + guard !holdsCatalogs(connectionId) else { throw TransferConnectionError.connectionHeld } + let driver: any DatabaseDriver + if let owner = session.driver as? DedicatedTransferConnection { + driver = try await owner.openTransferDriver() + guard driver !== session.driver else { return driver } + } else { + driver = try await Self.openTransferDriver( + dial: dial, database: database ?? "", factory: driverFactory, secureStore: secureStore + ) + } + guard registerTransferDriver(driver, for: connectionId, session: session) else { + try? await driver.disconnect() + throw holdsCatalogs(connectionId) ? TransferConnectionError.connectionHeld : ConnectionError.notConnected + } + return driver + } + + // Leaves the session's own driver alone. + public func closeDedicatedDriver(_ driver: any DatabaseDriver, for connectionId: UUID) async { + guard takeTransferDriver(driver, for: connectionId) else { return } + try? await driver.disconnect() + } + + private func sessionAndDial(for connectionId: UUID) -> (ConnectionSession, SessionDial)? { + lock.lock() + defer { lock.unlock() } + guard let session = sessions[connectionId], let dial = dials[connectionId] else { return nil } + return (session, dial) + } + + private func holdsCatalogs(_ connectionId: UUID) -> Bool { + lock.lock() + defer { lock.unlock() } + return catalogHolds[connectionId] != nil + } + + // Refused once the session it was opened for has gone or a hold began while it opened. + private func registerTransferDriver( + _ driver: any DatabaseDriver, + for connectionId: UUID, + session: ConnectionSession + ) -> Bool { + lock.lock() + defer { lock.unlock() } + guard catalogHolds[connectionId] == nil, let current = sessions[connectionId], + current.driver === session.driver else { return false } + transferDrivers[connectionId, default: [:]][ObjectIdentifier(driver)] = driver + return true + } + + private func takeTransferDriver(_ driver: any DatabaseDriver, for connectionId: UUID) -> Bool { + lock.lock() + defer { lock.unlock() } + guard transferDrivers[connectionId]?.removeValue(forKey: ObjectIdentifier(driver)) != nil else { return false } + if transferDrivers[connectionId]?.isEmpty == true { + transferDrivers.removeValue(forKey: connectionId) + } + return true + } + + private func takeTransferDrivers(for connectionId: UUID) -> [any DatabaseDriver] { + lock.lock() + defer { lock.unlock() } + return transferDrivers.removeValue(forKey: connectionId).map { Array($0.values) } ?? [] + } + + // A transfer reads as long as it takes: no catalog preparation (SQL Server's LOCK_TIMEOUT) and no query timeout, + // as the Mac lifts the timeout for an export. + private static func openTransferDriver( + dial: SessionDial, + database: String, + factory: DriverFactory, + secureStore: SecureStore + ) async throws -> any DatabaseDriver { + var target = dial.effective + if !database.isEmpty { + target.database = database + } + target.queryTimeoutSeconds = nil + let stored = try secureStore.retrieve(forKey: passwordKey(for: dial.saved.id)) + let password = try await factory.resolvePassword(for: dial.saved, stored: stored, secureStore: secureStore) + let driver = try factory.createDriver( + for: target, + password: password, + preTunnelHost: dial.preTunnelHost, + refreshPassword: passwordRefresh(for: dial.saved, stored: stored, factory: factory, secureStore: secureStore) + ) + try await driver.connect() + return driver + } + /// Closes the tunnel before the driver, because a driver blocked reading through a tunnel the server /// dropped only returns once its socket's peer closes; the reverse order queues `disconnect()` behind /// a read that never completes. The lock is held past the `teardowns` write, so the task's own @@ -262,10 +391,15 @@ public final class ConnectionManager: @unchecked Sendable { if let catalog { Task { await catalog.close() } } + let transfers = transferDrivers.removeValue(forKey: connectionId).map { Array($0.values) } ?? [] Task { [weak self] in if let sshProvider { try? await sshProvider.closeTunnel(for: connectionId) } + // Before the session: a DuckDB transfer connection reads the database the session's close ends. + for transfer in transfers { + try? await transfer.disconnect() + } try? await session.driver.disconnect() self?.retireTeardown(teardown.id, for: connectionId) teardown.finish() @@ -345,8 +479,12 @@ public final class ConnectionManager: @unchecked Sendable { // SQL Server cannot drop or take single-user a database another connection sits in, so a run that may do either // closes the catalog connection first and keeps it closed until the run ends. + // A transfer's own connection sits in the database too, so the hold ends it. public func holdCatalogConnections(for connectionId: UUID) async { await takeCatalogForHold(for: connectionId)?.close() + for transfer in takeTransferDrivers(for: connectionId) { + try? await transfer.disconnect() + } } public func releaseCatalogConnections(for connectionId: UUID) { @@ -495,6 +633,46 @@ public final class ConnectionManager: @unchecked Sendable { } } +// A session driver whose engine gives no second connection through the factory makes the transfer's own: a SQLite +// reader beside a WAL session or over a snapshot, a DuckDB connection on the open database. It answers with itself +// when it has none to give now. +public protocol DedicatedTransferConnection: DatabaseDriver { + func openTransferDriver() async throws -> any DatabaseDriver +} + +// Keeps the connection's file open through a background release while a transfer runs. +public final class TransferHold: @unchecked Sendable { + private let lock = NSLock() + private var onRelease: (@Sendable () -> Void)? + + init(onRelease: @escaping @Sendable () -> Void) { + self.onRelease = onRelease + } + + deinit { + release() + } + + public func release() { + lock.lock() + let release = onRelease + onRelease = nil + lock.unlock() + release?() + } +} + +public enum TransferConnectionError: Error, Equatable, Sendable, LocalizedError { + case connectionHeld + + public var errorDescription: String? { + switch self { + case .connectionHeld: + return String(localized: "Another task needs this connection alone. Try again when it ends.") + } + } +} + /// One teardown, and the waiters parked on it. /// /// `await task.value` on a `Task<_, Never>` ignores the awaiting task's cancellation entirely: measured, diff --git a/apple/Packages/TableProCore/Sources/TableProDatabase/DatabaseDriver.swift b/apple/Packages/TableProCore/Sources/TableProDatabase/DatabaseDriver.swift index 5fe26dfd00..58a4ef2690 100644 --- a/apple/Packages/TableProCore/Sources/TableProDatabase/DatabaseDriver.swift +++ b/apple/Packages/TableProCore/Sources/TableProDatabase/DatabaseDriver.swift @@ -60,6 +60,15 @@ public protocol DatabaseDriver: AnyObject, Sendable { var settledLexicalFacts: SQLSessionLexicalFacts? { get } func escapeStringLiteral(_ value: String) -> String + + // One statement with bound values, sent the way this engine's macOS driver sends them. The default refuses: + // a literal fallback would quote binary and typed values differently from the Mac. + func executeParameterized(query: String, parameters: [PluginCellValue], typeNames: [String?]?) async throws + -> QueryResult + + // The statements that turn the engine's foreign key checks off or back on for this session, empty where it + // has none. + func foreignKeyCheckStatements(enabled: Bool) -> [String] } public extension DatabaseDriver { @@ -105,4 +114,11 @@ public extension DatabaseDriver { func executeStreaming(query: String, options: StreamOptions = .default) -> AsyncThrowingStream { QueryResultStreaming.stream(options: options) { try await self.execute(query: query) } } + + func executeParameterized(query: String, parameters: [PluginCellValue], typeNames: [String?]?) async throws + -> QueryResult { + throw DatabaseDriverCapabilityError.unsupported(.parameterizedStatements) + } + + func foreignKeyCheckStatements(enabled: Bool) -> [String] { [] } } diff --git a/apple/Packages/TableProCore/Sources/TableProDatabase/TransferCapabilities.swift b/apple/Packages/TableProCore/Sources/TableProDatabase/TransferCapabilities.swift new file mode 100644 index 0000000000..22c4a1ad05 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDatabase/TransferCapabilities.swift @@ -0,0 +1,39 @@ +import Foundation +import TableProPluginKit + +public enum DatabaseDriverCapabilityError: Error, Equatable, Sendable, LocalizedError { + public enum Capability: Sendable, Equatable { + case parameterizedStatements + case catalogQueries + case exportRows + } + + case unsupported(Capability) + + public var errorDescription: String? { + switch self { + case .unsupported(.parameterizedStatements): + return String(localized: "This connection cannot send statements with bound values.") + case .unsupported(.catalogQueries): + return String(localized: "This connection cannot read its catalog for an export.") + case .unsupported(.exportRows): + return String(localized: "This connection cannot stream rows to a file.") + } + } +} + +// Raw catalog reads for the shared export catalogs: every row, bytes kept, no display truncation. +public protocol CatalogQuerying: DatabaseDriver { + func catalogQuery(_ sql: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult +} + +public extension CatalogQuerying { + func catalogQuery(_ sql: String) async throws -> PluginQueryResult { + try await catalogQuery(sql, parameters: []) + } +} + +// Rows for a file export: pulled one batch at a time, bytes kept, no row cap, and a cancel ends the server read. +public protocol ExportRowStreaming: DatabaseDriver { + func exportRows(query: String, readOnly: Bool) -> AsyncThrowingStream +} diff --git a/apple/Packages/TableProCore/Sources/TableProDuckDBCore/DuckDBObjectQueries.swift b/apple/Packages/TableProCore/Sources/TableProDuckDBCore/DuckDBObjectQueries.swift new file mode 100644 index 0000000000..34c4404556 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDuckDBCore/DuckDBObjectQueries.swift @@ -0,0 +1,57 @@ +import Foundation +import TableProSQLQuoting + +// DuckDB has macros where other engines have functions, and no procedures or triggers. +public enum DuckDBObjectQueries { + // Macros only, since duckdb_functions() also lists every built-in. On the catalog, since it spans every attached one + // and temp, whose overloads would merge into one statement. One row per parameter, read as a list element: the + // list's text form quotes an element only when it holds certain characters. WITH ORDINALITY, since + // generate_subscripts needs the core_functions extension the bundled library does not link. Overloads sort by + // their parameters, so a rebuilt statement lists them in the same order every time. + public static func macroList(catalog: String, schema: String?) -> String { + let schemaPredicate = schema.map { " AND schema_name = \(SQLQuoting.stringLiteral($0))" } ?? "" + return """ + SELECT f.macro_index, f.schema_name, f.function_name, f.macro_definition, f.function_type, p.parameter, + f.parameter_types[p.position] AS parameter_type + FROM ( + SELECT row_number() OVER ( + ORDER BY schema_name, function_name, function_type, parameters, parameter_types + ) AS macro_index, + schema_name, function_name, macro_definition, function_type, parameters, parameter_types + FROM duckdb_functions() + WHERE database_name = \(SQLQuoting.stringLiteral(catalog))\(schemaPredicate) + AND function_type IN ('macro', 'table_macro') AND NOT internal + ) AS f + LEFT JOIN LATERAL unnest(f.parameters) WITH ORDINALITY AS p(parameter, position) ON true + ORDER BY f.macro_index, p.position + """ + } + + // An empty or missing schema leaves the name to resolve in the session's own. + public static func qualifiedName(schema: String?, name: String) -> String { + [schema, name].compactMap { $0 }.filter { !$0.isEmpty } + .map { SQLQuoting.identifier($0) } + .joined(separator: ".") + } + + // DuckDB has no `DROP ROUTINE`. Measured on 2.0.0 and 1.5.4: `DROP MACRO` drops a table macro too, unless a scalar + // macro shares its name, which it drops instead; `DROP MACRO TABLE` drops only a table macro. + public static func dropMacro(name: String, schema: String?, isTableMacro: Bool) -> String { + let keyword = isTableMacro ? "DROP MACRO TABLE" : "DROP MACRO" + return "\(keyword) IF EXISTS \(qualifiedName(schema: schema, name: name))" + } + + static let tableMacroSuffix = " AS TABLE" + + // A scalar and a table macro may share a name and parameters, and only the signature reaches the drop, so a table + // macro's names its kind the way CREATE MACRO spells it. + public static func macroSignature(parameterNames: [String], isTableMacro: Bool) -> String { + let parameters = "(\(parameterNames.joined(separator: ", ")))" + return isTableMacro ? parameters + tableMacroSuffix : parameters + } + + // A scalar macro's signature ends with its closing parenthesis. + public static func namesTableMacro(_ signature: String?) -> Bool { + signature?.hasSuffix(tableMacroSuffix) ?? false + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProDuckDBCore/DuckDBViewDefinition.swift b/apple/Packages/TableProCore/Sources/TableProDuckDBCore/DuckDBViewDefinition.swift new file mode 100644 index 0000000000..735e74936e --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDuckDBCore/DuckDBViewDefinition.swift @@ -0,0 +1,17 @@ +import Foundation + +public enum DuckDBViewDefinition { + // DuckDB stores the `CREATE VIEW` the view was made with, which fails when run again, so a leading one becomes + // `CREATE OR REPLACE VIEW`. Anything else comes back untouched. + public static func makeReplaceable(_ sql: String) -> String { + let trimmed = sql.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return sql } + + let prefix = "CREATE VIEW" + guard trimmed.count >= prefix.count else { return sql } + let head = String(trimmed.prefix(prefix.count)) + guard head.uppercased() == prefix else { return sql } + + return "CREATE OR REPLACE VIEW" + trimmed.dropFirst(prefix.count) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProGzip/GzipDeflater.swift b/apple/Packages/TableProCore/Sources/TableProGzip/GzipDeflater.swift new file mode 100644 index 0000000000..4d36359c83 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProGzip/GzipDeflater.swift @@ -0,0 +1,92 @@ +import Foundation +import zlib + +// zlib's gzip mode (window bits + 16) writes the header and the CRC and length trailer that gzip -d checks. +public final class GzipDeflater { + private let stream: UnsafeMutablePointer + private let chunk: UnsafeMutableBufferPointer + private var finished = false + + private static let chunkSize = 256 * 1_024 + + public init() throws(GzipError) { + stream = .allocate(capacity: 1) + stream.initialize(to: z_stream()) + chunk = .allocate(capacity: Self.chunkSize) + let status = deflateInit2_( + stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 16 + MAX_WBITS, 8, Z_DEFAULT_STRATEGY, + ZLIB_VERSION, Int32(MemoryLayout.size) + ) + guard status == Z_OK else { + chunk.deallocate() + stream.deinitialize(count: 1) + stream.deallocate() + throw .compressionFailed + } + } + + deinit { + deflateEnd(stream) + stream.deinitialize(count: 1) + stream.deallocate() + chunk.deallocate() + } + + public func deflate( + _ input: UnsafeRawBufferPointer, + into sink: (UnsafeRawBufferPointer) throws -> Void + ) throws { + guard !finished else { throw GzipError.compressionFailed } + guard let base = input.bindMemory(to: UInt8.self).baseAddress, !input.isEmpty else { return } + var cursor = 0 + while cursor < input.count { + let take = min(input.count - cursor, Int(UInt32.max)) + stream.pointee.next_in = UnsafeMutablePointer(mutating: base + cursor) + stream.pointee.avail_in = uInt(take) + try drain(flush: Z_NO_FLUSH, into: sink) + cursor += take + } + } + + public func finish(into sink: (UnsafeRawBufferPointer) throws -> Void) throws { + guard !finished else { return } + stream.pointee.next_in = nil + stream.pointee.avail_in = 0 + try drain(flush: Z_FINISH, into: sink) + finished = true + } + + // Runs deflate until it has taken all its input and, on Z_FINISH, written the trailer. + private func drain(flush: Int32, into sink: (UnsafeRawBufferPointer) throws -> Void) throws { + while true { + stream.pointee.next_out = chunk.baseAddress + stream.pointee.avail_out = uInt(chunk.count) + let status = zlib.deflate(stream, flush) + let produced = chunk.count - Int(stream.pointee.avail_out) + switch status { + case Z_OK, Z_BUF_ERROR, Z_STREAM_END: + break + default: + throw GzipError.compressionFailed + } + if produced > 0 { + try sink(UnsafeRawBufferPointer(start: chunk.baseAddress, count: produced)) + } + if flush == Z_FINISH { + if status == Z_STREAM_END { return } + guard produced > 0 else { throw GzipError.compressionFailed } + } else if stream.pointee.avail_in == 0, stream.pointee.avail_out > 0 { + return + } + } + } + + public static func compress(_ data: Data) throws -> Data { + var output = Data() + let deflater = try GzipDeflater() + let sink: (UnsafeRawBufferPointer) throws -> Void = { output.append($0.assumingMemoryBound(to: UInt8.self)) } + try data.withUnsafeBytes { try deflater.deflate($0, into: sink) } + try deflater.finish(into: sink) + return output + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift b/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift index d25cea91e8..0b51516c47 100644 --- a/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift +++ b/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift @@ -57,6 +57,58 @@ public enum GzipFile { } } + // Throws CancellationError once `isCancelled` answers true, checked before every chunk; the caller maps it. + public static func compress( + source: URL, + into output: FileHandle, + isCancelled: () -> Bool, + progress: ((_ bytesRead: Int64, _ totalBytes: Int64) -> Void)? = nil + ) throws { + let input = try FileHandle(forReadingFrom: source) + defer { try? input.close() } + var info = stat() + let total = fstat(input.fileDescriptor, &info) == 0 ? Int64(info.st_size) : 0 + let deflater = try GzipDeflater() + let sink: (UnsafeRawBufferPointer) throws -> Void = { try output.write(contentsOf: $0) } + var bytesRead: Int64 = 0 + progress?(0, total) + // Each read returns an autoreleased buffer, and this loop never suspends, so nothing drains them until it ends. + while let block = try autoreleasepool(invoking: { try input.read(upToCount: readSize) }), !block.isEmpty { + if isCancelled() { throw CancellationError() } + try block.withUnsafeBytes { try deflater.deflate($0, into: sink) } + bytesRead += Int64(block.count) + progress?(bytesRead, total) + } + if isCancelled() { throw CancellationError() } + try deflater.finish(into: sink) + } + + // The destination is removed on any failure, so a stopped or failed compression leaves nothing behind. + public static func compress( + source: URL, + to destination: URL, + isCancelled: () -> Bool, + progress: ((_ bytesRead: Int64, _ totalBytes: Int64) -> Void)? = nil + ) throws { + guard FileManager.default.createFile(atPath: destination.path(percentEncoded: false), contents: nil) else { + throw CocoaError(.fileWriteUnknown, userInfo: [NSFilePathErrorKey: destination.path(percentEncoded: false)]) + } + do { + let output = try FileHandle(forWritingTo: destination) + do { + try compress(source: source, into: output, isCancelled: isCancelled, progress: progress) + } catch { + try? output.close() + throw error + } + // A volume can report a deferred write failure only at close. + try output.close() + } catch { + try? FileManager.default.removeItem(at: destination) + throw error + } + } + private static func sizeLimit(_ limit: Int) -> DecompressionBudget { DecompressionBudget(limit: limit, ratioLimit: .max, grace: .max) } diff --git a/apple/Packages/TableProCore/Sources/TableProGzip/GzipInflater.swift b/apple/Packages/TableProCore/Sources/TableProGzip/GzipInflater.swift index 89f0237669..63568fb0bf 100644 --- a/apple/Packages/TableProCore/Sources/TableProGzip/GzipInflater.swift +++ b/apple/Packages/TableProCore/Sources/TableProGzip/GzipInflater.swift @@ -7,6 +7,7 @@ public enum GzipError: Error, Equatable, Sendable, LocalizedError { case truncated case expandsTooFar case needsMoreDiskSpace + case compressionFailed public var errorDescription: String? { switch self { @@ -18,6 +19,8 @@ public enum GzipError: Error, Equatable, Sendable, LocalizedError { return String(localized: "The compressed data expands to far more than it holds, so it was not read.") case .needsMoreDiskSpace: return String(localized: "The decompressed file needs more free disk space than this Mac has.") + case .compressionFailed: + return String(localized: "The data could not be compressed.") } } } diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDropStatements.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDropStatements.swift new file mode 100644 index 0000000000..688ffa9365 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDropStatements.swift @@ -0,0 +1,18 @@ +import Foundation + +// T-SQL has no DROP ROUTINE: a procedure, a function and a CLR aggregate each have their own verb. +public enum MSSQLDropStatements { + public static func routine(schema: String, name: String, isFunction: Bool, signature: String?) -> String { + let keyword = switch (isFunction, MSSQLRoutineSignature.namesAggregate(signature)) { + case (true, true): "AGGREGATE" + case (true, false): "FUNCTION" + case (false, _): "PROCEDURE" + } + return "DROP \(keyword) IF EXISTS \(MSSQLSchemaQueries.bracketed(schema: schema, table: name))" + } + + // A DML trigger lives in its table's schema and is named without the table. + public static func trigger(schema: String, name: String) -> String { + "DROP TRIGGER \(MSSQLSchemaQueries.bracketed(schema: schema, table: name))" + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLRoutineSignature.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLRoutineSignature.swift new file mode 100644 index 0000000000..067159bb2f --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLRoutineSignature.swift @@ -0,0 +1,22 @@ +import Foundation + +// A CLR aggregate (sys.objects.type AF) is listed as a function and PluginKit has no aggregate kind, so its +// signature names it: only the signature reaches the drop, which has to be DROP AGGREGATE. +public enum MSSQLRoutineSignature { + static let aggregateSuffix = " AGGREGATE" + + public static func listed(parameters: String, objectType: String) -> String { + let signature = "(\(parameters))" + return isAggregate(objectType: objectType) ? signature + aggregateSuffix : signature + } + + // sys.objects.type is char(2), so a code can arrive padded. + public static func isAggregate(objectType: String) -> Bool { + objectType.trimmingCharacters(in: .whitespaces).uppercased() == "AF" + } + + // Every other listed signature ends with its closing parenthesis. + public static func namesAggregate(_ signature: String?) -> Bool { + signature?.hasSuffix(aggregateSuffix) ?? false + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift index 1c23bef903..44335cbf25 100644 --- a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift @@ -609,6 +609,16 @@ public enum MSSQLSchemaQueries { return "SELECT CONVERT(nvarchar(128), DATABASEPROPERTYEX(\(database), 'Collation'))" } + // The heap or clustered index partitions only, so a row is not counted once per index. + public static func approximateRowCount(schema: String, table: String) -> String { + let object = MSSQLStringLiteral.quoted(bracketed(schema: schema, table: table)) + return """ + SELECT SUM(p.rows) + FROM sys.partitions p + WHERE p.object_id = OBJECT_ID(\(object)) AND p.index_id IN (0, 1) + """ + } + private static func catalogObject(schema: String, table: String) -> (catalog: String, object: String) { if namesTemporaryTable(table) { return ("tempdb.", temporaryObjectID(table)) diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLTypeDefinition.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLTypeDefinition.swift similarity index 76% rename from apple/Plugins/MSSQLDriverPlugin/MSSQLTypeDefinition.swift rename to apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLTypeDefinition.swift index 5399457f57..e91794d5de 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLTypeDefinition.swift +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLTypeDefinition.swift @@ -1,18 +1,7 @@ -// -// MSSQLTypeDefinition.swift -// MSSQLDriverPlugin -// -// Rebuilds a CREATE TYPE statement from the catalog. Pure, so it is testable without a server. -// - import Foundation -import TableProMSSQLCore -import TableProPluginKit import TableProSQLQuoting -/// A type has no `sys.sql_modules` row on any engine path, so unlike a procedure there is no stored -/// text to read back. Every statement here is synthesized from the catalog, and both shapes were -/// executed against SQL Server 2022 to prove they parse. +// A type has no `sys.sql_modules` row, so every statement here is rebuilt from the catalog. public enum MSSQLTypeDefinition { public struct Column: Sendable, Equatable { public let name: String @@ -52,8 +41,6 @@ public enum MSSQLTypeDefinition { } } - /// Keys stay structured rather than comma-joined: a column name may legally contain a comma, - /// and only a per-key flag can carry `DESC`. public struct Index: Sendable, Equatable { public let name: String? public let isPrimaryKey: Bool @@ -85,6 +72,62 @@ public enum MSSQLTypeDefinition { SQLQuoting.identifier(identifier, quote: .bracket) } + // Rows are what `MSSQLTypeQueries.tableTypeColumns` selects. + public static func columns(fromRows rows: [[String?]]) -> [Column] { + rows.compactMap { row -> Column? in + guard let name = cell(row, 0) else { return nil } + return Column( + name: name, + type: cell(row, 1), + isNullable: cell(row, 2) == "1", + identitySpec: cell(row, 3), + computedDefinition: cell(row, 4), + defaultDefinition: cell(row, 5), + collation: cell(row, 6) + ) + } + } + + // Rows are what `MSSQLTypeQueries.tableTypeIndexes` selects, one per key column, folded by `index_id`. + public static func indexes(fromRows rows: [[String?]]) -> [Index] { + var order: [String] = [] + var byId: [String: Index] = [:] + for row in rows { + guard let indexId = cell(row, 6), let column = cell(row, 4) else { continue } + let key = IndexKey(column: column, isDescending: cell(row, 5) == "1") + if let existing = byId[indexId] { + byId[indexId] = Index( + name: existing.name, + isPrimaryKey: existing.isPrimaryKey, + isUnique: existing.isUnique, + typeDescription: existing.typeDescription, + keys: existing.keys + [key], + bucketCount: existing.bucketCount + ) + continue + } + order.append(indexId) + byId[indexId] = Index( + name: cell(row, 0), + isPrimaryKey: cell(row, 1) == "1", + isUnique: cell(row, 2) == "1", + typeDescription: cell(row, 3), + keys: [key], + bucketCount: cell(row, 7).flatMap(Int.init) ?? 0 + ) + } + return order.compactMap { byId[$0] } + } + + // Rows are what `MSSQLTypeQueries.tableTypeCheckConstraints` selects. + public static func checkConstraints(fromRows rows: [[String?]]) -> [String] { + rows.compactMap { cell($0, 0) } + } + + private static func cell(_ row: [String?], _ index: Int) -> String? { + row.indices.contains(index) ? row[index] : nil + } + // Nullability is always spelled out: the default differs with `ANSI_NULL_DFLT_ON`. public static func aliasStatement(schema: String, name: String, baseType: String?, isNullable: Bool) throws -> String { if let baseType, !MSSQLDefinitionText.isColumnType(baseType) { @@ -94,10 +137,7 @@ public enum MSSQLTypeDefinition { return "CREATE TYPE \(bracketed(schema)).\(bracketed(name))\(base) \(isNullable ? "NULL" : "NOT NULL");" } - /// A CLR type's body is compiled into a .NET assembly, so this names the assembly rather than - /// pretending to a definition. The managed class is `sys.assembly_types.assembly_class` and is - /// free to differ from the SQL type's name; substituting the SQL name produced an - /// `EXTERNAL NAME` pointing at a class that does not exist. + // The managed class may differ from the SQL type's name, so it comes from `sys.assembly_types`. public static func clrStatement( schema: String, name: String, @@ -145,8 +185,7 @@ public enum MSSQLTypeDefinition { return "CREATE TYPE \(bracketed(schema)).\(bracketed(name)) AS TABLE (\n\(body)\(tail)" } - /// SQL Server defaults a primary key to CLUSTERED, so a NONCLUSTERED one replays with a - /// different layout, and a clustered secondary index then makes the replay fail outright. + // A primary key defaults to CLUSTERED, so a NONCLUSTERED one has to say so. private static func clusteringClause(_ typeDescription: String?) -> String { guard let description = typeDescription?.uppercased(), description == "CLUSTERED" || description == "NONCLUSTERED" diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLTypeQueries.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLTypeQueries.swift similarity index 78% rename from apple/Plugins/MSSQLDriverPlugin/MSSQLTypeQueries.swift rename to apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLTypeQueries.swift index d465be5408..063c7ef9ab 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLTypeQueries.swift +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLTypeQueries.swift @@ -1,26 +1,14 @@ -// -// MSSQLTypeQueries.swift -// MSSQLDriverPlugin -// -// Catalog SQL for user-defined types. Pure, so it is testable without a server. -// - import Foundation -import TableProMSSQLCore public enum MSSQLTypeQueries { - /// SQL Server has three shapes of user-defined type and `sys.types` tells them apart with two - /// flags rather than a kind column. + // `sys.types` tells the three shapes apart with two flags rather than a kind column. public enum Kind: String { case alias = "ALIAS" case table = "TABLE" case clr = "CLR" } - /// Rebuilds the base type the way the user wrote it. `max_length` is in BYTES, so an - /// `nvarchar(320)` reports 640 and has to be halved, and -1 means MAX. Measured against a - /// server holding `nvarchar(320)`, `nvarchar(max)`, `varchar(32)` and `decimal(18,4)`: every - /// spelling came back matching the original `CREATE TYPE`. + // `max_length` is in bytes, so an `nvarchar(320)` reports 640, and -1 means MAX. private static let baseTypeSpelling = """ CASE WHEN t.is_table_type = 1 OR t.is_assembly_type = 1 THEN NULL WHEN bt.name IN ('nvarchar', 'nchar') @@ -36,8 +24,7 @@ public enum MSSQLTypeQueries { ELSE bt.name END """ - /// The identity is `user_type_id`, which is stable for the life of the type and survives a - /// rename, so a re-fetch never keys on the name or the kind. + // The identity is `user_type_id`, which survives a rename. public static func userDefinedTypeList(schema: String) -> String { """ SELECT @@ -64,8 +51,7 @@ public enum MSSQLTypeQueries { """ } - /// A table type's columns, in declaration order. `sys.table_types.type_table_object_id` is the - /// hidden table behind the type, which is what `sys.columns` is keyed on. + // `type_table_object_id` is the hidden table behind the type, which `sys.columns` is keyed on. public static func tableTypeColumns(schema: String, name: String) -> String { """ SELECT @@ -106,10 +92,7 @@ public enum MSSQLTypeQueries { """ } - /// One row per index key column rather than a comma-joined aggregate. A column name may legally - /// contain a comma (`[a,b]` is a valid identifier), which splitting turns into two columns, and - /// an aggregate has nowhere to carry `is_descending_key`, so a `DESC` key silently replayed as - /// ascending. Structured rows lose neither. + // One row per key column: a column name may hold a comma, and only a row can carry `is_descending_key`. public static func tableTypeIndexes(schema: String, name: String) -> String { """ SELECT @@ -136,8 +119,6 @@ public enum MSSQLTypeQueries { """ } - /// A CHECK on a table type is part of what the type validates, so a rebuilt statement that - /// drops it recreates a type with weaker validation than the original. public static func tableTypeCheckConstraints(schema: String, name: String) -> String { """ SELECT cc.definition @@ -151,8 +132,6 @@ public enum MSSQLTypeQueries { """ } - /// The database's own collation. A column keeps its collation in `sys.columns` whether or not - /// it differs from the database's, so without this every column would carry a `COLLATE` clause - /// the user never wrote. + // A column keeps its collation in `sys.columns` even when it is the database's, which would write a `COLLATE` nobody wrote. public static let databaseCollation = "SELECT CONVERT(varchar(128), DATABASEPROPERTYEX(DB_NAME(), 'Collation'))" } diff --git a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift index b174813116..2ff8a95f57 100644 --- a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift +++ b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift @@ -377,4 +377,37 @@ public enum MySQLObjectQueries { let host = String(scalars[scalars.index(after: separator)...]) return "\(quoteIdentifier(user))@\(quoteIdentifier(host))" } + + public static func columnCatalog(schema: String, table: String?, projectsGenerationExpression: Bool) -> String { + let tableFilter = table.map { " AND TABLE_NAME = '\(escapeLiteral($0))'" } ?? "" + let generationProjection = projectsGenerationExpression ? "GENERATION_EXPRESSION" : "NULL" + return """ + SELECT + TABLE_NAME, COLUMN_NAME, COLUMN_TYPE, COLLATION_NAME, + IS_NULLABLE, COLUMN_KEY, COLUMN_DEFAULT, EXTRA, COLUMN_COMMENT, + \(generationProjection) + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = '\(escapeLiteral(schema))'\(tableFilter) + ORDER BY TABLE_NAME, ORDINAL_POSITION + """ + } + + public static func approximateRowCount(schema: String, table: String) -> String { + """ + SELECT TABLE_ROWS + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = '\(escapeLiteral(schema))' + AND TABLE_NAME = '\(escapeLiteral(table))' + """ + } + + // MySQL has no `DROP ROUTINE`, and takes no argument list: a database holds one routine of a kind per name. + public static func dropRoutine(isFunction: Bool, schema: String?, name: String, quote: (String) -> String) -> String { + let kind = isFunction ? "FUNCTION" : "PROCEDURE" + return "DROP \(kind) IF EXISTS \(qualifiedIdentifier(schema: schema, name: name, quote: quote))" + } + + public static func dropTrigger(schema: String?, name: String, quote: (String) -> String) -> String { + "DROP TRIGGER \(qualifiedIdentifier(schema: schema, name: name, quote: quote))" + } } diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLForeignKeyChecks.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLForeignKeyChecks.swift new file mode 100644 index 0000000000..0d8480c9c6 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLForeignKeyChecks.swift @@ -0,0 +1,13 @@ +import Foundation + +// Setting `session_replication_role` takes a superuser or a `SET` grant, and a bare refusal aborts the transaction it +// runs in. The block swallows the refusal and keeps the transaction, and an allowed `SET` outlives it (measured on +// 17.11 and 18.6). +nonisolated public enum PostgreSQLForeignKeyChecks { + public static let disableStatements = [settingReplicationRole("replica")] + public static let enableStatements = [settingReplicationRole("DEFAULT")] + + private static func settingReplicationRole(_ value: String) -> String { + "DO $$ BEGIN SET session_replication_role = \(value); EXCEPTION WHEN insufficient_privilege THEN NULL; END $$" + } +} diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLChunkDecoder.swift b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLChunkDecoder.swift similarity index 61% rename from apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLChunkDecoder.swift rename to apple/Packages/TableProCore/Sources/TableProSQLFile/SQLChunkDecoder.swift index 86ed187924..087337d417 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLChunkDecoder.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLChunkDecoder.swift @@ -1,26 +1,9 @@ import Foundation +import TableProTabularIO -/// Turns a file's bytes into text one chunk at a time, holding back whatever runs off the end of -/// a chunk until the next one arrives. -/// -/// `String(data:encoding:)` is the whole of Foundation's decoding API and it has no way to report -/// a partial character, so three measured behaviours have to be worked around rather than caught. -/// -/// A chunk of UTF-16 carrying no byte order mark decodes as big-endian. The mark is at the start -/// of the file, so only the first chunk has one, and every chunk after it in a little-endian file -/// came back byte-swapped: a UTF-16 dump over 64 KiB imported as CJK from its first chunk -/// boundary on, with nothing raised. The byte order is therefore settled once, from the mark, and -/// every chunk after that is decoded with the explicit variant. Stripping the mark becomes this -/// type's job at the same time, because only the `.utf16` spelling consumes one. -/// -/// A chunk holding an odd number of UTF-16 bytes decodes the even part and drops the last byte -/// without failing, so the alignment is held back here rather than left for the decoder to -/// notice. -/// -/// A UTF-16 surrogate pair split across a boundary fails the whole chunk, as does a partial UTF-8 -/// sequence, so a chunk that will not decode gives its last few bytes back to the next one. How -/// many is derived from the encoding rather than assumed: only UTF-8 was handled before, so a -/// character split across a boundary in any other multi-byte encoding failed the import outright. +// String(data:encoding:) cannot report a partial character. Measured: unmarked UTF-16 decodes big-endian, so the +// byte order is settled once from the first chunk's mark; an odd UTF-16 byte count drops the last byte silently; a +// character split across chunks fails the whole chunk, so its last bytes go back on the front of the next one. public struct SQLChunkDecoder { private let declaredEncoding: String.Encoding private var resolvedEncoding: String.Encoding? @@ -94,10 +77,8 @@ public struct SQLChunkDecoder { return Resolution(encoding: encoding.unmarkedByteOrder, byteOrderMarkLength: 0) } - /// `CFStringGetMaximumSizeForEncoding` counts bytes per UTF-16 code unit, so it is asked for - /// two of them: that is one Unicode scalar, which is the largest thing a boundary can cut in - /// half. It over-reports for several encodings, which costs a decode attempt that fails and - /// never loses a byte, since a trimmed byte goes back on the front of the next chunk. + // CFStringGetMaximumSizeForEncoding counts per UTF-16 unit, so two of them cover one scalar, the most a boundary + // can cut. An over-report costs a failed decode attempt, never a byte. private static func maximumTrim(of encoding: String.Encoding, unitSize: Int) -> Int { let cfEncoding = CFStringConvertNSStringEncodingToEncoding(encoding.rawValue) guard cfEncoding != kCFStringEncodingInvalidId else { return 0 } diff --git a/apple/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLClientEncodingRewriter.swift similarity index 95% rename from apple/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift rename to apple/Packages/TableProCore/Sources/TableProSQLFile/SQLClientEncodingRewriter.swift index a501f6cbb9..639926ab12 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLClientEncodingRewriter.swift @@ -1,13 +1,7 @@ -// -// SQLClientEncodingRewriter.swift -// TablePro -// - import Foundation import TableProSQLGrammar -import TableProUtilities -internal enum SQLClientEncodingRewriter { +public enum SQLClientEncodingRewriter { private struct Replacement { let range: NSRange let text: String @@ -22,9 +16,9 @@ internal enum SQLClientEncodingRewriter { private static let postgresScopes: Set = ["SESSION", "LOCAL"] private static let clientCharacterSetVariable = "CHARACTER_SET_CLIENT" - internal static func rewritten( + public static func rewritten( _ statement: String, - family: TransactionEngineFamily, + family: SQLDumpFamily, grammar: SQLLexicalGrammar ) -> String? { let text = statement as NSString @@ -34,9 +28,9 @@ internal enum SQLClientEncodingRewriter { switch family { case .mysql: replacements = mysqlClientCharacterSets(&cursor, in: text) - case .postgres, .redshift, .cockroach: + case .postgres: replacements = postgresClientEncoding(&cursor, in: text).map { [Replacement(range: $0, text: postgresEncoding)] } ?? [] - case .sqlite, .duckdb, .sqlServer, .oracle, .redis, .teradata, .other: + case .other: return nil } guard !replacements.isEmpty else { return nil } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLDumpEncodingChoice.swift b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLDumpEncodingChoice.swift new file mode 100644 index 0000000000..2ddcf1d3fd --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLDumpEncodingChoice.swift @@ -0,0 +1,88 @@ +import Foundation +import TableProSQLGrammar +import TableProTabularIO + +public enum SQLDumpEncodingChoice { + private static let declarationScanLength = 65_536 + private static let declarationPattern = try? NSRegularExpression(pattern: [ + #"(?i)\A(?:\s+|--[^\n]*(?:\n|\z)|#[^\n]*(?:\n|\z)|/\*(?!M?!)[\s\S]*?\*/)*"#, + #"(?:/\*M?!\d*\s*)?SET\s+(?:(?:SESSION|LOCAL)\s+)?"#, + #"(?:NAMES|CHARACTER\s+SET|CHARSET|character_set_client\s*=|client_encoding\s*(?:=|TO))"#, + #"\s*['"]?([A-Za-z0-9_-]+)"# + ].joined()) + + private static let declaredNames: [String: String.Encoding] = [ + "UTF8": .utf8, "UTF8MB4": .utf8, "UTF8MB3": .utf8, "UNICODE": .utf8, + "CP1252": .windowsCP1252, "WIN1252": .windowsCP1252, "WINDOWS1252": .windowsCP1252, + "LATIN1MYSQL": .windowsCP1252, "LATIN1": .isoLatin1, "ISO88591": .isoLatin1, + "ASCII": .ascii, "SQLASCII": .ascii, + "CP932": shiftJIS, "SJIS": shiftJIS, "SHIFTJIS": shiftJIS, "SHIFTJIS2004": shiftJIS, + "UJIS": eucJP, "EUCJPMS": eucJP, "EUCJP": eucJP, + "GBK": gb18030, "GB2312": gb18030, "GB18030": gb18030, "EUCCN": gb18030, + "BIG5": big5, "CP950": big5, + "EUCKR": eucKR, "UHC": eucKR, "CP949": eucKR + ] + + private static let shiftJIS = TabularTextEncoding.shiftJIS.foundationEncoding + private static let eucJP = TabularTextEncoding.eucJP.foundationEncoding + private static let gb18030 = TabularTextEncoding.gb18030.foundationEncoding + private static let big5 = TabularTextEncoding.big5.foundationEncoding + private static let eucKR = TabularTextEncoding.eucKR.foundationEncoding + + // A trail byte of these can be 0x5C, so a guess at one could end a literal early inside a character. + private static let hidesABackslashInsideACharacter: Set = [.shiftJIS, .gb18030, .big5] + + public static func encoding( + replacing selected: String.Encoding, + forPreview preview: Data, + isWholeFile: Bool, + family: SQLDumpFamily, + grammar: SQLLexicalGrammar + ) -> String.Encoding? { + if let declared = declaredEncoding(in: preview, family: family, grammar: grammar) { + guard declared != selected, previewDecodes(preview, as: declared, isWholeFile: isWholeFile) else { return nil } + return declared + } + let detected = TabularEncodingDetector.sniff(preview, isWholeFile: isWholeFile).encoding + guard detected.foundationEncoding != selected, !hidesABackslashInsideACharacter.contains(detected), + previewDecodes(preview, as: detected.foundationEncoding, isWholeFile: isWholeFile) else { + return nil + } + return detected.foundationEncoding + } + + public static func declaredEncoding( + in preview: Data, + family: SQLDumpFamily, + grammar: SQLLexicalGrammar + ) -> String.Encoding? { + guard let declarationPattern, + let head = String(data: preview.prefix(declarationScanLength), encoding: .isoLatin1) else { + return nil + } + for statement in SQLStatementScanner.allStatements(in: head, grammar: grammar) { + let text = statement as NSString + guard let match = declarationPattern.firstMatch(in: statement, range: NSRange(location: 0, length: text.length)) else { + continue + } + let name = text.substring(with: match.range(at: 1)).uppercased().filter { $0.isLetter || $0.isNumber } + let key = name == "LATIN1" && family == .mysql ? "LATIN1MYSQL" : name + if let encoding = declaredNames[key] { + return encoding + } + } + return nil + } + + // The decoded preview, or nil when it does not read as `encoding`. A whole file has no next chunk, so bytes held + // back as the start of a split character never decode. + public static func decodedPreview(_ preview: Data, as encoding: String.Encoding, isWholeFile: Bool) -> String? { + var decoder = SQLChunkDecoder(encoding: encoding) + guard let text = decoder.decode(preview), !(isWholeFile && decoder.hasPendingBytes) else { return nil } + return text + } + + public static func previewDecodes(_ preview: Data, as encoding: String.Encoding, isWholeFile: Bool) -> Bool { + decodedPreview(preview, as: encoding, isWholeFile: isWholeFile) != nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLDumpFamily.swift b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLDumpFamily.swift new file mode 100644 index 0000000000..744abb6e9d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLDumpFamily.swift @@ -0,0 +1,8 @@ +import Foundation + +// The client a dump was written for, which decides the statements an import rewrites before it sends them. +public enum SQLDumpFamily: Sendable, Hashable, CaseIterable { + case mysql + case postgres + case other +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLDumpStatementRewriter.swift b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLDumpStatementRewriter.swift new file mode 100644 index 0000000000..72aa481507 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLDumpStatementRewriter.swift @@ -0,0 +1,111 @@ +import Foundation +import os +import TableProSQLGrammar + +public enum SqlFileImportError: LocalizedError, Equatable { + case unrecoverableBinaryLiteral(line: Int, encoding: String) + + public var errorDescription: String? { + switch self { + case .unrecoverableBinaryLiteral(let line, let encoding): + return String( + format: String(localized: "Line %@ holds binary data that cannot be read back exactly from a %@ file. Export the dump with --hex-blob, then import it again."), + line.formatted(), + encoding + ) + } + } +} + +// The import sends UTF-8, so a dump decoded from any other encoding has its literals and its client encoding +// statements rewritten to read back the same. +public struct SQLDumpStatementRewriter: Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "SQLDumpStatementRewriter") + + public let encoding: String.Encoding + public let family: SQLDumpFamily + public let grammar: @Sendable () -> SQLLexicalGrammar + + public init(encoding: String.Encoding, family: SQLDumpFamily, grammar: @escaping @Sendable () -> SQLLexicalGrammar) { + self.encoding = encoding + self.family = family + self.grammar = grammar + } + + // UTF-8 and ASCII reach the server as the file holds them. + public static func rewrites(fileDecodedFrom encoding: String.Encoding) -> Bool { + encoding != .utf8 && encoding != .ascii + } + + public func rewritten(_ statement: String, at lineNumber: Int) throws -> String { + let grammar = self.grammar() + let literals = try rewrittenLiterals(of: statement, at: lineNumber, grammar: grammar) ?? statement + guard let rewritten = SQLClientEncodingRewriter.rewritten(literals, family: family, grammar: grammar) else { + return literals + } + Self.logger.info("Kept the session's client encoding at line \(lineNumber, privacy: .public), since the import sends UTF-8") + return rewritten + } + + public func rewriting( + _ statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error> + ) -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + let rewriting = RewrittenStatements(statements, rewriter: self) + return AsyncThrowingStream(unfolding: { + try await rewriting.next() + }) + } + + public func rewriting(_ elements: AsyncThrowingStream) -> AsyncThrowingStream { + let rewriting = RewrittenElements(elements, rewriter: self) + return AsyncThrowingStream(unfolding: { + try await rewriting.next() + }) + } + + private func rewrittenLiterals(of statement: String, at lineNumber: Int, grammar: SQLLexicalGrammar) throws -> String? { + guard family == .mysql else { return nil } + do { + return try SQLTranscodedLiteralRewriter.rewritten( + statement, + decodedFrom: encoding, + backslashEscapesAreOn: grammar.contains(.backslashEscapesInSingleQuotes), + grammar: grammar + ) + } catch SQLTranscodedLiteralError.unrecoverableBinaryLiteral { + throw SqlFileImportError.unrecoverableBinaryLiteral(line: lineNumber, encoding: String.localizedName(of: encoding)) + } + } +} + +private final class RewrittenStatements: @unchecked Sendable { + private var statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>.AsyncIterator + private let rewriter: SQLDumpStatementRewriter + + init(_ statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>, rewriter: SQLDumpStatementRewriter) { + self.statements = statements.makeAsyncIterator() + self.rewriter = rewriter + } + + func next() async throws -> (statement: String, lineNumber: Int)? { + guard let (statement, lineNumber) = try await statements.next() else { return nil } + return (try rewriter.rewritten(statement, at: lineNumber), lineNumber) + } +} + +// COPY rows pass unchanged: the parser hands them out as UTF-8 already. +private final class RewrittenElements: @unchecked Sendable { + private var elements: AsyncThrowingStream.AsyncIterator + private let rewriter: SQLDumpStatementRewriter + + init(_ elements: AsyncThrowingStream, rewriter: SQLDumpStatementRewriter) { + self.elements = elements.makeAsyncIterator() + self.rewriter = rewriter + } + + func next() async throws -> SQLFileElement? { + guard let element = try await elements.next() else { return nil } + guard case let .statement(statement, line) = element else { return element } + return .statement(try rewriter.rewritten(statement, at: line), line: line) + } +} diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLFileBatchLines.swift b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLFileBatchLines.swift similarity index 72% rename from apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLFileBatchLines.swift rename to apple/Packages/TableProCore/Sources/TableProSQLFile/SQLFileBatchLines.swift index e3da684f9d..8b198ff272 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLFileBatchLines.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLFileBatchLines.swift @@ -1,15 +1,13 @@ import Foundation -/// What the import parser needs to know about lines to cut a SQL Server script at its `GO` lines, read off a buffer -/// that holds only the part of the file the last chunks brought in. +// Line facts for cutting a SQL Server script at its GO lines, read off a buffer holding only the last chunks. public enum SQLFileBatchLines { private static let lineFeed: unichar = 0x0A private static let carriageReturn: unichar = 0x0D private static let space: unichar = 0x20 private static let tab: unichar = 0x09 - /// Whether only spaces and tabs follow the last line break once the units in `start.. Bool { var index = end - 1 while index >= start { @@ -37,8 +35,7 @@ public enum SQLFileBatchLines { return nil } - /// How many line feeds the blanks at the start of `text` hold, which is how far below its first line the text the - /// server receives begins once those blanks are trimmed. + // How far below its first line the text the server receives starts once its leading blanks are trimmed. public static func leadingLineFeeds(in text: NSString?) -> Int { guard let text else { return 0 } var count = 0 diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLFileParser+CopyRows.swift similarity index 97% rename from apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift rename to apple/Packages/TableProCore/Sources/TableProSQLFile/SQLFileParser+CopyRows.swift index ace89683df..113d03f52c 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLFileParser+CopyRows.swift @@ -1,16 +1,11 @@ -// -// SQLFileParser+CopyRows.swift -// TablePro -// - import Foundation import TableProSQLGrammar extension SQLFileParser { - static let defaultChunkSize = 65_536 + public static let defaultChunkSize = 65_536 // A UTF-16 unit takes at most three UTF-8 bytes, so a piece of COPY rows stays within 1 MiB. - static let defaultCopyPieceLength = 1_048_576 / 3 + public static let defaultCopyPieceLength = 1_048_576 / 3 struct ReadLimits { let batchCutLength: Int diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLFileParser.swift similarity index 85% rename from apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift rename to apple/Packages/TableProCore/Sources/TableProSQLFile/SQLFileParser.swift index eb76fd8404..fc1ebc8d3f 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLFileParser.swift @@ -1,20 +1,21 @@ -// -// SQLFileParser.swift -// TablePro -// - import Foundation import os -import TableProPluginKit import TableProSQLGrammar -import TableProUtilities -/// A file the import refuses before it sends the statement that needs what it cannot give. -enum SQLFileParseError: LocalizedError, Equatable { +// What a file holds, a statement at a time: a `COPY ... FROM STDIN` comes with a reader over the rows after it, which +// a driver pulls on its own thread until it answers nil. Rows left unread are passed over once the next element is +// asked for. +public enum SQLFileElement: Sendable { + case statement(String, line: Int) + case copyFromStandardInput(statement: String, line: Int, readRows: @Sendable () throws -> Data?) +} + +public enum SQLFileParseError: LocalizedError, Equatable { case copyFromStandardInput(line: Int) case spanTooLong(line: Int) + case fileReadFailed(String) - var errorDescription: String? { + public var errorDescription: String? { switch self { case .copyFromStandardInput(let line): return String( @@ -28,26 +29,25 @@ enum SQLFileParseError: LocalizedError, Equatable { format: String(localized: "The string or comment that starts on line %lld is longer than the import reads."), line ) + case .fileReadFailed(let message): + return String(format: String(localized: "Failed to read file: %@"), message) } } } -final class SQLFileParser: Sendable { +public final class SQLFileParser: Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "SQLFileParser") - /// SQL Server takes at most 65,536 packets of 4,096 bytes in one request, about 134 million UTF-16 units of batch - /// text: measured on Azure SQL Edge 15, a 133,900,000-unit batch ran and a 133,960,000-unit one closed the - /// connection. A batch that reaches half of that ends at its next `;`, which leaves the statement it ends inside - /// as much room again. A script whose `GO` lines stand that far apart is then sent in pieces the server takes, - /// and held one piece at a time rather than whole. - static let defaultBatchCutLength = 67_108_864 + // SQL Server takes about 134 million UTF-16 units in one request (Azure SQL Edge 15: 133,900,000 ran, 133,960,000 + // closed the connection). A batch half that long ends at its next `;`, leaving the statement inside it as much again. + public static let defaultBatchCutLength = 67_108_864 private let limits: ReadLimits // As much as the core's script reader holds of one statement: a string or comment longer is not one a server takes. - static let defaultSpanLimit = 134_217_728 + public static let defaultSpanLimit = 134_217_728 - init( + public init( batchCutLength: Int = SQLFileParser.defaultBatchCutLength, chunkSize: Int = SQLFileParser.defaultChunkSize, copyPieceLength: Int = SQLFileParser.defaultCopyPieceLength, @@ -61,8 +61,6 @@ final class SQLFileParser: Sendable { ) } - /// One statement the file holds, or on a grammar that cuts scripts into batches, one batch, with how many times - /// the script runs it. struct ParsedStatement { let statement: String let lineNumber: Int @@ -72,7 +70,7 @@ final class SQLFileParser: Sendable { enum ParserState { case normal - /// A psql `\restrict` line, which runs to its line feed whatever ends a comment. + // A psql `\restrict` line, which runs to its line feed whatever ends a comment. case inClientLine } @@ -142,7 +140,7 @@ final class SQLFileParser: Sendable { return statement.range(of: "\n", options: [], range: rest).location == NSNotFound } - /// The `DELIMITER` line the statement read so far is, where it opened its line as the mysql client requires. + // Only where the statement opened its line, as the mysql client requires. private static func delimiterDirective(_ ctx: ParserContext) -> SQLDelimiterDirective? { guard ctx.grammar.contains(.delimiterDirective), ctx.statementOpensLine, let statement = ctx.currentStatement @@ -171,58 +169,43 @@ final class SQLFileParser: Sendable { // The next read takes as much again, so a literal of many megabytes is lexed a few times, not once per chunk. var heldSpanLength = 0 - /// SQL Server's tools read a script the way sqlcmd does: a line holding only `GO` ends a batch, the batch goes - /// to the server whole with its `;` and its comments, and the `GO` line goes nowhere. The comments stay so the - /// server's line numbers count the file's lines and a routine keeps the comments written inside it. - /// - /// Only a file that holds a `GO` line is read so. One that holds none was written to run a statement at a - /// time, cut at each `;`: that is how every SQL Server dump TablePro wrote before it wrote `GO` lines reads, - /// and read as one batch it fails on its first view with Msg 111, having run none of it. + // As sqlcmd reads a script: a `GO` line ends a batch, which goes whole with its `;` and comments, so the + // server's line numbers count the file's. Only a file holding a `GO` line: older TablePro dumps hold none, and + // read as one batch they fail on their first view with Msg 111. let readsBatches: Bool let batchCutLength: Int - /// Set once a `GO` line has ended a batch, which is what settles whether a file is read in batches. var sawBatchSeparator = false - /// Whether only spaces and tabs stand between the last line break and the unit being read, which is where a - /// `GO` line can start. Tracked here because the buffer no longer holds the line's start once a chunk is - /// consumed. + // Where a `GO` line can start; tracked because the buffer drops the line's start once a chunk is consumed. var lineHoldsOnlyBlanks = true - /// How much of the line after a `G` held back for the rest of its line is already known to hold no line break, - /// so a long line is searched once rather than once per chunk. + // So a long line after a held `G` is searched once rather than once per chunk. var separatorLineSearched = 0 - /// The line the batch text starts on before its leading whitespace, and how many units were read before it. var batchTextStartLine = 1 var batchStartUnit = 0 var unitsBeforeBuffer = 0 - /// The statement grammar, for a dialect whose statement can own a `;`: a PL/SQL unit arrives whole with - /// its own `;`, and so do a T-SQL `MERGE` and a `BEGIN...END` routine body, and a CQL batch arrives whole - /// with every `;` inside it. A batch keeps every `;` it holds, so a file read in batches needs none. Every - /// other dialect has always split an import at each `;` and relies on `DELIMITER` or dollar quoting for a - /// routine body, and keeps doing so. + // Only for a dialect whose statement can own a `;` (a PL/SQL unit, a T-SQL `MERGE`, a routine body, a CQL + // batch); a file read in batches keeps every `;` and needs none. var boundaries: (any SQLStatementBoundaryTracking)? - /// A `;` that ended the statement while the tracker could still take it back at the next token. + // A `;` the tracker can still take back at the next token. var heldEnd: HeldEnd? var word: [unichar] = [] - /// Where the word being read starts in the statement text, and on which line. var wordStart = 0 var wordStartLine = 1 var lineHasCode = false - /// Whether the statement's first unit was the first code on its line, where a `DELIMITER` can stand. + // A `DELIMITER` stands only where the statement's first unit was the first code on its line. var statementOpensLine = false var pendingSlashLine = false var pendingSlashTrailing: [unichar] = [] - /// Whether the unit before the one being read belongs to a word or a name, which a letter glued to it continues. A number is neither, - /// so `1MERGE` is a number and a keyword, as the statement scanner and SQL Server both read it. + // A number is neither word nor name, so `1MERGE` is a number and a keyword, as the scanner and SQL Server read it. var previousUnitInWord = false - /// Set for the last pass over the buffer, once the file has nothing more to give. A character held back for - /// the one after it is settled with nothing after it, instead of being left in the buffer and dropped. + // On the last pass a unit held back for the one after it is settled, not left in the buffer and dropped. var atEndOfInput = false init(grammar: SQLLexicalGrammar, currentStatement: NSMutableString?, batchCutLength: Int, readsBatches: Bool) { @@ -256,14 +239,12 @@ final class SQLFileParser: Sendable { } } - /// The statement's text before the held `;`, and whether the `;` belongs to it. struct HeldEnd { let textLength: Int let keepsSemicolon: Bool } - /// Settles a held end at the first token after it, `word` being that token when it is a word. When the `;` ended - /// the statement, the text up to it is yielded and the word, if any, starts the next statement. + // When the held `;` ended the statement, the word after it, if any, starts the next one. private static func settleHeldEnd(_ ctx: inout ParserContext, word: String?) { guard let held = ctx.heldEnd else { return } ctx.heldEnd = nil @@ -301,8 +282,7 @@ final class SQLFileParser: Sendable { char == kSpace || char == kTab || char == kCarriageReturn } - /// Hands the word that just ended to the tracker. A word is only assembled while the tracker still reads words, - /// so a statement it has already classified as plain SQL costs nothing per character. + // A word is assembled only while the tracker reads words, so plain SQL costs nothing per character. private static func flushWord(_ ctx: inout ParserContext) { guard !ctx.word.isEmpty else { return } let word = String(utf16CodeUnits: ctx.word, count: ctx.word.count).uppercased() @@ -341,9 +321,7 @@ final class SQLFileParser: Sendable { return true } - /// Settles a `/` held back at the start of a line once the line's end shows whether it stood alone. - /// - /// Returns true when `char` was consumed as trailing whitespace on the `/` line. + // True when `char` was a blank trailing a `/` held back at the start of its line. private static func settlePendingSlashLine(_ ctx: inout ParserContext, char: unichar) -> Bool { guard ctx.pendingSlashLine else { return false } if isLineBlank(char) { @@ -524,9 +502,7 @@ final class SQLFileParser: Sendable { appendChar(char, to: ctx.currentStatement) } - /// A `;` ends the statement unless the grammar holds it inside a PL/SQL unit or a routine body, and a statement - /// that owns the `;` that ends it keeps it. In a batch it ends nothing, unless the batch has grown past what the - /// server takes in one request. + // In a batch a `;` ends nothing unless the batch has grown past what the server takes in one request. private static func processSemicolon(_ ctx: inout ParserContext, at i: Int) { if readsDelimiterDirective(ctx) { appendChar(kSemicolon, to: ctx.currentStatement) @@ -585,10 +561,7 @@ final class SQLFileParser: Sendable { resetStatement(&ctx) } - /// Ends the batch read so far, which is sent only when it holds code: a batch of comments and blanks runs nothing. - /// - /// Its line is the one its text starts on once the leading blanks are trimmed, because that is the line the server - /// counts as the batch's first. + // A batch of comments and blanks runs nothing. Its line is where its trimmed text starts, as the server counts it. private static func yieldBatch(_ ctx: inout ParserContext, repeatCount: Int, nextBatchStart: Int) { if ctx.hasStatementContent { let lineNumber = ctx.batchTextStartLine + SQLFileBatchLines.leadingLineFeeds(in: ctx.currentStatement) @@ -601,7 +574,6 @@ final class SQLFileParser: Sendable { ctx.batchStartUnit = nextBatchStart } - /// Ends the batch at the `GO` line starting at `i` and steps over the line, or answers nil when the line is code. private static func endBatchAtSeparatorLine( _ ctx: inout ParserContext, i: inout Int, @@ -628,8 +600,7 @@ final class SQLFileParser: Sendable { case needsMoreData } - /// Reads the line a `G` starts once the buffer holds all of it: what follows `GO` decides, and a line cut off at - /// the end of a chunk would read as ending there. + // Waits for the whole line: what follows `GO` decides, and a line a chunk cuts would read as ending there. private static func batchSeparator( _ ctx: inout ParserContext, at i: Int, @@ -659,12 +630,16 @@ final class SQLFileParser: Sendable { target.append(buffer.substring(with: NSRange(location: start, length: end - start))) } - func parseFile(url: URL, encoding: String.Encoding, grammar: SQLLexicalGrammar) -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + public func parseFile( + url: URL, + encoding: String.Encoding, + grammar: SQLLexicalGrammar + ) -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { parseFile(url: url, encoding: encoding, grammar: { grammar }) } // Asked before each statement is cut, once the one before it ran: `SET sql_mode` changes how the next is read. - func parseFile( + public func parseFile( url: URL, encoding: String.Encoding, grammar: @escaping @Sendable () -> SQLLexicalGrammar, @@ -676,21 +651,21 @@ final class SQLFileParser: Sendable { }) } - func parseElements( + public func parseElements( url: URL, encoding: String.Encoding, grammar: SQLLexicalGrammar - ) -> AsyncThrowingStream { + ) -> AsyncThrowingStream { parseElements(url: url, encoding: encoding, grammar: { grammar }) } // The statements as parseFile yields them, and each `COPY ... FROM STDIN` with a reader over its rows. - func parseElements( + public func parseElements( url: URL, encoding: String.Encoding, grammar: @escaping @Sendable () -> SQLLexicalGrammar, onRead: (@Sendable (_ bytesRead: Int64, _ fileSize: Int64) -> Void)? = nil - ) -> AsyncThrowingStream { + ) -> AsyncThrowingStream { let session = ParseSession(url: url, encoding: encoding, grammar: grammar, countOnly: false, limits: limits, onRead: onRead) return AsyncThrowingStream(unfolding: { try await session.nextElement() @@ -698,7 +673,7 @@ final class SQLFileParser: Sendable { } // Runs are added up rather than walked: a `GO n` count can reach `Int32.max`. A COPY counts as one. - func countStatements( + public func countStatements( url: URL, encoding: String.Encoding, grammar: SQLLexicalGrammar @@ -716,12 +691,9 @@ final class SQLFileParser: Sendable { } extension SQLFileParser { - /// How a session cuts its file. private enum Reading { - /// Settled from the file before its first statement: in batches when it holds a `GO` line and the grammar - /// reads them, else a statement at a time. + // In batches when the file holds a `GO` line and the grammar reads them, else a statement at a time. case fromFile - /// At the file's `GO` lines, as sqlcmd cuts a script. case batches } @@ -795,22 +767,22 @@ extension SQLFileParser { return (statement, lineNumber) } - func nextElement() async throws -> PluginImportElement? { + func nextElement() async throws -> SQLFileElement? { if let repeating, remainingRuns > 0 { remainingRuns -= 1 - return .statement(repeating.statement, lineNumber: repeating.lineNumber) + return .statement(repeating.statement, line: repeating.lineNumber) } guard let next = try await nextStatement() else { return nil } guard next.readsCopyRows else { repeating = next remainingRuns = next.repeatCount - 1 - return .statement(next.statement, lineNumber: next.lineNumber) + return .statement(next.statement, line: next.lineNumber) } let generation = copyGeneration // The driver reads pieces on its own queue while the import awaits the copy, and the import asks for the // next element only once the copy returns, so the session is never used from two places at once. - let rows = PluginCopyData { [self] in try nextRowsPiece(generation: generation) } - return .copyFromStandardInput(statement: next.statement, lineNumber: next.lineNumber, rows: rows) + let readRows: @Sendable () throws -> Data? = { [self] in try nextRowsPiece(generation: generation) } + return .copyFromStandardInput(statement: next.statement, line: next.lineNumber, readRows: readRows) } func nextStatement() async throws -> ParsedStatement? { @@ -925,13 +897,13 @@ extension SQLFileParser { let isFinalChunk = rawData.isEmpty guard let chunk = decoder.decode(rawData) else { - throw DecompressionError.fileReadFailed( + throw SQLFileParseError.fileReadFailed( "Failed to decode file with \(encoding.description) encoding" ) } if isFinalChunk && decoder.hasPendingBytes { - throw DecompressionError.fileReadFailed( + throw SQLFileParseError.fileReadFailed( "Trailing bytes did not form a valid \(encoding.description) sequence at end of file" ) } diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLTranscodedLiteralRewriter.swift b/apple/Packages/TableProCore/Sources/TableProSQLFile/SQLTranscodedLiteralRewriter.swift similarity index 100% rename from apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLTranscodedLiteralRewriter.swift rename to apple/Packages/TableProCore/Sources/TableProSQLFile/SQLTranscodedLiteralRewriter.swift diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLTokenCursor.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLTokenCursor.swift similarity index 82% rename from apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLTokenCursor.swift rename to apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLTokenCursor.swift index 2ec12d14c2..cd05f41065 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLTokenCursor.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLTokenCursor.swift @@ -1,19 +1,7 @@ import Foundation -import TableProSQLGrammar - -/// A lazy reader over the head of one statement, in the vocabulary a rule about that statement -/// needs: words, quoted identifiers, literals and single symbols. -/// -/// It never materializes the statement. A rule pulls the two or three tokens it cares about and -/// stops, which is what keeps a 100,000-statement dump affordable, and it shares ``SQLNonCodeSpan`` -/// with every other scanner so a comment, a string and a dollar-quoted body end in the same place. -/// -/// The cursor ends at a depth-0 `;`, so a statement carrying a trailing fragment cannot be read as -/// two. A MySQL conditional comment is code, not a comment, so its `/*!NNNNN` and `/*M!NNNNN` -/// openers and the `*/` that closes them are stepped over and the body is read. -/// -/// It is a struct with value semantics, so a rule that needs to look ahead copies it, reads, and -/// either adopts the copy or drops it. + +// Lazy, so a rule reads the two or three tokens it needs and a 100,000-statement dump stays affordable. It ends at a +// depth-0 `;`, and reads a MySQL conditional comment's body as code. Copy it to look ahead. public struct SQLTokenCursor { public enum Token: Equatable { case word(String) @@ -47,8 +35,6 @@ public struct SQLTokenCursor { NSRange(location: tokenStart, length: index - tokenStart) } - /// The token `next()` returned last as written, for a rule that needs a word's spelling rather - /// than its uppercased form. public var lastTokenText: String { text.substring(with: tokenRange) } @@ -172,8 +158,7 @@ public extension SQLTokenCursor.Token { return word } - /// The name this token spells, whichever way it was written. A quoted identifier is uppercased - /// so `` `sql_log_bin` `` and `SQL_LOG_BIN` answer the same rule. + // Uppercased when quoted, so `` `sql_log_bin` `` and `SQL_LOG_BIN` answer the same rule. var identifier: String? { switch self { case .word(let word): diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/Text/ByteOrderMark.swift b/apple/Packages/TableProCore/Sources/TableProTabularIO/ByteOrderMark.swift similarity index 100% rename from apple/Packages/TableProMacCore/Sources/TableProUtilities/Text/ByteOrderMark.swift rename to apple/Packages/TableProCore/Sources/TableProTabularIO/ByteOrderMark.swift diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataObjectQueries.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataObjectQueries.swift index 1f9e76e6d0..cc0fc88078 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataObjectQueries.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataObjectQueries.swift @@ -8,8 +8,7 @@ import Foundation public enum TeradataObjectQueries { - /// DBC.TablesV.TableKind is a single character. These are the ones that are routines or - /// triggers rather than tables, views or indexes. + // The DBC.TablesV.TableKind codes of routines and triggers, as opposed to tables, views and indexes. public enum TableKind { public static let storedProcedure = "P" public static let externalProcedure = "E" @@ -54,8 +53,7 @@ public enum TeradataObjectQueries { return query + "\nORDER BY TableName, TriggerName" } - /// SHOW PROCEDURE only returns text when the procedure was created with SPL retention on, so - /// DBC.TablesV.RequestText is tried first and this is the fallback. + // The fallback after DBC.TablesV.RequestText: SHOW PROCEDURE has text only for a procedure created with SPL on. public static func routineDefinition(kind: String, database: String, name: String) -> String { let qualified = TeradataSchemaQueries.qualifiedName(database: database, table: name) return TableKind.procedures.contains(kind) @@ -89,4 +87,9 @@ public enum TeradataObjectQueries { public static func orientation(fromKind kind: String?) -> String { kind?.trimmingCharacters(in: .whitespaces).uppercased() == "R" ? "ROW" : "STATEMENT" } + + // Teradata's DROP TRIGGER takes no IF EXISTS and names the trigger in its database, not on its table. + public static func dropTriggerStatement(name: String, database: String?) -> String { + "DROP TRIGGER \(TeradataSchemaQueries.qualifiedName(database: database, table: name))" + } } diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraDropTriggerTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraDropTriggerTests.swift new file mode 100644 index 0000000000..345dbee184 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraDropTriggerTests.swift @@ -0,0 +1,20 @@ +import TableProCassandraCore +import Testing + +struct CassandraDropTriggerTests { + @Test("A trigger drop names its table and skips a trigger that is gone") + func dropNamesTheTable() { + #expect( + CassandraObjectQueries.dropTriggerStatement(name: "audit", keyspace: "shop", table: "orders") + == #"DROP TRIGGER IF EXISTS "audit" ON "shop"."orders""# + ) + } + + @Test("A quote inside a name is doubled") + func quoteInsideANameIsDoubled() { + #expect( + CassandraObjectQueries.dropTriggerStatement(name: #"a"b"#, keyspace: #"k"s"#, table: #"t"1"#) + == #"DROP TRIGGER IF EXISTS "a""b" ON "k""s"."t""1""# + ) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProDatabaseTests/ConnectionManagerTransferTests.swift b/apple/Packages/TableProCore/Tests/TableProDatabaseTests/ConnectionManagerTransferTests.swift new file mode 100644 index 0000000000..1d8419deb0 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProDatabaseTests/ConnectionManagerTransferTests.swift @@ -0,0 +1,236 @@ +import Foundation +@testable import TableProDatabase +@testable import TableProModels +import TableProTestSupport +import Testing + +@Suite(.timeLimit(.minutes(1))) +struct ConnectionManagerTransferTests { + private static let type = DatabaseType(rawValue: "mock") + + private func connection() -> DatabaseConnection { + var connection = DatabaseConnection(name: "Test", type: Self.type, host: "db.example.com", port: 5_432, database: "app") + connection.queryTimeoutSeconds = 30 + return connection + } + + @Test("A transfer gets a driver of its own on the asked database, with no catalog preparation and no query timeout") + func opensItsOwnDriver() async throws { + let factory = TransferDriverFactory() + let manager = ConnectionManager(driverFactory: factory, secureStore: MockSecureStore()) + let connection = connection() + let session = try await manager.connect(connection) + + let transfer = try await manager.openDedicatedDriver(for: connection.id, database: "sales") + + let dialed = factory.dialed + #expect(transfer !== session.driver) + #expect((transfer as? MockDatabaseDriver)?.isConnected == true) + #expect(dialed.last?.database == "sales") + #expect(dialed.last?.queryTimeoutSeconds == nil) + #expect(factory.prepared.isEmpty) + } + + @Test("Closing the transfer's driver leaves the session alone, and a second close does nothing") + func closeLeavesTheSession() async throws { + let factory = TransferDriverFactory() + let manager = ConnectionManager(driverFactory: factory, secureStore: MockSecureStore()) + let connection = connection() + let session = try await manager.connect(connection) + let transfer = try #require(try await manager.openDedicatedDriver(for: connection.id, database: nil) as? MockDatabaseDriver) + + await manager.closeDedicatedDriver(transfer, for: connection.id) + await manager.closeDedicatedDriver(transfer, for: connection.id) + await manager.closeDedicatedDriver(session.driver, for: connection.id) + + #expect(transfer.disconnectCount == 1) + #expect((session.driver as? MockDatabaseDriver)?.isConnected == true) + } + + @Test("A disconnect ends the transfer's driver before the session's") + func disconnectEndsTheTransferFirst() async throws { + let factory = TransferDriverFactory() + let order = DisconnectOrder() + let manager = ConnectionManager(driverFactory: factory, secureStore: MockSecureStore()) + let connection = connection() + let session = try await manager.connect(connection) + let sessionDriver = try #require(session.driver as? MockDatabaseDriver) + let transfer = try #require(try await manager.openDedicatedDriver(for: connection.id, database: nil) as? MockDatabaseDriver) + transfer.beforeDisconnect = { await order.record("transfer") } + sessionDriver.beforeDisconnect = { await order.record("session") } + + await manager.disconnect(connection.id) + + #expect(transfer.disconnectCount == 1) + #expect(await order.names == ["transfer", "session"]) + } + + @Test("A catalog hold ends the transfer's driver and opens none until it is released") + func holdEndsTheTransfer() async throws { + let factory = TransferDriverFactory() + let manager = ConnectionManager(driverFactory: factory, secureStore: MockSecureStore()) + let connection = connection() + _ = try await manager.connect(connection) + let transfer = try #require(try await manager.openDedicatedDriver(for: connection.id, database: nil) as? MockDatabaseDriver) + + await manager.holdCatalogConnections(for: connection.id) + + #expect(transfer.disconnectCount == 1) + await #expect(throws: TransferConnectionError.connectionHeld) { + _ = try await manager.openDedicatedDriver(for: connection.id, database: nil) + } + manager.releaseCatalogConnections(for: connection.id) + _ = try await manager.openDedicatedDriver(for: connection.id, database: nil) + } + + @Test("A session driver that gives its own transfer connection is asked, and its answer of itself is not owned") + func sessionDriverGivesItsOwn() async throws { + let session = OwnTransferDriver() + let factory = TransferDriverFactory(sessionDriver: session) + let manager = ConnectionManager(driverFactory: factory, secureStore: MockSecureStore()) + let connection = connection() + _ = try await manager.connect(connection) + + let own = try await manager.openDedicatedDriver(for: connection.id, database: nil) + session.answersWithItself = true + let itself = try await manager.openDedicatedDriver(for: connection.id, database: nil) + await manager.disconnect(connection.id) + + #expect(own === session.given) + #expect(itself === session) + #expect(factory.dialed.count == 1) + #expect(session.given.disconnectCount == 1) + #expect(session.disconnectCount == 1) + } + + @Test("No session, no transfer driver") + func noSession() async { + let manager = ConnectionManager(driverFactory: TransferDriverFactory(), secureStore: MockSecureStore()) + + await #expect(throws: ConnectionError.notConnected) { + _ = try await manager.openDedicatedDriver(for: UUID(), database: nil) + } + } + + @Test("A held connection keeps its file through a background release, and the last release says so") + func holdKeepsTheFile() async throws { + let session = MockDatabaseDriver() + session.holdsSuspensionBlockingResource = true + let manager = ConnectionManager( + driverFactory: TransferDriverFactory(sessionDriver: session), secureStore: MockSecureStore() + ) + let connection = connection() + _ = try await manager.connect(connection) + let ended = EndedHolds() + manager.observeTransferHoldsEnded { ended.record($0) } + + let first = manager.holdForTransfer(connection.id) + let second = manager.holdForTransfer(connection.id) + let releasedWhileHeld = await manager.releaseSuspensionBlockingResources() + let blockingWhileHeld = manager.hasSuspensionBlockingResources + first.release() + let endedAfterOne = ended.ids + second.release() + second.release() + + #expect(releasedWhileHeld.isEmpty) + #expect(!blockingWhileHeld) + #expect(session.isConnected) + #expect(endedAfterOne.isEmpty) + #expect(ended.ids == [connection.id]) + #expect(manager.hasSuspensionBlockingResources) + } +} + +private actor DisconnectOrder { + private(set) var names: [String] = [] + + func record(_ name: String) { + names.append(name) + } +} + +private final class EndedHolds: @unchecked Sendable { + private let lock = NSLock() + private var recorded: [UUID] = [] + + var ids: [UUID] { + lock.withLock { recorded } + } + + func record(_ id: UUID) { + lock.withLock { recorded.append(id) } + } +} + +// The session's driver first, then a new one for each later dial. +private final class TransferDriverFactory: DriverFactory, @unchecked Sendable { + private let lock = NSLock() + private var sessionDriver: (any DatabaseDriver)? + private var connections: [DatabaseConnection] = [] + private var preparedDrivers: [any DatabaseDriver] = [] + + init(sessionDriver: (any DatabaseDriver)? = nil) { + self.sessionDriver = sessionDriver ?? MockDatabaseDriver() + } + + var dialed: [DatabaseConnection] { + lock.withLock { connections } + } + + var prepared: [any DatabaseDriver] { + lock.withLock { preparedDrivers } + } + + func createDriver(for connection: DatabaseConnection, password: String?) throws -> any DatabaseDriver { + lock.withLock { + connections.append(connection) + defer { sessionDriver = nil } + return sessionDriver ?? MockDatabaseDriver() + } + } + + func supportedTypes() -> [DatabaseType] { [DatabaseType(rawValue: "mock")] } + + func opensCatalogConnections(for connection: DatabaseConnection) -> Bool { true } + + func prepareCatalogDriver(_ driver: any DatabaseDriver, for connection: DatabaseConnection) async throws { + lock.withLock { preparedDrivers.append(driver) } + } +} + +// SQLite and DuckDB's shape: the session makes the transfer's connection itself, or answers with itself. +private final class OwnTransferDriver: DatabaseDriver, DedicatedTransferConnection, @unchecked Sendable { + let given = MockDatabaseDriver() + var answersWithItself = false + private(set) var disconnectCount = 0 + + func openTransferDriver() async throws -> any DatabaseDriver { + if answersWithItself { return self } + try await given.connect() + return given + } + + func connect() async throws {} + func disconnect() async throws { disconnectCount += 1 } + func ping() async throws -> Bool { true } + func execute(query: String) async throws -> QueryResult { + QueryResult(columns: [], rows: [], rowsAffected: 0, executionTime: 0) + } + func cancelCurrentQuery() async throws {} + func fetchTables(schema: String?) async throws -> [TableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [ColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [IndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [ForeignKeyInfo] { [] } + func fetchDatabases() async throws -> [String] { [] } + func switchDatabase(to name: String) async throws {} + var supportsSchemas: Bool { false } + func switchSchema(to name: String) async throws {} + func fetchSchemas() async throws -> [String] { [] } + var currentSchema: String? { nil } + var supportsTransactions: Bool { false } + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} + var serverVersion: String? { nil } +} diff --git a/apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBObjectQueriesTests.swift b/apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBObjectQueriesTests.swift new file mode 100644 index 0000000000..3f6120440a --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBObjectQueriesTests.swift @@ -0,0 +1,58 @@ +import Foundation +import TableProDuckDBCore +import Testing + +@Test func aScalarMacroDropsWithDropMacro() { + #expect(DuckDBObjectQueries.dropMacro(name: "add_one", schema: "main", isTableMacro: false) + == #"DROP MACRO IF EXISTS "main"."add_one""#) +} + +@Test func aTableMacroDropsWithDropMacroTable() { + #expect(DuckDBObjectQueries.dropMacro(name: "rows_of", schema: "main", isTableMacro: true) + == #"DROP MACRO TABLE IF EXISTS "main"."rows_of""#) +} + +@Test func aMacroWithNoSchemaDropsByItsBareName() { + #expect(DuckDBObjectQueries.dropMacro(name: "m", schema: nil, isTableMacro: false) == #"DROP MACRO IF EXISTS "m""#) + #expect(DuckDBObjectQueries.dropMacro(name: "m", schema: "", isTableMacro: false) == #"DROP MACRO IF EXISTS "m""#) +} + +@Test func aQuoteInAMacroNameStaysInsideTheIdentifier() { + #expect(DuckDBObjectQueries.qualifiedName(schema: #"s"x"#, name: #"a"b"#) == #""s""x"."a""b""#) +} + +@Test func theMacroListNamesTheCatalogAndTheSchemaAsLiterals() { + let sql = DuckDBObjectQueries.macroList(catalog: "it's", schema: "main") + #expect(sql.contains("WHERE database_name = 'it''s' AND schema_name = 'main'")) + #expect(sql.contains("function_type IN ('macro', 'table_macro') AND NOT internal")) +} + +@Test func theMacroListWithNoSchemaReadsEverySchemaOfTheCatalog() { + let sql = DuckDBObjectQueries.macroList(catalog: "memory", schema: nil) + #expect(sql.contains("WHERE database_name = 'memory'\n")) + #expect(!sql.contains("schema_name = ")) +} + +@Test func aTableMacroSignatureNamesItsKind() { + #expect(DuckDBObjectQueries.macroSignature(parameterNames: ["a", "b"], isTableMacro: true) == "(a, b) AS TABLE") + #expect(DuckDBObjectQueries.macroSignature(parameterNames: ["a", "b"], isTableMacro: false) == "(a, b)") + #expect(DuckDBObjectQueries.macroSignature(parameterNames: [], isTableMacro: false) == "()") +} + +@Test func aScalarAndATableMacroOfOneNameAndParametersGetTwoSignatures() { + #expect(DuckDBObjectQueries.macroSignature(parameterNames: ["a"], isTableMacro: true) + != DuckDBObjectQueries.macroSignature(parameterNames: ["a"], isTableMacro: false)) +} + +@Test func aTableMacroSignatureDropsWithDropMacroTable() { + let signature = DuckDBObjectQueries.macroSignature(parameterNames: ["a"], isTableMacro: true) + #expect(DuckDBObjectQueries.namesTableMacro(signature)) + #expect(DuckDBObjectQueries.dropMacro( + name: "rows_of", schema: "main", isTableMacro: DuckDBObjectQueries.namesTableMacro(signature) + ) == #"DROP MACRO TABLE IF EXISTS "main"."rows_of""#) +} + +@Test(arguments: ["(a)", "()", "(x) AS TABLE)", nil] as [String?]) +func aScalarMacroSignatureDropsWithDropMacro(signature: String?) { + #expect(!DuckDBObjectQueries.namesTableMacro(signature)) +} diff --git a/apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBViewDefinitionTests.swift b/apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBViewDefinitionTests.swift new file mode 100644 index 0000000000..2ba14437b7 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBViewDefinitionTests.swift @@ -0,0 +1,18 @@ +import Foundation +import TableProDuckDBCore +import Testing + +@Test func aStoredCreateViewRunsAgainAsCreateOrReplace() { + #expect(DuckDBViewDefinition.makeReplaceable("CREATE VIEW v AS SELECT 1;") == "CREATE OR REPLACE VIEW v AS SELECT 1;") +} + +@Test func theLeadingKeywordsMatchInAnyCaseAndSurroundingSpaceIsTrimmed() { + #expect(DuckDBViewDefinition.makeReplaceable(" create view v AS SELECT 1\n") == "CREATE OR REPLACE VIEW v AS SELECT 1") +} + +@Test func aDefinitionThatIsNotAPlainCreateViewComesBackUntouched() { + let replaceable = "CREATE OR REPLACE VIEW v AS SELECT 1" + #expect(DuckDBViewDefinition.makeReplaceable(replaceable) == replaceable) + #expect(DuckDBViewDefinition.makeReplaceable("SELECT 1") == "SELECT 1") + #expect(DuckDBViewDefinition.makeReplaceable("") == "") +} diff --git a/apple/Packages/TableProCore/Tests/TableProGzipTests/GzipDeflaterTests.swift b/apple/Packages/TableProCore/Tests/TableProGzipTests/GzipDeflaterTests.swift new file mode 100644 index 0000000000..a7a34f5549 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProGzipTests/GzipDeflaterTests.swift @@ -0,0 +1,127 @@ +import Foundation +import TableProGzip +import TableProWireIO +import Testing + +private enum DeflaterFixture { + static func directory() throws -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + return url + } + + static func randomBytes(_ count: Int) -> Data { + var generator = SystemRandomNumberGenerator() + return Data((0.. Data { + Data((0.. Int32 { + FileManager.default.createFile(atPath: destination.path(percentEncoded: false), contents: nil) + let output = try FileHandle(forWritingTo: destination) + defer { try? output.close() } + let process = Process() + process.executableURL = URL(fileURLWithPath: gzipPath) + process.arguments = ["-dc", source.path(percentEncoded: false)] + // Apple's gzip reads extra options from GZIP. + process.environment = [:] + process.standardOutput = output + try process.run() + process.waitUntilExit() + return process.terminationStatus + } +} + +@Test("Compressed data inflates back to itself", arguments: [0, 1, 4_096, 3 << 20]) +func deflatedDataRoundTrips(count: Int) throws { + let input = DeflaterFixture.dumpText(lines: count / 40 + 1).prefix(count) + let packed = try GzipDeflater.compress(Data(input)) + #expect(packed.prefix(2) == Data([0x1F, 0x8B])) + #expect(try GzipInflater.decompress(packed, budget: .init(limit: .max, ratioLimit: .max, grace: .max)) == Data(input)) +} + +@Test("Bytes that do not compress still round trip") +func incompressibleBytesRoundTrip() throws { + let input = DeflaterFixture.randomBytes(1 << 20) + let packed = try GzipDeflater.compress(input) + #expect(try GzipInflater.decompress(packed) == input) +} + +@Test("A compressed file larger than one read chunk inflates back to its source") +func compressedFileRoundTrips() throws { + let directory = try DeflaterFixture.directory() + defer { try? FileManager.default.removeItem(at: directory) } + let source = directory.appendingPathComponent("dump.sql") + let input = DeflaterFixture.dumpText(lines: 60_000) + DeflaterFixture.randomBytes(300_000) + #expect(input.count > 1 << 20) + try input.write(to: source) + let destination = directory.appendingPathComponent("dump.sql.gz") + + var reported: [(Int64, Int64)] = [] + try GzipFile.compress(source: source, to: destination, isCancelled: { false }) { reported.append(($0, $1)) } + + #expect(try GzipInflater.decompress(Data(contentsOf: destination)) == input) + #expect(reported.first.map { $0.0 == 0 } == true) + #expect(reported.last.map { $0.0 == Int64(input.count) && $0.1 == Int64(input.count) } == true) +} + +@Test( + "gzip -d reads what the deflater writes", + .enabled(if: FileManager.default.isExecutableFile(atPath: DeflaterFixture.gzipPath)), + arguments: [0, 5_000, 2 << 20] +) +func systemGzipReadsTheOutput(count: Int) throws { + let directory = try DeflaterFixture.directory() + defer { try? FileManager.default.removeItem(at: directory) } + let source = directory.appendingPathComponent("dump.sql") + let input = Data((DeflaterFixture.dumpText(lines: count / 40 + 1) + DeflaterFixture.randomBytes(count / 4)).prefix(count)) + try input.write(to: source) + let packed = directory.appendingPathComponent("dump.sql.gz") + try GzipFile.compress(source: source, to: packed, isCancelled: { false }) + let unpacked = directory.appendingPathComponent("unpacked.sql") + + #expect(try DeflaterFixture.gunzip(packed, to: unpacked) == 0) + #expect(try Data(contentsOf: unpacked) == input) +} + +@Test("A stop partway through a file throws and leaves nothing at the destination") +func cancellationLeavesNoPartialFile() throws { + let directory = try DeflaterFixture.directory() + defer { try? FileManager.default.removeItem(at: directory) } + let source = directory.appendingPathComponent("dump.sql") + try DeflaterFixture.randomBytes(3 << 20).write(to: source) + let destination = directory.appendingPathComponent("dump.sql.gz") + + var checks = 0 + #expect(throws: CancellationError.self) { + try GzipFile.compress(source: source, to: destination, isCancelled: { + checks += 1 + return checks > 2 + }) + } + #expect(checks == 3) + #expect(!FileManager.default.fileExists(atPath: destination.path(percentEncoded: false))) +} + +@Test("A stop before the trailer is written throws rather than leaving a cut stream") +func cancellationAfterTheLastChunkThrows() throws { + let directory = try DeflaterFixture.directory() + defer { try? FileManager.default.removeItem(at: directory) } + let source = directory.appendingPathComponent("dump.sql") + try DeflaterFixture.dumpText(lines: 10).write(to: source) + let destination = directory.appendingPathComponent("dump.sql.gz") + + var checks = 0 + #expect(throws: CancellationError.self) { + try GzipFile.compress(source: source, to: destination, isCancelled: { + checks += 1 + return checks > 1 + }) + } + #expect(!FileManager.default.fileExists(atPath: destination.path(percentEncoded: false))) +} diff --git a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDropStatementsTests.swift b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDropStatementsTests.swift new file mode 100644 index 0000000000..5ba9612a32 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDropStatementsTests.swift @@ -0,0 +1,58 @@ +import Foundation +import TableProMSSQLCore +import Testing + +struct MSSQLDropStatementsTests { + @Test("A function drops with DROP FUNCTION, qualified by its schema") + func functionDrop() { + let signature = MSSQLRoutineSignature.listed(parameters: "@id int", objectType: "FN") + #expect(MSSQLDropStatements.routine(schema: "dbo", name: "fn_total", isFunction: true, signature: signature) + == "DROP FUNCTION IF EXISTS [dbo].[fn_total]") + } + + @Test("A procedure drops with DROP PROCEDURE, never DROP ROUTINE") + func procedureDrop() { + let statement = MSSQLDropStatements.routine(schema: "sales", name: "usp_close", isFunction: false, signature: "()") + #expect(statement == "DROP PROCEDURE IF EXISTS [sales].[usp_close]") + #expect(!statement.contains("ROUTINE")) + } + + @Test("A closing bracket in a routine or schema name is doubled") + func bracketInNameIsEscaped() { + #expect(MSSQLDropStatements.routine(schema: "a]b", name: "p]q", isFunction: false, signature: nil) + == "DROP PROCEDURE IF EXISTS [a]]b].[p]]q]") + } + + @Test("A CLR aggregate listed as a function drops with DROP AGGREGATE", arguments: ["AF", "af", "AF "]) + func aggregateDrop(objectType: String) { + let signature = MSSQLRoutineSignature.listed(parameters: "@value nvarchar", objectType: objectType) + #expect(MSSQLDropStatements.routine(schema: "dbo", name: "Concatenate", isFunction: true, signature: signature) + == "DROP AGGREGATE IF EXISTS [dbo].[Concatenate]") + } + + @Test("A CLR scalar or table-valued function still drops with DROP FUNCTION", arguments: ["FS", "FT", "IF", "TF"]) + func clrFunctionDrop(objectType: String) { + let signature = MSSQLRoutineSignature.listed(parameters: "", objectType: objectType) + #expect(MSSQLDropStatements.routine(schema: "dbo", name: "f", isFunction: true, signature: signature) + == "DROP FUNCTION IF EXISTS [dbo].[f]") + } + + @Test("A parameter named like the marker leaves a function a function") + func parameterNamedAggregate() { + let signature = MSSQLRoutineSignature.listed(parameters: "@x AGGREGATE", objectType: "FN") + #expect(signature == "(@x AGGREGATE)") + #expect(!MSSQLRoutineSignature.namesAggregate(signature)) + } + + @Test("An aggregate's signature keeps its parameter list") + func aggregateSignature() { + #expect(MSSQLRoutineSignature.listed(parameters: "@value nvarchar", objectType: "AF") + == "(@value nvarchar) AGGREGATE") + #expect(MSSQLRoutineSignature.listed(parameters: "@id int", objectType: "FN") == "(@id int)") + } + + @Test("A trigger drops by its schema and name, without its table") + func triggerDrop() { + #expect(MSSQLDropStatements.trigger(schema: "dbo", name: "trg_audit") == "DROP TRIGGER [dbo].[trg_audit]") + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift index 86af97fc78..3df7231bd4 100644 --- a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift @@ -463,6 +463,15 @@ struct MSSQLSchemaQueriesTests { #expect(MSSQLSchemaQueries.schemas.contains("'information_schema'")) } + @Test("The row estimate reads the heap or clustered partitions of the bracketed object") + func approximateRowCountQuery() { + #expect(MSSQLSchemaQueries.approximateRowCount(schema: "sales", table: "O'rders]") == """ + SELECT SUM(p.rows) + FROM sys.partitions p + WHERE p.object_id = OBJECT_ID(N'[sales].[O''rders]]]') AND p.index_id IN (0, 1) + """) + } + @Test("A rename brackets each part of the column it names, so a schema called [dbo] is not dbo") func renameColumnBracketsEachPart() { #expect( diff --git a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLTypeDefinitionTests.swift b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLTypeDefinitionTests.swift new file mode 100644 index 0000000000..ad1990361d --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLTypeDefinitionTests.swift @@ -0,0 +1,147 @@ +import Foundation +import TableProMSSQLCore +import Testing + +struct MSSQLTypeDefinitionTests { + @Test("An alias type names its base type and spells out its nullability") + func aliasStatement() throws { + #expect(try MSSQLTypeDefinition.aliasStatement( + schema: "dbo", name: "Email", baseType: "nvarchar(320)", isNullable: false + ) == "CREATE TYPE [dbo].[Email] FROM nvarchar(320) NOT NULL;") + #expect(try MSSQLTypeDefinition.aliasStatement( + schema: "dbo", name: "Code", baseType: "char(3)", isNullable: true + ) == "CREATE TYPE [dbo].[Code] FROM char(3) NULL;") + } + + @Test("A base type that would start a second statement is refused") + func aliasRefusesEscapingBaseType() { + #expect(throws: MSSQLDefinitionRefusal.self) { + try MSSQLTypeDefinition.aliasStatement( + schema: "dbo", name: "Bad", baseType: "int; DROP TABLE t", isNullable: true + ) + } + } + + @Test("A CLR type names its managed class, not the SQL type name") + func clrStatementUsesAssemblyClass() { + #expect(MSSQLTypeDefinition.clrStatement( + schema: "dbo", name: "Point", assembly: "Geo", assemblyClass: "Geo.PointType" + ) == "CREATE TYPE [dbo].[Point] EXTERNAL NAME [Geo].[Geo.PointType];") + #expect(MSSQLTypeDefinition.clrStatement( + schema: "dbo", name: "Point", assembly: "Geo", assemblyClass: "" + ) == "CREATE TYPE [dbo].[Point] EXTERNAL NAME [Geo].[Point];") + } + + @Test("A table type inlines a one-column key and leaves out the database's own collation") + func tableStatementInlinesSingleKey() throws { + let statement = try MSSQLTypeDefinition.tableStatement( + schema: "dbo", + name: "IdList", + columns: [ + MSSQLTypeDefinition.Column(name: "Id", type: "int", isNullable: false), + MSSQLTypeDefinition.Column( + name: "Label", type: "nvarchar(50)", isNullable: true, collation: "SQL_Latin1_General_CP1_CI_AS" + ), + ], + indexes: [ + MSSQLTypeDefinition.Index( + name: "PK__IdList", isPrimaryKey: true, isUnique: true, typeDescription: "CLUSTERED", + keys: [MSSQLTypeDefinition.IndexKey(column: "Id", isDescending: false)] + ), + ], + databaseCollation: "SQL_Latin1_General_CP1_CI_AS" + ) + #expect(statement == """ + CREATE TYPE [dbo].[IdList] AS TABLE ( + [Id] int NOT NULL PRIMARY KEY CLUSTERED, + [Label] nvarchar(50) NULL + ); + """) + } + + @Test("A memory-optimized table type keeps its hash index and its WITH clause") + func tableStatementMemoryOptimized() throws { + let statement = try MSSQLTypeDefinition.tableStatement( + schema: "dbo", + name: "Pairs", + columns: [ + MSSQLTypeDefinition.Column(name: "A", type: "int", isNullable: false), + MSSQLTypeDefinition.Column(name: "B", type: "int", isNullable: false), + ], + indexes: [ + MSSQLTypeDefinition.Index( + name: "IX_Pairs", isPrimaryKey: false, isUnique: false, typeDescription: "NONCLUSTERED HASH", + keys: [ + MSSQLTypeDefinition.IndexKey(column: "A", isDescending: false), + MSSQLTypeDefinition.IndexKey(column: "B", isDescending: true), + ], + bucketCount: 1_024 + ), + ], + checkConstraints: ["([A]>(0))"], + isMemoryOptimized: true, + databaseCollation: nil + ) + #expect(statement == """ + CREATE TYPE [dbo].[Pairs] AS TABLE ( + [A] int NOT NULL, + [B] int NOT NULL, + INDEX [IX_Pairs] HASH ([A], [B] DESC) WITH (BUCKET_COUNT = 1024), + CHECK ([A]>(0)) + ) + WITH (MEMORY_OPTIMIZED = ON); + """) + } + + @Test("Column rows map by position, and a row without a name is skipped") + func columnsFromRows() { + let columns = MSSQLTypeDefinition.columns(fromRows: [ + ["Id", "int", "0", "1,1", nil, nil, nil], + ["Total", nil, "1", nil, "([A]+[B])", nil, nil], + ["Note", "nvarchar(10)", "1", nil, nil, "(N'x')", "Latin1_General_CI_AS"], + [nil, "int", "1", nil, nil, nil, nil], + ]) + #expect(columns == [ + MSSQLTypeDefinition.Column(name: "Id", type: "int", isNullable: false, identitySpec: "1,1"), + MSSQLTypeDefinition.Column(name: "Total", type: nil, isNullable: true, computedDefinition: "([A]+[B])"), + MSSQLTypeDefinition.Column( + name: "Note", type: "nvarchar(10)", isNullable: true, defaultDefinition: "(N'x')", + collation: "Latin1_General_CI_AS" + ), + ]) + } + + @Test("Index rows fold into one index per index_id, keys in row order") + func indexesFromRows() { + let indexes = MSSQLTypeDefinition.indexes(fromRows: [ + ["PK_T", "1", "1", "CLUSTERED", "Id", "0", "1", "0"], + ["IX_T", "0", "0", "NONCLUSTERED", "A", "0", "2", "0"], + ["IX_T", "0", "0", "NONCLUSTERED", "B", "1", "2", "0"], + ["IX_H", "0", "0", "NONCLUSTERED HASH", "C", "0", "3", "256"], + ["IX_X", "0", "0", "NONCLUSTERED", nil, "0", "4", "0"], + ]) + #expect(indexes == [ + MSSQLTypeDefinition.Index( + name: "PK_T", isPrimaryKey: true, isUnique: true, typeDescription: "CLUSTERED", + keys: [MSSQLTypeDefinition.IndexKey(column: "Id", isDescending: false)] + ), + MSSQLTypeDefinition.Index( + name: "IX_T", isPrimaryKey: false, isUnique: false, typeDescription: "NONCLUSTERED", + keys: [ + MSSQLTypeDefinition.IndexKey(column: "A", isDescending: false), + MSSQLTypeDefinition.IndexKey(column: "B", isDescending: true), + ] + ), + MSSQLTypeDefinition.Index( + name: "IX_H", isPrimaryKey: false, isUnique: false, typeDescription: "NONCLUSTERED HASH", + keys: [MSSQLTypeDefinition.IndexKey(column: "C", isDescending: false)], + bucketCount: 256 + ), + ]) + } + + @Test("Check rows give their definitions and skip a NULL one") + func checkConstraintsFromRows() { + #expect(MSSQLTypeDefinition.checkConstraints(fromRows: [["([A]>(0))"], [nil], []]) == ["([A]>(0))"]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLTypeQueriesTests.swift b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLTypeQueriesTests.swift new file mode 100644 index 0000000000..52d0ed1fb1 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLTypeQueriesTests.swift @@ -0,0 +1,51 @@ +import Foundation +import TableProMSSQLCore +import Testing + +struct MSSQLTypeQueriesTests { + @Test("The type list filters on the schema as an N literal and keys each type by user_type_id") + func typeListFiltersOnSchema() { + let sql = MSSQLTypeQueries.userDefinedTypeList(schema: "o'neil") + #expect(sql.contains("WHERE t.is_user_defined = 1 AND s.name = N'o''neil'")) + #expect(sql.contains("CONVERT(varchar(11), t.user_type_id) AS identity_id")) + #expect(sql.contains("ORDER BY t.name")) + } + + @Test("The type list names its kind in the spellings Kind reads") + func typeListKindSpellings() { + let sql = MSSQLTypeQueries.userDefinedTypeList(schema: "dbo") + for kind in [MSSQLTypeQueries.Kind.alias, .table, .clr] { + #expect(sql.contains("'\(kind.rawValue)'")) + } + #expect(MSSQLTypeQueries.Kind(rawValue: "TABLE") == .table) + } + + @Test("A unicode base type's byte length is halved back to characters") + func baseTypeHalvesUnicodeLength() { + let sql = MSSQLTypeQueries.userDefinedTypeList(schema: "dbo") + #expect(sql.contains("CONVERT(varchar(11), t.max_length / 2)")) + } + + @Test("Each table type read filters on the schema and the type name", arguments: [ + MSSQLTypeQueries.tableTypeColumns(schema: "dbo", name: "Id]List"), + MSSQLTypeQueries.tableTypeIndexes(schema: "dbo", name: "Id]List"), + MSSQLTypeQueries.tableTypeCheckConstraints(schema: "dbo", name: "Id]List"), + ]) + func tableTypeReadsFilterOnNames(sql: String) { + #expect(sql.contains("AND s.name = N'dbo'")) + #expect(sql.contains("AND tt.name = N'Id]List'")) + } + + @Test("Index rows come one per key column in key order, included columns left out") + func tableTypeIndexesOnePerKey() { + let sql = MSSQLTypeQueries.tableTypeIndexes(schema: "dbo", name: "T") + #expect(sql.contains("AND ic.is_included_column = 0")) + #expect(sql.contains("ORDER BY i.index_id, ic.key_ordinal")) + } + + @Test("The database collation is read for the current database") + func databaseCollationReadsCurrentDatabase() { + #expect(MSSQLTypeQueries.databaseCollation + == "SELECT CONVERT(varchar(128), DATABASEPROPERTYEX(DB_NAME(), 'Collation'))") + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLExportQueryTests.swift b/apple/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLExportQueryTests.swift new file mode 100644 index 0000000000..61604371d3 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLExportQueryTests.swift @@ -0,0 +1,76 @@ +import Foundation +import Testing + +@testable import TableProMySQLCore + +struct MySQLExportQueryTests { + @Test("A function drop names the function kind, qualified by its database") + func functionDrop() { + let sql = MySQLObjectQueries.dropRoutine( + isFunction: true, schema: "app", name: "total", quote: MySQLObjectQueries.quoteIdentifier + ) + #expect(sql == "DROP FUNCTION IF EXISTS `app`.`total`") + } + + @Test("A procedure drop says PROCEDURE, never ROUTINE") + func procedureDrop() { + let sql = MySQLObjectQueries.dropRoutine( + isFunction: false, schema: "app", name: "cleanup", quote: MySQLObjectQueries.quoteIdentifier + ) + #expect(sql == "DROP PROCEDURE IF EXISTS `app`.`cleanup`") + #expect(!sql.contains("ROUTINE")) + } + + @Test("A routine drop with no database is unqualified") + func unqualifiedRoutineDrop() { + let sql = MySQLObjectQueries.dropRoutine( + isFunction: false, schema: nil, name: "p", quote: MySQLObjectQueries.quoteIdentifier + ) + #expect(sql == "DROP PROCEDURE IF EXISTS `p`") + } + + @Test("A drop quotes with the caller's quoter") + func dropUsesCallerQuote() { + let sql = MySQLObjectQueries.dropRoutine(isFunction: true, schema: "d", name: "f", quote: { "\"\($0)\"" }) + #expect(sql == "DROP FUNCTION IF EXISTS \"d\".\"f\"") + } + + @Test("A trigger drop is qualified and doubles a backtick in the name") + func triggerDrop() { + let sql = MySQLObjectQueries.dropTrigger(schema: "app", name: "a`b", quote: MySQLObjectQueries.quoteIdentifier) + #expect(sql == "DROP TRIGGER `app`.`a``b`") + } + + @Test("The bulk column read covers the database, ordered for the per-table grouping") + func columnCatalogForDatabase() { + let sql = MySQLObjectQueries.columnCatalog(schema: "app", table: nil, projectsGenerationExpression: true) + #expect(sql == """ + SELECT + TABLE_NAME, COLUMN_NAME, COLUMN_TYPE, COLLATION_NAME, + IS_NULLABLE, COLUMN_KEY, COLUMN_DEFAULT, EXTRA, COLUMN_COMMENT, + GENERATION_EXPRESSION + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = 'app' + ORDER BY TABLE_NAME, ORDINAL_POSITION + """) + } + + @Test("The column read narrows to one table and projects NULL where the catalog has no expression column") + func columnCatalogForTable() { + let sql = MySQLObjectQueries.columnCatalog(schema: "a'b", table: "t'1", projectsGenerationExpression: false) + #expect(sql.contains("WHERE TABLE_SCHEMA = 'a''b' AND TABLE_NAME = 't''1'\n")) + #expect(sql.contains("COLUMN_COMMENT,\n NULL\n")) + #expect(!sql.contains("GENERATION_EXPRESSION")) + } + + @Test("The row count read escapes both names") + func approximateRowCount() { + let sql = MySQLObjectQueries.approximateRowCount(schema: "app", table: "o'rders") + #expect(sql == """ + SELECT TABLE_ROWS + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = 'app' + AND TABLE_NAME = 'o''rders' + """) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SqlDialectTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SqlDialectTests.swift new file mode 100644 index 0000000000..23709ca7cc --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SqlDialectTests.swift @@ -0,0 +1,25 @@ +import TableProPluginKit +import Testing + +struct SqlDialectTests { + @Test("Dialect.from maps known database type ids") + func dialect_from_database_type_id() { + #expect(SqlDialect.from(databaseTypeId: "PostgreSQL") == .postgres) + #expect(SqlDialect.from(databaseTypeId: "Redshift") == .postgres) + #expect(SqlDialect.from(databaseTypeId: "Greenplum") == .postgres) + #expect(SqlDialect.from(databaseTypeId: "AlloyDB") == .postgres) + #expect(SqlDialect.from(databaseTypeId: "Citus") == .postgres) + #expect(SqlDialect.from(databaseTypeId: "CockroachDB") == .postgres) + #expect(SqlDialect.from(databaseTypeId: "PGlite") == .postgres) + #expect(SqlDialect.from(databaseTypeId: "MySQL") == .mysql) + #expect(SqlDialect.from(databaseTypeId: "MariaDB") == .mysql) + #expect(SqlDialect.from(databaseTypeId: "TiDB") == .mysql) + #expect(SqlDialect.from(databaseTypeId: "Databend") == .generic) + #expect(SqlDialect.from(databaseTypeId: "SQLite") == .sqlite) + #expect(SqlDialect.from(databaseTypeId: "DuckDB") == .sqlite) + #expect(SqlDialect.from(databaseTypeId: "Cloudflare D1") == .sqlite) + #expect(SqlDialect.from(databaseTypeId: "Oracle") == .oracle) + #expect(SqlDialect.from(databaseTypeId: "Dameng") == .generic) + #expect(SqlDialect.from(databaseTypeId: "Unknown Whatever") == .generic) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPostgreSQLCoreTests/PostgreSQLForeignKeyChecksTests.swift b/apple/Packages/TableProCore/Tests/TableProPostgreSQLCoreTests/PostgreSQLForeignKeyChecksTests.swift new file mode 100644 index 0000000000..6a482d0d77 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPostgreSQLCoreTests/PostgreSQLForeignKeyChecksTests.swift @@ -0,0 +1,15 @@ +import Foundation +import TableProPostgreSQLCore +import Testing + +struct PostgreSQLForeignKeyChecksTests { + @Test("The foreign key switch swallows a refused SET, both ways") + func switchSwallowsRefusal() { + #expect(PostgreSQLForeignKeyChecks.disableStatements == [ + "DO $$ BEGIN SET session_replication_role = replica; EXCEPTION WHEN insufficient_privilege THEN NULL; END $$", + ]) + #expect(PostgreSQLForeignKeyChecks.enableStatements == [ + "DO $$ BEGIN SET session_replication_role = DEFAULT; EXCEPTION WHEN insufficient_privilege THEN NULL; END $$", + ]) + } +} diff --git a/apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/SQL/SQLChunkDecoderTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLChunkDecoderTests.swift similarity index 81% rename from apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/SQL/SQLChunkDecoderTests.swift rename to apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLChunkDecoderTests.swift index 5e18848950..d6d138941f 100644 --- a/apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/SQL/SQLChunkDecoderTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLChunkDecoderTests.swift @@ -1,7 +1,7 @@ import Foundation import Testing -@testable import TableProUtilities +@testable import TableProSQLFile struct SQLChunkDecoderTests { private func decodeInChunks(_ data: Data, encoding: String.Encoding, chunk size: Int) -> String? { @@ -17,10 +17,10 @@ struct SQLChunkDecoderTests { return decoder.hasPendingBytes ? nil : text } - /// The byte order mark is at the start of the file, so only the first chunk carries one, and - /// a chunk of UTF-16 without one decodes as big-endian. Every chunk after the first in a - /// little-endian file therefore came back byte-swapped, and a dump over 64 KiB imported as - /// CJK from its first boundary on with nothing raised. + // The byte order mark is at the start of the file, so only the first chunk carries one, and + // a chunk of UTF-16 without one decodes as big-endian. Every chunk after the first in a + // little-endian file therefore came back byte-swapped, and a dump over 64 KiB imported as + // CJK from its first boundary on with nothing raised. @Test("A UTF-16 file keeps its byte order past the first chunk") func utf16KeepsItsByteOrder() throws { let text = "SELECT 'a';\nSELECT 'b';\nSELECT 'c';\n" @@ -50,8 +50,8 @@ struct SQLChunkDecoderTests { #expect(decodeInChunks(data, encoding: .utf16LittleEndian, chunk: 3) == "\u{0}ab") } - /// Without a mark there is nothing to read the order from, and big-endian is what Foundation - /// itself assumes. + // Without a mark there is nothing to read the order from, and big-endian is what Foundation + // itself assumes. @Test("UTF-16 with no mark stays big-endian, and an explicit choice is honoured") func utf16WithoutAMark() throws { let text = "SELECT 1;" @@ -61,8 +61,8 @@ struct SQLChunkDecoderTests { #expect(decodeInChunks(littleEndian, encoding: .utf16LittleEndian, chunk: 6) == text) } - /// A mark left in the text is a zero-width no-break space in front of the first statement, - /// which the server answers with a syntax error. Only the `.utf16` spelling drops one. + // A mark left in the text is a zero-width no-break space in front of the first statement, + // which the server answers with a syntax error. Only the `.utf16` spelling drops one. @Test("A byte order mark never reaches the text") func markIsStripped() throws { let data = try Data([0xFF, 0xFE]) + #require("SELECT 1;".data(using: .utf16LittleEndian)) @@ -72,8 +72,8 @@ struct SQLChunkDecoderTests { } } - /// A chunk holding an odd number of UTF-16 bytes decodes the even part and drops the last - /// byte without failing, so the alignment has to be held back rather than noticed. + // A chunk holding an odd number of UTF-16 bytes decodes the even part and drops the last + // byte without failing, so the alignment has to be held back rather than noticed. @Test("An odd chunk boundary loses no byte") func oddBoundariesLoseNothing() throws { let text = "SELECT 'abcdefgh';" @@ -96,8 +96,8 @@ struct SQLChunkDecoderTests { } } - /// Shift JIS is not in the import menu, and the decoder must not be the reason it could never - /// be: only UTF-8 carried a partial character across a boundary before. + // Shift JIS is not in the import menu, and the decoder must not be the reason it could never + // be: only UTF-8 carried a partial character across a boundary before. @Test("A multi-byte encoding outside the menu is carried too") func otherMultiByteEncodingsAreCarried() throws { let text = "SELECT '日本語';" @@ -107,8 +107,8 @@ struct SQLChunkDecoderTests { } } - /// Latin-1 and Windows-1252 disagree over 0x80 to 0x9F, which is where a MySQL dump keeps its - /// curly quotes and its euro sign. + // Latin-1 and Windows-1252 disagree over 0x80 to 0x9F, which is where a MySQL dump keeps its + // curly quotes and its euro sign. @Test("Latin-1 and Windows-1252 decode the C1 range differently") func singleByteEncodingsDiffer() { let data = Data([0x27, 0x80, 0x92, 0x27]) diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLClientEncodingRewriterTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLClientEncodingRewriterTests.swift similarity index 94% rename from apple/TableProTests/Core/Utilities/SQL/SQLClientEncodingRewriterTests.swift rename to apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLClientEncodingRewriterTests.swift index dae26ab968..4d2a99acea 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLClientEncodingRewriterTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLClientEncodingRewriterTests.swift @@ -1,10 +1,6 @@ -// -// SQLClientEncodingRewriterTests.swift -// TableProTests -// - import Foundation -@testable import TablePro +@testable import TableProSQLFile +import TableProSQLGrammar import Testing struct SQLClientEncodingRewriterTests { @@ -74,7 +70,7 @@ struct SQLClientEncodingRewriterTests { @Test("Engines without a client encoding statement are left alone") func otherEnginesLeftAlone() { - #expect(SQLClientEncodingRewriter.rewritten("SET NAMES cp932", family: .sqlite, grammar: TestGrammar.sqlite) == nil) - #expect(SQLClientEncodingRewriter.rewritten("SET client_encoding = 'SJIS'", family: .sqlServer, grammar: TestGrammar.sqlServer) == nil) + #expect(SQLClientEncodingRewriter.rewritten("SET NAMES cp932", family: .other, grammar: TestGrammar.sqlite) == nil) + #expect(SQLClientEncodingRewriter.rewritten("SET client_encoding = 'SJIS'", family: .other, grammar: TestGrammar.sqlServer) == nil) } } diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLDumpEncodingChoiceTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLDumpEncodingChoiceTests.swift similarity index 63% rename from apple/TableProTests/Core/Utilities/SQL/SQLDumpEncodingChoiceTests.swift rename to apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLDumpEncodingChoiceTests.swift index 070c13d1a8..5779485f21 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLDumpEncodingChoiceTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLDumpEncodingChoiceTests.swift @@ -1,15 +1,17 @@ -// -// SQLDumpEncodingChoiceTests.swift -// TableProTests -// - import Foundation -@testable import TablePro +@testable import TableProSQLFile +import TableProSQLGrammar import TableProTabularIO import Testing struct SQLDumpEncodingChoiceTests { - private func choice(_ bytes: Data, replacing selected: ImportEncoding = .utf8, family: TransactionEngineFamily = .mysql) -> ImportEncoding? { + private static let hidesABackslash: Set = [ + TabularTextEncoding.shiftJIS.foundationEncoding, + TabularTextEncoding.gb18030.foundationEncoding, + TabularTextEncoding.big5.foundationEncoding, + ] + + private func choice(_ bytes: Data, replacing selected: String.Encoding = .utf8, family: SQLDumpFamily = .mysql) -> String.Encoding? { SQLDumpEncodingChoice.encoding( replacing: selected, forPreview: bytes, @@ -29,11 +31,11 @@ struct SQLDumpEncodingChoiceTests { bytes.append(try shiftJIS("日本語テストのデータです")) bytes.append(contentsOf: Array("'),(2,'".utf8) + [0xE9, 0x5C, 0x27] + Array("); DROP TABLE users; -- '),(3,'bob');\n".utf8)) #expect(TabularEncodingDetector.sniff(bytes).encoding == .shiftJIS) - #expect(choice(bytes).map { !$0.canHideABackslashInsideACharacter } ?? true) + #expect(choice(bytes).map { !Self.hidesABackslash.contains($0) } ?? true) let latin1 = Data([0x2F, 0x2A, 0x21, 0x34, 0x30, 0x31, 0x30, 0x31] + Array(" SET NAMES latin1 */;\nINSERT INTO t VALUES ('Caf".utf8) + [0xE9, 0x5C, 0x27] + Array("s');\n".utf8)) - #expect(choice(latin1) == .windows1252) + #expect(choice(latin1) == .windowsCP1252) } @Test("Without a declaration, an encoding that can hide a backslash inside a character is never picked for you") @@ -47,17 +49,17 @@ struct SQLDumpEncodingChoiceTests { @Test("A Shift JIS mysqldump or pg_dump that says so is read as Shift JIS") func declaredShiftJISIsPicked() throws { let mysql = try shiftJIS("/*!40101 SET NAMES cp932 */;\nINSERT INTO t VALUES ('日本語');\n") - #expect(choice(mysql) == .shiftJIS) + #expect(choice(mysql) == TabularTextEncoding.shiftJIS.foundationEncoding) let postgres = try shiftJIS("SET client_encoding = 'SJIS';\nINSERT INTO t VALUES ('日本語');\n") - #expect(choice(postgres, family: .postgres) == .shiftJIS) + #expect(choice(postgres, family: .postgres) == TabularTextEncoding.shiftJIS.foundationEncoding) } @Test("An undeclared EUC-JP or Latin-1 script is read in the encoding it looks like") func undeclaredSafeEncodingsArePicked() throws { let euc = try #require("INSERT INTO t VALUES ('日本語のテキストです', '東京都港区');\n".data(using: .japaneseEUC)) - #expect(choice(euc) == .eucJP) + #expect(choice(euc) == TabularTextEncoding.eucJP.foundationEncoding) let western = try #require("INSERT INTO t VALUES ('Müller', 'Straße', 'Café crème');\n".data(using: .windowsCP1252)) - #expect(choice(western) == .windows1252) + #expect(choice(western) == .windowsCP1252) } @Test("A declaration matching the chosen encoding changes nothing") @@ -66,4 +68,16 @@ struct SQLDumpEncodingChoiceTests { #expect(choice(bytes) == nil) #expect(SQLDumpEncodingChoice.declaredEncoding(in: bytes, family: .mysql, grammar: TestGrammar.mysql) == .utf8) } + + @Test("A non-UTF-8 byte near the end of a whole file does not read as UTF-8, though a chunk would hold it back") + func heldBackBytesAtTheEndOfAWholeFileDoNotDecode() { + var bytes = Data("SET NAMES latin1;\nINSERT INTO t VALUES ('caf".utf8) + bytes.append(0xE9) + bytes.append(contentsOf: Data("');\n".utf8)) + + #expect(SQLDumpEncodingChoice.decodedPreview(bytes, as: .utf8, isWholeFile: true) == nil) + #expect(SQLDumpEncodingChoice.decodedPreview(bytes, as: .utf8, isWholeFile: false) != nil) + #expect(SQLDumpEncodingChoice.decodedPreview(bytes, as: .windowsCP1252, isWholeFile: true)?.hasSuffix("café');\n") == true) + #expect(choice(bytes) == .windowsCP1252) + } } diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserBatchTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserBatchTests.swift similarity index 93% rename from apple/TableProTests/Core/Utilities/SQL/SQLFileParserBatchTests.swift rename to apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserBatchTests.swift index 22157d165b..8c2f8a4023 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserBatchTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserBatchTests.swift @@ -1,17 +1,12 @@ +// A SQL Server script imported from a file is read the way sqlcmd reads it: a line holding only `GO` ends a batch, +// the batch reaches the server whole, and the `GO` line reaches it not at all. The parser streams the file in 64 KiB +// chunks, so a `GO` line can be cut in two by a chunk boundary, and the split must not depend on where it falls. // -// SQLFileParserBatchTests.swift -// TableProTests -// -// A SQL Server script imported from a file is read the way sqlcmd reads it: a line holding only `GO` ends a batch, -// the batch reaches the server whole, and the `GO` line reaches it not at all. The parser streams the file in 64 KiB -// chunks, so a `GO` line can be cut in two by a chunk boundary, and the split must not depend on where it falls. -// -// A file with no `GO` line at all was written to run a statement at a time, the way TablePro wrote every SQL Server -// dump before it wrote `GO` lines, so it is still cut at each `;`. -// +// A file with no `GO` line at all was written to run a statement at a time, the way TablePro wrote every SQL Server +// dump before it wrote `GO` lines, so it is still cut at each `;`. import Foundation -@testable import TablePro +@testable import TableProSQLFile import TableProSQLGrammar import Testing @@ -92,8 +87,8 @@ struct SQLFileParserBatchTests { ]) } - /// Read as one batch, this dump fails on its view with Msg 111, "'CREATE VIEW' must be the first statement in a - /// query batch", and runs none of its statements. + // Read as one batch, this dump fails on its view with Msg 111, "'CREATE VIEW' must be the first statement in a + // query batch", and runs none of its statements. @Test("A dump with no GO line runs a statement at a time, and the count agrees") func dumpWithoutGoRunsStatements() async throws { let dump = """ diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserBytesReadTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserBytesReadTests.swift similarity index 96% rename from apple/TableProTests/Core/Utilities/SQL/SQLFileParserBytesReadTests.swift rename to apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserBytesReadTests.swift index a517d3b548..9003640378 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserBytesReadTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserBytesReadTests.swift @@ -1,12 +1,6 @@ -// -// SQLFileParserBytesReadTests.swift -// TableProTests -// - import Foundation import os -@testable import TablePro -import TableProPluginKit +@testable import TableProSQLFile import TableProSQLGrammar import Testing @@ -115,7 +109,7 @@ struct SQLFileParserBytesReadTests { copiesRead += 1 let before = reports.all var data = Data() - while let piece = try copyRows.nextPiece() { + while let piece = try copyRows() { data.append(piece) } let after = reports.all diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserCopyRowsTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserCopyRowsTests.swift similarity index 92% rename from apple/TableProTests/Core/Utilities/SQL/SQLFileParserCopyRowsTests.swift rename to apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserCopyRowsTests.swift index e227718272..fafeaf649b 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserCopyRowsTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserCopyRowsTests.swift @@ -1,14 +1,8 @@ -// -// SQLFileParserCopyRowsTests.swift -// TableProTests -// - import Foundation -import TableProPluginKit import TableProSQLGrammar import Testing -@testable import TablePro +@testable import TableProSQLFile struct SQLFileParserCopyRowsTests { private enum Element: Equatable { @@ -28,23 +22,20 @@ struct SQLFileParserCopyRowsTests { return try await body(url) } - private static func pieces(of rows: PluginCopyData) throws -> [Data] { + private static func pieces(of rows: () throws -> Data?) throws -> [Data] { var pieces: [Data] = [] - while let piece = try rows.nextPiece() { + while let piece = try rows() { pieces.append(piece) } return pieces } - private static func read(_ element: PluginImportElement) throws -> Element { + private static func read(_ element: SQLFileElement) throws -> Element { switch element { - case let .statement(statement, lineNumber): - return .statement(statement, line: lineNumber) - case let .copyFromStandardInput(statement, lineNumber, rows): - return .copy(statement, line: lineNumber, rows: Data(try pieces(of: rows).joined())) - @unknown default: - Issue.record("An element this test does not read: \(element)") - return .statement("", line: 0) + case let .statement(statement, line): + return .statement(statement, line: line) + case let .copyFromStandardInput(statement, line, rows): + return .copy(statement, line: line, rows: Data(try pieces(of: rows).joined())) } } @@ -141,7 +132,7 @@ struct SQLFileParserCopyRowsTests { } let next = try #require(try await iterator.next()) #expect(try Self.read(next) == .statement("SELECT 5", line: 5)) - #expect(try rows.nextPiece() == nil) + #expect(try rows() == nil) let end = try await iterator.next().map { try Self.read($0) } #expect(end == nil) } @@ -158,10 +149,10 @@ struct SQLFileParserCopyRowsTests { Issue.record("Expected the COPY, got \(copy)") return } - #expect(try rows.nextPiece() == Data("1\n2\n".utf8)) + #expect(try rows() == Data("1\n2\n".utf8)) let next = try #require(try await iterator.next()) #expect(try Self.read(next) == .statement("SELECT 5", line: 8)) - #expect(try rows.nextPiece() == nil) + #expect(try rows() == nil) } } diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserSharedCorpusTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserSharedCorpusTests.swift similarity index 61% rename from apple/TableProTests/Core/Utilities/SQL/SQLFileParserSharedCorpusTests.swift rename to apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserSharedCorpusTests.swift index bf4ac203b8..c9bffcd1ad 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserSharedCorpusTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserSharedCorpusTests.swift @@ -1,16 +1,49 @@ -// -// SQLFileParserSharedCorpusTests.swift -// TableProTests -// -// The corpus the statement scanner and the core's script reader run, read by the import a chunk at a time. The import -// drops the comments between statements' code, so each statement is compared as its code without comments or blanks. -// +// The corpus the statement scanner and the core's script reader run, read by the import a chunk at a time. The import +// drops the comments between statements' code, so each statement is compared as its code without comments or blanks. import Foundation import TableProSQLGrammar import Testing -@testable import TablePro +@testable import TableProSQLFile + +private struct SharedSQLScriptCorpus: Decodable { + struct Batch: Decodable { + let text: String + let `repeat`: Int + } + + struct Case: Decodable { + let name: String + let dialect: String + let sql: String + let statements: [String] + let batches: [Batch]? + + var grammar: SQLLexicalGrammar? { + Self.databaseTypeIds[dialect].map { TestGrammar.of($0) } + } + + private static let databaseTypeIds = [ + "postgresql": "PostgreSQL", "mysql": "MySQL", "sqlite": "SQLite", "mssql": "SQL Server", + "clickhouse": "ClickHouse", "duckdb": "DuckDB", "cql": "Cassandra", + ] + } + + let cases: [Case] + + static func load() throws -> [Case] { + let url = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("core/crates/core/tests/fixtures/sql_script_corpus.json") + return try JSONDecoder().decode(SharedSQLScriptCorpus.self, from: Data(contentsOf: url)).cases + } +} struct SQLFileParserSharedCorpusTests { // The code a statement holds: comments dropped, blanks outside literals dropped. @@ -52,8 +85,7 @@ struct SQLFileParserSharedCorpusTests { @Test("The import cuts each corpus case where the engine does, however the file is read") func importCutsEachCase() async throws { for testCase in try SharedSQLScriptCorpus.load() { - let type = try #require(testCase.databaseType, "\(testCase.dialect)") - let grammar = type.lexicalGrammar + let grammar = try #require(testCase.grammar, "\(testCase.dialect)") let runs = testCase.batches.map { batches in batches.flatMap { batch in Array(repeating: batch.text, count: batch.repeat) } } ?? testCase.statements diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserTests.swift similarity index 92% rename from apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift rename to apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserTests.swift index 4523f695f1..aacf4ed29c 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLFileParserTests.swift @@ -1,15 +1,9 @@ -// -// SQLFileParserTests.swift -// TableProTests -// - import Foundation import os -import TableProPluginKit import TableProSQLGrammar import Testing -@testable import TablePro +@testable import TableProSQLFile struct SQLFileParserTests { private static func parse(_ sql: String, grammar: SQLLexicalGrammar) async throws -> [String] { @@ -124,7 +118,7 @@ struct SQLFileParserTests { SELECT b'has\\'quote;', xb'\\'; SELECT 3; """ - let stmts = try await Self.parse(sql, grammar: DatabaseType.cockroachdb.lexicalGrammar) + let stmts = try await Self.parse(sql, grammar: TestGrammar.cockroachDB) #expect(stmts.count == 2) #expect(stmts[0].contains("b'has\\'quote;'")) #expect(stmts[1] == "SELECT 3") @@ -258,8 +252,8 @@ struct SQLFileParserTests { MERGE dbo.t AS t USING dbo.s AS s ON t.id = s.id WHEN NOT MATCHED THEN INSERT (id, v) VALUES (s.id, s.v) """ - /// SQL import sends a SQL Server file with no `GO` line a statement at a time, and SQL Server fails a `MERGE` sent - /// without its `;` with Msg 10713. + // SQL import sends a SQL Server file with no `GO` line a statement at a time, and SQL Server fails a `MERGE` sent + // without its `;` with Msg 10713. @Test("SQL Server: a MERGE keeps its ; in the middle of a file and at its end") func sqlServerMergeKeepsItsTerminator() async throws { let sql = """ @@ -278,7 +272,7 @@ struct SQLFileParserTests { ]) } - /// SQL Server reads `1MERGE` as a number and a `MERGE`, as the statement scanner does. + // SQL Server reads `1MERGE` as a number and a `MERGE`, as the statement scanner does. @Test("SQL Server: a MERGE glued to a number keeps its ;") func sqlServerMergeGluedToNumberKeepsItsTerminator() async throws { let sql = "SELECT 1\(Self.merge);\nSELECT 1.\(Self.merge);" @@ -301,8 +295,8 @@ struct SQLFileParserTests { #expect(stmts == ["SELECT '\(padding)'", Self.merge + ";"]) } - /// A T-SQL routine has no `DELIMITER` and no dollar quoting, so in a file with no `GO` line only the statement - /// grammar keeps its body whole. + // A T-SQL routine has no `DELIMITER` and no dollar quoting, so in a file with no `GO` line only the statement + // grammar keeps its body whole. @Test("SQL Server: a procedure body arrives whole") func sqlServerProcedureBodyArrivesWhole() async throws { let procedure = "CREATE PROCEDURE dbo.p AS BEGIN UPDATE dbo.t SET v = 1; \(Self.merge); END" @@ -310,7 +304,7 @@ struct SQLFileParserTests { #expect(stmts == [procedure, "SELECT 1"]) } - /// A T-SQL routine runs to the end of its batch, as sqlcmd sends it, so its body arrives whole. + // A T-SQL routine runs to the end of its batch, as sqlcmd sends it, so its body arrives whole. @Test("SQL Server: a procedure body arrives whole in its batch") func sqlServerProcedureBodyArrivesWholeInItsBatch() async throws { let procedure = "CREATE PROCEDURE dbo.p AS BEGIN UPDATE dbo.t SET v = 1; \(Self.merge); END;" @@ -544,32 +538,11 @@ struct SQLFileParserTests { @Test("A UTF-8 file cut inside its last character fails rather than dropping the bytes") func fileCutInsideItsLastCharacter() async throws { let cut = Data(Data("SELECT 'é".utf8).dropLast()) - await #expect(throws: DecompressionError.self) { + await #expect(throws: SQLFileParseError.self) { _ = try await Self.parse(cut, encoding: .utf8) } } - @Test("Dialect.from maps known database type ids") - func dialect_from_database_type_id() { - #expect(SqlDialect.from(databaseTypeId: "PostgreSQL") == .postgres) - #expect(SqlDialect.from(databaseTypeId: "Redshift") == .postgres) - #expect(SqlDialect.from(databaseTypeId: "Greenplum") == .postgres) - #expect(SqlDialect.from(databaseTypeId: "AlloyDB") == .postgres) - #expect(SqlDialect.from(databaseTypeId: "Citus") == .postgres) - #expect(SqlDialect.from(databaseTypeId: "CockroachDB") == .postgres) - #expect(SqlDialect.from(databaseTypeId: "PGlite") == .postgres) - #expect(SqlDialect.from(databaseTypeId: "MySQL") == .mysql) - #expect(SqlDialect.from(databaseTypeId: "MariaDB") == .mysql) - #expect(SqlDialect.from(databaseTypeId: "TiDB") == .mysql) - #expect(SqlDialect.from(databaseTypeId: "Databend") == .generic) - #expect(SqlDialect.from(databaseTypeId: "SQLite") == .sqlite) - #expect(SqlDialect.from(databaseTypeId: "DuckDB") == .sqlite) - #expect(SqlDialect.from(databaseTypeId: "Cloudflare D1") == .sqlite) - #expect(SqlDialect.from(databaseTypeId: "Oracle") == .oracle) - #expect(SqlDialect.from(databaseTypeId: "Dameng") == .generic) - #expect(SqlDialect.from(databaseTypeId: "Unknown Whatever") == .generic) - } - private static func parse(_ data: Data, encoding: String.Encoding) async throws -> [String] { let url = FileManager.default.temporaryDirectory .appendingPathComponent(UUID().uuidString + ".sql") @@ -584,9 +557,9 @@ struct SQLFileParserTests { return statements } - /// The parser reads 64 KiB at a time, and a chunk of UTF-16 without a byte order mark decodes - /// as big-endian, so every statement past the first boundary used to arrive byte-swapped: a - /// dump this size imported as CJK from 64 KiB on, with nothing raised. + // The parser reads 64 KiB at a time, and a chunk of UTF-16 without a byte order mark decodes + // as big-endian, so every statement past the first boundary used to arrive byte-swapped: a + // dump this size imported as CJK from 64 KiB on, with nothing raised. @Test("A UTF-16 dump larger than one chunk parses to the statements it holds") func utf16DumpLargerThanOneChunk() async throws { let lines = (0..<4_000).map { "INSERT INTO t (a) VALUES ('メール \($0)');" } @@ -601,8 +574,8 @@ struct SQLFileParserTests { #expect(withoutMark == lines.map { String($0.dropLast()) }) } - /// A multi-byte character landing on a chunk boundary is the ordinary case in a dump of - /// non-Latin text, and only UTF-8 used to carry one across. + // A multi-byte character landing on a chunk boundary is the ordinary case in a dump of + // non-Latin text, and only UTF-8 used to carry one across. @Test("A Shift JIS dump larger than one chunk parses to the statements it holds") func shiftJISDumpLargerThanOneChunk() async throws { let lines = (0..<4_000).map { "INSERT INTO t (a) VALUES ('日本語 \($0)');" } @@ -634,7 +607,7 @@ struct SQLFileParserTests { @Test("An imported CQL batch arrives whole, as the editor sends it") func cqlBatchImportsWhole() async throws { - let grammar = DatabaseType.cassandra.lexicalGrammar + let grammar = TestGrammar.cassandra #expect(try await Self.parse(Self.cqlScript, grammar: grammar) == Self.cqlStatements) let edited = SQLStatementScanner.executableStatements(in: Self.cqlScript, grammar: grammar).map { $0.sql } #expect(edited == Self.cqlStatements) @@ -657,9 +630,9 @@ struct SQLFileParserTests { let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") try dump.write(to: url, atomically: true, encoding: .utf8) defer { try? FileManager.default.removeItem(at: url) } - let noEscapes = SQLLexicalResolver.readings(for: .mysql, session: PluginSessionLexicalState( + let noEscapes = TestGrammar.of("MySQL", session: SQLSessionLexicalFacts( determined: [.backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes], enabled: [] - )).execution + )) let escapesOff = OSAllocatedUnfairLock(initialState: false) var statements: [String] = [] @@ -678,7 +651,7 @@ struct SQLFileParserTests { @Test("A chunk boundary anywhere in a CQL batch changes nothing") func cqlBatchChunkBoundaryAnywhere() async throws { - let grammar = DatabaseType.cassandra.lexicalGrammar + let grammar = TestGrammar.cassandra let chunkSize = 65_536 for boundary in 0..<(Self.cqlScript as NSString).length { let padding = "--" + String(repeating: "x", count: chunkSize - boundary - 3) + "\n" @@ -722,7 +695,7 @@ struct SQLFileParserTests { #expect(try await Self.parse("DELIMITER // -- set\nSELECT 1//\nSELECT 2//\n", grammar: TestGrammar.mysql) == [ "SELECT 1", "SELECT 2", ]) - let cockroach = DatabaseType.cockroachdb.lexicalGrammar + let cockroach = TestGrammar.cockroachDB #expect(try await Self.parse("INSERT INTO t VALUES (b'\\''); DELETE FROM x; SELECT 'a';", grammar: cockroach) == [ "INSERT INTO t VALUES (b'\\'')", "DELETE FROM x", "SELECT 'a'", ]) diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLTranscodedLiteralRewriterTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLTranscodedLiteralRewriterTests.swift similarity index 96% rename from apple/TableProTests/Core/Utilities/SQL/SQLTranscodedLiteralRewriterTests.swift rename to apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLTranscodedLiteralRewriterTests.swift index e35717fdb0..63fe476f2c 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLTranscodedLiteralRewriterTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/SQLTranscodedLiteralRewriterTests.swift @@ -1,11 +1,6 @@ -// -// SQLTranscodedLiteralRewriterTests.swift -// TableProTests -// - import Foundation -@testable import TablePro -@testable import TableProUtilities +@testable import TableProSQLFile +import TableProSQLGrammar import Testing struct SQLTranscodedLiteralRewriterTests { diff --git a/apple/Packages/TableProCore/Tests/TableProSQLFileTests/TestGrammar.swift b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/TestGrammar.swift new file mode 100644 index 0000000000..0221f51b90 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLFileTests/TestGrammar.swift @@ -0,0 +1,16 @@ +import TableProSQLGrammar + +// The grammar each engine is split with, resolved as both apps resolve it for an engine TablePro curates. +enum TestGrammar { + static func of(_ databaseTypeId: String, session: SQLSessionLexicalFacts? = nil) -> SQLLexicalGrammar { + SQLLexicalReadings.resolve(databaseTypeId: databaseTypeId, declared: nil, session: session).execution + } + + static let postgres = of("PostgreSQL") + static let mysql = of("MySQL") + static let sqlite = of("SQLite") + static let duckdb = of("DuckDB") + static let sqlServer = of("SQL Server") + static let cockroachDB = of("CockroachDB") + static let cassandra = of("Cassandra") +} diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLTokenCursorTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLTokenCursorTests.swift similarity index 93% rename from apple/TableProTests/Core/Utilities/SQL/SQLTokenCursorTests.swift rename to apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLTokenCursorTests.swift index 51e23e1f2f..aff0318e1b 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLTokenCursorTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLTokenCursorTests.swift @@ -1,15 +1,19 @@ -// -// SQLTokenCursorTests.swift -// TableProTests -// - import Foundation -@testable import TablePro -import TableProPluginKit import TableProSQLGrammar -@testable import TableProUtilities import Testing +private enum TestGrammar { + static let postgres = grammar(of: "PostgreSQL") + static let mysql = grammar(of: "MySQL") + static let sqlite = grammar(of: "SQLite") + static let oracle = grammar(of: "Oracle") + static let standard = SQLLexicalGrammar.ansi + + private static func grammar(of databaseTypeId: String) -> SQLLexicalGrammar { + SQLLexicalReadings.resolve(databaseTypeId: databaseTypeId, declared: nil, session: nil).execution + } +} + struct SQLTokenCursorTests { private static func tokens(_ sql: String, grammar: SQLLexicalGrammar) -> [SQLTokenCursor.Token] { var cursor = SQLTokenCursor(sql, grammar: grammar) diff --git a/apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/Text/ByteOrderMarkTests.swift b/apple/Packages/TableProCore/Tests/TableProTabularIOTests/ByteOrderMarkTests.swift similarity index 98% rename from apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/Text/ByteOrderMarkTests.swift rename to apple/Packages/TableProCore/Tests/TableProTabularIOTests/ByteOrderMarkTests.swift index 8338c5f75a..bcf3d651b2 100644 --- a/apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/Text/ByteOrderMarkTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProTabularIOTests/ByteOrderMarkTests.swift @@ -1,5 +1,5 @@ import Foundation -@testable import TableProUtilities +@testable import TableProTabularIO import Testing struct ByteOrderMarkTests { diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataDropTriggerTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataDropTriggerTests.swift new file mode 100644 index 0000000000..034a04e7d8 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataDropTriggerTests.swift @@ -0,0 +1,19 @@ +import TableProTeradataCore +import Testing + +struct TeradataDropTriggerTests { + @Test("A trigger drop names the trigger in its database") + func dropIsQualifiedByDatabase() { + #expect(TeradataObjectQueries.dropTriggerStatement(name: "audit", database: "shop") == #"DROP TRIGGER "shop"."audit""#) + } + + @Test("A trigger drop with no database names the trigger alone", arguments: [nil, ""] as [String?]) + func dropWithoutDatabase(database: String?) { + #expect(TeradataObjectQueries.dropTriggerStatement(name: "audit", database: database) == #"DROP TRIGGER "audit""#) + } + + @Test("A quote inside a name is doubled") + func quoteInsideANameIsDoubled() { + #expect(TeradataObjectQueries.dropTriggerStatement(name: #"a"b"#, database: #"d"b"#) == #"DROP TRIGGER "d""b"."a""b""#) + } +} diff --git a/apple/Packages/TableProMacCore/Package.swift b/apple/Packages/TableProMacCore/Package.swift index 016b422475..8bfcceb9d6 100644 --- a/apple/Packages/TableProMacCore/Package.swift +++ b/apple/Packages/TableProMacCore/Package.swift @@ -106,6 +106,7 @@ let package = Package( name: "TableProUtilitiesTests", dependencies: [ "TableProUtilities", + .product(name: "TableProTabularIO", package: "TableProCore"), .product(name: "TableProTestSupport", package: "TableProCore") ], path: "Tests/TableProUtilitiesTests", diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/TextPrefixDecoder.swift b/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/TextPrefixDecoder.swift index 262bb5d0d2..91bfabd9ad 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/TextPrefixDecoder.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/TextPrefixDecoder.swift @@ -1,4 +1,5 @@ import Foundation +import TableProTabularIO internal enum TextPrefixDecoder { struct Decoded { diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLSetAssignments.swift b/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLSetAssignments.swift index dc37e612a7..b9dcb10b44 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLSetAssignments.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/SQLSetAssignments.swift @@ -1,4 +1,5 @@ import Foundation +import TableProSQLGrammar /// One `name = value` element of a `SET` statement, as it was written. /// diff --git a/apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/TextPrefixDecoderTests.swift b/apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/TextPrefixDecoderTests.swift index 4bf5de0bf1..3117a7aef2 100644 --- a/apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/TextPrefixDecoderTests.swift +++ b/apple/Packages/TableProMacCore/Tests/TableProUtilitiesTests/TextPrefixDecoderTests.swift @@ -1,4 +1,5 @@ import Foundation +import TableProTabularIO @testable import TableProUtilities import Testing diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleDropStatements.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleDropStatements.swift new file mode 100644 index 0000000000..3c0f8990cf --- /dev/null +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleDropStatements.swift @@ -0,0 +1,68 @@ +import Foundation + +// A dump drops each object before creating it, and has to restore into a schema that never had it. 23ai takes +// `DROP ... IF EXISTS`; before it only a block that ignores the object's own "does not exist" error does. +public enum OracleDropStatements { + public enum Object: Sendable, CaseIterable { + case table + case view + case materializedView + case trigger + + var keyword: String { + switch self { + case .table: "TABLE" + case .view: "VIEW" + case .materializedView: "MATERIALIZED VIEW" + case .trigger: "TRIGGER" + } + } + + // ORA-00942 table or view, ORA-12003 materialized view, ORA-04080 trigger: "does not exist". + public var missingObjectCode: Int { + switch self { + case .table, .view: -942 + case .materializedView: -12_003 + case .trigger: -4_080 + } + } + } + + // An unknown release gets the block, which every release runs. + public static func drop( + _ object: Object, + name: String, + schema: String?, + currentSchema: String?, + release: OracleServerRelease? + ) -> String { + let target = target(name: name, schema: schema, currentSchema: currentSchema) + if let release, release.hasDropIfExists { + return "DROP \(object.keyword) IF EXISTS \(target)" + } + return ignoringMissingObject("DROP \(object.keyword) \(target)", code: object.missingObjectCode) + } + + // Unqualified in the current schema, so the dump restores into a schema of another name. + static func target(name: String, schema: String?, currentSchema: String?) -> String { + guard let owner = schema?.trimmingCharacters(in: .whitespacesAndNewlines), !owner.isEmpty, + owner != currentSchema + else { + return OracleSchemaQueries.quoteIdentifier(name) + } + return "\(OracleSchemaQueries.quoteIdentifier(owner)).\(OracleSchemaQueries.quoteIdentifier(name))" + } + + static func ignoringMissingObject(_ statement: String, code: Int) -> String { + """ + BEGIN + EXECUTE IMMEDIATE '\(OracleSchemaQueries.escapeLiteral(statement))'; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != \(code) THEN + RAISE; + END IF; + END; + """ + } +} diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleRoutineStatements.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleRoutineStatements.swift new file mode 100644 index 0000000000..5cb7534d8d --- /dev/null +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleRoutineStatements.swift @@ -0,0 +1,14 @@ +import Foundation + +public enum OracleRoutineStatements { + // ORA-04043: the object does not exist. + public static let missingObjectCode = -4_043 + + // A block on every release, so a dump that drops a routine before creating it also restores into a schema that + // never had it. + public static func drop(name: String, schema: String?, currentSchema: String?, isFunction: Bool) -> String { + let keyword = isFunction ? "FUNCTION" : "PROCEDURE" + let target = OracleDropStatements.target(name: name, schema: schema, currentSchema: currentSchema) + return OracleDropStatements.ignoringMissingObject("DROP \(keyword) \(target)", code: missingObjectCode) + } +} diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleServerRelease.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleServerRelease.swift index e7732e76a0..9a33ce4b0b 100644 --- a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleServerRelease.swift +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleServerRelease.swift @@ -1,14 +1,10 @@ import Foundation -/// The release of the server a session is on, which decides the dictionary columns a query may name. -/// -/// It comes from the login reply (`AUTH_VERSION_NO`), so every connected session has it without a query. A dictionary -/// column the release does not have fails the whole statement with ORA-00904, so a catalog read written for a newer -/// dictionary has to be written differently for an older one rather than sent and forgiven. +// From the login reply (`AUTH_VERSION_NO`), so no query reads it. A dictionary column the release lacks fails the whole +// statement with ORA-00904, so a catalog read is written per release. public struct OracleServerRelease: Sendable, Equatable { public let major: Int - /// The release update, the second field of `23.4.0.24.05`. 23ai kept its major through every update, and some of - /// its dictionary columns arrived in an update rather than with the major (26ai still reports `23.26`). + // The second field of `23.4.0.24.05`: some 23ai dictionary columns came with an update (26ai reports `23.26`). public let update: Int public init(major: Int, update: Int = 0) { @@ -16,19 +12,19 @@ public struct OracleServerRelease: Sendable, Equatable { self.update = update } - /// 12.1 brought identity columns, `DEFAULT ON NULL` and invisible columns, and with them the `IDENTITY_COLUMN`, - /// `DEFAULT_ON_NULL` and `USER_GENERATED` columns of `ALL_TAB_COLS`. 11g has none of the three. + // 12.1 brought identity, `DEFAULT ON NULL` and invisible columns, with `ALL_TAB_COLS.IDENTITY_COLUMN`, + // `DEFAULT_ON_NULL` and `USER_GENERATED`. 11g has none of the three. public var hasIdentityColumns: Bool { major >= 12 } - /// 23ai brought `DEFAULT ON NULL FOR INSERT AND UPDATE` and the `DEFAULT_ON_NULL_UPD` column that reports it. + // 23ai brought `DEFAULT ON NULL FOR INSERT AND UPDATE` and the `DEFAULT_ON_NULL_UPD` column that reports it. public var hasDefaultOnNullForUpdate: Bool { major >= 23 } - /// `VECTOR_INFO` came with the `VECTOR` type in 23.4, the first release with AI Vector Search. The 23.2 and 23.3 - /// developer releases are 23 too, so the major alone would name the column on a server that may not have it. + // `VECTOR_INFO` came with the `VECTOR` type in 23.4. The 23.2 and 23.3 developer releases are 23 too, so the major + // alone would name the column on a server that may not have it. public var hasVectorInfo: Bool { major > 23 || (major == 23 && update >= 4) } @@ -42,4 +38,9 @@ public struct OracleServerRelease: Sendable, Equatable { public var hasBooleanType: Bool { major >= 23 } + + // 23ai brought `IF EXISTS` on `DROP`. Before it the clause does not parse. + public var hasDropIfExists: Bool { + major >= 23 + } } diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleDropStatementsTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleDropStatementsTests.swift new file mode 100644 index 0000000000..ae9a409a18 --- /dev/null +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleDropStatementsTests.swift @@ -0,0 +1,81 @@ +import Foundation +import TableProOracleCore +import Testing + +struct OracleDropStatementsTests { + private let release19c = OracleServerRelease(major: 19, update: 3) + private let release23ai = OracleServerRelease(major: 23, update: 4) + + @Test func dropIfExistsArrivesIn23ai() { + #expect(!OracleServerRelease(major: 21).hasDropIfExists) + #expect(OracleServerRelease(major: 23).hasDropIfExists) + #expect(OracleServerRelease(major: 23, update: 26).hasDropIfExists) + } + + @Test func aTableBefore23aiDropsInABlockThatIgnoresOnlyAMissingTable() { + let sql = OracleDropStatements.drop(.table, name: "ORDERS", schema: "APP", currentSchema: "APP", release: release19c) + + #expect(sql == """ + BEGIN + EXECUTE IMMEDIATE 'DROP TABLE "ORDERS"'; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; + END; + """) + } + + @Test(arguments: [ + (OracleDropStatements.Object.view, "DROP VIEW", -942), + (.materializedView, "DROP MATERIALIZED VIEW", -12_003), + (.trigger, "DROP TRIGGER", -4_080), + ]) + func eachKindIgnoresItsOwnMissingObjectError(object: OracleDropStatements.Object, verb: String, code: Int) { + let sql = OracleDropStatements.drop(object, name: "X", schema: nil, currentSchema: "APP", release: release19c) + + #expect(sql.contains("EXECUTE IMMEDIATE '\(verb) \"X\"';")) + #expect(sql.contains("IF SQLCODE != \(code) THEN")) + #expect(object.missingObjectCode == code) + } + + @Test(arguments: OracleDropStatements.Object.allCases) + func from23aiTheDropTakesIfExists(object: OracleDropStatements.Object) { + let sql = OracleDropStatements.drop(object, name: "X", schema: "APP", currentSchema: "APP", release: release23ai) + + #expect(sql.hasPrefix("DROP ")) + #expect(sql.hasSuffix(" IF EXISTS \"X\"")) + #expect(!sql.contains("BEGIN")) + } + + @Test func aTriggerFrom23aiDrops() { + #expect(OracleDropStatements.drop(.trigger, name: "TRG", schema: "HR", currentSchema: "APP", release: release23ai) + == #"DROP TRIGGER IF EXISTS "HR"."TRG""#) + } + + @Test func anUnknownReleaseGetsTheBlockEveryReleaseRuns() { + let sql = OracleDropStatements.drop(.view, name: "V", schema: nil, currentSchema: nil, release: nil) + + #expect(sql.hasPrefix("BEGIN")) + #expect(sql.contains("EXECUTE IMMEDIATE 'DROP VIEW \"V\"';")) + } + + @Test func anObjectAnotherSchemaOwnsIsQualified() { + let sql = OracleDropStatements.drop(.table, name: "T", schema: "HR", currentSchema: "APP", release: release19c) + + #expect(sql.contains("EXECUTE IMMEDIATE 'DROP TABLE \"HR\".\"T\"';")) + } + + @Test func aBlankSchemaMeansTheCurrentOne() { + let sql = OracleDropStatements.drop(.table, name: "T", schema: " ", currentSchema: "APP", release: release23ai) + + #expect(sql == #"DROP TABLE IF EXISTS "T""#) + } + + @Test func aQuoteInTheNameIsDoubledInsideTheLiteral() { + let sql = OracleDropStatements.drop(.table, name: "it's", schema: nil, currentSchema: nil, release: release19c) + + #expect(sql.contains("EXECUTE IMMEDIATE 'DROP TABLE \"it''s\"';")) + } +} diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleRoutineStatementsTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleRoutineStatementsTests.swift new file mode 100644 index 0000000000..6b8ce33f11 --- /dev/null +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleRoutineStatementsTests.swift @@ -0,0 +1,53 @@ +import Foundation +import TableProOracleCore +import Testing + +struct OracleRoutineStatementsTests { + @Test func dropsAProcedureInTheCurrentSchemaByItsBareName() { + let sql = OracleRoutineStatements.drop(name: "P_SYNC", schema: "APP", currentSchema: "APP", isFunction: false) + + #expect(sql == """ + BEGIN + EXECUTE IMMEDIATE 'DROP PROCEDURE "P_SYNC"'; + EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -4043 THEN + RAISE; + END IF; + END; + """) + } + + @Test func dropsAFunctionWithTheFunctionKeyword() { + let sql = OracleRoutineStatements.drop(name: "F_TOTAL", schema: nil, currentSchema: "APP", isFunction: true) + + #expect(sql.contains("EXECUTE IMMEDIATE 'DROP FUNCTION \"F_TOTAL\"';")) + #expect(!sql.contains("PROCEDURE")) + } + + @Test func qualifiesARoutineAnotherSchemaOwns() { + let sql = OracleRoutineStatements.drop(name: "P_SYNC", schema: "HR", currentSchema: "APP", isFunction: false) + + #expect(sql.contains("EXECUTE IMMEDIATE 'DROP PROCEDURE \"HR\".\"P_SYNC\"';")) + } + + @Test func anEmptySchemaMeansTheCurrentOne() { + let sql = OracleRoutineStatements.drop(name: "P_SYNC", schema: " ", currentSchema: "APP", isFunction: false) + + #expect(sql.contains("EXECUTE IMMEDIATE 'DROP PROCEDURE \"P_SYNC\"';")) + } + + @Test func aQuoteInTheNameIsDoubledInsideTheLiteral() { + let sql = OracleRoutineStatements.drop(name: "it's", schema: nil, currentSchema: nil, isFunction: true) + + #expect(sql.contains("EXECUTE IMMEDIATE 'DROP FUNCTION \"it''s\"';")) + } + + @Test func onlyAMissingObjectIsIgnored() { + let sql = OracleRoutineStatements.drop(name: "P", schema: nil, currentSchema: nil, isFunction: false) + + #expect(sql.contains("IF SQLCODE != -4043 THEN")) + #expect(sql.contains("RAISE;")) + #expect(sql.hasSuffix("END;")) + } +} diff --git a/apple/Plugins/CSVExportPlugin/CSVEncodingReport.swift b/apple/Plugins/CSVExportPlugin/CSVEncodingReport.swift index 8267476783..36ce3b18b4 100644 --- a/apple/Plugins/CSVExportPlugin/CSVEncodingReport.swift +++ b/apple/Plugins/CSVExportPlugin/CSVEncodingReport.swift @@ -12,7 +12,7 @@ import TableProPluginKit /// file that lost half its names looks exactly like one that lost nothing. This collects the /// distinct characters as the rows go past and turns them into the warning the export dialog /// already knows how to show. -struct CSVEncodingReport { +nonisolated struct CSVEncodingReport { /// Enough to recognise what was lost without filling an alert. A result holding thousands of /// distinct unrepresentable characters says the same thing as one holding twelve: the encoding /// is wrong for this data. diff --git a/apple/Plugins/CSVExportPlugin/CSVExportModels.swift b/apple/Plugins/CSVExportPlugin/CSVExportModels.swift index cd7942fb4b..1c5f721394 100644 --- a/apple/Plugins/CSVExportPlugin/CSVExportModels.swift +++ b/apple/Plugins/CSVExportPlugin/CSVExportModels.swift @@ -6,7 +6,7 @@ import Foundation import TableProPluginKit -public enum CSVDelimiter: String, CaseIterable, Identifiable, Codable, Sendable { +nonisolated public enum CSVDelimiter: String, CaseIterable, Identifiable, Codable, Sendable { case comma = "," case semicolon = ";" case tab = "\\t" @@ -28,7 +28,7 @@ public enum CSVDelimiter: String, CaseIterable, Identifiable, Codable, Sendable } } -public enum CSVQuoteHandling: String, CaseIterable, Identifiable, Codable, Sendable { +nonisolated public enum CSVQuoteHandling: String, CaseIterable, Identifiable, Codable, Sendable { case always = "Always" case asNeeded = "Quote if needed" case never = "Never" @@ -46,7 +46,7 @@ public enum CSVQuoteHandling: String, CaseIterable, Identifiable, Codable, Senda } } -public enum CSVLineBreak: String, CaseIterable, Identifiable, Codable, Sendable { +nonisolated public enum CSVLineBreak: String, CaseIterable, Identifiable, Codable, Sendable { case lf = "\\n" case crlf = "\\r\\n" case cr = "\\r" @@ -62,7 +62,7 @@ public enum CSVLineBreak: String, CaseIterable, Identifiable, Codable, Sendable } } -public enum CSVDecimalFormat: String, CaseIterable, Identifiable, Codable, Sendable { +nonisolated public enum CSVDecimalFormat: String, CaseIterable, Identifiable, Codable, Sendable { case period = "." case comma = "," @@ -71,7 +71,7 @@ public enum CSVDecimalFormat: String, CaseIterable, Identifiable, Codable, Senda public var separator: String { rawValue } } -public struct CSVExportOptions: Equatable, Codable, Sendable { +nonisolated public struct CSVExportOptions: Equatable, Codable, Sendable { public var convertNullToEmpty: Bool = true public var convertLineBreakToSpace: Bool = false public var includeFieldNames: Bool = true diff --git a/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift b/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift index f38791eced..f898216e46 100644 --- a/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift +++ b/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift @@ -112,3 +112,10 @@ struct CSVExportOptionsView: View { } } } + +extension CSVExportPlugin { + @MainActor + func settingsView() -> AnyView? { + AnyView(CSVExportOptionsView(plugin: self)) + } +} diff --git a/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift b/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift index f4f2bf17b3..53141d916e 100644 --- a/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift +++ b/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift @@ -5,10 +5,9 @@ import Combine import Foundation -import SwiftUI import TableProPluginKit -final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugin, @unchecked Sendable { +nonisolated final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugin, @unchecked Sendable { static let pluginName = "CSV Export" static let pluginVersion = "1.0.0" static let pluginDescription = "Export data to CSV format" @@ -23,17 +22,14 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi typealias Settings = CSVExportOptions static let settingsStorageId = "csv" - @Published var settings = CSVExportOptions() { + // A nonisolated class cannot hold an @Published property, so the change is announced by hand. + var settings = CSVExportOptions() { + willSet { objectWillChange.send() } didSet { saveSettings() } } required init() { loadSettings() } - @MainActor - func settingsView() -> AnyView? { - AnyView(CSVExportOptionsView(plugin: self)) - } - func resetSettingsToDefaults() { settings = CSVExportOptions() } diff --git a/apple/Plugins/CSVImportPlugin/CSVImportOptions.swift b/apple/Plugins/CSVImportPlugin/CSVImportOptions.swift index 8b0c9379e7..aee95f2017 100644 --- a/apple/Plugins/CSVImportPlugin/CSVImportOptions.swift +++ b/apple/Plugins/CSVImportPlugin/CSVImportOptions.swift @@ -7,7 +7,7 @@ import Foundation import TableProPluginKit import TableProTabularIO -struct CSVImportOptions: Equatable, Codable { +nonisolated struct CSVImportOptions: Equatable, Codable { enum Delimiter: String, Codable, CaseIterable, Identifiable { case auto case comma diff --git a/apple/Plugins/CSVImportPlugin/CSVImportOptionsView.swift b/apple/Plugins/CSVImportPlugin/CSVImportOptionsView.swift index 31c4f92778..19fff562e3 100644 --- a/apple/Plugins/CSVImportPlugin/CSVImportOptionsView.swift +++ b/apple/Plugins/CSVImportPlugin/CSVImportOptionsView.swift @@ -96,3 +96,10 @@ struct CSVImportOptionsView: View { .font(.system(size: 13)) } } + +extension CSVImportPlugin { + @MainActor + func settingsView() -> AnyView? { + AnyView(CSVImportOptionsView(plugin: self)) + } +} diff --git a/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift b/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift index a319f836b9..7a0b786f3d 100644 --- a/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift +++ b/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift @@ -12,7 +12,7 @@ import Foundation import TableProPluginKit -enum CSVImportParsing { +nonisolated enum CSVImportParsing { static let detectionSampleLimit = 200 static func resolveDialect(in text: Data, options: CSVImportOptions) -> CSVDialect { diff --git a/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift b/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift index 6839ce8e7d..bea3ab1b0f 100644 --- a/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift +++ b/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift @@ -5,11 +5,10 @@ import Combine import Foundation -import SwiftUI import TableProPluginKit import TableProTabularIO -final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugin, @unchecked Sendable { +nonisolated final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugin, @unchecked Sendable { static let pluginName = "CSV Import" static let pluginVersion = "1.0.0" static let pluginDescription = "Import data from CSV and TSV files" @@ -23,7 +22,9 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi typealias Settings = CSVImportOptions static let settingsStorageId = "csv-import" - @Published var settings = CSVImportOptions() { + // A nonisolated class cannot hold an @Published property, so the change is announced by hand. + var settings = CSVImportOptions() { + willSet { objectWillChange.send() } didSet { saveSettings() } } @@ -41,11 +42,6 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi required init() { loadSettings() } - @MainActor - func settingsView() -> AnyView? { - AnyView(CSVImportOptionsView(plugin: self)) - } - private typealias MalformedRow = (line: Int, error: CSVImportParsing.RowShapeError) private static let detectionPrefixBytes = 1_048_576 diff --git a/apple/Plugins/CSVImportPlugin/CSVImportText.swift b/apple/Plugins/CSVImportPlugin/CSVImportText.swift index 4465602a03..966c8f39f0 100644 --- a/apple/Plugins/CSVImportPlugin/CSVImportText.swift +++ b/apple/Plugins/CSVImportPlugin/CSVImportText.swift @@ -6,7 +6,7 @@ import Foundation import TableProTabularIO -struct CSVImportText { +nonisolated struct CSVImportText { let data: Data let encoding: TabularTextEncoding let firstUndecodableLine: Int? diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+Routines.swift b/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+Routines.swift index ea8905015a..9e37428b35 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+Routines.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+Routines.swift @@ -23,23 +23,23 @@ extension CassandraPluginDriver { return definition } - /// False until the whole-schema scope is checked against a live server. This driver has no - /// per-table trigger read, so the protocol default answers with nothing and a comparison has - /// never listed its triggers; opting in here would change what is compared rather than only - /// how fast it is read. + // Unchecked on a live server: with no per-table read, opting in would change what Compare lists, not its speed. var providesBulkTriggerFetch: Bool { false } func fetchAllTriggers(schema: String?) async throws -> [PluginTriggerInfo] { try await cassandraTriggerList(keyspace: resolveKeyspace(schema), table: nil) } - /// A Cassandra trigger is a pointer to a Java class the server loads, so there is no body to - /// show. The class name is presented as what it is rather than as an empty source pane. + // A Cassandra trigger points at a Java class the server loads, so its DDL names the class instead of a body. func fetchTriggerDDL(_ trigger: PluginTriggerInfo) async throws -> String { if let definition = trigger.definition, !definition.isEmpty { return definition } throw PluginObjectSourceError.unsupported(trigger.name) } + func generateDropTriggerSQL(name: String, table: String, schema: String?) -> String? { + CassandraObjectQueries.dropTriggerStatement(name: name, keyspace: resolveKeyspace(schema), table: table) + } + private func fetchCassandraFunctions(keyspace: String) async throws -> [PluginRoutineInfo] { let result = try await executeCatalogQuery(CassandraObjectQueries.functionList(keyspace: keyspace)) return result.rows.compactMap { row -> PluginRoutineInfo? in diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Routines.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Routines.swift index 4f7ed58eb5..119642d74d 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Routines.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePluginDriver+Routines.swift @@ -42,6 +42,11 @@ nonisolated public enum ClickHouseObjectQueries { public static func functionDefinition(name: String) -> String { "SELECT create_query FROM system.functions WHERE name = '\(escapeLiteral(name))'" } + + // A user function belongs to no database, so the name is never qualified. + public static func dropFunction(name: String) -> String { + "DROP FUNCTION IF EXISTS \(quoteIdentifier(name))" + } } nonisolated extension ClickHousePluginDriver { @@ -85,4 +90,8 @@ nonisolated extension ClickHousePluginDriver { } return ddl } + + func generateDropRoutineSQL(name: String, signature: String?, schema: String?, isFunction: Bool) -> String? { + ClickHouseObjectQueries.dropFunction(name: name) + } } diff --git a/apple/Plugins/ClickHouseDriverTests/ClickHouseDropRoutineTests.swift b/apple/Plugins/ClickHouseDriverTests/ClickHouseDropRoutineTests.swift new file mode 100644 index 0000000000..31fb6bd18c --- /dev/null +++ b/apple/Plugins/ClickHouseDriverTests/ClickHouseDropRoutineTests.swift @@ -0,0 +1,34 @@ +// +// ClickHouseDropRoutineTests.swift +// ClickHouseDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +// ClickHouse has no DROP ROUTINE, so a dump with Drop ticked on a user function restores only through this statement. +struct ClickHouseDropRoutineTests { + private var driver: ClickHousePluginDriver { + ClickHousePluginDriver(config: DriverConnectionConfig( + host: "localhost", + port: 8_123, + username: "default", + password: "", + database: "analytics", + ssl: SSLConfiguration(), + additionalFields: [:] + )) + } + + @Test("A user function drops by its bare name, whatever database the export names") + func dropsFunctionByBareName() { + let sql = driver.generateDropRoutineSQL(name: "linear_eq", signature: nil, schema: "analytics", isFunction: true) + #expect(sql == "DROP FUNCTION IF EXISTS `linear_eq`") + } + + @Test("A backtick and a backslash in the name stay inside the identifier") + func quotesTheName() { + #expect(ClickHouseObjectQueries.dropFunction(name: #"a`b\c"#) == #"DROP FUNCTION IF EXISTS `a\`b\\c`"#) + } +} diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBCreateTableSQL.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBCreateTableSQL.swift new file mode 100644 index 0000000000..fe88efe1bd --- /dev/null +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBCreateTableSQL.swift @@ -0,0 +1,136 @@ +// +// DuckDBCreateTableSQL.swift +// DuckDBDriverPlugin +// + +import Foundation +import TableProPluginKit +import TableProSQLGrammar + +// The id `DuckDBPlugin.databaseTypeId` holds, for the files iOS compiles without the plugin class. +nonisolated internal let duckdbDatabaseTypeId = "DuckDB" + +// Each auto-increment column's sequence first, then the table, its indexes and its column comments. Nil when a +// column or index holds text that is not one type or expression, or a foreign key action DuckDB cannot spell. +nonisolated internal func duckdbCreateTableStatements( + definition: PluginCreateTableDefinition, + schema: String, + quote: (String) -> String +) -> [String]? { + guard !definition.columns.isEmpty, + definition.columns.allSatisfy({ duckdbColumnTextRefusal(for: $0) == nil }), + definition.indexes.allSatisfy({ duckdbIndexExpressionRefusal(for: $0) == nil }) + else { return nil } + + let table = definition.tableName + let qualifiedTable = "\(quote(schema)).\(quote(table))" + let pkColumns = definition.columns.filter { $0.isPrimaryKey } + let inlinePK = pkColumns.count == 1 + var parts: [String] = definition.columns.map { + duckdbColumnDefinition($0, schema: schema, table: table, inlinePK: inlinePK, inCreateTable: true, quote: quote) + } + + if pkColumns.count > 1 { + let pkCols = pkColumns.map { quote($0.name) }.joined(separator: ", ") + parts.append("PRIMARY KEY (\(pkCols))") + } + + for fk in definition.foreignKeys { + guard let foreignKey = duckdbForeignKeyDefinition(fk, quote: quote) else { return nil } + parts.append(foreignKey) + } + + let createTable = "CREATE TABLE \(qualifiedTable) (\n " + parts.joined(separator: ",\n ") + "\n)" + let indexStatements = definition.indexes.map { + DuckDBIndexClauses.createStatement(for: $0, qualifiedTable: qualifiedTable, quote: quote) + } + let sequenceStatements = definition.columns.filter(\.autoIncrement).map { + DuckDBSchemaQueries.createSequenceStatement( + schema: schema, + sequence: DuckDBSchemaQueries.autoIncrementSequenceName(table: table, column: $0.name) + ) + } + let commentStatements = definition.columns.compactMap { column -> String? in + guard let comment = column.comment, !comment.isEmpty else { return nil } + return DuckDBSchemaQueries.columnCommentStatement( + schema: schema, table: table, column: column.name, comment: comment + ) + } + return sequenceStatements + [createTable] + indexStatements + commentStatements +} + +// The statements take these as SQL, whether the catalog or the user wrote them. +nonisolated internal func duckdbColumnTextRefusal(name: String, dataType: String, defaultValue: String?) -> String? { + guard SQLExpressionFragment.isSingleType(dataType, databaseTypeId: duckdbDatabaseTypeId) else { + return String(format: String(localized: "The type of column %@ cannot be written as SQL."), name) + } + if let defaultValue, !SQLExpressionFragment.isSingleExpression(defaultValue, databaseTypeId: duckdbDatabaseTypeId) { + return String(format: String(localized: "The default of column %@ cannot be written as SQL."), name) + } + return nil +} + +nonisolated internal func duckdbColumnTextRefusal(for column: PluginColumnDefinition) -> String? { + duckdbColumnTextRefusal(name: column.name, dataType: column.dataType, defaultValue: column.defaultValue) +} + +nonisolated internal func duckdbIndexExpressionRefusal(for index: PluginIndexDefinition) -> String? { + let expressions = index.expressions ?? [] + guard !expressions.allSatisfy({ SQLExpressionFragment.isSingleExpression($0, databaseTypeId: duckdbDatabaseTypeId) }) + else { return nil } + return String(format: String(localized: "Index %@ has a key part that is not one SQL expression."), index.name) +} + +nonisolated internal func duckdbReferentialAction(_ action: String) -> String? { + ["NO ACTION", "CASCADE", "SET NULL", "SET DEFAULT", "RESTRICT"].first { $0 == action.uppercased() } +} + +// `ADD COLUMN` refuses every constraint, NOT NULL included, so an added auto-increment column carries its default +// alone, and each existing row takes a value from the sequence. +nonisolated internal func duckdbColumnDefinition( + _ col: PluginColumnDefinition, + schema: String, + table: String, + inlinePK: Bool, + inCreateTable: Bool, + quote: (String) -> String +) -> String { + var def = "\(quote(col.name)) \(col.dataType)" + if col.autoIncrement { + if inCreateTable { + def += " NOT NULL" + } + let sequence = DuckDBSchemaQueries.autoIncrementSequenceName(table: table, column: col.name) + def += " DEFAULT \(DuckDBSchemaQueries.nextvalDefault(schema: schema, sequence: sequence))" + } else { + def += col.isNullable ? " NULL" : " NOT NULL" + if let defaultValue = col.defaultValue { + def += " DEFAULT \(defaultValue)" + } + } + if inlinePK && col.isPrimaryKey { + def += " PRIMARY KEY" + } + return def +} + +nonisolated internal func duckdbForeignKeyDefinition( + _ fk: PluginForeignKeyDefinition, + quote: (String) -> String +) -> String? { + guard let onDelete = duckdbReferentialAction(fk.onDelete), let onUpdate = duckdbReferentialAction(fk.onUpdate) + else { return nil } + let cols = fk.columns.map { quote($0) }.joined(separator: ", ") + let constraint = fk.name.isEmpty ? "" : "CONSTRAINT \(quote(fk.name)) " + var def = "\(constraint)FOREIGN KEY (\(cols)) REFERENCES \(quote(fk.referencedTable))" + if !fk.referencedColumns.isEmpty { + def += " (\(fk.referencedColumns.map { quote($0) }.joined(separator: ", ")))" + } + if onDelete != "NO ACTION" { + def += " ON DELETE \(onDelete)" + } + if onUpdate != "NO ACTION" { + def += " ON UPDATE \(onUpdate)" + } + return def +} diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBExportCatalog.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBExportCatalog.swift new file mode 100644 index 0000000000..3f1a20d3a0 --- /dev/null +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBExportCatalog.swift @@ -0,0 +1,336 @@ +// +// DuckDBExportCatalog.swift +// DuckDBDriverPlugin +// + +import Foundation +import TableProDuckDBCore +import TableProPluginKit +import TableProSQLGrammar + +// The catalog reads and the statements rebuilt from them, over whichever session the app hands in. Every read names +// the catalog because `duckdb_*()` spans every attached database. +nonisolated enum DuckDBExportCatalog { + // Binds `$1..$3`; the macro list binds nothing. + typealias Run = @Sendable (String, [PluginCellValue]) async throws -> PluginQueryResult + + static let tableDDLIncludesForeignKeys = true + + static func tables(catalog: String, schema: String, run: Run) async throws -> [PluginTableInfo] { + let result = try await run(DuckDBSchemaQueries.listTables(in: .schema), [.text(catalog), .text(schema)]) + return tableInfos(from: result) + } + + static func tableInfos(from result: PluginQueryResult) -> [PluginTableInfo] { + result.rows.compactMap { row in + guard let name = row[safe: 0]?.asText else { return nil } + let typeString = (row[safe: 1]?.asText) ?? "BASE TABLE" + let tableType = typeString.uppercased().contains("VIEW") ? "VIEW" : "TABLE" + return PluginTableInfo(name: name, type: tableType, schema: row[safe: 2]?.asText) + } + } + + static func columns(catalog: String, schema: String, table: String, run: Run) async throws -> [PluginColumnInfo] { + let result = try await run(DuckDBSchemaQueries.columnsForTable, [.text(catalog), .text(schema), .text(table)]) + let pkResult = try await run( + DuckDBSchemaQueries.primaryKeyColumnsForTable, + [.text(catalog), .text(schema), .text(table)] + ) + let pkColumns = Set(pkResult.rows.compactMap { $0[safe: 0]?.asText }) + + return result.rows.compactMap { row in + guard let name = row[safe: 0]?.asText, + let dataType = row[safe: 1]?.asText else { + return nil + } + return PluginColumnInfo( + name: name, + dataType: dataType, + isNullable: isNullableFlag(row[safe: 2]?.asText), + isPrimaryKey: pkColumns.contains(name), + defaultValue: row[safe: 3]?.asText, + comment: row[safe: 5]?.asText?.nilIfEmpty, + allowedValues: enumValues(dataType: dataType) + ) + } + } + + static func allColumns(catalog: String, schema: String, run: Run) async throws -> [String: [PluginColumnInfo]] { + let result = try await run(DuckDBSchemaQueries.columnsForSchema, [.text(catalog), .text(schema)]) + let pkResult = try await run(DuckDBSchemaQueries.primaryKeyColumnsForSchema, [.text(catalog), .text(schema)]) + var pkMap: [String: Set] = [:] + for row in pkResult.rows { + if let tableName = row[safe: 0]?.asText, let colName = row[safe: 1]?.asText { + pkMap[tableName, default: []].insert(colName) + } + } + + var allColumns: [String: [PluginColumnInfo]] = [:] + for row in result.rows { + guard let tableName = row[safe: 0]?.asText, + let columnName = row[safe: 1]?.asText, + let dataType = row[safe: 2]?.asText else { + continue + } + let column = PluginColumnInfo( + name: columnName, + dataType: dataType, + isNullable: isNullableFlag(row[safe: 3]?.asText), + isPrimaryKey: pkMap[tableName]?.contains(columnName) ?? false, + defaultValue: row[safe: 4]?.asText, + comment: row[safe: 6]?.asText?.nilIfEmpty, + allowedValues: enumValues(dataType: dataType) + ) + allColumns[tableName, default: []].append(column) + } + return allColumns + } + + // A failed read lists no keys rather than failing the caller. + static func foreignKeys( + catalog: String, + schema: String, + table: String, + run: Run + ) async throws -> [PluginForeignKeyInfo] { + do { + let result = try await run( + DuckDBSchemaQueries.foreignKeysForTable, + [.text(catalog), .text(schema), .text(table)] + ) + return result.rows.compactMap { row in + guard let name = row[safe: 0]?.asText, + let column = row[safe: 1]?.asText, + let refTable = row[safe: 2]?.asText, + let refColumn = row[safe: 3]?.asText else { + return nil + } + return PluginForeignKeyInfo( + name: name, + column: column, + referencedTable: refTable, + referencedColumn: refColumn, + onDelete: (row[safe: 4]?.asText) ?? "NO ACTION", + onUpdate: (row[safe: 5]?.asText) ?? "NO ACTION" + ) + } + } catch { + return [] + } + } + + // The recorded statement first, since only it keeps LIST, STRUCT and MAP types; then a view's, since + // `duckdb_tables()` has no row for a view; then one rebuilt from the columns. + static func tableDDL(catalog: String, schema: String, table: String, run: Run) async throws -> String { + let nativeResult = try await run(DuckDBSchemaQueries.tableDDL, [.text(catalog), .text(schema), .text(table)]) + if let firstRow = nativeResult.rows.first, let sql = firstRow[0].asText { + return sql.hasSuffix(";") ? sql : sql + ";" + } + + let viewResult = try await run(DuckDBSchemaQueries.viewDefinition, [.text(catalog), .text(schema), .text(table)]) + if let definition = viewResult.rows.first?[safe: 0]?.asText?.nilIfEmpty { + return DuckDBViewDefinition.makeReplaceable(definition) + } + + let columns = try await Self.columns(catalog: catalog, schema: schema, table: table, run: run) + let fks = try await Self.foreignKeys(catalog: catalog, schema: schema, table: table, run: run) + let refusal = columns.lazy.compactMap { + duckdbColumnTextRefusal(name: $0.name, dataType: $0.dataType, defaultValue: $0.defaultValue) + }.first + if let refusal { throw DuckDBCatalogError.unreadable(refusal) } + + let quote = DuckDBSchemaQueries.quoteIdentifier + var ddl = "CREATE TABLE \(quote(schema)).\(quote(table)) (\n" + var allDefs = columns.map { col in + var def = " \(quote(col.name)) \(col.dataType)" + if !col.isNullable { def += " NOT NULL" } + if let defaultVal = col.defaultValue { def += " DEFAULT \(defaultVal)" } + return def + } + + let pkColumns = columns.filter(\.isPrimaryKey) + if !pkColumns.isEmpty { + allDefs.append(" PRIMARY KEY (\(pkColumns.map { quote($0.name) }.joined(separator: ", ")))") + } + + for fk in fks { + guard let onDelete = duckdbReferentialAction(fk.onDelete), let onUpdate = duckdbReferentialAction(fk.onUpdate) + else { + throw DuckDBCatalogError.unreadable(String( + format: String(localized: "The action of foreign key %@ cannot be written as SQL."), fk.name + )) + } + allDefs.append( + " FOREIGN KEY (\(quote(fk.column)))" + + " REFERENCES \(quote(fk.referencedTable))" + + "(\(quote(fk.referencedColumn)))" + + " ON DELETE \(onDelete) ON UPDATE \(onUpdate)" + ) + } + + ddl += allDefs.joined(separator: ",\n") + ddl += "\n);" + return ddl + } + + static func viewDefinition(catalog: String, schema: String, view: String, run: Run) async throws -> String { + let result = try await run(DuckDBSchemaQueries.viewDefinition, [.text(catalog), .text(schema), .text(view)]) + guard let firstRow = result.rows.first, + let definition = firstRow[0].asText?.nilIfEmpty else { + throw DuckDBCatalogError.unreadable("Failed to fetch definition for view '\(view)'") + } + return DuckDBViewDefinition.makeReplaceable(definition) + } + + // A constraint's backing index is not a row of `duckdb_indexes()`, so nothing is filtered out. + static func indexDDL(catalog: String, schema: String, table: String, run: Run) async throws -> [String] { + let result = try await run(DuckDBSchemaQueries.indexDDLForTable, [.text(catalog), .text(schema), .text(table)]) + return result.rows.compactMap { $0[safe: 0]?.asText } + .map { $0.hasSuffix(";") ? $0 : $0 + ";" } + } + + static func routines(catalog: String, schema: String?, run: Run) async throws -> [PluginRoutineInfo] { + try await macros(catalog: catalog, schema: schema, run: run).map(\.routine) + } + + // DuckDB keeps the parsed macro, not the CREATE MACRO text, so the statement is rebuilt from the catalog. + static func routineDDL(_ routine: PluginRoutineInfo, catalog: String, run: Run) async throws -> String { + let signature = routine.argumentSignature ?? "()" + // A scalar and a table macro may share a name and parameters. + let kind = routine.attributes.first { $0.label == "Kind" }?.value + let macros = try await Self.macros(catalog: catalog, schema: routine.schema, run: run) + guard let macro = macros.first(where: { + $0.name == routine.name && $0.signature == signature && (kind == nil || $0.kind == kind) + }) else { + throw PluginObjectSourceError.notFound(routine.name) + } + let overloads = macros.filter { $0.schema == macro.schema && $0.name == macro.name && $0.kind == macro.kind } + guard let statement = DuckDBMacro.createStatement(overloads: overloads) else { + throw DuckDBCatalogError.unreadable(String( + format: String(localized: "TablePro cannot rebuild the definition of %@ from what the server reported."), + routine.name + )) + } + return statement + } + + static func macros(catalog: String, schema: String?, run: Run) async throws -> [DuckDBMacro] { + let result = try await run(DuckDBObjectQueries.macroList(catalog: catalog, schema: schema), []) + var macros: [DuckDBMacro] = [] + var currentIndex: String? + for row in result.rows { + guard let index = row[safe: 0]?.asText, let name = row[safe: 2]?.asText, !name.isEmpty else { continue } + if index != currentIndex { + currentIndex = index + macros.append(DuckDBMacro( + schema: row[safe: 1]?.asText, + name: name, + kind: row[safe: 4]?.asText, + body: row[safe: 3]?.asText + )) + } + if let parameter = row[safe: 5]?.asText { + macros[macros.count - 1].parameters.append( + DuckDBMacro.Parameter(name: parameter, type: row[safe: 6]?.asText) + ) + } + } + return macros + } + + // `duckdb_columns()` types `is_nullable` as BOOLEAN, so the value arrives as `true`. + private static func isNullableFlag(_ value: String?) -> Bool { + value == "true" + } + + // DuckDB spells an ENUM column's type with its members, so they need no second read. + private static func enumValues(dataType: String) -> [String]? { + EnumValueParser.parseMySQLEnumOrSet(from: dataType) + } +} + +nonisolated enum DuckDBCatalogError: PluginDriverError { + case unreadable(String) + + var pluginErrorMessage: String { + switch self { + case .unreadable(let message): return message + } + } +} + +nonisolated struct DuckDBMacro { + struct Parameter { + let name: String + let type: String? + } + + let schema: String? + let name: String + let kind: String? + let body: String? + var parameters: [Parameter] = [] + + // The identity Copy Objects and the export match on. + var signature: String { + DuckDBObjectQueries.macroSignature(parameterNames: parameters.map(\.name), isTableMacro: isTableMacro) + } + + var isTableMacro: Bool { + kind == "table_macro" + } + + var routine: PluginRoutineInfo { + var attributes: [PluginObjectAttribute] = [] + if let kind, !kind.isEmpty { + attributes.append(PluginObjectAttribute(label: "Kind", value: kind)) + } + return PluginRoutineInfo( + name: name, + kind: .function, + schema: schema, + returnType: nil, + language: "SQL", + argumentSignature: signature, + identity: nil, + definition: body, + attributes: attributes + ) + } + + // Nil when the body does not read as one expression, or one query for a table macro, or a parameter type as one + // type. DuckDB needs a table macro's query in parentheses once another overload follows it. + var overloadDefinition: String? { + guard let body else { return nil } + let databaseTypeId = duckdbDatabaseTypeId + let isSingle = isTableMacro + ? SQLExpressionFragment.isSingleQuery(body, databaseTypeId: databaseTypeId) + : SQLExpressionFragment.isSingleExpression(body, databaseTypeId: databaseTypeId) + guard isSingle else { return nil } + var parameterList: [String] = [] + for parameter in parameters { + let quotedName = DuckDBSchemaQueries.quoteIdentifier(parameter.name) + guard let type = parameter.type else { + parameterList.append(quotedName) + continue + } + guard SQLExpressionFragment.isSingleType(type, databaseTypeId: databaseTypeId) else { return nil } + parameterList.append("\(quotedName) \(type)") + } + let parameterText = parameterList.joined(separator: ", ") + return isTableMacro ? "(\(parameterText)) AS TABLE (\(body))" : "(\(parameterText)) AS \(body)" + } + + // DuckDB keeps every overload of a name as one macro, so one statement creates them all; one per overload would + // leave only the last. The catalog keeps no parameter defaults, so the statement has none. + static func createStatement(overloads: [DuckDBMacro]) -> String? { + guard let first = overloads.first else { return nil } + var definitions: [String] = [] + for overload in overloads { + guard let definition = overload.overloadDefinition else { return nil } + definitions.append(definition) + } + let target = DuckDBObjectQueries.qualifiedName(schema: first.schema, name: first.name) + return "CREATE OR REPLACE MACRO \(target)\(definitions.joined(separator: ", "));" + } +} diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBIndexClauses.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBIndexClauses.swift index d660720a3e..5b1e202f7d 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBIndexClauses.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBIndexClauses.swift @@ -6,7 +6,7 @@ import Foundation import TableProPluginKit -enum DuckDBIndexClauses { +nonisolated enum DuckDBIndexClauses { struct KeyParts: Equatable { let columns: [String] let expressions: [String] diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBLexicalFeatures.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBLexicalFeatures.swift index 0ea05bbda3..03765e3590 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBLexicalFeatures.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBLexicalFeatures.swift @@ -6,12 +6,10 @@ import Foundation import TableProPluginKit -/// How DuckDB lexes a statement, for the batches the plugin reads itself to track its transaction. -/// -/// Measured on 2.0.0, the linked library: `$$` and `$tag$` bodies with non-ASCII tags, nested block comments, -/// `E'...'` escapes, backtick-quoted names, a carriage return ending `--`, and a backslash that never escapes a plain -/// string or a name. The app's curated table holds the same facts. -enum DuckDBLexicalFeatures { +// Measured on 2.0.0, the linked library: `$$` and `$tag$` bodies with non-ASCII tags, nested block comments, +// `E'...'` escapes, backtick-quoted names, a carriage return ending `--`, and a backslash that never escapes a plain +// string or a name. The app's curated table holds the same facts. +nonisolated enum DuckDBLexicalFeatures { static let features: SQLLexicalFeatures = [ .taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix, .backtickQuotes, .carriageReturnEndsLineComments, diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift index b25bb6fef5..0cbd35a75a 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift @@ -815,11 +815,7 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Schema Operations func fetchTables(schema: String?) async throws -> [PluginTableInfo] { - let result = try await executeParameterized( - query: DuckDBSchemaQueries.listTables(in: .schema), - parameters: [.text(try requireCatalog()), .text(resolveSchema(schema))] - ) - return Self.tableInfos(from: result) + try await DuckDBExportCatalog.tables(catalog: try requireCatalog(), schema: resolveSchema(schema), run: catalogRun) } /// A remote catalog answers its schema list best-effort, falling back to `main` when it cannot, @@ -830,7 +826,7 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { query: DuckDBSchemaQueries.listTables(in: .allSchemas), parameters: [.text(try requireCatalog())] ) - return Self.tableInfos(from: result) + return DuckDBExportCatalog.tableInfos(from: result) } /// A remote catalog answers metadata best-effort, so it stays per schema, where a failed read @@ -842,112 +838,21 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { query: DuckDBSchemaQueries.listTables(in: .schemas(schemas)), parameters: [.text(try requireCatalog())] ) - return Self.tableInfos(from: result) - } - - private static func tableInfos(from result: PluginQueryResult) -> [PluginTableInfo] { - result.rows.compactMap { row in - guard let name = row[safe: 0]?.asText else { return nil } - let typeString = (row[safe: 1]?.asText) ?? "BASE TABLE" - let tableType = typeString.uppercased().contains("VIEW") ? "VIEW" : "TABLE" - return PluginTableInfo(name: name, type: tableType, schema: row[safe: 2]?.asText) - } + return DuckDBExportCatalog.tableInfos(from: result) } func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { let schemaName = resolveSchema(schema) - let catalog = try requireCatalog() - let result = try await executeParameterized( - query: DuckDBSchemaQueries.columnsForTable, - parameters: [.text(catalog), .text(schemaName), .text(table)] + return try await DuckDBExportCatalog.columns( + catalog: try requireCatalog(), schema: schemaName, table: table, run: catalogRun ) - - let pkColumns = try await fetchPrimaryKeyColumns(table: table, schema: schemaName) - - return result.rows.compactMap { row in - guard let name = row[safe: 0]?.asText, - let dataType = row[safe: 1]?.asText else { - return nil - } - - let isNullable = Self.isNullableFlag(row[safe: 2]?.asText) - let defaultValue = row[safe: 3]?.asText - let isPrimaryKey = pkColumns.contains(name) - - return PluginColumnInfo( - name: name, - dataType: dataType, - isNullable: isNullable, - isPrimaryKey: isPrimaryKey, - defaultValue: defaultValue, - comment: row[safe: 5]?.asText?.nilIfEmpty, - allowedValues: resolveEnumValues(dataType: dataType) - ) - } } var fetchesAllColumnsInOneRequest: Bool { true } func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { let schemaName = resolveSchema(schema) - let catalog = try requireCatalog() - let result = try await executeParameterized( - query: DuckDBSchemaQueries.columnsForSchema, - parameters: [.text(catalog), .text(schemaName)] - ) - - let pkResult = try await executeParameterized( - query: DuckDBSchemaQueries.primaryKeyColumnsForSchema, - parameters: [.text(catalog), .text(schemaName)] - ) - var pkMap: [String: Set] = [:] - for row in pkResult.rows { - if let tableName = row[safe: 0]?.asText, let colName = row[safe: 1]?.asText { - pkMap[tableName, default: []].insert(colName) - } - } - - var allColumns: [String: [PluginColumnInfo]] = [:] - - for row in result.rows { - guard let tableName = row[safe: 0]?.asText, - let columnName = row[safe: 1]?.asText, - let dataType = row[safe: 2]?.asText else { - continue - } - - let isNullable = Self.isNullableFlag(row[safe: 3]?.asText) - let defaultValue = row[safe: 4]?.asText - let isPrimaryKey = pkMap[tableName]?.contains(columnName) ?? false - - let column = PluginColumnInfo( - name: columnName, - dataType: dataType, - isNullable: isNullable, - isPrimaryKey: isPrimaryKey, - defaultValue: defaultValue, - comment: row[safe: 6]?.asText?.nilIfEmpty, - allowedValues: resolveEnumValues(dataType: dataType) - ) - - allColumns[tableName, default: []].append(column) - } - - return allColumns - } - - /// `duckdb_columns()` types `is_nullable` as BOOLEAN, so the value arrives as `true`. - private static func isNullableFlag(_ value: String?) -> Bool { - value == "true" - } - - /// DuckDB spells an ENUM column's type as `ENUM('ok', 'bad')`, members included, so the - /// allowed values are already in hand. The driver used to also query `duckdb_types()` for - /// them and key the result on `type_name`, which is `mood`, so that map never matched a - /// column and every value came from this parse anyway. The query is gone rather than - /// repaired: it cost a round trip per column fetch and answered nothing. - private func resolveEnumValues(dataType: String) -> [String]? { - EnumValueParser.parseMySQLEnumOrSet(from: dataType) + return try await DuckDBExportCatalog.allColumns(catalog: try requireCatalog(), schema: schemaName, run: catalogRun) } func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { @@ -985,137 +890,44 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } - var tableDDLIncludesForeignKeys: Bool { true } + var tableDDLIncludesForeignKeys: Bool { DuckDBExportCatalog.tableDDLIncludesForeignKeys } func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { let schemaName = resolveSchema(schema) - let catalog = try requireCatalog() - do { - let result = try await executeParameterized( - query: DuckDBSchemaQueries.foreignKeysForTable, - parameters: [.text(catalog), .text(schemaName), .text(table)] - ) - return result.rows.compactMap { row in - guard let name = row[safe: 0]?.asText, - let column = row[safe: 1]?.asText, - let refTable = row[safe: 2]?.asText, - let refColumn = row[safe: 3]?.asText else { - return nil - } - - let onDelete = (row[safe: 4]?.asText) ?? "NO ACTION" - let onUpdate = (row[safe: 5]?.asText) ?? "NO ACTION" - - return PluginForeignKeyInfo( - name: name, - column: column, - referencedTable: refTable, - referencedColumn: refColumn, - onDelete: onDelete, - onUpdate: onUpdate - ) - } - } catch { - return [] - } + return try await DuckDBExportCatalog.foreignKeys( + catalog: try requireCatalog(), schema: schemaName, table: table, run: catalogRun + ) } func fetchTableDDL(table: String, schema: String?) async throws -> String { let schemaName = resolveSchema(schema) - - // Try native DDL from duckdb_tables() first (preserves complex types like LIST, STRUCT, MAP) - let nativeResult = try await executeParameterized( - query: DuckDBSchemaQueries.tableDDL, - parameters: [.text(try requireCatalog()), .text(schemaName), .text(table)] + return try await DuckDBExportCatalog.tableDDL( + catalog: try requireCatalog(), schema: schemaName, table: table, run: catalogRun ) - - if let firstRow = nativeResult.rows.first, let sql = firstRow[0].asText { - return sql.hasSuffix(";") ? sql : sql + ";" - } - - // `duckdb_tables()` has no row for a view while `duckdb_columns()` lists its columns. - let viewResult = try await executeParameterized( - query: DuckDBSchemaQueries.viewDefinition, - parameters: [.text(try requireCatalog()), .text(schemaName), .text(table)] - ) - if let definition = viewResult.rows.first?[safe: 0]?.asText?.nilIfEmpty { - return DuckDBViewDefinition.makeReplaceable(definition) - } - - // Fallback: synthesize DDL from schema metadata, whose text goes in only as one type or expression. - let columns = try await fetchColumns(table: table, schema: schemaName) - let fks = try await fetchForeignKeys(table: table, schema: schemaName) - let refusal = columns.lazy.compactMap { - Self.columnTextRefusal(name: $0.name, dataType: $0.dataType, defaultValue: $0.defaultValue) - }.first - if let refusal { throw DuckDBPluginError.queryFailed(refusal) } - - var ddl = "CREATE TABLE \(quoteIdentifier(schemaName)).\(quoteIdentifier(table)) (\n" - - let columnDefs = columns.map { col in - var def = " \(quoteIdentifier(col.name)) \(col.dataType)" - if !col.isNullable { def += " NOT NULL" } - if let defaultVal = col.defaultValue { def += " DEFAULT \(defaultVal)" } - return def - } - - var allDefs = columnDefs - - let pkColumns = columns.filter(\.isPrimaryKey) - if !pkColumns.isEmpty { - let pkCols = pkColumns.map { quoteIdentifier($0.name) } - .joined(separator: ", ") - allDefs.append(" PRIMARY KEY (\(pkCols))") - } - - for fk in fks { - guard let onDelete = Self.referentialAction(fk.onDelete), let onUpdate = Self.referentialAction(fk.onUpdate) - else { - throw DuckDBPluginError.queryFailed(String( - format: String(localized: "The action of foreign key %@ cannot be written as SQL."), fk.name - )) - } - let fkDef = " FOREIGN KEY (\(quoteIdentifier(fk.column)))" - + " REFERENCES \(quoteIdentifier(fk.referencedTable))" - + "(\(quoteIdentifier(fk.referencedColumn)))" - + " ON DELETE \(onDelete) ON UPDATE \(onUpdate)" - allDefs.append(fkDef) - } - - ddl += allDefs.joined(separator: ",\n") - ddl += "\n);" - - return ddl } - /// `duckdb_indexes()` carries each index's own `CREATE INDEX` text, which reproduces an - /// expression key that no column list can. It lists user indexes only: an index DuckDB builds - /// to back a PRIMARY KEY or UNIQUE constraint is not a row there, so nothing has to be filtered - /// out of the result. func fetchIndexDDL(table: String, schema: String?) async throws -> [String] { - let result = try await executeParameterized( - query: DuckDBSchemaQueries.indexDDLForTable, - parameters: [.text(try requireCatalog()), .text(resolveSchema(schema)), .text(table)] + try await DuckDBExportCatalog.indexDDL( + catalog: try requireCatalog(), schema: resolveSchema(schema), table: table, run: catalogRun ) - return result.rows.compactMap { $0[safe: 0]?.asText } - .map { $0.hasSuffix(";") ? $0 : $0 + ";" } } func fetchViewDefinition(view: String, schema: String?) async throws -> String { let schemaName = resolveSchema(schema) - let result = try await executeParameterized( - query: DuckDBSchemaQueries.viewDefinition, - parameters: [.text(try requireCatalog()), .text(schemaName), .text(view)] + return try await DuckDBExportCatalog.viewDefinition( + catalog: try requireCatalog(), schema: schemaName, view: view, run: catalogRun ) + } - guard let firstRow = result.rows.first, - let definition = firstRow[0].asText?.nilIfEmpty else { - throw DuckDBPluginError.queryFailed( - "Failed to fetch definition for view '\(view)'" - ) + // The catalog reads bind the catalog, schema and name; the macro list binds nothing and runs unprepared, as it + // always has. + var catalogRun: DuckDBExportCatalog.Run { + { [self] query, parameters in + if parameters.isEmpty { + return try await execute(query: query) + } + return try await executeParameterized(query: query, parameters: parameters) } - - return DuckDBViewDefinition.makeReplaceable(definition) } func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { @@ -1266,50 +1078,10 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return path } - private func fetchPrimaryKeyColumns( - table: String, - schema: String - ) async throws -> Set { - let result = try await executeParameterized( - query: DuckDBSchemaQueries.primaryKeyColumnsForTable, - parameters: [.text(try requireCatalog()), .text(schema), .text(table)] - ) - return Set(result.rows.compactMap { $0[safe: 0]?.asText }) - } - // MARK: - Create Table DDL func generateCreateTableStatements(definition: PluginCreateTableDefinition, schema: String?) -> [String]? { - guard !definition.columns.isEmpty, - definition.columns.allSatisfy({ Self.columnTextRefusal(for: $0) == nil }), - definition.indexes.allSatisfy({ Self.indexExpressionRefusal(for: $0) == nil }) - else { return nil } - - let schema = resolveSchema(schema) - let table = definition.tableName - let qualifiedTable = "\(quoteIdentifier(schema)).\(quoteIdentifier(table))" - let pkColumns = definition.columns.filter { $0.isPrimaryKey } - let inlinePK = pkColumns.count == 1 - var parts: [String] = definition.columns.map { - duckdbColumnDefinition($0, schema: schema, table: table, inlinePK: inlinePK, inCreateTable: true) - } - - if pkColumns.count > 1 { - let pkCols = pkColumns.map { quoteIdentifier($0.name) }.joined(separator: ", ") - parts.append("PRIMARY KEY (\(pkCols))") - } - - for fk in definition.foreignKeys { - guard let foreignKey = duckdbForeignKeyDefinition(fk) else { return nil } - parts.append(foreignKey) - } - - let createTable = "CREATE TABLE \(qualifiedTable) (\n " + parts.joined(separator: ",\n ") + "\n)" - let indexStatements = definition.indexes.map { duckdbIndexDefinition($0, qualifiedTable: qualifiedTable) } - return sequenceStatements(for: definition.columns, schema: schema, table: table) - + [createTable] - + indexStatements - + columnCommentStatements(for: definition.columns, schema: schema, table: table) + duckdbCreateTableStatements(definition: definition, schema: resolveSchema(schema), quote: quoteIdentifier) } func schemaOperationRefusal(_ operation: PluginSchemaOperation) -> String? { @@ -1319,114 +1091,16 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { !defaultValue.trimmingCharacters(in: .whitespaces).isEmpty { return String(localized: "An auto-increment column takes its values from a sequence, so it cannot also have a default.") } - return Self.columnTextRefusal(for: column) + return duckdbColumnTextRefusal(for: column) case .modifyColumn(_, let column): - return Self.columnTextRefusal(for: column) + return duckdbColumnTextRefusal(for: column) case .addIndex(let index): - return Self.indexExpressionRefusal(for: index) + return duckdbIndexExpressionRefusal(for: index) default: return nil } } - // The statements take these as SQL, whether the catalog or the user wrote them. - private static func columnTextRefusal(name: String, dataType: String, defaultValue: String?) -> String? { - guard SQLExpressionFragment.isSingleType(dataType, databaseTypeId: DuckDBPlugin.databaseTypeId) else { - return String(format: String(localized: "The type of column %@ cannot be written as SQL."), name) - } - if let defaultValue, !SQLExpressionFragment.isSingleExpression(defaultValue, databaseTypeId: DuckDBPlugin.databaseTypeId) { - return String(format: String(localized: "The default of column %@ cannot be written as SQL."), name) - } - return nil - } - - private static func columnTextRefusal(for column: PluginColumnDefinition) -> String? { - columnTextRefusal(name: column.name, dataType: column.dataType, defaultValue: column.defaultValue) - } - - private static func indexExpressionRefusal(for index: PluginIndexDefinition) -> String? { - let expressions = index.expressions ?? [] - guard !expressions.allSatisfy({ SQLExpressionFragment.isSingleExpression($0, databaseTypeId: DuckDBPlugin.databaseTypeId) }) - else { return nil } - return String(format: String(localized: "Index %@ has a key part that is not one SQL expression."), index.name) - } - - private static func referentialAction(_ action: String) -> String? { - ["NO ACTION", "CASCADE", "SET NULL", "SET DEFAULT", "RESTRICT"].first { $0 == action.uppercased() } - } - - private func sequenceStatements(for columns: [PluginColumnDefinition], schema: String, table: String) -> [String] { - columns.filter(\.autoIncrement).map { - DuckDBSchemaQueries.createSequenceStatement( - schema: schema, - sequence: DuckDBSchemaQueries.autoIncrementSequenceName(table: table, column: $0.name) - ) - } - } - - private func columnCommentStatements( - for columns: [PluginColumnDefinition], - schema: String, - table: String - ) -> [String] { - columns.compactMap { column in - guard let comment = column.comment, !comment.isEmpty else { return nil } - return DuckDBSchemaQueries.columnCommentStatement( - schema: schema, table: table, column: column.name, comment: comment - ) - } - } - - /// `ADD COLUMN` refuses every constraint, NOT NULL included, so an added auto-increment column - /// carries its default alone, and each existing row takes a value from the sequence. - private func duckdbColumnDefinition( - _ col: PluginColumnDefinition, - schema: String, - table: String, - inlinePK: Bool, - inCreateTable: Bool - ) -> String { - var def = "\(quoteIdentifier(col.name)) \(col.dataType)" - if col.autoIncrement { - if inCreateTable { - def += " NOT NULL" - } - let sequence = DuckDBSchemaQueries.autoIncrementSequenceName(table: table, column: col.name) - def += " DEFAULT \(DuckDBSchemaQueries.nextvalDefault(schema: schema, sequence: sequence))" - } else { - def += col.isNullable ? " NULL" : " NOT NULL" - if let defaultValue = col.defaultValue { - def += " DEFAULT \(defaultValue)" - } - } - if inlinePK && col.isPrimaryKey { - def += " PRIMARY KEY" - } - return def - } - - private func duckdbIndexDefinition(_ index: PluginIndexDefinition, qualifiedTable: String) -> String { - DuckDBIndexClauses.createStatement(for: index, qualifiedTable: qualifiedTable, quote: quoteIdentifier) - } - - private func duckdbForeignKeyDefinition(_ fk: PluginForeignKeyDefinition) -> String? { - guard let onDelete = Self.referentialAction(fk.onDelete), let onUpdate = Self.referentialAction(fk.onUpdate) - else { return nil } - let cols = fk.columns.map { quoteIdentifier($0) }.joined(separator: ", ") - let constraint = fk.name.isEmpty ? "" : "CONSTRAINT \(quoteIdentifier(fk.name)) " - var def = "\(constraint)FOREIGN KEY (\(cols)) REFERENCES \(quoteIdentifier(fk.referencedTable))" - if !fk.referencedColumns.isEmpty { - def += " (\(fk.referencedColumns.map { quoteIdentifier($0) }.joined(separator: ", ")))" - } - if onDelete != "NO ACTION" { - def += " ON DELETE \(onDelete)" - } - if onUpdate != "NO ACTION" { - def += " ON UPDATE \(onUpdate)" - } - return def - } - private func qualifiedTableName(_ table: String, schema: String?) -> String { "\(quoteIdentifier(resolveSchema(schema))).\(quoteIdentifier(table))" } @@ -1435,9 +1109,9 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // One statement only; an auto-increment column needs its sequence created first, so it is unsupported here. func generateAddColumnSQL(table: String, schema: String?, column: PluginColumnDefinition) -> String? { - guard !column.autoIncrement, Self.columnTextRefusal(for: column) == nil else { return nil } + guard !column.autoIncrement, duckdbColumnTextRefusal(for: column) == nil else { return nil } let colDef = duckdbColumnDefinition( - column, schema: resolveSchema(schema), table: table, inlinePK: false, inCreateTable: false + column, schema: resolveSchema(schema), table: table, inlinePK: false, inCreateTable: false, quote: quoteIdentifier ) return "ALTER TABLE \(qualifiedTableName(table, schema: schema)) ADD COLUMN \(colDef)" } @@ -1448,7 +1122,7 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { oldColumn: PluginColumnDefinition, newColumn: PluginColumnDefinition ) -> String? { - guard Self.columnTextRefusal(for: newColumn) == nil else { return nil } + guard duckdbColumnTextRefusal(for: newColumn) == nil else { return nil } let qt = qualifiedTableName(table, schema: schema) var stmts: [String] = [] @@ -1483,8 +1157,10 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func generateAddIndexSQL(table: String, schema: String?, index: PluginIndexDefinition) -> String? { - guard Self.indexExpressionRefusal(for: index) == nil else { return nil } - return duckdbIndexDefinition(index, qualifiedTable: qualifiedTableName(table, schema: schema)) + guard duckdbIndexExpressionRefusal(for: index) == nil else { return nil } + return DuckDBIndexClauses.createStatement( + for: index, qualifiedTable: qualifiedTableName(table, schema: schema), quote: quoteIdentifier + ) } func generateDropIndexSQL(table: String, schema: String?, indexName: String) -> String? { @@ -1492,7 +1168,9 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func generateAddForeignKeySQL(table: String, schema: String?, fk: PluginForeignKeyDefinition) -> String? { - duckdbForeignKeyDefinition(fk).map { "ALTER TABLE \(qualifiedTableName(table, schema: schema)) ADD \($0)" } + duckdbForeignKeyDefinition(fk, quote: quoteIdentifier).map { + "ALTER TABLE \(qualifiedTableName(table, schema: schema)) ADD \($0)" + } } func generateDropForeignKeySQL(table: String, schema: String?, constraintName: String) -> String? { diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBPluginDriver+Routines.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBPluginDriver+Routines.swift index 755ac00ee9..03b6099611 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBPluginDriver+Routines.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBPluginDriver+Routines.swift @@ -4,161 +4,19 @@ // import Foundation +import TableProDuckDBCore import TableProPluginKit -import TableProSQLGrammar - -// DuckDB has macros where other engines have functions, and no procedures or triggers. -public enum DuckDBObjectQueries { - // Macros only, since duckdb_functions() also lists every built-in. On the catalog, since it spans every attached one - // and temp, whose overloads would merge into one statement. One row per parameter, read as a list element: the - // list's text form quotes an element only when it holds certain characters. WITH ORDINALITY, since - // generate_subscripts needs the core_functions extension the bundled library does not link. Overloads sort by - // their parameters, so a rebuilt statement lists them in the same order every time. - public static func macroList(catalog: String, schema: String?) -> String { - let schemaPredicate = schema.map { " AND schema_name = \(DuckDBSchemaQueries.quoteLiteral($0))" } ?? "" - return """ - SELECT f.macro_index, f.schema_name, f.function_name, f.macro_definition, f.function_type, p.parameter, - f.parameter_types[p.position] AS parameter_type - FROM ( - SELECT row_number() OVER ( - ORDER BY schema_name, function_name, function_type, parameters, parameter_types - ) AS macro_index, - schema_name, function_name, macro_definition, function_type, parameters, parameter_types - FROM duckdb_functions() - WHERE database_name = \(DuckDBSchemaQueries.quoteLiteral(catalog))\(schemaPredicate) - AND function_type IN ('macro', 'table_macro') AND NOT internal - ) AS f - LEFT JOIN LATERAL unnest(f.parameters) WITH ORDINALITY AS p(parameter, position) ON true - ORDER BY f.macro_index, p.position - """ - } -} - -struct DuckDBMacro { - struct Parameter { - let name: String - let type: String? - } - - let schema: String? - let name: String - let kind: String? - let body: String? - var parameters: [Parameter] = [] - - // The identity Copy Objects and the export match on, so its format stays the one the listing always showed. - var signature: String { - "(\(parameters.map(\.name).joined(separator: ", ")))" - } - - var isTableMacro: Bool { - kind == "table_macro" - } - - var routine: PluginRoutineInfo { - var attributes: [PluginObjectAttribute] = [] - if let kind, !kind.isEmpty { - attributes.append(PluginObjectAttribute(label: "Kind", value: kind)) - } - return PluginRoutineInfo( - name: name, - kind: .function, - schema: schema, - returnType: nil, - language: "SQL", - argumentSignature: signature, - identity: nil, - definition: body, - attributes: attributes - ) - } - - // Nil when the body does not read as one expression, or one query for a table macro, or a parameter type as one - // type. DuckDB needs a table macro's query in parentheses once another overload follows it. - var overloadDefinition: String? { - guard let body else { return nil } - let databaseTypeId = DuckDBPlugin.databaseTypeId - let isSingle = isTableMacro - ? SQLExpressionFragment.isSingleQuery(body, databaseTypeId: databaseTypeId) - : SQLExpressionFragment.isSingleExpression(body, databaseTypeId: databaseTypeId) - guard isSingle else { return nil } - var parameterList: [String] = [] - for parameter in parameters { - let quotedName = DuckDBSchemaQueries.quoteIdentifier(parameter.name) - guard let type = parameter.type else { - parameterList.append(quotedName) - continue - } - guard SQLExpressionFragment.isSingleType(type, databaseTypeId: databaseTypeId) else { return nil } - parameterList.append("\(quotedName) \(type)") - } - let parameterText = parameterList.joined(separator: ", ") - return isTableMacro ? "(\(parameterText)) AS TABLE (\(body))" : "(\(parameterText)) AS \(body)" - } - - // DuckDB keeps every overload of a name as one macro, so one statement creates them all; one per overload would - // leave only the last. The catalog keeps no parameter defaults, so the statement has none. - static func createStatement(overloads: [DuckDBMacro]) -> String? { - guard let first = overloads.first else { return nil } - var definitions: [String] = [] - for overload in overloads { - guard let definition = overload.overloadDefinition else { return nil } - definitions.append(definition) - } - let target = [first.schema, first.name].compactMap { $0 }.filter { !$0.isEmpty } - .map(DuckDBSchemaQueries.quoteIdentifier) - .joined(separator: ".") - return "CREATE OR REPLACE MACRO \(target)\(definitions.joined(separator: ", "));" - } -} extension DuckDBPluginDriver { func fetchRoutines(schema: String?) async throws -> [PluginRoutineInfo] { - try await fetchMacros(schema: schema).map(\.routine) + try await DuckDBExportCatalog.routines(catalog: try requireCatalog(), schema: schema, run: catalogRun) } - // DuckDB keeps the parsed macro, not the CREATE MACRO text, so the statement is rebuilt from the catalog. func fetchRoutineDDL(_ routine: PluginRoutineInfo) async throws -> String { - let signature = routine.argumentSignature ?? "()" - // A scalar and a table macro may share a name and parameters. - let kind = routine.attributes.first { $0.label == "Kind" }?.value - let macros = try await fetchMacros(schema: routine.schema) - guard let macro = macros.first(where: { - $0.name == routine.name && $0.signature == signature && (kind == nil || $0.kind == kind) - }) else { - throw PluginObjectSourceError.notFound(routine.name) - } - let overloads = macros.filter { $0.schema == macro.schema && $0.name == macro.name && $0.kind == macro.kind } - guard let statement = DuckDBMacro.createStatement(overloads: overloads) else { - throw DuckDBPluginError.queryFailed(String( - format: String(localized: "TablePro cannot rebuild the definition of %@ from what the server reported."), - routine.name - )) - } - return statement + try await DuckDBExportCatalog.routineDDL(routine, catalog: try requireCatalog(), run: catalogRun) } - private func fetchMacros(schema: String?) async throws -> [DuckDBMacro] { - let result = try await execute(query: DuckDBObjectQueries.macroList(catalog: try requireCatalog(), schema: schema)) - var macros: [DuckDBMacro] = [] - var currentIndex: String? - for row in result.rows { - guard let index = row[safe: 0]?.asText, let name = row[safe: 2]?.asText, !name.isEmpty else { continue } - if index != currentIndex { - currentIndex = index - macros.append(DuckDBMacro( - schema: row[safe: 1]?.asText, - name: name, - kind: row[safe: 4]?.asText, - body: row[safe: 3]?.asText - )) - } - if let parameter = row[safe: 5]?.asText { - macros[macros.count - 1].parameters.append( - DuckDBMacro.Parameter(name: parameter, type: row[safe: 6]?.asText) - ) - } - } - return macros + func generateDropRoutineSQL(name: String, signature: String?, schema: String?, isFunction: Bool) -> String? { + DuckDBObjectQueries.dropMacro(name: name, schema: schema, isTableMacro: DuckDBObjectQueries.namesTableMacro(signature)) } } diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBSchemaQueries.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBSchemaQueries.swift index 81649cbccc..c4cf838e39 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBSchemaQueries.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBSchemaQueries.swift @@ -2,41 +2,24 @@ // DuckDBSchemaQueries.swift // DuckDBDriverPlugin // -// Static SQL used to enumerate catalogs, schemas and objects. Extracted so the -// queries can be exercised by unit tests via TableProTests, which never loads -// the plugin bundle. +// Static SQL used to enumerate catalogs, schemas and objects. // import Foundation import TableProPluginKit import TableProSQLQuoting -enum DuckDBTableListingScope: Sendable, Equatable { +nonisolated enum DuckDBTableListingScope: Sendable, Equatable { case schema case schemas([String]) case allSchemas } -/// DuckDB's namespace is `catalog.schema.table`, and the `duckdb_*` table functions span -/// every attached catalog. A predicate on the schema alone therefore matches same-named -/// schemas in other catalogs: with a second database attached, `WHERE schema_name = 'main'` -/// returns both catalogs' `main` tables. Every query here takes the catalog as its first -/// bound parameter for that reason, and the driver's `switchDatabase` is what moves it. -/// -/// Nothing here may call `current_database()`, `current_schema()`, or read -/// `information_schema.key_column_usage` or `information_schema.referential_constraints`. -/// All four live in DuckDB's `core_functions` extension, which the macOS build does not -/// link statically, so DuckDB fetches it from `extensions.duckdb.org` on first use. A Mac -/// that cannot reach that host answers every one of them with -/// `Binder Error: Referenced table "system" not found!`, which left the sidebar with no -/// databases, schemas, tables or columns at all. The `duckdb_*()` table functions and the -/// remaining `information_schema` views are part of the core engine and need no download. -/// `scripts/probes/check-duckdb-offline-metadata.sh` runs every query here against the shipped -/// static library with the extension directory pointed at an empty folder. -enum DuckDBSchemaQueries { - /// `system` and `temp` are DuckDB's built-in catalogs; `internal` marks them and - /// nothing else. Filtering catalogs is also what keeps `information_schema` and - /// `pg_catalog` out of the schema list, because they are schemas of `system`. +// `duckdb_*()` spans every attached catalog, so every query binds the catalog first. Nothing here may call +// `current_database()`, `current_schema()` or the key and constraint `information_schema` views: they live in the +// `core_functions` extension, which an offline Mac cannot fetch (`scripts/probes/check-duckdb-offline-metadata.sh`). +nonisolated enum DuckDBSchemaQueries { + // `internal` marks only `system` and `temp`, which also keeps `information_schema` and `pg_catalog` out. static let listDatabases = """ SELECT database_name FROM duckdb_databases() @@ -44,17 +27,14 @@ enum DuckDBSchemaQueries { ORDER BY database_name """ - /// `USE` writes the catalog and schema it landed on into the `search_path` setting, and - /// the `schema` setting carries the schema on its own. Reading them back is how the - /// driver learns its position without `current_database()` or `current_schema()`. + // Where `USE` landed, read without `current_database()` or `current_schema()`. static let currentPosition = """ SELECT name, value FROM duckdb_settings() WHERE name IN ('search_path', 'schema') """ - /// Never filters on the schema's own `internal` flag: DuckDB marks the auto-created - /// `main` internal in every catalog, so that predicate hides the default schema. + // Never on the schema's `internal` flag: DuckDB marks every catalog's `main` internal. static let listSchemas = """ SELECT schema_name FROM duckdb_schemas() @@ -62,10 +42,7 @@ enum DuckDBSchemaQueries { ORDER BY schema_name """ - /// One schema's objects, bound to the catalog and the schema; a named set's or every schema's, - /// bound to the catalog alone. Every schema's filters by `listSchemas` itself rather than dropping - /// the schema predicate, so an object is listed here exactly when its schema is listed there. - /// Both multi-schema listings project each row's schema as a third column. + // Every schema's listing filters by `listSchemas` itself, so an object is listed exactly when its schema is. static func listTables(in scope: DuckDBTableListingScope) -> String { let schemaFilter: String let schemaColumn: String @@ -119,8 +96,7 @@ enum DuckDBSchemaQueries { ORDER BY table_name, column_index """ - /// `constraint_column_names` is a LIST, so `UNNEST` turns a composite key into one row - /// per column and keeps the caller's row shape flat. + // `UNNEST` turns a composite key into one row per column. static let primaryKeyColumnsForSchema = """ SELECT table_name, UNNEST(constraint_column_names) AS column_name FROM duckdb_constraints() @@ -146,9 +122,7 @@ enum DuckDBSchemaQueries { AND table_name = $3 """ - /// The statement DuckDB itself recorded, which is the only form that reproduces an expression - /// index. `duckdb_indexes()` lists user indexes only, so a constraint's backing index is absent - /// rather than needing to be filtered out. + // Only the recorded statement reproduces an expression index. static let indexDDLForTable = """ SELECT sql FROM duckdb_indexes() @@ -159,11 +133,7 @@ enum DuckDBSchemaQueries { ORDER BY index_name """ - /// The referencing and referenced column lists are parallel, so unnesting both in one - /// SELECT pairs them by position: a two-column key yields `(x, a)` then `(y, b)`. - /// DuckDB rejects `CASCADE`, `SET NULL` and `SET DEFAULT` at parse time, so a foreign - /// key can only ever be `NO ACTION` and the literals below say exactly what the engine - /// reports. + // The two parallel lists unnest pairwise by position. DuckDB parses no action but `NO ACTION`. static let foreignKeysForTable = """ SELECT constraint_name, @@ -209,13 +179,8 @@ enum DuckDBSchemaQueries { AND view_name = $3 """ - /// The app runs this one itself rather than through the driver's parameterized path, so - /// both values are interpolated. - /// - /// They need opposite treatment, which is why they are named apart. The schema reaches the - /// driver through `SchemaSwitchable.escapedSchema`, which has already run - /// `escapeStringLiteral` over it, so escaping it again turns a schema called `it's` into - /// `it''''s` and the query matches nothing. The catalog comes from `currentDatabase` raw. + // Run by the app itself, so both values are interpolated: the schema arrives already escaped through + // `SchemaSwitchable.escapedSchema`, the catalog raw from `currentDatabase`. static func allTablesMetadata(catalog: String, escapedSchema: String) -> String { let catalogLiteral = quoteLiteral(catalog) let schemaLiteral = "'\(escapedSchema)'" @@ -255,15 +220,14 @@ enum DuckDBSchemaQueries { return "COMMENT ON COLUMN \(target) IS \(commentValue(comment))" } - /// `IS ''` stores an empty comment; only `IS NULL` removes one. + // `IS ''` stores an empty comment; only `IS NULL` removes one. static func commentValue(_ comment: String?) -> String { guard let comment, !comment.isEmpty else { return "NULL" } return quoteLiteral(comment) } - /// DuckDB 1.5.2 has no SERIAL or identity column, so an auto-increment column takes `nextval` of - /// a sequence of its own. `nextval` cannot resolve a name holding `"`, so none is kept, and the - /// hash of the exact pair keeps `a_b.id` and `a.b_id` from sharing one sequence. + // DuckDB 1.5.2 has no SERIAL or identity, so each such column gets a sequence. `nextval` cannot resolve a name + // holding `"`, and the hash keeps `a_b.id` and `a.b_id` apart. static func autoIncrementSequenceName(table: String, column: String) -> String { let readable = "\(table)_\(column)".unicodeScalars.filter { $0 != "\"" } return "\(String(String.UnicodeScalarView(readable)))_\(fnv1a32(table + "\u{0}" + column))_seq" @@ -279,7 +243,7 @@ enum DuckDBSchemaQueries { return String(repeating: "0", count: 8 - hex.count) + hex } - /// `DROP TABLE` leaves the sequence behind, so a table created again under the same name reuses it. + // `DROP TABLE` leaves the sequence behind, so a table created again under the same name reuses it. static func createSequenceStatement(schema: String, sequence: String) -> String { "CREATE SEQUENCE IF NOT EXISTS \(quoteIdentifier(schema)).\(quoteIdentifier(sequence))" } @@ -292,24 +256,9 @@ enum DuckDBSchemaQueries { return "nextval(\(quoteLiteral(name)))" } - /// What the session is holding that closing the handle would destroy, in one round trip. - /// - /// `internal = false` is not optional on either count. DuckDB keeps its own machinery in the - /// `temp` catalog and marks it temporary: measured on the shipped v1.5.2, a connection that - /// has run nothing already reports 46 temporary views. Without the predicate the answer is - /// never zero and the lock is never released. - /// - /// Tables and views are not the whole of it. Measured on the shipped v1.5.2, a `CREATE TEMP - /// MACRO`, a `PREPARE` and a `SET VARIABLE` each leave both temporary counts at zero while - /// living in `duckdb_functions()`, `duckdb_prepared_statements()` and `duckdb_variables()`, and - /// each is destroyed by a close. All five are summed because the gate only asks whether the - /// session holds anything at all. - /// - /// One catalog is the file itself, so a count above one means the user attached something, - /// and an `ATTACH` does not survive a reopen. A catalog the connection attached itself is left - /// out of the count only while it still holds the same file, under the same name and access, - /// and one that is gone counts as held: a reopen attaches every one of them again, which would - /// undo a `DETACH` or replace whatever the user attached under that name. + // Whether closing the handle would destroy anything. Measured on 1.5.2: without `internal = false` an idle + // connection reports 46 temporary views; a TEMP MACRO, a PREPARE and a SET VARIABLE count nowhere else. A + // catalog the driver attached itself is left out only while it still holds the same file, name and access. static let sessionHeldState = sessionHeldState(excluding: []) static func sessionHeldState(excluding attachments: [AttachedDatabase], readOnly: Bool = false) -> String { @@ -333,15 +282,13 @@ enum DuckDBSchemaQueries { """ } - /// `ATTACH` takes no bound parameters, so the path goes in as a quoted literal. + // `ATTACH` takes no bound parameters, so the path goes in as a quoted literal. static func attach(_ database: AttachedDatabase, readOnly: Bool) -> String { let options = readOnly ? " (READ_ONLY)" : "" return "ATTACH \(quoteLiteral(database.expandedPath)) AS \(quoteIdentifier(database.name))\(options)" } - /// `duckdb_settings()` reports no default, so a setting the user changed is only visible by - /// comparing against what the connection opened with. The driver snapshots this at connect - /// and again before releasing, and any difference it does not own itself blocks the release. + // `duckdb_settings()` reports no default, so the driver compares this against its snapshot from connect. static let allSettings = """ SELECT name, value FROM duckdb_settings() @@ -351,9 +298,7 @@ enum DuckDBSchemaQueries { "USE \(quoteIdentifier(database))" } - /// `SET schema` splits its value on `.`, so it cannot address a schema whose name - /// contains one, and it cannot say which catalog it means. The two-part `USE` form - /// is unambiguous for both. + // `SET schema` splits on `.` and names no catalog; the two-part `USE` does neither. static func useSchema(_ schema: String, in database: String?) -> String { guard let database, !database.isEmpty else { return "USE \(quoteIdentifier(schema))" diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBViewDefinition.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBViewDefinition.swift deleted file mode 100644 index 44f2a568b9..0000000000 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBViewDefinition.swift +++ /dev/null @@ -1,28 +0,0 @@ -// -// DuckDBViewDefinition.swift -// DuckDBDriverPlugin -// -// Turns DuckDB's stored view SQL into something the user can run back. -// - -import Foundation - -enum DuckDBViewDefinition { - /// DuckDB stores a view as the `CREATE VIEW ...` it was made with. TablePro opens that - /// text in a query tab for editing, and running it again fails because the view already - /// exists, so the statement is promoted to `CREATE OR REPLACE VIEW`. - /// - /// Only a leading `CREATE VIEW` is rewritten, and only when the stored SQL does not - /// already say `OR REPLACE`. Anything else is returned untouched rather than guessed at. - static func makeReplaceable(_ sql: String) -> String { - let trimmed = sql.trimmingCharacters(in: .whitespacesAndNewlines) - guard !trimmed.isEmpty else { return sql } - - let prefix = "CREATE VIEW" - guard trimmed.count >= prefix.count else { return sql } - let head = String(trimmed.prefix(prefix.count)) - guard head.uppercased() == prefix else { return sql } - - return "CREATE OR REPLACE VIEW" + trimmed.dropFirst(prefix.count) - } -} diff --git a/apple/Plugins/FormatPluginTests/ExportFormatEscapingTests.swift b/apple/Plugins/FormatPluginTests/ExportFormatEscapingTests.swift index d2e6fbf9ef..5bb380d519 100644 --- a/apple/Plugins/FormatPluginTests/ExportFormatEscapingTests.swift +++ b/apple/Plugins/FormatPluginTests/ExportFormatEscapingTests.swift @@ -49,6 +49,22 @@ struct MarkdownExportEscapingTests { #expect(MarkdownTableRenderer.row(["abcdef"], widths: [2]) == "| abcdef |") } + @Test("Aligned cells keep emoji, flags and combining marks whole and pad them by Character") + func alignedCellsKeepGraphemes() { + let rows = [["😀😀😀", "Cafe\u{301}"], ["🇻🇳", "x"]] + let widths = MarkdownTableRenderer.widths(header: ["e", "name"], sample: rows) + #expect(widths == [3, 4]) + #expect(MarkdownTableRenderer.row(rows[0], widths: widths) == "| 😀😀😀 | Cafe\u{301} |") + #expect(MarkdownTableRenderer.row(rows[1], widths: widths) == "| 🇻🇳 | x |") + } + + @Test("A header wider in UTF-16 than in Characters is written whole") + func wideHeaderKeepsItsEmoji() { + let widths = MarkdownTableRenderer.widths(header: ["👩‍👩‍👧"], sample: [["ab"]]) + #expect(widths == [2]) + #expect(MarkdownTableRenderer.row(["👩‍👩‍👧"], widths: widths) == "| 👩‍👩‍👧 |") + } + @Test("The separator is at least three dashes per column") func separatorMinimumWidth() { #expect(MarkdownTableRenderer.separator(columnCount: 2, widths: nil) == "| --- | --- |") diff --git a/apple/Plugins/FormatPluginTests/ExportPause.swift b/apple/Plugins/FormatPluginTests/ExportPause.swift new file mode 100644 index 0000000000..d5a4261af5 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/ExportPause.swift @@ -0,0 +1,34 @@ +// +// ExportPause.swift +// FormatPluginTests +// + +import Foundation + +// Holds an export where its data source calls `hold()` until the test resumes it, so a second export on the same +// plugin can run to its end in between. +actor ExportPause { + private var reached = false + private var arrival: CheckedContinuation? + private var released = false + private var release: CheckedContinuation? + + func hold() async { + reached = true + arrival?.resume() + arrival = nil + guard !released else { return } + await withCheckedContinuation { release = $0 } + } + + func untilReached() async { + guard !reached else { return } + await withCheckedContinuation { arrival = $0 } + } + + func resume() { + released = true + release?.resume() + release = nil + } +} diff --git a/apple/Plugins/FormatPluginTests/ExportStopAndStatusTests.swift b/apple/Plugins/FormatPluginTests/ExportStopAndStatusTests.swift new file mode 100644 index 0000000000..ace7eaf342 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/ExportStopAndStatusTests.swift @@ -0,0 +1,197 @@ +// +// ExportStopAndStatusTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +// Pulled, so the stop lands after the plugin has read the last row and before the stream reports its end. +private final class StopAfterLastRowSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId = "SQLite" + private let progress: PluginExportProgress + + init(progress: PluginExportProgress) { + self.progress = progress + } + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + let progress = progress + let pulls = PullCount() + return AsyncThrowingStream(unfolding: { + switch await pulls.next() { + case 0: + return .header(PluginStreamHeader(columns: ["id", "name"], columnTypeNames: ["INTEGER", "TEXT"])) + case 1: + return .rows([[.text("1"), .text("Ada")], [.text("2"), .text("Grace")]]) + default: + progress.cancel() + return nil + } + }) + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { "" } + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { "\"\(identifier)\"" } + + func escapeStringLiteral(_ value: String) -> String { value } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { nil } +} + +private actor PullCount { + private var value = 0 + + func next() -> Int { + defer { value += 1 } + return value + } +} + +private final class DefinitionSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId = "PostgreSQL" + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + continuation.yield(.header(PluginStreamHeader(columns: ["id"], columnTypeNames: ["INTEGER"]))) + continuation.yield(.rows([[.text("1")]])) + continuation.finish() + } + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + "CREATE TABLE \(table) (id INTEGER)" + } + + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + switch object.kind { + case .view: return "CREATE VIEW \(object.name) AS SELECT 1" + case .routine: return "CREATE FUNCTION \(object.name)() RETURNS integer AS $$ SELECT 1 $$ LANGUAGE sql" + default: return try await fetchTableDDL(table: object.name, databaseName: object.databaseName) + } + } + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { "\"\(identifier)\"" } + + func escapeStringLiteral(_ value: String) -> String { value } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { nil } +} + +private final class StatusLog: @unchecked Sendable { + private let lock = NSLock() + private var entries: [String] = [] + + func append(_ status: String) { + lock.lock() + entries.append(status) + lock.unlock() + } + + var all: [String] { + lock.lock() + defer { lock.unlock() } + return entries + } +} + +struct ExportStopAndStatusTests { + private let table = PluginExportTable(name: "t", databaseName: "", tableType: "table", schema: nil, kind: .table) + + private func temporaryURL(_ fileExtension: String) -> URL { + FileManager.default.temporaryDirectory.appendingPathComponent("\(UUID().uuidString).\(fileExtension)") + } + + private func expectStopAfterLastRowWritesNothing( + _ plugin: any ExportFormatPlugin, + fileExtension: String, + sourceLocation: SourceLocation = #_sourceLocation + ) async { + let progress = PluginExportProgress(progress: Progress(totalUnitCount: 2)) + let destination = temporaryURL(fileExtension) + defer { try? FileManager.default.removeItem(at: destination) } + + await #expect(throws: PluginExportCancellationError.self, sourceLocation: sourceLocation) { + _ = try await plugin.export( + tables: [table], + dataSource: StopAfterLastRowSource(progress: progress), + destination: destination, + progress: progress + ) + } + #expect( + !FileManager.default.fileExists(atPath: destination.path(percentEncoded: false)), + sourceLocation: sourceLocation) + } + + @Test("XLSX stopped after the last row writes no workbook") + func xlsxStoppedAfterLastRow() async { + await expectStopAfterLastRowWritesNothing(XLSXExportPlugin(), fileExtension: "xlsx") + } + + @Test("Markdown stopped after the last row writes no file") + func markdownStoppedAfterLastRow() async { + await expectStopAfterLastRowWritesNothing(MarkdownExportPlugin(), fileExtension: "md") + } + + @Test("XLSX stopped while it copies the sheets into the archive writes no workbook, after reporting its bytes") + func xlsxStoppedDuringArchive() async throws { + let writer = XLSXWriter() + try writer.beginSheet(name: "t", columns: ["v"], includeHeader: false, convertNullToEmpty: true) + let row: [PluginCellValue] = [.text(String(repeating: "x", count: 64 * 1_024))] + try writer.addRows(Array(repeating: row, count: 96), convertNullToEmpty: true) + try writer.finishSheet() + + let nsProgress = Progress(totalUnitCount: 1) + let progress = PluginExportProgress(progress: nsProgress) + // The first status arrives once 4 MiB of the 6 MiB sheet are copied, so the stop lands mid-copy. + let observation = nsProgress.observe(\.localizedAdditionalDescription) { observed, _ in + if observed.localizedAdditionalDescription?.isEmpty == false { observed.cancel() } + } + defer { observation.invalidate() } + let destination = temporaryURL("xlsx") + defer { try? FileManager.default.removeItem(at: destination) } + + await #expect(throws: PluginExportCancellationError.self) { + try await writer.write(to: destination, progress: progress) + } + #expect(nsProgress.localizedAdditionalDescription?.isEmpty == false) + #expect(!FileManager.default.fileExists(atPath: destination.path(percentEncoded: false))) + } + + @Test("A SQL dump moves its status once for each object it fetches after the rows") + func sqlDefinitionsMoveTheStatus() async throws { + let nsProgress = Progress(totalUnitCount: 1) + let progress = PluginExportProgress(progress: nsProgress) + let log = StatusLog() + let observation = nsProgress.observe(\.localizedAdditionalDescription) { observed, _ in + log.append(observed.localizedAdditionalDescription ?? "") + } + defer { observation.invalidate() } + let kinds: [(name: String, kind: PluginExportObjectKind)] = [ + ("orders", .table), ("v_orders", .view), ("p_orders", .routine) + ] + let objects = kinds.map { entry in + PluginExportTable( + name: entry.name, databaseName: "", tableType: entry.kind.rawValue, optionValues: [true, true, true], + schema: nil, kind: entry.kind) + } + + _ = try await SQLExportHarness.shared.dump(tables: objects, dataSource: DefinitionSource(), progress: progress) + + // The table's index read, then the view's and the routine's definitions. + let statuses = log.all.filter { !$0.isEmpty } + #expect(Set(statuses).count == 3) + #expect(statuses.last?.hasSuffix("3 of 3") == true) + } +} diff --git a/apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift b/apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift index f995008f40..320cca3692 100644 --- a/apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift +++ b/apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift @@ -75,6 +75,24 @@ private final class ForeignKeyRecordingSink: PluginImportDataSink, @unchecked Se func enableForeignKeyChecks() async throws { enabled += 1 } } +// The JSON import reads the target table while it parses the document, after the run has started. +private final class TableReadingSink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "SQLite" + var onTargetTableRead: () -> Void = {} + private(set) var rows: [[String: PluginCellValue]] = [] + + var targetTable: String? { + onTargetTableRead() + return "people" + } + + func execute(statement: String) async throws {} + func insertRow(_ values: [String: PluginCellValue]) async throws { rows.append(values) } + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} +} + private final class OneStatementSource: PluginImportSource, @unchecked Sendable { func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { AsyncThrowingStream { continuation in @@ -182,4 +200,30 @@ struct FormatSettingsChangedMidRunTests { #expect(sink.disabled == 1) #expect(sink.enabled == 1) } + + @Test("A JSON import keeps skipping an element that is not an object after another window picks a stop mode") + func jsonImportKeepsItsSkipChoice() async throws { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("\(UUID().uuidString).json") + try Data(#"[{"name": "Ada"}, 5, {"name": "Grace"}]"#.utf8).write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + + let plugin = JSONImportPlugin() + let stored = plugin.settings + defer { plugin.settings = stored } + var options = JSONImportOptions() + options.errorHandling = .skipAndContinue + options.wrapInTransaction = true + options.deleteExistingRows = false + plugin.settings = options + + let sink = TableReadingSink() + sink.onTargetTableRead = { plugin.settings.errorHandling = .stopAndRollback } + let result = try await plugin.performImport( + source: RoundTripImportFile(url), + sink: sink, + progress: PluginImportProgress(progress: Progress()) + ) + #expect(sink.rows.compactMap { $0["name"] } == [.text("Ada"), .text("Grace")]) + #expect(result.skippedStatements == 1) + } } diff --git a/apple/Plugins/FormatPluginTests/SQLExportBatchSeparatorTests.swift b/apple/Plugins/FormatPluginTests/SQLExportBatchSeparatorTests.swift index 66eabd3342..4187e41634 100644 --- a/apple/Plugins/FormatPluginTests/SQLExportBatchSeparatorTests.swift +++ b/apple/Plugins/FormatPluginTests/SQLExportBatchSeparatorTests.swift @@ -12,34 +12,6 @@ import Foundation import TableProPluginKit import Testing -/// Holds an export inside its first table definition fetch until the test lets it go, so a second export can run to -/// its end on the same plugin in between. -private actor ExportPause { - private var reached = false - private var arrival: CheckedContinuation? - private var released = false - private var release: CheckedContinuation? - - func hold() async { - reached = true - arrival?.resume() - arrival = nil - guard !released else { return } - await withCheckedContinuation { release = $0 } - } - - func untilReached() async { - guard !reached else { return } - await withCheckedContinuation { arrival = $0 } - } - - func resume() { - released = true - release?.resume() - release = nil - } -} - struct SQLExportBatchSeparatorTests { private enum Engine: String { case sqlServer = "SQL Server" @@ -164,7 +136,8 @@ struct SQLExportBatchSeparatorTests { #expect(batches.contains("SET IDENTITY_INSERT [orders] ON;")) #expect(batches.contains("SET IDENTITY_INSERT [orders] OFF;")) #expect(batches.contains { $0.hasPrefix("INSERT INTO [orders]") && $0.hasSuffix(";") }) - #expect(batches.filter { $0.hasPrefix("DROP ") }.count == 3) + // The routine gets no drop: the source writes none, and SQL Server has no DROP ROUTINE. + #expect(batches.filter { $0.hasPrefix("DROP ") }.count == 2) } @Test("A SQL Server dump ends each statement on a GO line, as SQL Server Management Studio writes one") @@ -204,7 +177,7 @@ struct SQLExportBatchSeparatorTests { on: plugin, tables: tables, dataSource: ServerDataSource(engine: otherEngine) ) await pause.resume() - return (try await held, other) + return (try await held.text, other.text) } #expect(goLines(in: held) == expectedGoLines[heldEngine], "\(heldEngine.rawValue) held while another ran") #expect(goLines(in: other) == expectedGoLines[otherEngine], "\(otherEngine.rawValue) run during another") diff --git a/apple/Plugins/FormatPluginTests/SQLExportConcurrentRunTests.swift b/apple/Plugins/FormatPluginTests/SQLExportConcurrentRunTests.swift new file mode 100644 index 0000000000..6c6864dfae --- /dev/null +++ b/apple/Plugins/FormatPluginTests/SQLExportConcurrentRunTests.swift @@ -0,0 +1,115 @@ +// +// SQLExportConcurrentRunTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +private struct IndexReadFailure: Error {} + +// A PostgreSQL table whose id defaults from a sequence, and whose indexes cannot be read. +private final class SequenceTableSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId = "PostgreSQL" + private let sequenceDDL: String + private let pause: ExportPause? + + init(sequenceDDL: String, pause: ExportPause? = nil) { + self.sequenceDDL = sequenceDDL + self.pause = pause + } + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + continuation.yield(.header(PluginStreamHeader(columns: ["id"], columnTypeNames: ["INTEGER"]))) + continuation.yield(.rows([[.text("1")]])) + continuation.finish() + } + } + + func fetchDependentSequences(table: String, databaseName: String) async throws -> [PluginSequenceInfo] { + await pause?.hold() + return [PluginSequenceInfo(name: "id_seq", ddl: sequenceDDL)] + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + "CREATE TABLE \(quoteIdentifier(table)) (id INTEGER DEFAULT nextval('id_seq'))" + } + + func fetchIndexDDL(table: String, databaseName: String) async throws -> [String] { + throw IndexReadFailure() + } + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { + "\"\(identifier.replacingOccurrences(of: "\"", with: "\"\""))\"" + } + + func escapeStringLiteral(_ value: String) -> String { + value.replacingOccurrences(of: "'", with: "''") + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { nil } +} + +struct SQLExportConcurrentRunTests { + // One name, two definitions: the name is what a run records once written. + private static let ordersSequence = #"CREATE SEQUENCE "id_seq" START WITH 100"# + private static let invoicesSequence = #"CREATE SEQUENCE "id_seq" START WITH 200"# + + private func table(_ name: String) -> PluginExportTable { + PluginExportTable( + name: name, + databaseName: "", + tableType: "table", + optionValues: [true, true, true], + schema: nil, + kind: .table) + } + + private func withoutTimestamp(_ dump: String) -> String { + dump.components(separatedBy: "\n") + .filter { !$0.hasPrefix("-- Generated: ") } + .joined(separator: "\n") + } + + // Every macOS window and iOS scene gets the same plugin instance, so a second export can start while one runs. + @Test( + "Two SQL exports on one plugin each write their own sequence and report only their own failures", + .timeLimit(.minutes(1))) + func overlappingExportsKeepTheirOwnBookkeeping() async throws { + let orders = [table("orders")] + let invoices = [table("invoices")] + let ordersAlone = try await SQLExportHarness.shared.dump( + tables: orders, dataSource: SequenceTableSource(sequenceDDL: Self.ordersSequence)) + let invoicesAlone = try await SQLExportHarness.shared.dump( + tables: invoices, dataSource: SequenceTableSource(sequenceDDL: Self.invoicesSequence)) + + let pause = ExportPause() + let (held, other) = try await SQLExportHarness.shared.withPlugin { plugin in + async let held = SQLExportHarness.dump( + on: plugin, tables: orders, + dataSource: SequenceTableSource(sequenceDDL: Self.ordersSequence, pause: pause)) + await pause.untilReached() + let other = try await SQLExportHarness.dump( + on: plugin, tables: invoices, dataSource: SequenceTableSource(sequenceDDL: Self.invoicesSequence)) + await pause.resume() + return (try await held, other) + } + + #expect(held.text.contains(Self.ordersSequence)) + #expect(other.text.contains(Self.invoicesSequence)) + #expect(held.result.warnings.contains { $0.contains("orders") }) + #expect(!held.result.warnings.contains { $0.contains("invoices") }) + #expect(other.result.warnings.contains { $0.contains("invoices") }) + #expect(!other.result.warnings.contains { $0.contains("orders") }) + #expect(held.result.warnings == ordersAlone.result.warnings) + #expect(other.result.warnings == invoicesAlone.result.warnings) + #expect(withoutTimestamp(held.text) == withoutTimestamp(ordersAlone.text)) + #expect(withoutTimestamp(other.text) == withoutTimestamp(invoicesAlone.text)) + } +} diff --git a/apple/Plugins/FormatPluginTests/SQLExportDropClauseTests.swift b/apple/Plugins/FormatPluginTests/SQLExportDropClauseTests.swift index a6910c8d51..08e5d40416 100644 --- a/apple/Plugins/FormatPluginTests/SQLExportDropClauseTests.swift +++ b/apple/Plugins/FormatPluginTests/SQLExportDropClauseTests.swift @@ -13,14 +13,19 @@ import Testing /// easier"); sqlite3 rejects `DROP TABLE IF EXISTS "fields" CASCADE;` outright with /// `near "CASCADE": syntax error`. The clause was previously emitted for every engine. struct SQLExportDropClauseTests { - private final class StubExportDataSource: PluginExportDataSource, @unchecked Sendable { let databaseTypeId: String let supportsCascadeDrop: Bool + private let routineDrop: String? - init(databaseTypeId: String, supportsCascadeDrop: Bool) { + init(databaseTypeId: String, supportsCascadeDrop: Bool, routineDrop: String? = nil) { self.databaseTypeId = databaseTypeId self.supportsCascadeDrop = supportsCascadeDrop + self.routineDrop = routineDrop + } + + func dropStatement(for object: PluginExportTable) -> String? { + object.kind == .routine ? routineDrop : nil } func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { @@ -99,6 +104,27 @@ struct SQLExportDropClauseTests { #expect(sql.contains("DROP TABLE IF EXISTS `fields` CASCADE;")) } + private func routineDump(routineDrop: String?) async throws -> [String] { + let routine = PluginExportTable( + name: "p", databaseName: "", tableType: "routine", optionValues: [true, true, true], schema: nil, kind: .routine) + let source = StubExportDataSource(databaseTypeId: "MySQL", supportsCascadeDrop: false, routineDrop: routineDrop) + return try await SQLExportHarness.shared.dump(tables: [routine], dataSource: source) + .text.components(separatedBy: "\n") + } + + // MariaDB stops a restore at `DROP ROUTINE IF EXISTS` with error 1064, and only PostgreSQL has the statement. + @Test("A routine the driver writes no drop for gets no DROP line") + func routineWithoutDriverDropGetsNoDrop() async throws { + let lines = try await routineDump(routineDrop: nil) + #expect(!lines.contains { $0.hasPrefix("DROP") }) + } + + @Test("A routine keeps the drop its driver writes") + func routineKeepsTheDriverDrop() async throws { + let lines = try await routineDump(routineDrop: "DROP PROCEDURE IF EXISTS `p`") + #expect(lines.filter { $0.hasPrefix("DROP") } == ["DROP PROCEDURE IF EXISTS `p`;"]) + } + /// The default is the answer that is never a syntax error. This asserts the protocol /// extension's own value, so flipping that default fails here rather than shipping a dump /// SQLite refuses. diff --git a/apple/Plugins/FormatPluginTests/SQLExportHarness.swift b/apple/Plugins/FormatPluginTests/SQLExportHarness.swift index 06337648ec..1b2c69056f 100644 --- a/apple/Plugins/FormatPluginTests/SQLExportHarness.swift +++ b/apple/Plugins/FormatPluginTests/SQLExportHarness.swift @@ -59,22 +59,22 @@ internal actor SQLExportHarness { return try await body(plugin) } - /// One export on `plugin`, read back as text. Only for a plugin `withPlugin` hands out, which holds known settings. + // Only for a plugin `withPlugin` hands out, which holds known settings. internal static func dump( on plugin: SQLExportPlugin, tables: [PluginExportTable], dataSource: any PluginExportDataSource - ) async throws -> String { + ) async throws -> (text: String, result: ExportFormatResult) { let destination = FileManager.default.temporaryDirectory .appendingPathComponent("\(UUID().uuidString).sql") defer { try? FileManager.default.removeItem(at: destination) } - _ = try await plugin.export( + let result = try await plugin.export( tables: tables, dataSource: dataSource, destination: destination, progress: PluginExportProgress(progress: Progress(totalUnitCount: 1)) ) - return try String(contentsOf: destination, encoding: .utf8) + return (try String(contentsOf: destination, encoding: .utf8), result) } /// The parts a split export wrote, in restore order, and the whole dump as one part when it did diff --git a/apple/Plugins/FormatPluginTests/XLSXImportTests.swift b/apple/Plugins/FormatPluginTests/XLSXImportTests.swift index fe5b08b0d1..f975448de9 100644 --- a/apple/Plugins/FormatPluginTests/XLSXImportTests.swift +++ b/apple/Plugins/FormatPluginTests/XLSXImportTests.swift @@ -5,6 +5,7 @@ import Foundation import TableProPluginKit +import TableProTabularIO import Testing private final class RecordingSink: PluginImportDataSink, @unchecked Sendable { @@ -31,8 +32,8 @@ struct XLSXImportSheetTests { return try XLSXImportSheet.read(at: url, hasHeaderRow: hasHeaderRow, isCancelled: { false }) } - private func values(_ sheet: XLSXImportSheet) -> [[String?]] { - sheet.rows(0 ..< sheet.rowCount).map(\.values) + private func values(_ sheet: XLSXImportSheet) throws -> [[String?]] { + try sheet.rows(0 ..< sheet.rowCount).map(\.values) } private func number(_ reference: String, _ value: String) -> String { @@ -77,7 +78,7 @@ struct XLSXImportSheetTests { func cellsArePlacedByReference() throws { let cells = [("A1", "a"), ("B1", "b"), ("Z1", "z"), ("AA1", "aa"), ("AB1", "ab"), ("BA1", "ba")] let sheet = try read(XLSXImportFixture(rows: "\(cells.map { inline($0.0, $0.1) }.joined())"), hasHeaderRow: false) - let row = try #require(values(sheet).first) + let row = try #require(try values(sheet).first) #expect(row.count == 53) #expect([0, 1, 25, 26, 27, 52].map { row[$0] } == ["a", "b", "z", "aa", "ab", "ba"]) #expect(row.compactMap { $0 }.count == 6) @@ -102,7 +103,7 @@ struct XLSXImportSheetTests { """ let sheet = try read(XLSXImportFixture(rows: rows)) #expect(sheet.header == ["id", "name"]) - #expect(values(sheet) == [["1", nil]]) + #expect(try values(sheet) == [["1", nil]]) } @Test("A cell at the last column and many empty rows import as one row") @@ -116,7 +117,7 @@ struct XLSXImportSheetTests { #expect(header[1] == "column2") #expect(header.last == "far") #expect(sheet.rowCount == 1) - let row = try #require(sheet.rows(0 ..< 1).first) + let row = try #require(try sheet.rows(0 ..< 1).first) #expect(row.line == 100_002) #expect(row.values.first == "7") #expect(row.values.dropFirst().allSatisfy { $0 == nil }) @@ -141,38 +142,38 @@ struct XLSXImportSheetTests { @Test("A shared-string index the table lacks keeps the raw value") func outOfRangeIndexKeepsRawValue() throws { let rows = "99" - #expect(values(try read(XLSXImportFixture(rows: rows, sharedStrings: ["only"]), hasHeaderRow: false)) == [["99"]]) + #expect(try values(read(XLSXImportFixture(rows: rows, sharedStrings: ["only"]), hasHeaderRow: false)) == [["99"]]) } @Test("A gap in a row is filled from the cell references") func gapsArePlacedByReference() throws { let rows = "\(number("A1", "1"))\(number("C1", "3"))" - #expect(values(try read(XLSXImportFixture(rows: rows), hasHeaderRow: false)) == [["1", nil, "3"]]) + #expect(try values(read(XLSXImportFixture(rows: rows), hasHeaderRow: false)) == [["1", nil, "3"]]) } @Test("Rows are padded to the widest row") func rowsArePadded() throws { let rows = "\(number("A1", "1"))\(number("B1", "2"))\(number("A2", "3"))" - #expect(values(try read(XLSXImportFixture(rows: rows), hasHeaderRow: false)) == [["1", "2"], ["3", nil]]) + #expect(try values(read(XLSXImportFixture(rows: rows), hasHeaderRow: false)) == [["1", "2"], ["3", nil]]) } @Test("Numbers keep their text and booleans read as 1 and 0") func valuesKeepTheirStoredText() throws { let rows = "\(number("A1", "12.50"))\(number("B1", "007"))10" - #expect(values(try read(XLSXImportFixture(rows: rows), hasHeaderRow: false)) == [["12.50", "007", "1", "0"]]) + #expect(try values(read(XLSXImportFixture(rows: rows), hasHeaderRow: false)) == [["12.50", "007", "1", "0"]]) } @Test("A number of nineteen digits reads as its text") func nineteenDigitNumberIsRead() throws { let rows = "\(number("A1", "9999999999999999999"))\(number("B1", "-9223372036854775808"))" - #expect(values(try read(XLSXImportFixture(rows: rows), hasHeaderRow: false)) == [["9999999999999999999", "-9223372036854775808"]]) + #expect(try values(read(XLSXImportFixture(rows: rows), hasHeaderRow: false)) == [["9999999999999999999", "-9223372036854775808"]]) } @Test("A row's line is its row number in the sheet") func linesFollowTheSheet() throws { let rows = "\(inline("A3", "id"))\(number("A5", "1"))" let sheet = try read(XLSXImportFixture(rows: rows)) - #expect(sheet.rows(0 ..< sheet.rowCount).map(\.line) == [5]) + #expect(try sheet.rows(0 ..< sheet.rowCount).map(\.line) == [5]) } @Test("An empty sheet reads as no rows rather than failing") @@ -189,8 +190,8 @@ struct XLSXImportSheetTests { func emptyTextIsNotAMissingCell() throws { let sheet = try read(emptyCells) #expect(sheet.header == ["id", "note", "code"]) - #expect(values(sheet) == [["1", "", ""], [nil, "", ""], ["2", nil, nil]]) - #expect(sheet.rows(0 ..< sheet.rowCount).map(\.line) == [2, 3, 5]) + #expect(try values(sheet) == [["1", "", ""], [nil, "", ""], ["2", nil, nil]]) + #expect(try sheet.rows(0 ..< sheet.rowCount).map(\.line) == [2, 3, 5]) } @Test("With Empty as NULL off, a cell that stores empty text imports as empty text") @@ -220,7 +221,7 @@ struct XLSXImportSheetTests { 0.51.5\(number("E1", "45292")) """ let fixture = XLSXImportFixture(rows: rows, cellFormatIDs: [0, 14, 22, 20, 46]) - #expect(values(try read(fixture, hasHeaderRow: false)) == [["2024-01-01", "2024-01-01 12:00:00", "12:00:00", "36:00:00", "45292"]]) + #expect(try values(read(fixture, hasHeaderRow: false)) == [["2024-01-01", "2024-01-01 12:00:00", "12:00:00", "36:00:00", "45292"]]) } @Test("Field detection types a date column as text") @@ -257,4 +258,66 @@ struct XLSXImportSheetTests { #expect(fields.map(\.sampleValue) == ["1", "Ada"]) #expect(fields.map(\.inferredType) == [.integer, .text]) } + + private func sharedStringCell(_ reference: String) -> String { + "0" + } + + // One shared string of `length` bytes, referenced by every cell the rows name: stored once, read once per cell. + private func repeatedSharedString(length: Int, rows: String) -> XLSXImportFixture { + XLSXImportFixture(rows: rows, sharedStrings: ["\(String(repeating: "x", count: length))"]) + } + + private func headerReferencing(_ count: Int) -> String { + let cells = (0..\(cells.joined())" + } + + @Test("A shared string past the cell limit fails the header before any cell referencing it is copied") + func oversizedSharedStringInTheHeaderFails() throws { + let fixture = repeatedSharedString(length: XLSXImportSheet.cellByteLimit + 1, rows: headerReferencing(256)) + #expect(throws: PluginImportError.self) { try read(fixture) } + } + + @Test("Field detection fails on a header that repeats an oversized shared string") + func detectionFailsOnAnOversizedSharedString() throws { + let url = try repeatedSharedString(length: XLSXImportSheet.cellByteLimit + 1, rows: headerReferencing(256)).write() + defer { try? FileManager.default.removeItem(at: url) } + let plugin = XLSXImportPlugin() + let saved = plugin.settings + defer { plugin.settings = saved } + plugin.settings = XLSXImportOptions() + #expect(throws: PluginImportError.self) { try plugin.detectSourceFields(at: url, targetTable: nil) } + } + + @Test("A header whose cells fit the cell limit fails once their references pass the read limit") + func repeatedSharedStringPastTheReadLimitFails() throws { + let references = XLSXImportSheet.readByteLimit / XLSXImportSheet.cellByteLimit + 1 + let fixture = repeatedSharedString(length: XLSXImportSheet.cellByteLimit, rows: headerReferencing(references)) + #expect(throws: PluginImportError.self) { try read(fixture) } + } + + @Test("An import whose batch repeats a shared string past the read limit fails and inserts nothing") + func importFailsOnABatchPastTheReadLimit() async throws { + let references = XLSXImportSheet.readByteLimit / XLSXImportSheet.cellByteLimit + 1 + let dataRows = (2...(references + 1)).map { "\(sharedStringCell("A\($0)"))" } + let fixture = repeatedSharedString( + length: XLSXImportSheet.cellByteLimit, + rows: "\(inline("A1", "note"))\(dataRows.joined())") + let url = try fixture.write() + defer { try? FileManager.default.removeItem(at: url) } + let plugin = XLSXImportPlugin() + let saved = plugin.settings + defer { plugin.settings = saved } + plugin.settings = XLSXImportOptions() + let sink = RecordingSink() + await #expect(throws: PluginImportError.self) { + _ = try await plugin.performImport( + source: RoundTripImportFile(url), + sink: sink, + progress: PluginImportProgress(progress: Progress(totalUnitCount: 1)) + ) + } + #expect(sink.rows.isEmpty) + } } diff --git a/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift b/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift index 367fe36762..1b856a11fc 100644 --- a/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift +++ b/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift @@ -129,6 +129,20 @@ struct XLSXWriterCellTypeTests { + "") } + @Test("A string with leading or trailing whitespace asks Excel to keep it", arguments: [ + " A1", "Ada ", " ", "\tx", "x\n" + ]) + func outerWhitespaceIsPreserved(value: String) async throws { + let xml = try await sheetXML(columnTypeNames: ["text"], row: [.text(value)]) + #expect(xml == "\(value)") + } + + @Test("A string with whitespace only inside it gets no space attribute") + func innerWhitespaceNeedsNoAttribute() async throws { + let xml = try await sheetXML(columnTypeNames: ["text"], row: [.text("Ada Lovelace")]) + #expect(xml == "Ada Lovelace") + } + @Test("A missing or unknown type name is untyped", arguments: ["", " ", "interval", "geometry", "variant"]) func untypedTypeNames(typeName: String) { #expect(XLSXWriter.ColumnNumbers(typeName: typeName) == .untyped) diff --git a/apple/Plugins/JSONExportPlugin/JSONExportModels.swift b/apple/Plugins/JSONExportPlugin/JSONExportModels.swift index 0a055cc2a6..d9d1bba309 100644 --- a/apple/Plugins/JSONExportPlugin/JSONExportModels.swift +++ b/apple/Plugins/JSONExportPlugin/JSONExportModels.swift @@ -6,7 +6,7 @@ import Foundation /// How the rows are laid out in the file. -public enum JSONExportLayout: String, Codable, CaseIterable, Sendable, Identifiable { +nonisolated public enum JSONExportLayout: String, Codable, CaseIterable, Sendable, Identifiable { /// One object per table, each holding an array of rows. The whole file is one JSON value. case object /// One row per line, with no wrapping array. A stream reader can process it a line at a time, @@ -30,7 +30,7 @@ public enum JSONExportLayout: String, Codable, CaseIterable, Sendable, Identifia } } -public struct JSONExportOptions: Equatable, Codable { +nonisolated public struct JSONExportOptions: Equatable, Codable { public var prettyPrint: Bool = true public var includeNullValues: Bool = true public var preserveAllAsStrings: Bool = false diff --git a/apple/Plugins/JSONExportPlugin/JSONExportOptionsView.swift b/apple/Plugins/JSONExportPlugin/JSONExportOptionsView.swift index aa0bfa297e..023f89d974 100644 --- a/apple/Plugins/JSONExportPlugin/JSONExportOptionsView.swift +++ b/apple/Plugins/JSONExportPlugin/JSONExportOptionsView.swift @@ -40,3 +40,10 @@ struct JSONExportOptionsView: View { .font(.system(size: 13)) } } + +extension JSONExportPlugin { + @MainActor + func settingsView() -> AnyView? { + AnyView(JSONExportOptionsView(plugin: self)) + } +} diff --git a/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift b/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift index 60adecb627..293eda2027 100644 --- a/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift +++ b/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift @@ -5,10 +5,9 @@ import Combine import Foundation -import SwiftUI import TableProPluginKit -final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugin, @unchecked Sendable { +nonisolated final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugin, @unchecked Sendable { static let pluginName = "JSON Export" static let pluginVersion = "1.0.0" static let pluginDescription = "Export data to JSON format" @@ -20,7 +19,9 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug typealias Settings = JSONExportOptions static let settingsStorageId = "json" - @Published var settings = JSONExportOptions() { + // A nonisolated class cannot hold an @Published property, so the change is announced by hand. + var settings = JSONExportOptions() { + willSet { objectWillChange.send() } didSet { saveSettings() } } @@ -30,11 +31,6 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug settings.layout.fileExtension } - @MainActor - func settingsView() -> AnyView? { - AnyView(JSONExportOptionsView(plugin: self)) - } - func resetSettingsToDefaults() { settings = JSONExportOptions() } @@ -115,7 +111,7 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug let colTypeName = colIndex < (columnTypeNames ?? []).count ? (columnTypeNames ?? [])[colIndex] : "" - let jsonValue = formatJSONValue( + let jsonValue = Self.formatJSONValue( value, columnTypeName: colTypeName, preserveAsString: options.preserveAllAsStrings @@ -162,9 +158,7 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug return ExportFormatResult() } - // MARK: - Private - - private func formatJSONValue(_ value: PluginCellValue, columnTypeName: String, preserveAsString: Bool) -> String { + static func formatJSONValue(_ value: PluginCellValue, columnTypeName: String, preserveAsString: Bool) -> String { switch value { case .null: return "null" @@ -175,13 +169,13 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug } } - private func formatJSONTextValue(_ val: String, columnTypeName: String, preserveAsString: Bool) -> String { + private static func formatJSONTextValue(_ val: String, columnTypeName: String, preserveAsString: Bool) -> String { if preserveAsString { return "\"\(PluginExportUtilities.escapeJSONString(val))\"" } let folded = val.lowercased() - if Self.isBooleanColumnType(columnTypeName), folded == "true" || folded == "false" { + if isBooleanColumnType(columnTypeName), folded == "true" || folded == "false" { return folded } diff --git a/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift b/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift index 4aa0e25e01..05ecee102e 100644 --- a/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift +++ b/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift @@ -6,7 +6,7 @@ import Foundation import TableProPluginKit -enum JSONDocumentRows: Equatable { +nonisolated enum JSONDocumentRows: Equatable { // The rows are the elements of the array that opens at `offset`. `slot` is the one `visit` was given for them. case array(offset: Int, elementCount: Int, slot: Int) case singleObject(offset: Int) @@ -14,7 +14,7 @@ enum JSONDocumentRows: Equatable { // Finds which array of a `.json` file holds the rows, by the rules `JSONImportParsing.extractElements` // applies to a parsed file, in one pass that builds nothing but member names. -enum JSONDocumentOutline { +nonisolated enum JSONDocumentOutline { private struct Member { let name: String let slot: Int diff --git a/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift b/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift index 8bace6ac1c..707cfdf69b 100644 --- a/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift +++ b/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift @@ -5,7 +5,7 @@ import Foundation -enum JSONDocumentError: LocalizedError, Equatable { +nonisolated enum JSONDocumentError: LocalizedError, Equatable { case endsEarly case unexpectedCharacter(byteOffset: Int) case invalidText @@ -27,7 +27,7 @@ enum JSONDocumentError: LocalizedError, Equatable { // each value a caller asks for, so every value reads exactly as it would in a whole-file parse. // UTF-16 and UTF-32 text is turned into UTF-8 as it is read, so `offset` counts UTF-8 bytes and // `fileOffset` the file's own. -struct JSONDocumentReader { +nonisolated struct JSONDocumentReader { static let defaultChunkSize = 1 << 20 private static let quote: UInt8 = 0x22 diff --git a/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift b/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift index d11b26c8b3..b92df0b788 100644 --- a/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift +++ b/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift @@ -8,13 +8,13 @@ import TableProPluginKit // The values of a `.json` file that become rows. A value `JSONSerialization` cannot read throws in // every error mode, as it failed the whole-file parse the import used to run. -protocol JSONElementSequence { +nonisolated protocol JSONElementSequence { mutating func next() throws -> Any? func close() } // The elements of one array of the file, each read and parsed as it is reached. -struct JSONStreamedElements: JSONElementSequence { +nonisolated struct JSONStreamedElements: JSONElementSequence { private var reader: JSONDocumentReader private var hasMore: Bool @@ -40,7 +40,7 @@ struct JSONStreamedElements: JSONElementSequence { } // A file whose top-level object is the one row, built member by member. -struct JSONSingleRow: JSONElementSequence { +nonisolated struct JSONSingleRow: JSONElementSequence { private var row: [String: PluginCellValue]? init(_ row: [String: PluginCellValue]) { @@ -57,7 +57,7 @@ struct JSONSingleRow: JSONElementSequence { // Hands a `.json` file's rows to `RowImportRunner` a batch at a time, numbered by their place in the // array, and keeps the elements that are not objects, which Skip and Continue passes over. -struct JSONElementBatches { +nonisolated struct JSONElementBatches { private var elements: any JSONElementSequence private let elementsPerBatch: Int private let skipsUnreadableElements: Bool diff --git a/apple/Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift b/apple/Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift index ab5ac80531..9564c229b1 100644 --- a/apple/Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift +++ b/apple/Plugins/JSONImportPlugin/JSONFieldDetectionCache.swift @@ -12,7 +12,7 @@ import TableProPluginKit /// A JSON Lines file's fields do not depend on the table they are matched against, and reading /// them means reading every line. The import sheet asks again on every destination table pick, so /// each pick used to read the whole file again. -final class JSONFieldDetectionCache: Sendable { +nonisolated final class JSONFieldDetectionCache: Sendable { /// A file counts as unchanged while its path, file number, size and modification date all /// match. The identity is taken before the read, so an edit made during one leaves a stale /// identity behind and the next request reads the file again. It describes the file a link diff --git a/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift b/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift index f4c5acf4a8..349126904d 100644 --- a/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift +++ b/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift @@ -9,7 +9,7 @@ import TableProPluginKit /// Every field any row of the file names, each with the first value it holds and a type that fits /// every value it holds. It keeps one small record per field rather than the values themselves, /// so it can read every row of a file of any size. -struct JSONFieldSurvey { +nonisolated struct JSONFieldSurvey { private var profiles: [String: JSONFieldProfile] = [:] mutating func add(_ row: NSDictionary) { @@ -35,7 +35,7 @@ struct JSONFieldSurvey { } } -struct JSONFieldProfile { +nonisolated struct JSONFieldProfile { private(set) var sampleValue: String? private(set) var kinds = JSONValueKinds() @@ -49,7 +49,7 @@ struct JSONFieldProfile { } } -struct JSONValueKinds { +nonisolated struct JSONValueKinds { private var sawValue = false private var allNested = true private var allBoolean = true @@ -76,7 +76,7 @@ struct JSONValueKinds { /// Reads a parsed JSON value's kind from its Core Foundation type. A survey classifies every value /// in the file, and a Swift `is` or `as?` cast from `Any` costs about ten times as much. -enum JSONValueKind: Equatable { +nonisolated enum JSONValueKind: Equatable { case null case nested case boolean diff --git a/apple/Plugins/JSONImportPlugin/JSONImportOptions.swift b/apple/Plugins/JSONImportPlugin/JSONImportOptions.swift index 81d2ec992e..613ee12a95 100644 --- a/apple/Plugins/JSONImportPlugin/JSONImportOptions.swift +++ b/apple/Plugins/JSONImportPlugin/JSONImportOptions.swift @@ -6,7 +6,7 @@ import Foundation import TableProPluginKit -struct JSONImportOptions: Equatable, Codable { +nonisolated struct JSONImportOptions: Equatable, Codable { var errorHandling: ImportErrorHandling = .stopAndRollback var wrapInTransaction: Bool = true var deleteExistingRows: Bool = false diff --git a/apple/Plugins/JSONImportPlugin/JSONImportOptionsView.swift b/apple/Plugins/JSONImportPlugin/JSONImportOptionsView.swift index fb9d481504..69f913762e 100644 --- a/apple/Plugins/JSONImportPlugin/JSONImportOptionsView.swift +++ b/apple/Plugins/JSONImportPlugin/JSONImportOptionsView.swift @@ -32,3 +32,10 @@ struct JSONImportOptionsView: View { } } } + +extension JSONImportPlugin { + @MainActor + func settingsView() -> AnyView? { + AnyView(JSONImportOptionsView(plugin: self)) + } +} diff --git a/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift b/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift index e92faf3392..bca0b68db9 100644 --- a/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift +++ b/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift @@ -7,7 +7,7 @@ import Foundation import TableProNumberFormatting import TableProPluginKit -enum JSONImportParsing { +nonisolated enum JSONImportParsing { static func isLineDelimited(_ url: URL) -> Bool { ["jsonl", "ndjson"].contains(url.pathExtension.lowercased()) } diff --git a/apple/Plugins/JSONImportPlugin/JSONImportPlugin.swift b/apple/Plugins/JSONImportPlugin/JSONImportPlugin.swift index 78a8533992..43ff5d8b93 100644 --- a/apple/Plugins/JSONImportPlugin/JSONImportPlugin.swift +++ b/apple/Plugins/JSONImportPlugin/JSONImportPlugin.swift @@ -5,10 +5,9 @@ import Combine import Foundation -import SwiftUI import TableProPluginKit -final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugin, @unchecked Sendable { +nonisolated final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugin, @unchecked Sendable { static let pluginName = "JSON Import" static let pluginVersion = "1.0.0" static let pluginDescription = "Import data from JSON files" @@ -21,27 +20,24 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug typealias Settings = JSONImportOptions static let settingsStorageId = "json-import" - @Published var settings = JSONImportOptions() { + // A nonisolated class cannot hold an @Published property, so the change is announced by hand. + var settings = JSONImportOptions() { + willSet { objectWillChange.send() } didSet { saveSettings() } } required init() { loadSettings() } - var rowImportConfiguration: RowImportRunner.Configuration? { runConfiguration } + var rowImportConfiguration: RowImportRunner.Configuration? { Self.runConfiguration(settings) } - private var runConfiguration: RowImportRunner.Configuration { + private static func runConfiguration(_ options: JSONImportOptions) -> RowImportRunner.Configuration { RowImportRunner.Configuration( - errorHandling: settings.errorHandling, - wrapInTransaction: settings.wrapInTransaction, - deleteExistingRows: settings.deleteExistingRows + errorHandling: options.errorHandling, + wrapInTransaction: options.wrapInTransaction, + deleteExistingRows: options.deleteExistingRows ) } - @MainActor - func settingsView() -> AnyView? { - AnyView(JSONImportOptionsView(plugin: self)) - } - func resetSettingsToDefaults() { settings = JSONImportOptions() } @@ -61,7 +57,9 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug ) async throws -> PluginImportResult { let startTime = Date() let url = source.fileURL() - let configuration = runConfiguration + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-import. + let options = settings + let configuration = Self.runConfiguration(options) let outcome: RowImportRunner.Outcome var unreadableLines: [PluginImportResult.ImportStatementError] = [] @@ -71,7 +69,7 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug var batches = JSONLineBatches( lines: try JSONLineReader(url: url, checkCancellation: progress.checkCancellation), linesPerBatch: Self.batchSize, - skipsUnreadableLines: settings.errorHandling == .skipAndContinue, + skipsUnreadableLines: options.errorHandling == .skipAndContinue, maxRecordedErrors: Self.maxRecordedErrors ) defer { batches.close() } @@ -90,7 +88,7 @@ final class JSONImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug var batches = JSONElementBatches( elements: document.elements, elementsPerBatch: Self.batchSize, - skipsUnreadableElements: settings.errorHandling == .skipAndContinue, + skipsUnreadableElements: options.errorHandling == .skipAndContinue, maxRecordedErrors: Self.maxRecordedErrors ) defer { batches.close() } diff --git a/apple/Plugins/JSONImportPlugin/JSONLineBatches.swift b/apple/Plugins/JSONImportPlugin/JSONLineBatches.swift index b377a77c3c..925fe93657 100644 --- a/apple/Plugins/JSONImportPlugin/JSONLineBatches.swift +++ b/apple/Plugins/JSONImportPlugin/JSONLineBatches.swift @@ -12,7 +12,7 @@ import TableProPluginKit /// A batch ends after a set number of lines, not rows. The runner checks for a stop between /// batches, so a batch that waited for its rows read on through a run of lines holding none, up /// to a whole chunk of the file, before a stop was seen. -struct JSONLineBatches { +nonisolated struct JSONLineBatches { private var lines: JSONLineReader private let linesPerBatch: Int private let skipsUnreadableLines: Bool diff --git a/apple/Plugins/JSONImportPlugin/JSONLineReader.swift b/apple/Plugins/JSONImportPlugin/JSONLineReader.swift index 4195b4f3be..4fc1eb4a4b 100644 --- a/apple/Plugins/JSONImportPlugin/JSONLineReader.swift +++ b/apple/Plugins/JSONImportPlugin/JSONLineReader.swift @@ -15,7 +15,7 @@ import Foundation /// /// A stop is checked before every chunk rather than between lines, because one line can run to /// the end of a file that holds no newline at all. -struct JSONLineReader { +nonisolated struct JSONLineReader { static let defaultChunkSize = 1 << 20 private let handle: FileHandle diff --git a/apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift b/apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift index 6d937f2826..f6f6c754a2 100644 --- a/apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift +++ b/apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift @@ -6,7 +6,7 @@ import Foundation // The five encodings `JSONSerialization` reads, with or without a byte order mark. -enum JSONTextEncoding: Equatable { +nonisolated enum JSONTextEncoding: Equatable { case utf8 case utf16BigEndian case utf16LittleEndian @@ -43,7 +43,7 @@ enum JSONTextEncoding: Equatable { // Turns UTF-16 or UTF-32 text into UTF-8 a chunk at a time. A code unit, or a surrogate pair, that a // chunk cuts in two waits for the next chunk. -struct JSONTextDecoder { +nonisolated struct JSONTextDecoder { let encoding: JSONTextEncoding private var pending: [UInt8] = [] diff --git a/apple/Plugins/MQLExportPlugin/MQLCollectionDefinition.swift b/apple/Plugins/MQLExportPlugin/MQLCollectionDefinition.swift index 8f854a700e..2e32307583 100644 --- a/apple/Plugins/MQLExportPlugin/MQLCollectionDefinition.swift +++ b/apple/Plugins/MQLExportPlugin/MQLCollectionDefinition.swift @@ -20,7 +20,7 @@ import TableProJavaScriptText /// validator only as the first, where the driver writes it. A statement that fails to read is left /// out up to the next line, not one token at a time: stepping through it found a statement a name /// had spelled inside it and wrote that out as one of the driver's own. -enum MQLCollectionDefinition { +nonisolated enum MQLCollectionDefinition { static let skippedStatementComment = "// Skipped a statement that is not an index or a validator" static func script(fromDDL ddl: String, collection: String) -> String { diff --git a/apple/Plugins/MQLExportPlugin/MQLExportHelpers.swift b/apple/Plugins/MQLExportPlugin/MQLExportHelpers.swift index 09e8913e45..38e4c2eec7 100644 --- a/apple/Plugins/MQLExportPlugin/MQLExportHelpers.swift +++ b/apple/Plugins/MQLExportPlugin/MQLExportHelpers.swift @@ -8,7 +8,7 @@ import TableProJavaScriptText import TableProNumberFormatting import TableProPluginKit -enum MQLExportHelpers { +nonisolated enum MQLExportHelpers { /// Spelled by the export's own rules rather than `MongoCollectionAccessor`'s, because this /// plugin can be released for an app that shipped an older copy of that helper. static func collectionAccessor(for name: String) -> String { @@ -75,7 +75,7 @@ enum MQLExportHelpers { } // A command has a BSON size limit too; reserving space keeps its array keys and metadata outside the document budget. -struct MQLDocumentBatch { +nonisolated struct MQLDocumentBatch { static let documentByteLimit = 15 * 1_024 * 1_024 private let countLimit: Int diff --git a/apple/Plugins/MQLExportPlugin/MQLExportModels.swift b/apple/Plugins/MQLExportPlugin/MQLExportModels.swift index 1e7d39471c..d909fc94f5 100644 --- a/apple/Plugins/MQLExportPlugin/MQLExportModels.swift +++ b/apple/Plugins/MQLExportPlugin/MQLExportModels.swift @@ -5,7 +5,9 @@ import Foundation -public struct MQLExportOptions: Equatable, Codable { +nonisolated public struct MQLExportOptions: Equatable, Codable { + public static let batchSizeChoices = [100, 500, 1_000, 5_000] + public var batchSize: Int = 500 public init() {} diff --git a/apple/Plugins/MQLExportPlugin/MQLExportOptionsView.swift b/apple/Plugins/MQLExportPlugin/MQLExportOptionsView.swift index 64f3cbab17..9a56cdf95f 100644 --- a/apple/Plugins/MQLExportPlugin/MQLExportOptionsView.swift +++ b/apple/Plugins/MQLExportPlugin/MQLExportOptionsView.swift @@ -8,8 +8,6 @@ import SwiftUI struct MQLExportOptionsView: View { @ObservedObject var plugin: MQLExportPlugin - private static let batchSizeOptions = [100, 500, 1_000, 5_000] - var body: some View { VStack(alignment: .leading, spacing: 8) { Text("Exports data as mongosh-compatible scripts. Drop, Indexes, and Data options are configured per collection in the collection list.") @@ -27,7 +25,7 @@ struct MQLExportOptionsView: View { Spacer() Picker(String(localized: "Rows per batch", bundle: .main), selection: $plugin.settings.batchSize) { - ForEach(Self.batchSizeOptions, id: \.self) { size in + ForEach(MQLExportOptions.batchSizeChoices, id: \.self) { size in Text("\(size)") .tag(size) } @@ -40,3 +38,10 @@ struct MQLExportOptionsView: View { } } } + +extension MQLExportPlugin { + @MainActor + func settingsView() -> AnyView? { + AnyView(MQLExportOptionsView(plugin: self)) + } +} diff --git a/apple/Plugins/MQLExportPlugin/MQLExportPlugin.swift b/apple/Plugins/MQLExportPlugin/MQLExportPlugin.swift index eb15636a60..426c0fdc72 100644 --- a/apple/Plugins/MQLExportPlugin/MQLExportPlugin.swift +++ b/apple/Plugins/MQLExportPlugin/MQLExportPlugin.swift @@ -5,10 +5,9 @@ import Combine import Foundation -import SwiftUI import TableProPluginKit -final class MQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugin, @unchecked Sendable { +nonisolated final class MQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugin, @unchecked Sendable { static let pluginName = "MQL Export" static let pluginVersion = "1.0.0" static let pluginDescription = "Export data to MongoDB Query Language format" @@ -28,7 +27,9 @@ final class MQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi typealias Settings = MQLExportOptions static let settingsStorageId = "mql" - @Published var settings = MQLExportOptions() { + // A nonisolated class cannot hold an @Published property, so the change is announced by hand. + var settings = MQLExportOptions() { + willSet { objectWillChange.send() } didSet { saveSettings() } } @@ -47,11 +48,6 @@ final class MQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi optionValues.count <= 2 || optionValues[2] } - @MainActor - func settingsView() -> AnyView? { - AnyView(MQLExportOptionsView(plugin: self)) - } - func resetSettingsToDefaults() { settings = MQLExportOptions() } diff --git a/apple/Plugins/MQLExportPlugin/MQLScriptLexer.swift b/apple/Plugins/MQLExportPlugin/MQLScriptLexer.swift index 519bc5c2d6..928e9e108c 100644 --- a/apple/Plugins/MQLExportPlugin/MQLScriptLexer.swift +++ b/apple/Plugins/MQLExportPlugin/MQLScriptLexer.swift @@ -5,7 +5,7 @@ import Foundation -enum MQLScriptToken: Equatable { +nonisolated enum MQLScriptToken: Equatable { case identifier(String) case string(String) case number(String) @@ -14,7 +14,7 @@ enum MQLScriptToken: Equatable { case unreadable } -struct MQLScriptLexeme: Equatable { +nonisolated struct MQLScriptLexeme: Equatable { let token: MQLScriptToken let followsLineBreak: Bool let followsBlankLine: Bool @@ -26,7 +26,7 @@ struct MQLScriptLexeme: Equatable { /// Only what a collection definition needs is read. Anything else, a regular expression literal, /// a template string or an operator, comes back as `unreadable`, and a string with an escape it /// does not know or a raw line break in it does too. -struct MQLScriptLexer { +nonisolated struct MQLScriptLexer { private static let punctuators: Set = ["{", "}", "[", "]", "(", ")", ",", ":", ";", ".", "-"] private static let lineTerminators: Set = ["\n", "\r", "\u{2028}", "\u{2029}"] private static let spaces: Set = [" ", "\t", "\u{0B}", "\u{0C}", "\u{A0}", "\u{FEFF}"] diff --git a/apple/Plugins/MQLExportPlugin/MQLScriptValue.swift b/apple/Plugins/MQLExportPlugin/MQLScriptValue.swift index a31a372e03..3aeb9f32e5 100644 --- a/apple/Plugins/MQLExportPlugin/MQLScriptValue.swift +++ b/apple/Plugins/MQLExportPlugin/MQLScriptValue.swift @@ -10,7 +10,7 @@ import TableProPluginKit /// A literal value read out of shell text, which the export writes again by its own rules rather /// than copying the text it came from. -indirect enum MQLScriptValue: Equatable { +nonisolated indirect enum MQLScriptValue: Equatable { case string(String) case number(String) case boolean(Bool) @@ -150,7 +150,7 @@ indirect enum MQLScriptValue: Equatable { } } -struct MQLScriptMember: Equatable { +nonisolated struct MQLScriptMember: Equatable { let key: String let value: MQLScriptValue } @@ -159,7 +159,7 @@ struct MQLScriptMember: Equatable { /// /// Each read either returns what it matched and moves past it, or returns nil. A caller that needs /// to try one shape and then another works on a copy, which is cheap because the tokens are shared. -struct MQLScriptReader { +nonisolated struct MQLScriptReader { private let tokens: [MQLScriptToken] private(set) var index: Int diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLExportCatalog.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLExportCatalog.swift new file mode 100644 index 0000000000..5688fe5565 --- /dev/null +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLExportCatalog.swift @@ -0,0 +1,625 @@ +import Foundation +import os +import TableProMSSQLCore +import TableProPluginKit + +nonisolated struct MSSQLPrincipalState: Sendable { + var directory: MSSQLPrincipalDirectory? + var catalog: PluginPrivilegeCatalog? +} + +nonisolated struct MSSQLExportCatalogError: PluginDriverError { + let pluginErrorMessage: String + + init(_ refusal: MSSQLDefinitionRefusal) { + pluginErrorMessage = refusal.reason + } +} + +// The catalog reads both apps run for SQL Server, over whatever connection the caller hands in. +nonisolated struct MSSQLExportCatalog: Sendable { + typealias Run = @Sendable (String) async throws -> PluginQueryResult + + let run: Run + let currentSchema: String + let principalState: OSAllocatedUnfairLock + + init( + run: @escaping Run, + currentSchema: String, + principalState: OSAllocatedUnfairLock = OSAllocatedUnfairLock( + initialState: MSSQLPrincipalState() + ) + ) { + self.run = run + self.currentSchema = currentSchema + self.principalState = principalState + } + + static let tableDDLIncludesForeignKeys = true + + // An empty schema is the current one, never a schema named "". + static func effectiveSchema(_ schema: String?, current: String) -> String { + guard let schema, !schema.isEmpty else { return current } + return schema + } + + func effectiveSchema(_ schema: String?) -> String { + Self.effectiveSchema(schema, current: currentSchema) + } + + func textRows(_ query: String) async throws -> [[String?]] { + try await run(query).rows.map { $0.map(\.asText) } + } + + // MARK: - Routines + + func fetchRoutines(schema: String?) async throws -> [PluginRoutineInfo] { + let resolvedSchema = effectiveSchema(schema) + let result = try await run(MSSQLObjectQueries.routineList(schema: resolvedSchema)) + return result.rows.compactMap { row -> PluginRoutineInfo? in + guard let name = row[safe: 0]?.asText else { return nil } + let objectType = row[safe: 2]?.asText ?? "" + let isProcedure = MSSQLObjectQueries.routineKind(forObjectType: objectType) == "PROCEDURE" + var attributes: [PluginObjectAttribute] = [] + attributes.append(PluginObjectAttribute(label: "Object Type", value: objectType.trimmingCharacters(in: .whitespaces))) + let hasSQLSource = MSSQLObjectQueries.routineHasSQLSource(forObjectType: objectType) + if row[safe: 3]?.asText == "1" { + attributes.append(PluginObjectAttribute(label: "Encrypted", value: "YES")) + } else if hasSQLSource, row[safe: 4]?.asText == "1" { + attributes.append(PluginObjectAttribute( + label: String(localized: "Source"), + value: String(localized: "Not readable with your permissions") + )) + } + let parameters = row[safe: 5]?.asText ?? "" + return PluginRoutineInfo( + name: name, + kind: isProcedure ? .procedure : .function, + schema: row[safe: 1]?.asText ?? resolvedSchema, + returnType: isProcedure ? nil : row[safe: 6]?.asText, + language: MSSQLObjectQueries.routineLanguage(forObjectType: objectType), + argumentSignature: MSSQLRoutineSignature.listed(parameters: parameters, objectType: objectType), + identity: nil, + attributes: attributes + ) + } + } + + func fetchRoutineDDL(_ routine: PluginRoutineInfo) async throws -> String { + let resolvedSchema = effectiveSchema(routine.schema) + let query = MSSQLObjectQueries.moduleDefinition(schema: resolvedSchema, name: routine.name) + let result = try await run(query) + guard let row = result.rows.first else { + throw PluginObjectSourceError.notFound(routine.name) + } + guard let definition = row[safe: 0]?.asText, !definition.isEmpty else { + let isEncrypted = row[safe: 1]?.asText == "1" + let hasSQLSource = MSSQLObjectQueries.routineHasSQLSource(forObjectType: row[safe: 2]?.asText ?? "") + throw isEncrypted || !hasSQLSource + ? PluginObjectSourceError.unsupported(routine.name) + : PluginObjectSourceError.insufficientPrivilege(routine.name) + } + return definition + } + + // SQL Server routines do not overload, so the signature only says whether a function is a CLR aggregate. + func generateDropRoutineSQL(name: String, signature: String?, schema: String?, isFunction: Bool) -> String? { + MSSQLDropStatements.routine( + schema: effectiveSchema(schema), + name: name, + isFunction: isFunction, + signature: signature + ) + } + + // MARK: - Triggers + + func fetchAllTriggers(schema: String?) async throws -> [PluginTriggerInfo] { + try await triggerList(schema: effectiveSchema(schema), table: nil) + } + + func fetchTriggers(table: String, schema: String?) async throws -> [PluginTriggerInfo] { + try await triggerList(schema: effectiveSchema(schema), table: table) + } + + func fetchTriggerDDL(_ trigger: PluginTriggerInfo) async throws -> String { + if let definition = trigger.definition, !definition.isEmpty { return definition } + let listed = try await triggerList(schema: effectiveSchema(trigger.schema), table: trigger.table) + guard let definition = listed.first(where: { $0.name == trigger.name })?.definition, + !definition.isEmpty + else { + throw PluginObjectSourceError.insufficientPrivilege(trigger.name) + } + return definition + } + + func generateDropTriggerSQL(name: String, table: String, schema: String?) -> String? { + MSSQLDropStatements.trigger(schema: effectiveSchema(schema), name: name) + } + + // sys.trigger_events has one row per event, so a trigger's events are folded back together in the order listed. + private func triggerList(schema: String, table: String?) async throws -> [PluginTriggerInfo] { + let result = try await run(MSSQLObjectQueries.triggerList(schema: schema, table: table)) + var order: [String] = [] + var byKey: [String: (info: PluginTriggerInfo, events: [String])] = [:] + for row in result.rows { + guard let name = row[safe: 0]?.asText else { continue } + let owningTable = row[safe: 1]?.asText + let key = "\(owningTable ?? "")|\(name)" + let event = row[safe: 4]?.asText ?? "" + if byKey[key] == nil { + order.append(key) + let definition = row[safe: 6]?.asText ?? "" + byKey[key] = ( + info: PluginTriggerInfo( + name: name, + table: owningTable, + schema: row[safe: 2]?.asText ?? schema, + timing: row[safe: 3]?.asText == "1" ? "INSTEAD OF" : "AFTER", + event: "", + orientation: "STATEMENT", + statement: definition, + definition: definition, + enabled: row[safe: 5]?.asText != "1", + attributes: [] + ), + events: [] + ) + } + if !event.isEmpty { + byKey[key]?.events.append(event) + } + } + return order.compactMap { key in + guard let entry = byKey[key] else { return nil } + let info = entry.info + return PluginTriggerInfo( + name: info.name, + table: info.table, + schema: info.schema, + timing: info.timing, + event: entry.events.joined(separator: " OR "), + orientation: info.orientation, + statement: info.statement, + definition: info.definition, + enabled: info.enabled, + attributes: info.attributes + ) + } + } + + // MARK: - Tables and Views + + // A view answers with its own definition: a column over an identity column reports `IsIdentity`. + func fetchTableDDL(table: String, schema: String?) async throws -> String { + let resolvedSchema = effectiveSchema(schema) + let module = try await storedModule(schema: resolvedSchema, name: table) + if MSSQLObjectQueries.isView(objectType: module?[safe: 2].flatMap { $0 } ?? "") { + return try MSSQLObjectQueries.viewSource(row: module, schema: resolvedSchema, name: table) + } + + let columns = try await columnRows(table: table, schema: schema) + let keyConstraints = try await textRows(MSSQLSchemaQueries.keyConstraints(schema: resolvedSchema, table: table)) + .compactMap(MSSQLSchemaQueries.parseKeyConstraintRow) + let checkConstraints = try await textRows(MSSQLSchemaQueries.checkConstraints(schema: resolvedSchema, table: table)) + .compactMap(MSSQLSchemaQueries.parseCheckConstraintRow) + let foreignKeys = try await foreignKeyRows(table: table, schema: schema) + let collation = await databaseCollation(forTable: table) + do { + return try MSSQLSchemaQueries.createTableStatement( + schema: resolvedSchema, + table: table, + columns: columns, + keyConstraints: keyConstraints, + checkConstraints: checkConstraints, + foreignKeys: foreignKeys, + databaseCollation: collation + ) + } catch let refusal as MSSQLDefinitionRefusal { + throw MSSQLExportCatalogError(refusal) + } + } + + func fetchViewDefinition(view: String, schema: String?) async throws -> String { + let resolvedSchema = effectiveSchema(schema) + let module = try await storedModule(schema: resolvedSchema, name: view) + return try MSSQLObjectQueries.viewSource(row: module, schema: resolvedSchema, name: view) + } + + // Key and UNIQUE indexes come with `fetchTableDDL`; this also re-enables checks the source enforced unchecked. + func fetchIndexDDL(table: String, schema: String?) async throws -> [String] { + let resolvedSchema = effectiveSchema(schema) + let rows = try await textRows(MSSQLSchemaQueries.indexDefinitions(schema: resolvedSchema, table: table)) + let checks = try await textRows(MSSQLSchemaQueries.checkConstraints(schema: resolvedSchema, table: table)) + .compactMap(MSSQLSchemaQueries.parseCheckConstraintRow) + do { + return try MSSQLSchemaQueries.indexStatements(rows: rows, schema: resolvedSchema, table: table) + + MSSQLSchemaQueries.checkConstraintReenableStatements( + schema: resolvedSchema, table: table, checkConstraints: checks + ) + } catch let refusal as MSSQLDefinitionRefusal { + throw MSSQLExportCatalogError(refusal) + } + } + + func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { + let result = try await run(MSSQLSchemaQueries.approximateRowCount(schema: effectiveSchema(schema), table: table)) + if let row = result.rows.first, let cell = row.first, let str = cell.asText { + return Int(str) + } + return nil + } + + // Unread, every column states its collation, which keeps it rather than changing it. + private func databaseCollation(forTable table: String) async -> String? { + guard let result = try? await run(MSSQLSchemaQueries.databaseCollation(forTable: table)) else { + return nil + } + return result.rows.first?[safe: 0]?.asText + } + + private func storedModule(schema: String, name: String) async throws -> [String?]? { + let result = try await run(MSSQLObjectQueries.moduleDefinition(schema: schema, name: name)) + return result.rows.first.map { $0.map(\.asText) } + } + + // MARK: - Columns + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + try await columnRows(table: table, schema: schema).map(\.pluginColumnInfo) + } + + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { + let result = try await run(MSSQLSchemaQueries.allColumns(schema: effectiveSchema(schema))) + var columnsByTable: [String: [PluginColumnInfo]] = [:] + for row in result.rows { + let cells = row.map(\.asText) + guard let tableName = cells[safe: MSSQLSchemaQueries.allColumnsTableNameIndex] ?? nil, + let column = MSSQLSchemaQueries.parseColumnRow(cells) else { continue } + columnsByTable[tableName, default: []].append(column.pluginColumnInfo) + } + return columnsByTable + } + + private func columnRows(table: String, schema: String?) async throws -> [MSSQLColumnRow] { + let sql = MSSQLSchemaQueries.namesTemporaryTable(table) + ? MSSQLSchemaQueries.temporaryColumns(table: table) + : MSSQLSchemaQueries.columns(schema: effectiveSchema(schema), table: table) + let result = try await run(sql) + return result.rows.compactMap { row in + MSSQLSchemaQueries.parseColumnRow(row.map(\.asText)) + } + } + + // MARK: - Foreign Keys + + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + try await foreignKeyRows(table: table, schema: schema).map(Self.foreignKey) + } + + func fetchAllForeignKeys(schema: String?) async throws -> [String: [PluginForeignKeyInfo]] { + let result = try await run(MSSQLSchemaQueries.foreignKeys(schema: effectiveSchema(schema), table: nil)) + var fksByTable: [String: [PluginForeignKeyInfo]] = [:] + for row in result.rows { + guard let parsed = MSSQLSchemaQueries.parseForeignKeyRow(row.map { $0.asText }), + let table = parsed.table else { continue } + fksByTable[table, default: []].append(Self.foreignKey(parsed)) + } + return fksByTable + } + + private func foreignKeyRows(table: String, schema: String?) async throws -> [MSSQLForeignKeyRow] { + let sql = MSSQLSchemaQueries.foreignKeys(schema: effectiveSchema(schema), table: table) + let result = try await run(sql) + return result.rows.compactMap { row in + MSSQLSchemaQueries.parseForeignKeyRow(row.map { $0.asText }) + } + } + + private static func foreignKey(_ row: MSSQLForeignKeyRow) -> PluginForeignKeyInfo { + PluginForeignKeyInfo( + name: row.constraintName, + column: row.columnName, + referencedTable: row.referencedTable, + referencedColumn: row.referencedColumn, + referencedSchema: row.referencedSchema, + onDelete: row.onDelete, + onUpdate: row.onUpdate + ) + } + + // MARK: - User-Defined Types + + // The listing carries no definitions: a table type's statement costs two more reads per type. + func fetchUserDefinedTypes(schema: String?) async throws -> [PluginUserDefinedTypeInfo] { + let resolvedSchema = effectiveSchema(schema) + let result = try await run(MSSQLTypeQueries.userDefinedTypeList(schema: resolvedSchema)) + return result.rows.compactMap { row in + listedType(from: row, schema: resolvedSchema) + } + } + + // Addressed by `user_type_id`, so a type renamed since the listing still resolves. + func fetchUserDefinedType(_ type: PluginUserDefinedTypeInfo) async throws -> PluginUserDefinedTypeInfo { + let resolvedSchema = effectiveSchema(type.schema) + let result = try await run(MSSQLTypeQueries.userDefinedTypeList(schema: resolvedSchema)) + let listed = result.rows.compactMap { listedType(from: $0, schema: resolvedSchema) } + let match = type.identity.flatMap { identity in listed.first { $0.identity == identity } } + ?? listed.first { $0.name == type.name } + guard let match else { throw PluginObjectSourceError.notFound(type.name) } + + do { + switch match.kind { + case .tableType: + return try await tableType(match, schema: resolvedSchema) + case .clrType: + return match.withDefinition(MSSQLTypeDefinition.clrStatement( + schema: resolvedSchema, + name: match.name, + assembly: match.attributes.first { $0.label == "Assembly" }?.value, + assemblyClass: match.attributes.first { $0.label == "Class" }?.value + )) + default: + return try match.withDefinition(MSSQLTypeDefinition.aliasStatement( + schema: resolvedSchema, + name: match.name, + baseType: match.baseType, + isNullable: match.attributes.first { $0.label == "Nullable" }?.value != "NO" + )) + } + } catch let refusal as MSSQLDefinitionRefusal { + throw MSSQLExportCatalogError(refusal) + } + } + + private func tableType( + _ type: PluginUserDefinedTypeInfo, + schema: String + ) async throws -> PluginUserDefinedTypeInfo { + let columnRows = try await textRows(MSSQLTypeQueries.tableTypeColumns(schema: schema, name: type.name)) + let indexRows = try await textRows(MSSQLTypeQueries.tableTypeIndexes(schema: schema, name: type.name)) + let checkRows = try await textRows(MSSQLTypeQueries.tableTypeCheckConstraints(schema: schema, name: type.name)) + let collation = try? await run(MSSQLTypeQueries.databaseCollation).rows.first?[safe: 0]?.asText + + let columns = MSSQLTypeDefinition.columns(fromRows: columnRows) + let definition = try MSSQLTypeDefinition.tableStatement( + schema: schema, + name: type.name, + columns: columns, + indexes: MSSQLTypeDefinition.indexes(fromRows: indexRows), + checkConstraints: MSSQLTypeDefinition.checkConstraints(fromRows: checkRows), + isMemoryOptimized: type.attributes.contains { $0.label == "Memory Optimized" && $0.value == "YES" }, + databaseCollation: collation ?? nil + ) + + return PluginUserDefinedTypeInfo( + name: type.name, + kind: .tableType, + schema: schema, + identity: type.identity, + fields: columns.map { + PluginUserDefinedTypeField(name: $0.name, type: $0.type ?? "", collation: $0.collation) + }, + columnTypeSpelling: type.columnTypeSpelling, + definition: definition, + attributes: type.attributes + ) + } + + private func listedType(from row: [PluginCellValue], schema: String) -> PluginUserDefinedTypeInfo? { + guard let name = row[safe: 0]?.asText else { return nil } + let kindCode = row[safe: 3]?.asText ?? "" + let kind: PluginUserDefinedTypeKind = switch MSSQLTypeQueries.Kind(rawValue: kindCode) { + case .table: .tableType + case .clr: .clrType + default: .aliasType + } + + var attributes: [PluginObjectAttribute] = [] + if kind == .aliasType { + attributes.append(PluginObjectAttribute( + label: "Nullable", + value: row[safe: 5]?.asText == "1" ? "YES" : "NO" + )) + } + if let collation = row[safe: 6]?.asText, !collation.isEmpty { + attributes.append(PluginObjectAttribute(label: "Collation", value: collation)) + } + if let assembly = row[safe: 7]?.asText, !assembly.isEmpty { + attributes.append(PluginObjectAttribute(label: "Assembly", value: assembly)) + } + if let assemblyClass = row[safe: 8]?.asText, !assemblyClass.isEmpty { + attributes.append(PluginObjectAttribute(label: "Class", value: assemblyClass)) + } + if kind == .tableType, row[safe: 9]?.asText == "1" { + attributes.append(PluginObjectAttribute(label: "Memory Optimized", value: "YES")) + } + + return PluginUserDefinedTypeInfo( + name: name, + kind: kind, + schema: row[safe: 1]?.asText ?? schema, + identity: row[safe: 2]?.asText, + baseType: row[safe: 4]?.asText, + columnTypeSpelling: Self.columnTypeSpelling(kind: kind, schema: row[safe: 1]?.asText ?? schema, name: name), + attributes: attributes + ) + } + + // A table type is never a column type, so the column picker gets no spelling for it. + private static func columnTypeSpelling(kind: PluginUserDefinedTypeKind, schema: String, name: String) -> String? { + guard kind != .tableType else { return nil } + return "\(MSSQLTypeDefinition.bracketed(schema)).\(MSSQLTypeDefinition.bracketed(name))" + } + + // MARK: - Principals + + func fetchPrincipals() async throws -> [PluginPrincipalInfo] { + try await loadPrincipalDirectory().entries.map { entry in + PluginPrincipalInfo( + ref: PluginPrincipalRef(name: entry.name), + isRole: entry.kind.isRole, + canLogin: !entry.kind.isRole && entry.isEnabled, + memberOf: entry.memberOf + ) + } + } + + func fetchGrants(for principal: PluginPrincipalRef) async throws -> [PluginGrantInfo] { + let directory = try await principalDirectory() + var grants: [MSSQLGrant] = [] + switch directory.kind(of: principal.name) { + case let .login(user): + grants += try await serverGrants(principal.name) + if let user { + grants += try await databaseGrants(user, database: directory.database) + } + case .serverRole: + grants += try await serverGrants(principal.name) + case .databaseRole, .databaseUser: + grants += try await databaseGrants(principal.name, database: directory.database) + case nil: + return [] + } + return grants.map { Self.pluginGrant($0, database: directory.database) } + } + + func generateGrantSQL(changeSet: PluginPrincipalChangeSet) -> [String]? { + guard let grants = mssqlGrants(changeSet.grantsToAdd) else { return nil } + return principalStatements.grantStatements(grants, to: changeSet.principal.name) + } + + func generateRevokeSQL(changeSet: PluginPrincipalChangeSet) -> [String]? { + guard let grants = mssqlGrants(changeSet.grantsToRemove) else { return nil } + return principalStatements.revokeStatements(grants, from: changeSet.principal.name) + } + + var principalStatements: MSSQLPrincipalStatements { + MSSQLPrincipalStatements(directory: principalState.withLock { $0.directory }) + } + + func principalDirectory() async throws -> MSSQLPrincipalDirectory { + if let cached = principalState.withLock({ $0.directory }) { + return cached + } + return try await loadPrincipalDirectory() + } + + func names(_ query: String) async throws -> [String] { + MSSQLPrincipalQueries.parseNames(try await textRows(query)) + } + + private func loadPrincipalDirectory() async throws -> MSSQLPrincipalDirectory { + let database = try await names(MSSQLPrincipalQueries.currentDatabase).first ?? "" + let directory = MSSQLPrincipalDirectory( + database: database, + logins: MSSQLPrincipalQueries.parseLogins(try await textRows(MSSQLPrincipalQueries.logins(database: database))), + serverRoles: MSSQLPrincipalQueries.parseNames(try await textRows(MSSQLPrincipalQueries.serverRoles)), + databasePrincipals: MSSQLPrincipalQueries.parseDatabasePrincipals( + try await textRows(MSSQLPrincipalQueries.databasePrincipals(database: database)) + ), + serverMemberships: MSSQLPrincipalQueries.parseMemberships( + try await textRows(MSSQLPrincipalQueries.serverRoleMembers) + ), + databaseMemberships: MSSQLPrincipalQueries.parseMemberships( + try await textRows(MSSQLPrincipalQueries.databaseRoleMembers(database: database)) + ) + ) + principalState.withLock { $0.directory = directory } + return directory + } + + private func serverGrants(_ name: String) async throws -> [MSSQLGrant] { + MSSQLPrincipalQueries.parseServerPermissions( + try await textRows(MSSQLPrincipalQueries.serverPermissions(grantee: name)) + ) + } + + private func databaseGrants(_ name: String, database: String) async throws -> [MSSQLGrant] { + MSSQLPrincipalQueries.parseDatabasePermissions( + try await textRows(MSSQLPrincipalQueries.databasePermissions(grantee: name, database: database)) + ) + } + + // Statements run in the database the list was read from, so a scope naming another one is refused. + private func mssqlGrants(_ grants: [PluginGrantInfo]) -> [MSSQLGrant]? { + let database = principalState.withLock { $0.directory?.database } + var converted: [MSSQLGrant] = [] + for grant in grants { + if let database, let scopeDatabase = grant.scope.databaseName, scopeDatabase != database { + return nil + } + let securable: MSSQLSecurable + switch grant.scope { + case .server: + securable = .server + case .database: + securable = .database + case let .schema(_, schema): + securable = .schema(schema) + case let .table(_, schema, table), let .column(_, schema, table, _): + securable = .object(schema: effectiveSchema(schema), name: table) + @unknown default: + return nil + } + converted.append(MSSQLGrant( + permission: grant.privilege, + securable: securable, + column: grant.scope.columnName, + withGrantOption: grant.isGrantable + )) + } + return converted + } + + private static func pluginGrant(_ grant: MSSQLGrant, database: String) -> PluginGrantInfo { + let scope: PluginPrivilegeScope + switch grant.securable { + case .server: + scope = .server + case .database: + scope = .database(database) + case let .schema(schema): + scope = .schema(database: database, schema: schema) + case let .object(schema, name): + scope = grant.column.map { + .column(database: database, schema: schema, table: name, column: $0) + } ?? .table(database: database, schema: schema, table: name) + } + return PluginGrantInfo(privilege: grant.permission, scope: scope, isGrantable: grant.withGrantOption) + } +} + +nonisolated extension MSSQLColumnRow { + var pluginColumnInfo: PluginColumnInfo { + PluginColumnInfo( + name: name, + dataType: displayType, + isNullable: isNullable, + isPrimaryKey: isPrimaryKey, + defaultValue: defaultValue, + extra: isIdentity ? "IDENTITY" : nil, + collation: collation, + identityKind: isIdentity ? .always : nil, + isGenerated: isGenerated + ) + } +} + +nonisolated private extension PluginUserDefinedTypeInfo { + func withDefinition(_ definition: String) -> PluginUserDefinedTypeInfo { + PluginUserDefinedTypeInfo( + name: name, + kind: kind, + schema: schema, + identity: identity, + enumLabels: enumLabels, + fields: fields, + baseType: baseType, + columnTypeSpelling: columnTypeSpelling, + definition: definition, + attributes: attributes + ) + } +} diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLObjectQueries.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLObjectQueries.swift index a2e94ba002..1e3e7f63b7 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLObjectQueries.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLObjectQueries.swift @@ -9,7 +9,7 @@ import Foundation import TableProMSSQLCore import TableProPluginKit -public enum MSSQLObjectQueries { +nonisolated public enum MSSQLObjectQueries { public static func escapeLiteral(_ value: String) -> String { MSSQLStringLiteral.escaped(value) } diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift index 16ed36aa7e..30ea7331d0 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift @@ -790,18 +790,7 @@ final class MSSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { - let objectLiteral = MSSQLStringLiteral.quoted( - MSSQLSchemaQueries.bracketed(schema: effectiveSchema(schema), table: table)) - let sql = """ - SELECT SUM(p.rows) - FROM sys.partitions p - WHERE p.object_id = OBJECT_ID(\(objectLiteral)) AND p.index_id IN (0, 1) - """ - let result = try await execute(query: sql) - if let row = result.rows.first, let cell = row.first, let str = cell.asText { - return Int(str) - } - return nil + try await exportCatalog.fetchApproximateRowCount(table: table, schema: schema) } // MARK: - Query Building @@ -959,9 +948,16 @@ final class MSSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { MSSQLStringLiteral.escaped(value) } + var exportCatalog: MSSQLExportCatalog { + MSSQLExportCatalog( + run: { [self] query in try await execute(query: query) }, + currentSchema: currentSchemaName, + principalState: principalState + ) + } + func effectiveSchema(_ schema: String?) -> String { - guard let schema, !schema.isEmpty else { return currentSchemaName } - return schema + MSSQLExportCatalog.effectiveSchema(schema, current: currentSchemaName) } func effectiveSchemaQuoted(_ schema: String?) -> String { @@ -997,12 +993,6 @@ enum MSSQLPluginError: Error { case queryFailed(String) } -extension MSSQLPluginError { - init(_ refusal: MSSQLDefinitionRefusal) { - self = .queryFailed(refusal.reason) - } -} - extension MSSQLPluginError: PluginDriverError { var pluginErrorMessage: String { switch self { diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Principals.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Principals.swift index 9fd7ec9ae6..ef81ed1785 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Principals.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Principals.swift @@ -8,11 +8,6 @@ import os import TableProMSSQLCore import TableProPluginKit -struct MSSQLPrincipalState: Sendable { - var directory: MSSQLPrincipalDirectory? - var catalog: PluginPrivilegeCatalog? -} - extension MSSQLPluginDriver: PluginPrincipalManagement { var supportsRoleMembership: Bool { true } var restrictsGrantBrowsingToCurrentDatabase: Bool { true } @@ -28,14 +23,7 @@ extension MSSQLPluginDriver: PluginPrincipalManagement { } func fetchPrincipals() async throws -> [PluginPrincipalInfo] { - try await loadPrincipalDirectory().entries.map { entry in - PluginPrincipalInfo( - ref: PluginPrincipalRef(name: entry.name), - isRole: entry.kind.isRole, - canLogin: !entry.kind.isRole && entry.isEnabled, - memberOf: entry.memberOf - ) - } + try await exportCatalog.fetchPrincipals() } func fetchPrivilegeCatalog() async throws -> PluginPrivilegeCatalog { @@ -61,40 +49,26 @@ extension MSSQLPluginDriver: PluginPrincipalManagement { } func fetchGrants(for principal: PluginPrincipalRef) async throws -> [PluginGrantInfo] { - let directory = try await principalDirectory() - var grants: [MSSQLGrant] = [] - switch directory.kind(of: principal.name) { - case let .login(user): - grants += try await serverGrants(principal.name) - if let user { - grants += try await databaseGrants(user, database: directory.database) - } - case .serverRole: - grants += try await serverGrants(principal.name) - case .databaseRole, .databaseUser: - grants += try await databaseGrants(principal.name, database: directory.database) - case nil: - return [] - } - return grants.map { Self.pluginGrant($0, database: directory.database) } + try await exportCatalog.fetchGrants(for: principal) } func fetchGrantableChildren(of scope: PluginPrivilegeScope) async throws -> [PluginPrivilegeScope] { - let database = try await principalDirectory().database + let catalog = exportCatalog + let database = try await catalog.principalDirectory().database switch scope { case let .database(name): guard name == database else { return [] } - return try await names(MSSQLPrincipalQueries.schemas(database: name)).map { + return try await catalog.names(MSSQLPrincipalQueries.schemas(database: name)).map { .schema(database: name, schema: $0) } case let .schema(name, schema): guard name == database else { return [] } - return try await names(MSSQLPrincipalQueries.objects(schema: schema, database: name)).map { + return try await catalog.names(MSSQLPrincipalQueries.objects(schema: schema, database: name)).map { .table(database: name, schema: schema, table: $0) } case let .table(name, schema, table): guard name == database, let schema else { return [] } - return try await names(MSSQLPrincipalQueries.columns(schema: schema, object: table, database: name)).map { + return try await catalog.names(MSSQLPrincipalQueries.columns(schema: schema, object: table, database: name)).map { .column(database: name, schema: schema, table: table, column: $0) } case .server, .column: @@ -105,7 +79,7 @@ extension MSSQLPluginDriver: PluginPrincipalManagement { } func searchGrantableScopes(matching query: String, limit: Int) async throws -> [PluginPrivilegeScope] { - let database = try await principalDirectory().database + let database = try await exportCatalog.principalDirectory().database let result = try await execute( query: MSSQLPrincipalQueries.searchObjects(matching: query, limit: limit, database: database) ) @@ -116,7 +90,7 @@ extension MSSQLPluginDriver: PluginPrincipalManagement { } func currentPrincipalRef() async throws -> PluginPrincipalRef? { - try await names(MSSQLPrincipalQueries.currentLogin).first.map { PluginPrincipalRef(name: $0) } + try await exportCatalog.names(MSSQLPrincipalQueries.currentLogin).first.map { PluginPrincipalRef(name: $0) } } // MARK: - Statements @@ -150,118 +124,17 @@ extension MSSQLPluginDriver: PluginPrincipalManagement { } func generateGrantSQL(changeSet: PluginPrincipalChangeSet) -> [String]? { - guard let grants = mssqlGrants(changeSet.grantsToAdd) else { return nil } - return principalStatements.grantStatements(grants, to: changeSet.principal.name) + exportCatalog.generateGrantSQL(changeSet: changeSet) } func generateRevokeSQL(changeSet: PluginPrincipalChangeSet) -> [String]? { - guard let grants = mssqlGrants(changeSet.grantsToRemove) else { return nil } - return principalStatements.revokeStatements(grants, from: changeSet.principal.name) + exportCatalog.generateRevokeSQL(changeSet: changeSet) } // MARK: - Helpers private var principalStatements: MSSQLPrincipalStatements { - MSSQLPrincipalStatements(directory: principalState.withLock { $0.directory }) - } - - private func principalDirectory() async throws -> MSSQLPrincipalDirectory { - if let cached = principalState.withLock({ $0.directory }) { - return cached - } - return try await loadPrincipalDirectory() - } - - private func loadPrincipalDirectory() async throws -> MSSQLPrincipalDirectory { - let database = try await currentPrincipalDatabase() - let directory = MSSQLPrincipalDirectory( - database: database, - logins: MSSQLPrincipalQueries.parseLogins(try await textRows(MSSQLPrincipalQueries.logins(database: database))), - serverRoles: MSSQLPrincipalQueries.parseNames(try await textRows(MSSQLPrincipalQueries.serverRoles)), - databasePrincipals: MSSQLPrincipalQueries.parseDatabasePrincipals( - try await textRows(MSSQLPrincipalQueries.databasePrincipals(database: database)) - ), - serverMemberships: MSSQLPrincipalQueries.parseMemberships( - try await textRows(MSSQLPrincipalQueries.serverRoleMembers) - ), - databaseMemberships: MSSQLPrincipalQueries.parseMemberships( - try await textRows(MSSQLPrincipalQueries.databaseRoleMembers(database: database)) - ) - ) - principalState.withLock { $0.directory = directory } - return directory - } - - private func serverGrants(_ name: String) async throws -> [MSSQLGrant] { - MSSQLPrincipalQueries.parseServerPermissions( - try await textRows(MSSQLPrincipalQueries.serverPermissions(grantee: name)) - ) - } - - private func databaseGrants(_ name: String, database: String) async throws -> [MSSQLGrant] { - MSSQLPrincipalQueries.parseDatabasePermissions( - try await textRows(MSSQLPrincipalQueries.databasePermissions(grantee: name, database: database)) - ) - } - - private func currentPrincipalDatabase() async throws -> String { - try await names(MSSQLPrincipalQueries.currentDatabase).first ?? "" - } - - private func textRows(_ query: String) async throws -> [[String?]] { - try await execute(query: query).rows.map { $0.map(\.asText) } - } - - private func names(_ query: String) async throws -> [String] { - MSSQLPrincipalQueries.parseNames(try await textRows(query)) - } - - // Statements run in the database the list was read from, so a scope naming another one is refused. - private func mssqlGrants(_ grants: [PluginGrantInfo]) -> [MSSQLGrant]? { - let database = principalState.withLock { $0.directory?.database } - var converted: [MSSQLGrant] = [] - for grant in grants { - if let database, let scopeDatabase = grant.scope.databaseName, scopeDatabase != database { - return nil - } - let securable: MSSQLSecurable - switch grant.scope { - case .server: - securable = .server - case .database: - securable = .database - case let .schema(_, schema): - securable = .schema(schema) - case let .table(_, schema, table), let .column(_, schema, table, _): - securable = .object(schema: effectiveSchema(schema), name: table) - @unknown default: - return nil - } - converted.append(MSSQLGrant( - permission: grant.privilege, - securable: securable, - column: grant.scope.columnName, - withGrantOption: grant.isGrantable - )) - } - return converted - } - - private static func pluginGrant(_ grant: MSSQLGrant, database: String) -> PluginGrantInfo { - let scope: PluginPrivilegeScope - switch grant.securable { - case .server: - scope = .server - case .database: - scope = .database(database) - case let .schema(schema): - scope = .schema(database: database, schema: schema) - case let .object(schema, name): - scope = grant.column.map { - .column(database: database, schema: schema, table: name, column: $0) - } ?? .table(database: database, schema: schema, table: name) - } - return PluginGrantInfo(privilege: grant.permission, scope: scope, isGrantable: grant.withGrantOption) + exportCatalog.principalStatements } private static func descriptors(_ names: [String]) -> [PluginPrivilegeDescriptor] { diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Routines.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Routines.swift index d55203b809..1a837448d0 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Routines.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Routines.swift @@ -8,120 +8,24 @@ import TableProPluginKit extension MSSQLPluginDriver { func fetchRoutines(schema: String?) async throws -> [PluginRoutineInfo] { - let resolvedSchema = effectiveSchema(schema) - let result = try await execute(query: MSSQLObjectQueries.routineList(schema: resolvedSchema)) - return result.rows.compactMap { row -> PluginRoutineInfo? in - guard let name = row[safe: 0]?.asText else { return nil } - let objectType = row[safe: 2]?.asText ?? "" - let isProcedure = MSSQLObjectQueries.routineKind(forObjectType: objectType) == "PROCEDURE" - var attributes: [PluginObjectAttribute] = [] - attributes.append(PluginObjectAttribute(label: "Object Type", value: objectType.trimmingCharacters(in: .whitespaces))) - let hasSQLSource = MSSQLObjectQueries.routineHasSQLSource(forObjectType: objectType) - if row[safe: 3]?.asText == "1" { - attributes.append(PluginObjectAttribute(label: "Encrypted", value: "YES")) - } else if hasSQLSource, row[safe: 4]?.asText == "1" { - attributes.append(PluginObjectAttribute( - label: String(localized: "Source"), - value: String(localized: "Not readable with your permissions") - )) - } - let parameters = row[safe: 5]?.asText ?? "" - return PluginRoutineInfo( - name: name, - kind: isProcedure ? .procedure : .function, - schema: row[safe: 1]?.asText ?? resolvedSchema, - returnType: isProcedure ? nil : row[safe: 6]?.asText, - language: MSSQLObjectQueries.routineLanguage(forObjectType: objectType), - argumentSignature: "(\(parameters))", - identity: nil, - attributes: attributes - ) - } + try await exportCatalog.fetchRoutines(schema: schema) } func fetchRoutineDDL(_ routine: PluginRoutineInfo) async throws -> String { - let resolvedSchema = effectiveSchema(routine.schema) - let query = MSSQLObjectQueries.moduleDefinition(schema: resolvedSchema, name: routine.name) - let result = try await execute(query: query) - guard let row = result.rows.first else { - throw PluginObjectSourceError.notFound(routine.name) - } - guard let definition = row[safe: 0]?.asText, !definition.isEmpty else { - let isEncrypted = row[safe: 1]?.asText == "1" - let hasSQLSource = MSSQLObjectQueries.routineHasSQLSource(forObjectType: row[safe: 2]?.asText ?? "") - throw isEncrypted || !hasSQLSource - ? PluginObjectSourceError.unsupported(routine.name) - : PluginObjectSourceError.insufficientPrivilege(routine.name) - } - return definition + try await exportCatalog.fetchRoutineDDL(routine) + } + + func generateDropRoutineSQL(name: String, signature: String?, schema: String?, isFunction: Bool) -> String? { + exportCatalog.generateDropRoutineSQL(name: name, signature: signature, schema: schema, isFunction: isFunction) } var providesBulkTriggerFetch: Bool { true } func fetchAllTriggers(schema: String?) async throws -> [PluginTriggerInfo] { - try await triggerList(schema: effectiveSchema(schema), table: nil) + try await exportCatalog.fetchAllTriggers(schema: schema) } func fetchTriggerDDL(_ trigger: PluginTriggerInfo) async throws -> String { - if let definition = trigger.definition, !definition.isEmpty { return definition } - let listed = try await triggerList(schema: effectiveSchema(trigger.schema), table: trigger.table) - guard let definition = listed.first(where: { $0.name == trigger.name })?.definition, - !definition.isEmpty - else { - throw PluginObjectSourceError.insufficientPrivilege(trigger.name) - } - return definition - } - - /// sys.trigger_events has one row per event, so a trigger on INSERT and UPDATE arrives twice - /// and its events are folded back together here in the order the server listed them. - func triggerList(schema: String, table: String?) async throws -> [PluginTriggerInfo] { - let result = try await execute(query: MSSQLObjectQueries.triggerList(schema: schema, table: table)) - var order: [String] = [] - var byKey: [String: (info: PluginTriggerInfo, events: [String])] = [:] - for row in result.rows { - guard let name = row[safe: 0]?.asText else { continue } - let owningTable = row[safe: 1]?.asText - let key = "\(owningTable ?? "")|\(name)" - let event = row[safe: 4]?.asText ?? "" - if byKey[key] == nil { - order.append(key) - let definition = row[safe: 6]?.asText ?? "" - byKey[key] = ( - info: PluginTriggerInfo( - name: name, - table: owningTable, - schema: row[safe: 2]?.asText ?? schema, - timing: row[safe: 3]?.asText == "1" ? "INSTEAD OF" : "AFTER", - event: "", - orientation: "STATEMENT", - statement: definition, - definition: definition, - enabled: row[safe: 5]?.asText != "1", - attributes: [] - ), - events: [] - ) - } - if !event.isEmpty { - byKey[key]?.events.append(event) - } - } - return order.compactMap { key in - guard let entry = byKey[key] else { return nil } - let info = entry.info - return PluginTriggerInfo( - name: info.name, - table: info.table, - schema: info.schema, - timing: info.timing, - event: entry.events.joined(separator: " OR "), - orientation: info.orientation, - statement: info.statement, - definition: info.definition, - enabled: info.enabled, - attributes: info.attributes - ) - } + try await exportCatalog.fetchTriggerDDL(trigger) } } diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift index 98e731ebcd..db11acf359 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift @@ -49,17 +49,7 @@ extension MSSQLPluginDriver { } func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { - try await columnRows(table: table, schema: schema).map(\.pluginColumnInfo) - } - - private func columnRows(table: String, schema: String?) async throws -> [MSSQLColumnRow] { - let sql = MSSQLSchemaQueries.namesTemporaryTable(table) - ? MSSQLSchemaQueries.temporaryColumns(table: table) - : MSSQLSchemaQueries.columns(schema: effectiveSchema(schema), table: table) - let result = try await execute(query: sql) - return result.rows.compactMap { row in - MSSQLSchemaQueries.parseColumnRow(row.map(\.asText)) - } + try await exportCatalog.fetchColumns(table: table, schema: schema) } func fetchViewRowEditing(view: String, schema: String?) async throws -> PluginViewRowEditing? { @@ -127,7 +117,7 @@ extension MSSQLPluginDriver { } private func catalogRows(_ query: String) async throws -> [[String?]] { - try await execute(query: query).rows.map { $0.map(\.asText) } + try await exportCatalog.textRows(query) } func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { @@ -175,50 +165,18 @@ extension MSSQLPluginDriver { }.sorted { $0.name < $1.name } } - // The primary key and UNIQUE constraints are declared in `fetchTableDDL`, which builds their indexes. What runs - // after the rows also turns back on the checks the source enforced without having checked its rows. func fetchIndexDDL(table: String, schema: String?) async throws -> [String] { - let resolvedSchema = effectiveSchema(schema) - let rows = try await catalogRows(MSSQLSchemaQueries.indexDefinitions(schema: resolvedSchema, table: table)) - let checks = try await catalogRows(MSSQLSchemaQueries.checkConstraints(schema: resolvedSchema, table: table)) - .compactMap(MSSQLSchemaQueries.parseCheckConstraintRow) - do { - return try MSSQLSchemaQueries.indexStatements(rows: rows, schema: resolvedSchema, table: table) - + MSSQLSchemaQueries.checkConstraintReenableStatements( - schema: resolvedSchema, table: table, checkConstraints: checks - ) - } catch let refusal as MSSQLDefinitionRefusal { - throw MSSQLPluginError(refusal) - } + try await exportCatalog.fetchIndexDDL(table: table, schema: schema) } func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { - try await foreignKeyRows(table: table, schema: schema).map(Self.foreignKey) - } - - private func foreignKeyRows(table: String, schema: String?) async throws -> [MSSQLForeignKeyRow] { - let sql = MSSQLSchemaQueries.foreignKeys(schema: effectiveSchema(schema), table: table) - let result = try await execute(query: sql) - return result.rows.compactMap { row in - MSSQLSchemaQueries.parseForeignKeyRow(row.map { $0.asText }) - } - } - - private static func foreignKey(_ row: MSSQLForeignKeyRow) -> PluginForeignKeyInfo { - PluginForeignKeyInfo( - name: row.constraintName, - column: row.columnName, - referencedTable: row.referencedTable, - referencedColumn: row.referencedColumn, - referencedSchema: row.referencedSchema, - onDelete: row.onDelete, - onUpdate: row.onUpdate - ) + try await exportCatalog.fetchForeignKeys(table: table, schema: schema) } func fetchTriggers(table: String, schema: String?) async throws -> [PluginTriggerInfo] { - try await triggerList(schema: effectiveSchema(schema), table: table) + try await exportCatalog.fetchTriggers(table: table, schema: schema) } + var triggerEditUsesReplace: Bool { true } var supportsTransactionalDDL: Bool { true } @@ -249,37 +207,21 @@ extension MSSQLPluginDriver { } func generateDropTriggerSQL(name: String, table: String, schema: String?) -> String? { - let resolved = effectiveSchema(schema) - return "DROP TRIGGER \(quoteIdentifier(resolved)).\(quoteIdentifier(name))" + exportCatalog.generateDropTriggerSQL(name: name, table: table, schema: schema) } var fetchesAllColumnsInOneRequest: Bool { true } func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { - let result = try await execute(query: MSSQLSchemaQueries.allColumns(schema: effectiveSchema(schema))) - var columnsByTable: [String: [PluginColumnInfo]] = [:] - for row in result.rows { - let cells = row.map(\.asText) - guard let tableName = cells[safe: MSSQLSchemaQueries.allColumnsTableNameIndex] ?? nil, - let column = MSSQLSchemaQueries.parseColumnRow(cells) else { continue } - columnsByTable[tableName, default: []].append(column.pluginColumnInfo) - } - return columnsByTable + try await exportCatalog.fetchAllColumns(schema: schema) } var providesBulkForeignKeyFetch: Bool { true } - var tableDDLIncludesForeignKeys: Bool { true } + var tableDDLIncludesForeignKeys: Bool { MSSQLExportCatalog.tableDDLIncludesForeignKeys } func fetchAllForeignKeys(schema: String?) async throws -> [String: [PluginForeignKeyInfo]] { - let result = try await execute(query: MSSQLSchemaQueries.foreignKeys(schema: effectiveSchema(schema), table: nil)) - var fksByTable: [String: [PluginForeignKeyInfo]] = [:] - for row in result.rows { - guard let parsed = MSSQLSchemaQueries.parseForeignKeyRow(row.map { $0.asText }), - let table = parsed.table else { continue } - fksByTable[table, default: []].append(Self.foreignKey(parsed)) - } - return fksByTable + try await exportCatalog.fetchAllForeignKeys(schema: schema) } private static let metadataLogger = Logger(subsystem: "com.TablePro", category: "MSSQLPluginDriver") @@ -325,53 +267,12 @@ extension MSSQLPluginDriver { } } - // A view answers with its own definition: a column over an identity column reports `IsIdentity`. func fetchTableDDL(table: String, schema: String?) async throws -> String { - let resolvedSchema = effectiveSchema(schema) - let module = try await storedModule(schema: resolvedSchema, name: table) - if MSSQLObjectQueries.isView(objectType: module?[safe: 2].flatMap { $0 } ?? "") { - return try MSSQLObjectQueries.viewSource(row: module, schema: resolvedSchema, name: table) - } - - let columns = try await columnRows(table: table, schema: schema) - let keyConstraints = try await catalogRows(MSSQLSchemaQueries.keyConstraints(schema: resolvedSchema, table: table)) - .compactMap(MSSQLSchemaQueries.parseKeyConstraintRow) - let checkConstraints = try await catalogRows(MSSQLSchemaQueries.checkConstraints(schema: resolvedSchema, table: table)) - .compactMap(MSSQLSchemaQueries.parseCheckConstraintRow) - let foreignKeys = try await foreignKeyRows(table: table, schema: schema) - let collation = await databaseCollation(forTable: table) - do { - return try MSSQLSchemaQueries.createTableStatement( - schema: resolvedSchema, - table: table, - columns: columns, - keyConstraints: keyConstraints, - checkConstraints: checkConstraints, - foreignKeys: foreignKeys, - databaseCollation: collation - ) - } catch let refusal as MSSQLDefinitionRefusal { - throw MSSQLPluginError(refusal) - } - } - - // Unread, every column states its collation, which keeps it rather than changing it. - private func databaseCollation(forTable table: String) async -> String? { - guard let result = try? await execute(query: MSSQLSchemaQueries.databaseCollation(forTable: table)) else { - return nil - } - return result.rows.first?[safe: 0]?.asText + try await exportCatalog.fetchTableDDL(table: table, schema: schema) } func fetchViewDefinition(view: String, schema: String?) async throws -> String { - let resolvedSchema = effectiveSchema(schema) - let module = try await storedModule(schema: resolvedSchema, name: view) - return try MSSQLObjectQueries.viewSource(row: module, schema: resolvedSchema, name: view) - } - - private func storedModule(schema: String, name: String) async throws -> [String?]? { - let result = try await execute(query: MSSQLObjectQueries.moduleDefinition(schema: schema, name: name)) - return result.rows.first.map { $0.map(\.asText) } + try await exportCatalog.fetchViewDefinition(view: view, schema: schema) } func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { @@ -487,22 +388,6 @@ extension MSSQLPluginDriver { } } -extension MSSQLColumnRow { - var pluginColumnInfo: PluginColumnInfo { - PluginColumnInfo( - name: name, - dataType: displayType, - isNullable: isNullable, - isPrimaryKey: isPrimaryKey, - defaultValue: defaultValue, - extra: isIdentity ? "IDENTITY" : nil, - collation: collation, - identityKind: isIdentity ? .always : nil, - isGenerated: isGenerated - ) - } -} - private extension PluginViewReadOnlyReason { init(_ refusal: MSSQLViewSources.Refusal) { switch refusal { diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Types.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Types.swift index b16cda7969..f4c4a868b8 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Types.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Types.swift @@ -8,194 +8,15 @@ import TableProMSSQLCore import TableProPluginKit extension MSSQLPluginDriver { - /// The listing is one query and carries no definitions: a table type's statement needs two more - /// round trips per type, and a schema with a hundred types would pay for all of them to draw a - /// list of names. `fetchUserDefinedType` fills in the definition for the one the reader opens. func fetchUserDefinedTypes(schema: String?) async throws -> [PluginUserDefinedTypeInfo] { - let resolvedSchema = effectiveSchema(schema) - let result = try await execute(query: MSSQLTypeQueries.userDefinedTypeList(schema: resolvedSchema)) - return result.rows.compactMap { row in - listedType(from: row, schema: resolvedSchema) - } + try await exportCatalog.fetchUserDefinedTypes(schema: schema) } - // Addressed by `user_type_id`, so a type renamed since the listing still resolves. func fetchUserDefinedType(_ type: PluginUserDefinedTypeInfo) async throws -> PluginUserDefinedTypeInfo { - let resolvedSchema = effectiveSchema(type.schema) - let result = try await execute(query: MSSQLTypeQueries.userDefinedTypeList(schema: resolvedSchema)) - let listed = result.rows.compactMap { listedType(from: $0, schema: resolvedSchema) } - let match = type.identity.flatMap { identity in listed.first { $0.identity == identity } } - ?? listed.first { $0.name == type.name } - guard let match else { throw PluginObjectSourceError.notFound(type.name) } - - do { - switch match.kind { - case .tableType: - return try await tableType(match, schema: resolvedSchema) - case .clrType: - return match.withDefinition(MSSQLTypeDefinition.clrStatement( - schema: resolvedSchema, - name: match.name, - assembly: match.attributes.first { $0.label == "Assembly" }?.value, - assemblyClass: match.attributes.first { $0.label == "Class" }?.value - )) - default: - return try match.withDefinition(MSSQLTypeDefinition.aliasStatement( - schema: resolvedSchema, - name: match.name, - baseType: match.baseType, - isNullable: match.attributes.first { $0.label == "Nullable" }?.value != "NO" - )) - } - } catch let refusal as MSSQLDefinitionRefusal { - throw MSSQLPluginError(refusal) - } + try await exportCatalog.fetchUserDefinedType(type) } func generateDropTypeSQL(type: PluginUserDefinedTypeInfo) -> String? { "DROP TYPE \(MSSQLTypeDefinition.bracketed(effectiveSchema(type.schema))).\(MSSQLTypeDefinition.bracketed(type.name))" } - - private func tableType( - _ type: PluginUserDefinedTypeInfo, - schema: String - ) async throws -> PluginUserDefinedTypeInfo { - let columnRows = try await execute( - query: MSSQLTypeQueries.tableTypeColumns(schema: schema, name: type.name) - ).rows - let indexRows = try await execute( - query: MSSQLTypeQueries.tableTypeIndexes(schema: schema, name: type.name) - ).rows - let checkRows = try await execute( - query: MSSQLTypeQueries.tableTypeCheckConstraints(schema: schema, name: type.name) - ).rows - let collation = try? await execute(query: MSSQLTypeQueries.databaseCollation).rows.first?[safe: 0]?.asText - - let columns = columnRows.compactMap { row -> MSSQLTypeDefinition.Column? in - guard let name = row[safe: 0]?.asText else { return nil } - return MSSQLTypeDefinition.Column( - name: name, - type: row[safe: 1]?.asText, - isNullable: row[safe: 2]?.asText == "1", - identitySpec: row[safe: 3]?.asText, - computedDefinition: row[safe: 4]?.asText, - defaultDefinition: row[safe: 5]?.asText, - collation: row[safe: 6]?.asText - ) - } - var indexOrder: [String] = [] - var indexesById: [String: MSSQLTypeDefinition.Index] = [:] - for row in indexRows { - guard let indexId = row[safe: 6]?.asText, let column = row[safe: 4]?.asText else { continue } - let key = MSSQLTypeDefinition.IndexKey(column: column, isDescending: row[safe: 5]?.asText == "1") - if let existing = indexesById[indexId] { - indexesById[indexId] = MSSQLTypeDefinition.Index( - name: existing.name, - isPrimaryKey: existing.isPrimaryKey, - isUnique: existing.isUnique, - typeDescription: existing.typeDescription, - keys: existing.keys + [key], - bucketCount: existing.bucketCount - ) - continue - } - indexOrder.append(indexId) - indexesById[indexId] = MSSQLTypeDefinition.Index( - name: row[safe: 0]?.asText, - isPrimaryKey: row[safe: 1]?.asText == "1", - isUnique: row[safe: 2]?.asText == "1", - typeDescription: row[safe: 3]?.asText, - keys: [key], - bucketCount: row[safe: 7]?.asText.flatMap(Int.init) ?? 0 - ) - } - let indexes = indexOrder.compactMap { indexesById[$0] } - let checks = checkRows.compactMap { $0[safe: 0]?.asText } - let definition = try MSSQLTypeDefinition.tableStatement( - schema: schema, - name: type.name, - columns: columns, - indexes: indexes, - checkConstraints: checks, - isMemoryOptimized: type.attributes.contains { $0.label == "Memory Optimized" && $0.value == "YES" }, - databaseCollation: collation ?? nil - ) - - return PluginUserDefinedTypeInfo( - name: type.name, - kind: .tableType, - schema: schema, - identity: type.identity, - fields: columns.map { - PluginUserDefinedTypeField(name: $0.name, type: $0.type ?? "", collation: $0.collation) - }, - columnTypeSpelling: type.columnTypeSpelling, - definition: definition, - attributes: type.attributes - ) - } - - private func listedType(from row: [PluginCellValue], schema: String) -> PluginUserDefinedTypeInfo? { - guard let name = row[safe: 0]?.asText else { return nil } - let kindCode = row[safe: 3]?.asText ?? "" - let kind: PluginUserDefinedTypeKind = switch MSSQLTypeQueries.Kind(rawValue: kindCode) { - case .table: .tableType - case .clr: .clrType - default: .aliasType - } - - var attributes: [PluginObjectAttribute] = [] - if kind == .aliasType { - attributes.append(PluginObjectAttribute( - label: "Nullable", - value: row[safe: 5]?.asText == "1" ? "YES" : "NO" - )) - } - if let collation = row[safe: 6]?.asText, !collation.isEmpty { - attributes.append(PluginObjectAttribute(label: "Collation", value: collation)) - } - if let assembly = row[safe: 7]?.asText, !assembly.isEmpty { - attributes.append(PluginObjectAttribute(label: "Assembly", value: assembly)) - } - if let assemblyClass = row[safe: 8]?.asText, !assemblyClass.isEmpty { - attributes.append(PluginObjectAttribute(label: "Class", value: assemblyClass)) - } - if kind == .tableType, row[safe: 9]?.asText == "1" { - attributes.append(PluginObjectAttribute(label: "Memory Optimized", value: "YES")) - } - - return PluginUserDefinedTypeInfo( - name: name, - kind: kind, - schema: row[safe: 1]?.asText ?? schema, - identity: row[safe: 2]?.asText, - baseType: row[safe: 4]?.asText, - columnTypeSpelling: columnTypeSpelling(kind: kind, schema: row[safe: 1]?.asText ?? schema, name: name), - attributes: attributes - ) - } - - /// A table type is never a column type, so it gets no spelling at all rather than one the - /// column picker would offer and the server would reject. - private func columnTypeSpelling(kind: PluginUserDefinedTypeKind, schema: String, name: String) -> String? { - guard kind != .tableType else { return nil } - return "\(MSSQLTypeDefinition.bracketed(schema)).\(MSSQLTypeDefinition.bracketed(name))" - } -} - -private extension PluginUserDefinedTypeInfo { - func withDefinition(_ definition: String) -> PluginUserDefinedTypeInfo { - PluginUserDefinedTypeInfo( - name: name, - kind: kind, - schema: schema, - identity: identity, - enumLabels: enumLabels, - fields: fields, - baseType: baseType, - columnTypeSpelling: columnTypeSpelling, - definition: definition, - attributes: attributes - ) - } } diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLTableDefinitionSQL.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLTableDefinitionSQL.swift index 945552275e..6c1467303a 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLTableDefinitionSQL.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLTableDefinitionSQL.swift @@ -11,7 +11,7 @@ import TableProMSSQLCore import TableProPluginKit import TableProSQLQuoting -enum MSSQLTableDefinitionSQL { +nonisolated enum MSSQLTableDefinitionSQL { static func createTable(_ definition: PluginCreateTableDefinition, schema: String) -> String? { guard let createTable = createTableStatement(definition, schema: schema) else { return nil } var sql = createTable + ";" diff --git a/apple/Plugins/MarkdownExportPlugin/MarkdownExportModels.swift b/apple/Plugins/MarkdownExportPlugin/MarkdownExportModels.swift index 8874578f84..f55e42e0f8 100644 --- a/apple/Plugins/MarkdownExportPlugin/MarkdownExportModels.swift +++ b/apple/Plugins/MarkdownExportPlugin/MarkdownExportModels.swift @@ -5,7 +5,7 @@ import Foundation -public struct MarkdownExportOptions: Equatable, Codable { +nonisolated public struct MarkdownExportOptions: Equatable, Codable { /// Pads every cell so the columns line up in the raw text. Off writes the narrowest table a /// renderer still reads correctly, which is what a large result wants. public var alignsColumns: Bool = true @@ -33,12 +33,7 @@ public struct MarkdownExportOptions: Equatable, Codable { } } -/// Renders GitHub-flavoured Markdown tables. -/// -/// Pure and separate from the plugin so the escaping is testable without a database: a value -/// holding a pipe or a line break has to be neutralised or it ends the cell early and every column -/// after it shifts. -public enum MarkdownTableRenderer { +nonisolated public enum MarkdownTableRenderer { // A pipe closes a cell and a line break a row, and no renderer takes a raw newline inside a cell. public static func cell(_ value: String) -> String { value @@ -53,8 +48,9 @@ public enum MarkdownTableRenderer { guard let widths, widths.count == cells.count else { return "| \(cells.joined(separator: " | ")) |" } + // Widths count Characters; NSString padding counts UTF-16 units and truncates an emoji or a combining mark. let padded = zip(cells, widths).map { cell, width in - cell.padding(toLength: max(width, cell.count), withPad: " ", startingAt: 0) + cell + String(repeating: " ", count: max(0, width - cell.count)) } return "| \(padded.joined(separator: " | ")) |" } @@ -80,7 +76,7 @@ public enum MarkdownTableRenderer { } } -private extension Array { +nonisolated private extension Array { subscript(safe index: Int) -> Element? { indices.contains(index) ? self[index] : nil } diff --git a/apple/Plugins/MarkdownExportPlugin/MarkdownExportOptionsView.swift b/apple/Plugins/MarkdownExportPlugin/MarkdownExportOptionsView.swift index 38d64617d7..c5c814559f 100644 --- a/apple/Plugins/MarkdownExportPlugin/MarkdownExportOptionsView.swift +++ b/apple/Plugins/MarkdownExportPlugin/MarkdownExportOptionsView.swift @@ -29,3 +29,10 @@ struct MarkdownExportOptionsView: View { .font(.system(size: 13)) } } + +extension MarkdownExportPlugin { + @MainActor + func settingsView() -> AnyView? { + AnyView(MarkdownExportOptionsView(plugin: self)) + } +} diff --git a/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift b/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift index 8fc08e0fd3..aacc2e8dfa 100644 --- a/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift +++ b/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift @@ -6,10 +6,9 @@ import Combine import Foundation import os -import SwiftUI import TableProPluginKit -final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugin, @unchecked Sendable { +nonisolated final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugin, @unchecked Sendable { static let pluginName = "Markdown Export" static let pluginVersion = "1.0.0" static let pluginDescription = "Export data to Markdown tables" @@ -25,7 +24,9 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable /// would mean holding the whole table in memory to decide a column width. private static let alignmentSampleRows = 200 - @Published var settings = MarkdownExportOptions() { + // A nonisolated class cannot hold an @Published property, so the change is announced by hand. + var settings = MarkdownExportOptions() { + willSet { objectWillChange.send() } didSet { saveSettings() } } @@ -33,11 +34,6 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable required init() { loadSettings() } - @MainActor - func settingsView() -> AnyView? { - AnyView(MarkdownExportOptionsView(plugin: self)) - } - func resetSettingsToDefaults() { settings = MarkdownExportOptions() } @@ -66,6 +62,8 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable try fileHandle.write(contentsOf: "\n".toUTF8Data()) } + // The rows buffered for alignment are written after the stream ends, past its own checks. + try progress.checkCancellation() try fileHandle.close() try PluginExportUtilities.commitAtomicWrite(from: tempURL, to: destination) committed = true diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 9728bffdb9..cd043708c3 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -476,15 +476,19 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se throw MongoDBPluginError.notConnected } - return try await indexEntries(of: table, conn: conn).map(\.pluginIndexInfo) + return try await indexEntries(of: table, in: currentDb, conn: conn).map(\.pluginIndexInfo) } /// A view has no indexes, and the server says so by refusing `listIndexes` with /// CommandNotSupportedOnView. libmongoc answers a missing collection the same way, with an /// empty list, which is what the Enumerate Indexes spec asks for. - private func indexEntries(of collection: String, conn: MongoDBConnection) async throws -> [MongoDBIndexEntry] { + private func indexEntries( + of collection: String, + in database: String, + conn: MongoDBConnection + ) async throws -> [MongoDBIndexEntry] { do { - return try await conn.listIndexes(database: currentDb, collection: collection) + return try await conn.listIndexes(database: database, collection: collection) .compactMap(MongoDBIndexEntry.init(json:)) } catch let error as MongoDBError where error.code == MongoDBServerErrorCode.commandNotSupportedOnView { return [] @@ -549,10 +553,12 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se guard let conn = mongoConnection else { throw MongoDBPluginError.notConnected } + // An export names the collection's database, which need not be the one the session is on. + let database = schema?.nilIfEmpty ?? currentDb let entry: MongoDBNamespaceEntry? do { - entry = try await namespaceEntry(named: table, conn: conn) + entry = try await namespaceEntry(named: table, in: database, conn: conn) } catch { Self.logger.debug("Failed to fetch collection info for \(table): \(error.localizedDescription)") entry = nil @@ -563,7 +569,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se let indexes: [MongoDBIndexEntry] do { - indexes = try await indexEntries(of: table, conn: conn) + indexes = try await indexEntries(of: table, in: database, conn: conn) } catch { Self.logger.debug("Failed to fetch indexes for \(table): \(error.localizedDescription)") indexes = [] @@ -577,14 +583,18 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se guard let conn = mongoConnection else { throw MongoDBPluginError.notConnected } - guard let statement = try await namespaceEntry(named: view, conn: conn)?.collModStatement() else { + guard let statement = try await namespaceEntry(named: view, in: currentDb, conn: conn)?.collModStatement() else { throw MongoDBPluginError.viewNotFound(view) } return statement } - private func namespaceEntry(named name: String, conn: MongoDBConnection) async throws -> MongoDBNamespaceEntry? { - try await conn.listNamespaces(database: currentDb, named: name) + private func namespaceEntry( + named name: String, + in database: String, + conn: MongoDBConnection + ) async throws -> MongoDBNamespaceEntry? { + try await conn.listNamespaces(database: database, named: name) .lazy .compactMap(MongoDBNamespaceEntry.init(json:)) .first { $0.name == name } diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift index b4082e663f..a40d1bfcd4 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift @@ -40,12 +40,12 @@ nonisolated struct MongoDBQueryBuilder { // MARK: - Base Query - /// Build: db.collection.find({}).sort({}).skip(offset).limit(limit) + // A nil limit reads every document, as an export of the browsed collection does. func buildBaseQuery( collection: String, sortColumns: [(columnIndex: Int, ascending: Bool)] = [], columns: [String] = [], - limit: Int = 200, + limit: Int? = 200, offset: Int = 0 ) -> String { decorate( @@ -59,7 +59,7 @@ nonisolated struct MongoDBQueryBuilder { filterDocument: String, sortColumns: [(columnIndex: Int, ascending: Bool)] = [], columns: [String] = [], - limit: Int = 200, + limit: Int? = 200, offset: Int = 0 ) -> String { decorate( @@ -72,7 +72,7 @@ nonisolated struct MongoDBQueryBuilder { _ query: String, sortColumns: [(columnIndex: Int, ascending: Bool)], columns: [String], - limit: Int, + limit: Int?, offset: Int ) -> String { var result = query @@ -85,7 +85,9 @@ nonisolated struct MongoDBQueryBuilder { result += ".skip(\(offset))" } - result += ".limit(\(limit))" + if let limit { + result += ".limit(\(limit))" + } return result } diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLColumnDefinitionSQL.swift b/apple/Plugins/MySQLDriverPlugin/MySQLColumnDefinitionSQL.swift index df0e250f29..64cb4bfede 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLColumnDefinitionSQL.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLColumnDefinitionSQL.swift @@ -9,14 +9,14 @@ import TableProPluginKit import TableProSQLGrammar import TableProSQLQuoting -internal func mysqlQuoteIdentifier(_ name: String) -> String { +nonisolated internal func mysqlQuoteIdentifier(_ name: String) -> String { SQLQuoting.identifier(name, quote: .backtick) } /// MySQL and MariaDB take a `BLOB`, `TEXT`, `JSON` or `GEOMETRY` default "only if the value is /// written as an expression, even if the expression value is a literal", so the parentheses are /// required by the grammar rather than chosen by the caller. -internal func mysqlRequiresParenthesisedDefault(dataType: String) -> Bool { +nonisolated internal func mysqlRequiresParenthesisedDefault(dataType: String) -> Bool { let upper = dataType.uppercased() let base = upper.split(separator: "(", maxSplits: 1).first.map(String.init)? .trimmingCharacters(in: .whitespaces) ?? upper @@ -24,7 +24,7 @@ internal func mysqlRequiresParenthesisedDefault(dataType: String) -> Bool { } // MySQL requires parentheses around every default that is not one whole literal; MariaDB accepts them bare. -internal func mysqlDefaultIsLiteral(_ value: String) -> Bool { +nonisolated internal func mysqlDefaultIsLiteral(_ value: String) -> Bool { let upper = value.uppercased() if upper == "NULL" || upper == "TRUE" || upper == "FALSE" { return true } return PluginNumericLiteral.isValid(value) @@ -46,7 +46,7 @@ internal func mysqlDefaultIsLiteral(_ value: String) -> Bool { /// /// Nothing else is rewritten: the value already holds the SQL, and re-quoting it is what turned /// `(UUID())` into the six-character string `uuid()`. -internal func mysqlDefaultValueLiteral(_ value: String, dataType: String, isMariaDB: Bool) -> String { +nonisolated internal func mysqlDefaultValueLiteral(_ value: String, dataType: String, isMariaDB: Bool) -> String { if mysqlIsNullLiteral(value) { return "NULL" } if mysqlTemporalType(dataType), let expression = mysqlCurrentTimestampExpression(value, dataType: dataType) { return expression @@ -58,7 +58,7 @@ internal func mysqlDefaultValueLiteral(_ value: String, dataType: String, isMari } // The statement takes these as SQL, whether the catalog or the user wrote them, so each has to be one expression. -internal func mysqlColumnTextRefusal(for column: PluginColumnDefinition, flavor: MySQLServerFlavor) -> String? { +nonisolated internal func mysqlColumnTextRefusal(for column: PluginColumnDefinition, flavor: MySQLServerFlavor) -> String? { let expressions = [column.defaultValue, column.generationExpression, column.columnCheck].compactMap { $0 } let acceptsAll = SQLExpressionFragment.isSingleType(column.dataType, databaseTypeId: flavor.databaseTypeId) && expressions.allSatisfy { $0.isEmpty || mysqlIsSingleExpression($0, flavor: flavor) } @@ -71,14 +71,14 @@ internal func mysqlColumnTextRefusal(for column: PluginColumnDefinition, flavor: ) } -internal func mysqlIsNullLiteral(_ value: String) -> Bool { +nonisolated internal func mysqlIsNullLiteral(_ value: String) -> Bool { value.trimmingCharacters(in: .whitespaces).caseInsensitiveCompare("NULL") == .orderedSame } /// Which of `CHARACTER SET` and `COLLATE` a column type takes. Measured on MySQL 8.4 and MariaDB 11.4: /// `CHARACTER SET` on `INT`, `VARBINARY`, `BLOB` or `JSON` is ERROR 1064, and so is `CHARACTER SET` on a /// national spelling (`NVARCHAR`), which takes `COLLATE` alone. A type outside the list takes neither. -internal enum MySQLColumnTextType: Equatable { +nonisolated internal enum MySQLColumnTextType: Equatable { case none case collationOnly case characterSetAndCollation @@ -112,17 +112,17 @@ internal enum MySQLColumnTextType: Equatable { /// A name the catalog lists is a bare word; anything else is backticked, so the server refuses it as /// unknown rather than reading the rest of it as SQL. -internal func mysqlCharsetName(_ name: String) -> String { +nonisolated internal func mysqlCharsetName(_ name: String) -> String { mysqlIsSafeCharsetIdentifier(name) ? name : mysqlQuoteIdentifier(name) } -internal func mysqlIsSafeCharsetIdentifier(_ value: String) -> Bool { +nonisolated internal func mysqlIsSafeCharsetIdentifier(_ value: String) -> Bool { guard !value.isEmpty else { return false } let allowed = CharacterSet.alphanumerics.union(CharacterSet(charactersIn: "_")) return value.unicodeScalars.allSatisfy { allowed.contains($0) } } -internal func mysqlCharacterSetClauses(_ column: PluginColumnDefinition) -> String { +nonisolated internal func mysqlCharacterSetClauses(_ column: PluginColumnDefinition) -> String { let textType = MySQLColumnTextType(dataType: column.dataType) var clauses = "" if textType.takesCharacterSet, let charset = column.charset?.nilIfEmpty { @@ -134,7 +134,7 @@ internal func mysqlCharacterSetClauses(_ column: PluginColumnDefinition) -> Stri return clauses } -internal func mysqlColumnAttributesSQL(_ column: PluginColumnDefinition, isMariaDB: Bool) -> String { +nonisolated internal func mysqlColumnAttributesSQL(_ column: PluginColumnDefinition, isMariaDB: Bool) -> String { var def = "" if column.unsigned { @@ -162,7 +162,7 @@ internal func mysqlColumnAttributesSQL(_ column: PluginColumnDefinition, isMaria /// `columnCheck` is the column's own `CHECK` as the statement restates it, printed last on the line as /// MariaDB prints it. Only a `MODIFY` or a move of a column that already has one passes it. -internal func mysqlColumnDefinitionSQL( +nonisolated internal func mysqlColumnDefinitionSQL( _ column: PluginColumnDefinition, isMariaDB: Bool = false, columnCheck: String? = nil @@ -171,7 +171,7 @@ internal func mysqlColumnDefinitionSQL( return mysqlColumnBodySQL(column, isMariaDB: isMariaDB) + check } -private func mysqlColumnBodySQL(_ column: PluginColumnDefinition, isMariaDB: Bool) -> String { +nonisolated private func mysqlColumnBodySQL(_ column: PluginColumnDefinition, isMariaDB: Bool) -> String { let name = mysqlQuoteIdentifier(column.name) guard let expression = column.generationExpression?.nilIfEmpty else { return "\(name) \(column.dataType)" + mysqlColumnAttributesSQL(column, isMariaDB: isMariaDB) diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLCreateTableSQL.swift b/apple/Plugins/MySQLDriverPlugin/MySQLCreateTableSQL.swift index 3b11d25a7e..2de6dc8101 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLCreateTableSQL.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLCreateTableSQL.swift @@ -12,7 +12,7 @@ import TableProPluginKit /// `isMariaDB` is threaded rather than defaulted. MariaDB and MySQL spell an expression default /// differently, the flag is driver state that `PluginCreateTableDefinition` does not carry, and a /// default here would silently give every MariaDB table MySQL's parentheses. -internal func mysqlCreateTableSQL( +nonisolated internal func mysqlCreateTableSQL( definition: PluginCreateTableDefinition, isMariaDB: Bool ) -> String? { @@ -53,7 +53,7 @@ internal func mysqlCreateTableSQL( return sql + ";" } -internal enum MySQLIndexKeyPart: Equatable { +nonisolated internal enum MySQLIndexKeyPart: Equatable { case column(String, prefixLength: Int?) case expression(String) @@ -73,7 +73,7 @@ internal enum MySQLIndexKeyPart: Equatable { } } -internal struct MySQLIndexKey: Equatable { +nonisolated internal struct MySQLIndexKey: Equatable { let part: MySQLIndexKeyPart let isDescending: Bool @@ -82,7 +82,7 @@ internal struct MySQLIndexKey: Equatable { } } -internal func mysqlIndexKeyClause(_ keys: [MySQLIndexKey], type: String?) -> String { +nonisolated internal func mysqlIndexKeyClause(_ keys: [MySQLIndexKey], type: String?) -> String { var clause = "(\(keys.map(\.sql).joined(separator: ", ")))" let upperType = type?.uppercased() ?? "" if upperType == "BTREE" || upperType == "HASH" { @@ -91,7 +91,7 @@ internal func mysqlIndexKeyClause(_ keys: [MySQLIndexKey], type: String?) -> Str return clause } -internal func mysqlIndexKeys(of index: PluginIndexDefinition) -> [MySQLIndexKey] { +nonisolated internal func mysqlIndexKeys(of index: PluginIndexDefinition) -> [MySQLIndexKey] { let expressions = Set(index.expressions ?? []) return index.columns.map { column in let part: MySQLIndexKeyPart = expressions.contains(column) @@ -101,7 +101,7 @@ internal func mysqlIndexKeys(of index: PluginIndexDefinition) -> [MySQLIndexKey] } } -internal func mysqlIndexDefinitionSQL(_ index: PluginIndexDefinition) -> String { +nonisolated internal func mysqlIndexDefinitionSQL(_ index: PluginIndexDefinition) -> String { let upperType = index.indexType?.uppercased() ?? "" let kind: String switch upperType { @@ -115,11 +115,11 @@ internal func mysqlIndexDefinitionSQL(_ index: PluginIndexDefinition) -> String } // `spelling` is the session's: a key part is the catalog's backslash-escaped text. -internal func mysqlAddIndexSQL(table: String, index: PluginIndexDefinition, spelling: MySQLLiteralSpelling) -> String { +nonisolated internal func mysqlAddIndexSQL(table: String, index: PluginIndexDefinition, spelling: MySQLLiteralSpelling) -> String { spelling.respelled("ALTER TABLE \(mysqlQuoteIdentifier(table)) ADD \(mysqlIndexDefinitionSQL(index))") } -internal func mysqlModifyIndexSQL( +nonisolated internal func mysqlModifyIndexSQL( table: String, oldIndexName: String, newIndex: PluginIndexDefinition, @@ -136,7 +136,7 @@ internal func mysqlModifyIndexSQL( /// `PRIMARY`, in any case, names the primary key and nothing else: MariaDB 13 refuses /// `ADD INDEX primary (b)` with ERROR 1280. Refused before the save, because a save that splits a /// replacement of the key's row drops the key first and then cannot add anything back. -internal func mysqlReservedIndexNameRefusal(for index: PluginIndexDefinition) -> String? { +nonisolated internal func mysqlReservedIndexNameRefusal(for index: PluginIndexDefinition) -> String? { guard index.name.caseInsensitiveCompare("PRIMARY") == .orderedSame else { return nil } return String( localized: "PRIMARY is the primary key's name, and no other index can take it. Rename the index, or change the key on the Columns tab." @@ -144,7 +144,7 @@ internal func mysqlReservedIndexNameRefusal(for index: PluginIndexDefinition) -> } // A key part, and the key list read from the catalog, go into the statement as SQL. -internal func mysqlIndexExpressionRefusal(for index: PluginIndexDefinition, flavor: MySQLServerFlavor) -> String? { +nonisolated internal func mysqlIndexExpressionRefusal(for index: PluginIndexDefinition, flavor: MySQLServerFlavor) -> String? { let texts = (index.expressions ?? []) + [index.ddlMethodAndKeys].compactMap { $0 } guard !texts.allSatisfy({ $0.isEmpty || mysqlIsSingleExpression($0, flavor: flavor) }) else { return nil } return String(format: String(localized: "Index %@ has a key part that is not one SQL expression."), index.name) @@ -152,7 +152,7 @@ internal func mysqlIndexExpressionRefusal(for index: PluginIndexDefinition, flav /// `CONSTRAINT name` is optional in MySQL's grammar and the server invents one when it is left out, /// so a blank name writes no clause rather than `CONSTRAINT `` `` ``, which is a syntax error. -internal func mysqlForeignKeyDefinitionSQL(_ foreignKey: PluginForeignKeyDefinition) -> String { +nonisolated internal func mysqlForeignKeyDefinitionSQL(_ foreignKey: PluginForeignKeyDefinition) -> String { let columns = foreignKey.columns.map(mysqlQuoteIdentifier).joined(separator: ", ") let referencedColumns = foreignKey.referencedColumns.map(mysqlQuoteIdentifier).joined(separator: ", ") let referencedTable: String @@ -184,7 +184,7 @@ internal func mysqlForeignKeyDefinitionSQL(_ foreignKey: PluginForeignKeyDefinit } // Catalog text that is no referential action goes in as a name, which the server refuses. -private func mysqlReferentialAction(_ action: String) -> String { +nonisolated private func mysqlReferentialAction(_ action: String) -> String { let actions = ["CASCADE", "SET NULL", "SET DEFAULT", "RESTRICT", "NO ACTION"] return actions.first { $0 == action.uppercased() } ?? mysqlQuoteIdentifier(action) } diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLExportCatalog.swift b/apple/Plugins/MySQLDriverPlugin/MySQLExportCatalog.swift new file mode 100644 index 0000000000..54bf618f31 --- /dev/null +++ b/apple/Plugins/MySQLDriverPlugin/MySQLExportCatalog.swift @@ -0,0 +1,852 @@ +// +// MySQLExportCatalog.swift +// MySQLDriverPlugin +// + +import Foundation +import os +import TableProMySQLCore +import TableProPluginKit + +// The catalog reads and statements an export takes, over the caller's runner, so the Mac plugin and the iOS driver +// send the same SQL and map the same rows. +nonisolated struct MySQLExportCatalog: Sendable { + typealias Run = @Sendable (String) async throws -> PluginQueryResult + + struct SessionHooks: Sendable { + // Runs outside the footprint and the idle clock, spelled for the session like `run`. + let runSetup: Run + let transactionState: @Sendable () async -> PluginSessionTransactionState + let adjustFootprint: @Sendable ((inout MySQLSessionFootprint) -> Void) -> Void + } + + static let tableDDLIncludesForeignKeys = true + static let defaultHost = "%" + + let banner: String? + let flavor: MySQLServerFlavor + let activeDatabase: String + let quote: @Sendable (String) -> String + // Takes a statement spelled with backslash escapes and runs it spelled for the session that reads it. + let run: Run + let schemaNamesAreCaseSensitive: @Sendable () async -> Bool + // True for a server refusal that says the catalog cannot answer, false for a failed connection. + let settlesBlindness: @Sendable (any Error) -> Bool + + private static let logger = Logger(subsystem: "com.TablePro", category: "MySQLExportCatalog") + + private static let excludedPrivileges: Set = ["GRANT OPTION", "PROXY"] + private static let tableContextMarkers = ["TABLE", "INDEX", "VIEW", "TRIGGER"] + private static let databaseContextMarkers = [ + "DATABASE", "TABLE", "INDEX", "VIEW", "TRIGGER", "EVENT", "FUNCTION", "PROCEDURE" + ] + private static let columnGrantablePrivileges: Set = [ + "SELECT", "INSERT", "UPDATE", "REFERENCES" + ] + + // MARK: - Server facts + + func holds(_ floor: (String?, MySQLServerFlavor) -> Bool) -> Bool { + floor(banner, flavor) + } + + var hasInformationSchema: Bool { + holds(MySQLServerVersion.hasInformationSchema(banner:flavor:)) + } + + // The catalog read needs `REFERENTIAL_CONSTRAINTS` as well as `KEY_COLUMN_USAGE`; 5.0.96 has only the second. + var hasForeignKeyCatalog: Bool { + hasInformationSchema && holds(MySQLServerVersion.hasReferentialConstraintsCatalog(banner:flavor:)) + } + + var omittedForeignKeyAction: String { + MySQLServerVersion.omittedForeignKeyAction(banner: banner, flavor: flavor) + } + + func effectiveSchema(_ schema: String?) -> String { + MySQLObjectQueries.effectiveSchema(schema, activeDatabase: activeDatabase) + } + + // With no database selected an unqualified name is the only form the server takes. + func qualifiedName(_ name: String, schema: String?) -> String { + MySQLObjectQueries.qualifiedIdentifier(schema: effectiveSchema(schema).nilIfEmpty, name: name, quote: quote) + } + + // True under `lower_case_table_names = 0`; nil when the server does not answer. + static func schemaNameCaseSensitivity(run: Run) async -> Bool? { + guard let result = try? await run(MySQLObjectQueries.lowerCaseTableNames), + let setting = result.rows.first?[safe: 0]?.asText.flatMap({ Int($0) }) + else { return nil } + return setting == 0 + } + + // MARK: - Table and view DDL + + // Nil when the server answered without the statement. + func tableDDL(table: String, schema: String?) async throws -> String? { + let result = try await run("SHOW CREATE TABLE \(qualifiedName(table, schema: schema))") + guard let ddl = result.rows.first?[safe: 1]?.asText else { return nil } + return ddl.hasSuffix(";") ? ddl : ddl + ";" + } + + func viewDefinition(view: String, schema: String?) async throws -> String? { + // StarRocks's `SHOW CREATE VIEW` rewrites a materialized view as a plain `CREATE VIEW`. + let show = flavor.isStarRocks ? "SHOW CREATE TABLE" : "SHOW CREATE VIEW" + let result = try await run("\(show) \(qualifiedName(view, schema: schema))") + return result.rows.first?[safe: 1]?.asText + } + + func approximateRowCount(table: String, schema: String?) async throws -> Int? { + let result = try await run(MySQLObjectQueries.approximateRowCount(schema: effectiveSchema(schema), table: table)) + guard let value = result.rows.first?[safe: 0]?.asText, let count = Int(value) else { return nil } + return count + } + + // MARK: - Routines + + // The routines and their parameters are two reads; `MySQLRoutineSignatures` says why. + func routines(schema: String?) async throws -> [PluginRoutineInfo] { + guard flavor.hasMySQLObjectCatalogs, hasInformationSchema else { return [] } + let resolvedSchema = effectiveSchema(schema) + let exactSchema = await schemaNamesAreCaseSensitive() + let result = try await run(MySQLObjectQueries.routineList(schema: resolvedSchema, exactSchema: exactSchema)) + let signatures = result.rows.isEmpty + ? nil + : try await routineSignatures(schema: resolvedSchema, exactSchema: exactSchema) + return result.rows.compactMap { row -> PluginRoutineInfo? in + guard let name = row[safe: 0]?.asText else { return nil } + let type = row[safe: 1]?.asText ?? "" + let isProcedure = type.uppercased() == "PROCEDURE" + let owner = row[safe: 7]?.asText ?? resolvedSchema + var attributes: [PluginObjectAttribute] = [] + if let access = row[safe: 3]?.asText, !access.isEmpty { + attributes.append(PluginObjectAttribute(label: "Data Access", value: access)) + } + if let deterministic = row[safe: 4]?.asText, !deterministic.isEmpty { + attributes.append(PluginObjectAttribute(label: "Deterministic", value: deterministic)) + } + if let security = row[safe: 5]?.asText, !security.isEmpty { + attributes.append(PluginObjectAttribute(label: "Security", value: security)) + } + if let definer = row[safe: 6]?.asText, !definer.isEmpty { + attributes.append(PluginObjectAttribute(label: "Definer", value: definer)) + } + return PluginRoutineInfo( + name: name, + kind: isProcedure ? .procedure : .function, + schema: owner, + returnType: isProcedure ? nil : row[safe: 2]?.asText, + language: "SQL", + argumentSignature: signatures?.signature(schema: owner, type: type, routine: name), + identity: nil, + attributes: attributes + ) + } + } + + // Nil before 5.5.3, which has no `PARAMETERS` table, so a signature there is unknown rather than empty. + private func routineSignatures(schema: String, exactSchema: Bool) async throws -> MySQLRoutineSignatures? { + guard holds(MySQLServerVersion.hasParametersCatalog(banner:flavor:)) else { return nil } + let result = try await run(MySQLObjectQueries.routineParameters(schema: schema, exactSchema: exactSchema)) + var signatures = MySQLRoutineSignatures() + for row in result.rows { + guard let owner = row[safe: 0]?.asText, + let type = row[safe: 1]?.asText, + let routine = row[safe: 2]?.asText + else { continue } + signatures.append( + schema: owner, + type: type, + routine: routine, + mode: row[safe: 3]?.asText, + name: row[safe: 4]?.asText, + dataType: row[safe: 5]?.asText + ) + } + return signatures + } + + func routineDDL(_ routine: PluginRoutineInfo) async throws -> String { + let kind = routine.kind == .procedure ? "PROCEDURE" : "FUNCTION" + let query = MySQLObjectQueries.routineDefinition( + kind: kind, + schema: effectiveSchema(routine.schema), + name: routine.name + ) + let result = try await run(query) + guard let row = result.rows.first else { + throw PluginObjectSourceError.notFound(routine.name) + } + // A NULL body is the server hiding it from an account without SHOW_ROUTINE, not a missing routine. + guard let ddl = row[safe: 2]?.asText, !ddl.isEmpty else { + throw PluginObjectSourceError.insufficientPrivilege(routine.name) + } + return ddl + } + + // Nil for a flavor without stored routines, which leaves the caller's own statement. + func dropRoutineStatement(name: String, schema: String?, isFunction: Bool) -> String? { + guard flavor.hasMySQLObjectCatalogs else { return nil } + return MySQLObjectQueries.dropRoutine( + isFunction: isFunction, + schema: effectiveSchema(schema).nilIfEmpty, + name: name, + quote: quote + ) + } + + // MARK: - Triggers + + func allTriggers(schema: String?) async throws -> [PluginTriggerInfo] { + guard flavor.hasMySQLObjectCatalogs else { return [] } + return try await triggerList(schema: effectiveSchema(schema), table: nil) + } + + func triggerDDL(_ trigger: PluginTriggerInfo) async throws -> String { + if let definition = trigger.definition, !definition.isEmpty { return definition } + let listed = try await triggerList(schema: effectiveSchema(trigger.schema), table: trigger.table) + guard let match = listed.first(where: { $0.name == trigger.name }) else { + throw PluginObjectSourceError.notFound(trigger.name) + } + guard let definition = match.definition, !definition.isEmpty else { + throw PluginObjectSourceError.unsupported(trigger.name) + } + return definition + } + + // The sidebar list, the per-table read and the DDL lookup all come through here. 5.0.10 added `TRIGGERS`. + func triggerList(schema: String, table: String?) async throws -> [PluginTriggerInfo] { + guard holds(MySQLServerVersion.hasTriggerCatalog(banner:flavor:)) else { return [] } + let result = try await run(MySQLObjectQueries.triggerList(schema: schema, table: table)) + let flavor = self.flavor + return result.rows.compactMap { row -> PluginTriggerInfo? in + guard let name = row[safe: 0]?.asText, + let owningTable = row[safe: 1]?.asText, + let timing = row[safe: 3]?.asText, + let event = row[safe: 4]?.asText, + let body = row[safe: 6]?.asText + else { return nil } + let resolvedSchema = row[safe: 2]?.asText ?? schema + let orientation = row[safe: 5]?.asText + let condition = row[safe: 7]?.asText + let definer = row[safe: 8]?.asText + // A condition that is not one expression could end the statement it is assembled into. + let assembles = condition.map { $0.isEmpty || mysqlIsSingleExpression($0, flavor: flavor) } ?? true + let header = MySQLObjectQueries.triggerStatement( + name: name, + table: owningTable, + schema: resolvedSchema, + timing: timing, + event: event, + orientation: orientation, + condition: condition, + definer: definer + ) + var attributes: [PluginObjectAttribute] = [] + if let definer, !definer.isEmpty { + attributes.append(PluginObjectAttribute(label: "Definer", value: definer)) + } + if let order = row[safe: 9]?.asText, !order.isEmpty, order != "0" { + attributes.append(PluginObjectAttribute(label: "Action Order", value: order)) + } + return PluginTriggerInfo( + name: name, + table: owningTable, + schema: resolvedSchema, + timing: timing, + event: event, + orientation: orientation, + statement: body, + definition: assembles ? "\(header)\n\(body)" : nil, + enabled: nil, + attributes: attributes + ) + } + } + + func dropTriggerStatement(name: String, schema: String?) -> String { + MySQLObjectQueries.dropTrigger(schema: effectiveSchema(schema).nilIfEmpty, name: name, quote: quote) + } + + // MARK: - Events + + // `information_schema.EVENTS` lists them, and `SHOW CREATE EVENT` is the only runnable definition. + func events(schema: String?) async throws -> [PluginEventInfo] { + guard flavor.hasMySQLObjectCatalogs, + holds(MySQLServerVersion.hasPartitionAndEventCatalogs(banner:flavor:)) + else { return [] } + let exactSchema = await schemaNamesAreCaseSensitive() + let result = try await run(MySQLObjectQueries.eventList(schema: effectiveSchema(schema), exactSchema: exactSchema)) + return result.rows.compactMap { row in + guard let name = row[safe: 0]?.asText else { return nil } + return PluginEventInfo( + name: name, + schema: row[safe: 3]?.asText, + kind: row[safe: 1]?.asText, + isEnabled: row[safe: 2]?.asText?.uppercased() == "ENABLED" + ) + } + } + + func eventDDL(_ event: PluginEventInfo) async throws -> String { + let result = try await run("SHOW CREATE EVENT \(qualifiedName(event.name, schema: event.schema))") + guard let row = result.rows.first, let ddl = row[safe: 3]?.asText else { + throw PluginObjectSourceError.unsupported(event.name) + } + return ddl + } + + // MARK: - Creation scopes + + func routineCreationScope(_ routine: PluginRoutineInfo) async throws -> PluginStatementScope? { + guard flavor.hasMySQLObjectCatalogs, hasInformationSchema else { return nil } + let exactSchema = await schemaNamesAreCaseSensitive() + let query = MySQLObjectQueries.routineCreationSettings( + schema: effectiveSchema(routine.schema), + name: routine.name, + kind: routine.kind == .procedure ? "PROCEDURE" : "FUNCTION", + exactSchema: exactSchema + ) + let row = try await creationSettingsRow(query, named: routine.name) + return Self.creationScope(sqlMode: row?[safe: 1]?.asText, timeZone: nil) + } + + func triggerCreationScope(_ trigger: PluginTriggerInfo) async throws -> PluginStatementScope? { + guard flavor.hasMySQLObjectCatalogs, holds(MySQLServerVersion.hasTriggerCatalog(banner:flavor:)) else { + return nil + } + let query = MySQLObjectQueries.triggerCreationSettings(schema: effectiveSchema(trigger.schema), name: trigger.name) + let row = try await creationSettingsRow(query, named: trigger.name) + return Self.creationScope(sqlMode: row?[safe: 1]?.asText, timeZone: nil) + } + + func eventCreationScope(_ event: PluginEventInfo) async throws -> PluginStatementScope? { + guard flavor.hasMySQLObjectCatalogs, holds(MySQLServerVersion.hasPartitionAndEventCatalogs(banner:flavor:)) else { + return nil + } + let exactSchema = await schemaNamesAreCaseSensitive() + let query = MySQLObjectQueries.eventCreationSettings( + schema: effectiveSchema(event.schema), + name: event.name, + exactSchema: exactSchema + ) + let row = try await creationSettingsRow(query, named: event.name) + return Self.creationScope(sqlMode: row?[safe: 1]?.asText, timeZone: row?[safe: 2]?.asText) + } + + // Nil when the catalog has no row for the object, or no recorded mode. + static func creationScope(sqlMode: String?, timeZone: String?) -> PluginStatementScope? { + guard let sqlMode, let settings = MySQLCreationSettings(sqlMode: sqlMode, timeZone: timeZone) else { return nil } + return PluginStatementScope(opening: settings.opening, closing: settings.closing) + } + + private func creationSettingsRow(_ query: String, named name: String) async throws -> [PluginCellValue]? { + let result = try await run(query) + return result.rows.first { $0[safe: 0]?.asText == name } + } + + // MARK: - Foreign keys + + // Two single-catalog reads merged here rather than one join: ShardingSphere-Proxy 5.5.3 answers any join of two + // `information_schema` tables with an OK packet and no columns, and each read on its own correctly. + func catalogForeignKeys(database: String, table: String?) async throws -> [String: [PluginForeignKeyInfo]] { + let columns = try await run(MySQLObjectQueries.foreignKeyColumns(schema: database, table: table)) + let columnRows = columns.rows.compactMap { row -> MySQLForeignKeyCatalog.ColumnRow? in + guard let tableName = row[safe: 0]?.asText, + let constraint = row[safe: 1]?.asText, + let column = row[safe: 2]?.asText, + let referencedTable = row[safe: 4]?.asText, + let referencedColumn = row[safe: 5]?.asText + else { return nil } + return MySQLForeignKeyCatalog.ColumnRow( + table: tableName, + constraint: constraint, + column: column, + referencedSchema: row[safe: 3]?.asText, + referencedTable: referencedTable, + referencedColumn: referencedColumn + ) + } + guard !columnRows.isEmpty else { return [:] } + + let actions = try await run(MySQLObjectQueries.referentialActions(schema: database, table: table)) + let actionRows = actions.rows.compactMap { row -> MySQLForeignKeyCatalog.ActionRow? in + guard let tableName = row[safe: 0]?.asText, + let constraint = row[safe: 1]?.asText + else { return nil } + return MySQLForeignKeyCatalog.ActionRow( + table: tableName, + constraint: constraint, + onDelete: row[safe: 2]?.asText ?? "", + onUpdate: row[safe: 3]?.asText ?? "" + ) + } + + return MySQLForeignKeyCatalog.group( + columnRows: columnRows, + actionRows: actionRows.filter { !$0.onDelete.isEmpty && !$0.onUpdate.isEmpty }, + defaultAction: omittedForeignKeyAction + ) + } + + // MARK: - Columns + + // `GENERATION_EXPRESSION` is projected here so the bulk read agrees with the per-table one on generated columns. + func informationSchemaColumns(schema: String?, table: String?) async throws -> [String: [PluginColumnInfo]] { + let query = MySQLObjectQueries.columnCatalog( + schema: effectiveSchema(schema), + table: table, + projectsGenerationExpression: holds(MySQLServerVersion.hasGenerationExpression(banner:flavor:)) + ) + let result = try await run(query) + let createTableClausesByTable = try await oceanbaseDefaultClausesByTable( + forRows: result.rows, tableColumn: 0, typeColumn: 2, defaultColumn: 6, schema: schema + ) + let createTableDefaultsByTable = try await createTableDefaultsByTable( + forCatalogRows: result.rows, schema: schema + ) + + var allColumns: [String: [PluginColumnInfo]] = [:] + for row in result.rows { + guard let tableName = row[safe: 0]?.asText, + let name = row[safe: 1]?.asText, + let dataType = row[safe: 2]?.asText + else { continue } + + let collation = row[safe: 3]?.asText + let isNullable = (row[safe: 4]?.asText) == "YES" + let isPrimaryKey = (row[safe: 5]?.asText) == "PRI" + let rawDefault = row[safe: 6]?.asText + let extra = row[safe: 7]?.asText + let comment = row[safe: 8]?.asText + + let charset: String? = { + guard let coll = collation, coll != "NULL" else { return nil } + return coll.components(separatedBy: "_").first + }() + + let upperType = dataType.uppercased() + let normalizedType = (upperType.hasPrefix("ENUM(") || upperType.hasPrefix("SET(")) + ? dataType : upperType + let allowedValues = EnumValueParser.parseMySQLEnumOrSet(from: normalizedType) + let defaultValue = columnDefaultValue( + catalogDefault: rawDefault, + extra: extra, + dataType: normalizedType, + isNullable: isNullable, + column: name, + createTableClauses: createTableClausesByTable[tableName], + createTableDefaults: createTableDefaultsByTable[tableName] + ) + + let column = PluginColumnInfo( + name: name, + dataType: normalizedType, + isNullable: isNullable, + isPrimaryKey: isPrimaryKey, + defaultValue: defaultValue, + extra: extra, + charset: charset, + collation: collation == "NULL" ? nil : collation, + comment: comment?.isEmpty == false ? comment : nil, + identityKind: mysqlIdentityKind(extra: extra), + isGenerated: mysqlColumnIsGenerated(extra: extra), + allowedValues: allowedValues, + generationExpression: row[safe: 9]?.asText?.nilIfEmpty, + generationKind: mysqlGenerationKind(extra: extra) + ) + + allColumns[tableName, default: []].append(column) + } + + return allColumns + } + + // Nil when the statement is refused, or does not describe a table: a view's leaves its columns on the catalog's + // answer. + func createTableDefaultClauses(table: String, schema: String?) async throws -> [String: String]? { + do { + let result = try await run("SHOW CREATE TABLE \(qualifiedName(table, schema: schema))") + guard let createTable = result.rows.first?[safe: 1]?.asText else { return nil } + return MySQLCreateTableScanner.columnDefaultClauses(fromCreateTable: createTable) + } catch let error where settlesBlindness(error) { + Self.logger.warning("create table default read refused: \(error.localizedDescription)") + return nil + } + } + + // One `SHOW CREATE TABLE` per table holding a default its catalog row cannot recreate. On a MariaDB from 10.2.7 + // the catalog rows are already exact. + private func createTableDefaultsByTable( + forCatalogRows rows: [[PluginCellValue]], + schema: String? + ) async throws -> [String: MySQLCreateTableDefaults] { + guard !flavor.isOceanBase else { return [:] } + let scope: MySQLCreateTableDefaults.Scope + let needsCreateTable: ([PluginCellValue]) -> Bool + if flavor.isMariaDB { + guard holds(MySQLServerVersion.mariaDBDefaultsCanBeExpressions(banner:flavor:)), + !holds(MySQLServerVersion.quotesColumnDefault(banner:flavor:)) + else { return [:] } + scope = .everyColumn + needsCreateTable = { row in + mariaDBBareDefaultMayBeExpression(row[safe: 6]?.asText, dataType: row[safe: 2]?.asText ?? "") + } + } else { + scope = .expressionDefaults + needsCreateTable = { row in + mysqlExpressionDefaultNeedsCreateTable( + row[safe: 6]?.asText, extra: row[safe: 7]?.asText, dataType: row[safe: 2]?.asText ?? "" + ) + } + } + let tables = Set(rows.filter(needsCreateTable).compactMap { $0[safe: 0]?.asText }) + var defaults: [String: MySQLCreateTableDefaults] = [:] + for table in tables.sorted() { + guard let clauses = try await createTableDefaultClauses(table: table, schema: schema) else { continue } + defaults[table] = MySQLCreateTableDefaults(clauses: clauses, scope: scope) + } + return defaults + } + + private func oceanbaseDefaultClausesByTable( + forRows rows: [[PluginCellValue]], + tableColumn: Int, + typeColumn: Int, + defaultColumn: Int, + schema: String? + ) async throws -> [String: [String: String]] { + guard flavor.isOceanBase else { return [:] } + let candidates = Set(rows.compactMap { row -> String? in + guard let table = row[safe: tableColumn]?.asText, + let dataType = row[safe: typeColumn]?.asText, + OceanBaseColumnDefaults.catalogDefaultNeedsCreateTable( + row[safe: defaultColumn]?.asText, dataType: dataType + ) + else { return nil } + return table + }) + guard !candidates.isEmpty else { return [:] } + var clausesByTable: [String: [String: String]] = [:] + for table in try await baseTableNames(among: candidates, schema: schema).sorted() { + clausesByTable[table] = try await oceanbaseDefaultClauses(table: table, schema: schema) + } + return clausesByTable + } + + private func columnDefaultValue( + catalogDefault: String?, + extra: String?, + dataType: String, + isNullable: Bool, + column: String, + createTableClauses: [String: String]?, + createTableDefaults: MySQLCreateTableDefaults? + ) -> String? { + guard flavor.isOceanBase else { + let quotesDefaults = holds(MySQLServerVersion.quotesColumnDefault(banner:flavor:)) + return mysqlColumnDefault( + createTableDefaults?.catalogDefault(forColumn: column, extra: extra) + ?? (quotesDefaults ? .quoted(catalogDefault) : .bare(catalogDefault)), + extra: extra, + dataType: dataType, + isNullable: isNullable, + banner: banner, + flavor: flavor + ) + } + if let catalogDefault, let createTableClauses, + OceanBaseColumnDefaults.catalogDefaultNeedsCreateTable(catalogDefault, dataType: dataType) { + let resolution = OceanBaseColumnDefaults.resolve( + clause: createTableClauses[column], catalogDefault: catalogDefault + ) + if case .value(let value) = resolution { + return value + } + Self.logger.warning( + "OceanBase default of \(column, privacy: .private(mask: .hash)) is not in SHOW CREATE TABLE as reported" + ) + } + return OceanBaseColumnDefaults.columnDefault( + catalogDefault, extra: extra, dataType: dataType, isNullable: isNullable + ) + } + + private func baseTableNames(among tables: Set, schema: String?) async throws -> Set { + let names = tables.sorted().map { "'\(mysqlEscapeStringLiteral($0))'" }.joined(separator: ", ") + let result = try await run(""" + SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = '\(mysqlEscapeStringLiteral(effectiveSchema(schema)))' + AND TABLE_TYPE = 'BASE TABLE' + AND TABLE_NAME IN (\(names)) + """) + return Set(result.rows.compactMap { $0[safe: 0]?.asText }) + } + + private func oceanbaseDefaultClauses(table: String, schema: String?) async throws -> [String: String] { + let result = try await run("SHOW CREATE TABLE \(qualifiedName(table, schema: schema))") + guard let createTable = result.rows.first?[safe: 1]?.asText else { return [:] } + return MySQLCreateTableScanner.columnDefaultClauses(fromCreateTable: createTable) ?? [:] + } + + // MARK: - Session scope + + // Databend has no `sql_mode` or foreign key checks to hold, and a server before 5.0.3 cannot filter `SHOW` with + // `WHERE`. + func openSessionScope(for purpose: PluginSessionPurpose, hooks: SessionHooks) async throws -> PluginSessionScope? { + let names = MySQLSessionScopePlan.savedNames(for: purpose) + guard flavor.hasMySQLObjectCatalogs, + holds(MySQLServerVersion.showAcceptsWhere(banner:flavor:)), + !names.isEmpty + else { return nil } + let saved = try await Self.readSessionSettings(names, hooks: hooks) + guard let applied = MySQLSessionScopePlan.applied(for: purpose, saved: saved) else { return nil } + guard !applied.isEmpty else { + return PluginSessionScope(purpose: purpose, saved: saved, applied: []) + } + let statements = applied.compactMap { MySQLSessionStatements.assignment(name: $0.name, value: $0.value) } + let savedNames = Set(saved.map(\.name)) + // A setting with no saved value could not be put back, so the scope sets none of them. + guard statements.count == applied.count, applied.allSatisfy({ savedNames.contains($0.name) }) else { + Self.logger.warning("The server did not report every setting a session scope sets; running without one") + return nil + } + // Held before the first `SET`, so the idle release cannot hand the session back between two of them. + hooks.adjustFootprint({ $0.holdSessionScope() }) + do { + for statement in statements { + _ = try await hooks.runSetup(statement) + } + } catch { + let isRestored = await Self.restoreAfterFailedOpen(saved, hooks: hooks) + hooks.adjustFootprint({ $0.releaseSessionScope(isRestored: isRestored) }) + throw error + } + return PluginSessionScope(purpose: purpose, saved: saved, applied: applied) + } + + func closeSessionScope(_ scope: PluginSessionScope, hooks: SessionHooks) async throws { + guard flavor.hasMySQLObjectCatalogs else { return } + let heldSession = !scope.applied.isEmpty + let isRestored: Bool + do { + isRestored = try await Self.restoreSessionSettings(scope.saved, hooks: hooks) + } catch { + Self.noteClosedScope(heldSession: heldSession, isRestored: false, hooks: hooks) + throw error + } + Self.noteClosedScope(heldSession: heldSession, isRestored: isRestored, hooks: hooks) + } + + private static func noteClosedScope(heldSession: Bool, isRestored: Bool, hooks: SessionHooks) { + hooks.adjustFootprint({ footprint in + if heldSession { + footprint.releaseSessionScope(isRestored: isRestored) + } else if !isRestored { + footprint.noteUnrestoredSessionSettings() + } + }) + } + + private static func readSessionSettings(_ names: [String], hooks: SessionHooks) async throws -> [PluginSessionSetting] { + guard let query = MySQLSessionStatements.captureQuery(names: names) else { return [] } + let result = try await hooks.runSetup(query) + return MySQLSessionScopePlan.captured(rows: result.rows, names: names) + } + + // Every statement is tried, so one refused value does not leave the others changed, and the first refusal is + // what the caller hears. False when a setting had to stay changed. Any state but `.idle`, `.unknown` included, + // keeps `autocommit` from committing a transaction that may be the user's. + private static func restoreSessionSettings(_ saved: [PluginSessionSetting], hooks: SessionHooks) async throws -> Bool { + guard !saved.isEmpty else { return true } + let current = try await readSessionSettings(saved.map(\.name), hooks: hooks) + let isTransactionOpen = await hooks.transactionState() != .idle + let restore = MySQLSessionScopePlan.restore(saved: saved, current: current, isTransactionOpen: isTransactionOpen) + if !restore.isComplete { + logger.info("Left autocommit as the imported file set it, because the session holds an open transaction") + } + var firstFailure: Error? + for statement in restore.statements { + do { + _ = try await hooks.runSetup(statement) + } catch { + logger.error( + "Could not put a session setting back with \(statement, privacy: .private): \(error.localizedDescription)" + ) + firstFailure = firstFailure ?? error + } + } + if let firstFailure { + throw firstFailure + } + return restore.isComplete + } + + private static func restoreAfterFailedOpen(_ saved: [PluginSessionSetting], hooks: SessionHooks) async -> Bool { + do { + return try await restoreSessionSettings(saved, hooks: hooks) + } catch { + logger.error("A session scope that failed to open could not put its settings back") + return false + } + } + + // MARK: - Principals and grants + + func principals() async throws -> [PluginPrincipalInfo] { + let readsLimit = !flavor.isTiDB && holds(MySQLServerVersion.hasUserConnectionLimit(banner:flavor:)) + let query = readsLimit + ? "SELECT User, Host, max_user_connections FROM mysql.user ORDER BY User, Host" + : "SELECT User, Host FROM mysql.user ORDER BY User, Host" + let result = try await run(query) + + return result.rows.compactMap { row -> PluginPrincipalInfo? in + guard let name = row[safe: 0]?.asText else { return nil } + let host = row[safe: 1]?.asText ?? Self.defaultHost + let limit = row[safe: 2]?.asText.flatMap(Int.init) + + return PluginPrincipalInfo( + ref: PluginPrincipalRef(name: name, host: host), + isRole: false, + canLogin: true, + connectionLimit: (limit ?? 0) == 0 ? nil : limit + ) + } + } + + func privilegeCatalog() async throws -> PluginPrivilegeCatalog { + let result = try await run("SHOW PRIVILEGES") + + var server: [PluginPrivilegeDescriptor] = [] + var database: [PluginPrivilegeDescriptor] = [] + var table: [PluginPrivilegeDescriptor] = [] + var column: [PluginPrivilegeDescriptor] = [] + var hasDynamicPrivileges = false + + for row in result.rows { + guard let rawName = row[safe: 0]?.asText, + let name = PluginPrivilegeName.sanitized(rawName), + !Self.excludedPrivileges.contains(name) else { continue } + + let isDynamic = MySQLPrivilegeCatalog.isDynamic(name) + hasDynamicPrivileges = hasDynamicPrivileges || isDynamic + + let descriptor = PluginPrivilegeDescriptor( + name: name, + label: rawName, + category: MySQLPrivilegeCatalog.category(for: name) + ) + server.append(descriptor) + + guard !isDynamic else { continue } + let context = (row[safe: 1]?.asText ?? "").uppercased() + + if Self.databaseContextMarkers.contains(where: { context.contains($0) }) { + database.append(descriptor) + } + if Self.tableContextMarkers.contains(where: { context.contains($0) }) { + table.append(descriptor) + } + if Self.columnGrantablePrivileges.contains(name) { + column.append(descriptor) + } + } + + return PluginPrivilegeCatalog( + serverPrivileges: server, + databasePrivileges: database, + schemaPrivileges: [], + tablePrivileges: table, + columnPrivileges: column, + supportsDynamicPrivileges: hasDynamicPrivileges + ) + } + + func grants(for principal: PluginPrincipalRef, privileges: PluginPrivilegeCatalog) async throws -> [PluginGrantInfo] { + let result = try await run("SHOW GRANTS FOR \(Self.grantAccount(principal, quote: quote))") + return result.rows.flatMap { row -> [PluginGrantInfo] in + guard let line = row[safe: 0]?.asText, + let parsed = MySQLGrantParser.parseGrant(line) else { return [] } + return Self.grants(from: parsed, catalog: privileges) + } + } + + func grantStatements(for principal: PluginPrincipalRef, grants: [PluginGrantInfo]) -> [String] { + Self.grantBuilder(for: principal, quote: quote).grantStatements(grants) + } + + static func grantBuilder( + for principal: PluginPrincipalRef, + quote: @escaping @Sendable (String) -> String + ) -> PluginGrantSQLBuilder { + PluginGrantSQLBuilder( + grantee: grantAccount(principal, quote: quote), + quoteIdentifier: quote, + target: { grantTarget(for: $0, quote: quote) } + ) + } + + static func grantAccount(_ principal: PluginPrincipalRef, quote: (String) -> String) -> String { + let name = quote(principal.name) + let host = quote(principal.host ?? defaultHost) + return "\(name)@\(host)" + } + + // MySQL treats `_` and `%` in the database position as LIKE wildcards only for global and database-level grants. + // In a table-level target the database name is a literal identifier, so escaping it there would grant on a + // database whose name contains a backslash. + static func grantTarget(for scope: PluginPrivilegeScope, quote: (String) -> String) -> String? { + switch scope { + case .server: + "*.*" + case let .database(name): + "\(quotedDatabasePattern(name, quote: quote)).*" + case let .schema(database, _): + "\(quotedDatabasePattern(database, quote: quote)).*" + case let .table(database, _, table): + "\(quote(database)).\(quote(table))" + case let .column(database, _, table, _): + "\(quote(database)).\(quote(table))" + @unknown default: + nil + } + } + + static func quotedDatabasePattern(_ name: String, quote: (String) -> String) -> String { + quote(MySQLGrantPatternEscaping.escapeDatabasePattern(name)) + } + + private static func grants( + from parsed: MySQLParsedGrant, + catalog: PluginPrivilegeCatalog + ) -> [PluginGrantInfo] { + parsed.privileges.flatMap { privilege -> [PluginGrantInfo] in + guard privilege.columns.isEmpty else { + return columnGrants(privilege, in: parsed) + } + let names = privilege.name == MySQLGrantParser.allPrivileges + ? catalog.privileges(for: parsed.scope).map(\.name) + : [privilege.name] + + return names.map { + PluginGrantInfo(privilege: $0, scope: parsed.scope, isGrantable: parsed.isGrantable) + } + } + } + + private static func columnGrants( + _ privilege: MySQLParsedPrivilege, + in parsed: MySQLParsedGrant + ) -> [PluginGrantInfo] { + guard let database = parsed.scope.databaseName, + let table = parsed.scope.tableName else { return [] } + + return privilege.columns.map { column in + PluginGrantInfo( + privilege: privilege.name, + scope: .column(database: database, schema: nil, table: table, column: column), + isGrantable: parsed.isGrantable + ) + } + } +} diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLForeignKeyCatalog.swift b/apple/Plugins/MySQLDriverPlugin/MySQLForeignKeyCatalog.swift index 7cf231e1a4..7ad71d9c09 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLForeignKeyCatalog.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLForeignKeyCatalog.swift @@ -8,7 +8,7 @@ import Foundation import TableProPluginKit -internal enum MySQLForeignKeyCatalog { +nonisolated internal enum MySQLForeignKeyCatalog { /// One `KEY_COLUMN_USAGE` row: a single column of a single foreign key. struct ColumnRow { let table: String diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+BulkMetadata.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+BulkMetadata.swift index 9ac9e58ffb..82c0670298 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+BulkMetadata.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+BulkMetadata.swift @@ -29,7 +29,7 @@ extension MySQLPluginDriver { /// would depend on how the driver rendered an integer cell. func fetchAllIndexes(schema: String?) async throws -> [String: [PluginIndexInfo]] { guard flavor.hasMySQLObjectCatalogs else { return [:] } - let database = routineSchema(schema) + let database = effectiveSchema(schema) return try await catalogOrShow( database: database, catalogExists: serverHasInformationSchema, @@ -79,9 +79,9 @@ extension MySQLPluginDriver { /// per-table read indexes into. The database is named rather than inherited from the session, /// so a caller asking about another one is answered about the one it asked about. func fetchAllTableMetadata(schema: String?) async throws -> [String: PluginTableMetadata] { - guard !flavor.isDatabend else { return try await databendAllTableMetadata(database: routineSchema(schema)) } - guard !flavor.isStarRocks else { return try await starrocksTableStatus(database: routineSchema(schema)) } - let result = try await execute(query: Self.tableStatusStatement(database: routineSchema(schema))) + guard !flavor.isDatabend else { return try await databendAllTableMetadata(database: effectiveSchema(schema)) } + guard !flavor.isStarRocks else { return try await starrocksTableStatus(database: effectiveSchema(schema)) } + let result = try await execute(query: Self.tableStatusStatement(database: effectiveSchema(schema))) let appendsInnoDBStatus = serverAppendsInnoDBStatus var metadata: [String: PluginTableMetadata] = [:] for row in result.rows { diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+CreationScopes.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+CreationScopes.swift index 59ef877757..f30988f79d 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+CreationScopes.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+CreationScopes.swift @@ -4,54 +4,18 @@ // import Foundation -import TableProMySQLCore import TableProPluginKit internal extension MySQLPluginDriver { func fetchRoutineCreationScope(_ routine: PluginRoutineInfo) async throws -> PluginStatementScope? { - guard flavor.hasMySQLObjectCatalogs, serverHasInformationSchema else { return nil } - let exactSchema = await schemaNamesAreCaseSensitive() - let query = MySQLObjectQueries.routineCreationSettings( - schema: routineSchema(routine.schema), - name: routine.name, - kind: routine.kind == .procedure ? "PROCEDURE" : "FUNCTION", - exactSchema: exactSchema - ) - let row = try await creationSettingsRow(query, named: routine.name) - return Self.creationScope(sqlMode: row?[safe: 1]?.asText, timeZone: nil) + try await exportCatalog.routineCreationScope(routine) } func fetchTriggerCreationScope(_ trigger: PluginTriggerInfo) async throws -> PluginStatementScope? { - guard flavor.hasMySQLObjectCatalogs, holdsForServer(MySQLServerVersion.hasTriggerCatalog(banner:flavor:)) else { - return nil - } - let query = MySQLObjectQueries.triggerCreationSettings(schema: routineSchema(trigger.schema), name: trigger.name) - let row = try await creationSettingsRow(query, named: trigger.name) - return Self.creationScope(sqlMode: row?[safe: 1]?.asText, timeZone: nil) + try await exportCatalog.triggerCreationScope(trigger) } func fetchEventCreationScope(_ event: PluginEventInfo) async throws -> PluginStatementScope? { - guard flavor.hasMySQLObjectCatalogs, holdsForServer(MySQLServerVersion.hasPartitionAndEventCatalogs(banner:flavor:)) else { - return nil - } - let exactSchema = await schemaNamesAreCaseSensitive() - let query = MySQLObjectQueries.eventCreationSettings( - schema: effectiveSchema(event.schema), - name: event.name, - exactSchema: exactSchema - ) - let row = try await creationSettingsRow(query, named: event.name) - return Self.creationScope(sqlMode: row?[safe: 1]?.asText, timeZone: row?[safe: 2]?.asText) - } - - /// Nil when the catalog has no row for the object, or no recorded mode. - static func creationScope(sqlMode: String?, timeZone: String?) -> PluginStatementScope? { - guard let sqlMode, let settings = MySQLCreationSettings(sqlMode: sqlMode, timeZone: timeZone) else { return nil } - return PluginStatementScope(opening: settings.opening, closing: settings.closing) - } - - private func creationSettingsRow(_ query: String, named name: String) async throws -> [PluginCellValue]? { - let result = try await execute(ownStatement: query) - return result.rows.first { $0[safe: 0]?.asText == name } + try await exportCatalog.eventCreationScope(event) } } diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+ForeignKeys.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+ForeignKeys.swift index e7c93cb4e6..5425480a0f 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+ForeignKeys.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+ForeignKeys.swift @@ -13,7 +13,7 @@ import TableProPluginKit extension MySQLPluginDriver { var providesBulkForeignKeyFetch: Bool { true } - var tableDDLIncludesForeignKeys: Bool { true } + var tableDDLIncludesForeignKeys: Bool { MySQLExportCatalog.tableDDLIncludesForeignKeys } func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { guard flavor.hasMySQLObjectCatalogs else { return [] } @@ -26,10 +26,11 @@ extension MySQLPluginDriver { guard await !resolvesToTemporaryTable(table, schema: schema) else { return try await ddlForeignKeys(table: table, database: database, omittedAction: omittedAction) } + let reader = exportCatalog let byTable = try await catalogOrShow( database: database, - catalogExists: hasForeignKeyCatalog, - catalog: { try await self.catalogForeignKeys(database: database, table: table) }, + catalogExists: reader.hasForeignKeyCatalog, + catalog: { try await reader.catalogForeignKeys(database: database, table: table) }, show: { let keys = try await self.ddlForeignKeys( table: table, database: database, omittedAction: omittedAction @@ -43,69 +44,12 @@ extension MySQLPluginDriver { func fetchAllForeignKeys(schema: String?) async throws -> [String: [PluginForeignKeyInfo]] { guard flavor.hasMySQLObjectCatalogs else { return [:] } let database = effectiveSchema(schema) + let reader = exportCatalog return try await catalogOrShow( database: database, - catalogExists: hasForeignKeyCatalog, - catalog: { try await self.catalogForeignKeys(database: database, table: nil) }, + catalogExists: reader.hasForeignKeyCatalog, + catalog: { try await reader.catalogForeignKeys(database: database, table: nil) }, show: { try await self.showForeignKeysByTable(database: database) } ) } - - /// The catalog read needs `REFERENTIAL_CONSTRAINTS` as well as `KEY_COLUMN_USAGE`, and 5.0 has - /// only the second: measured `1109` on 5.0.96. - private var hasForeignKeyCatalog: Bool { - serverHasInformationSchema - && holdsForServer(MySQLServerVersion.hasReferentialConstraintsCatalog(banner:flavor:)) - } - - /// Two single-catalog reads merged here rather than one join run by the server. - /// - /// The join answered nothing at all through ShardingSphere-Proxy 5.5.3, which returns an OK - /// packet with no columns for any join of two `information_schema` tables while answering each - /// of these two reads with the same rows a direct server does. - private func catalogForeignKeys( - database: String, - table: String? - ) async throws -> [String: [PluginForeignKeyInfo]] { - let columns = try await execute(ownStatement: MySQLObjectQueries.foreignKeyColumns(schema: database, table: table)) - let columnRows = columns.rows.compactMap { row -> MySQLForeignKeyCatalog.ColumnRow? in - guard let tableName = row[safe: 0]?.asText, - let constraint = row[safe: 1]?.asText, - let column = row[safe: 2]?.asText, - let referencedTable = row[safe: 4]?.asText, - let referencedColumn = row[safe: 5]?.asText - else { return nil } - return MySQLForeignKeyCatalog.ColumnRow( - table: tableName, - constraint: constraint, - column: column, - referencedSchema: row[safe: 3]?.asText, - referencedTable: referencedTable, - referencedColumn: referencedColumn - ) - } - guard !columnRows.isEmpty else { return [:] } - - let actions = try await execute(ownStatement: MySQLObjectQueries.referentialActions(schema: database, table: table)) - let actionRows = actions.rows.compactMap { row -> MySQLForeignKeyCatalog.ActionRow? in - guard let tableName = row[safe: 0]?.asText, - let constraint = row[safe: 1]?.asText - else { return nil } - return MySQLForeignKeyCatalog.ActionRow( - table: tableName, - constraint: constraint, - onDelete: row[safe: 2]?.asText ?? "", - onUpdate: row[safe: 3]?.asText ?? "" - ) - } - - return MySQLForeignKeyCatalog.group( - columnRows: columnRows, - actionRows: actionRows.filter { !$0.onDelete.isEmpty && !$0.onUpdate.isEmpty }, - defaultAction: MySQLServerVersion.omittedForeignKeyAction( - banner: serverIdentity.banner, - flavor: serverIdentity.flavor - ) - ) - } } diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+OceanBaseDefaults.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+OceanBaseDefaults.swift deleted file mode 100644 index dfbc4c8ed0..0000000000 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+OceanBaseDefaults.swift +++ /dev/null @@ -1,91 +0,0 @@ -// -// MySQLPluginDriver+OceanBaseDefaults.swift -// MySQLDriverPlugin -// - -import Foundation -import os -import TableProMySQLCore -import TableProPluginKit - -internal extension MySQLPluginDriver { - func oceanbaseDefaultClausesByTable( - forRows rows: [[PluginCellValue]], - tableColumn: Int, - typeColumn: Int, - defaultColumn: Int, - schema: String? - ) async throws -> [String: [String: String]] { - guard flavor.isOceanBase else { return [:] } - let candidates = Set(rows.compactMap { row -> String? in - guard let table = row[safe: tableColumn]?.asText, - let dataType = row[safe: typeColumn]?.asText, - OceanBaseColumnDefaults.catalogDefaultNeedsCreateTable( - row[safe: defaultColumn]?.asText, dataType: dataType - ) - else { return nil } - return table - }) - guard !candidates.isEmpty else { return [:] } - var clausesByTable: [String: [String: String]] = [:] - for table in try await baseTableNames(among: candidates, schema: schema).sorted() { - clausesByTable[table] = try await oceanbaseDefaultClauses(table: table, schema: schema) - } - return clausesByTable - } - - func columnDefaultValue( - catalogDefault: String?, - extra: String?, - dataType: String, - isNullable: Bool, - column: String, - createTableClauses: [String: String]?, - createTableDefaults: MySQLCreateTableDefaults? - ) -> String? { - guard flavor.isOceanBase else { - let identity = serverIdentity - return mysqlColumnDefault( - createTableDefaults?.catalogDefault(forColumn: column, extra: extra) - ?? (catalogQuotesDefaults ? .quoted(catalogDefault) : .bare(catalogDefault)), - extra: extra, - dataType: dataType, - isNullable: isNullable, - banner: identity.banner, - flavor: identity.flavor - ) - } - if let catalogDefault, let createTableClauses, - OceanBaseColumnDefaults.catalogDefaultNeedsCreateTable(catalogDefault, dataType: dataType) { - let resolution = OceanBaseColumnDefaults.resolve( - clause: createTableClauses[column], catalogDefault: catalogDefault - ) - if case .value(let value) = resolution { - return value - } - Self.logger.warning( - "OceanBase default of \(column, privacy: .private(mask: .hash)) is not in SHOW CREATE TABLE as reported" - ) - } - return OceanBaseColumnDefaults.columnDefault( - catalogDefault, extra: extra, dataType: dataType, isNullable: isNullable - ) - } - - private func baseTableNames(among tables: Set, schema: String?) async throws -> Set { - let names = tables.sorted().map { "'\(mysqlEscapeStringLiteral($0))'" }.joined(separator: ", ") - let result = try await execute(ownStatement: """ - SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES - WHERE TABLE_SCHEMA = '\(effectiveSchemaLiteral(schema))' - AND TABLE_TYPE = 'BASE TABLE' - AND TABLE_NAME IN (\(names)) - """) - return Set(result.rows.compactMap { $0[safe: 0]?.asText }) - } - - private func oceanbaseDefaultClauses(table: String, schema: String?) async throws -> [String: String] { - let result = try await execute(query: "SHOW CREATE TABLE \(qualifiedName(table, schema: schema))") - guard let createTable = result.rows.first?[safe: 1]?.asText else { return [:] } - return MySQLCreateTableScanner.columnDefaultClauses(fromCreateTable: createTable) ?? [:] - } -} diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+PrincipalSQL.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+PrincipalSQL.swift index 4123454c53..8ae6c8d993 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+PrincipalSQL.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+PrincipalSQL.swift @@ -48,10 +48,6 @@ extension MySQLPluginDriver { } private func grantBuilder(for principal: PluginPrincipalRef) -> PluginGrantSQLBuilder { - PluginGrantSQLBuilder( - grantee: grantAccount(principal), - quoteIdentifier: { self.quoteIdentifier($0) }, - target: { self.grantTarget(for: $0) } - ) + MySQLExportCatalog.grantBuilder(for: principal, quote: { [self] in self.quoteIdentifier($0) }) } } diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Principals.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Principals.swift index caa60ecbd9..d0444b37ef 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Principals.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Principals.swift @@ -15,18 +15,6 @@ extension MySQLPluginDriver: PluginPrincipalManagement { var supportsGrantableScopeSearch: Bool { true } var rollsBackPrincipalStatements: Bool { false } - static let defaultHost = "%" - - private static let excludedPrivileges: Set = ["GRANT OPTION", "PROXY"] - - private static let tableContextMarkers = ["TABLE", "INDEX", "VIEW", "TRIGGER"] - private static let databaseContextMarkers = [ - "DATABASE", "TABLE", "INDEX", "VIEW", "TRIGGER", "EVENT", "FUNCTION", "PROCEDURE" - ] - private static let columnGrantablePrivileges: Set = [ - "SELECT", "INSERT", "UPDATE", "REFERENCES" - ] - func privilegeCascades( from ancestor: PluginPrivilegeScope, to descendant: PluginPrivilegeScope @@ -35,92 +23,21 @@ extension MySQLPluginDriver: PluginPrincipalManagement { } func fetchPrincipals() async throws -> [PluginPrincipalInfo] { - let readsLimit = !flavor.isTiDB && holdsForServer(MySQLServerVersion.hasUserConnectionLimit(banner:flavor:)) - let query = readsLimit - ? "SELECT User, Host, max_user_connections FROM mysql.user ORDER BY User, Host" - : "SELECT User, Host FROM mysql.user ORDER BY User, Host" - let result = try await execute(query: query) - - return result.rows.compactMap { row -> PluginPrincipalInfo? in - guard let name = row[safe: 0]?.asText else { return nil } - let host = row[safe: 1]?.asText ?? Self.defaultHost - let limit = row[safe: 2]?.asText.flatMap(Int.init) - - return PluginPrincipalInfo( - ref: PluginPrincipalRef(name: name, host: host), - isRole: false, - canLogin: true, - connectionLimit: (limit ?? 0) == 0 ? nil : limit - ) - } + try await exportCatalog.principals() } func fetchPrivilegeCatalog() async throws -> PluginPrivilegeCatalog { if let cached = cachedPrivilegeCatalog { return cached } - let catalog = try await loadPrivilegeCatalog() + let catalog = try await exportCatalog.privilegeCatalog() cachedPrivilegeCatalog = catalog return catalog } - private func loadPrivilegeCatalog() async throws -> PluginPrivilegeCatalog { - let result = try await execute(query: "SHOW PRIVILEGES") - - var server: [PluginPrivilegeDescriptor] = [] - var database: [PluginPrivilegeDescriptor] = [] - var table: [PluginPrivilegeDescriptor] = [] - var column: [PluginPrivilegeDescriptor] = [] - var hasDynamicPrivileges = false - - for row in result.rows { - guard let rawName = row[safe: 0]?.asText, - let name = PluginPrivilegeName.sanitized(rawName), - !Self.excludedPrivileges.contains(name) else { continue } - - let isDynamic = MySQLPrivilegeCatalog.isDynamic(name) - hasDynamicPrivileges = hasDynamicPrivileges || isDynamic - - let descriptor = PluginPrivilegeDescriptor( - name: name, - label: rawName, - category: MySQLPrivilegeCatalog.category(for: name) - ) - server.append(descriptor) - - guard !isDynamic else { continue } - let context = (row[safe: 1]?.asText ?? "").uppercased() - - if Self.databaseContextMarkers.contains(where: { context.contains($0) }) { - database.append(descriptor) - } - if Self.tableContextMarkers.contains(where: { context.contains($0) }) { - table.append(descriptor) - } - if Self.columnGrantablePrivileges.contains(name) { - column.append(descriptor) - } - } - - return PluginPrivilegeCatalog( - serverPrivileges: server, - databasePrivileges: database, - schemaPrivileges: [], - tablePrivileges: table, - columnPrivileges: column, - supportsDynamicPrivileges: hasDynamicPrivileges - ) - } - func fetchGrants(for principal: PluginPrincipalRef) async throws -> [PluginGrantInfo] { let catalog = try await fetchPrivilegeCatalog() - let result = try await execute(query: "SHOW GRANTS FOR \(grantAccount(principal))") - - return result.rows.flatMap { row -> [PluginGrantInfo] in - guard let line = row[safe: 0]?.asText, - let parsed = MySQLGrantParser.parseGrant(line) else { return [] } - return grants(from: parsed, catalog: catalog) - } + return try await exportCatalog.grants(for: principal, privileges: catalog) } func fetchGrantableChildren(of scope: PluginPrivilegeScope) async throws -> [PluginPrivilegeScope] { @@ -168,7 +85,7 @@ extension MySQLPluginDriver: PluginPrincipalManagement { let parts = value.split(separator: "@", maxSplits: 1, omittingEmptySubsequences: false) guard let name = parts.first else { return nil } - let host = parts.count > 1 ? String(parts[1]) : Self.defaultHost + let host = parts.count > 1 ? String(parts[1]) : MySQLExportCatalog.defaultHost return PluginPrincipalRef( name: MySQLGrantParser.unquoteIdentifier(String(name)), @@ -193,67 +110,11 @@ extension MySQLPluginDriver: PluginPrincipalManagement { } } - private func grants( - from parsed: MySQLParsedGrant, - catalog: PluginPrivilegeCatalog - ) -> [PluginGrantInfo] { - parsed.privileges.flatMap { privilege -> [PluginGrantInfo] in - guard privilege.columns.isEmpty else { - return columnGrants(privilege, in: parsed) - } - let names = privilege.name == MySQLGrantParser.allPrivileges - ? catalog.privileges(for: parsed.scope).map(\.name) - : [privilege.name] - - return names.map { - PluginGrantInfo(privilege: $0, scope: parsed.scope, isGrantable: parsed.isGrantable) - } - } - } - - private func columnGrants( - _ privilege: MySQLParsedPrivilege, - in parsed: MySQLParsedGrant - ) -> [PluginGrantInfo] { - guard let database = parsed.scope.databaseName, - let table = parsed.scope.tableName else { return [] } - - return privilege.columns.map { column in - PluginGrantInfo( - privilege: privilege.name, - scope: .column(database: database, schema: nil, table: table, column: column), - isGrantable: parsed.isGrantable - ) - } - } - func grantAccount(_ principal: PluginPrincipalRef) -> String { - let name = quoteIdentifier(principal.name) - let host = quoteIdentifier(principal.host ?? Self.defaultHost) - return "\(name)@\(host)" + MySQLExportCatalog.grantAccount(principal, quote: quoteIdentifier) } - // MySQL treats `_` and `%` in the database position as LIKE wildcards only for global and - // database-level grants. In a table-level target the database name is a literal identifier, so - // escaping it there would grant on a database whose name contains a backslash. func grantTarget(for scope: PluginPrivilegeScope) -> String? { - switch scope { - case .server: - "*.*" - case let .database(name): - "\(quotedDatabasePattern(name)).*" - case let .schema(database, _): - "\(quotedDatabasePattern(database)).*" - case let .table(database, _, table): - "\(quoteIdentifier(database)).\(quoteIdentifier(table))" - case let .column(database, _, table, _): - "\(quoteIdentifier(database)).\(quoteIdentifier(table))" - @unknown default: - nil - } - } - - func quotedDatabasePattern(_ name: String) -> String { - quoteIdentifier(MySQLGrantPatternEscaping.escapeDatabasePattern(name)) + MySQLExportCatalog.grantTarget(for: scope, quote: quoteIdentifier) } } diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Routines.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Routines.swift index 3da8e395c0..6607ee3595 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Routines.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Routines.swift @@ -4,165 +4,28 @@ // import Foundation -import TableProMySQLCore import TableProPluginKit extension MySQLPluginDriver { - /// The routines and their parameters are two reads; `MySQLRoutineSignatures` says why. func fetchRoutines(schema: String?) async throws -> [PluginRoutineInfo] { - guard flavor.hasMySQLObjectCatalogs, serverHasInformationSchema else { return [] } - let resolvedSchema = routineSchema(schema) - let exactSchema = await schemaNamesAreCaseSensitive() - let query = MySQLObjectQueries.routineList(schema: resolvedSchema, exactSchema: exactSchema) - let result = try await execute(ownStatement: query) - let signatures = result.rows.isEmpty - ? nil - : try await routineSignatures(schema: resolvedSchema, exactSchema: exactSchema) - return result.rows.compactMap { row -> PluginRoutineInfo? in - guard let name = row[safe: 0]?.asText else { return nil } - let type = row[safe: 1]?.asText ?? "" - let isProcedure = type.uppercased() == "PROCEDURE" - let owner = row[safe: 7]?.asText ?? resolvedSchema - var attributes: [PluginObjectAttribute] = [] - if let access = row[safe: 3]?.asText, !access.isEmpty { - attributes.append(PluginObjectAttribute(label: "Data Access", value: access)) - } - if let deterministic = row[safe: 4]?.asText, !deterministic.isEmpty { - attributes.append(PluginObjectAttribute(label: "Deterministic", value: deterministic)) - } - if let security = row[safe: 5]?.asText, !security.isEmpty { - attributes.append(PluginObjectAttribute(label: "Security", value: security)) - } - if let definer = row[safe: 6]?.asText, !definer.isEmpty { - attributes.append(PluginObjectAttribute(label: "Definer", value: definer)) - } - return PluginRoutineInfo( - name: name, - kind: isProcedure ? .procedure : .function, - schema: owner, - returnType: isProcedure ? nil : row[safe: 2]?.asText, - language: "SQL", - argumentSignature: signatures?.signature(schema: owner, type: type, routine: name), - identity: nil, - attributes: attributes - ) - } + try await exportCatalog.routines(schema: schema) } - /// Nil before 5.5.3, which has no `PARAMETERS` table, so a signature there is unknown rather - /// than empty. - private func routineSignatures(schema: String, exactSchema: Bool) async throws -> MySQLRoutineSignatures? { - guard holdsForServer(MySQLServerVersion.hasParametersCatalog(banner:flavor:)) else { return nil } - let query = MySQLObjectQueries.routineParameters(schema: schema, exactSchema: exactSchema) - let result = try await execute(ownStatement: query) - var signatures = MySQLRoutineSignatures() - for row in result.rows { - guard let owner = row[safe: 0]?.asText, - let type = row[safe: 1]?.asText, - let routine = row[safe: 2]?.asText - else { continue } - signatures.append( - schema: owner, - type: type, - routine: routine, - mode: row[safe: 3]?.asText, - name: row[safe: 4]?.asText, - dataType: row[safe: 5]?.asText - ) - } - return signatures + func fetchRoutineDDL(_ routine: PluginRoutineInfo) async throws -> String { + try await exportCatalog.routineDDL(routine) } - func fetchRoutineDDL(_ routine: PluginRoutineInfo) async throws -> String { - let resolvedSchema = routineSchema(routine.schema) - let kind = routine.kind == .procedure ? "PROCEDURE" : "FUNCTION" - let query = MySQLObjectQueries.routineDefinition( - kind: kind, - schema: resolvedSchema, - name: routine.name - ) - let result = try await execute(ownStatement: query) - guard let row = result.rows.first else { - throw PluginObjectSourceError.notFound(routine.name) - } - /// The server returns a NULL body rather than an error when the account lacks SHOW_ROUTINE, - /// so reporting this as "not found" would tell the user their routine is gone. - guard let ddl = row[safe: 2]?.asText, !ddl.isEmpty else { - throw PluginObjectSourceError.insufficientPrivilege(routine.name) - } - return ddl + func generateDropRoutineSQL(name: String, signature: String?, schema: String?, isFunction: Bool) -> String? { + exportCatalog.dropRoutineStatement(name: name, schema: schema, isFunction: isFunction) } var providesBulkTriggerFetch: Bool { true } func fetchAllTriggers(schema: String?) async throws -> [PluginTriggerInfo] { - guard flavor.hasMySQLObjectCatalogs else { return [] } - return try await triggerList(schema: routineSchema(schema), table: nil) + try await exportCatalog.allTriggers(schema: schema) } func fetchTriggerDDL(_ trigger: PluginTriggerInfo) async throws -> String { - if let definition = trigger.definition, !definition.isEmpty { return definition } - let listed = try await triggerList(schema: routineSchema(trigger.schema), table: trigger.table) - guard let match = listed.first(where: { $0.name == trigger.name }) else { - throw PluginObjectSourceError.notFound(trigger.name) - } - guard let definition = match.definition, !definition.isEmpty else { - throw PluginObjectSourceError.unsupported(trigger.name) - } - return definition - } - - // The sidebar list, the per-table read and the DDL lookup all come through here. 5.0.10 added `TRIGGERS`. - func triggerList(schema: String, table: String?) async throws -> [PluginTriggerInfo] { - guard holdsForServer(MySQLServerVersion.hasTriggerCatalog(banner:flavor:)) else { return [] } - let result = try await execute(ownStatement: MySQLObjectQueries.triggerList(schema: schema, table: table)) - let flavor = self.flavor - return result.rows.compactMap { row -> PluginTriggerInfo? in - guard let name = row[safe: 0]?.asText, - let owningTable = row[safe: 1]?.asText, - let timing = row[safe: 3]?.asText, - let event = row[safe: 4]?.asText, - let body = row[safe: 6]?.asText - else { return nil } - let resolvedSchema = row[safe: 2]?.asText ?? schema - let orientation = row[safe: 5]?.asText - let condition = row[safe: 7]?.asText - let definer = row[safe: 8]?.asText - // A condition that is not one expression could end the statement it is assembled into. - let assembles = condition.map { $0.isEmpty || mysqlIsSingleExpression($0, flavor: flavor) } ?? true - let header = MySQLObjectQueries.triggerStatement( - name: name, - table: owningTable, - schema: resolvedSchema, - timing: timing, - event: event, - orientation: orientation, - condition: condition, - definer: definer - ) - var attributes: [PluginObjectAttribute] = [] - if let definer, !definer.isEmpty { - attributes.append(PluginObjectAttribute(label: "Definer", value: definer)) - } - if let order = row[safe: 9]?.asText, !order.isEmpty, order != "0" { - attributes.append(PluginObjectAttribute(label: "Action Order", value: order)) - } - return PluginTriggerInfo( - name: name, - table: owningTable, - schema: resolvedSchema, - timing: timing, - event: event, - orientation: orientation, - statement: body, - definition: assembles ? "\(header)\n\(body)" : nil, - enabled: nil, - attributes: attributes - ) - } - } - - func routineSchema(_ schema: String?) -> String { - effectiveSchema(schema) + try await exportCatalog.triggerDDL(trigger) } } diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Schema.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Schema.swift index 6ba43b5c93..670b742b14 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Schema.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Schema.swift @@ -51,7 +51,7 @@ internal extension MySQLPluginDriver { return try await databendColumns(table: table, schema: schema) } guard !flavor.isOceanBase else { - let columnsByTable = try await informationSchemaColumns(schema: schema, table: table) + let columnsByTable = try await exportCatalog.informationSchemaColumns(schema: schema, table: table) return columnsByTable[table] ?? (columnsByTable.count == 1 ? columnsByTable.values.first ?? [] : []) } guard !flavor.isStarRocks else { @@ -202,63 +202,11 @@ internal extension MySQLPluginDriver { catalogDetails: [String: MySQLCatalogColumnDetail] ) async throws -> MySQLCreateTableDefaults? { guard let scope = createTableDefaultsScope(showRows: showRows, catalogDetails: catalogDetails), - let clauses = try await createTableDefaultClauses(table: table, schema: schema) + let clauses = try await exportCatalog.createTableDefaultClauses(table: table, schema: schema) else { return nil } return MySQLCreateTableDefaults(clauses: clauses, scope: scope) } - /// The whole-schema read's counterpart of `createTableDefaults`: one `SHOW CREATE TABLE` for each - /// table holding a default its catalog row cannot recreate, and none for the rest. The rows are - /// `INFORMATION_SCHEMA.COLUMNS` rows, which on a MariaDB from 10.2.7 are already exact. - private func createTableDefaultsByTable( - forCatalogRows rows: [[PluginCellValue]], - schema: String? - ) async throws -> [String: MySQLCreateTableDefaults] { - let identity = serverIdentity - guard !identity.flavor.isOceanBase else { return [:] } - let scope: MySQLCreateTableDefaults.Scope - let needsCreateTable: ([PluginCellValue]) -> Bool - if identity.flavor.isMariaDB { - guard MySQLServerVersion.mariaDBDefaultsCanBeExpressions(banner: identity.banner, flavor: identity.flavor), - !MySQLServerVersion.quotesColumnDefault(banner: identity.banner, flavor: identity.flavor) - else { return [:] } - scope = .everyColumn - needsCreateTable = { row in - mariaDBBareDefaultMayBeExpression(row[safe: 6]?.asText, dataType: row[safe: 2]?.asText ?? "") - } - } else { - scope = .expressionDefaults - needsCreateTable = { row in - mysqlExpressionDefaultNeedsCreateTable( - row[safe: 6]?.asText, extra: row[safe: 7]?.asText, dataType: row[safe: 2]?.asText ?? "" - ) - } - } - let tables = Set(rows.filter(needsCreateTable).compactMap { $0[safe: 0]?.asText }) - var defaults: [String: MySQLCreateTableDefaults] = [:] - for table in tables.sorted() { - guard let clauses = try await createTableDefaultClauses(table: table, schema: schema) else { continue } - defaults[table] = MySQLCreateTableDefaults(clauses: clauses, scope: scope) - } - return defaults - } - - /// Nil when the statement is refused, or does not describe a table: a view's leaves its columns on - /// the catalog's answer. - private func createTableDefaultClauses(table: String, schema: String?) async throws -> [String: String]? { - do { - let result = try await execute(query: "SHOW CREATE TABLE \(qualifiedName(table, schema: schema))") - guard let createTable = result.rows.first?[safe: 1]?.asText else { return nil } - return MySQLCreateTableScanner.columnDefaultClauses(fromCreateTable: createTable) - } catch let error as MariaDBPluginError - where MySQLCatalogVisibilityRule.settlesBlindness(code: error.code) { - Self.logger.warning( - "create table default read refused code=\(error.code, privacy: .public) message=\(error.message)" - ) - return nil - } - } - private func createTableDefaultsScope( showRows: [[PluginCellValue]], catalogDetails: [String: MySQLCatalogColumnDetail] @@ -324,10 +272,6 @@ internal extension MySQLPluginDriver { // StarRocks answers the bulk read by loading every table's schema from the lake behind an external catalog. var providesBulkColumnFetch: Bool { !flavor.isStarRocks } - /// `GENERATION_EXPRESSION` is projected here rather than looked up per table, because a caller - /// that takes the bulk list has to receive what `fetchColumns` would have given it. Without the - /// column the two reads disagree on generated columns alone, and a schema comparison built on - /// the bulk read reports a changed generation expression as no difference at all. func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { guard !flavor.isDatabend else { return try await databendAllColumns(schema: schema) } let database = effectiveSchema(schema) @@ -335,95 +279,10 @@ internal extension MySQLPluginDriver { return try await catalogOrShow( database: database, catalogExists: serverHasInformationSchema, - catalog: { try await self.informationSchemaColumns(schema: schema, table: nil) }, + catalog: { try await self.exportCatalog.informationSchemaColumns(schema: schema, table: nil) }, show: { try await self.showColumnsByTable(database: database) } ) } - - private func informationSchemaColumns( - schema: String?, - table: String? - ) async throws -> [String: [PluginColumnInfo]] { - let escapedDb = effectiveSchemaLiteral(schema) - let tableFilter = table.map { " AND TABLE_NAME = '\(mysqlEscapeStringLiteral($0))'" } ?? "" - let identity = serverIdentity - let hasGenerationExpression = MySQLServerVersion.hasGenerationExpression( - banner: identity.banner, flavor: identity.flavor - ) - let generationProjection = hasGenerationExpression ? "GENERATION_EXPRESSION" : "NULL" - let query = """ - SELECT - TABLE_NAME, COLUMN_NAME, COLUMN_TYPE, COLLATION_NAME, - IS_NULLABLE, COLUMN_KEY, COLUMN_DEFAULT, EXTRA, COLUMN_COMMENT, - \(generationProjection) - FROM INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_SCHEMA = '\(escapedDb)'\(tableFilter) - ORDER BY TABLE_NAME, ORDINAL_POSITION - """ - - let result = try await execute(ownStatement: query) - let createTableClausesByTable = try await oceanbaseDefaultClausesByTable( - forRows: result.rows, tableColumn: 0, typeColumn: 2, defaultColumn: 6, schema: schema - ) - let createTableDefaultsByTable = try await createTableDefaultsByTable( - forCatalogRows: result.rows, schema: schema - ) - - var allColumns: [String: [PluginColumnInfo]] = [:] - for row in result.rows { - guard let tableName = row[safe: 0]?.asText, - let name = row[safe: 1]?.asText, - let dataType = row[safe: 2]?.asText - else { continue } - - let collation = row[safe: 3]?.asText - let isNullable = (row[safe: 4]?.asText) == "YES" - let isPrimaryKey = (row[safe: 5]?.asText) == "PRI" - let rawDefault = row[safe: 6]?.asText - let extra = row[safe: 7]?.asText - let comment = row[safe: 8]?.asText - - let charset: String? = { - guard let coll = collation, coll != "NULL" else { return nil } - return coll.components(separatedBy: "_").first - }() - - let upperType = dataType.uppercased() - let normalizedType = (upperType.hasPrefix("ENUM(") || upperType.hasPrefix("SET(")) - ? dataType : upperType - let allowedValues = EnumValueParser.parseMySQLEnumOrSet(from: normalizedType) - let defaultValue = columnDefaultValue( - catalogDefault: rawDefault, - extra: extra, - dataType: normalizedType, - isNullable: isNullable, - column: name, - createTableClauses: createTableClausesByTable[tableName], - createTableDefaults: createTableDefaultsByTable[tableName] - ) - - let column = PluginColumnInfo( - name: name, - dataType: normalizedType, - isNullable: isNullable, - isPrimaryKey: isPrimaryKey, - defaultValue: defaultValue, - extra: extra, - charset: charset, - collation: collation == "NULL" ? nil : collation, - comment: comment?.isEmpty == false ? comment : nil, - identityKind: mysqlIdentityKind(extra: extra), - isGenerated: mysqlColumnIsGenerated(extra: extra), - allowedValues: allowedValues, - generationExpression: row[safe: 9]?.asText?.nilIfEmpty, - generationKind: mysqlGenerationKind(extra: extra) - ) - - allColumns[tableName, default: []].append(column) - } - - return allColumns - } } /// What the catalog adds to one `SHOW FULL COLUMNS` row. `catalogDefault` is nil when the read did diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+ServerSupport.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+ServerSupport.swift index 91adb63525..5de901197d 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+ServerSupport.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+ServerSupport.swift @@ -29,16 +29,26 @@ internal extension MySQLPluginDriver { holdsForServer(MySQLServerVersion.appendsInnoDBStatusToComment(banner:flavor:)) } - /// Whether a catalog read has to compare the schema name byte for byte: true under - /// `lower_case_table_names = 0`. A server that will not answer keeps the plain comparison, for - /// this call only. + // A server that will not answer keeps the plain comparison, for this call only. func schemaNamesAreCaseSensitive() async -> Bool { if let known = cachedSchemaNamesAreCaseSensitive { return known } - guard let result = try? await execute(ownStatement: MySQLObjectQueries.lowerCaseTableNames), - let setting = result.rows.first?[safe: 0]?.asText.flatMap({ Int($0) }) - else { return false } - let sensitive = setting == 0 + let run: MySQLExportCatalog.Run = { [self] in try await self.execute(ownStatement: $0) } + guard let sensitive = await MySQLExportCatalog.schemaNameCaseSensitivity(run: run) else { return false } cachedSchemaNamesAreCaseSensitive = sensitive return sensitive } + + // Built per call, so each read sees the banner, flavor and database of the session it runs on. + var exportCatalog: MySQLExportCatalog { + let identity = serverIdentity + return MySQLExportCatalog( + banner: identity.banner, + flavor: identity.flavor, + activeDatabase: activeDatabaseName, + quote: { [self] in self.quoteIdentifier($0) }, + run: { [self] in try await self.execute(ownStatement: $0) }, + schemaNamesAreCaseSensitive: { [self] in await self.schemaNamesAreCaseSensitive() }, + settlesBlindness: { MySQLPluginDriver.settlesBlindness($0) } + ) + } } diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+SessionScope.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+SessionScope.swift index cb28b9b8ac..31867244f5 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+SessionScope.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+SessionScope.swift @@ -4,110 +4,22 @@ // import Foundation -import os -import TableProMySQLCore import TableProPluginKit internal extension MySQLPluginDriver { - /// Databend has no `sql_mode` or foreign key checks to hold, and a server before 5.0.3 cannot filter `SHOW` with - /// `WHERE`. func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { - let names = MySQLSessionScopePlan.savedNames(for: purpose) - guard flavor.hasMySQLObjectCatalogs, - holdsForServer(MySQLServerVersion.showAcceptsWhere(banner:flavor:)), - !names.isEmpty - else { return nil } - let saved = try await readSessionSettings(names) - guard let applied = MySQLSessionScopePlan.applied(for: purpose, saved: saved) else { return nil } - guard !applied.isEmpty else { - return PluginSessionScope(purpose: purpose, saved: saved, applied: []) - } - let statements = applied.compactMap { MySQLSessionStatements.assignment(name: $0.name, value: $0.value) } - let savedNames = Set(saved.map(\.name)) - /// A setting with no saved value could not be put back, so the scope sets none of them. - guard statements.count == applied.count, applied.allSatisfy({ savedNames.contains($0.name) }) else { - Self.logger.warning("The server did not report every setting a session scope sets; running without one") - return nil - } - /// Held before the first `SET`, so the idle release cannot hand the session back between two of them. - adjustFootprint { $0.holdSessionScope() } - do { - for statement in statements { - _ = try await executeSetup(statement) - } - } catch { - let isRestored = await restoreAfterFailedOpen(saved) - adjustFootprint { $0.releaseSessionScope(isRestored: isRestored) } - throw error - } - return PluginSessionScope(purpose: purpose, saved: saved, applied: applied) + try await exportCatalog.openSessionScope(for: purpose, hooks: sessionHooks) } func closeSessionScope(_ scope: PluginSessionScope) async throws { - guard flavor.hasMySQLObjectCatalogs else { return } - let heldSession = !scope.applied.isEmpty - let isRestored: Bool - do { - isRestored = try await restoreSessionSettings(scope.saved) - } catch { - noteClosedScope(heldSession: heldSession, isRestored: false) - throw error - } - noteClosedScope(heldSession: heldSession, isRestored: isRestored) + try await exportCatalog.closeSessionScope(scope, hooks: sessionHooks) } - private func noteClosedScope(heldSession: Bool, isRestored: Bool) { - adjustFootprint { footprint in - if heldSession { - footprint.releaseSessionScope(isRestored: isRestored) - } else if !isRestored { - footprint.noteUnrestoredSessionSettings() - } - } - } - - private func readSessionSettings(_ names: [String]) async throws -> [PluginSessionSetting] { - guard let query = MySQLSessionStatements.captureQuery(names: names) else { return [] } - let result = try await executeSetup(query) - return MySQLSessionScopePlan.captured(rows: result.rows, names: names) - } - - /// Every statement is tried, so one refused value does not leave the others changed, and the first refusal is what - /// the caller hears. False when a setting had to stay changed. - /// - /// The transaction state comes from the status flags of the `SHOW` just read, and anything but `.idle`, `.unknown` - /// included, keeps `autocommit` from committing a transaction that may be the user's. - private func restoreSessionSettings(_ saved: [PluginSessionSetting]) async throws -> Bool { - guard !saved.isEmpty else { return true } - let current = try await readSessionSettings(saved.map(\.name)) - let isTransactionOpen = await sessionTransactionState() != .idle - let restore = MySQLSessionScopePlan.restore(saved: saved, current: current, isTransactionOpen: isTransactionOpen) - if !restore.isComplete { - Self.logger.info("Left autocommit as the imported file set it, because the session holds an open transaction") - } - var firstFailure: Error? - for statement in restore.statements { - do { - _ = try await executeSetup(statement) - } catch { - Self.logger.error( - "Could not put a session setting back with \(statement, privacy: .private): \(error.localizedDescription)" - ) - firstFailure = firstFailure ?? error - } - } - if let firstFailure { - throw firstFailure - } - return restore.isComplete - } - - private func restoreAfterFailedOpen(_ saved: [PluginSessionSetting]) async -> Bool { - do { - return try await restoreSessionSettings(saved) - } catch { - Self.logger.error("A session scope that failed to open could not put its settings back") - return false - } + private var sessionHooks: MySQLExportCatalog.SessionHooks { + MySQLExportCatalog.SessionHooks( + runSetup: { [self] in try await self.executeSetup($0) }, + transactionState: { [self] in await self.sessionTransactionState() }, + adjustFootprint: { [self] change in self.adjustFootprint(change) } + ) } } diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift index 6da86b8e0d..f7e404b18a 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift @@ -905,7 +905,7 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // A temporary table cannot carry a trigger. guard flavor.hasMySQLObjectCatalogs, await !resolvesToTemporaryTable(table, schema: schema) else { return [] } let dbName = effectiveSchema(schema) - let triggers = try await triggerList(schema: dbName, table: table) + let triggers = try await exportCatalog.triggerList(schema: dbName, table: table) Self.logger.info("[trigger] mysql fetchTriggers db=\(dbName, privacy: .private(mask: .hash)) table=\(table, privacy: .private(mask: .hash)) parsed=\(triggers.count)") return triggers } @@ -922,7 +922,7 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func generateDropTriggerSQL(name: String, table: String, schema: String?) -> String? { - "DROP TRIGGER \(qualifiedName(name, schema: schema))" + exportCatalog.dropTriggerStatement(name: name, schema: schema) } func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { @@ -933,78 +933,29 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { guard serverHasInformationSchema else { return try await tableStatusRow(table: table, schema: schema)?[safe: 4]?.asText.flatMap { Int($0) } } - let escapedDb = effectiveSchemaLiteral(schema) - let escapedTable = mysqlEscapeStringLiteral(table) - - let query = """ - SELECT TABLE_ROWS - FROM information_schema.TABLES - WHERE TABLE_SCHEMA = '\(escapedDb)' - AND TABLE_NAME = '\(escapedTable)' - """ - - let result = try await execute(ownStatement: query) - guard let firstRow = result.rows.first, - let value = firstRow[safe: 0]?.asText, - let count = Int(value) - else { return nil } - - return count + return try await exportCatalog.approximateRowCount(table: table, schema: schema) } func fetchTableDDL(table: String, schema: String?) async throws -> String { - let result = try await execute(query: "SHOW CREATE TABLE \(qualifiedName(table, schema: schema))") - - guard let firstRow = result.rows.first, - let ddl = firstRow[safe: 1]?.asText - else { + guard let ddl = try await exportCatalog.tableDDL(table: table, schema: schema) else { throw MariaDBPluginError(code: 0, message: "Failed to fetch DDL for table '\(table)'", sqlState: nil) } - - return ddl.hasSuffix(";") ? ddl : ddl + ";" + return ddl } - /// Scheduled events. `information_schema.EVENTS` lists them for the current database, and - /// `SHOW CREATE EVENT` is the only thing that produces a runnable definition. func fetchEvents(schema: String?) async throws -> [PluginEventInfo] { - guard flavor.hasMySQLObjectCatalogs, - holdsForServer(MySQLServerVersion.hasPartitionAndEventCatalogs(banner:flavor:)) - else { return [] } - let exactSchema = await schemaNamesAreCaseSensitive() - let result = try await execute( - ownStatement: MySQLObjectQueries.eventList(schema: effectiveSchema(schema), exactSchema: exactSchema) - ) - return result.rows.compactMap { row in - guard let name = row[safe: 0]?.asText else { return nil } - return PluginEventInfo( - name: name, - schema: row[safe: 3]?.asText, - kind: row[safe: 1]?.asText, - isEnabled: row[safe: 2]?.asText?.uppercased() == "ENABLED" - ) - } + try await exportCatalog.events(schema: schema) } func fetchEventDDL(_ event: PluginEventInfo) async throws -> String { - let result = try await execute(query: "SHOW CREATE EVENT \(qualifiedName(event.name, schema: event.schema))") - guard let row = result.rows.first, let ddl = row[safe: 3]?.asText else { - throw PluginObjectSourceError.unsupported(event.name) - } - return ddl + try await exportCatalog.eventDDL(event) } func fetchViewDefinition(view: String, schema: String?) async throws -> String { guard !flavor.isDatabend else { return try await databendViewDefinition(view: view, schema: schema) } - // StarRocks's `SHOW CREATE VIEW` rewrites a materialized view as a plain `CREATE VIEW`. - let show = flavor.isStarRocks ? "SHOW CREATE TABLE" : "SHOW CREATE VIEW" - let result = try await execute(query: "\(show) \(qualifiedName(view, schema: schema))") - - guard let firstRow = result.rows.first, - let ddl = firstRow[safe: 1]?.asText - else { + guard let ddl = try await exportCatalog.viewDefinition(view: view, schema: schema) else { throw MariaDBPluginError(code: 0, message: "Failed to fetch definition for view '\(view)'", sqlState: nil) } - return ddl } @@ -1426,11 +1377,11 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Foreign Key Checks func foreignKeyDisableStatements() -> [String]? { - flavor.hasMySQLObjectCatalogs ? ["SET FOREIGN_KEY_CHECKS=0"] : nil + flavor.foreignKeyCheckStatements(enabled: false) } func foreignKeyEnableStatements() -> [String]? { - flavor.hasMySQLObjectCatalogs ? ["SET FOREIGN_KEY_CHECKS=1"] : nil + flavor.foreignKeyCheckStatements(enabled: true) } } diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPrivilegeCatalog.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPrivilegeCatalog.swift index 17750b4379..c713cca09d 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPrivilegeCatalog.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPrivilegeCatalog.swift @@ -11,7 +11,7 @@ import Foundation import TableProPluginKit -enum MySQLPrivilegeCatalog { +nonisolated enum MySQLPrivilegeCatalog { static let dataPrivileges: Set = [ "SELECT", "INSERT", "UPDATE", "DELETE", "EXECUTE", "SHOW VIEW", "LOCK TABLES" ] diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLServerFlavor.swift b/apple/Plugins/MySQLDriverPlugin/MySQLServerFlavor.swift index fc3db7ad3c..464105ad81 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLServerFlavor.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLServerFlavor.swift @@ -117,6 +117,11 @@ nonisolated internal enum MySQLServerFlavor: Equatable, Sendable { // No indexes, foreign keys, triggers, routines, events or partitions to read the MySQL way. var hasMySQLObjectCatalogs: Bool { !isDatabend && !isStarRocks } + // What a SQL file import runs around its statements; nil where the flavor has no foreign key checks. + func foreignKeyCheckStatements(enabled: Bool) -> [String]? { + hasMySQLObjectCatalogs ? ["SET FOREIGN_KEY_CHECKS=\(enabled ? 1 : 0)"] : nil + } + var tidbVersion: MySQLEngineVersion? { guard case .tidb(let version) = self else { return nil } return version diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLSessionScopePlan.swift b/apple/Plugins/MySQLDriverPlugin/MySQLSessionScopePlan.swift index 6bca4c79ea..a9c1301fc8 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLSessionScopePlan.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLSessionScopePlan.swift @@ -8,7 +8,7 @@ import TableProMySQLCore import TableProPluginKit /// Which session settings each purpose saves and sets on MySQL, and what putting them back takes. -internal enum MySQLSessionScopePlan { +nonisolated internal enum MySQLSessionScopePlan { /// What a mysqldump or mariadb-dump file sets and puts back only at its end, which an imported file leaves changed /// when it stops early. `SHOW` reports only the ones the server has: `note_verbosity` is MariaDB's. /// diff --git a/apple/Plugins/MySQLDriverTests/MySQLExportCatalogTests.swift b/apple/Plugins/MySQLDriverTests/MySQLExportCatalogTests.swift new file mode 100644 index 0000000000..5acae244bc --- /dev/null +++ b/apple/Plugins/MySQLDriverTests/MySQLExportCatalogTests.swift @@ -0,0 +1,378 @@ +import Foundation +import TableProMySQLCore +import TableProPluginKit +import Testing + +private actor CatalogServer { + private(set) var sent: [String] = [] + private var answers: [[[PluginCellValue]]] + + init(answers: [[[PluginCellValue]]] = []) { + self.answers = answers + } + + func answer(_ sql: String) -> PluginQueryResult { + sent.append(sql) + let rows = answers.isEmpty ? [] : answers.removeFirst() + return PluginQueryResult(columns: [], columnTypeNames: [], rows: rows, rowsAffected: 0, executionTime: 0) + } +} + +private final class FootprintBox: @unchecked Sendable { + private let lock = NSLock() + private var footprint = MySQLSessionFootprint() + + func adjust(_ change: (inout MySQLSessionFootprint) -> Void) { + lock.withLock { change(&footprint) } + } + + var heldSessionScopes: Int { + lock.withLock { footprint.heldSessionScopes } + } +} + +private func makeCatalog( + _ server: CatalogServer, + flavor: MySQLServerFlavor = .mysql, + banner: String? = "8.0.36", + activeDatabase: String = "shop", + exactSchema: Bool = false +) -> MySQLExportCatalog { + MySQLExportCatalog( + banner: banner, + flavor: flavor, + activeDatabase: activeDatabase, + quote: { mysqlQuoteIdentifier($0) }, + run: { await server.answer($0) }, + schemaNamesAreCaseSensitive: { exactSchema }, + settlesBlindness: { _ in false } + ) +} + +struct MySQLExportCatalogTests { + @Test("Routines are listed, then their parameters read once, and keep the signature as identity") + func routinesReadListThenParameters() async throws { + let server = CatalogServer(answers: [ + [ + ["total", "FUNCTION", "decimal(10,2)", "READS SQL DATA", "YES", "DEFINER", "root@%", "shop"], + ["cleanup", "PROCEDURE", nil, "", "NO", "INVOKER", "", "shop"], + ], + [ + ["shop", "FUNCTION", "total", nil, "order_id", "int"], + ["shop", "PROCEDURE", "cleanup", "IN", "days", "int"], + ], + ]) + + let routines = try await makeCatalog(server, exactSchema: true).routines(schema: nil) + + #expect(await server.sent == [ + MySQLObjectQueries.routineList(schema: "shop", exactSchema: true), + MySQLObjectQueries.routineParameters(schema: "shop", exactSchema: true), + ]) + #expect(routines.map(\.name) == ["total", "cleanup"]) + #expect(routines.map(\.kind) == [.function, .procedure]) + #expect(routines[0].returnType == "decimal(10,2)") + #expect(routines[1].returnType == nil) + #expect(routines[0].argumentSignature == "(order_id int)") + #expect(routines[1].argumentSignature == "(IN days int)") + #expect(routines[0].attributes.map(\.label) == ["Data Access", "Deterministic", "Security", "Definer"]) + #expect(routines[1].attributes.map(\.label) == ["Deterministic", "Security"]) + } + + @Test("A database with no routines reads no parameters") + func noRoutinesNoParameterRead() async throws { + let server = CatalogServer() + let routines = try await makeCatalog(server).routines(schema: "app") + #expect(routines.isEmpty) + #expect(await server.sent == [MySQLObjectQueries.routineList(schema: "app", exactSchema: false)]) + } + + @Test("A flavor without MySQL object catalogs lists no routines, triggers or events and sends nothing") + func databendSendsNothing() async throws { + let server = CatalogServer() + let catalog = makeCatalog(server, flavor: .databend) + #expect(try await catalog.routines(schema: nil).isEmpty) + #expect(try await catalog.allTriggers(schema: nil).isEmpty) + #expect(try await catalog.events(schema: nil).isEmpty) + #expect(await server.sent.isEmpty) + } + + @Test("A NULL routine body is reported as a missing privilege, not a missing routine") + func nullRoutineBodyIsPrivilege() async throws { + let server = CatalogServer(answers: [[["cleanup", "", nil]]]) + let routine = PluginRoutineInfo(name: "cleanup", kind: .procedure, schema: "app") + await #expect(throws: PluginObjectSourceError.self) { + try await makeCatalog(server).routineDDL(routine) + } + #expect(await server.sent == ["SHOW CREATE PROCEDURE `app`.`cleanup`"]) + } + + @Test("Table DDL is the second column of SHOW CREATE TABLE, ended with one semicolon") + func tableDDL() async throws { + let server = CatalogServer(answers: [ + [["orders", "CREATE TABLE `orders` (`id` int)"]], + [["orders", "CREATE TABLE `orders` (`id` int);"]], + [], + ]) + let catalog = makeCatalog(server) + #expect(try await catalog.tableDDL(table: "orders", schema: nil) == "CREATE TABLE `orders` (`id` int);") + #expect(try await catalog.tableDDL(table: "orders", schema: "app") == "CREATE TABLE `orders` (`id` int);") + #expect(try await catalog.tableDDL(table: "gone", schema: nil) == nil) + #expect(await server.sent == [ + "SHOW CREATE TABLE `shop`.`orders`", + "SHOW CREATE TABLE `app`.`orders`", + "SHOW CREATE TABLE `shop`.`gone`", + ]) + } + + @Test("With no database selected the DDL read names the table alone") + func unqualifiedWithoutDatabase() async throws { + let server = CatalogServer() + _ = try await makeCatalog(server, activeDatabase: "").tableDDL(table: "t", schema: nil) + #expect(await server.sent == ["SHOW CREATE TABLE `t`"]) + } + + @Test("A view reads SHOW CREATE VIEW, and StarRocks reads SHOW CREATE TABLE") + func viewDefinition() async throws { + let server = CatalogServer(answers: [[["v", "CREATE VIEW `v` AS SELECT 1"]], []]) + #expect(try await makeCatalog(server).viewDefinition(view: "v", schema: nil) == "CREATE VIEW `v` AS SELECT 1") + #expect(try await makeCatalog(server, flavor: .starrocks).viewDefinition(view: "v", schema: nil) == nil) + #expect(await server.sent == ["SHOW CREATE VIEW `shop`.`v`", "SHOW CREATE TABLE `shop`.`v`"]) + } + + @Test("A trigger row becomes a runnable CREATE TRIGGER with its definer and order") + func triggerList() async throws { + let server = CatalogServer(answers: [[ + ["audit", "orders", "shop", "BEFORE", "INSERT", "ROW", "SET NEW.at = NOW()", nil, "root@%", "2"], + ]]) + + let triggers = try await makeCatalog(server).allTriggers(schema: nil) + + #expect(await server.sent == [MySQLObjectQueries.triggerList(schema: "shop", table: nil)]) + let trigger = try #require(triggers.first) + #expect(trigger.name == "audit") + #expect(trigger.table == "orders") + #expect(trigger.definition == """ + CREATE DEFINER = `root`@`%` TRIGGER `shop`.`audit` BEFORE INSERT ON `shop`.`orders` FOR EACH ROW + SET NEW.at = NOW() + """) + #expect(trigger.attributes.map(\.label) == ["Definer", "Action Order"]) + } + + @Test("A trigger that already carries its definition is returned without a read") + func triggerDDLWithDefinition() async throws { + let server = CatalogServer() + let trigger = PluginTriggerInfo( + name: "audit", table: "orders", timing: "BEFORE", event: "INSERT", + statement: "BEGIN END", definition: "CREATE TRIGGER audit BEFORE INSERT ON orders FOR EACH ROW BEGIN END" + ) + #expect(try await makeCatalog(server).triggerDDL(trigger).hasPrefix("CREATE TRIGGER audit")) + #expect(await server.sent.isEmpty) + } + + @Test("Events are listed with their state, and their DDL is the fourth column of SHOW CREATE EVENT") + func events() async throws { + let server = CatalogServer(answers: [ + [["nightly", "RECURRING", "ENABLED", "shop"], ["once", "ONE TIME", "DISABLED", "shop"]], + [["nightly", "", "", "CREATE EVENT `nightly` ON SCHEDULE EVERY 1 DAY DO SELECT 1"]], + ]) + let catalog = makeCatalog(server) + + let events = try await catalog.events(schema: nil) + let ddl = try await catalog.eventDDL(events[0]) + + #expect(events.map(\.name) == ["nightly", "once"]) + #expect(events.map(\.isEnabled) == [true, false]) + #expect(ddl.hasPrefix("CREATE EVENT `nightly`")) + #expect(await server.sent == [ + MySQLObjectQueries.eventList(schema: "shop", exactSchema: false), + "SHOW CREATE EVENT `shop`.`nightly`", + ]) + } + + @Test("A creation scope takes the row named exactly, not a case variant the catalog also returned") + func creationScopeExactName() async throws { + let server = CatalogServer(answers: [[["CLEANUP", "ANSI_QUOTES"], ["cleanup", "STRICT_TRANS_TABLES"]]]) + let routine = PluginRoutineInfo(name: "cleanup", kind: .procedure, schema: "app") + + let scope = try await makeCatalog(server).routineCreationScope(routine) + + #expect(await server.sent == [MySQLObjectQueries.routineCreationSettings( + schema: "app", name: "cleanup", kind: "PROCEDURE", exactSchema: false + )]) + #expect(scope == MySQLExportCatalog.creationScope(sqlMode: "STRICT_TRANS_TABLES", timeZone: nil)) + } + + @Test("Foreign keys are two single-catalog reads merged by table and constraint") + func catalogForeignKeys() async throws { + let server = CatalogServer(answers: [ + [ + ["lines", "fk_order", "order_id", "shop", "orders", "id"], + ["lines", "fk_item", "item_id", nil, "items", "id"], + ], + [["lines", "fk_order", "CASCADE", "RESTRICT"]], + ]) + + let keys = try await makeCatalog(server).catalogForeignKeys(database: "shop", table: nil) + + #expect(await server.sent == [ + MySQLObjectQueries.foreignKeyColumns(schema: "shop", table: nil), + MySQLObjectQueries.referentialActions(schema: "shop", table: nil), + ]) + let lines = try #require(keys["lines"]) + #expect(lines.map(\.name) == ["fk_order", "fk_item"]) + #expect(lines[0].onDelete == "CASCADE") + #expect(lines[0].onUpdate == "RESTRICT") + let omitted = MySQLServerVersion.omittedForeignKeyAction(banner: "8.0.36", flavor: .mysql) + #expect(lines[1].onDelete == omitted) + } + + @Test("A database with no foreign key columns skips the actions read") + func noForeignKeysSkipsActions() async throws { + let server = CatalogServer() + let keys = try await makeCatalog(server).catalogForeignKeys(database: "shop", table: "orders") + #expect(keys.isEmpty) + #expect(await server.sent == [MySQLObjectQueries.foreignKeyColumns(schema: "shop", table: "orders")]) + } + + @Test("The bulk column read marks keys, generated and auto-increment columns per table") + func informationSchemaColumns() async throws { + let server = CatalogServer(answers: [[ + ["orders", "id", "int", nil, "NO", "PRI", nil, "auto_increment", "", nil], + ["orders", "total", "decimal(10,2)", nil, "YES", "", nil, "", "Order total", nil], + ["orders", "doubled", "decimal(10,2)", nil, "YES", "", nil, "VIRTUAL GENERATED", "", "(`total` * 2)"], + ["items", "state", "enum('a','b')", "utf8mb4_bin", "NO", "", "a", "", "", nil], + ]]) + + let columns = try await makeCatalog(server).informationSchemaColumns(schema: nil, table: nil) + + #expect(await server.sent == [ + MySQLObjectQueries.columnCatalog(schema: "shop", table: nil, projectsGenerationExpression: true), + ]) + let orders = try #require(columns["orders"]) + #expect(orders.map(\.name) == ["id", "total", "doubled"]) + #expect(orders.map(\.isPrimaryKey) == [true, false, false]) + #expect(orders[0].identityKind != nil) + #expect(orders.map(\.isGenerated) == [false, false, true]) + #expect(orders[2].generationExpression == "(`total` * 2)") + #expect(orders[1].comment == "Order total") + #expect(orders[1].dataType == "DECIMAL(10,2)") + let state = try #require(columns["items"]?.first) + #expect(state.dataType == "enum('a','b')") + #expect(state.allowedValues == ["a", "b"]) + #expect(state.charset == "utf8mb4") + } + + @Test("Opening a verbatim read scope saves the settings, holds the session, then sets them") + func openVerbatimReadScope() async throws { + let server = CatalogServer(answers: [[["sql_mode", "STRICT_TRANS_TABLES"], ["time_zone", "SYSTEM"]]]) + let footprint = FootprintBox() + let hooks = MySQLExportCatalog.SessionHooks( + runSetup: { await server.answer($0) }, + transactionState: { .idle }, + adjustFootprint: { change in footprint.adjust(change) } + ) + + let scope = try await makeCatalog(server).openSessionScope(for: .verbatimRead, hooks: hooks) + + let capture = try #require(MySQLSessionStatements.captureQuery(names: MySQLSessionScopePlan.verbatimNames)) + let setMode = try #require(MySQLSessionStatements.assignment(name: "sql_mode", value: "")) + let setZone = try #require(MySQLSessionStatements.assignment(name: "time_zone", value: "+00:00")) + #expect(await server.sent == [capture, setMode, setZone]) + #expect(scope?.applied.map(\.name) == ["sql_mode", "time_zone"]) + #expect(footprint.heldSessionScopes == 1) + } + + @Test("Principals read the connection limit where the server has it, and default the host") + func principals() async throws { + let server = CatalogServer(answers: [[["app", "%", "0"], ["batch", nil, "5"]]]) + + let principals = try await makeCatalog(server).principals() + + #expect(await server.sent == ["SELECT User, Host, max_user_connections FROM mysql.user ORDER BY User, Host"]) + #expect(principals.map(\.ref.host) == ["%", "%"]) + #expect(principals.map(\.connectionLimit) == [nil, 5]) + } + + @Test("Grants are read for the quoted account and written back as GRANT statements") + func grantsRoundTrip() async throws { + let server = CatalogServer(answers: [[["GRANT SELECT, INSERT ON `shop`.* TO `app`@`%`"]]]) + let catalog = makeCatalog(server) + let principal = PluginPrincipalRef(name: "app", host: "%") + let privileges = PluginPrivilegeCatalog( + serverPrivileges: [], databasePrivileges: [], schemaPrivileges: [], + tablePrivileges: [], columnPrivileges: [], supportsDynamicPrivileges: false + ) + + let grants = try await catalog.grants(for: principal, privileges: privileges) + + #expect(await server.sent == ["SHOW GRANTS FOR `app`@`%`"]) + #expect(Set(grants.map(\.privilege)) == ["SELECT", "INSERT"]) + #expect(grants.allSatisfy { $0.scope == .database("shop") }) + let statements = catalog.grantStatements( + for: principal, + grants: [PluginGrantInfo(privilege: "SELECT", scope: .table(database: "shop", schema: nil, table: "orders"))] + ) + #expect(statements == ["GRANT SELECT ON `shop`.`orders` TO `app`@`%`"]) + } + + @Test("A database-level grant target escapes LIKE wildcards; a table-level one does not") + func grantTargets() { + let quote: (String) -> String = { mysqlQuoteIdentifier($0) } + #expect(MySQLExportCatalog.grantTarget(for: .database("my_db"), quote: quote) == "`my\\_db`.*") + #expect(MySQLExportCatalog.grantTarget( + for: .table(database: "my_db", schema: nil, table: "t"), quote: quote + ) == "`my_db`.`t`") + #expect(MySQLExportCatalog.grantTarget(for: .server, quote: quote) == "*.*") + } + + @Test("A routine drop names its kind and never DROP ROUTINE") + func dropRoutineStatement() { + let catalog = makeCatalog(CatalogServer()) + #expect(catalog.dropRoutineStatement(name: "total", schema: nil, isFunction: true) + == "DROP FUNCTION IF EXISTS `shop`.`total`") + #expect(catalog.dropRoutineStatement(name: "cleanup", schema: "app", isFunction: false) + == "DROP PROCEDURE IF EXISTS `app`.`cleanup`") + #expect(makeCatalog(CatalogServer(), flavor: .databend) + .dropRoutineStatement(name: "f", schema: nil, isFunction: true) == nil) + } + + @Test("A trigger drop is qualified by the effective database") + func dropTriggerStatement() { + let catalog = makeCatalog(CatalogServer()) + #expect(catalog.dropTriggerStatement(name: "audit", schema: nil) == "DROP TRIGGER `shop`.`audit`") + } + + @Test("The row count read is the catalog's TABLE_ROWS") + func approximateRowCount() async throws { + let server = CatalogServer(answers: [[["42"]], [[nil]]]) + let catalog = makeCatalog(server) + #expect(try await catalog.approximateRowCount(table: "orders", schema: nil) == 42) + #expect(try await catalog.approximateRowCount(table: "orders", schema: "app") == nil) + #expect(await server.sent == [ + MySQLObjectQueries.approximateRowCount(schema: "shop", table: "orders"), + MySQLObjectQueries.approximateRowCount(schema: "app", table: "orders"), + ]) + } + + @Test("A SQL file import turns foreign key checks off and on where the flavor has them") + func foreignKeyCheckStatements() { + #expect(MySQLServerFlavor.mysql.foreignKeyCheckStatements(enabled: false) == ["SET FOREIGN_KEY_CHECKS=0"]) + #expect(MySQLServerFlavor.mariadb.foreignKeyCheckStatements(enabled: true) == ["SET FOREIGN_KEY_CHECKS=1"]) + #expect(MySQLServerFlavor.databend.foreignKeyCheckStatements(enabled: false) == nil) + #expect(MySQLServerFlavor.starrocks.foreignKeyCheckStatements(enabled: true) == nil) + } + + @Test("The plugin driver forwards drop statements and foreign key checks to the shared catalog") + func driverForwards() { + let driver = MySQLPluginDriver(config: DriverConnectionConfig( + host: "", port: 0, username: "", password: "", database: "shop", additionalFields: [:] + )) + #expect(driver.generateDropRoutineSQL(name: "p", signature: "(IN a int)", schema: nil, isFunction: false) + == "DROP PROCEDURE IF EXISTS `shop`.`p`") + #expect(driver.generateDropTriggerSQL(name: "t", table: "orders", schema: "app") == "DROP TRIGGER `app`.`t`") + #expect(driver.foreignKeyDisableStatements() == ["SET FOREIGN_KEY_CHECKS=0"]) + #expect(driver.foreignKeyEnableStatements() == ["SET FOREIGN_KEY_CHECKS=1"]) + #expect(driver.tableDDLIncludesForeignKeys) + } +} diff --git a/apple/Plugins/MySQLDriverTests/MySQLSessionScopeTests.swift b/apple/Plugins/MySQLDriverTests/MySQLSessionScopeTests.swift index ea60bb8367..7f8ed02892 100644 --- a/apple/Plugins/MySQLDriverTests/MySQLSessionScopeTests.swift +++ b/apple/Plugins/MySQLDriverTests/MySQLSessionScopeTests.swift @@ -246,17 +246,17 @@ struct MySQLSessionScopeTests { @Test("A creation scope needs a recorded mode, and carries the zone only when there is one") func creationScopeNeedsRecordedMode() { - #expect(MySQLPluginDriver.creationScope(sqlMode: nil, timeZone: "SYSTEM") == nil) - let routine = MySQLPluginDriver.creationScope(sqlMode: "STRICT_TRANS_TABLES", timeZone: nil) + #expect(MySQLExportCatalog.creationScope(sqlMode: nil, timeZone: "SYSTEM") == nil) + let routine = MySQLExportCatalog.creationScope(sqlMode: "STRICT_TRANS_TABLES", timeZone: nil) #expect(routine?.opening == [ "/*!50003 SET @saved_sql_mode = @@sql_mode */", "/*!50003 SET sql_mode = 'STRICT_TRANS_TABLES' */", ]) #expect(routine?.closing == ["/*!50003 SET sql_mode = @saved_sql_mode */"]) - let event = MySQLPluginDriver.creationScope(sqlMode: "", timeZone: "SYSTEM") + let event = MySQLExportCatalog.creationScope(sqlMode: "", timeZone: "SYSTEM") #expect(event?.opening.count == 4) #expect(event?.closing.first == "/*!50003 SET time_zone = @saved_time_zone */") - let mariaDBRoutine = MySQLPluginDriver.creationScope( + let mariaDBRoutine = MySQLExportCatalog.creationScope( sqlMode: "STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION", timeZone: nil ) diff --git a/apple/Plugins/OracleDriverPlugin/OracleBindPlaceholders.swift b/apple/Plugins/OracleDriverPlugin/OracleBindPlaceholders.swift index 92029328e5..b17c458203 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleBindPlaceholders.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleBindPlaceholders.swift @@ -18,7 +18,7 @@ import TableProSQLQuoting // Only a `?` that is a placeholder is replaced. One inside a string literal, an alternative-quoted `q'[...]'` literal // (any delimiter, with or without the national `n` prefix), a double-quoted name such as `"Paid?"`, or a `--` or // `/* */` comment is text, and replacing it would change the statement and shift every value after it. -internal enum OracleBindPlaceholders { +nonisolated internal enum OracleBindPlaceholders { /// The longest string literal Oracle parses, in bytes of the value it holds rather than of its doubled quotes /// (measured on 23ai with `MAX_STRING_SIZE = STANDARD`: 4,000 bytes parse, 4,001 fail with ORA-01704). static let maxLiteralBytes = 4_000 @@ -46,7 +46,7 @@ internal enum OracleBindPlaceholders { return ":\(binds.count)" } ) else { - throw OraclePluginError(core: .queryFailed(OracleValueKind.unwritableBFileReason)) + throw OracleStatementError(message: OracleValueKind.unwritableBFileReason) } return written } diff --git a/apple/Plugins/OracleDriverPlugin/OracleColumnStatements.swift b/apple/Plugins/OracleDriverPlugin/OracleColumnStatements.swift index 58a850006c..f329ac14d9 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleColumnStatements.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleColumnStatements.swift @@ -16,7 +16,7 @@ import TableProPluginKit /// /// The type is written as it was given. A type change is still detected without regard to case, but folding the /// written type would turn a quoted mixed-case object type such as `"Hunt_Mixed"` into a name that does not exist. -internal enum OracleColumnStatements { +nonisolated internal enum OracleColumnStatements { /// Auto-increment is `GENERATED BY DEFAULT ON NULL AS IDENTITY`, not `ALWAYS`, so an explicit key from a copy or /// an import still inserts (ORA-32795 otherwise). An identity takes no `DEFAULT`, and `ON NULL` makes it NOT NULL. internal static func definition( diff --git a/apple/Plugins/OracleDriverPlugin/OracleCreateTableSQL.swift b/apple/Plugins/OracleDriverPlugin/OracleCreateTableSQL.swift new file mode 100644 index 0000000000..1d9d2ab8cc --- /dev/null +++ b/apple/Plugins/OracleDriverPlugin/OracleCreateTableSQL.swift @@ -0,0 +1,102 @@ +// +// OracleCreateTableSQL.swift +// OracleDriverPlugin +// + +import Foundation +import TableProPluginKit + +// Oracle runs one statement per call: sent as one text, the table and its indexes fail with ORA-03405 and create +// nothing. So each element is one statement. +nonisolated enum OracleCreateTableSQL { + static func statements( + definition: PluginCreateTableDefinition, + schema: String, + supportsIdentity: Bool + ) -> [String]? { + guard let statements = tableAndIndexStatements( + definition: definition, schema: schema, supportsIdentity: supportsIdentity + ) else { return nil } + let comments = OracleColumnStatements.commentStatements( + qualifiedTable: qualifiedTable(definition.tableName, schema: schema), columns: definition.columns + ) + return statements + comments + } + + static func tableAndIndexStatements( + definition: PluginCreateTableDefinition, + schema: String, + supportsIdentity: Bool + ) -> [String]? { + guard !definition.columns.isEmpty else { return nil } + + let qualifiedTable = Self.qualifiedTable(definition.tableName, schema: schema) + let pkColumns = definition.columns.filter { $0.isPrimaryKey } + let inlinePK = pkColumns.count == 1 + var parts: [String] = definition.columns.map { + columnDefinition($0, inlinePrimaryKey: inlinePK, supportsIdentity: supportsIdentity) + } + + if pkColumns.count > 1 { + let pkCols = pkColumns.map { quote($0.name) }.joined(separator: ", ") + parts.append("PRIMARY KEY (\(pkCols))") + } + + for fk in definition.foreignKeys { + parts.append(foreignKeyConstraint(fk)) + } + + let createTable = "CREATE TABLE \(qualifiedTable) (\n " + + parts.joined(separator: ",\n ") + + "\n)" + let indexStatements = definition.indexes.map { indexDefinition($0, qualifiedTable: qualifiedTable) } + return [createTable] + indexStatements + } + + static func qualifiedTable(_ table: String, schema: String) -> String { + "\(quote(schema)).\(quote(table))" + } + + static func columnDefinition( + _ column: PluginColumnDefinition, + inlinePrimaryKey: Bool, + supportsIdentity: Bool + ) -> String { + OracleColumnStatements.definition( + column, + inlinePrimaryKey: inlinePrimaryKey, + supportsIdentity: supportsIdentity, + quote: quote + ) + } + + static func indexDefinition(_ index: PluginIndexDefinition, qualifiedTable: String) -> String { + let cols = index.columns.map { quote($0) }.joined(separator: ", ") + let unique = index.isUnique ? "UNIQUE " : "" + return "CREATE \(unique)INDEX \(quote(index.name)) ON \(qualifiedTable) (\(cols))" + } + + static func foreignKeyConstraint(_ fk: PluginForeignKeyDefinition) -> String { + let cols = fk.columns.map { quote($0) }.joined(separator: ", ") + let refCols = fk.referencedColumns.map { quote($0) }.joined(separator: ", ") + let refTable: String + if let schema = fk.referencedSchema, !schema.isEmpty { + refTable = "\(quote(schema)).\(quote(fk.referencedTable))" + } else { + refTable = quote(fk.referencedTable) + } + let constraint = fk.name.isEmpty ? "" : "CONSTRAINT \(quote(fk.name)) " + var def = "\(constraint)FOREIGN KEY (\(cols)) REFERENCES \(refTable)" + if !refCols.isEmpty { + def += " (\(refCols))" + } + if fk.onDelete != "NO ACTION" { + def += " ON DELETE \(fk.onDelete)" + } + return def + } + + private static func quote(_ name: String) -> String { + OracleObjectQueries.quoteIdentifier(name) + } +} diff --git a/apple/Plugins/OracleDriverPlugin/OracleExportCatalog.swift b/apple/Plugins/OracleDriverPlugin/OracleExportCatalog.swift new file mode 100644 index 0000000000..415da40802 --- /dev/null +++ b/apple/Plugins/OracleDriverPlugin/OracleExportCatalog.swift @@ -0,0 +1,472 @@ +// +// OracleExportCatalog.swift +// OracleDriverPlugin +// + +import Foundation +import os +import TableProOracleCore +import TableProPluginKit + +nonisolated struct OraclePrincipalState: Sendable { + var access: OracleDictionaryAccess? + var roles: Set = [] + var catalog: PluginPrivilegeCatalog? +} + +// The catalog reads both apps run against Oracle, over the connection each app holds. +nonisolated struct OracleExportCatalog: Sendable { + typealias Run = @Sendable (String) async throws -> OracleRawResult + + static let tableDDLIncludesForeignKeys = false + + let run: Run + let currentSchema: String? + let fallbackSchema: String + let release: OracleServerRelease + let principalState: OSAllocatedUnfairLock + // The server error inside what `run` threw, so a dictionary view the account cannot read is told apart. + let coreError: @Sendable (Error) -> OracleCoreError? + + init( + run: @escaping Run, + currentSchema: String?, + fallbackSchema: String, + release: OracleServerRelease, + principalState: OSAllocatedUnfairLock, + coreError: @escaping @Sendable (Error) -> OracleCoreError? + ) { + self.run = run + self.currentSchema = currentSchema + self.fallbackSchema = fallbackSchema + self.release = release + self.principalState = principalState + self.coreError = coreError + } + + func effectiveSchema(_ schema: String?) -> String { + schema ?? currentSchema ?? fallbackSchema + } + + // MARK: - Columns and keys + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + let result = try await run( + OracleSchemaQueries.columns(schema: effectiveSchema(schema), table: table, release: release) + ) + return result.rows.compactMap(OracleSchemaQueries.parseColumnRow).map(\.pluginColumnInfo) + } + + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { + let result = try await run(OracleSchemaQueries.allColumns(schema: effectiveSchema(schema), release: release)) + var columnsByTable: [String: [PluginColumnInfo]] = [:] + for (table, column) in result.rows.compactMap(OracleSchemaQueries.parseTableColumnRow) { + columnsByTable[table, default: []].append(column.pluginColumnInfo) + } + return columnsByTable + } + + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + let result = try await run(OracleSchemaQueries.foreignKeys(schema: effectiveSchema(schema), table: table)) + return result.rows.compactMap(OracleSchemaQueries.parseForeignKeyRow).map { + PluginForeignKeyInfo( + name: $0.constraintName, + column: $0.columnName, + referencedTable: $0.referencedTable, + referencedColumn: $0.referencedColumn, + referencedSchema: $0.referencedSchema, + onDelete: $0.deleteRule, + onUpdate: "NO ACTION" + ) + } + } + + func fetchAllForeignKeys(schema: String?) async throws -> [String: [PluginForeignKeyInfo]] { + let rows = try await cells(OracleSchemaQueries.allForeignKeys(schema: effectiveSchema(schema))) + var fksByTable: [String: [PluginForeignKeyInfo]] = [:] + for row in rows { + guard let tableName = row[safe: 0]?.asText, + let constraintName = row[safe: 1]?.asText, + let columnName = row[safe: 2]?.asText, + let refTable = row[safe: 3]?.asText, + let refColumn = row[safe: 4]?.asText else { continue } + let deleteRule = (row[safe: 5]?.asText) ?? "NO ACTION" + let fk = PluginForeignKeyInfo( + name: constraintName, + column: columnName, + referencedTable: refTable, + referencedColumn: refColumn, + referencedSchema: row[safe: 6]?.asText, + onDelete: deleteRule, + onUpdate: "NO ACTION" + ) + fksByTable[tableName, default: []].append(fk) + } + return fksByTable + } + + // MARK: - Definitions + + // `ALL_TAB_COLS` lists a view's columns as it lists a table's, so a view is looked up first. + func fetchTableDDL(table: String, schema: String?) async throws -> String { + let owner = effectiveSchema(schema) + let view = try await cells(OracleSchemaQueries.viewDefinition(schema: owner, view: table)) + if !view.isEmpty { + return try await viewStatement(from: view, schema: owner, view: table) + } + let columns = try await fetchColumns(table: table, schema: schema) + let constraints = try await run( + OracleSchemaQueries.tableConstraints(schema: owner, table: table, release: release) + ) + return try OracleTableDDL.createTable( + qualifiedTable: Self.qualified(owner, table), + columns: columns, + constraints: OracleSchemaQueries.parseTableConstraints(constraints.rows), + quote: OracleObjectQueries.quoteIdentifier + ) + } + + // ALL_VIEWS.TEXT is a LONG, which crashes OracleNIO, and DBMS_METADATA.GET_DDL on the wrong object type raises + // ORA-31603, which breaks its connection state machine. TEXT_VC is a VARCHAR2(4000). + func fetchViewDefinition(view: String, schema: String?) async throws -> String { + let owner = effectiveSchema(schema) + let rows = try await cells(OracleSchemaQueries.viewDefinition(schema: owner, view: view)) + return try await viewStatement(from: rows, schema: owner, view: view) + } + + // TEXT_VC is the view's SELECT alone; written bare, a restore ran the query and created no view. + private func viewStatement(from rows: [[PluginCellValue]], schema: String, view: String) async throws -> String { + guard let row = rows.first, let body = row.first?.asText, + !body.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + throw OracleStatementError( + message: String(format: String(localized: "Oracle returned no definition for view '%@'."), view) + ) + } + guard !OracleSchemaQueries.isViewTextTruncated( + storedLength: row[safe: 1]?.asText, returnedLength: row[safe: 2]?.asText + ) else { + throw OracleStatementError(message: String( + localized: "Oracle returns only the first 4000 bytes of this view's definition, so the full text cannot be read." + )) + } + return OracleTableDDL.createView( + qualifiedView: Self.qualified(schema, view), + columns: try await fetchColumns(table: view, schema: schema).map(\.name), + query: body, + quote: OracleObjectQueries.quoteIdentifier + ) + } + + func fetchIndexDDL(table: String, schema: String?) async throws -> [String] { + let owner = effectiveSchema(schema) + let result = try await run(OracleIndexStatements.query(schema: owner, table: table)) + let indexes = OracleIndexStatements.render( + rows: result.rows.map { row in row.map { $0.stringValue } }, + schema: owner, + table: table, + quote: OracleObjectQueries.quoteIdentifier) + let constraints = try await run( + OracleSchemaQueries.tableConstraints(schema: owner, table: table, release: release) + ) + return indexes + (try OracleTableDDL.afterRowsStatements( + qualifiedTable: Self.qualified(owner, table), + constraints: OracleSchemaQueries.parseTableConstraints(constraints.rows), + quote: OracleObjectQueries.quoteIdentifier + )) + } + + // MARK: - Routines + + func fetchRoutines(schema: String?) async throws -> [PluginRoutineInfo] { + let resolvedSchema = routineSchema(schema) + let rows = try await cells(OracleObjectQueries.routineList(schema: resolvedSchema)) + return rows.compactMap { row -> PluginRoutineInfo? in + guard let name = row[safe: 0]?.asText else { return nil } + let objectType = (row[safe: 2]?.asText ?? "").uppercased() + var attributes: [PluginObjectAttribute] = [] + if let status = row[safe: 3]?.asText, !status.isEmpty { + attributes.append(PluginObjectAttribute(label: "Status", value: status)) + } + return PluginRoutineInfo( + name: name, + kind: objectType == "PROCEDURE" ? .procedure : .function, + schema: row[safe: 1]?.asText ?? resolvedSchema, + returnType: nil, + language: "PL/SQL", + argumentSignature: nil, + identity: objectType, + attributes: attributes + ) + } + } + + // ALL_SOURCE returns no rows at all when the caller cannot see the object, a privilege answer. + func fetchRoutineDDL(_ routine: PluginRoutineInfo) async throws -> String { + let query = OracleObjectQueries.routineSource( + schema: routineSchema(routine.schema), + name: routine.name, + type: routine.kind == .procedure ? "PROCEDURE" : "FUNCTION" + ) + let rows = try await cells(query) + guard !rows.isEmpty else { + throw PluginObjectSourceError.insufficientPrivilege(routine.name) + } + let body = rows + .compactMap { $0[safe: 0]?.asText } + .joined() + .trimmingCharacters(in: .whitespacesAndNewlines) + guard !body.isEmpty else { + throw PluginObjectSourceError.insufficientPrivilege(routine.name) + } + return body.uppercased().hasPrefix("CREATE") ? body : "CREATE OR REPLACE \(body)" + } + + static func dropRoutineStatement(name: String, schema: String?, currentSchema: String?, isFunction: Bool) -> String { + OracleRoutineStatements.drop(name: name, schema: schema, currentSchema: currentSchema, isFunction: isFunction) + } + + // MARK: - Triggers + + // A schema-wide read scopes on OWNER, the trigger's own schema. + func fetchAllTriggers(schema: String?) async throws -> [PluginTriggerInfo] { + try await triggerList(schema: routineSchema(schema), table: nil) + } + + // A per-table read scopes on TABLE_OWNER, the subject table's schema. + func fetchTriggers(table: String, schema: String?) async throws -> [PluginTriggerInfo] { + try await triggerList(schema: effectiveSchema(schema), table: table) + } + + func fetchTriggerDDL(_ trigger: PluginTriggerInfo) async throws -> String { + if let definition = trigger.definition, !definition.isEmpty { return definition } + let listed = try await triggerList(schema: routineSchema(trigger.schema), table: trigger.table) + guard let definition = listed.first(where: { $0.name == trigger.name })?.definition, + !definition.isEmpty + else { + throw PluginObjectSourceError.insufficientPrivilege(trigger.name) + } + return definition + } + + static func dropTriggerStatement( + name: String, + schema: String?, + currentSchema: String?, + release: OracleServerRelease? + ) -> String { + OracleDropStatements.drop(.trigger, name: name, schema: schema, currentSchema: currentSchema, release: release) + } + + // The export's own `DROP TABLE IF EXISTS` parses only on 23ai and later. + static func dropRelationStatement( + kind: PluginExportObjectKind, + name: String, + schema: String?, + currentSchema: String?, + release: OracleServerRelease? + ) -> String? { + let relation: OracleDropStatements.Object + switch kind { + case .table: relation = .table + case .view: relation = .view + case .materializedView: relation = .materializedView + default: return nil + } + return OracleDropStatements.drop(relation, name: name, schema: schema, currentSchema: currentSchema, release: release) + } + + // Every drop a dump writes for Oracle. Nil for a routine whose `tableType` is not its `PluginRoutineKind`. + func dropStatement(for object: PluginExportTable, schema: String?) -> String? { + switch object.kind { + case .trigger: + return Self.dropTriggerStatement( + name: object.name, schema: schema, currentSchema: currentSchema, release: release + ) + case .routine: + guard let kind = PluginRoutineKind(rawValue: object.tableType.lowercased()) else { return nil } + return Self.dropRoutineStatement( + name: object.name, schema: schema, currentSchema: currentSchema, isFunction: kind == .function + ) + default: + return Self.dropRelationStatement( + kind: object.kind, name: object.name, schema: schema, currentSchema: currentSchema, release: release + ) + } + } + + private func triggerList(schema: String, table: String?) async throws -> [PluginTriggerInfo] { + let rows = try await cells(OracleObjectQueries.triggerList(schema: schema, table: table)) + return rows.compactMap { row -> PluginTriggerInfo? in + guard let name = row[safe: 0]?.asText else { return nil } + let triggerType = row[safe: 3]?.asText ?? "" + let owner = row[safe: 2]?.asText + let body = row[safe: 10]?.asText + let definition = OracleObjectQueries.triggerDefinition(OracleTriggerSource( + name: name, + owner: owner, + tableOwner: row[safe: 9]?.asText, + description: row[safe: 7]?.asText, + whenClause: row[safe: 6]?.asText, + actionType: row[safe: 8]?.asText, + status: row[safe: 5]?.asText, + body: body + )) + var attributes: [PluginObjectAttribute] = [] + if let whenClause = row[safe: 6]?.asText, !whenClause.isEmpty { + attributes.append(PluginObjectAttribute(label: "When", value: whenClause)) + } + return PluginTriggerInfo( + name: name, + table: row[safe: 1]?.asText, + schema: owner ?? schema, + timing: OracleObjectQueries.timing(fromTriggerType: triggerType), + event: row[safe: 4]?.asText ?? "", + orientation: OracleObjectQueries.orientation(fromTriggerType: triggerType), + statement: body ?? definition, + definition: definition, + enabled: (row[safe: 5]?.asText ?? "").uppercased() == "ENABLED", + attributes: attributes + ) + } + } + + // MARK: - Principals + + // A schema is what the driver lists as a database, so a table's scope is `.table(database: owner, schema: nil)`. + func fetchPrincipals() async throws -> [PluginPrincipalInfo] { + let access = try await dictionaryAccess() + let users = OraclePrincipalQueries.parseUsers( + try await textRows(OraclePrincipalQueries.users(access: access, release: release)) + ) + let roles = OraclePrincipalQueries.parseNames(try await textRows(OraclePrincipalQueries.roles(access: access))) + let grants = OraclePrincipalQueries.parseRoleGrants( + try await textRows(OraclePrincipalQueries.roleGrants(access: access)) + ) + let roleNames = Set(roles) + principalState.withLock { $0.roles = roleNames } + + var memberOf: [String: Set] = [:] + for grant in grants where roleNames.contains(grant.role) { + memberOf[grant.grantee, default: []].insert(grant.role) + } + + let userInfos = users.filter { !roleNames.contains($0.name) }.map { user in + PluginPrincipalInfo( + ref: PluginPrincipalRef(name: user.name), + canLogin: !user.isLocked, + memberOf: (memberOf[user.name] ?? []).sorted() + ) + } + let roleInfos = roles.map { role in + PluginPrincipalInfo( + ref: PluginPrincipalRef(name: role), + isRole: true, + canLogin: false, + memberOf: (memberOf[role] ?? []).sorted() + ) + } + return userInfos + roleInfos + } + + func fetchGrants(for principal: PluginPrincipalRef) async throws -> [PluginGrantInfo] { + let access = try await dictionaryAccess() + let name = principal.name + let system = OraclePrincipalQueries.parseSystemPrivileges( + try await textRows(OraclePrincipalQueries.systemPrivileges(grantee: name, access: access)) + ) + let objects = OraclePrincipalQueries.parseObjectPrivileges( + try await textRows(OraclePrincipalQueries.objectPrivileges(grantee: name, access: access)) + ) + let columns = OraclePrincipalQueries.parseColumnPrivileges( + try await textRows(OraclePrincipalQueries.columnPrivileges(grantee: name, access: access)) + ) + return (system + objects + columns).map(Self.pluginGrant) + } + + // `roles` are the names `fetchPrincipals` found to be roles, which a grant spells without a privilege keyword. + static func generateGrantSQL(changeSet: PluginPrincipalChangeSet, roles: Set) -> [String]? { + guard let grants = oracleGrants(changeSet.grantsToAdd) else { return nil } + return OraclePrincipalStatements(roles: roles).grantStatements(grants, to: changeSet.principal.name) + } + + static func generateRevokeSQL(changeSet: PluginPrincipalChangeSet, roles: Set) -> [String]? { + guard let grants = oracleGrants(changeSet.grantsToRemove) else { return nil } + return OraclePrincipalStatements(roles: roles).revokeStatements(grants, from: changeSet.principal.name) + } + + func textRows(_ query: String) async throws -> [[String?]] { + try await run(query).rows.map { $0.map(\.stringValue) } + } + + func names(_ query: String) async throws -> [String] { + OraclePrincipalQueries.parseNames(try await textRows(query)) + } + + // Probed once per connection: an account that cannot read DBA_USERS reads the ALL_ and USER_ views instead. + private func dictionaryAccess() async throws -> OracleDictionaryAccess { + if let access = principalState.withLock({ $0.access }) { + return access + } + let access: OracleDictionaryAccess + do { + _ = try await run(OraclePrincipalQueries.dbaProbe) + access = .dba + } catch let error where coreError(error).map(OraclePrincipalQueries.isMissingView) == true { + access = .session + } + principalState.withLock { $0.access = access } + return access + } + + private static func oracleGrants(_ grants: [PluginGrantInfo]) -> [OracleGrant]? { + var converted: [OracleGrant] = [] + for grant in grants { + let securable: OracleSecurable + switch grant.scope { + case .server: + securable = .system + case let .table(owner, _, table), let .column(owner, _, table, _): + securable = .object(owner: owner, name: table) + case .database, .schema: + return nil + @unknown default: + return nil + } + converted.append(OracleGrant( + privilege: grant.privilege, + securable: securable, + column: grant.scope.columnName, + isGrantable: grant.isGrantable + )) + } + return converted + } + + private static func pluginGrant(_ grant: OracleGrant) -> PluginGrantInfo { + let scope: PluginPrivilegeScope + switch grant.securable { + case .system: + scope = .server + case let .object(owner, name): + scope = grant.column.map { + .column(database: owner, schema: nil, table: name, column: $0) + } ?? .table(database: owner, schema: nil, table: name) + } + return PluginGrantInfo(privilege: grant.privilege, scope: scope, isGrantable: grant.isGrantable) + } + + // MARK: - Helpers + + private func routineSchema(_ schema: String?) -> String { + effectiveSchema(schema?.isEmpty == false ? schema : nil) + } + + // The plugin's catalog reads took their text from the PluginKit cells, so a byte cell reads as no text here too. + private func cells(_ query: String) async throws -> [[PluginCellValue]] { + try await run(query).rows.map { $0.map(\.asPluginCell) } + } + + private static func qualified(_ schema: String, _ name: String) -> String { + "\(OracleObjectQueries.quoteIdentifier(schema)).\(OracleObjectQueries.quoteIdentifier(name))" + } +} diff --git a/apple/Plugins/OracleDriverPlugin/OracleIndexStatements.swift b/apple/Plugins/OracleDriverPlugin/OracleIndexStatements.swift index df7ff44f6b..77162dd6e1 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleIndexStatements.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleIndexStatements.swift @@ -11,7 +11,7 @@ import TableProSQLQuoting // PRIMARY KEY or UNIQUE constraint owns is left out: `fetchTableDDL` writes the constraint, which makes its own index, // and a second index on the same columns fails with ORA-01408. A disabled or NOVALIDATE constraint is written disabled // and makes none, so an index it kept is written, and a NOVALIDATE one is enabled on it after the rows. -internal enum OracleIndexStatements { +nonisolated internal enum OracleIndexStatements { // NORMAL and BITMAP only: a function-based index keeps its key in a LONG column OracleNIO cannot decode. A // constraint row with no `INDEX_OWNER` is read as naming an index of its own schema. internal static func query(schema: String, table: String) -> String { @@ -75,7 +75,7 @@ internal enum OracleIndexStatements { } } -private extension Array where Element == String? { +nonisolated private extension Array where Element == String? { func element(at index: Int) -> String? { indices.contains(index) ? self[index] : nil } diff --git a/apple/Plugins/OracleDriverPlugin/OracleNationalText.swift b/apple/Plugins/OracleDriverPlugin/OracleNationalText.swift index 46191bf144..85a10f21e4 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleNationalText.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleNationalText.swift @@ -12,7 +12,7 @@ import Foundation /// (measured on 23ai). `UNISTR` carries each character as ASCII or a `\XXXX` UTF-16 escape, which the server turns /// straight into the national character set. Text that is all ASCII is in every database character set and keeps its /// plain literal. -internal enum OracleNationalText { +nonisolated internal enum OracleNationalText { /// The most a UNISTR argument may hold. It is a string literal: 800 escapes parse, 801 fail with ORA-01704. static let maxChunkBytes = 4_000 diff --git a/apple/Plugins/OracleDriverPlugin/OracleObjectQueries.swift b/apple/Plugins/OracleDriverPlugin/OracleObjectQueries.swift index 09ecb2ff0d..2b0be7c3df 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleObjectQueries.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleObjectQueries.swift @@ -8,13 +8,10 @@ import Foundation import TableProSQLQuoting -/// Catalog SQL for routines and triggers. Pure, so it is testable without a server. -/// -/// Every dictionary view is named with its `SYS` owner. Oracle resolves an unqualified name to a current-schema object -/// before the public synonym, so a user who owns the schema the reader is in could plant a same-named table or view and -/// have the app read it. The rule and its measurement live in `OracleDictionary` in TableProOracleCore, which this file -/// cannot import: it compiles into the test target, which does not link that package, so the prefix is written out here. -public enum OracleObjectQueries { +// Every dictionary view is named with its `SYS` owner: an unqualified name resolves to a current-schema object before +// the public synonym, so a planted same-named view would be read. The rule is `OracleDictionary`'s in TableProOracleCore, +// written out here because a target compiling this file does not link that package. +nonisolated public enum OracleObjectQueries { public static func escapeLiteral(_ value: String) -> String { SQLQuoting.stringLiteralBody(value) } @@ -23,7 +20,7 @@ public enum OracleObjectQueries { SQLQuoting.identifier(value) } - /// Oracle's grammar is `COMMENT ON TABLE [schema.]{table | view}`: there is no `VIEW` keyword. + // Oracle's grammar is `COMMENT ON TABLE [schema.]{table | view}`: there is no `VIEW` keyword. public static func commentStatement(qualifiedName: String, objectType: String, comment: String?) -> String? { let keyword: String switch objectType.uppercased() { @@ -43,13 +40,8 @@ public enum OracleObjectQueries { SQLQuoting.stringLiteral(comment ?? "") } - /// Standalone procedures and functions only. A packaged routine is addressed through its - /// package, which is a different object with a different DDL call, so listing it here would - /// produce rows whose source cannot be fetched. - /// - /// No argument list is built. Oracle only allows overloading inside a package, so a standalone - /// routine is identified by its name alone, and the LISTAGG that would assemble a signature - /// raises ORA-01489 past 4000 bytes, which fails the whole listing over one wide signature. + // Standalone routines only: a packaged one is read through its package, so its row would have no source. No + // argument list: only a package overloads, and the LISTAGG that builds one raises ORA-01489 past 4000 bytes. public static func routineList(schema: String) -> String { let schemaLiteral = escapeLiteral(schema) return """ @@ -65,9 +57,8 @@ public enum OracleObjectQueries { """ } - /// ALL_SOURCE stores one row per line, so the body has to be reassembled in LINE order. - /// DBMS_METADATA.GET_DDL is nicer but returns nothing rather than raising when the caller - /// lacks SELECT_CATALOG_ROLE for another schema, which reads as a routine that vanished. + // ALL_SOURCE, one row per line, in LINE order. DBMS_METADATA.GET_DDL returns nothing rather than raising without + // SELECT_CATALOG_ROLE for another schema, which reads as a routine that vanished. public static func routineSource(schema: String, name: String, type: String) -> String { """ SELECT TEXT @@ -79,13 +70,11 @@ public enum OracleObjectQueries { """ } - /// TRIGGER_BODY is the part the previous query never selected, which left the viewer showing a - /// CREATE OR REPLACE header with no body under it. It stays last because it is a LONG. + // TRIGGER_BODY stays last because it is a LONG. public static func triggerList(schema: String, table: String?) -> String { let schemaLiteral = escapeLiteral(schema) - /// A schema browse asks for the triggers this schema owns, which is OWNER. A per-table - /// fetch asks for the triggers on that table, which is TABLE_OWNER plus TABLE_NAME. They - /// differ for a trigger one schema owns on another schema's table. + // A schema browse reads OWNER, a per-table fetch TABLE_OWNER and TABLE_NAME: they differ for a trigger one + // schema owns on another schema's table. let scope = table.map { "TABLE_OWNER = '\(schemaLiteral)' AND TABLE_NAME = '\(escapeLiteral($0))'" } ?? "OWNER = '\(schemaLiteral)'" @@ -119,21 +108,11 @@ public enum OracleObjectQueries { triggerType.uppercased().contains("EACH ROW") ? "ROW" : "STATEMENT" } - /// The statement that recreates a trigger, assembled from the columns ALL_TRIGGERS splits it into. - /// - /// DESCRIPTION holds the header up to the body: the name, timing, events, subject, REFERENCING, - /// FOR EACH ROW and FOLLOWS. Measured on Oracle 23ai, it holds neither the WHEN clause nor - /// DISABLE, and a trigger whose body is a CALL keeps only the call's target in TRIGGER_BODY, - /// with no CALL keyword and a `;` Oracle added. Written back as they were read, the WHEN clause - /// and the disabled state were lost and a CALL trigger failed with ORA-04079, after the sync had - /// already dropped the one in the target. - /// - /// The owner's own schema is taken out of the header, the way ALL_SOURCE already gives a - /// procedure without it, so the trigger lands in whatever schema the statement runs in rather - /// than going back to the one it was read from. That holds only for a trigger on its own - /// schema's table, or on the schema itself: one schema's trigger on another's table is listed - /// with that table and replayed from its schema, where only the owner's qualifier still names - /// the same trigger. A qualifier naming any other schema stays. + // Measured on 23ai: DESCRIPTION holds the header up to the body but neither the WHEN clause nor DISABLE, and a CALL + // trigger's TRIGGER_BODY holds only the call's target with a `;` Oracle added, so those are written back here or the + // trigger fails with ORA-04079. The owner's schema is taken out of the header, as ALL_SOURCE gives a procedure, so + // the trigger lands in the schema the statement runs in. Not for a trigger on another schema's table, which is + // replayed from that schema, where only the owner's qualifier still names it. public static func triggerDefinition(_ trigger: OracleTriggerSource) -> String { let prefix = "CREATE OR REPLACE TRIGGER " let body = triggerBody(trigger) @@ -157,21 +136,8 @@ public enum OracleObjectQueries { return lines.joined(separator: "\n") } - /// `DROP TRIGGER`, qualified when the trigger lives outside the session's current schema. An - /// unqualified name is looked up in the current schema, so a trigger another schema owns was - /// dropped under the wrong owner or not found. A trigger in the current schema stays unqualified, - /// which keeps a dump restorable into a schema of another name. - public static func dropTrigger(name: String, schema: String?, currentSchema: String?) -> String { - guard let schema = nonEmpty(schema), schema != currentSchema else { - return "DROP TRIGGER \(quoteIdentifier(name))" - } - return "DROP TRIGGER \(quoteIdentifier(schema)).\(quoteIdentifier(name))" - } - - /// `header` with every qualifier that names `schema` taken out: `probe.t`, `"PROBE"."T"` and - /// `PROBE.SCHEMA` all lose their `PROBE.`. An unquoted name matches however it is cased, since - /// Oracle folds it to upper case; a quoted one matches exactly. Comments and literals are copied - /// as they are. + // `probe.t`, `"PROBE"."T"` and `PROBE.SCHEMA` all lose their `PROBE.`: Oracle folds an unquoted name to upper case, + // and a quoted one matches exactly. Comments and literals are copied as they are. public static func strippingSchema(_ schema: String, from header: String) -> String { var reader = OracleHeaderScanner(header) var output = "" @@ -207,8 +173,8 @@ public enum OracleObjectQueries { } } -/// One ALL_TRIGGERS row, as ``OracleObjectQueries/triggerList(schema:table:)`` selects it. -public struct OracleTriggerSource: Sendable, Equatable { +// One ALL_TRIGGERS row, as `OracleObjectQueries.triggerList(schema:table:)` selects it. +nonisolated public struct OracleTriggerSource: Sendable, Equatable { public let name: String public let owner: String? public let tableOwner: String? @@ -239,10 +205,8 @@ public struct OracleTriggerSource: Sendable, Equatable { } } -/// Reads a trigger header one token at a time: identifiers, quoted identifiers, comments, literals -/// and single characters, each with the exact text it covers so the header can be written back -/// unchanged around what is taken out. -private struct OracleHeaderScanner { +// Each token keeps the exact text it covers, so the header is written back unchanged around what is taken out. +nonisolated private struct OracleHeaderScanner { enum Kind { case identifier(String, quoted: Bool) case other @@ -280,7 +244,7 @@ private struct OracleHeaderScanner { return Token(kind: kind, text: text, end: index, followsPeriod: followsPeriod) } - /// Where the name after a `.` starts, when the token ending at `position` is followed by one. + // Where the name after a `.` starts, when the token ending at `position` is followed by one. func positionAfterPeriod(from position: Int) -> Int? { var cursor = skippingWhitespace(from: position) guard cursor < scalars.count, scalars[cursor] == "." else { return nil } diff --git a/apple/Plugins/OracleDriverPlugin/OraclePlugin+Bridging.swift b/apple/Plugins/OracleDriverPlugin/OraclePlugin+Bridging.swift index 7e173cbbdc..2fb1cbc930 100644 --- a/apple/Plugins/OracleDriverPlugin/OraclePlugin+Bridging.swift +++ b/apple/Plugins/OracleDriverPlugin/OraclePlugin+Bridging.swift @@ -34,61 +34,6 @@ extension OracleTLSFailureKind { } } -extension OracleRawCell { - var asPluginCell: PluginCellValue { - switch self { - case .null: return .null - case .string(let value): return .text(value) - case .bytes(let data): return .bytes(data) - } - } -} - -extension PluginCellValue { - var oracleBindValue: OracleBindValue { - switch self { - case .null: return .null - case .text(let value): return .text(value) - case .bytes(let data): return .bytes(data) - } - } -} - -extension OracleColumnRow { - var pluginColumnInfo: PluginColumnInfo { - PluginColumnInfo( - name: name, - dataType: displayType, - isNullable: isNullable, - isPrimaryKey: isPrimaryKey, - defaultValue: defaultValue, - extra: nil, - charset: nil, - collation: nil, - comment: nil, - identityKind: identityGeneration.map(\.identityKind), - isGenerated: isVirtual, - allowedValues: nil, - generationExpression: generationExpression, - generationKind: isVirtual ? .virtual : nil, - ddlSpelling: OracleTypeCatalog.ddlSpelling(forDeclaredType: displayType), - ddlDefault: nil, - ddlGenerationExpression: nil, - ddlCollation: nil, - classificationTypeName: classificationTypeName - ) - } -} - -extension OracleIdentityGeneration { - var identityKind: IdentityKind { - switch self { - case .always: return .always - case .byDefault: return .byDefault - } - } -} - extension OracleRawResult { func toPluginResult(executionTime: TimeInterval) -> PluginQueryResult { let names = columns.map(\.name) diff --git a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift index 5fff067069..7ac04331ef 100644 --- a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift +++ b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift @@ -325,11 +325,8 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { try ensureAlive(connection) } - /// A probe that only failed to answer is survivable, and both have a fallback. One that killed the - /// connection is not: the core is left disconnected, so the next call silently redials with the full - /// login budget while the caller's own, shorter deadline blames whatever step it happened to be on. - /// Checked after each optional probe, because a redial inside the next one would mask the - /// previous probe's failure. + // A probe that killed the connection leaves the core disconnected, and the next call would redial with the full + // login budget under the caller's shorter deadline. Checked after each probe, so a redial cannot mask its failure. private func ensureAlive(_ connection: OracleCoreConnection) throws { guard connection.isConnected else { core = nil @@ -442,8 +439,7 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { ) } - /// An anonymous block reports no row count of its own; whatever oracle-nio counted belongs to the statements - /// inside it. + // An anonymous block reports no row count of its own; what oracle-nio counted belongs to the statements inside it. private func pluginResult( of query: String, _ result: OracleRawResult, @@ -454,8 +450,7 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { .toPluginResult(executionTime: executionTime) } - /// At most this many lines are read after one statement. A loop that prints more is reported as truncated - /// rather than read into memory, and the rest of its buffer is discarded on the server. + // Lines read after one statement. More is reported as truncated, and the rest is discarded on the server. static let serverOutputLineLimit = 10_000 func fetchServerOutput() async throws -> PluginServerOutput { @@ -468,11 +463,8 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } - /// Turns a `CREATE` that stored an INVALID unit into the failure it is. - /// - /// Oracle accepts the statement and flags the compile failure only as a warning, which oracle-nio drops, so the - /// unit's own errors are read back from `ALL_ERRORS`. A unit the header does not name, or one that compiled, adds - /// nothing. + // Oracle stores an INVALID unit with only a warning, which oracle-nio drops, so its errors are read from + // `ALL_ERRORS`. A unit the header does not name, or one that compiled, adds nothing. func reportCompilationErrors(of query: String) async throws { guard let unit = OraclePLSQLUnit.definition(in: query) else { return } let errors = try await rawQuery(unit.errorsQuery).rows.compactMap(OracleCompilationError.init(row:)) @@ -589,10 +581,7 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { - let result = try await rawQuery( - OracleSchemaQueries.columns(schema: effectiveSchema(schema), table: table, release: serverRelease()) - ) - return result.rows.compactMap(OracleSchemaQueries.parseColumnRow).map(\.pluginColumnInfo) + try await exportCatalog.fetchColumns(table: table, schema: schema) } func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { @@ -619,42 +608,15 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func fetchIndexDDL(table: String, schema: String?) async throws -> [String] { - let owner = effectiveSchema(schema) - let result = try await rawQuery(OracleIndexStatements.query(schema: owner, table: table)) - let indexes = OracleIndexStatements.render( - rows: result.rows.map { row in row.map { $0.stringValue } }, - schema: owner, - table: table, - quote: quoteIdentifier) - let constraints = try await rawQuery( - OracleSchemaQueries.tableConstraints(schema: owner, table: table, release: serverRelease()) - ) - return indexes + (try OracleTableDDL.afterRowsStatements( - qualifiedTable: "\(quoteIdentifier(owner)).\(quoteIdentifier(table))", - constraints: OracleSchemaQueries.parseTableConstraints(constraints.rows), - quote: quoteIdentifier - )) + try await exportCatalog.fetchIndexDDL(table: table, schema: schema) } func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { - let result = try await rawQuery( - OracleSchemaQueries.foreignKeys(schema: effectiveSchema(schema), table: table) - ) - return result.rows.compactMap(OracleSchemaQueries.parseForeignKeyRow).map { - PluginForeignKeyInfo( - name: $0.constraintName, - column: $0.columnName, - referencedTable: $0.referencedTable, - referencedColumn: $0.referencedColumn, - referencedSchema: $0.referencedSchema, - onDelete: $0.deleteRule, - onUpdate: "NO ACTION" - ) - } + try await exportCatalog.fetchForeignKeys(table: table, schema: schema) } func fetchTriggers(table: String, schema: String?) async throws -> [PluginTriggerInfo] { - try await triggerList(schema: effectiveSchema(schema), table: table) + try await exportCatalog.fetchTriggers(table: table, schema: schema) } var triggerEditUsesReplace: Bool { true } @@ -675,47 +637,33 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func generateDropTriggerSQL(name: String, table: String, schema: String?) -> String? { - OracleObjectQueries.dropTrigger(name: name, schema: schema, currentSchema: currentSchema) + OracleExportCatalog.dropTriggerStatement( + name: name, schema: schema, currentSchema: currentSchema, release: core?.serverRelease + ) + } + + func generateDropRoutineSQL(name: String, signature: String?, schema: String?, isFunction: Bool) -> String? { + OracleExportCatalog.dropRoutineStatement( + name: name, schema: schema, currentSchema: currentSchema, isFunction: isFunction + ) + } + + func generateDropRelationSQL(name: String, kind: PluginExportObjectKind, schema: String?) -> String? { + OracleExportCatalog.dropRelationStatement( + kind: kind, name: name, schema: schema, currentSchema: currentSchema, release: core?.serverRelease + ) } var fetchesAllColumnsInOneRequest: Bool { true } func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { - let result = try await rawQuery( - OracleSchemaQueries.allColumns(schema: effectiveSchema(schema), release: serverRelease()) - ) - var columnsByTable: [String: [PluginColumnInfo]] = [:] - for (table, column) in result.rows.compactMap(OracleSchemaQueries.parseTableColumnRow) { - columnsByTable[table, default: []].append(column.pluginColumnInfo) - } - return columnsByTable + try await exportCatalog.fetchAllColumns(schema: schema) } var providesBulkForeignKeyFetch: Bool { true } func fetchAllForeignKeys(schema: String?) async throws -> [String: [PluginForeignKeyInfo]] { - let sql = OracleSchemaQueries.allForeignKeys(schema: effectiveSchema(schema)) - let result = try await execute(query: sql) - var fksByTable: [String: [PluginForeignKeyInfo]] = [:] - for row in result.rows { - guard let tableName = row[safe: 0]?.asText, - let constraintName = row[safe: 1]?.asText, - let columnName = row[safe: 2]?.asText, - let refTable = row[safe: 3]?.asText, - let refColumn = row[safe: 4]?.asText else { continue } - let deleteRule = (row[safe: 5]?.asText) ?? "NO ACTION" - let fk = PluginForeignKeyInfo( - name: constraintName, - column: columnName, - referencedTable: refTable, - referencedColumn: refColumn, - referencedSchema: row[safe: 6]?.asText, - onDelete: deleteRule, - onUpdate: "NO ACTION" - ) - fksByTable[tableName, default: []].append(fk) - } - return fksByTable + try await exportCatalog.fetchAllForeignKeys(schema: schema) } func fetchAllDatabaseMetadata() async throws -> [PluginDatabaseMetadata] { @@ -728,9 +676,8 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } - /// The per-schema segment sizes, or an empty map when the reader lacks the DBA privilege the view needs. It is a - /// separate best-effort read because a non-DBA cannot query `DBA_SEGMENTS` and joining it would fail the whole - /// summary; `ALL_SEGMENTS` cannot stand in for it because Oracle has no such view. + // A separate best-effort read: a non-DBA cannot query `DBA_SEGMENTS`, joining it would fail the whole summary, and + // Oracle has no `ALL_SEGMENTS`. Empty without the privilege. private func schemaSegmentSizes() async -> [String: Int64] { guard let result = try? await execute(query: OracleSchemaQueries.schemaSegmentSizes) else { return [:] } var sizes: [String: Int64] = [:] @@ -743,56 +690,12 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { return sizes } - /// `ALL_TAB_COLS` lists a view's columns as it lists a table's, so a view is looked up first: - /// without it Show DDL rebuilt a view as a `CREATE TABLE` of its columns. func fetchTableDDL(table: String, schema: String?) async throws -> String { - let owner = effectiveSchema(schema) - let view = try await execute(query: OracleSchemaQueries.viewDefinition(schema: owner, view: table)) - if !view.rows.isEmpty { - return try await viewStatement(from: view, schema: owner, view: table) - } - let columns = try await fetchColumns(table: table, schema: schema) - let constraints = try await rawQuery( - OracleSchemaQueries.tableConstraints(schema: owner, table: table, release: serverRelease()) - ) - return try OracleTableDDL.createTable( - qualifiedTable: "\(quoteIdentifier(owner)).\(quoteIdentifier(table))", - columns: columns, - constraints: OracleSchemaQueries.parseTableConstraints(constraints.rows), - quote: quoteIdentifier - ) + try await exportCatalog.fetchTableDDL(table: table, schema: schema) } func fetchViewDefinition(view: String, schema: String?) async throws -> String { - // ALL_VIEWS.TEXT is LONG (crashes OracleNIO). TEXT_VC is VARCHAR2(4000), safe. - // Do NOT use DBMS_METADATA.GET_DDL: the wrong object type triggers ORA-31603, - // which corrupts OracleNIO's connection state machine. - let owner = effectiveSchema(schema) - let result = try await execute(query: OracleSchemaQueries.viewDefinition(schema: owner, view: view)) - return try await viewStatement(from: result, schema: owner, view: view) - } - - /// `TEXT_VC` is the view's `SELECT` and nothing else, so it needs the header a dump replays. - /// Written bare it made the restore run a query and create no view. - private func viewStatement(from result: PluginQueryResult, schema: String, view: String) async throws -> String { - guard let row = result.rows.first, let body = row.first?.asText, - !body.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { - throw OraclePluginError(core: .queryFailed( - String(format: String(localized: "Oracle returned no definition for view '%@'."), view))) - } - guard !OracleSchemaQueries.isViewTextTruncated( - storedLength: row[safe: 1]?.asText, returnedLength: row[safe: 2]?.asText - ) else { - throw OraclePluginError(core: .queryFailed(String( - localized: "Oracle returns only the first 4000 bytes of this view's definition, so the full text cannot be read." - ))) - } - return OracleTableDDL.createView( - qualifiedView: "\(quoteIdentifier(schema)).\(quoteIdentifier(view))", - columns: try await fetchColumns(table: view, schema: schema).map(\.name), - query: body, - quote: quoteIdentifier - ) + try await exportCatalog.fetchViewDefinition(view: view, schema: schema) } func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { @@ -821,11 +724,8 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { return PluginTableMetadata(tableName: table) } - /// The segment bytes of a table, or of the whole schema when `table` is nil, best-effort. - /// - /// The reader's own schema always answers through `USER_SEGMENTS`; another schema needs `DBA_SEGMENTS`, which a - /// non-DBA cannot read, so a refusal returns nil rather than failing the metadata read. `ALL_SEGMENTS` is never - /// used because Oracle has no such view (ORA-00942 even as `SYSTEM`). + // The reader's own schema answers through `USER_SEGMENTS`; another needs `DBA_SEGMENTS`, so a non-DBA gets nil. + // Oracle has no `ALL_SEGMENTS` (ORA-00942 even as `SYSTEM`). private func segmentSize(schema: String, table: String?) async -> Int64? { let ownedByCurrentSchema = schema.caseInsensitiveCompare(effectiveSchema(nil)) == .orderedSame let sql = OracleSchemaQueries.segmentSize( @@ -920,8 +820,8 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { ) } - /// Nothing to write is nil, never an empty list: a non-nil answer to an empty update is how a driver claims the - /// restore of deleted rows, and this one has no identity-preserving insert to restore them with. + // Nothing to write is nil, never an empty list: a non-nil answer to an empty update claims the restore of deleted + // rows, and this driver has no identity-preserving insert to restore them with. func generateRowWrites( table: String, schema: String?, @@ -957,14 +857,11 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Create Table DDL - /// Oracle runs one statement per call, and sent as one text the table and its indexes fail with ORA-03405 and - /// create nothing. + // Oracle runs one statement per call: sent as one text, the table and its indexes fail with ORA-03405. func generateCreateTableStatements(definition: PluginCreateTableDefinition, schema: String?) -> [String]? { - guard let statements = createTableAndIndexStatements(definition: definition, schema: schema) else { return nil } - let comments = OracleColumnStatements.commentStatements( - qualifiedTable: oracleQualifiedTable(definition.tableName, schema: schema), columns: definition.columns + OracleCreateTableSQL.statements( + definition: definition, schema: effectiveSchema(schema), supportsIdentity: supportsIdentityColumns ) - return statements + comments } func objectCommentStatement(name: String, objectType: String, schema: String?, comment: String?) -> String? { @@ -983,30 +880,6 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { return String(localized: "An auto-increment column takes its values from a sequence, so it cannot also have a default.") } - private func createTableAndIndexStatements(definition: PluginCreateTableDefinition, schema: String?) -> [String]? { - guard !definition.columns.isEmpty else { return nil } - - let qualifiedTable = oracleQualifiedTable(definition.tableName, schema: schema) - let pkColumns = definition.columns.filter { $0.isPrimaryKey } - let inlinePK = pkColumns.count == 1 - var parts: [String] = definition.columns.map { oracleColumnDefinition($0, inlinePK: inlinePK) } - - if pkColumns.count > 1 { - let pkCols = pkColumns.map { quoteIdentifier($0.name) }.joined(separator: ", ") - parts.append("PRIMARY KEY (\(pkCols))") - } - - for fk in definition.foreignKeys { - parts.append(oracleForeignKeyConstraint(fk)) - } - - let createTable = "CREATE TABLE \(qualifiedTable) (\n " + - parts.joined(separator: ",\n ") + - "\n)" - let indexStatements = definition.indexes.map { oracleIndexDefinition($0, qualifiedTable: qualifiedTable) } - return [createTable] + indexStatements - } - // MARK: - Definition SQL (clipboard copy) func generateColumnDefinitionSQL(column: PluginColumnDefinition) -> String? { @@ -1048,18 +921,10 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { "ALTER TABLE \(oracleQualifiedTable(table, schema: schema)) DROP COLUMN \(quoteIdentifier(columnName))" } - /// Oracle has no positional clause, but making a column invisible and visible again moves it to - /// the end of the visible order, so any order is reachable by appending the right suffix. - /// - /// Measured against Oracle Free 23: the cycle works on the primary key, on an identity column - /// and on a virtual column; the rows, the default, the NOT NULL, the comment, the identity - /// sequence, the constraints, the indexes and the foreign keys pointing at the table all - /// survive it, and no data is read or written. Needs 12.1, where invisible columns arrived; an - /// older server rejects the statement and the error is reported as it is. - /// - /// The two halves of a cycle are separate statements because Oracle commits each DDL on its - /// own, so a column is invisible for the width of one statement. Cycling one column at a time - /// keeps that window as small as it can be. + // Making a column invisible and visible again moves it to the end, so any order is reachable. Measured on Oracle + // Free 23: works on a primary key, identity and virtual column, keeps rows, defaults, NOT NULL, comments, identity + // sequences, constraints, indexes and incoming foreign keys, and reads no data. Needs 12.1. One column at a time, + // since Oracle commits each DDL on its own and the column is invisible between the two statements. func generateColumnReorderPlan( table: String, schema: String?, @@ -1078,11 +943,8 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } guard !statements.isEmpty else { return nil } - /// Oracle commits each DDL statement on its own, so there is no transaction to roll back: - /// a cycle whose `VISIBLE` half fails, on a dropped connection or a server error, leaves - /// that column hidden for good. Every cycled column gets a compensating `VISIBLE` that the - /// executor runs on any mid-plan failure, which is idempotent on a column that is already - /// visible and puts back the one that is not. + // No transaction to roll back: a failed `VISIBLE` half leaves the column hidden, so every cycled column gets a + // compensating `VISIBLE`, idempotent on one already visible, that the executor runs on a mid-plan failure. return PluginColumnReorderPlan( statements: statements, compensation: cycled.map { "ALTER TABLE \(qt) MODIFY (\(quoteIdentifier($0)) VISIBLE)" }, @@ -1108,44 +970,25 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - DDL Helpers + // An unknown release writes the identity: `schemaOperationRefusal` stops auto-increment on a known 11g. + private var supportsIdentityColumns: Bool { + core?.serverRelease?.hasIdentityColumns ?? true + } + private func oracleQualifiedTable(_ table: String, schema: String?) -> String { - "\(quoteIdentifier(effectiveSchema(schema))).\(quoteIdentifier(table))" + OracleCreateTableSQL.qualifiedTable(table, schema: effectiveSchema(schema)) } - /// An unknown release writes the identity: `schemaOperationRefusal` stops auto-increment on a known 11g. private func oracleColumnDefinition(_ col: PluginColumnDefinition, inlinePK: Bool) -> String { - OracleColumnStatements.definition( - col, - inlinePrimaryKey: inlinePK, - supportsIdentity: core?.serverRelease?.hasIdentityColumns ?? true, - quote: quoteIdentifier - ) + OracleCreateTableSQL.columnDefinition(col, inlinePrimaryKey: inlinePK, supportsIdentity: supportsIdentityColumns) } private func oracleIndexDefinition(_ index: PluginIndexDefinition, qualifiedTable: String) -> String { - let cols = index.columns.map { quoteIdentifier($0) }.joined(separator: ", ") - let unique = index.isUnique ? "UNIQUE " : "" - return "CREATE \(unique)INDEX \(quoteIdentifier(index.name)) ON \(qualifiedTable) (\(cols))" + OracleCreateTableSQL.indexDefinition(index, qualifiedTable: qualifiedTable) } private func oracleForeignKeyConstraint(_ fk: PluginForeignKeyDefinition) -> String { - let cols = fk.columns.map { quoteIdentifier($0) }.joined(separator: ", ") - let refCols = fk.referencedColumns.map { quoteIdentifier($0) }.joined(separator: ", ") - let refTable: String - if let schema = fk.referencedSchema, !schema.isEmpty { - refTable = "\(quoteIdentifier(schema)).\(quoteIdentifier(fk.referencedTable))" - } else { - refTable = quoteIdentifier(fk.referencedTable) - } - let constraint = fk.name.isEmpty ? "" : "CONSTRAINT \(quoteIdentifier(fk.name)) " - var def = "\(constraint)FOREIGN KEY (\(cols)) REFERENCES \(refTable)" - if !refCols.isEmpty { - def += " (\(refCols))" - } - if fk.onDelete != "NO ACTION" { - def += " ON DELETE \(fk.onDelete)" - } - return def + OracleCreateTableSQL.foreignKeyConstraint(fk) } // MARK: - Schema Switching @@ -1161,9 +1004,7 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { core.noteSessionSchema(schema) } - /// Oracle has no real database concept; "switch database" is a schema switch. - /// Aliases to keep `coordinator.switchDatabase` working from tab restore paths - /// without relying on a manager-side kludge. + // Oracle has no database apart from its schemas, so tab restore's database switch is a schema switch. func switchDatabase(to database: String) async throws { try await switchSchema(to: database) } @@ -1278,12 +1119,26 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Private Helpers - /// Set at login before `core` is, so a driver with a core always has it. + // Set at login before `core` is, so a driver with a core always has it. func serverRelease() throws -> OracleServerRelease { guard let release = core?.serverRelease else { throw OraclePluginError(core: .notConnected) } return release } + // Without a core every read fails as not connected, so building the catalog fails the same way. + var exportCatalog: OracleExportCatalog { + get throws { + OracleExportCatalog( + run: { [self] query in try await rawQuery(query) }, + currentSchema: currentSchema, + fallbackSchema: config.username.uppercased(), + release: try serverRelease(), + principalState: principalState, + coreError: { ($0 as? OraclePluginError)?.core } + ) + } + } + func effectiveSchema(_ schema: String?) -> String { schema ?? currentSchema ?? config.username.uppercased() } diff --git a/apple/Plugins/OracleDriverPlugin/OraclePluginDriver+Principals.swift b/apple/Plugins/OracleDriverPlugin/OraclePluginDriver+Principals.swift index 237b7d66c4..0d1412926b 100644 --- a/apple/Plugins/OracleDriverPlugin/OraclePluginDriver+Principals.swift +++ b/apple/Plugins/OracleDriverPlugin/OraclePluginDriver+Principals.swift @@ -8,12 +8,6 @@ import os import TableProOracleCore import TableProPluginKit -struct OraclePrincipalState: Sendable { - var access: OracleDictionaryAccess? - var roles: Set = [] - var catalog: PluginPrivilegeCatalog? -} - // A schema is what this driver lists as a database, so a table's scope is `.table(database: owner, schema: nil, ...)`. extension OraclePluginDriver: PluginPrincipalManagement { var supportsRoleMembership: Bool { true } @@ -29,38 +23,7 @@ extension OraclePluginDriver: PluginPrincipalManagement { } func fetchPrincipals() async throws -> [PluginPrincipalInfo] { - let access = try await dictionaryAccess() - let users = OraclePrincipalQueries.parseUsers( - try await textRows(OraclePrincipalQueries.users(access: access, release: principalRelease)) - ) - let roles = OraclePrincipalQueries.parseNames(try await textRows(OraclePrincipalQueries.roles(access: access))) - let grants = OraclePrincipalQueries.parseRoleGrants( - try await textRows(OraclePrincipalQueries.roleGrants(access: access)) - ) - let roleNames = Set(roles) - principalState.withLock { $0.roles = roleNames } - - var memberOf: [String: Set] = [:] - for grant in grants where roleNames.contains(grant.role) { - memberOf[grant.grantee, default: []].insert(grant.role) - } - - let userInfos = users.filter { !roleNames.contains($0.name) }.map { user in - PluginPrincipalInfo( - ref: PluginPrincipalRef(name: user.name), - canLogin: !user.isLocked, - memberOf: (memberOf[user.name] ?? []).sorted() - ) - } - let roleInfos = roles.map { role in - PluginPrincipalInfo( - ref: PluginPrincipalRef(name: role), - isRole: true, - canLogin: false, - memberOf: (memberOf[role] ?? []).sorted() - ) - } - return userInfos + roleInfos + try await exportCatalog.fetchPrincipals() } func fetchPrivilegeCatalog() async throws -> PluginPrivilegeCatalog { @@ -80,18 +43,7 @@ extension OraclePluginDriver: PluginPrincipalManagement { } func fetchGrants(for principal: PluginPrincipalRef) async throws -> [PluginGrantInfo] { - let access = try await dictionaryAccess() - let name = principal.name - let system = OraclePrincipalQueries.parseSystemPrivileges( - try await textRows(OraclePrincipalQueries.systemPrivileges(grantee: name, access: access)) - ) - let objects = OraclePrincipalQueries.parseObjectPrivileges( - try await textRows(OraclePrincipalQueries.objectPrivileges(grantee: name, access: access)) - ) - let columns = OraclePrincipalQueries.parseColumnPrivileges( - try await textRows(OraclePrincipalQueries.columnPrivileges(grantee: name, access: access)) - ) - return (system + objects + columns).map(Self.pluginGrant) + try await exportCatalog.fetchGrants(for: principal) } func fetchGrantableChildren(of scope: PluginPrivilegeScope) async throws -> [PluginPrivilegeScope] { @@ -154,13 +106,11 @@ extension OraclePluginDriver: PluginPrincipalManagement { } func generateGrantSQL(changeSet: PluginPrincipalChangeSet) -> [String]? { - guard let grants = oracleGrants(changeSet.grantsToAdd) else { return nil } - return principalStatements.grantStatements(grants, to: changeSet.principal.name) + OracleExportCatalog.generateGrantSQL(changeSet: changeSet, roles: principalState.withLock { $0.roles }) } func generateRevokeSQL(changeSet: PluginPrincipalChangeSet) -> [String]? { - guard let grants = oracleGrants(changeSet.grantsToRemove) else { return nil } - return principalStatements.revokeStatements(grants, from: changeSet.principal.name) + OracleExportCatalog.generateRevokeSQL(changeSet: changeSet, roles: principalState.withLock { $0.roles }) } // MARK: - Helpers @@ -173,65 +123,12 @@ extension OraclePluginDriver: PluginPrincipalManagement { try? serverRelease() } - // Probed once per connection: an account that cannot read DBA_USERS reads the ALL_ and USER_ views instead. - private func dictionaryAccess() async throws -> OracleDictionaryAccess { - if let access = principalState.withLock({ $0.access }) { - return access - } - let access: OracleDictionaryAccess - do { - _ = try await rawQuery(OraclePrincipalQueries.dbaProbe) - access = .dba - } catch let error as OraclePluginError where OraclePrincipalQueries.isMissingView(error.core) { - access = .session - } - principalState.withLock { $0.access = access } - return access - } - private func textRows(_ query: String) async throws -> [[String?]] { - try await rawQuery(query).rows.map { $0.map(\.stringValue) } + try await exportCatalog.textRows(query) } private func names(_ query: String) async throws -> [String] { - OraclePrincipalQueries.parseNames(try await textRows(query)) - } - - private func oracleGrants(_ grants: [PluginGrantInfo]) -> [OracleGrant]? { - var converted: [OracleGrant] = [] - for grant in grants { - let securable: OracleSecurable - switch grant.scope { - case .server: - securable = .system - case let .table(owner, _, table), let .column(owner, _, table, _): - securable = .object(owner: owner, name: table) - case .database, .schema: - return nil - @unknown default: - return nil - } - converted.append(OracleGrant( - privilege: grant.privilege, - securable: securable, - column: grant.scope.columnName, - isGrantable: grant.isGrantable - )) - } - return converted - } - - private static func pluginGrant(_ grant: OracleGrant) -> PluginGrantInfo { - let scope: PluginPrivilegeScope - switch grant.securable { - case .system: - scope = .server - case let .object(owner, name): - scope = grant.column.map { - .column(database: owner, schema: nil, table: name, column: $0) - } ?? .table(database: owner, schema: nil, table: name) - } - return PluginGrantInfo(privilege: grant.privilege, scope: scope, isGrantable: grant.isGrantable) + try await exportCatalog.names(query) } private static func systemDescriptor(_ name: String) -> PluginPrivilegeDescriptor { diff --git a/apple/Plugins/OracleDriverPlugin/OraclePluginDriver+Routines.swift b/apple/Plugins/OracleDriverPlugin/OraclePluginDriver+Routines.swift index e697058980..37da96cbea 100644 --- a/apple/Plugins/OracleDriverPlugin/OraclePluginDriver+Routines.swift +++ b/apple/Plugins/OracleDriverPlugin/OraclePluginDriver+Routines.swift @@ -8,111 +8,23 @@ import TableProPluginKit extension OraclePluginDriver { func fetchRoutines(schema: String?) async throws -> [PluginRoutineInfo] { - let resolvedSchema = routineSchema(schema) - let result = try await execute(query: OracleObjectQueries.routineList(schema: resolvedSchema)) - return result.rows.compactMap { row -> PluginRoutineInfo? in - guard let name = row[safe: 0]?.asText else { return nil } - let objectType = (row[safe: 2]?.asText ?? "").uppercased() - var attributes: [PluginObjectAttribute] = [] - if let status = row[safe: 3]?.asText, !status.isEmpty { - attributes.append(PluginObjectAttribute(label: "Status", value: status)) - } - return PluginRoutineInfo( - name: name, - kind: objectType == "PROCEDURE" ? .procedure : .function, - schema: row[safe: 1]?.asText ?? resolvedSchema, - returnType: nil, - language: "PL/SQL", - argumentSignature: nil, - identity: objectType, - attributes: attributes - ) - } + try await exportCatalog.fetchRoutines(schema: schema) } func fetchRoutineDDL(_ routine: PluginRoutineInfo) async throws -> String { - let resolvedSchema = routineSchema(routine.schema) - let type = routine.kind == .procedure ? "PROCEDURE" : "FUNCTION" - let query = OracleObjectQueries.routineSource( - schema: resolvedSchema, - name: routine.name, - type: type - ) - let result = try await execute(query: query) - /// ALL_SOURCE holds one line per row and returns no rows at all when the caller cannot see - /// the object, which is a privilege answer rather than a missing one. - guard !result.rows.isEmpty else { - throw PluginObjectSourceError.insufficientPrivilege(routine.name) - } - let body = result.rows - .compactMap { $0[safe: 0]?.asText } - .joined() - .trimmingCharacters(in: .whitespacesAndNewlines) - guard !body.isEmpty else { - throw PluginObjectSourceError.insufficientPrivilege(routine.name) - } - return body.uppercased().hasPrefix("CREATE") ? body : "CREATE OR REPLACE \(body)" + try await exportCatalog.fetchRoutineDDL(routine) } - /// False on purpose. `triggerList` scopes a whole-schema read on OWNER, the trigger's own - /// schema, and a per-table read on TABLE_OWNER, the subject table's. Those are different - /// questions for a trigger one schema owns on another's table, and a caller holding a bare - /// table name cannot tell them apart, so a comparison would accept a trigger from outside its - /// scope and miss one inside it. + // False on purpose: a whole-schema trigger read scopes on OWNER and a per-table one on TABLE_OWNER, which differ + // for a trigger one schema owns on another's table, so a comparison would take one from outside its scope and + // miss one inside it. var providesBulkTriggerFetch: Bool { false } func fetchAllTriggers(schema: String?) async throws -> [PluginTriggerInfo] { - try await triggerList(schema: routineSchema(schema), table: nil) + try await exportCatalog.fetchAllTriggers(schema: schema) } func fetchTriggerDDL(_ trigger: PluginTriggerInfo) async throws -> String { - if let definition = trigger.definition, !definition.isEmpty { return definition } - let listed = try await triggerList(schema: routineSchema(trigger.schema), table: trigger.table) - guard let definition = listed.first(where: { $0.name == trigger.name })?.definition, - !definition.isEmpty - else { - throw PluginObjectSourceError.insufficientPrivilege(trigger.name) - } - return definition - } - - func triggerList(schema: String, table: String?) async throws -> [PluginTriggerInfo] { - let result = try await execute(query: OracleObjectQueries.triggerList(schema: schema, table: table)) - return result.rows.compactMap { row -> PluginTriggerInfo? in - guard let name = row[safe: 0]?.asText else { return nil } - let triggerType = row[safe: 3]?.asText ?? "" - let owner = row[safe: 2]?.asText - let body = row[safe: 10]?.asText - let definition = OracleObjectQueries.triggerDefinition(OracleTriggerSource( - name: name, - owner: owner, - tableOwner: row[safe: 9]?.asText, - description: row[safe: 7]?.asText, - whenClause: row[safe: 6]?.asText, - actionType: row[safe: 8]?.asText, - status: row[safe: 5]?.asText, - body: body - )) - var attributes: [PluginObjectAttribute] = [] - if let whenClause = row[safe: 6]?.asText, !whenClause.isEmpty { - attributes.append(PluginObjectAttribute(label: "When", value: whenClause)) - } - return PluginTriggerInfo( - name: name, - table: row[safe: 1]?.asText, - schema: owner ?? schema, - timing: OracleObjectQueries.timing(fromTriggerType: triggerType), - event: row[safe: 4]?.asText ?? "", - orientation: OracleObjectQueries.orientation(fromTriggerType: triggerType), - statement: body ?? definition, - definition: definition, - enabled: (row[safe: 5]?.asText ?? "").uppercased() == "ENABLED", - attributes: attributes - ) - } - } - - private func routineSchema(_ schema: String?) -> String { - effectiveSchema(schema?.isEmpty == false ? schema : nil) + try await exportCatalog.fetchTriggerDDL(trigger) } } diff --git a/apple/Plugins/OracleDriverPlugin/OraclePluginValues.swift b/apple/Plugins/OracleDriverPlugin/OraclePluginValues.swift new file mode 100644 index 0000000000..6aad4218fe --- /dev/null +++ b/apple/Plugins/OracleDriverPlugin/OraclePluginValues.swift @@ -0,0 +1,63 @@ +// +// OraclePluginValues.swift +// OracleDriverPlugin +// + +import Foundation +import TableProOracleCore +import TableProPluginKit + +nonisolated extension OracleRawCell { + var asPluginCell: PluginCellValue { + switch self { + case .null: return .null + case .string(let value): return .text(value) + case .bytes(let data): return .bytes(data) + } + } +} + +nonisolated extension PluginCellValue { + var oracleBindValue: OracleBindValue { + switch self { + case .null: return .null + case .text(let value): return .text(value) + case .bytes(let data): return .bytes(data) + } + } +} + +nonisolated extension OracleColumnRow { + var pluginColumnInfo: PluginColumnInfo { + PluginColumnInfo( + name: name, + dataType: displayType, + isNullable: isNullable, + isPrimaryKey: isPrimaryKey, + defaultValue: defaultValue, + extra: nil, + charset: nil, + collation: nil, + comment: nil, + identityKind: identityGeneration.map(\.identityKind), + isGenerated: isVirtual, + allowedValues: nil, + generationExpression: generationExpression, + generationKind: isVirtual ? .virtual : nil, + ddlSpelling: OracleTypeCatalog.ddlSpelling(forDeclaredType: displayType), + ddlDefault: nil, + ddlGenerationExpression: nil, + ddlCollation: nil, + classificationTypeName: classificationTypeName + ) + } +} + +nonisolated extension OracleIdentityGeneration { + var identityKind: IdentityKind { + switch self { + case .always: return .always + case .byDefault: return .byDefault + } + } +} diff --git a/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift b/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift index 14ee2761f0..eed435d5d5 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift @@ -20,7 +20,7 @@ import TableProSQLQuoting /// because a bound `123.45` fails with ORA-01722 once `NLS_NUMERIC_CHARACTERS` is `,.`; NULL is the literal, /// because a NULL bind into an object-type column fails with ORA-00932; and text for an NCHAR, NVARCHAR2 or NCLOB /// column is `UNISTR`, because a literal or bind passes through the database character set (``OracleNationalText``). -internal struct OracleRowWriter { +nonisolated internal struct OracleRowWriter { /// A keyed delete carries at most this many key values. One statement with 32,767 binds took 30 seconds on 23ai /// and one with 1,000 took 0.07, and a statement that outlives the query timeout closes the session. static let maxValuesPerDelete = 1_000 @@ -289,7 +289,7 @@ internal struct OracleRowWriter { } /// How a value is written for the type of the column it goes into. -internal enum OracleValueKind: Equatable { +nonisolated internal enum OracleValueKind: Equatable { case date case timestamp case timestampWithTimeZone @@ -399,7 +399,7 @@ internal enum OracleValueKind: Equatable { /// An Oracle numeric literal in ASCII: an optional sign, digits with at most one decimal point, and an optional /// exponent. It is written into the statement unquoted, so nothing else may pass. -internal enum OracleNumericLiteral { +nonisolated internal enum OracleNumericLiteral { static func isValid(_ text: String) -> Bool { var bytes = Array(text.utf8)[...] if let first = bytes.first, first == UInt8(ascii: "+") || first == UInt8(ascii: "-") { @@ -434,7 +434,7 @@ internal enum OracleNumericLiteral { /// The `BFILENAME('DIR', 'file')` text a BFILE reads as, taken apart only in exactly that shape: two single-quoted /// literals with `''` escapes and nothing else. Both names are quoted again, so nothing from the text reaches the /// statement unescaped. -internal enum OracleBFileText { +nonisolated internal enum OracleBFileText { static func call(from text: String) -> String? { guard let names = names(in: text), let directory = OracleBindPlaceholders.quotedLiteral(names.directory), diff --git a/apple/Plugins/OracleDriverPlugin/OracleStatementError.swift b/apple/Plugins/OracleDriverPlugin/OracleStatementError.swift new file mode 100644 index 0000000000..ca12d49342 --- /dev/null +++ b/apple/Plugins/OracleDriverPlugin/OracleStatementError.swift @@ -0,0 +1,14 @@ +// +// OracleStatementError.swift +// OracleDriverPlugin +// + +import Foundation +import TableProPluginKit + +// Thrown by the files the iOS app compiles too, so it reads the same as the driver's own query failure. +nonisolated struct OracleStatementError: PluginDriverError { + let message: String + + var pluginErrorMessage: String { message } +} diff --git a/apple/Plugins/OracleDriverPlugin/OracleTableDDL.swift b/apple/Plugins/OracleDriverPlugin/OracleTableDDL.swift index 215d9e4ee4..bd61ba8c23 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleTableDDL.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleTableDDL.swift @@ -16,7 +16,7 @@ import TableProSQLGrammar /// Each column is written in the order Oracle's grammar takes it, measured on 23ai: the type verbatim, then /// `GENERATED ALWAYS AS (...) VIRTUAL` or `DEFAULT`, then `NOT NULL`. `NOT NULL` before `DEFAULT` fails with /// ORA-03076, and a type folded to upper case names no type at all when it is a quoted mixed-case object type. -internal enum OracleTableDDL { +nonisolated internal enum OracleTableDDL { // The keys and checks follow the columns. A NOT NULL the column definition writes is not written again as a check. static func createTable( qualifiedTable: String, @@ -47,10 +47,10 @@ internal enum OracleTableDDL { private static func checkCondition(of constraint: OracleTableConstraint) throws -> String { let condition = SQLExpressionFragment.strippingComments(constraint.condition ?? "", databaseTypeId: "Oracle") guard !constraint.conditionMayBeCut, !condition.isEmpty, isSingleExpression(condition) else { - throw OraclePluginError(core: .queryFailed(String( + throw OracleStatementError(message: String( format: String(localized: "The condition of check constraint %@ cannot be written as SQL."), constraint.name - ))) + )) } return condition } @@ -114,7 +114,7 @@ internal enum OracleTableDDL { SQLExpressionFragment.isSingleExpression(text, databaseTypeId: "Oracle") } - private static func refusal(_ format: String, _ column: PluginColumnInfo) -> OraclePluginError { - OraclePluginError(core: .queryFailed(String(format: format, column.name))) + private static func refusal(_ format: String, _ column: PluginColumnInfo) -> OracleStatementError { + OracleStatementError(message: String(format: format, column.name)) } } diff --git a/apple/Plugins/OracleDriverPlugin/OracleTypeCatalog.swift b/apple/Plugins/OracleDriverPlugin/OracleTypeCatalog.swift index 84feff894b..2ae9a09bab 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleTypeCatalog.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleTypeCatalog.swift @@ -7,7 +7,7 @@ import Foundation import TableProPluginKit import TableProSQLVocabulary -internal enum OracleTypeCatalog { +nonisolated internal enum OracleTypeCatalog { static let columnTypesByCategory: [String: [String]] = Dictionary( uniqueKeysWithValues: SQLEngineVocabulary.oracleDataTypeCategories.map { ($0.name, $0.types) } ) @@ -85,7 +85,7 @@ internal enum OracleTypeCatalog { } } -extension PluginQueryResult { +nonisolated extension PluginQueryResult { /// The same result with the Oracle classification hints its columns need, for a result built without them. func withOracleClassificationHints() -> PluginQueryResult { guard columnMeta == nil, diff --git a/apple/Plugins/PostgreSQLDriverPlugin/ColumnQueryShape.swift b/apple/Plugins/PostgreSQLDriverPlugin/ColumnQueryShape.swift index 241724d40e..ef836a8e38 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/ColumnQueryShape.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/ColumnQueryShape.swift @@ -5,7 +5,7 @@ import Foundation -enum ColumnQueryShape { +nonisolated enum ColumnQueryShape { struct Fragments { let selectPrefix: String let tableLiteral: String? diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQCellDecoding.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQCellDecoding.swift index ba054532ba..f08239bca5 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQCellDecoding.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQCellDecoding.swift @@ -7,7 +7,7 @@ import Foundation import TableProPluginKit import TableProPostgreSQLCore -internal enum LibPQCellDecoding { +nonisolated internal enum LibPQCellDecoding { private static let booleanOid: UInt32 = 16 private static let byteaOid: UInt32 = 17 diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQParameterBuffers.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQParameterBuffers.swift index d353374ac2..674f1f14d8 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQParameterBuffers.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQParameterBuffers.swift @@ -8,7 +8,7 @@ import TableProPluginKit // The C buffers `PQexecParams` and `PQsendQueryParams` read the parameters from: text as a C string, bytes in binary // format, null as a null pointer. -internal enum LibPQParameterBuffers { +nonisolated internal enum LibPQParameterBuffers { // libpq reads the buffers only while the call sends them, so they live for `body` alone. static func with( _ parameters: [PluginCellValue], @@ -35,7 +35,7 @@ internal enum LibPQParameterBuffers { formats.append(0) case .text(let text): guard let cString = strdup(text) else { - throw LibPQPluginError(message: "Failed to allocate parameter buffer", sqlState: nil, detail: nil) + throw LibPQParameterBufferError() } allocations.append(UnsafeMutableRawPointer(cString)) values.append(UnsafePointer(cString)) @@ -44,7 +44,7 @@ internal enum LibPQParameterBuffers { case .bytes(let data): let byteCount = data.count guard let raw = malloc(max(byteCount, 1)) else { - throw LibPQPluginError(message: "Failed to allocate parameter buffer", sqlState: nil, detail: nil) + throw LibPQParameterBufferError() } allocations.append(raw) if byteCount > 0 { @@ -63,3 +63,7 @@ internal enum LibPQParameterBuffers { } } } + +nonisolated struct LibPQParameterBufferError: PluginDriverError { + var pluginErrorMessage: String { "Failed to allocate parameter buffer" } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift index 9c37479dd3..e983703ad0 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift @@ -863,24 +863,19 @@ final class LibPQPluginConnection: @unchecked Sendable { guard state == .inTransaction else { return try executeParameterizedQuerySync(statementToRun, parameters: params) } - _ = try executeQuerySync(Self.sessionSettingSavepoint) + _ = try executeQuerySync(PostgreSQLSessionScopeSQL.settingSavepoint) let result: LibPQPluginQueryResult do { result = try executeParameterizedQuerySync(statementToRun, parameters: params) } catch { - _ = try? executeQuerySync(Self.sessionSettingRollback) + _ = try? executeQuerySync(PostgreSQLSessionScopeSQL.settingRollback) throw error } - _ = try executeQuerySync(Self.sessionSettingRelease) + _ = try executeQuerySync(PostgreSQLSessionScopeSQL.settingRelease) return result } } - private static let sessionSettingSavepoint = "SAVEPOINT tablepro_session_setting" - private static let sessionSettingRelease = "RELEASE SAVEPOINT tablepro_session_setting" - private static let sessionSettingRollback = - "ROLLBACK TO SAVEPOINT tablepro_session_setting; RELEASE SAVEPOINT tablepro_session_setting" - // Whether the session is inside a transaction block, including one a failed statement has // aborted, so a statement sent now joins it. Called on the connection's queue only: one `PGconn` // may not be used from two threads at once, and the lock guards the pointer rather than the call. diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCatalogExport.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCatalogExport.swift new file mode 100644 index 0000000000..a92d5f8fa6 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCatalogExport.swift @@ -0,0 +1,910 @@ +// +// PostgreSQLCatalogExport.swift +// PostgreSQLDriverPlugin +// + +import Foundation +import os +import TableProLogRedaction +import TableProPluginKit +import TableProPostgreSQLCore + +// The catalog reads and statements an export needs, over whichever connection runs them. The plugin driver and the +// iOS driver each build one with their own read. +nonisolated struct PostgreSQLCatalogExport: Sendable { + // `namesRelativeTo` nil runs the read with only pg_catalog on the path; a schema puts that schema first. + typealias Read = @Sendable (_ statement: String, _ namesRelativeTo: String?) async throws -> [[PluginCellValue]] + + static let tableDDLIncludesForeignKeys = false + + private static let logger = Logger(subsystem: "com.TablePro.PostgreSQLDriver", category: "PostgreSQLPluginDriver") + private static let sequenceLogger = Logger(subsystem: "com.TablePro.PostgreSQLDriver", category: "Sequences") + + let read: Read + let currentSchema: String + // The server version the session resolved: what the reads' column choices follow. + let capabilities: PostgreSQLCapabilities + // The reported version, read as modern when unknown: what the listing reads follow. + let catalogCapabilities: PostgreSQLCapabilities + let catalogPresence: PostgreSQLCatalogPresence? + + init( + read: @escaping Read, + currentSchema: String, + capabilities: PostgreSQLCapabilities, + catalogCapabilities: PostgreSQLCapabilities, + catalogPresence: PostgreSQLCatalogPresence? + ) { + self.read = read + self.currentSchema = currentSchema + self.capabilities = capabilities + self.catalogCapabilities = catalogCapabilities + self.catalogPresence = catalogPresence + } + + var includesMaterializedViews: Bool { + catalogPresence?.hasMaterializedViews ?? capabilities.hasMaterializedViewsCatalog + } + + var includesSequencesCatalog: Bool { + catalogPresence?.hasSequences ?? capabilities.hasSequencesCatalog + } + + private func rows(_ statement: String, namesRelativeTo schema: String? = nil) async throws -> [[PluginCellValue]] { + try await read(statement, schema) + } + + // MARK: - Routines and triggers + + func fetchRoutines(schema: String?) async throws -> [PluginRoutineInfo] { + let resolvedSchema = schema ?? currentSchema + let query = PostgreSQLObjectQueries.routineList(schema: resolvedSchema, capabilities: catalogCapabilities) + return try await rows(query).compactMap { row -> PluginRoutineInfo? in + guard let name = row[safe: 1]?.asText else { return nil } + let kind: PluginRoutineKind = row[safe: 6]?.asText == "p" ? .procedure : .function + return PluginRoutineInfo( + name: name, + kind: kind, + schema: row[safe: 2]?.asText ?? resolvedSchema, + returnType: row[safe: 4]?.asText, + language: row[safe: 5]?.asText, + argumentSignature: row[safe: 3]?.asText, + identity: row[safe: 0]?.asText, + attributes: Self.routineAttributes(row) + ) + } + } + + func fetchRoutineDDL(_ routine: PluginRoutineInfo) async throws -> String { + let resolvedSchema = routine.schema ?? currentSchema + let query: String + if let identity = routine.identity, !identity.isEmpty, Int(identity) != nil { + query = PostgreSQLObjectQueries.routineDefinition(identity: identity) + } else { + query = PostgreSQLObjectQueries.routineDefinitionByName( + name: routine.name, + schema: resolvedSchema, + arguments: routine.argumentSignature + ) + } + guard let ddl = try await rows(query).first?[safe: 0]?.asText, !ddl.isEmpty else { + throw PluginObjectSourceError.notFound(routine.name) + } + return ddl + } + + func fetchAllTriggers(schema: String?) async throws -> [PluginTriggerInfo] { + let resolvedSchema = schema ?? currentSchema + let query = PostgreSQLObjectQueries.triggerList(schema: resolvedSchema, table: nil) + return try await rows(query).compactMap { Self.trigger(from: $0, fallbackSchema: resolvedSchema) } + } + + // The schema-wide builder with one more predicate, so one table's triggers never disagree with the list. + func fetchTriggers(table: String, schema: String?) async throws -> [PluginTriggerInfo] { + let resolvedSchema = schema ?? currentSchema + let query = PostgreSQLObjectQueries.triggerList(schema: resolvedSchema, table: table) + let result = try await rows(query) + let triggers = result.compactMap { Self.trigger(from: $0, fallbackSchema: resolvedSchema) } + Self.logger.info("[trigger] postgres fetchTriggers schema=\(resolvedSchema, privacy: .private(mask: .hash)) table=\(table, privacy: .private(mask: .hash)) rows=\(result.count) parsed=\(triggers.count)") + return triggers + } + + func fetchTriggerDDL(_ trigger: PluginTriggerInfo) async throws -> String { + if let definition = trigger.definition, !definition.isEmpty { return definition } + let resolvedSchema = trigger.schema ?? currentSchema + let query = PostgreSQLObjectQueries.triggerList(schema: resolvedSchema, table: trigger.table) + let match = try await rows(query) + .compactMap { Self.trigger(from: $0, fallbackSchema: resolvedSchema) } + .first { $0.name == trigger.name } + guard let definition = match?.definition, !definition.isEmpty else { + throw PluginObjectSourceError.notFound(trigger.name) + } + return definition + } + + static func trigger(from row: [PluginCellValue], fallbackSchema: String) -> PluginTriggerInfo? { + guard let name = row[safe: 0]?.asText, + let definition = row[safe: 7]?.asText + else { return nil } + var attributes: [PluginObjectAttribute] = [] + if let owner = row[safe: 8]?.asText, !owner.isEmpty { + attributes.append(PluginObjectAttribute(label: "Owner", value: owner)) + } + return PluginTriggerInfo( + name: name, + table: row[safe: 1]?.asText, + schema: row[safe: 2]?.asText ?? fallbackSchema, + timing: row[safe: 3]?.asText ?? "", + event: row[safe: 4]?.asText ?? "", + orientation: row[safe: 5]?.asText, + statement: definition, + definition: definition, + enabled: PostgreSQLCatalogBoolean.isTrue(row[safe: 6]?.asText), + attributes: attributes + ) + } + + private static func routineAttributes(_ row: [PluginCellValue]) -> [PluginObjectAttribute] { + var attributes: [PluginObjectAttribute] = [] + if let volatility = row[safe: 7]?.asText, !volatility.isEmpty { + attributes.append(PluginObjectAttribute(label: "Volatility", value: volatility)) + } + if let security = row[safe: 8]?.asText, !security.isEmpty { + attributes.append(PluginObjectAttribute(label: "Security", value: security)) + } + if let owner = row[safe: 9]?.asText, !owner.isEmpty { + attributes.append(PluginObjectAttribute(label: "Owner", value: owner)) + } + return attributes + } + + // PostgreSQL allows `f(integer)` and `f(text)` in one schema, so a drop that names only `f` is ambiguous. + func generateDropRoutineSQL(name: String, signature: String?, schema: String?, isFunction: Bool) -> String { + let keyword = isFunction ? "FUNCTION" : "PROCEDURE" + let arguments = (signature ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + let qualified = PostgreSQLObjectQueries.qualifiedName(schema: schema ?? currentSchema, name: name) + return "DROP \(keyword) IF EXISTS \(qualified)\(arguments)" + } + + func generateDropTriggerSQL(name: String, table: String, schema: String?) -> String { + let qualified = PostgreSQLObjectQueries.qualifiedName(schema: schema ?? currentSchema, name: table) + return "DROP TRIGGER IF EXISTS \(PostgreSQLObjectQueries.quoteIdentifier(name)) ON \(qualified)" + } + + // MARK: - Types + + func fetchUserDefinedTypes(schema: String?) async throws -> [PluginUserDefinedTypeInfo] { + let query = PostgreSQLObjectQueries.userDefinedTypeList( + schema: schema ?? currentSchema, + identity: nil, + capabilities: catalogCapabilities + ) + return try await rows(query) + .compactMap(PostgreSQLTypeDefinition.record(from:)) + .map(PostgreSQLTypeDefinition.info(from:)) + } + + // An identity is the oid this catalog handed out, looked up without a schema predicate because a moved type keeps + // it; one that is not an oid is refused rather than widened into a listing. + func fetchUserDefinedType(_ type: PluginUserDefinedTypeInfo) async throws -> PluginUserDefinedTypeInfo { + if let identity = type.identity { + guard let oid = UInt32(identity) else { throw PluginObjectSourceError.notFound(type.name) } + let query = PostgreSQLObjectQueries.userDefinedTypeList( + schema: nil, + identity: String(oid), + capabilities: catalogCapabilities + ) + let match = try await rows(query) + .compactMap(PostgreSQLTypeDefinition.record(from:)) + .first { $0.identity == String(oid) } + guard let match else { throw PluginObjectSourceError.notFound(type.name) } + return PostgreSQLTypeDefinition.info(from: match) + } + let query = PostgreSQLObjectQueries.userDefinedTypeList( + schema: type.schema ?? currentSchema, + identity: nil, + capabilities: catalogCapabilities + ) + let match = try await rows(query) + .compactMap(PostgreSQLTypeDefinition.record(from:)) + .first { $0.name == type.name } + guard let match else { throw PluginObjectSourceError.notFound(type.name) } + return PostgreSQLTypeDefinition.info(from: match) + } + + func fetchDependentTypes(table: String, schema: String?) async throws -> [(name: String, labels: [String])] { + let query = PostgreSQLSchemaQueries.dependentEnumTypes(schema: schema ?? currentSchema, table: table) + return try await rows(query).compactMap { row -> (name: String, labels: [String])? in + guard let typeName = row[0].asText, let labelsStr = row[1].asText else { return nil } + return (name: typeName, labels: PostgreSQLTextArray.values(labelsStr)) + } + } + + // MARK: - Sequences + + func fetchDependentSequences(table: String, schema: String?) async throws -> [(name: String, ddl: String)] { + let definitions = try await sequenceDefinitions(schema: schema ?? currentSchema, dependentOnTable: table) + return definitions.map { (name: $0.name, ddl: $0.ddl) } + } + + func fetchSequences(schema: String?) async throws -> [PluginSequenceInfo] { + let schemaName = schema ?? currentSchema + let definitions = try await sequenceDefinitions(schema: schemaName, dependentOnTable: nil) + return definitions.map { PluginSequenceInfo(name: $0.name, ddl: $0.ddl, schema: schemaName) } + } + + func fetchSequenceDetails(sequence: String, schema: String?) async throws -> PluginSequenceDetails? { + let schemaName = schema ?? currentSchema + let definitions = try await sequenceDefinitions(schema: schemaName, dependentOnTable: nil, named: sequence) + guard let definition = definitions.first else { return nil } + return definition.details(schema: schemaName, position: await position(schema: schemaName, sequence: sequence)) + } + + func sequenceDDL(sequence: String, schema: String) async throws -> String? { + let definitions = try await sequenceDefinitions(schema: schema, dependentOnTable: nil, named: sequence) + return definitions.first?.qualifiedDDL(schema: schema) + } + + // Needs SELECT on the sequence; a role without it gets no position rather than a guess. + private func position(schema: String, sequence: String) async -> PostgreSQLSequencePosition? { + do { + let query = PostgreSQLSequencePosition.query(schema: schema, sequence: sequence) + return PostgreSQLSequencePosition(row: try await rows(query).first) + } catch { + Self.sequenceLogger.debug( + "Sequence position unavailable for \(sequence, privacy: .private(mask: .hash)): \(error.localizedDescription)" + ) + return nil + } + } + + private func sequenceDefinitions( + schema: String, + dependentOnTable table: String?, + named sequence: String? = nil + ) async throws -> [PostgreSQLSequenceDefinition] { + let source = PostgreSQLSequenceQueries.source(hasSequencesCatalog: includesSequencesCatalog) + let listing = try await rows( + PostgreSQLSequenceQueries.sequenceList(schema: schema, dependentOnTable: table, source: source, named: sequence) + ) + var definitions: [PostgreSQLSequenceDefinition] = [] + for definition in PostgreSQLSequenceQueries.definitions(from: listing) { + guard definition.needsLastValueRead else { + definitions.append(definition) + continue + } + definitions.append(definition.withLastValue(await lastValue(schema: schema, sequence: definition.name))) + } + return definitions + } + + private func lastValue(schema: String, sequence: String) async -> String? { + do { + let query = PostgreSQLSequenceQueries.lastValue(schema: schema, sequence: sequence) + return try await rows(query).first?[safe: 0]?.asText + } catch { + Self.sequenceLogger.debug( + "Sequence last value unavailable for \(sequence, privacy: .private(mask: .hash)): \(error.localizedDescription)" + ) + return nil + } + } + + // MARK: - Columns + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + let resolvedSchema = schema ?? currentSchema + let catalog = try await fetchTypeCatalog(schema: resolvedSchema, table: table) + let query = PostgreSQLSchemaQueries.columnsQuery( + schema: resolvedSchema, + table: table, + capabilities: capabilities, + includeMaterializedViews: includesMaterializedViews + ) + let result = try await rows(query) + let spellings = try await fetchColumnSpellings(schema: resolvedSchema, table: table)[table] ?? [:] + let columnDDL = try await fetchColumnDDL(schema: resolvedSchema, table: table)[table] ?? [:] + return result.compactMap { row in + Self.column(row, tableNameOffset: 0, catalog: catalog, spellings: spellings, columnDDL: columnDDL) + } + } + + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { + let resolvedSchema = schema ?? currentSchema + let catalog = try await fetchTypeCatalog(schema: resolvedSchema, table: nil) + let query = PostgreSQLSchemaQueries.columnsQuery( + schema: resolvedSchema, + table: nil, + capabilities: capabilities, + includeMaterializedViews: includesMaterializedViews + ) + let result = try await rows(query) + let spellings = try await fetchColumnSpellings(schema: resolvedSchema, table: nil) + let columnDDL = try await fetchColumnDDL(schema: resolvedSchema, table: nil) + var allColumns: [String: [PluginColumnInfo]] = [:] + for row in result { + guard row.count >= 5, let tableName = row[0].asText else { continue } + let tableDDL = columnDDL[tableName] ?? [:] + let tableSpellings = spellings[tableName] ?? [:] + if let column = Self.column( + row, tableNameOffset: 1, catalog: catalog, spellings: tableSpellings, columnDDL: tableDDL + ) { + allColumns[tableName, default: []].append(column) + } + } + return allColumns + } + + private func fetchColumnSpellings( + schema: String, + table: String? + ) async throws -> [String: [String: PostgreSQLColumnSpelling]] { + let query = PostgreSQLSchemaQueries.columnSpellingQuery(schema: schema, table: table, capabilities: capabilities) + return PostgreSQLSchemaQueries.columnSpellings(rows: try await rows(query, namesRelativeTo: schema)) + } + + private func fetchColumnDDL( + schema: String, + table: String? + ) async throws -> [String: [String: PostgreSQLCatalogColumnDDL]] { + let query = PostgreSQLSchemaQueries.columnDDLQuery(schema: schema, table: table, capabilities: capabilities) + return PostgreSQLSchemaQueries.columnDDL(rows: try await rows(query)) + } + + private func fetchEnumLabelMap(schema: String, table: String?) async throws -> [String: [String]] { + var map: [String: [String]] = [:] + for row in try await rows(PostgreSQLSchemaQueries.enumLabelQuery(schema: schema, table: table)) { + guard let schemaName = row[safe: 0]?.asText, + let typeName = row[safe: 1]?.asText else { continue } + let key = PostgresColumnTypeResolver.qualifiedName(schema: schemaName, name: typeName) + var labels = map[key] ?? [] + if let label = row[safe: 2]?.asText { + labels.append(label) + } + map[key] = labels + } + return map + } + + private func fetchArrayTypeMap(schema: String, table: String?) async throws -> [String: PostgresArrayTypeInfo] { + var map: [String: PostgresArrayTypeInfo] = [:] + for row in try await rows(PostgreSQLSchemaQueries.arrayTypeQuery(schema: schema, table: table)) { + guard let schemaName = row[safe: 0]?.asText, + let arrayTypeName = row[safe: 1]?.asText, + let elementTypeName = row[safe: 2]?.asText, + let elementKind = row[safe: 3]?.asText?.first else { continue } + let key = PostgresColumnTypeResolver.qualifiedName(schema: schemaName, name: arrayTypeName) + map[key] = PostgresArrayTypeInfo(elementTypeName: elementTypeName, elementTypeKind: elementKind) + } + return map + } + + // Scoped to the types the read's own columns use and read fresh: a cache goes stale on the next `CREATE TYPE`. + private func fetchTypeCatalog(schema: String, table: String?) async throws -> PostgresTypeCatalog { + let enumLabels = try await fetchEnumLabelMap(schema: schema, table: table) + let arrayTypes = try await fetchArrayTypeMap(schema: schema, table: table) + return PostgresTypeCatalog(enumLabels: enumLabels, arrayTypes: arrayTypes) + } + + private static func column( + _ row: [PluginCellValue], + tableNameOffset: Int, + catalog: PostgresTypeCatalog, + spellings: [String: PostgreSQLColumnSpelling], + columnDDL: [String: PostgreSQLCatalogColumnDDL] + ) -> PluginColumnInfo? { + let nameIdx = tableNameOffset + let typeIdx = tableNameOffset + 1 + let nullableIdx = tableNameOffset + 2 + let collationIdx = tableNameOffset + 4 + let commentIdx = tableNameOffset + 5 + let udtIdx = tableNameOffset + 6 + let pkIdx = tableNameOffset + 7 + let identityIdx = tableNameOffset + 8 + let generatedIdx = tableNameOffset + 9 + let udtSchemaIdx = tableNameOffset + 10 + let domainNameIdx = tableNameOffset + 13 + + guard row.count > typeIdx, + let name = row[nameIdx].asText, + let rawDataType = row[typeIdx].asText + else { return nil } + let spelling = spellings[name] + + let udtName = row.count > udtIdx ? row[udtIdx].asText : nil + let udtSchema = row.count > udtSchemaIdx ? row[udtSchemaIdx].asText : nil + let resolution = PostgresColumnTypeResolver.resolve( + rawDataType: rawDataType, + udtSchema: udtSchema, + udtName: udtName, + enumLabelsByQualifiedName: catalog.enumLabels, + arrayTypesByQualifiedName: catalog.arrayTypes + ) + let typeSpelling = PostgreSQLColumnTypeSpelling.resolve( + declaredType: spelling?.declaredType, + informationSchemaType: rawDataType, + domainName: row.count > domainNameIdx ? row[domainNameIdx].asText : nil, + udtSchema: udtSchema, + resolved: resolution + ) + + let isNullable = row.count > nullableIdx && row[nullableIdx].asText == "YES" + let collation = row.count > collationIdx ? row[collationIdx].asText : nil + let comment = row.count > commentIdx ? row[commentIdx].asText : nil + let isPk = row.count > pkIdx && row[pkIdx].asText == "YES" + let attidentity = row.count > identityIdx ? row[identityIdx].asText : nil + let attgenerated = row.count > generatedIdx ? row[generatedIdx].asText : nil + + let charset: String? = { + guard let coll = collation, coll.contains(".") else { return nil } + return coll.components(separatedBy: ".").last + }() + + return PluginColumnInfo( + name: name, + dataType: typeSpelling.dataType, + isNullable: isNullable, + isPrimaryKey: isPk, + defaultValue: spelling?.defaultValue, + charset: charset, + collation: collation, + comment: comment?.isEmpty == false ? comment : nil, + identityKind: identityKind(attidentity), + isGenerated: attgenerated == "s" || attgenerated == "v", + allowedValues: resolution.allowedValues, + generationExpression: spelling?.generationExpression, + generationKind: generationKind(attgenerated), + ddlSpelling: columnDDL[name]?.typeSpelling, + ddlDefault: columnDDL[name]?.defaultExpression, + ddlGenerationExpression: columnDDL[name]?.generationExpression, + ddlCollation: columnDDL[name]?.collation, + classificationTypeName: typeSpelling.classificationTypeName + ) + } + + // PostgreSQL 18 added virtual generated columns ('v'); reading only 's' left them writable. + private static func generationKind(_ attgenerated: String?) -> GenerationKind? { + switch attgenerated { + case "s": return .stored + case "v": return .virtual + default: return nil + } + } + + private static func identityKind(_ attidentity: String?) -> IdentityKind? { + switch attidentity { + case "a": return .always + case "d": return .byDefault + default: return nil + } + } + + // MARK: - Foreign keys + + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + let resolvedSchema = schema ?? currentSchema + let query = PostgreSQLForeignKeyQueries.foreignKeyList( + schema: resolvedSchema, table: table, capabilities: catalogCapabilities + ) + let result = try await rows(query) + let foreignKeys = result.compactMap { PostgreSQLForeignKeyRow($0)?.foreignKey } + Self.logger.info("[fk] postgres fetchForeignKeys schema=\(resolvedSchema, privacy: .private(mask: .hash)) table=\(table, privacy: .private(mask: .hash)) rows=\(result.count) parsed=\(foreignKeys.count)") + return foreignKeys + } + + func fetchAllForeignKeys(schema: String?) async throws -> [String: [PluginForeignKeyInfo]] { + let query = PostgreSQLForeignKeyQueries.foreignKeyList( + schema: schema ?? currentSchema, table: nil, capabilities: catalogCapabilities + ) + var grouped: [String: [PluginForeignKeyInfo]] = [:] + for row in try await rows(query) { + guard let decoded = PostgreSQLForeignKeyRow(row) else { continue } + grouped[decoded.table, default: []].append(decoded.foreignKey) + } + return grouped + } + + // MARK: - Definitions + + func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { + let query = PostgreSQLSchemaQueries.approximateRowCount(schema: schema ?? currentSchema, table: table) + return PostgreSQLSchemaQueries.rowEstimate(reltuples: try await rows(query).first?[safe: 0]?.asText).map { Int($0) } + } + + func fetchTableDDL(table: String, schema: String?) async throws -> String { + let resolvedSchema = schema ?? currentSchema + let columnsQuery = PostgreSQLSchemaQueries.tableDDLColumnsQuery( + schema: resolvedSchema, table: table, capabilities: capabilities + ) + let constraintsQuery = PostgreSQLSchemaQueries.tableDDLConstraintsQuery(schema: resolvedSchema, table: table) + let shapeQuery = PostgreSQLTableDDL.shapeQuery(schema: resolvedSchema, table: table, capabilities: capabilities) + + async let columnsResult = rows(columnsQuery) + async let constraintsResult = rows(constraintsQuery) + async let shapeResult = try? rows(shapeQuery) + + let (cols, cons) = try await (columnsResult, constraintsResult) + let shapeRows = await shapeResult + + // pg_attribute covers views too, which would otherwise come back as a `CREATE TABLE` of their columns. + if let relkind = cols.first?[safe: 1]?.asText, + PostgreSQLViewDefinition.kind(forRelkind: relkind) != nil { + return try await fetchViewDefinition(view: table, schema: resolvedSchema) + } + // A sequence's attributes are its state columns, not a table to recreate. + if cols.first?[safe: 1]?.asText == "S", + let ddl = try await sequenceDDL(sequence: table, schema: resolvedSchema) { + return ddl + } + + let columnDefs = cols.compactMap { $0[0].asText } + guard !columnDefs.isEmpty else { + throw PostgreSQLCatalogReadError(message: "Failed to fetch DDL for table '\(table)'") + } + + var parts = columnDefs + parts.append(contentsOf: cons.compactMap { $0[0].asText }) + + var shape = PostgreSQLTableDDL.shape(fromRows: shapeRows?.map { $0.map(\.asText) }) + var partitions: [PostgreSQLTableDDL.Partition] = [] + var partitionIndexes: [String] = [] + var partitionComments: [String] = [] + if shape.partitionKey != nil { + let tree = try await rows(PostgreSQLTableDDL.partitionTreeQuery(schema: resolvedSchema, table: table)) + partitions = tree.compactMap { PostgreSQLTableDDL.partition(from: $0.map(\.asText)) } + if !partitions.isEmpty { + async let elementRows = rows(PostgreSQLTableDDL.partitionElementsQuery( + schema: resolvedSchema, table: table, scope: .below, capabilities: capabilities + )) + async let indexRows = rows(PostgreSQLTableDDL.partitionIndexesQuery(schema: resolvedSchema, table: table)) + async let commentRows = rows(PostgreSQLTableDDL.partitionCommentsQuery(schema: resolvedSchema, table: table)) + let elements = PostgreSQLTableDDL.elementsByPartition(try await elementRows.map { $0.map(\.asText) }) + partitions = partitions.map { partition in + var partition = partition + partition.elements = elements[partition.name] ?? [] + return partition + } + partitionIndexes = try await indexRows.compactMap { $0.first?.asText } + partitionComments = PostgreSQLTableDDL.partitionComments(try await commentRows.map { $0.map(\.asText) }) + } + } else if shape.partitionOf != nil { + let elementRows = try await rows(PostgreSQLTableDDL.partitionElementsQuery( + schema: resolvedSchema, table: table, scope: .itself, capabilities: capabilities + )) + shape.partitionElements = PostgreSQLTableDDL.elementsByPartition(elementRows.map { $0.map(\.asText) }) + .values.first ?? [] + } + + return PostgreSQLTableDDL.script( + name: PostgreSQLObjectQueries.qualifiedName(schema: resolvedSchema, name: table), + definitions: parts, + shape: shape, + partitions: partitions, + partitionIndexes: partitionIndexes, + partitionComments: partitionComments + ) + } + + // Rebuilt from the catalog with every name qualified: pg_views lost the options and the check option. + func fetchViewDefinition(view: String, schema: String?) async throws -> String { + let resolvedSchema = schema ?? currentSchema + let query = PostgreSQLViewDefinition.catalogQuery(name: view, schema: resolvedSchema) + guard let row = try await rows(query).first, + let catalogRow = PostgreSQLViewDefinition.parse(row: row.map(\.asText)) + else { + throw PostgreSQLCatalogReadError(message: "Failed to fetch definition for view '\(view)'") + } + return PostgreSQLViewDefinition.statement(name: view, schema: resolvedSchema, row: catalogRow) + } + + // `pg_get_indexdef` is what pg_dump emits, and it qualifies the table whatever the path holds. + func fetchIndexDDL(table: String, schema: String?) async throws -> [String] { + try await fetchStandaloneIndexes(table: table, schema: schema ?? currentSchema).definitions + .map(PostgreSQLTableDDL.indexCoveringPartitions) + } + + func fetchStandaloneIndexes(table: String, schema: String) async throws -> PostgreSQLStandaloneIndexes { + let query = PostgreSQLIndexQueries.standaloneIndexQuery(schema: schema, table: table) + let indexes = PostgreSQLIndexQueries.standaloneIndexes(rows: try await rows(query)) + if !indexes.invalidNames.isEmpty { + Self.logger.info( + "Left out \(indexes.invalidNames.count) invalid index(es) on \(table, privacy: .private(mask: .hash))" + ) + } + return indexes + } + + // A plain read: nothing here is deparsed by the server, so the session's path cannot change the answer. + func fetchCommentDDL(table: String, schema: String?) async throws -> [String] { + let resolvedSchema = schema ?? currentSchema + let query = PostgreSQLCommentStatements.catalogQuery(name: table, schema: resolvedSchema) + return PostgreSQLCommentStatements.statements( + name: table, + schema: resolvedSchema, + rows: try await rows(query).map { $0.map(\.asText) } + ) + } + + // MARK: - Principals + + func fetchPrincipals() async throws -> [PluginPrincipalInfo] { + let memberships = try await fetchMemberships() + let supportedAttributes = PostgreSQLVersionedStatements.roleAttributes(capabilities: capabilities) + let query = PostgreSQLPrincipalQueries.principals(includeBypassRLS: capabilities.hasBypassRLS) + return try await rows(query).compactMap { row -> PluginPrincipalInfo? in + guard let name = row[safe: 0]?.asText else { return nil } + let canLogin = PostgreSQLCatalogBoolean.isTrue(row[safe: 1]?.asText) + let connectionLimit = row[safe: 8]?.asText.flatMap(Int.init) + return PluginPrincipalInfo( + ref: PluginPrincipalRef(name: name), + isRole: !canLogin, + canLogin: canLogin, + attributes: Self.decodeAttributes(row: row, supported: supportedAttributes), + memberOf: memberships[name] ?? [], + connectionLimit: connectionLimit == -1 ? nil : connectionLimit, + comment: row[safe: 9]?.asText + ) + } + } + + func fetchGrants(for principal: PluginPrincipalRef) async throws -> [PluginGrantInfo] { + let role = principal.name + let database = try await currentDatabaseName() + let databaseGrants = try await fetchDatabaseGrants(role: role) + let schemaGrants = try await fetchSchemaGrants(role: role, database: database) + let tableGrants = try await fetchTableGrants(role: role, database: database) + let columnGrants = try await fetchColumnGrants(role: role, database: database) + return databaseGrants + schemaGrants + tableGrants + columnGrants + } + + func currentDatabaseName() async throws -> String { + try await rows(PostgreSQLPrincipalQueries.currentDatabase).first?[safe: 0]?.asText ?? "" + } + + static func generateGrantSQL(changeSet: PluginPrincipalChangeSet) -> [String]? { + grantBuilder(for: changeSet.principal).grantStatements(changeSet.grantsToAdd) + } + + static func generateRevokeSQL(changeSet: PluginPrincipalChangeSet) -> [String]? { + grantBuilder(for: changeSet.principal).revokeStatements(changeSet.grantsToRemove) + } + + private static func grantBuilder(for principal: PluginPrincipalRef) -> PluginGrantSQLBuilder { + PluginGrantSQLBuilder( + grantee: PostgreSQLObjectQueries.quoteIdentifier(principal.name), + quoteIdentifier: { PostgreSQLObjectQueries.quoteIdentifier($0) }, + target: { grantTarget(for: $0) } + ) + } + + private static func grantTarget(for scope: PluginPrivilegeScope) -> String? { + let quote = PostgreSQLObjectQueries.quoteIdentifier + switch scope { + case .server: + return nil + case let .database(name): + return "DATABASE \(quote(name))" + case let .schema(_, schema): + return "SCHEMA \(quote(schema))" + case let .table(_, schema, table), let .column(_, schema, table, _): + if let schema { + return "TABLE \(quote(schema)).\(quote(table))" + } + return "TABLE \(quote(table))" + @unknown default: + return nil + } + } + + private func fetchMemberships() async throws -> [String: [String]] { + var memberships: [String: [String]] = [:] + for row in try await rows(PostgreSQLPrincipalQueries.memberships) { + guard let member = row[safe: 0]?.asText, + let grantedRole = row[safe: 1]?.asText else { continue } + memberships[member, default: []].append(grantedRole) + } + return memberships + } + + private func fetchDatabaseGrants(role: String) async throws -> [PluginGrantInfo] { + try await rows(PostgreSQLPrincipalQueries.databaseGrants(role: role)).compactMap { row -> PluginGrantInfo? in + guard let database = row[safe: 0]?.asText, + let privilege = row[safe: 1]?.asText else { return nil } + return PluginGrantInfo( + privilege: privilege, + scope: .database(database), + isGrantable: PostgreSQLCatalogBoolean.isTrue(row[safe: 2]?.asText) + ) + } + } + + private func fetchSchemaGrants(role: String, database: String) async throws -> [PluginGrantInfo] { + try await rows(PostgreSQLPrincipalQueries.schemaGrants(role: role)).compactMap { row -> PluginGrantInfo? in + guard let schema = row[safe: 0]?.asText, + let privilege = row[safe: 1]?.asText else { return nil } + return PluginGrantInfo( + privilege: privilege, + scope: .schema(database: database, schema: schema), + isGrantable: PostgreSQLCatalogBoolean.isTrue(row[safe: 2]?.asText) + ) + } + } + + private func fetchTableGrants(role: String, database: String) async throws -> [PluginGrantInfo] { + try await rows(PostgreSQLPrincipalQueries.tableGrants(role: role)).compactMap { row -> PluginGrantInfo? in + guard let schema = row[safe: 0]?.asText, + let table = row[safe: 1]?.asText, + let privilege = row[safe: 2]?.asText else { return nil } + return PluginGrantInfo( + privilege: privilege, + scope: .table(database: database, schema: schema, table: table), + isGrantable: PostgreSQLCatalogBoolean.isTrue(row[safe: 3]?.asText) + ) + } + } + + private func fetchColumnGrants(role: String, database: String) async throws -> [PluginGrantInfo] { + try await rows(PostgreSQLPrincipalQueries.columnGrants(role: role)).compactMap { row -> PluginGrantInfo? in + guard let schema = row[safe: 0]?.asText, + let table = row[safe: 1]?.asText, + let column = row[safe: 2]?.asText, + let privilege = row[safe: 3]?.asText else { return nil } + return PluginGrantInfo( + privilege: privilege, + scope: .column(database: database, schema: schema, table: table, column: column), + isGrantable: PostgreSQLCatalogBoolean.isTrue(row[safe: 4]?.asText) + ) + } + } + + private static func decodeAttributes( + row: [PluginCellValue], + supported: Set + ) -> [PluginPrincipalAttribute] { + let columnOffsets: [(PostgreSQLRoleAttribute, Int)] = [ + (.superuser, 2), + (.createdb, 3), + (.createrole, 4), + (.replication, 5), + (.bypassrls, 6), + (.inherit, 7) + ] + return columnOffsets.filter { supported.contains($0.0) }.map { attribute, offset in + PluginPrincipalAttribute( + key: attribute.rawValue, + label: attribute.label, + isEnabled: PostgreSQLCatalogBoolean.isTrue(row[safe: offset]?.asText) + ) + } + } +} + +// The settings scope an export opens around its reads, on PostgreSQL's `pg_settings`, `set_config` and savepoints +// (measured on 17.11). CockroachDB and Redshift keep PluginKit's nil scopes. +nonisolated struct PostgreSQLSessionScope: Sendable { + // Runs one setting statement on the session: inside a transaction under `PostgreSQLSessionScopeSQL`'s savepoint, + // and throwing `PostgreSQLSessionScopeError.sessionNotIdle` when `requiresIdleSession` is set and it is not idle. + typealias Setting = @Sendable ( + _ statement: PostgreSQLBoundStatement, + _ requiresIdleSession: Bool + ) async throws -> [[PluginCellValue]] + + private static let logger = Logger(subsystem: "com.TablePro.PostgreSQLDriver", category: "PostgreSQLSessionScope") + + let setting: Setting + let transactionState: @Sendable () async -> PluginSessionTransactionState + + init( + setting: @escaping Setting, + transactionState: @escaping @Sendable () async -> PluginSessionTransactionState + ) { + self.setting = setting + self.transactionState = transactionState + } + + func open(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + switch purpose { + case .preserve: + guard let saved = await capture(PostgreSQLSessionScopeSQL.preservedNames), + !saved.isEmpty else { return nil } + return PluginSessionScope(purpose: .preserve, saved: saved, applied: []) + case .verbatimRead: + return await openVerbatimRead() + case .verbatimWrite: + // What the read pins reads back the same under any settings; nil would make the host drop the read too. + return PluginSessionScope(purpose: .verbatimWrite, saved: [], applied: []) + case .relaxedForeignKeys: + // `session_replication_role = replica` would also stop the target's user triggers, and needs a superuser. + return nil + @unknown default: + return nil + } + } + + func close(_ scope: PluginSessionScope) async throws { + guard !scope.saved.isEmpty else { return } + try await restoreChanged(scope.saved) + } + + private func openVerbatimRead() async -> PluginSessionScope? { + guard PostgreSQLSessionScopeSQL.canOpenReadScope(in: await transactionState()) else { + Self.logger.info("Read scope not opened: the session has a transaction open") + return nil + } + let requested = PostgreSQLSessionScopeSQL.verbatimReadSettings + let names = PostgreSQLSessionScopeSQL.verbatimReadNames + guard let captured = await capture(names), + let saved = PostgreSQLSessionScopeSQL.completeSettings(captured, names: names), + let statement = PostgreSQLSessionScopeSQL.setStatement(requested) + else { + Self.logger.info("Read scope not opened: the server lacks one of its settings or the capture failed") + return nil + } + do { + let rows = try await setting(statement, true) + let applied = PostgreSQLSessionScopeSQL.appliedSettings( + requested, + reportedRow: rows.first?.map(\.asText) ?? [] + ) + return PluginSessionScope(purpose: .verbatimRead, saved: saved, applied: applied) + } catch PostgreSQLSessionScopeError.sessionNotIdle { + Self.logger.info("Read scope not opened: a transaction opened before its settings were applied") + return nil + } catch { + // One statement outside a transaction: a refusal rolls back all of it, so nothing needs putting back. + Self.logger.error( + "Read scope not opened, its settings were refused: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + return nil + } + } + + private func capture(_ names: [String]) async -> [PluginSessionSetting]? { + guard PostgreSQLSessionScopeSQL.canRestore(in: await transactionState()) else { + Self.logger.info("Session scope not opened: the session is in an aborted transaction") + return nil + } + do { + let rows = try await setting(PostgreSQLSessionScopeSQL.captureStatement(names: names), false) + return PostgreSQLSessionScopeSQL.settings(fromRows: rows.map { $0.map(\.asText) }, names: names) + } catch { + Self.logger.error( + "Session settings capture failed: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + return nil + } + } + + // Nothing is sent in an aborted transaction: the server would refuse it, and the rollback the session still needs + // reverts only what the transaction itself set, so the caller is told rather than left to assume. + private func restoreChanged(_ saved: [PluginSessionSetting]) async throws { + guard PostgreSQLSessionScopeSQL.canRestore(in: await transactionState()) else { + Self.logger.error("Session settings not restored: the session is in an aborted transaction") + throw PostgreSQLSessionScopeError.abortedTransaction + } + let names = saved.map(\.name) + let rows = try await setting(PostgreSQLSessionScopeSQL.captureStatement(names: names), false) + let current = PostgreSQLSessionScopeSQL.settings(fromRows: rows.map { $0.map(\.asText) }, names: names) + + var refused: [String] = [] + for changed in PostgreSQLSessionScopeSQL.changedSettings(saved: saved, current: current) { + guard let statement = PostgreSQLSessionScopeSQL.setStatement([changed]) else { continue } + do { + _ = try await setting(statement, false) + } catch { + refused.append(changed.name) + Self.logger.error( + "Restoring \(changed.name, privacy: .public) failed: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + } + } + guard refused.isEmpty else { throw PostgreSQLSessionScopeError.restoreRefused(refused) } + } +} + +// Shown as the driver's own errors are, so a failed read reads the same on either app. +nonisolated struct PostgreSQLCatalogReadError: PluginDriverError { + let message: String + + var pluginErrorMessage: String { message } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLColumnClauses.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLColumnClauses.swift index 07ca27f158..1ed5eebdf2 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLColumnClauses.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLColumnClauses.swift @@ -10,7 +10,7 @@ import Foundation import TableProPluginKit -enum PostgreSQLColumnClauses { +nonisolated enum PostgreSQLColumnClauses { /// `ddlSpelling` first, because it is the one spelling that names the type's schema and keeps its /// modifier: `dataType` says `geometry` for `public.geometry(Point,4326)` and `ENUM` for an enum, /// and PostgreSQL rejects both under a `search_path` that is only the target schema. diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLColumnTypeSpelling.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLColumnTypeSpelling.swift index 8d48341f52..c575918680 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLColumnTypeSpelling.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLColumnTypeSpelling.swift @@ -9,7 +9,7 @@ import Foundation import TableProPluginKit -enum PostgreSQLColumnTypeSpelling { +nonisolated enum PostgreSQLColumnTypeSpelling { struct Resolution: Equatable { let dataType: String let classificationTypeName: String? diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCommentStatements.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCommentStatements.swift index 71c957c861..0c04e34d06 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCommentStatements.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCommentStatements.swift @@ -8,7 +8,7 @@ import Foundation -public enum PostgreSQLCommentStatements { +nonisolated public enum PostgreSQLCommentStatements { /// The relation's own comment first, then one row per commented column in `attnum` order, which /// is the order `pg_dump` writes them in. /// diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCreateTableSQL.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCreateTableSQL.swift new file mode 100644 index 0000000000..faa12945c4 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCreateTableSQL.swift @@ -0,0 +1,120 @@ +// +// PostgreSQLCreateTableSQL.swift +// PostgreSQLDriverPlugin +// + +import Foundation +import TableProPluginKit +import TableProPostgreSQLCore + +nonisolated enum PostgreSQLCreateTableSQL { + struct Parts: Equatable { + let qualifiedTable: String + let createTable: String + let indexStatements: [String] + } + + static func statements( + _ definition: PluginCreateTableDefinition, + schema: String, + capabilities: PostgreSQLCapabilities + ) -> [String]? { + guard let parts = parts(definition, schema: schema, capabilities: capabilities) else { return nil } + let columnComments = PostgreSQLRelationSQL.columnCommentStatements( + qualifiedTable: parts.qualifiedTable, + columns: definition.columns + ) + return [parts.createTable] + parts.indexStatements + columnComments + } + + static func parts( + _ definition: PluginCreateTableDefinition, + schema: String, + capabilities: PostgreSQLCapabilities + ) -> Parts? { + guard !definition.columns.isEmpty, + PostgreSQLVersionedStatements.refusal(for: definition, capabilities: capabilities) == nil + else { return nil } + + let quote = PostgreSQLObjectQueries.quoteIdentifier + let qualifiedTable = PostgreSQLObjectQueries.qualifiedName(schema: schema, name: definition.tableName) + let pkColumns = definition.columns.filter { $0.isPrimaryKey } + let inlinePK = pkColumns.count == 1 + var parts: [String] = definition.columns.map { + columnDefinition($0, inlinePK: inlinePK, capabilities: capabilities) + } + + if pkColumns.count > 1 { + parts.append("PRIMARY KEY (\(pkColumns.map { quote($0.name) }.joined(separator: ", ")))") + } + + for fk in definition.foreignKeys { + parts.append(foreignKeyDefinition(fk)) + } + + let createTable = "CREATE TABLE \(qualifiedTable) (\n " + parts.joined(separator: ",\n ") + "\n)" + let indexStatements = definition.indexes.map { + PostgreSQLIndexClauses.createStatement(for: $0, qualifiedTable: qualifiedTable) + } + return Parts(qualifiedTable: qualifiedTable, createTable: createTable, indexStatements: indexStatements) + } + + static func columnDefinition( + _ col: PluginColumnDefinition, + inlinePK: Bool, + capabilities: PostgreSQLCapabilities + ) -> String { + var def = "\(PostgreSQLObjectQueries.quoteIdentifier(col.name)) \(PostgreSQLColumnClauses.type(for: col))" + if let collation = PostgreSQLColumnClauses.collation(for: col) { + def += " COLLATE \(collation)" + } + if let expression = PostgreSQLColumnClauses.generationExpression(for: col) { + def += " GENERATED ALWAYS AS (\(expression)) \(generationKeyword(col.generationKind, capabilities: capabilities))" + if !col.isNullable { def += " NOT NULL" } + // A generated column can hold the primary key, and the caller relies on the inline key being written here. + if inlinePK && col.isPrimaryKey { def += " PRIMARY KEY" } + return def + } + if !col.autoIncrement { + def += col.isNullable ? " NULL" : " NOT NULL" + } + if let defaultValue = PostgreSQLColumnClauses.defaultExpression(for: col) { + def += " DEFAULT \(defaultValue)" + } + if inlinePK && col.isPrimaryKey { + def += " PRIMARY KEY" + } + return def + } + + // PostgreSQL 17 and earlier refuse VIRTUAL and require STORED, while 18 made VIRTUAL the default, so the keyword is + // always written. + static func generationKeyword(_ kind: GenerationKind?, capabilities: PostgreSQLCapabilities) -> String { + guard capabilities.hasVirtualGeneratedColumns else { return "STORED" } + return (kind ?? .virtual).rawValue + } + + static func foreignKeyDefinition(_ fk: PluginForeignKeyDefinition) -> String { + let quote = PostgreSQLObjectQueries.quoteIdentifier + let cols = fk.columns.map(quote).joined(separator: ", ") + let refCols = fk.referencedColumns.map(quote).joined(separator: ", ") + let refTable: String + if let schema = fk.referencedSchema, !schema.isEmpty { + refTable = "\(quote(schema)).\(quote(fk.referencedTable))" + } else { + refTable = quote(fk.referencedTable) + } + let constraint = fk.name.isEmpty ? "" : "CONSTRAINT \(quote(fk.name)) " + var def = "\(constraint)FOREIGN KEY (\(cols)) REFERENCES \(refTable)" + if !refCols.isEmpty { + def += " (\(refCols))" + } + if fk.onDelete != "NO ACTION" { + def += " ON DELETE \(fk.onDelete)" + } + if fk.onUpdate != "NO ACTION" { + def += " ON UPDATE \(fk.onUpdate)" + } + return def + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLForeignKeyQueries.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLForeignKeyQueries.swift index 7bbb36c419..0713c0e557 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLForeignKeyQueries.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLForeignKeyQueries.swift @@ -7,7 +7,7 @@ import Foundation import TableProPluginKit import TableProPostgreSQLCore -enum PostgreSQLForeignKeyQueries { +nonisolated enum PostgreSQLForeignKeyQueries { static func foreignKeyList(schema: String, table: String?, capabilities: PostgreSQLCapabilities) -> String { let schemaLiteral = PostgreSQLObjectQueries.quoteLiteral(schema) // The namespace comes from the constraint, so the whole-schema read never joins `pg_class` for @@ -65,7 +65,7 @@ enum PostgreSQLForeignKeyQueries { } } -struct PostgreSQLForeignKeyRow { +nonisolated struct PostgreSQLForeignKeyRow { let table: String let foreignKey: PluginForeignKeyInfo diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLIndexClauses.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLIndexClauses.swift index a8e34ac639..4fabdd7d2a 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLIndexClauses.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLIndexClauses.swift @@ -10,7 +10,7 @@ import Foundation import TableProPluginKit -enum PostgreSQLIndexClauses { +nonisolated enum PostgreSQLIndexClauses { static func createStatement(for index: PluginIndexDefinition, qualifiedTable: String) -> String { let unique = index.isUnique ? "UNIQUE " : "" let name = PostgreSQLObjectQueries.quoteIdentifier(index.name) diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Columns.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Columns.swift index 6fb587159d..f667fa852a 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Columns.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Columns.swift @@ -8,200 +8,10 @@ import TableProPluginKit extension PostgreSQLPluginDriver { func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { - let resolvedSchema = schema ?? core.currentSchema - let catalog = try await fetchTypeCatalog(schema: resolvedSchema, table: table) - let query = PostgreSQLSchemaQueries.columnsQuery( - schema: resolvedSchema, - table: table, - capabilities: versionedCapabilities, - includeMaterializedViews: includesMaterializedViews() - ) - let result = try await catalogRead(query) - let spellings = try await fetchColumnSpellings(schema: resolvedSchema, table: table)[table] ?? [:] - let columnDDL = try await fetchColumnDDL(schema: resolvedSchema, table: table)[table] ?? [:] - return result.rows.compactMap { row in - mapPgColumnRow(row, tableNameOffset: 0, catalog: catalog, spellings: spellings, columnDDL: columnDDL) - } + try await exportCatalog.fetchColumns(table: table, schema: schema) } func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { - let resolvedSchema = schema ?? core.currentSchema - let catalog = try await fetchTypeCatalog(schema: resolvedSchema, table: nil) - let query = PostgreSQLSchemaQueries.columnsQuery( - schema: resolvedSchema, - table: nil, - capabilities: versionedCapabilities, - includeMaterializedViews: includesMaterializedViews() - ) - let result = try await catalogRead(query) - let spellings = try await fetchColumnSpellings(schema: resolvedSchema, table: nil) - let columnDDL = try await fetchColumnDDL(schema: resolvedSchema, table: nil) - var allColumns: [String: [PluginColumnInfo]] = [:] - for row in result.rows { - guard row.count >= 5, let tableName = row[0].asText else { continue } - let tableDDL = columnDDL[tableName] ?? [:] - let tableSpellings = spellings[tableName] ?? [:] - if let column = mapPgColumnRow( - row, tableNameOffset: 1, catalog: catalog, spellings: tableSpellings, columnDDL: tableDDL - ) { - allColumns[tableName, default: []].append(column) - } - } - return allColumns - } - - fileprivate func fetchColumnSpellings( - schema: String, - table: String? - ) async throws -> [String: [String: PostgreSQLColumnSpelling]] { - let query = PostgreSQLSchemaQueries.columnSpellingQuery( - schema: schema, table: table, capabilities: versionedCapabilities - ) - let result = try await catalogRead(query, namesRelativeTo: schema) - return PostgreSQLSchemaQueries.columnSpellings(rows: result.rows) - } - - fileprivate func fetchColumnDDL( - schema: String, - table: String? - ) async throws -> [String: [String: PostgreSQLCatalogColumnDDL]] { - let query = PostgreSQLSchemaQueries.columnDDLQuery( - schema: schema, table: table, capabilities: versionedCapabilities - ) - let result = try await catalogRead(query) - return PostgreSQLSchemaQueries.columnDDL(rows: result.rows) - } - - fileprivate func fetchEnumLabelMap(schema: String, table: String?) async throws -> [String: [String]] { - let result = try await catalogRead(PostgreSQLSchemaQueries.enumLabelQuery(schema: schema, table: table)) - var map: [String: [String]] = [:] - for row in result.rows { - guard let schemaName = row[safe: 0]?.asText, - let typeName = row[safe: 1]?.asText else { continue } - let key = PostgresColumnTypeResolver.qualifiedName(schema: schemaName, name: typeName) - var labels = map[key] ?? [] - if let label = row[safe: 2]?.asText { - labels.append(label) - } - map[key] = labels - } - return map - } - - fileprivate func fetchArrayTypeMap(schema: String, table: String?) async throws -> [String: PostgresArrayTypeInfo] { - let result = try await catalogRead(PostgreSQLSchemaQueries.arrayTypeQuery(schema: schema, table: table)) - var map: [String: PostgresArrayTypeInfo] = [:] - for row in result.rows { - guard let schemaName = row[safe: 0]?.asText, - let arrayTypeName = row[safe: 1]?.asText, - let elementTypeName = row[safe: 2]?.asText, - let elementKind = row[safe: 3]?.asText?.first else { continue } - let key = PostgresColumnTypeResolver.qualifiedName(schema: schemaName, name: arrayTypeName) - map[key] = PostgresArrayTypeInfo(elementTypeName: elementTypeName, elementTypeKind: elementKind) - } - return map - } - - /// Scoped to the types the read's own columns use, and read fresh each time: a cache would go - /// stale on the next `CREATE TYPE`, and the scoped read costs about 0.4 ms for one table. - fileprivate func fetchTypeCatalog(schema: String, table: String?) async throws -> PostgresTypeCatalog { - let enumLabels = try await fetchEnumLabelMap(schema: schema, table: table) - let arrayTypes = try await fetchArrayTypeMap(schema: schema, table: table) - return PostgresTypeCatalog(enumLabels: enumLabels, arrayTypes: arrayTypes) - } - - fileprivate func mapPgColumnRow( - _ row: [PluginCellValue], - tableNameOffset: Int, - catalog: PostgresTypeCatalog, - spellings: [String: PostgreSQLColumnSpelling], - columnDDL: [String: PostgreSQLCatalogColumnDDL] - ) -> PluginColumnInfo? { - let nameIdx = tableNameOffset - let typeIdx = tableNameOffset + 1 - let nullableIdx = tableNameOffset + 2 - let collationIdx = tableNameOffset + 4 - let commentIdx = tableNameOffset + 5 - let udtIdx = tableNameOffset + 6 - let pkIdx = tableNameOffset + 7 - let identityIdx = tableNameOffset + 8 - let generatedIdx = tableNameOffset + 9 - let udtSchemaIdx = tableNameOffset + 10 - let domainNameIdx = tableNameOffset + 13 - - guard row.count > typeIdx, - let name = row[nameIdx].asText, - let rawDataType = row[typeIdx].asText - else { return nil } - let spelling = spellings[name] - - let udtName = row.count > udtIdx ? row[udtIdx].asText : nil - let udtSchema = row.count > udtSchemaIdx ? row[udtSchemaIdx].asText : nil - let resolution = PostgresColumnTypeResolver.resolve( - rawDataType: rawDataType, - udtSchema: udtSchema, - udtName: udtName, - enumLabelsByQualifiedName: catalog.enumLabels, - arrayTypesByQualifiedName: catalog.arrayTypes - ) - let allowedValues = resolution.allowedValues - let typeSpelling = PostgreSQLColumnTypeSpelling.resolve( - declaredType: spelling?.declaredType, - informationSchemaType: rawDataType, - domainName: row.count > domainNameIdx ? row[domainNameIdx].asText : nil, - udtSchema: udtSchema, - resolved: resolution - ) - - let isNullable = row.count > nullableIdx && row[nullableIdx].asText == "YES" - let collation = row.count > collationIdx ? row[collationIdx].asText : nil - let comment = row.count > commentIdx ? row[commentIdx].asText : nil - let isPk = row.count > pkIdx && row[pkIdx].asText == "YES" - let attidentity = row.count > identityIdx ? row[identityIdx].asText : nil - let attgenerated = row.count > generatedIdx ? row[generatedIdx].asText : nil - - let charset: String? = { - guard let coll = collation, coll.contains(".") else { return nil } - return coll.components(separatedBy: ".").last - }() - - return PluginColumnInfo( - name: name, - dataType: typeSpelling.dataType, - isNullable: isNullable, - isPrimaryKey: isPk, - defaultValue: spelling?.defaultValue, - charset: charset, - collation: collation, - comment: comment?.isEmpty == false ? comment : nil, - identityKind: pgIdentityKind(attidentity), - isGenerated: attgenerated == "s" || attgenerated == "v", - allowedValues: allowedValues, - generationExpression: spelling?.generationExpression, - generationKind: pgGenerationKind(attgenerated), - ddlSpelling: columnDDL[name]?.typeSpelling, - ddlDefault: columnDDL[name]?.defaultExpression, - ddlGenerationExpression: columnDDL[name]?.generationExpression, - ddlCollation: columnDDL[name]?.collation, - classificationTypeName: typeSpelling.classificationTypeName - ) - } - - /// PostgreSQL 18 added virtual generated columns ('v'). Treating only 's' as generated left - /// every virtual one looking writable, so it was included in INSERT and UPDATE. - fileprivate func pgGenerationKind(_ attgenerated: String?) -> GenerationKind? { - switch attgenerated { - case "s": return .stored - case "v": return .virtual - default: return nil - } - } - - fileprivate func pgIdentityKind(_ attidentity: String?) -> IdentityKind? { - switch attidentity { - case "a": return .always - case "d": return .byDefault - default: return nil - } + try await exportCatalog.fetchAllColumns(schema: schema) } } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Comments.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Comments.swift index de75ea4787..f1390cf274 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Comments.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Comments.swift @@ -6,20 +6,9 @@ import Foundation import TableProPluginKit +// PGlite inherits this. CockroachDB and Redshift conform to `LibPQBackedDriver` directly, so they do not. extension PostgreSQLPluginDriver { - /// A plain read rather than the `search_path`-emptied one `fetchViewDefinition` takes: nothing - /// here is deparsed by the server, so the session's path cannot change the answer. - /// - /// PGlite inherits this. CockroachDB and Redshift conform to `LibPQBackedDriver` directly, so - /// they do not. func fetchCommentDDL(table: String, schema: String?) async throws -> [String] { - let resolvedSchema = schema ?? core.currentSchema - let query = PostgreSQLCommentStatements.catalogQuery(name: table, schema: resolvedSchema) - let result = try await catalogRead(query) - return PostgreSQLCommentStatements.statements( - name: table, - schema: resolvedSchema, - rows: result.rows.map { $0.map(\.asText) } - ) + try await exportCatalog.fetchCommentDDL(table: table, schema: schema) } } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+PrincipalSQL.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+PrincipalSQL.swift index 1f05e90f7f..90b696933f 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+PrincipalSQL.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+PrincipalSQL.swift @@ -83,38 +83,11 @@ extension PostgreSQLPluginDriver { } func generateGrantSQL(changeSet: PluginPrincipalChangeSet) -> [String]? { - grantBuilder(for: changeSet.principal).grantStatements(changeSet.grantsToAdd) + PostgreSQLCatalogExport.generateGrantSQL(changeSet: changeSet) } func generateRevokeSQL(changeSet: PluginPrincipalChangeSet) -> [String]? { - grantBuilder(for: changeSet.principal).revokeStatements(changeSet.grantsToRemove) - } - - private func grantBuilder(for principal: PluginPrincipalRef) -> PluginGrantSQLBuilder { - PluginGrantSQLBuilder( - grantee: quoteIdentifier(principal.name), - quoteIdentifier: { self.quoteIdentifier($0) }, - target: { self.grantTarget(for: $0) } - ) - } - - private func grantTarget(for scope: PluginPrivilegeScope) -> String? { - switch scope { - case .server: - nil - case let .database(name): - "DATABASE \(quoteIdentifier(name))" - case let .schema(_, schema): - "SCHEMA \(quoteIdentifier(schema))" - case let .table(_, schema, table), let .column(_, schema, table, _): - if let schema { - "TABLE \(quoteIdentifier(schema)).\(quoteIdentifier(table))" - } else { - "TABLE \(quoteIdentifier(table))" - } - @unknown default: - nil - } + PostgreSQLCatalogExport.generateRevokeSQL(changeSet: changeSet) } private var supportedRoleAttributes: Set { diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Principals.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Principals.swift index a8d551357f..52b17ff09e 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Principals.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Principals.swift @@ -27,7 +27,7 @@ extension PostgreSQLPluginDriver: PluginPrincipalManagement { matching query: String, limit: Int ) async throws -> [PluginPrivilegeScope] { - let database = try await currentDatabaseName() + let database = try await exportCatalog.currentDatabaseName() let sql = PostgreSQLPrincipalQueries.searchObjects( pattern: query, limit: limit @@ -42,29 +42,7 @@ extension PostgreSQLPluginDriver: PluginPrincipalManagement { } func fetchPrincipals() async throws -> [PluginPrincipalInfo] { - let memberships = try await fetchMemberships() - let supportedAttributes = PostgreSQLVersionedStatements.roleAttributes(capabilities: versionedCapabilities) - let query = PostgreSQLPrincipalQueries.principals( - includeBypassRLS: versionedCapabilities.hasBypassRLS - ) - let result = try await catalogRead(query) - - return result.rows.compactMap { row -> PluginPrincipalInfo? in - guard let name = row[safe: 0]?.asText else { return nil } - let canLogin = PostgreSQLCatalogBoolean.isTrue(row[safe: 1]?.asText) - let attributes = Self.decodeAttributes(row: row, supported: supportedAttributes) - let connectionLimit = row[safe: 8]?.asText.flatMap(Int.init) - - return PluginPrincipalInfo( - ref: PluginPrincipalRef(name: name), - isRole: !canLogin, - canLogin: canLogin, - attributes: attributes, - memberOf: memberships[name] ?? [], - connectionLimit: connectionLimit == -1 ? nil : connectionLimit, - comment: row[safe: 9]?.asText - ) - } + try await exportCatalog.fetchPrincipals() } func fetchPrivilegeCatalog() async throws -> PluginPrivilegeCatalog { @@ -79,19 +57,11 @@ extension PostgreSQLPluginDriver: PluginPrincipalManagement { } func fetchGrants(for principal: PluginPrincipalRef) async throws -> [PluginGrantInfo] { - let role = principal.name - let database = try await currentDatabaseName() - - let databaseGrants = try await fetchDatabaseGrants(role: role) - let schemaGrants = try await fetchSchemaGrants(role: role, database: database) - let tableGrants = try await fetchTableGrants(role: role, database: database) - let columnGrants = try await fetchColumnGrants(role: role, database: database) - - return databaseGrants + schemaGrants + tableGrants + columnGrants + try await exportCatalog.fetchGrants(for: principal) } func fetchGrantableChildren(of scope: PluginPrivilegeScope) async throws -> [PluginPrivilegeScope] { - let currentDatabase = try await currentDatabaseName() + let currentDatabase = try await exportCatalog.currentDatabaseName() switch scope { case let .database(database): @@ -143,22 +113,6 @@ extension PostgreSQLPluginDriver: PluginPrincipalManagement { } } - private func fetchColumnGrants(role: String, database: String) async throws -> [PluginGrantInfo] { - let query = PostgreSQLPrincipalQueries.columnGrants(role: role) - let result = try await catalogRead(query) - return result.rows.compactMap { row -> PluginGrantInfo? in - guard let schema = row[safe: 0]?.asText, - let table = row[safe: 1]?.asText, - let column = row[safe: 2]?.asText, - let privilege = row[safe: 3]?.asText else { return nil } - return PluginGrantInfo( - privilege: privilege, - scope: .column(database: database, schema: schema, table: table, column: column), - isGrantable: PostgreSQLCatalogBoolean.isTrue(row[safe: 4]?.asText) - ) - } - } - func currentPrincipalRef() async throws -> PluginPrincipalRef? { let result = try await catalogRead(PostgreSQLPrincipalQueries.currentPrincipal) guard let name = result.rows.first?[safe: 0]?.asText else { return nil } @@ -172,84 +126,4 @@ extension PostgreSQLPluginDriver: PluginPrincipalManagement { let result = try await catalogRead(query) return PostgreSQLCatalogBoolean.isTrue(result.rows.first?[safe: 0]?.asText) } - - private func fetchMemberships() async throws -> [String: [String]] { - let result = try await catalogRead(PostgreSQLPrincipalQueries.memberships) - var memberships: [String: [String]] = [:] - for row in result.rows { - guard let member = row[safe: 0]?.asText, - let grantedRole = row[safe: 1]?.asText else { continue } - memberships[member, default: []].append(grantedRole) - } - return memberships - } - - private func currentDatabaseName() async throws -> String { - let result = try await catalogRead(PostgreSQLPrincipalQueries.currentDatabase) - return result.rows.first?[safe: 0]?.asText ?? "" - } - - private func fetchDatabaseGrants(role: String) async throws -> [PluginGrantInfo] { - let query = PostgreSQLPrincipalQueries.databaseGrants(role: role) - let result = try await catalogRead(query) - return result.rows.compactMap { row -> PluginGrantInfo? in - guard let database = row[safe: 0]?.asText, - let privilege = row[safe: 1]?.asText else { return nil } - return PluginGrantInfo( - privilege: privilege, - scope: .database(database), - isGrantable: PostgreSQLCatalogBoolean.isTrue(row[safe: 2]?.asText) - ) - } - } - - private func fetchSchemaGrants(role: String, database: String) async throws -> [PluginGrantInfo] { - let query = PostgreSQLPrincipalQueries.schemaGrants(role: role) - let result = try await catalogRead(query) - return result.rows.compactMap { row -> PluginGrantInfo? in - guard let schema = row[safe: 0]?.asText, - let privilege = row[safe: 1]?.asText else { return nil } - return PluginGrantInfo( - privilege: privilege, - scope: .schema(database: database, schema: schema), - isGrantable: PostgreSQLCatalogBoolean.isTrue(row[safe: 2]?.asText) - ) - } - } - - private func fetchTableGrants(role: String, database: String) async throws -> [PluginGrantInfo] { - let query = PostgreSQLPrincipalQueries.tableGrants(role: role) - let result = try await catalogRead(query) - return result.rows.compactMap { row -> PluginGrantInfo? in - guard let schema = row[safe: 0]?.asText, - let table = row[safe: 1]?.asText, - let privilege = row[safe: 2]?.asText else { return nil } - return PluginGrantInfo( - privilege: privilege, - scope: .table(database: database, schema: schema, table: table), - isGrantable: PostgreSQLCatalogBoolean.isTrue(row[safe: 3]?.asText) - ) - } - } - - private static func decodeAttributes( - row: [PluginCellValue], - supported: Set - ) -> [PluginPrincipalAttribute] { - let columnOffsets: [(PostgreSQLRoleAttribute, Int)] = [ - (.superuser, 2), - (.createdb, 3), - (.createrole, 4), - (.replication, 5), - (.bypassrls, 6), - (.inherit, 7) - ] - return columnOffsets.filter { supported.contains($0.0) }.map { attribute, offset in - PluginPrincipalAttribute( - key: attribute.rawValue, - label: attribute.label, - isEnabled: PostgreSQLCatalogBoolean.isTrue(row[safe: offset]?.asText) - ) - } - } } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Routines.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Routines.swift index 929d9c36d9..c69be6440b 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Routines.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Routines.swift @@ -5,107 +5,23 @@ import Foundation import TableProPluginKit -import TableProPostgreSQLCore extension PostgreSQLPluginDriver { func fetchRoutines(schema: String?) async throws -> [PluginRoutineInfo] { - let resolvedSchema = schema ?? currentSchema ?? "public" - let query = PostgreSQLObjectQueries.routineList( - schema: resolvedSchema, - capabilities: catalogCapabilities - ) - let result = try await catalogRead(query) - return result.rows.compactMap { row -> PluginRoutineInfo? in - guard let name = row[safe: 1]?.asText else { return nil } - let kind: PluginRoutineKind = row[safe: 6]?.asText == "p" ? .procedure : .function - return PluginRoutineInfo( - name: name, - kind: kind, - schema: row[safe: 2]?.asText ?? resolvedSchema, - returnType: row[safe: 4]?.asText, - language: row[safe: 5]?.asText, - argumentSignature: row[safe: 3]?.asText, - identity: row[safe: 0]?.asText, - attributes: Self.routineAttributes(row) - ) - } + try await exportCatalog.fetchRoutines(schema: schema) } func fetchRoutineDDL(_ routine: PluginRoutineInfo) async throws -> String { - let resolvedSchema = routine.schema ?? currentSchema ?? "public" - let query: String - if let identity = routine.identity, !identity.isEmpty, Int(identity) != nil { - query = PostgreSQLObjectQueries.routineDefinition(identity: identity) - } else { - query = PostgreSQLObjectQueries.routineDefinitionByName( - name: routine.name, - schema: resolvedSchema, - arguments: routine.argumentSignature - ) - } - let result = try await catalogRead(query) - guard let ddl = result.rows.first?[safe: 0]?.asText, !ddl.isEmpty else { - throw PluginObjectSourceError.notFound(routine.name) - } - return ddl + try await exportCatalog.fetchRoutineDDL(routine) } var providesBulkTriggerFetch: Bool { true } func fetchAllTriggers(schema: String?) async throws -> [PluginTriggerInfo] { - let resolvedSchema = schema ?? currentSchema ?? "public" - let query = PostgreSQLObjectQueries.triggerList(schema: resolvedSchema, table: nil) - let result = try await catalogRead(query) - return result.rows.compactMap { Self.trigger(from: $0, fallbackSchema: resolvedSchema) } + try await exportCatalog.fetchAllTriggers(schema: schema) } func fetchTriggerDDL(_ trigger: PluginTriggerInfo) async throws -> String { - if let definition = trigger.definition, !definition.isEmpty { return definition } - let resolvedSchema = trigger.schema ?? currentSchema ?? "public" - let query = PostgreSQLObjectQueries.triggerList(schema: resolvedSchema, table: trigger.table) - let result = try await catalogRead(query) - let match = result.rows - .compactMap { Self.trigger(from: $0, fallbackSchema: resolvedSchema) } - .first { $0.name == trigger.name } - guard let definition = match?.definition, !definition.isEmpty else { - throw PluginObjectSourceError.notFound(trigger.name) - } - return definition - } - - static func trigger(from row: [PluginCellValue], fallbackSchema: String) -> PluginTriggerInfo? { - guard let name = row[safe: 0]?.asText, - let definition = row[safe: 7]?.asText - else { return nil } - var attributes: [PluginObjectAttribute] = [] - if let owner = row[safe: 8]?.asText, !owner.isEmpty { - attributes.append(PluginObjectAttribute(label: "Owner", value: owner)) - } - return PluginTriggerInfo( - name: name, - table: row[safe: 1]?.asText, - schema: row[safe: 2]?.asText ?? fallbackSchema, - timing: row[safe: 3]?.asText ?? "", - event: row[safe: 4]?.asText ?? "", - orientation: row[safe: 5]?.asText, - statement: definition, - definition: definition, - enabled: PostgreSQLCatalogBoolean.isTrue(row[safe: 6]?.asText), - attributes: attributes - ) - } - - private static func routineAttributes(_ row: [PluginCellValue]) -> [PluginObjectAttribute] { - var attributes: [PluginObjectAttribute] = [] - if let volatility = row[safe: 7]?.asText, !volatility.isEmpty { - attributes.append(PluginObjectAttribute(label: "Volatility", value: volatility)) - } - if let security = row[safe: 8]?.asText, !security.isEmpty { - attributes.append(PluginObjectAttribute(label: "Security", value: security)) - } - if let owner = row[safe: 9]?.asText, !owner.isEmpty { - attributes.append(PluginObjectAttribute(label: "Owner", value: owner)) - } - return attributes + try await exportCatalog.fetchTriggerDDL(trigger) } } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Sequences.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Sequences.swift index 39aa1e8bc8..075ec5c759 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Sequences.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Sequences.swift @@ -4,47 +4,20 @@ // import Foundation -import os import TableProPluginKit import TableProPostgreSQLCore extension PostgreSQLPluginDriver { - static let sequenceLogger = Logger(subsystem: "com.TablePro.PostgreSQLDriver", category: "Sequences") - func fetchDependentSequences(table: String, schema: String?) async throws -> [(name: String, ddl: String)] { - let definitions = try await sequenceDefinitions(schema: schema ?? core.currentSchema, dependentOnTable: table) - return definitions.map { (name: $0.name, ddl: $0.ddl) } + try await exportCatalog.fetchDependentSequences(table: table, schema: schema) } func fetchSequences(schema: String?) async throws -> [PluginSequenceInfo] { - let schemaName = schema ?? core.currentSchema - let definitions = try await sequenceDefinitions(schema: schemaName, dependentOnTable: nil) - return definitions.map { PluginSequenceInfo(name: $0.name, ddl: $0.ddl, schema: schemaName) } + try await exportCatalog.fetchSequences(schema: schema) } func fetchSequenceDetails(sequence: String, schema: String?) async throws -> PluginSequenceDetails? { - let schemaName = schema ?? core.currentSchema - let definitions = try await sequenceDefinitions(schema: schemaName, dependentOnTable: nil, named: sequence) - guard let definition = definitions.first else { return nil } - return definition.details(schema: schemaName, position: await position(schema: schemaName, sequence: sequence)) - } - - // Needs SELECT on the sequence; a role without it gets no position rather than a guess. - private func position(schema: String, sequence: String) async -> PostgreSQLSequencePosition? { - do { - let query = PostgreSQLSequencePosition.query(schema: schema, sequence: sequence) - return PostgreSQLSequencePosition(row: try await catalogRead(query).rows.first) - } catch { - Self.sequenceLogger.debug( - "Sequence position unavailable for \(sequence, privacy: .private(mask: .hash)): \(error.localizedDescription)" - ) - return nil - } - } - - func sequenceDDL(sequence: String, schema: String) async throws -> String? { - let definitions = try await sequenceDefinitions(schema: schema, dependentOnTable: nil, named: sequence) - return definitions.first?.qualifiedDDL(schema: schema) + try await exportCatalog.fetchSequenceDetails(sequence: sequence, schema: schema) } func sequenceNextValueStatement( @@ -69,36 +42,4 @@ extension PostgreSQLPluginDriver { func createSequenceTemplate() -> String? { PostgreSQLSequenceStatements.createTemplate } - - private func sequenceDefinitions( - schema: String, - dependentOnTable table: String?, - named sequence: String? = nil - ) async throws -> [PostgreSQLSequenceDefinition] { - let source = PostgreSQLSequenceQueries.source(hasSequencesCatalog: includesSequencesCatalog()) - let listing = try await catalogRead( - PostgreSQLSequenceQueries.sequenceList(schema: schema, dependentOnTable: table, source: source, named: sequence) - ) - var definitions: [PostgreSQLSequenceDefinition] = [] - for definition in PostgreSQLSequenceQueries.definitions(from: listing.rows) { - guard definition.needsLastValueRead else { - definitions.append(definition) - continue - } - definitions.append(definition.withLastValue(await lastValue(schema: schema, sequence: definition.name))) - } - return definitions - } - - private func lastValue(schema: String, sequence: String) async -> String? { - do { - let query = PostgreSQLSequenceQueries.lastValue(schema: schema, sequence: sequence) - return try await catalogRead(query).rows.first?[safe: 0]?.asText - } catch { - Self.sequenceLogger.debug( - "Sequence last value unavailable for \(sequence, privacy: .private(mask: .hash)): \(error.localizedDescription)" - ) - return nil - } - } } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+SessionScope.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+SessionScope.swift index 73dd899b7d..7c9dd7b827 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+SessionScope.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+SessionScope.swift @@ -4,111 +4,24 @@ // import Foundation -import os -import TableProLogRedaction import TableProPluginKit -/// On this class rather than `LibPQBackedDriver`, so CockroachDB and Redshift keep PluginKit's nil scopes: this rests -/// on PostgreSQL's `pg_settings`, `set_config` and savepoints, measured on 17.11 only. +// On this class rather than `LibPQBackedDriver`, so CockroachDB and Redshift keep PluginKit's nil scopes. internal extension PostgreSQLPluginDriver { - private static let scopeLogger = Logger(subsystem: "com.TablePro.PostgreSQLDriver", category: "PostgreSQLSessionScope") + private var sessionScope: PostgreSQLSessionScope { + PostgreSQLSessionScope( + setting: { [core] statement, requiresIdleSession in + try await core.executeSessionSetting(statement, requiresIdleSession: requiresIdleSession) + }, + transactionState: { [core] in await core.sessionTransactionState() } + ) + } func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { - switch purpose { - case .preserve: - guard let saved = await captureSessionSettings(PostgreSQLSessionScopeSQL.preservedNames), - !saved.isEmpty else { return nil } - return PluginSessionScope(purpose: .preserve, saved: saved, applied: []) - case .verbatimRead: - return await openVerbatimReadScope() - case .verbatimWrite: - // What the read pins reads back the same under any settings; nil would make the host drop the read too. - return PluginSessionScope(purpose: .verbatimWrite, saved: [], applied: []) - case .relaxedForeignKeys: - // `session_replication_role = replica` would also stop the target's user triggers, and needs a superuser. - return nil - @unknown default: - return nil - } + try await sessionScope.open(for: purpose) } func closeSessionScope(_ scope: PluginSessionScope) async throws { - guard !scope.saved.isEmpty else { return } - try await restoreChangedSessionSettings(scope.saved) - } - - private func openVerbatimReadScope() async -> PluginSessionScope? { - guard PostgreSQLSessionScopeSQL.canOpenReadScope(in: await core.sessionTransactionState()) else { - Self.scopeLogger.info("Read scope not opened: the session has a transaction open") - return nil - } - let requested = PostgreSQLSessionScopeSQL.verbatimReadSettings - let names = PostgreSQLSessionScopeSQL.verbatimReadNames - guard let captured = await captureSessionSettings(names), - let saved = PostgreSQLSessionScopeSQL.completeSettings(captured, names: names), - let statement = PostgreSQLSessionScopeSQL.setStatement(requested) - else { - Self.scopeLogger.info("Read scope not opened: the server lacks one of its settings or the capture failed") - return nil - } - do { - let rows = try await core.executeSessionSetting(statement, requiresIdleSession: true) - let applied = PostgreSQLSessionScopeSQL.appliedSettings( - requested, - reportedRow: rows.first?.map(\.asText) ?? [] - ) - return PluginSessionScope(purpose: .verbatimRead, saved: saved, applied: applied) - } catch PostgreSQLSessionScopeError.sessionNotIdle { - Self.scopeLogger.info("Read scope not opened: a transaction opened before its settings were applied") - return nil - } catch { - // One statement outside a transaction: a refusal rolls back all of it, so nothing needs putting back. - Self.scopeLogger.error( - "Read scope not opened, its settings were refused: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" - ) - return nil - } - } - - private func captureSessionSettings(_ names: [String]) async -> [PluginSessionSetting]? { - guard PostgreSQLSessionScopeSQL.canRestore(in: await core.sessionTransactionState()) else { - Self.scopeLogger.info("Session scope not opened: the session is in an aborted transaction") - return nil - } - do { - let rows = try await core.executeSessionSetting(PostgreSQLSessionScopeSQL.captureStatement(names: names)) - return PostgreSQLSessionScopeSQL.settings(fromRows: rows.map { $0.map(\.asText) }, names: names) - } catch { - Self.scopeLogger.error( - "Session settings capture failed: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" - ) - return nil - } - } - - /// Nothing is sent in an aborted transaction: the server would refuse it, and the rollback the session still - /// needs reverts only what the transaction itself set, so the caller is told rather than left to assume. - private func restoreChangedSessionSettings(_ saved: [PluginSessionSetting]) async throws { - guard PostgreSQLSessionScopeSQL.canRestore(in: await core.sessionTransactionState()) else { - Self.scopeLogger.error("Session settings not restored: the session is in an aborted transaction") - throw PostgreSQLSessionScopeError.abortedTransaction - } - let names = saved.map(\.name) - let rows = try await core.executeSessionSetting(PostgreSQLSessionScopeSQL.captureStatement(names: names)) - let current = PostgreSQLSessionScopeSQL.settings(fromRows: rows.map { $0.map(\.asText) }, names: names) - - var refused: [String] = [] - for setting in PostgreSQLSessionScopeSQL.changedSettings(saved: saved, current: current) { - guard let statement = PostgreSQLSessionScopeSQL.setStatement([setting]) else { continue } - do { - _ = try await core.executeSessionSetting(statement) - } catch { - refused.append(setting.name) - Self.scopeLogger.error( - "Restoring \(setting.name, privacy: .public) failed: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" - ) - } - } - guard refused.isEmpty else { throw PostgreSQLSessionScopeError.restoreRefused(refused) } + try await sessionScope.close(scope) } } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+TriggerEditing.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+TriggerEditing.swift index dab2670b25..a5b6e08087 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+TriggerEditing.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+TriggerEditing.swift @@ -51,6 +51,6 @@ internal extension PostgreSQLPluginDriver { } func generateDropTriggerSQL(name: String, table: String, schema: String?) -> String? { - "DROP TRIGGER IF EXISTS \(quoteIdentifier(name)) ON \(qualifiedTable(table, schema: schema))" + exportCatalog.generateDropTriggerSQL(name: name, table: table, schema: schema) } } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Types.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Types.swift index 30bcd2c9cc..230149ace2 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Types.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Types.swift @@ -9,16 +9,7 @@ import TableProPostgreSQLCore extension PostgreSQLPluginDriver { func fetchUserDefinedTypes(schema: String?) async throws -> [PluginUserDefinedTypeInfo] { - let resolvedSchema = schema ?? currentSchema ?? "public" - let query = PostgreSQLObjectQueries.userDefinedTypeList( - schema: resolvedSchema, - identity: nil, - capabilities: catalogCapabilities - ) - let result = try await catalogRead(query) - return result.rows - .compactMap(PostgreSQLTypeDefinition.record(from:)) - .map(PostgreSQLTypeDefinition.info(from:)) + try await exportCatalog.fetchUserDefinedTypes(schema: schema) } func fetchUserDefinedTypes(inSchemas schemas: [String]) async throws -> [PluginUserDefinedTypeInfo]? { @@ -30,37 +21,8 @@ extension PostgreSQLPluginDriver { .map(PostgreSQLTypeDefinition.info(from:)) } - /// An identity is the oid this driver handed out, so one that is not an oid is refused rather - /// than widened into a schema listing that would hand back whichever type sorts first. The - /// oid is looked up without a schema predicate, because a type moved to another schema keeps - /// it, and the row that comes back is checked against it before it is believed. func fetchUserDefinedType(_ type: PluginUserDefinedTypeInfo) async throws -> PluginUserDefinedTypeInfo { - if let identity = type.identity { - guard let oid = UInt32(identity) else { throw PluginObjectSourceError.notFound(type.name) } - let query = PostgreSQLObjectQueries.userDefinedTypeList( - schema: nil, - identity: String(oid), - capabilities: catalogCapabilities - ) - let result = try await catalogRead(query) - let match = result.rows - .compactMap(PostgreSQLTypeDefinition.record(from:)) - .first { $0.identity == String(oid) } - guard let match else { throw PluginObjectSourceError.notFound(type.name) } - return PostgreSQLTypeDefinition.info(from: match) - } - let resolvedSchema = type.schema ?? currentSchema ?? "public" - let query = PostgreSQLObjectQueries.userDefinedTypeList( - schema: resolvedSchema, - identity: nil, - capabilities: catalogCapabilities - ) - let result = try await catalogRead(query) - let match = result.rows - .compactMap(PostgreSQLTypeDefinition.record(from:)) - .first { $0.name == type.name } - guard let match else { throw PluginObjectSourceError.notFound(type.name) } - return PostgreSQLTypeDefinition.info(from: match) + try await exportCatalog.fetchUserDefinedType(type) } func createTypeTemplate(schema: String?) -> String? { diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Views.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Views.swift index eb7340d2d0..cd19ab2a71 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Views.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver+Views.swift @@ -8,17 +8,8 @@ import TableProPluginKit import TableProPostgreSQLCore extension PostgreSQLPluginDriver { - // Rebuilt from the catalog with every name qualified: pg_views lost the options and the check option. func fetchViewDefinition(view: String, schema: String?) async throws -> String { - let resolvedSchema = schema ?? core.currentSchema - let query = PostgreSQLViewDefinition.catalogQuery(name: view, schema: resolvedSchema) - let result = try await catalogRead(query) - guard let row = result.rows.first, - let catalogRow = PostgreSQLViewDefinition.parse(row: row.map(\.asText)) - else { - throw LibPQPluginError(message: "Failed to fetch definition for view '\(view)'", sqlState: nil, detail: nil) - } - return PostgreSQLViewDefinition.statement(name: view, schema: resolvedSchema, row: catalogRow) + try await exportCatalog.fetchViewDefinition(view: view, schema: schema) } func fetchViewRowEditing(view: String, schema: String?) async throws -> PluginViewRowEditing? { diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift index b41131155e..c5686096f2 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift @@ -123,6 +123,16 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { } } + var exportCatalog: PostgreSQLCatalogExport { + PostgreSQLCatalogExport( + read: { [self] statement, schema in try await catalogRead(statement, namesRelativeTo: schema).rows }, + currentSchema: core.currentSchema, + capabilities: versionedCapabilities, + catalogCapabilities: catalogCapabilities, + catalogPresence: catalogPresence + ) + } + func includesMaterializedViews() -> Bool { catalogPresence?.hasMaterializedViews ?? versionedCapabilities.hasMaterializedViewsCatalog } @@ -131,24 +141,14 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { catalogPresence?.hasForeignTables ?? versionedCapabilities.hasForeignTablesCatalog } - func includesSequencesCatalog() -> Bool { - catalogPresence?.hasSequences ?? versionedCapabilities.hasSequencesCatalog - } - // MARK: - Foreign Keys func foreignKeyDisableStatements() -> [String]? { - [Self.settingReplicationRole("replica")] + PostgreSQLForeignKeyChecks.disableStatements } func foreignKeyEnableStatements() -> [String]? { - [Self.settingReplicationRole("DEFAULT")] - } - - /// Setting it takes a superuser or a `SET` grant, and a bare refusal aborts the transaction it runs in. The block - /// swallows the refusal and keeps the transaction, and an allowed `SET` outlives it (measured on 17.11 and 18.6). - private static func settingReplicationRole(_ value: String) -> String { - "DO $$ BEGIN SET session_replication_role = \(value); EXCEPTION WHEN insufficient_privilege THEN NULL; END $$" + PostgreSQLForeignKeyChecks.enableStatements } /// A duplicated database arrives with `public` alone, so every other schema its tables are @@ -356,170 +356,42 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { } func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { - let resolvedSchema = schema ?? core.currentSchema - let query = PostgreSQLForeignKeyQueries.foreignKeyList( - schema: resolvedSchema, table: table, capabilities: catalogCapabilities - ) - let result = try await catalogRead(query) - let foreignKeys = result.rows.compactMap { PostgreSQLForeignKeyRow($0)?.foreignKey } - Self.logger.info("[fk] postgres fetchForeignKeys schema=\(resolvedSchema, privacy: .private(mask: .hash)) table=\(table, privacy: .private(mask: .hash)) rows=\(result.rows.count) parsed=\(foreignKeys.count)") - return foreignKeys + try await exportCatalog.fetchForeignKeys(table: table, schema: schema) } - /// The same builder the schema-wide list uses, with one more predicate. Two hand-written - /// queries over pg_trigger would be two chances to disagree about one table's triggers. func fetchTriggers(table: String, schema: String?) async throws -> [PluginTriggerInfo] { - let resolvedSchema = schema ?? core.currentSchema - let query = PostgreSQLObjectQueries.triggerList(schema: resolvedSchema, table: table) - let result = try await catalogRead(query) - let triggers = result.rows.compactMap { - Self.trigger(from: $0, fallbackSchema: resolvedSchema) - } - Self.logger.info("[trigger] postgres fetchTriggers schema=\(resolvedSchema, privacy: .private(mask: .hash)) table=\(table, privacy: .private(mask: .hash)) rows=\(result.rows.count) parsed=\(triggers.count)") - return triggers + try await exportCatalog.fetchTriggers(table: table, schema: schema) } - /// PostgreSQL allows `f(integer)` and `f(text)` in one schema, so a drop that names only `f` - /// is ambiguous and the server refuses it. func generateDropRoutineSQL( name: String, signature: String?, schema: String?, isFunction: Bool ) -> String? { - let keyword = isFunction ? "FUNCTION" : "PROCEDURE" - let arguments = (signature ?? "").trimmingCharacters(in: .whitespacesAndNewlines) - return "DROP \(keyword) IF EXISTS \(qualifiedTable(name, schema: schema))\(arguments)" + exportCatalog.generateDropRoutineSQL(name: name, signature: signature, schema: schema, isFunction: isFunction) } var providesBulkForeignKeyFetch: Bool { true } func fetchAllForeignKeys(schema: String?) async throws -> [String: [PluginForeignKeyInfo]] { - let query = PostgreSQLForeignKeyQueries.foreignKeyList( - schema: schema ?? core.currentSchema, table: nil, capabilities: catalogCapabilities - ) - let result = try await catalogRead(query) - var grouped: [String: [PluginForeignKeyInfo]] = [:] - for row in result.rows { - guard let decoded = PostgreSQLForeignKeyRow(row) else { continue } - grouped[decoded.table, default: []].append(decoded.foreignKey) - } - return grouped + try await exportCatalog.fetchAllForeignKeys(schema: schema) } func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { - let query = PostgreSQLSchemaQueries.approximateRowCount( - schema: schema ?? core.currentSchema, table: table - ) - let result = try await catalogRead(query) - return PostgreSQLSchemaQueries.rowEstimate(reltuples: result.rows.first?[safe: 0]?.asText).map { Int($0) } + try await exportCatalog.fetchApproximateRowCount(table: table, schema: schema) } func fetchTableDDL(table: String, schema: String?) async throws -> String { - let resolvedSchema = schema ?? core.currentSchema - let quotedTable = quoteIdentifier(table) - let columnsQuery = PostgreSQLSchemaQueries.tableDDLColumnsQuery( - schema: resolvedSchema, table: table, capabilities: versionedCapabilities - ) - - let constraintsQuery = PostgreSQLSchemaQueries.tableDDLConstraintsQuery( - schema: resolvedSchema, table: table - ) - - let shapeQuery = PostgreSQLTableDDL.shapeQuery( - schema: resolvedSchema, table: table, capabilities: versionedCapabilities - ) - - async let columnsResult = catalogRead(columnsQuery) - async let constraintsResult = catalogRead(constraintsQuery) - async let shapeResult = try? catalogRead(shapeQuery) - - let (cols, cons) = try await (columnsResult, constraintsResult) - let shapeRows = await shapeResult - - /// `pg_attribute` covers views and materialized views as well as tables, so a view reached - /// here as a `CREATE TABLE` of its columns. The Structure tab's DDL and every other caller - /// that asks for a relation's DDL by name get the view's own statement instead. - if let relkind = cols.rows.first?[safe: 1]?.asText, - PostgreSQLViewDefinition.kind(forRelkind: relkind) != nil { - return try await fetchViewDefinition(view: table, schema: resolvedSchema) - } - // A sequence's attributes are its state columns, not a table to recreate. - if cols.rows.first?[safe: 1]?.asText == "S", - let ddl = try await sequenceDDL(sequence: table, schema: resolvedSchema) { - return ddl - } - - let columnDefs = cols.rows.compactMap { $0[0].asText } - guard !columnDefs.isEmpty else { - throw LibPQPluginError(message: "Failed to fetch DDL for table '\(table)'", sqlState: nil, detail: nil) - } - - let constraints = cons.rows.compactMap { $0[0].asText } - var parts = columnDefs - parts.append(contentsOf: constraints) - - var shape = PostgreSQLTableDDL.shape(fromRows: shapeRows?.rows.map { $0.map(\.asText) }) - var partitions: [PostgreSQLTableDDL.Partition] = [] - var partitionIndexes: [String] = [] - var partitionComments: [String] = [] - if shape.partitionKey != nil { - let tree = try await catalogRead(PostgreSQLTableDDL.partitionTreeQuery(schema: resolvedSchema, table: table)) - partitions = tree.rows.compactMap { PostgreSQLTableDDL.partition(from: $0.map(\.asText)) } - if !partitions.isEmpty { - async let elementRows = catalogRead(PostgreSQLTableDDL.partitionElementsQuery( - schema: resolvedSchema, table: table, scope: .below, capabilities: versionedCapabilities - )) - async let indexRows = catalogRead(PostgreSQLTableDDL.partitionIndexesQuery(schema: resolvedSchema, table: table)) - async let commentRows = catalogRead(PostgreSQLTableDDL.partitionCommentsQuery(schema: resolvedSchema, table: table)) - let elements = PostgreSQLTableDDL.elementsByPartition(try await elementRows.rows.map { $0.map(\.asText) }) - partitions = partitions.map { partition in - var partition = partition - partition.elements = elements[partition.name] ?? [] - return partition - } - partitionIndexes = try await indexRows.rows.compactMap { $0.first?.asText } - partitionComments = PostgreSQLTableDDL.partitionComments(try await commentRows.rows.map { $0.map(\.asText) }) - } - } else if shape.partitionOf != nil { - let elementRows = try await catalogRead(PostgreSQLTableDDL.partitionElementsQuery( - schema: resolvedSchema, table: table, scope: .itself, capabilities: versionedCapabilities - )) - shape.partitionElements = PostgreSQLTableDDL.elementsByPartition(elementRows.rows.map { $0.map(\.asText) }) - .values.first ?? [] - } - - let quotedSchema = quoteIdentifier(resolvedSchema) - return PostgreSQLTableDDL.script( - name: "\(quotedSchema).\(quotedTable)", - definitions: parts, - shape: shape, - partitions: partitions, - partitionIndexes: partitionIndexes, - partitionComments: partitionComments - ) + try await exportCatalog.fetchTableDDL(table: table, schema: schema) } - /// `pg_get_indexdef` is what `pg_dump` itself emits, and it round-trips an expression key, an - /// operator class, an `INCLUDE` list, a storage parameter, a partial predicate and a per-column - /// sort direction verbatim. It also qualifies the table whatever `search_path` holds, so a dump - /// spanning two schemas attaches each index to the right one. func fetchIndexDDL(table: String, schema: String?) async throws -> [String] { - try await fetchStandaloneIndexes(table: table, schema: schema ?? core.currentSchema).definitions - .map(PostgreSQLTableDDL.indexCoveringPartitions) + try await exportCatalog.fetchIndexDDL(table: table, schema: schema) } func fetchStandaloneIndexes(table: String, schema: String) async throws -> PostgreSQLStandaloneIndexes { - let query = PostgreSQLIndexQueries.standaloneIndexQuery(schema: schema, table: table) - let indexes = PostgreSQLIndexQueries.standaloneIndexes( - rows: try await catalogRead(query).rows - ) - if !indexes.invalidNames.isEmpty { - Self.logger.info( - "Left out \(indexes.invalidNames.count) invalid index(es) on \(table, privacy: .private(mask: .hash))" - ) - } - return indexes + try await exportCatalog.fetchStandaloneIndexes(table: table, schema: schema) } func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { @@ -580,12 +452,7 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { } func fetchDependentTypes(table: String, schema: String?) async throws -> [(name: String, labels: [String])] { - let query = PostgreSQLSchemaQueries.dependentEnumTypes(schema: schema ?? core.currentSchema, table: table) - let result = try await catalogRead(query) - return result.rows.compactMap { row -> (name: String, labels: [String])? in - guard let typeName = row[0].asText, let labelsStr = row[1].asText else { return nil } - return (name: typeName, labels: PostgreSQLTextArray.values(labelsStr)) - } + try await exportCatalog.fetchDependentTypes(table: table, schema: schema) } private static let supportedEncodings: [String] = [ @@ -843,72 +710,13 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { // MARK: - Create Table DDL func generateCreateTableStatements(definition: PluginCreateTableDefinition, schema: String?) -> [String]? { - guard let parts = createTableParts(definition, schema: schema) else { return nil } - let columnComments = PostgreSQLRelationSQL.columnCommentStatements( - qualifiedTable: parts.qualifiedTable, - columns: definition.columns + PostgreSQLCreateTableSQL.statements( + definition, schema: schema ?? core.currentSchema, capabilities: versionedCapabilities ) - return [parts.createTable] + parts.indexStatements + columnComments - } - - private func createTableParts( - _ definition: PluginCreateTableDefinition, - schema: String? - ) -> (qualifiedTable: String, createTable: String, indexStatements: [String])? { - guard !definition.columns.isEmpty, - PostgreSQLVersionedStatements.refusal(for: definition, capabilities: versionedCapabilities) == nil - else { return nil } - - let qualifiedTable = qualifiedTableName(definition.tableName, schema: schema) - let pkColumns = definition.columns.filter { $0.isPrimaryKey } - let inlinePK = pkColumns.count == 1 - var parts: [String] = definition.columns.map { pgColumnDefinition($0, inlinePK: inlinePK) } - - if pkColumns.count > 1 { - let pkCols = pkColumns.map { quoteIdentifier($0.name) }.joined(separator: ", ") - parts.append("PRIMARY KEY (\(pkCols))") - } - - for fk in definition.foreignKeys { - parts.append(pgForeignKeyDefinition(fk)) - } - - let createTable = "CREATE TABLE \(qualifiedTable) (\n " + - parts.joined(separator: ",\n ") + - "\n)" - let indexStatements = definition.indexes.map { - PostgreSQLIndexClauses.createStatement(for: $0, qualifiedTable: qualifiedTable) - } - return (qualifiedTable, createTable, indexStatements) } private func pgColumnDefinition(_ col: PluginColumnDefinition, inlinePK: Bool) -> String { - var def = "\(quoteIdentifier(col.name)) \(PostgreSQLColumnClauses.type(for: col))" - if let collation = PostgreSQLColumnClauses.collation(for: col) { - def += " COLLATE \(collation)" - } - if let expression = PostgreSQLColumnClauses.generationExpression(for: col) { - def += " GENERATED ALWAYS AS (\(expression)) \(pgGenerationKeyword(col.generationKind))" - if !col.isNullable { def += " NOT NULL" } - // PostgreSQL allows a primary key on a generated column, and the caller relies on the - // inline key being emitted here: returning early without it created no key at all. - if inlinePK && col.isPrimaryKey { def += " PRIMARY KEY" } - return def - } - if !col.autoIncrement { - if col.isNullable { - def += " NULL" - } else { - def += " NOT NULL" - } - } - if let defaultValue = PostgreSQLColumnClauses.defaultExpression(for: col) { - def += " DEFAULT \(defaultValue)" - } - if inlinePK && col.isPrimaryKey { - def += " PRIMARY KEY" - } - return def + PostgreSQLCreateTableSQL.columnDefinition(col, inlinePK: inlinePK, capabilities: versionedCapabilities) } /// PostgreSQL 17 added ALTER COLUMN ... SET EXPRESSION AS, which rewrites the column in place. @@ -930,36 +738,6 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { return "ALTER TABLE \(qt) ALTER COLUMN \(colName) SET EXPRESSION AS (\(expression))" } - /// Never emitted bare: PostgreSQL 17 and earlier reject VIRTUAL outright and require STORED, - /// while 18 made VIRTUAL the default, so the keyword has to be explicit either way. - private func pgGenerationKeyword(_ kind: GenerationKind?) -> String { - guard versionedCapabilities.hasVirtualGeneratedColumns else { return "STORED" } - return (kind ?? .virtual).rawValue - } - - private func pgForeignKeyDefinition(_ fk: PluginForeignKeyDefinition) -> String { - let cols = fk.columns.map { quoteIdentifier($0) }.joined(separator: ", ") - let refCols = fk.referencedColumns.map { quoteIdentifier($0) }.joined(separator: ", ") - let refTable: String - if let schema = fk.referencedSchema, !schema.isEmpty { - refTable = "\(quoteIdentifier(schema)).\(quoteIdentifier(fk.referencedTable))" - } else { - refTable = quoteIdentifier(fk.referencedTable) - } - let constraint = fk.name.isEmpty ? "" : "CONSTRAINT \(quoteIdentifier(fk.name)) " - var def = "\(constraint)FOREIGN KEY (\(cols)) REFERENCES \(refTable)" - if !refCols.isEmpty { - def += " (\(refCols))" - } - if fk.onDelete != "NO ACTION" { - def += " ON DELETE \(fk.onDelete)" - } - if fk.onUpdate != "NO ACTION" { - def += " ON UPDATE \(fk.onUpdate)" - } - return def - } - // MARK: - Definition SQL (clipboard copy) func generateColumnDefinitionSQL(column: PluginColumnDefinition) -> String? { @@ -974,7 +752,7 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { } func generateForeignKeyDefinitionSQL(fk: PluginForeignKeyDefinition) -> String? { - pgForeignKeyDefinition(fk) + PostgreSQLCreateTableSQL.foreignKeyDefinition(fk) } // MARK: - ALTER TABLE DDL @@ -1060,7 +838,7 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { } func generateAddForeignKeySQL(table: String, schema: String?, fk: PluginForeignKeyDefinition) -> String? { - "ALTER TABLE \(qualifiedTableName(table, schema: schema)) ADD \(pgForeignKeyDefinition(fk))" + "ALTER TABLE \(qualifiedTableName(table, schema: schema)) ADD \(PostgreSQLCreateTableSQL.foreignKeyDefinition(fk))" } func generateDropForeignKeySQL(table: String, schema: String?, constraintName: String) -> String? { diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPrincipalQueries.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPrincipalQueries.swift index e581f8f3f3..82c1a37a36 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPrincipalQueries.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPrincipalQueries.swift @@ -6,7 +6,7 @@ import Foundation import TableProPluginKit -enum PostgreSQLRoleAttribute: String, CaseIterable { +nonisolated enum PostgreSQLRoleAttribute: String, CaseIterable { case superuser = "SUPERUSER" case createdb = "CREATEDB" case createrole = "CREATEROLE" @@ -43,7 +43,7 @@ enum PostgreSQLRoleAttribute: String, CaseIterable { } } -enum PostgreSQLPrincipalQueries { +nonisolated enum PostgreSQLPrincipalQueries { private static let data = PluginPrivilegeCategoryKey.data private static let structure = PluginPrivilegeCategoryKey.structure private static let administration = PluginPrivilegeCategoryKey.administration diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLRelationSQL.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLRelationSQL.swift index 96bf6d7001..8355098b65 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLRelationSQL.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLRelationSQL.swift @@ -9,7 +9,7 @@ import Foundation import TableProPluginKit -public enum PostgreSQLRelationSQL { +nonisolated public enum PostgreSQLRelationSQL { /// The keyword `COMMENT ON` needs for each relation kind. PostgreSQL checks it against the /// relation's `relkind` and refuses a mismatch: `COMMENT ON TABLE` on a view fails with /// "is not a table", and `COMMENT ON VIEW` on a materialized view with "is not a view". diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSchemaQueries.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSchemaQueries.swift index 1058084322..196c0da39e 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSchemaQueries.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSchemaQueries.swift @@ -11,13 +11,13 @@ import TableProPluginKit import TableProPostgreSQLCore import TableProSQLQuoting -enum PostgreSQLSchemaProbe: Equatable { +nonisolated enum PostgreSQLSchemaProbe: Equatable { case schema(String) case empty case failed } -enum PostgreSQLSchemaQueries { +nonisolated enum PostgreSQLSchemaQueries { /// Returns the first schema on the effective search path, or SQL NULL /// when the path is empty (neither `$user` nor `public` exists). static let currentSchema = "SELECT pg_catalog.current_schema()" @@ -679,9 +679,8 @@ enum PostgreSQLSchemaQueries { """ } - /// The order `PostgreSQLPluginDriver.mapPgColumnRow` reads the row in. It maps by position, so - /// this list is the contract between the two arms of `columnsQuery` and the mapper. - /// `ordinal_position` stays inside the derived table, named only by the outer `ORDER BY`. + // The order `PostgreSQLCatalogExport.column` reads the row in, by position: the contract between both arms of + // `columnsQuery` and the mapper. `ordinal_position` stays inside the derived table, named only by the `ORDER BY`. private static func columnsOuterProjection(includesTableName: Bool) -> String { let columns = (includesTableName ? ["cols.table_name"] : []) + [ "cols.column_name", @@ -743,14 +742,14 @@ enum PostgreSQLSchemaQueries { } } -struct PostgreSQLColumnSpelling: Equatable { +nonisolated struct PostgreSQLColumnSpelling: Equatable { let defaultValue: String? let generationExpression: String? let declaredType: String? } /// One column's clauses as the server spells them, read under a narrowed `search_path`. -struct PostgreSQLCatalogColumnDDL: Equatable { +nonisolated struct PostgreSQLCatalogColumnDDL: Equatable { let typeSpelling: String let defaultExpression: String? let generationExpression: String? diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSequenceQueries.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSequenceQueries.swift index 521849ae11..c92fbd17bd 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSequenceQueries.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSequenceQueries.swift @@ -7,7 +7,7 @@ import Foundation import TableProPluginKit import TableProPostgreSQLCore -struct PostgreSQLSequenceDefinition: Equatable, Sendable { +nonisolated struct PostgreSQLSequenceDefinition: Equatable, Sendable { let name: String let startValue: String? let minValue: String? @@ -112,7 +112,7 @@ struct PostgreSQLSequenceDefinition: Equatable, Sendable { // The sequence relation's own row: pg_sequences reports NULL both before the first nextval and after // setval(..., false), so only last_value with is_called says where the sequence stands. -struct PostgreSQLSequencePosition: Equatable, Sendable { +nonisolated struct PostgreSQLSequencePosition: Equatable, Sendable { let lastValue: Int64 let isCalled: Bool @@ -140,7 +140,7 @@ struct PostgreSQLSequencePosition: Equatable, Sendable { } } -enum PostgreSQLSequenceQueries { +nonisolated enum PostgreSQLSequenceQueries { enum Source: Equatable { case sequencesView case sequenceParameters @@ -296,7 +296,7 @@ enum PostgreSQLSequenceQueries { } } -enum PostgreSQLSequenceStatements { +nonisolated enum PostgreSQLSequenceStatements { static let createTemplate = """ CREATE SEQUENCE sequence_name INCREMENT BY 1 diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSequenceReference.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSequenceReference.swift index 0596b1c563..912fb4ead0 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSequenceReference.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSequenceReference.swift @@ -10,7 +10,7 @@ import Foundation import TableProPluginKit import TableProSQLQuoting -struct PostgreSQLSequenceReference: Equatable { +nonisolated struct PostgreSQLSequenceReference: Equatable { /// `regclassout` under a `search_path` of `pg_catalog`: `sales.orders_id_seq`. let qualifiedName: String /// `regclassout` where the sequence's own schema is on the path: `orders_id_seq`. diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSessionScopeSQL.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSessionScopeSQL.swift index bdec181ea8..621c11199e 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSessionScopeSQL.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLSessionScopeSQL.swift @@ -6,18 +6,24 @@ import Foundation import TableProPluginKit -internal struct PostgreSQLBoundStatement: Equatable, Sendable { +nonisolated internal struct PostgreSQLBoundStatement: Equatable, Sendable { let sql: String let parameters: [PluginCellValue] } -internal enum PostgreSQLSessionScopeError: Error, Equatable { +nonisolated internal enum PostgreSQLSessionScopeError: Error, Equatable { case abortedTransaction case sessionNotIdle case restoreRefused([String]) } -internal enum PostgreSQLSessionScopeSQL { +nonisolated internal enum PostgreSQLSessionScopeSQL { + // A setting sent inside the user's transaction runs under this savepoint, so a refusal leaves the transaction open. + static let settingSavepoint = "SAVEPOINT tablepro_session_setting" + static let settingRelease = "RELEASE SAVEPOINT tablepro_session_setting" + static let settingRollback = + "ROLLBACK TO SAVEPOINT tablepro_session_setting; RELEASE SAVEPOINT tablepro_session_setting" + /// Every `SET` a pg_dump 17.11 plain file runs and never puts back, the settings a value's text form depends on, /// and `session_replication_role`, which the import's foreign key switch sets. static let preservedNames = [ diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift index 347c088d4d..2c9af55652 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift @@ -9,7 +9,7 @@ import TableProPostgreSQLCore // What a table's CREATE statement says outside its column list. The table listing hides the partitions of a // partitioned table, so the parent's script creates them, with what each holds of its own: without them a restore has // nowhere to put a row, and a partition's own check, key, default and index would be lost. -enum PostgreSQLTableDDL { +nonisolated enum PostgreSQLTableDDL { struct Shape: Equatable { var isUnlogged = false var partitionKey: String? diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTypeDefinition.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTypeDefinition.swift index 5d5564c3d2..ca1780263b 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTypeDefinition.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTypeDefinition.swift @@ -10,7 +10,7 @@ import Foundation import TableProPluginKit import TableProPostgreSQLCore -public struct PostgreSQLDomainConstraint: Sendable, Equatable { +nonisolated public struct PostgreSQLDomainConstraint: Sendable, Equatable { public let name: String /// `pg_get_constraintdef` without the ` NOT VALID` it appends to an unvalidated check. @@ -24,7 +24,7 @@ public struct PostgreSQLDomainConstraint: Sendable, Equatable { } } -public struct PostgreSQLUserDefinedTypeRecord: Sendable, Equatable { +nonisolated public struct PostgreSQLUserDefinedTypeRecord: Sendable, Equatable { public enum Kind: String, Sendable { case enumeration = "e" case composite = "c" @@ -108,7 +108,7 @@ public struct PostgreSQLUserDefinedTypeRecord: Sendable, Equatable { } } -public enum PostgreSQLTypeDefinition { +nonisolated public enum PostgreSQLTypeDefinition { /// The projection `PostgreSQLObjectQueries.userDefinedTypeList` selects, in order. The parser /// reads by these positions, so the query and the parser cannot drift apart silently. public enum Column: Int, CaseIterable { diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLVersionedStatements.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLVersionedStatements.swift index e1c1f827da..d41dac32b7 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLVersionedStatements.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLVersionedStatements.swift @@ -7,7 +7,7 @@ import Foundation import TableProPluginKit import TableProPostgreSQLCore -internal struct PostgreSQLSessionFacts: Sendable, Equatable { +nonisolated internal struct PostgreSQLSessionFacts: Sendable, Equatable { static let probeQuery = "SELECT pg_catalog.current_database(), pg_catalog.current_setting('server_version_num')" static let unknown = PostgreSQLSessionFacts(database: nil, serverVersion: nil) @@ -32,7 +32,7 @@ internal struct PostgreSQLSessionFacts: Sendable, Equatable { } } -internal enum PostgreSQLVersionedStatements { +nonisolated internal enum PostgreSQLVersionedStatements { static let mySQLOnlyIndexTypes: Set = ["FULLTEXT", "SPATIAL"] static func createSchema(_ name: String, capabilities: PostgreSQLCapabilities) -> String { diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLViewDefinition.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLViewDefinition.swift index a6fccf509a..83f46ce162 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLViewDefinition.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLViewDefinition.swift @@ -8,7 +8,7 @@ import Foundation -public enum PostgreSQLViewDefinition { +nonisolated public enum PostgreSQLViewDefinition { public enum Kind: Equatable, Sendable { case view case materializedView @@ -147,7 +147,7 @@ public enum PostgreSQLViewDefinition { } } -private extension String { +nonisolated private extension String { var nilIfBlank: String? { trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ? nil : self } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/RedshiftCatalogExport.swift b/apple/Plugins/PostgreSQLDriverPlugin/RedshiftCatalogExport.swift new file mode 100644 index 0000000000..8cbcec8ce1 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/RedshiftCatalogExport.swift @@ -0,0 +1,369 @@ +// +// RedshiftCatalogExport.swift +// PostgreSQLDriverPlugin +// + +import Foundation +import os +import TableProLogRedaction +import TableProPluginKit + +// Redshift's catalog reads for listing and export, over whichever connection runs them. External schemas list their +// tables and columns only in the svv_external_* views. +nonisolated struct RedshiftCatalogExport: Sendable { + typealias Read = @Sendable (_ statement: String) async throws -> [[PluginCellValue]] + + static let tableDDLIncludesForeignKeys = true + + private static let logger = Logger(subsystem: "com.TablePro.PostgreSQLDriver", category: "RedshiftPluginDriver") + + let read: Read + let currentSchema: String + // The database the connection is on: the svv_external_* views carry rows for every database on the cluster. + let database: String + let serverVersionNumber: Int32 + let externalSchemas: Set + + init( + read: @escaping Read, + currentSchema: String, + database: String, + serverVersionNumber: Int32, + externalSchemas: Set + ) { + self.read = read + self.currentSchema = currentSchema + self.database = database + self.serverVersionNumber = serverVersionNumber + self.externalSchemas = externalSchemas + } + + // A cluster with no external catalog answers this in one cheap read. + static func externalSchemaNames(read: Read) async throws -> Set { + Set(try await read(RedshiftExternalSchemaQueries.listExternalSchemaNames).compactMap { $0.first?.asText }) + } + + private func isExternalSchema(_ schema: String) -> Bool { + externalSchemas.contains(schema) + } + + // MARK: - Tables + + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { + let resolvedSchema = schema ?? currentSchema + let localTables = try await read(RedshiftTableCatalog.listingQuery(schema: resolvedSchema)) + .compactMap { RedshiftTableCatalog.table(fromListingRow: $0.map(\.asText)) } + + guard isExternalSchema(resolvedSchema) else { return localTables } + + let externalTables = await fetchExternalTables(schema: resolvedSchema) + guard !externalTables.isEmpty else { return localTables } + + let localNames = Set(localTables.map(\.name)) + let merged = localTables + externalTables.filter { !localNames.contains($0.name) } + return merged.sorted { $0.name.localizedStandardCompare($1.name) == .orderedAscending } + } + + func fetchTables(inSchemas schemas: [String]) async throws -> [PluginTableInfo] { + guard !schemas.isEmpty else { return [] } + var tables = try await read(RedshiftTableCatalog.listingQuery(inSchemas: schemas)) + .compactMap { RedshiftTableCatalog.table(fromListingRow: $0.map(\.asText)) } + for schema in schemas where isExternalSchema(schema) { + let localNames = Set(tables.lazy.filter { $0.schema == schema }.map(\.name)) + tables += await fetchExternalTables(schema: schema).filter { !localNames.contains($0.name) } + } + return tables + } + + private func fetchExternalTables(schema: String) async -> [PluginTableInfo] { + do { + let query = RedshiftExternalSchemaQueries.listExternalTables(schema: schema, database: database) + return try await read(query).compactMap { row -> PluginTableInfo? in + guard let name = row[0].asText else { return nil } + let rawType = row.count > 1 ? row[1].asText : nil + return PluginTableInfo( + name: name, + type: RedshiftExternalSchemaQueries.classifyTableType(rawTabletype: rawType), + schema: schema + ) + } + } catch { + Self.logger.warning( + "svv_external_tables failed for schema \(schema, privacy: .private(mask: .hash)); listing local tables only: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + return [] + } + } + + // MARK: - Columns + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + let resolvedSchema = schema ?? currentSchema + if isExternalSchema(resolvedSchema) { + let external = await fetchExternalColumns(schema: resolvedSchema, table: table) + if !external.isEmpty { return external } + } + return try await fetchLocalColumns(table: table, schema: resolvedSchema) + } + + func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { + let resolvedSchema = schema ?? currentSchema + if isExternalSchema(resolvedSchema) { + let external = await fetchExternalAllColumns(schema: resolvedSchema) + if !external.isEmpty { return external } + } + return try await fetchLocalAllColumns(schema: resolvedSchema) + } + + private func fetchExternalColumns(schema: String, table: String) async -> [PluginColumnInfo] { + do { + let query = RedshiftExternalSchemaQueries.listExternalColumns(schema: schema, table: table, database: database) + return try await read(query).compactMap { row -> PluginColumnInfo? in + guard row.count >= 2, let name = row[0].asText, let dataType = row[1].asText else { return nil } + return Self.externalColumn(name: name, dataType: dataType, row: row, typeIndex: 1) + } + } catch { + Self.logger.warning( + "svv_external_columns failed for schema \(schema, privacy: .private(mask: .hash)): \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + return [] + } + } + + private func fetchExternalAllColumns(schema: String) async -> [String: [PluginColumnInfo]] { + do { + let query = RedshiftExternalSchemaQueries.listExternalColumns(schema: schema, table: nil, database: database) + var allColumns: [String: [PluginColumnInfo]] = [:] + for row in try await read(query) { + guard row.count >= 3, + let tableName = row[0].asText, + let name = row[1].asText, + let dataType = row[2].asText + else { continue } + let column = Self.externalColumn(name: name, dataType: dataType, row: row, typeIndex: 2) + allColumns[tableName, default: []].append(column) + } + return allColumns + } catch { + Self.logger.warning( + "svv_external_columns failed for schema \(schema, privacy: .private(mask: .hash)): \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + return [:] + } + } + + // External columns carry no default, charset, collation, comment or key, and `external_type` is a Hive type + // string that reaches the UI unparsed so nested `struct` and `array` declarations survive. + private static func externalColumn( + name: String, + dataType: String, + row: [PluginCellValue], + typeIndex: Int + ) -> PluginColumnInfo { + let nullableIndex = typeIndex + 1 + let partKeyIndex = typeIndex + 2 + let rawNullable = row.count > nullableIndex ? row[nullableIndex].asText : nil + let rawPartKey = row.count > partKeyIndex ? row[partKeyIndex].asText : nil + return PluginColumnInfo( + name: name, + dataType: dataType, + isNullable: RedshiftExternalSchemaQueries.classifyIsNullable(raw: rawNullable), + isPrimaryKey: false, + extra: RedshiftExternalSchemaQueries.partitionKeyDescription(rawPartKey: rawPartKey) + ) + } + + private func fetchLocalColumns(table: String, schema: String) async throws -> [PluginColumnInfo] { + try await read(RedshiftSchemaQueries.columnsQuery(schema: schema, table: table)).compactMap { row in + guard row.count >= 4 else { return nil } + return Self.localColumn(row, tableNameOffset: 0) + } + } + + private func fetchLocalAllColumns(schema: String) async throws -> [String: [PluginColumnInfo]] { + var allColumns: [String: [PluginColumnInfo]] = [:] + for row in try await read(RedshiftSchemaQueries.columnsQuery(schema: schema, table: nil)) { + guard row.count >= 5, + let tableName = row[0].asText, + let column = Self.localColumn(row, tableNameOffset: 1) + else { continue } + allColumns[tableName, default: []].append(column) + } + return allColumns + } + + private static func localColumn(_ row: [PluginCellValue], tableNameOffset offset: Int) -> PluginColumnInfo? { + guard let name = row[offset].asText, + let rawDataType = row[offset + 1].asText + else { return nil } + + let udtName = row.count > offset + 6 ? row[offset + 6].asText : nil + let dataType: String + if rawDataType.uppercased() == "USER-DEFINED", let udt = udtName { + dataType = "ENUM(\(udt))" + } else { + dataType = rawDataType.uppercased() + } + + let collation = row.count > offset + 4 ? row[offset + 4].asText : nil + let comment = row.count > offset + 5 ? row[offset + 5].asText : nil + let charset: String? = { + guard let coll = collation, coll.contains(".") else { return nil } + return coll.components(separatedBy: ".").last + }() + + return PluginColumnInfo( + name: name, + dataType: dataType, + isNullable: row[offset + 2].asText == "YES", + isPrimaryKey: row.count > offset + 7 && row[offset + 7].asText == "YES", + defaultValue: row[offset + 3].asText, + charset: charset, + collation: collation, + comment: comment?.isEmpty == false ? comment : nil + ) + } + + // MARK: - Keys + + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { + let query = RedshiftTableCatalog.keysQuery(schema: schema ?? currentSchema, table: table) + return RedshiftTableCatalog.keys(fromRows: try await read(query).map { $0.map(\.asText) }) + } + + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { + let query = PostgreSQLCatalogForeignKeys.query( + schema: schema ?? currentSchema, + table: table, + excludesPartitionClones: PostgreSQLCatalogForeignKeys.excludesPartitionClones( + serverVersionNumber: serverVersionNumber + ) + ) + return PostgreSQLCatalogForeignKeys.foreignKeys(from: try await read(query).map { $0.map(\.asText) }) + } + + // MARK: - Definitions + + func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { + let resolvedSchema = schema ?? currentSchema + guard !isExternalSchema(resolvedSchema) else { return nil } + let query = """ + SELECT tbl_rows + FROM svv_table_info + WHERE "table" = \(PostgreSQLObjectQueries.quoteLiteral(table)) + AND schema = \(PostgreSQLObjectQueries.quoteLiteral(resolvedSchema)) + """ + guard let firstRow = try await read(query).first, let value = firstRow[0].asText, let count = Int(value) else { + return nil + } + return count >= 0 ? count : nil + } + + func fetchTableDDL(table: String, schema: String?) async throws -> String { + let resolvedSchema = schema ?? currentSchema + let quotedTable = PostgreSQLObjectQueries.quoteIdentifier(table) + let quotedSchema = PostgreSQLObjectQueries.quoteIdentifier(resolvedSchema) + + do { + let showRows = try await read("SHOW TABLE \(quotedSchema).\(quotedTable)") + if let firstRow = showRows.first, let ddl = firstRow[0].asText, !ddl.isEmpty { + return ddl + } + } catch { + Self.logger.debug("SHOW TABLE not available, falling back to manual reconstruction") + } + + // pg_attribute lists a view's columns as it lists a table's, so a view is looked up before the reconstruction. + let view = try await read(Self.viewDefinitionQuery(view: table, schema: resolvedSchema)) + if let ddl = view.first?[safe: 0]?.asText { + return ddl + } + + let columnDefs = try await read(Self.columnDefinitionsQuery(table: table, schema: resolvedSchema)) + .compactMap { $0[0].asText } + guard !columnDefs.isEmpty else { + throw PostgreSQLCatalogReadError(message: "Failed to fetch DDL for table '\(table)'") + } + + var parts = columnDefs + parts.append(contentsOf: try await foreignKeyClauses(table: table, schema: schema)) + + let ddl = "CREATE TABLE \(quotedSchema).\(quotedTable) (\n " + parts.joined(separator: ",\n ") + "\n);" + + do { + var suffixes: [String] = [] + for idx in try await fetchIndexes(table: table, schema: schema) { + if idx.type == "DISTKEY", let col = idx.columns.first { + suffixes.append("DISTKEY(\(col))") + } + if idx.type == "SORTKEY" { + suffixes.append("SORTKEY(\(idx.columns.joined(separator: ", ")))") + } + } + if !suffixes.isEmpty { + return ddl + "\n" + suffixes.joined(separator: "\n") + ";" + } + } catch { + Self.logger.debug("Could not fetch DISTKEY/SORTKEY info: \(error.localizedDescription)") + } + return ddl + } + + // `SHOW TABLE` declares foreign keys inline, so the reconstruction does too. A failed lookup throws: an empty + // answer would read as a table with no foreign keys. + private func foreignKeyClauses(table: String, schema: String?) async throws -> [String] { + let quote = PostgreSQLObjectQueries.quoteIdentifier + var orderedNames: [String] = [] + var grouped: [String: [PluginForeignKeyInfo]] = [:] + for foreignKey in try await fetchForeignKeys(table: table, schema: schema) { + if grouped[foreignKey.name] == nil { orderedNames.append(foreignKey.name) } + grouped[foreignKey.name, default: []].append(foreignKey) + } + return orderedNames.compactMap { name in + guard let group = grouped[name], let first = group.first else { return nil } + let columns = group.map { quote($0.column) }.joined(separator: ", ") + let referencedColumns = group.map { quote($0.referencedColumn) }.joined(separator: ", ") + let referencedSchema = first.referencedSchema.flatMap { $0.isEmpty ? nil : $0 } + let referencedTable = referencedSchema.map { + "\(quote($0)).\(quote(first.referencedTable))" + } ?? quote(first.referencedTable) + return "CONSTRAINT \(quote(name)) FOREIGN KEY (\(columns))" + + " REFERENCES \(referencedTable) (\(referencedColumns))" + } + } + + func fetchViewDefinition(view: String, schema: String?) async throws -> String { + let rows = try await read(Self.viewDefinitionQuery(view: view, schema: schema ?? currentSchema)) + guard let firstRow = rows.first, let ddl = firstRow[0].asText else { + throw PostgreSQLCatalogReadError(message: "Failed to fetch definition for view '\(view)'") + } + return ddl + } + + private static func viewDefinitionQuery(view: String, schema: String) -> String { + """ + SELECT 'CREATE OR REPLACE VIEW ' || pg_catalog.quote_ident(schemaname) || '.' || pg_catalog.quote_ident(viewname) || ' AS ' || E'\\n' || definition AS ddl + FROM pg_catalog.pg_views + WHERE viewname = \(PostgreSQLObjectQueries.quoteLiteral(view)) + AND schemaname = \(PostgreSQLObjectQueries.quoteLiteral(schema)) + """ + } + + private static func columnDefinitionsQuery(table: String, schema: String) -> String { + """ + SELECT + pg_catalog.quote_ident(a.attname) || ' ' || pg_catalog.format_type(a.atttypid, a.atttypmod) || + CASE WHEN a.attnotnull THEN ' NOT NULL' ELSE '' END || + CASE WHEN a.atthasdef THEN ' DEFAULT ' || pg_catalog.pg_get_expr(d.adbin, d.adrelid) ELSE '' END + FROM pg_catalog.pg_attribute a + JOIN pg_catalog.pg_class c ON c.oid = a.attrelid + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_catalog.pg_attrdef d ON d.adrelid = c.oid AND d.adnum = a.attnum + WHERE c.relname = \(PostgreSQLObjectQueries.quoteLiteral(table)) + AND n.nspname = \(PostgreSQLObjectQueries.quoteLiteral(schema)) + AND a.attnum > 0 + AND NOT a.attisdropped + ORDER BY a.attnum + """ + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/RedshiftExternalSchemaQueries.swift b/apple/Plugins/PostgreSQLDriverPlugin/RedshiftExternalSchemaQueries.swift index 63d4a1e3d0..9f9fedec61 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/RedshiftExternalSchemaQueries.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/RedshiftExternalSchemaQueries.swift @@ -16,7 +16,7 @@ import Foundation -enum RedshiftExternalSchemaQueries { +nonisolated enum RedshiftExternalSchemaQueries { static let listExternalSchemaNames = "SELECT schemaname FROM svv_external_schemas" /// Tables registered in one external schema. Both views carry rows for diff --git a/apple/Plugins/PostgreSQLDriverPlugin/RedshiftPluginDriver.swift b/apple/Plugins/PostgreSQLDriverPlugin/RedshiftPluginDriver.swift index 8b7c359b62..82e1a891fc 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/RedshiftPluginDriver.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/RedshiftPluginDriver.swift @@ -52,14 +52,11 @@ final class RedshiftPluginDriver: LibPQBackedDriver, @unchecked Sendable { // MARK: - Schema - /// Refreshed from `onPostConnect` and whenever the schema list is loaded, so - /// a schema created mid-session is classified without a reconnect. A failed - /// probe leaves the previous answer in place rather than replacing it with - /// an empty one. A cluster with no external catalog answers in one cheap read. + // Refreshed on connect and with every schema list, so a schema made mid-session is classified; a failed probe + // keeps the previous answer. private func probeExternalSchemas() async { do { - let result = try await catalogRead(RedshiftExternalSchemaQueries.listExternalSchemaNames) - let names = Set(result.rows.compactMap { $0.first?.asText }) + let names = try await RedshiftCatalogExport.externalSchemaNames(read: catalogRows) externalSchemaLock.withLock { externalSchemaCache = names } } catch { Self.logger.warning( @@ -68,6 +65,20 @@ final class RedshiftPluginDriver: LibPQBackedDriver, @unchecked Sendable { } } + private var catalogRows: RedshiftCatalogExport.Read { + { [self] statement in try await catalogRead(statement).rows } + } + + var exportCatalog: RedshiftCatalogExport { + RedshiftCatalogExport( + read: catalogRows, + currentSchema: core.currentSchema, + database: connectedDatabase, + serverVersionNumber: core.serverVersionNumber, + externalSchemas: externalSchemaLock.withLock { externalSchemaCache } ?? [] + ) + } + func fetchExternalSchemaNames() async throws -> Set { externalSchemaLock.withLock { externalSchemaCache } ?? [] } @@ -78,257 +89,31 @@ final class RedshiftPluginDriver: LibPQBackedDriver, @unchecked Sendable { } func fetchTables(schema: String?) async throws -> [PluginTableInfo] { - let resolvedSchema = schema ?? core.currentSchema - let result = try await catalogRead(RedshiftTableCatalog.listingQuery(schema: resolvedSchema)) - let localTables = result.rows.compactMap { RedshiftTableCatalog.table(fromListingRow: $0.map(\.asText)) } - - guard isExternalSchema(resolvedSchema) else { return localTables } - - let externalTables = await fetchExternalTables(schema: resolvedSchema) - guard !externalTables.isEmpty else { return localTables } - - let localNames = Set(localTables.map(\.name)) - let merged = localTables + externalTables.filter { !localNames.contains($0.name) } - return merged.sorted { $0.name.localizedStandardCompare($1.name) == .orderedAscending } + try await exportCatalog.fetchTables(schema: schema) } func fetchTables(inSchemas schemas: [String]) async throws -> [PluginTableInfo]? { - guard !schemas.isEmpty else { return [] } - let result = try await catalogRead(RedshiftTableCatalog.listingQuery(inSchemas: schemas)) - var tables = result.rows.compactMap { RedshiftTableCatalog.table(fromListingRow: $0.map(\.asText)) } - for schema in schemas where isExternalSchema(schema) { - let localNames = Set(tables.lazy.filter { $0.schema == schema }.map(\.name)) - tables += await fetchExternalTables(schema: schema).filter { !localNames.contains($0.name) } - } - return tables - } - - private func fetchExternalTables(schema: String) async -> [PluginTableInfo] { - do { - let result = try await catalogRead( - RedshiftExternalSchemaQueries.listExternalTables( - schema: schema, - database: connectedDatabase - ) - ) - return result.rows.compactMap { row -> PluginTableInfo? in - guard let name = row[0].asText else { return nil } - let rawType = row.count > 1 ? row[1].asText : nil - return PluginTableInfo( - name: name, - type: RedshiftExternalSchemaQueries.classifyTableType(rawTabletype: rawType), - schema: schema - ) - } - } catch { - Self.logger.warning( - "svv_external_tables failed for schema \(schema, privacy: .private(mask: .hash)); listing local tables only: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" - ) - return [] - } + try await exportCatalog.fetchTables(inSchemas: schemas) } func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { - let resolvedSchema = schema ?? core.currentSchema - if isExternalSchema(resolvedSchema) { - let external = await fetchExternalColumns(schema: resolvedSchema, table: table) - if !external.isEmpty { return external } - } - return try await fetchLocalColumns(table: table, schema: resolvedSchema) - } - - private func fetchExternalColumns(schema: String, table: String) async -> [PluginColumnInfo] { - do { - let result = try await catalogRead( - RedshiftExternalSchemaQueries.listExternalColumns( - schema: schema, - table: table, - database: connectedDatabase - ) - ) - return result.rows.compactMap { row -> PluginColumnInfo? in - guard row.count >= 2, let name = row[0].asText, let dataType = row[1].asText else { return nil } - return Self.externalColumn(name: name, dataType: dataType, row: row, typeIndex: 1) - } - } catch { - Self.logger.warning( - "svv_external_columns failed for schema \(schema, privacy: .private(mask: .hash)): \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" - ) - return [] - } - } - - /// External columns carry no default, charset, collation, comment, or key - /// information, and `external_type` is an opaque Hive type string that must - /// reach the UI unparsed so nested `struct`/`array` declarations survive. - private static func externalColumn( - name: String, - dataType: String, - row: [PluginCellValue], - typeIndex: Int - ) -> PluginColumnInfo { - let nullableIndex = typeIndex + 1 - let partKeyIndex = typeIndex + 2 - let rawNullable = row.count > nullableIndex ? row[nullableIndex].asText : nil - let rawPartKey = row.count > partKeyIndex ? row[partKeyIndex].asText : nil - return PluginColumnInfo( - name: name, - dataType: dataType, - isNullable: RedshiftExternalSchemaQueries.classifyIsNullable(raw: rawNullable), - isPrimaryKey: false, - extra: RedshiftExternalSchemaQueries.partitionKeyDescription(rawPartKey: rawPartKey) - ) - } - - private func fetchLocalColumns(table: String, schema: String) async throws -> [PluginColumnInfo] { - let query = RedshiftSchemaQueries.columnsQuery(schema: schema, table: table) - let result = try await catalogRead(query) - return result.rows.compactMap { row -> PluginColumnInfo? in - guard row.count >= 4, - let name = row[0].asText, - let rawDataType = row[1].asText - else { return nil } - - let udtName = row.count > 6 ? row[6].asText : nil - let dataType: String - if rawDataType.uppercased() == "USER-DEFINED", let udt = udtName { - dataType = "ENUM(\(udt))" - } else { - dataType = rawDataType.uppercased() - } - - let isNullable = row[2].asText == "YES" - let defaultValue = row[3].asText - let collation = row.count > 4 ? row[4].asText : nil - let comment = row.count > 5 ? row[5].asText : nil - let isPk = row.count > 7 && row[7].asText == "YES" - - let charset: String? = { - guard let coll = collation else { return nil } - if coll.contains(".") { - return coll.components(separatedBy: ".").last - } - return nil - }() - - return PluginColumnInfo( - name: name, - dataType: dataType, - isNullable: isNullable, - isPrimaryKey: isPk, - defaultValue: defaultValue, - charset: charset, - collation: collation, - comment: comment?.isEmpty == false ? comment : nil - ) - } + try await exportCatalog.fetchColumns(table: table, schema: schema) } var fetchesAllColumnsInOneRequest: Bool { true } func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { - let resolvedSchema = schema ?? core.currentSchema - if isExternalSchema(resolvedSchema) { - let external = await fetchExternalAllColumns(schema: resolvedSchema) - if !external.isEmpty { return external } - } - return try await fetchLocalAllColumns(schema: resolvedSchema) - } - - private func fetchExternalAllColumns(schema: String) async -> [String: [PluginColumnInfo]] { - do { - let result = try await catalogRead( - RedshiftExternalSchemaQueries.listExternalColumns( - schema: schema, - table: nil, - database: connectedDatabase - ) - ) - var allColumns: [String: [PluginColumnInfo]] = [:] - for row in result.rows { - guard row.count >= 3, - let tableName = row[0].asText, - let name = row[1].asText, - let dataType = row[2].asText - else { continue } - let column = Self.externalColumn(name: name, dataType: dataType, row: row, typeIndex: 2) - allColumns[tableName, default: []].append(column) - } - return allColumns - } catch { - Self.logger.warning( - "svv_external_columns failed for schema \(schema, privacy: .private(mask: .hash)): \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" - ) - return [:] - } - } - - private func fetchLocalAllColumns(schema: String) async throws -> [String: [PluginColumnInfo]] { - let query = RedshiftSchemaQueries.columnsQuery(schema: schema, table: nil) - let result = try await catalogRead(query) - var allColumns: [String: [PluginColumnInfo]] = [:] - for row in result.rows { - guard row.count >= 5, - let tableName = row[0].asText, - let name = row[1].asText, - let rawDataType = row[2].asText - else { continue } - - let udtName = row.count > 7 ? row[7].asText : nil - let dataType: String - if rawDataType.uppercased() == "USER-DEFINED", let udt = udtName { - dataType = "ENUM(\(udt))" - } else { - dataType = rawDataType.uppercased() - } - - let isNullable = row[3].asText == "YES" - let defaultValue = row[4].asText - let collation = row.count > 5 ? row[5].asText : nil - let comment = row.count > 6 ? row[6].asText : nil - let isPk = row.count > 8 && row[8].asText == "YES" - - let charset: String? = { - guard let coll = collation else { return nil } - if coll.contains(".") { - return coll.components(separatedBy: ".").last - } - return nil - }() - - let column = PluginColumnInfo( - name: name, - dataType: dataType, - isNullable: isNullable, - isPrimaryKey: isPk, - defaultValue: defaultValue, - charset: charset, - collation: collation, - comment: comment?.isEmpty == false ? comment : nil - ) - allColumns[tableName, default: []].append(column) - } - return allColumns + try await exportCatalog.fetchAllColumns(schema: schema) } func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { - let query = RedshiftTableCatalog.keysQuery(schema: schema ?? core.currentSchema, table: table) - let result = try await catalogRead(query) - return RedshiftTableCatalog.keys(fromRows: result.rows.map { $0.map(\.asText) }) + try await exportCatalog.fetchIndexes(table: table, schema: schema) } - var tableDDLIncludesForeignKeys: Bool { true } + var tableDDLIncludesForeignKeys: Bool { RedshiftCatalogExport.tableDDLIncludesForeignKeys } func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { - let query = PostgreSQLCatalogForeignKeys.query( - schema: schema ?? core.currentSchema, - table: table, - excludesPartitionClones: PostgreSQLCatalogForeignKeys.excludesPartitionClones( - serverVersionNumber: core.serverVersionNumber - ) - ) - let result = try await catalogRead(query) - return PostgreSQLCatalogForeignKeys.foreignKeys(from: result.rows.map { $0.map(\.asText) }) + try await exportCatalog.fetchForeignKeys(table: table, schema: schema) } func fetchCheckConstraints(table: String, schema: String?) async throws -> [PluginCheckConstraintInfo] { @@ -336,133 +121,15 @@ final class RedshiftPluginDriver: LibPQBackedDriver, @unchecked Sendable { } func fetchApproximateRowCount(table: String, schema: String?) async throws -> Int? { - let resolvedSchema = schema ?? core.currentSchema - guard !isExternalSchema(resolvedSchema) else { return nil } - let tableLiteral = PostgreSQLObjectQueries.quoteLiteral(table) - let schemaLiteral = PostgreSQLObjectQueries.quoteLiteral(resolvedSchema) - let query = """ - SELECT tbl_rows - FROM svv_table_info - WHERE "table" = \(tableLiteral) - AND schema = \(schemaLiteral) - """ - let result = try await catalogRead(query) - guard let firstRow = result.rows.first, let value = firstRow[0].asText, let count = Int(value) else { return nil } - return count >= 0 ? count : nil + try await exportCatalog.fetchApproximateRowCount(table: table, schema: schema) } func fetchTableDDL(table: String, schema: String?) async throws -> String { - let tableLiteral = PostgreSQLObjectQueries.quoteLiteral(table) - let resolvedSchema = schema ?? core.currentSchema - let schemaLiteral = PostgreSQLObjectQueries.quoteLiteral(resolvedSchema) - let quotedTable = quoteIdentifier(table) - let quotedSchema = quoteIdentifier(resolvedSchema) - - do { - let showResult = try await catalogRead("SHOW TABLE \(quotedSchema).\(quotedTable)") - if let firstRow = showResult.rows.first, let ddl = firstRow[0].asText, !ddl.isEmpty { - return ddl - } - } catch { - Self.logger.debug("SHOW TABLE not available, falling back to manual reconstruction") - } - - /// `pg_attribute` lists a view's columns as it lists a table's, so a view is looked up before - /// the reconstruction below, which rebuilt a view as a `CREATE TABLE` of its columns. - let view = try await catalogRead(Self.viewDefinitionQuery(view: table, schema: resolvedSchema)) - if let ddl = view.rows.first?[safe: 0]?.asText { - return ddl - } - - let columnsQuery = """ - SELECT - pg_catalog.quote_ident(a.attname) || ' ' || pg_catalog.format_type(a.atttypid, a.atttypmod) || - CASE WHEN a.attnotnull THEN ' NOT NULL' ELSE '' END || - CASE WHEN a.atthasdef THEN ' DEFAULT ' || pg_catalog.pg_get_expr(d.adbin, d.adrelid) ELSE '' END - FROM pg_catalog.pg_attribute a - JOIN pg_catalog.pg_class c ON c.oid = a.attrelid - JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace - LEFT JOIN pg_catalog.pg_attrdef d ON d.adrelid = c.oid AND d.adnum = a.attnum - WHERE c.relname = \(tableLiteral) - AND n.nspname = \(schemaLiteral) - AND a.attnum > 0 - AND NOT a.attisdropped - ORDER BY a.attnum - """ - let columnsResult = try await catalogRead(columnsQuery) - let columnDefs = columnsResult.rows.compactMap { $0[0].asText } - guard !columnDefs.isEmpty else { - throw LibPQPluginError(message: "Failed to fetch DDL for table '\(table)'", sqlState: nil, detail: nil) - } - - var parts = columnDefs - parts.append(contentsOf: try await foreignKeyClauses(table: table, schema: schema)) - - let ddl = "CREATE TABLE \(quotedSchema).\(quotedTable) (\n " + - parts.joined(separator: ",\n ") + - "\n);" - - do { - let indexes = try await fetchIndexes(table: table, schema: schema) - var suffixes: [String] = [] - for idx in indexes { - if idx.type == "DISTKEY", let col = idx.columns.first { - suffixes.append("DISTKEY(\(col))") - } - if idx.type == "SORTKEY" { - suffixes.append("SORTKEY(\(idx.columns.joined(separator: ", ")))") - } - } - if !suffixes.isEmpty { - return ddl + "\n" + suffixes.joined(separator: "\n") + ";" - } - } catch { - Self.logger.debug("Could not fetch DISTKEY/SORTKEY info: \(error.localizedDescription)") - } - return ddl - } - - /// `SHOW TABLE` declares foreign keys inline, so the reconstruction below has to as well or - /// `tableDDLIncludesForeignKeys` would be true of one path and false of the other, and a SQL - /// export would drop every constraint whenever the fallback ran. A failed lookup therefore - /// throws rather than returning nothing, because nothing here is indistinguishable from a - /// table that has no foreign keys. - private func foreignKeyClauses(table: String, schema: String?) async throws -> [String] { - let foreignKeys = try await fetchForeignKeys(table: table, schema: schema) - var orderedNames: [String] = [] - var grouped: [String: [PluginForeignKeyInfo]] = [:] - for foreignKey in foreignKeys { - if grouped[foreignKey.name] == nil { orderedNames.append(foreignKey.name) } - grouped[foreignKey.name, default: []].append(foreignKey) - } - return orderedNames.compactMap { name in - guard let group = grouped[name], let first = group.first else { return nil } - let columns = group.map { quoteIdentifier($0.column) }.joined(separator: ", ") - let referencedColumns = group.map { quoteIdentifier($0.referencedColumn) }.joined(separator: ", ") - let referencedSchema = first.referencedSchema.flatMap { $0.isEmpty ? nil : $0 } - let referencedTable = referencedSchema.map { - "\(quoteIdentifier($0)).\(quoteIdentifier(first.referencedTable))" - } ?? quoteIdentifier(first.referencedTable) - return "CONSTRAINT \(quoteIdentifier(name)) FOREIGN KEY (\(columns))" - + " REFERENCES \(referencedTable) (\(referencedColumns))" - } + try await exportCatalog.fetchTableDDL(table: table, schema: schema) } func fetchViewDefinition(view: String, schema: String?) async throws -> String { - let result = try await catalogRead(Self.viewDefinitionQuery(view: view, schema: schema ?? core.currentSchema)) - guard let firstRow = result.rows.first, let ddl = firstRow[0].asText else { - throw LibPQPluginError(message: "Failed to fetch definition for view '\(view)'", sqlState: nil, detail: nil) - } - return ddl - } - - private static func viewDefinitionQuery(view: String, schema: String) -> String { - """ - SELECT 'CREATE OR REPLACE VIEW ' || pg_catalog.quote_ident(schemaname) || '.' || pg_catalog.quote_ident(viewname) || ' AS ' || E'\\n' || definition AS ddl - FROM pg_catalog.pg_views - WHERE viewname = \(PostgreSQLObjectQueries.quoteLiteral(view)) - AND schemaname = \(PostgreSQLObjectQueries.quoteLiteral(schema)) - """ + try await exportCatalog.fetchViewDefinition(view: view, schema: schema) } func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { diff --git a/apple/Plugins/PostgreSQLDriverPlugin/RedshiftSchemaQueries.swift b/apple/Plugins/PostgreSQLDriverPlugin/RedshiftSchemaQueries.swift index b4095ef712..8724bbe5b3 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/RedshiftSchemaQueries.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/RedshiftSchemaQueries.swift @@ -5,7 +5,7 @@ import Foundation -enum RedshiftSchemaQueries { +nonisolated enum RedshiftSchemaQueries { static func columnsQuery(schema: String, table: String?) -> String { ColumnQueryShape.informationSchemaColumnsQuery(schema: schema, table: table) } diff --git a/apple/Plugins/PostgreSQLDriverTests/CatalogReadRecorder.swift b/apple/Plugins/PostgreSQLDriverTests/CatalogReadRecorder.swift new file mode 100644 index 0000000000..7ae8339b14 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/CatalogReadRecorder.swift @@ -0,0 +1,45 @@ +import Foundation +import TableProPluginKit + +// A fake catalog connection: answers each statement from `answer` and keeps what was sent, in order. +final class CatalogReadRecorder: @unchecked Sendable { + struct Read: Equatable { + let statement: String + let namesRelativeTo: String? + } + + private let lock = NSLock() + private var recorded: [Read] = [] + private let answer: @Sendable (String) throws -> [[PluginCellValue]] + + init(answer: @escaping @Sendable (String) throws -> [[PluginCellValue]] = { _ in [] }) { + self.answer = answer + } + + var reads: [Read] { + lock.withLock { recorded } + } + + var statements: [String] { + reads.map(\.statement) + } + + var postgreSQLRead: PostgreSQLCatalogExport.Read { + { [self] statement, schema in try respond(statement, namesRelativeTo: schema) } + } + + var redshiftRead: RedshiftCatalogExport.Read { + { [self] statement in try respond(statement, namesRelativeTo: nil) } + } + + private func respond(_ statement: String, namesRelativeTo schema: String?) throws -> [[PluginCellValue]] { + lock.withLock { recorded.append(Read(statement: statement, namesRelativeTo: schema)) } + return try answer(statement) + } +} + +extension Array where Element == String? { + var cells: [PluginCellValue] { + map { $0.map(PluginCellValue.text) ?? .null } + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/PostgreSQLCatalogExportTests.swift b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLCatalogExportTests.swift new file mode 100644 index 0000000000..5428761769 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLCatalogExportTests.swift @@ -0,0 +1,223 @@ +import Foundation +import TableProPluginKit +import TableProPostgreSQLCore +import Testing + +struct PostgreSQLCatalogExportTests { + private static let modern = PostgreSQLCapabilities(serverVersion: 170_011) + private static let legacy = PostgreSQLCapabilities(serverVersion: 90_600) + + private static func catalog( + _ recorder: CatalogReadRecorder, + capabilities: PostgreSQLCapabilities = modern, + catalogCapabilities: PostgreSQLCapabilities = modern + ) -> PostgreSQLCatalogExport { + PostgreSQLCatalogExport( + read: recorder.postgreSQLRead, + currentSchema: "app", + capabilities: capabilities, + catalogCapabilities: catalogCapabilities, + catalogPresence: nil + ) + } + + @Test("Routines read the listing under the catalog capabilities and keep the oid as identity") + func routinesMapTheListing() async throws { + let recorder = CatalogReadRecorder { _ in + [["16401", "total", "app", "(integer)", "integer", "plpgsql", "p", "VOLATILE", "INVOKER", "owner"].cells] + } + let routines = try await Self.catalog(recorder, catalogCapabilities: Self.legacy).fetchRoutines(schema: nil) + + #expect(recorder.reads == [.init( + statement: PostgreSQLObjectQueries.routineList(schema: "app", capabilities: Self.legacy), + namesRelativeTo: nil + )]) + let routine = try #require(routines.first) + #expect(routine.name == "total") + #expect(routine.kind == .procedure) + #expect(routine.schema == "app") + #expect(routine.returnType == "integer") + #expect(routine.language == "plpgsql") + #expect(routine.argumentSignature == "(integer)") + #expect(routine.identity == "16401") + #expect(routine.attributes.map { $0.label } == ["Volatility", "Security", "Owner"]) + } + + @Test("A routine with an oid identity reads its definition by oid") + func routineDefinitionByOid() async throws { + let recorder = CatalogReadRecorder { _ in [["CREATE FUNCTION app.total() ..."].cells] } + let routine = PluginRoutineInfo( + name: "total", kind: .function, schema: "app", argumentSignature: "()", identity: "16401" + ) + let ddl = try await Self.catalog(recorder).fetchRoutineDDL(routine) + + #expect(ddl == "CREATE FUNCTION app.total() ...") + #expect(recorder.statements == [PostgreSQLObjectQueries.routineDefinition(identity: "16401")]) + } + + @Test("A routine whose definition comes back empty is not found") + func emptyRoutineDefinitionIsNotFound() async { + let recorder = CatalogReadRecorder { _ in [[""].cells] } + let routine = PluginRoutineInfo(name: "gone", kind: .function) + await #expect(throws: PluginObjectSourceError.self) { + try await Self.catalog(recorder).fetchRoutineDDL(routine) + } + } + + @Test("The schema-wide trigger list maps each row and falls back to the current schema") + func triggersMapTheListing() async throws { + let recorder = CatalogReadRecorder { _ in + [["audit", "orders", nil, "AFTER", "INSERT", "ROW", "t", "CREATE TRIGGER audit ...", "owner"].cells] + } + let triggers = try await Self.catalog(recorder).fetchAllTriggers(schema: nil) + + #expect(recorder.statements == [PostgreSQLObjectQueries.triggerList(schema: "app", table: nil)]) + let trigger = try #require(triggers.first) + #expect(trigger.name == "audit") + #expect(trigger.table == "orders") + #expect(trigger.schema == "app") + #expect(trigger.definition == "CREATE TRIGGER audit ...") + #expect(trigger.enabled == true) + #expect(trigger.attributes == [PluginObjectAttribute(label: "Owner", value: "owner")]) + } + + @Test("Dropping a routine names its kind and argument list, qualified by its schema") + func dropRoutineStatement() { + let catalog = Self.catalog(CatalogReadRecorder()) + #expect(catalog.generateDropRoutineSQL(name: "total", signature: " (integer) ", schema: "sales", isFunction: true) + == #"DROP FUNCTION IF EXISTS "sales"."total"(integer)"#) + #expect(catalog.generateDropRoutineSQL(name: "p_proc", signature: "(integer)", schema: nil, isFunction: false) + == #"DROP PROCEDURE IF EXISTS "app"."p_proc"(integer)"#) + } + + @Test("Dropping a trigger names the table it is on") + func dropTriggerStatement() { + let catalog = Self.catalog(CatalogReadRecorder()) + #expect(catalog.generateDropTriggerSQL(name: "audit", table: "orders", schema: nil) + == #"DROP TRIGGER IF EXISTS "audit" ON "app"."orders""#) + } + + @Test("A table's DDL is its columns and constraints under its qualified name") + func tableDDLAssemblesColumnsAndConstraints() async throws { + let columns = PostgreSQLSchemaQueries.tableDDLColumnsQuery(schema: "app", table: "orders", capabilities: Self.modern) + let constraints = PostgreSQLSchemaQueries.tableDDLConstraintsQuery(schema: "app", table: "orders") + let recorder = CatalogReadRecorder { statement in + switch statement { + case columns: return [[#""id" integer NOT NULL"#, "r"].cells, [#""note" text"#, "r"].cells] + case constraints: return [[#"CONSTRAINT "orders_pkey" PRIMARY KEY ("id")"#].cells] + default: return [] + } + } + let ddl = try await Self.catalog(recorder).fetchTableDDL(table: "orders", schema: nil) + + #expect(ddl == """ + CREATE TABLE "app"."orders" ( + "id" integer NOT NULL, + "note" text, + CONSTRAINT "orders_pkey" PRIMARY KEY ("id") + ); + """) + } + + @Test("A view reached as a table answers with its own definition") + func tableDDLOfAViewIsTheView() async throws { + let columns = PostgreSQLSchemaQueries.tableDDLColumnsQuery(schema: "app", table: "v", capabilities: Self.modern) + let view = PostgreSQLViewDefinition.catalogQuery(name: "v", schema: "app") + let recorder = CatalogReadRecorder { statement in + statement == columns ? [[#""id" integer"#, "v"].cells] : [] + } + await #expect(throws: PostgreSQLCatalogReadError.self) { + try await Self.catalog(recorder).fetchTableDDL(table: "v", schema: nil) + } + #expect(recorder.statements.contains(view)) + } + + @Test("A table with no columns read fails with the driver's message") + func tableDDLWithoutColumnsFails() async { + let recorder = CatalogReadRecorder() + do { + _ = try await Self.catalog(recorder).fetchTableDDL(table: "orders", schema: nil) + Issue.record("Expected a failure") + } catch { + #expect(error.localizedDescription == "Failed to fetch DDL for table 'orders'") + } + } + + @Test("Column spellings read relative to the table's schema, every other column read pinned") + func columnSpellingsReadRelativeToTheSchema() async throws { + let recorder = CatalogReadRecorder() + _ = try await Self.catalog(recorder).fetchColumns(table: "orders", schema: "sales") + + let spelling = PostgreSQLSchemaQueries.columnSpellingQuery(schema: "sales", table: "orders", capabilities: Self.modern) + #expect(recorder.reads.filter { $0.namesRelativeTo != nil } == [.init(statement: spelling, namesRelativeTo: "sales")]) + } + + @Test("All foreign keys group by their table") + func allForeignKeysGroupByTable() async throws { + let recorder = CatalogReadRecorder { _ in + [ + ["orders", "orders_customer_fkey", "customer_id", "customers", "id", "app", "CASCADE", nil].cells, + ["lines", "lines_order_fkey", "order_id", "orders", "id", "app", nil, nil].cells + ] + } + let grouped = try await Self.catalog(recorder).fetchAllForeignKeys(schema: nil) + + #expect(recorder.statements == [ + PostgreSQLForeignKeyQueries.foreignKeyList(schema: "app", table: nil, capabilities: Self.modern) + ]) + #expect(grouped["orders"]?.map { $0.name } == ["orders_customer_fkey"]) + #expect(grouped["orders"]?.first?.onDelete == "CASCADE") + #expect(grouped["lines"]?.first?.onUpdate == "NO ACTION") + } + + @Test("The approximate row count reads the planner's estimate") + func approximateRowCount() async throws { + let recorder = CatalogReadRecorder { _ in [["1200"].cells] } + let count = try await Self.catalog(recorder).fetchApproximateRowCount(table: "orders", schema: nil) + + #expect(recorder.statements == [PostgreSQLSchemaQueries.approximateRowCount(schema: "app", table: "orders")]) + #expect(count == PostgreSQLSchemaQueries.rowEstimate(reltuples: "1200").map { Int($0) }) + } + + @Test("Grants render per object for the quoted grantee") + func grantStatements() { + let changeSet = PluginPrincipalChangeSet( + principal: PluginPrincipalRef(name: "reader"), + grantsToAdd: [ + PluginGrantInfo(privilege: "SELECT", scope: .table(database: "db", schema: "app", table: "orders")), + PluginGrantInfo(privilege: "USAGE", scope: .schema(database: "db", schema: "app"), isGrantable: true) + ] + ) + #expect(PostgreSQLCatalogExport.generateGrantSQL(changeSet: changeSet) == [ + #"GRANT SELECT ON TABLE "app"."orders" TO "reader""#, + #"GRANT USAGE ON SCHEMA "app" TO "reader" WITH GRANT OPTION"# + ]) + } + + @Test("Table DDL leaves foreign keys to the statements after the rows") + func foreignKeysFollowTheRows() { + #expect(!PostgreSQLCatalogExport.tableDDLIncludesForeignKeys) + } + + @Test("A session scope relaxing foreign keys opens nothing and runs nothing") + func relaxedForeignKeysOpenNoScope() async throws { + let recorder = CatalogReadRecorder() + let scope = PostgreSQLSessionScope( + setting: { statement, _ in try await recorder.postgreSQLRead(statement.sql, nil) }, + transactionState: { .idle } + ) + #expect(try await scope.open(for: .relaxedForeignKeys) == nil) + #expect(recorder.reads.isEmpty) + } + + @Test("A preserve scope sends nothing inside an aborted transaction") + func preserveScopeSkipsAnAbortedTransaction() async throws { + let recorder = CatalogReadRecorder() + let scope = PostgreSQLSessionScope( + setting: { statement, _ in try await recorder.postgreSQLRead(statement.sql, nil) }, + transactionState: { .abortedTransaction } + ) + #expect(try await scope.open(for: .preserve) == nil) + #expect(recorder.reads.isEmpty) + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/PostgreSQLCreateTableSQLTests.swift b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLCreateTableSQLTests.swift new file mode 100644 index 0000000000..00eb325dd1 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLCreateTableSQLTests.swift @@ -0,0 +1,98 @@ +import Foundation +import TableProPluginKit +import TableProPostgreSQLCore +import Testing + +struct PostgreSQLCreateTableSQLTests { + private static let pg17 = PostgreSQLCapabilities(serverVersion: 170_011) + private static let pg18 = PostgreSQLCapabilities(serverVersion: 180_000) + + @Test("A single key is declared inline and a column comment follows the table") + func singleKeyInlineWithComment() throws { + let definition = PluginCreateTableDefinition( + tableName: "orders", + columns: [ + PluginColumnDefinition(name: "id", dataType: "integer", isNullable: false, isPrimaryKey: true), + PluginColumnDefinition(name: "note", dataType: "text", defaultValue: "'x'", comment: "Free text") + ] + ) + let statements = try #require(PostgreSQLCreateTableSQL.statements(definition, schema: "app", capabilities: Self.pg17)) + + #expect(statements == [ + """ + CREATE TABLE "app"."orders" ( + "id" integer NOT NULL PRIMARY KEY, + "note" text NULL DEFAULT 'x' + ) + """, + #"COMMENT ON COLUMN "app"."orders"."note" IS 'Free text'"# + ]) + } + + @Test("A composite key and a foreign key are table constraints") + func compositeKeyAndForeignKey() throws { + let definition = PluginCreateTableDefinition( + tableName: "child", + columns: [ + PluginColumnDefinition(name: "a", dataType: "integer", isNullable: false, isPrimaryKey: true), + PluginColumnDefinition(name: "b", dataType: "integer", isNullable: false, isPrimaryKey: true) + ], + foreignKeys: [ + PluginForeignKeyDefinition( + name: "child_fk", columns: ["a"], referencedTable: "parents", referencedColumns: ["id"], + onDelete: "CASCADE", referencedSchema: "core" + ) + ] + ) + let parts = try #require(PostgreSQLCreateTableSQL.parts(definition, schema: "app", capabilities: Self.pg17)) + + #expect(parts.qualifiedTable == #""app"."child""#) + #expect(parts.createTable == """ + CREATE TABLE "app"."child" ( + "a" integer NOT NULL, + "b" integer NOT NULL, + PRIMARY KEY ("a", "b"), + CONSTRAINT "child_fk" FOREIGN KEY ("a") REFERENCES "core"."parents" ("id") ON DELETE CASCADE + ) + """) + #expect(parts.indexStatements.isEmpty) + } + + @Test("A serial column states no nullability") + func serialColumn() { + let column = PluginColumnDefinition(name: "id", dataType: "BIGINT", isNullable: false, autoIncrement: true) + #expect(PostgreSQLCreateTableSQL.columnDefinition(column, inlinePK: false, capabilities: Self.pg17) + == #""id" BIGSERIAL"#) + } + + @Test("A generated column is STORED before PostgreSQL 18 and keeps its kind from 18") + func generatedColumnKeyword() { + let column = PluginColumnDefinition( + name: "total", dataType: "integer", generationExpression: "a * 2", generationKind: nil, + ddlSpelling: nil, ddlDefault: nil, ddlGenerationExpression: nil, ddlCollation: nil, columnCheck: nil + ) + #expect(PostgreSQLCreateTableSQL.columnDefinition(column, inlinePK: false, capabilities: Self.pg17) + == #""total" integer GENERATED ALWAYS AS (a * 2) STORED"#) + #expect(PostgreSQLCreateTableSQL.columnDefinition(column, inlinePK: false, capabilities: Self.pg18) + == #""total" integer GENERATED ALWAYS AS (a * 2) VIRTUAL"#) + } + + @Test("A table with no columns writes nothing") + func noColumnsWritesNothing() { + let definition = PluginCreateTableDefinition(tableName: "empty", columns: []) + #expect(PostgreSQLCreateTableSQL.statements(definition, schema: "app", capabilities: Self.pg17) == nil) + } + + @Test("The plugin driver writes its table in the session's schema through the shared writer") + func driverForwardsToTheWriter() { + let driver = PostgreSQLPluginDriver( + config: DriverConnectionConfig(host: "", port: 0, username: "", password: "", database: "") + ) + let definition = PluginCreateTableDefinition( + tableName: "orders", + columns: [PluginColumnDefinition(name: "id", dataType: "integer", isPrimaryKey: true)] + ) + #expect(driver.generateCreateTableStatements(definition: definition, schema: nil) + == PostgreSQLCreateTableSQL.statements(definition, schema: "public", capabilities: driver.versionedCapabilities)) + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/RedshiftCatalogExportTests.swift b/apple/Plugins/PostgreSQLDriverTests/RedshiftCatalogExportTests.swift new file mode 100644 index 0000000000..301b0cb2a3 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/RedshiftCatalogExportTests.swift @@ -0,0 +1,101 @@ +import Foundation +import TableProPluginKit +import Testing + +struct RedshiftCatalogExportTests { + private static func catalog(_ recorder: CatalogReadRecorder, externalSchemas: Set = []) -> RedshiftCatalogExport { + RedshiftCatalogExport( + read: recorder.redshiftRead, + currentSchema: "public", + database: "dev", + serverVersionNumber: 80_002, + externalSchemas: externalSchemas + ) + } + + private struct Refused: Error {} + + @Test("SHOW TABLE answers the DDL when the cluster has it") + func showTableAnswers() async throws { + let recorder = CatalogReadRecorder { _ in [["CREATE TABLE public.orders (id integer)"].cells] } + let ddl = try await Self.catalog(recorder).fetchTableDDL(table: "orders", schema: nil) + + #expect(ddl == "CREATE TABLE public.orders (id integer)") + #expect(recorder.statements == [#"SHOW TABLE "public"."orders""#]) + } + + @Test("Without SHOW TABLE the DDL is rebuilt with its foreign keys inline") + func reconstructionDeclaresForeignKeys() async throws { + let recorder = CatalogReadRecorder { statement in + if statement.hasPrefix("SHOW TABLE") { throw Refused() } + if statement.contains("pg_catalog.pg_views") { return [] } + if statement.contains("pg_catalog.format_type") { return [[#""id" integer NOT NULL"#].cells] } + return [] + } + let ddl = try await Self.catalog(recorder).fetchTableDDL(table: "orders", schema: nil) + + #expect(ddl == """ + CREATE TABLE "public"."orders" ( + "id" integer NOT NULL + ); + """) + #expect(RedshiftCatalogExport.tableDDLIncludesForeignKeys) + } + + @Test("A rebuilt table with no columns read fails with the driver's message") + func reconstructionWithoutColumnsFails() async { + let recorder = CatalogReadRecorder { statement in + if statement.hasPrefix("SHOW TABLE") { throw Refused() } + return [] + } + do { + _ = try await Self.catalog(recorder).fetchTableDDL(table: "orders", schema: nil) + Issue.record("Expected a failure") + } catch { + #expect(error.localizedDescription == "Failed to fetch DDL for table 'orders'") + } + } + + @Test("Local columns map the information_schema row") + func localColumnsMap() async throws { + let recorder = CatalogReadRecorder { _ in + [["status", "USER-DEFINED", "NO", nil, "en_US.UTF-8", "", "mood", "YES"].cells] + } + let column = try #require(try await Self.catalog(recorder).fetchColumns(table: "orders", schema: nil).first) + + #expect(column.name == "status") + #expect(column.dataType == "ENUM(mood)") + #expect(!column.isNullable) + #expect(column.isPrimaryKey) + #expect(column.charset == "UTF-8") + #expect(column.comment == nil) + } + + @Test("An external schema's columns come from svv_external_columns of the connected database") + func externalColumnsReadTheExternalView() async throws { + let recorder = CatalogReadRecorder { _ in [["payload", "struct", "true", "1"].cells] } + let columns = try await Self.catalog(recorder, externalSchemas: ["spectrum"]) + .fetchColumns(table: "events", schema: "spectrum") + + #expect(recorder.statements == [ + RedshiftExternalSchemaQueries.listExternalColumns(schema: "spectrum", table: "events", database: "dev") + ]) + #expect(columns.first?.dataType == "struct") + } + + @Test("An external table has no row count to read") + func externalTableHasNoRowCount() async throws { + let recorder = CatalogReadRecorder() + let count = try await Self.catalog(recorder, externalSchemas: ["spectrum"]) + .fetchApproximateRowCount(table: "events", schema: "spectrum") + + #expect(count == nil) + #expect(recorder.reads.isEmpty) + } + + @Test("A negative tbl_rows is no count") + func negativeRowCountIsNone() async throws { + let recorder = CatalogReadRecorder { _ in [["-1"].cells] } + #expect(try await Self.catalog(recorder).fetchApproximateRowCount(table: "orders", schema: nil) == nil) + } +} diff --git a/apple/Plugins/SQLExportPlugin/SQLExportBinaryLiteral.swift b/apple/Plugins/SQLExportPlugin/SQLExportBinaryLiteral.swift index 8fcff51093..17f1d81dd6 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportBinaryLiteral.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportBinaryLiteral.swift @@ -41,7 +41,7 @@ import TableProPluginKit /// /// An engine this does not name keeps `X''`, which is what the export has always written, so no /// engine's output changes except the three measured to have been wrong. -internal enum SQLExportBinaryLiteral { +nonisolated internal enum SQLExportBinaryLiteral { /// The most binary bytes Oracle can carry in one `HEXTORAW` literal: 4,000 characters of hex, /// two per byte. Measured, not inferred. internal static let oracleLiteralByteCeiling = 2_000 diff --git a/apple/Plugins/SQLExportPlugin/SQLExportCompressor.swift b/apple/Plugins/SQLExportPlugin/SQLExportCompressor.swift index 677982bef7..e1aefcb41d 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportCompressor.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportCompressor.swift @@ -4,25 +4,20 @@ // import Foundation +import TableProGzip import TableProPluginKit -import TableProProcess // Compresses through the atomic pair, so a failed or stopped compression leaves the previous file whole (#2533). -internal enum SQLExportCompressor { - private static let gzipPath = "/usr/bin/gzip" - - // Stop calls `Progress.cancel()`, not a task cancellation, so it is polled. - private static let cancellationPollInterval = Duration.milliseconds(150) +nonisolated internal enum SQLExportCompressor { + // Often enough that whoever watches the status sees the run move, rarely enough to cost nothing. + private static let statusInterval = Duration.milliseconds(250) + @concurrent internal static func compress( source: URL, to destination: URL, progress: PluginExportProgress ) async throws { - guard FileManager.default.isExecutableFile(atPath: gzipPath) else { - throw PluginExportError.exportFailed( - String(format: String(localized: "Compression unavailable: gzip not found at %@"), gzipPath)) - } try progress.checkCancellation() let (handle, tempURL) = try PluginExportUtilities.beginAtomicWrite(for: destination) @@ -31,121 +26,36 @@ internal enum SQLExportCompressor { if !committed { PluginExportUtilities.rollbackAtomicWrite(at: tempURL) } } - try await runGzip(source: source, into: handle, progress: progress) - try PluginExportUtilities.commitAtomicWrite(from: tempURL, to: destination) - committed = true - } - - private static func runGzip( - source: URL, - into handle: FileHandle, - progress: PluginExportProgress - ) async throws { - let process: Process - do { - // Apple's gzip reads extra options from GZIP. - process = try ProcessLauncher.process(for: ProcessCommand( - executable: URL(fileURLWithPath: gzipPath), - arguments: ["-c", source.standardizedFileURL.path(percentEncoded: false)], - environment: ProcessEnvironment.minimal() - )) - } catch { - try? handle.close() - throw error - } - let errorPipe = Pipe() - process.standardOutput = handle - process.standardError = errorPipe - - let gate = ProcessGate() - let watcher = Task { - while !Task.isCancelled { - if progress.isCancelled { - gate.cancel() - return - } - try? await Task.sleep(for: cancellationPollInterval) - } - } - defer { watcher.cancel() } - + let clock = ContinuousClock() + var lastStatus: ContinuousClock.Instant? do { - try await withTaskCancellationHandler { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - process.terminationHandler = { finished in - // A volume can report a deferred write failure only at close, and a swallowed one - // would let a truncated temp replace a valid dump. - let closeError: (any Error)? - do { - try handle.close() - closeError = nil - } catch { - closeError = error - } - guard finished.terminationStatus == 0 else { - continuation.resume(throwing: failure( - status: finished.terminationStatus, from: errorPipe)) - return - } - if let closeError { - continuation.resume(throwing: closeError) - return - } - continuation.resume() - } - do { - try gate.launch(process) - } catch { - try? handle.close() - continuation.resume(throwing: error) - } - } - } onCancel: { - gate.cancel() + try GzipFile.compress(source: source, into: handle, isCancelled: { progress.isCancelled }) { done, total in + let now = clock.now + if done < total, let lastStatus, now - lastStatus < Self.statusInterval { return } + lastStatus = now + progress.setStatus(Self.status(done: done, total: total)) } + } catch is CancellationError { + try? handle.close() + throw PluginExportCancellationError() } catch { - // A Stop kills gzip, so its non-zero exit means the user stopped, not that compression failed. - guard !progress.isCancelled else { throw PluginExportCancellationError() } + try? handle.close() throw error } - - // A Stop that lands after gzip exits still abandons the file. + // A volume can report a deferred write failure only at close, and a swallowed one would let a truncated temp + // replace a valid dump. + try handle.close() + // A Stop that lands after the last chunk still abandons the file. try progress.checkCancellation() + try PluginExportUtilities.commitAtomicWrite(from: tempURL, to: destination) + committed = true } - private static func failure(status: Int32, from errorPipe: Pipe) -> any Error { - let text = String(data: errorPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8)? - .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" - guard !text.isEmpty else { - return PluginExportError.exportFailed(String( - format: String(localized: "Compression failed with exit status %lld"), Int64(status))) - } - return PluginExportError.exportFailed(String( - format: String(localized: "Compression failed with exit status %1$lld: %2$@"), - Int64(status), text)) - } - - // `terminate()` on a process never launched raises an uncaught exception, so the lock spans `run()` - // and is released before `terminate()`. - private final class ProcessGate: @unchecked Sendable { - private let lock = NSLock() - private var launched: Process? - private var cancelled = false - - func launch(_ process: Process) throws { - lock.lock() - defer { lock.unlock() } - guard !cancelled else { throw PluginExportCancellationError() } - try process.run() - launched = process - } - - func cancel() { - lock.lock() - cancelled = true - let running = launched - lock.unlock() - running?.terminate() - } + private static func status(done: Int64, total: Int64) -> String { + String( + format: String(localized: "Compressing\u{2026} %1$@ of %2$@"), + ByteCountFormatter.string(fromByteCount: done, countStyle: .file), + ByteCountFormatter.string(fromByteCount: total, countStyle: .file) + ) } } diff --git a/apple/Plugins/SQLExportPlugin/SQLExportDDLRewriter.swift b/apple/Plugins/SQLExportPlugin/SQLExportDDLRewriter.swift index e4124dfac1..31d0d1eb4f 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportDDLRewriter.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportDDLRewriter.swift @@ -24,7 +24,7 @@ import TableProPluginKit /// Only the clause forms carry `=`. The column-level `AUTO_INCREMENT` attribute and /// `SQL SECURITY DEFINER` do not, so both survive: dropping the latter, as `mysqlpump --skip-definer` /// does, would turn a view declared `SQL SECURITY INVOKER` into a definer-rights view. -internal struct SQLExportDDLRewriter { +nonisolated internal struct SQLExportDDLRewriter { internal let dialect: SqlDialect internal let excludesAutoIncrementValue: Bool internal let excludesDefiner: Bool diff --git a/apple/Plugins/SQLExportPlugin/SQLExportFileWriter.swift b/apple/Plugins/SQLExportPlugin/SQLExportFileWriter.swift index 4d1ba322f5..a0a241c319 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportFileWriter.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportFileWriter.swift @@ -19,7 +19,7 @@ import TableProPluginKit /// published part back would leave neither the file the user had nor the one just written. This /// used to claim it prevented half a dump, while `rollback()` removed temps only and left every /// already-renamed part in place unmentioned (#2533). -internal final class SQLExportFileWriter { +nonisolated internal final class SQLExportFileWriter { /// The name a part takes: `dump.sql` becomes `dump.part1.sql`, and a compound extension like /// `dump.sql.gz` becomes `dump.part1.sql.gz`. internal static func partURL(for destination: URL, part: Int) -> URL { @@ -211,7 +211,7 @@ internal final class SQLExportFileWriter { /// the earlier parts on disk. Naming them is the whole point of this error: the alert that reports /// the failure is the only place the user hears which files the export did write, and they cannot /// be taken back, because `replaceItemAt` keeps no backup of what stood at the path. -internal struct SQLExportPublishFailure: LocalizedError { +nonisolated internal struct SQLExportPublishFailure: LocalizedError { internal let published: [URL] internal let failed: URL internal let underlying: any Error diff --git a/apple/Plugins/SQLExportPlugin/SQLExportInsertMode.swift b/apple/Plugins/SQLExportPlugin/SQLExportInsertMode.swift index e39672b9e6..fa3ebbfc34 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportInsertMode.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportInsertMode.swift @@ -7,7 +7,7 @@ import Foundation import TableProPluginKit /// How a dump's `INSERT` behaves when a row it writes already exists. -public enum SQLExportInsertMode: String, Codable, CaseIterable, Sendable, Identifiable { +nonisolated public enum SQLExportInsertMode: String, Codable, CaseIterable, Sendable, Identifiable { case insert case ignoreExisting case replaceExisting @@ -33,7 +33,7 @@ public enum SQLExportInsertMode: String, Codable, CaseIterable, Sendable, Identi /// a resolution clause (`INSERT OR IGNORE`), and PostgreSQL puts it in a trailing `ON CONFLICT` /// that has to name the conflict target. An engine with no spelling at all falls back to a plain /// `INSERT` and says so, rather than writing a statement the restore would reject. -public struct SQLExportInsertRenderer { +nonisolated public struct SQLExportInsertRenderer { public struct Rendered: Equatable, Sendable { public let prefix: String public let suffix: String diff --git a/apple/Plugins/SQLExportPlugin/SQLExportModels.swift b/apple/Plugins/SQLExportPlugin/SQLExportModels.swift index 81cbaca7b3..b7a82f48be 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportModels.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportModels.swift @@ -5,7 +5,22 @@ import Foundation -public struct SQLExportOptions: Equatable, Codable { +nonisolated public struct SQLExportOptions: Equatable, Codable { + public static let batchSizeChoices = [1, 100, 500, 1_000] + + public static let splitSizeChoices = [0, 8, 32, 128, 512] + + // Past the first, each is a `max_allowed_packet` default a MySQL or MariaDB release shipped, so an untuned server takes it. + public static let statementSizeChoices = [0, 262_144, 1_048_576, 4_194_304, 16_777_216, 67_108_864] + + public static func statementSizeLabel(_ bytes: Int) -> String { + guard bytes > 0 else { return String(localized: "No limit", bundle: .main) } + let formatter = ByteCountFormatter() + formatter.countStyle = .binary + formatter.allowedUnits = [.useKB, .useMB] + return formatter.string(fromByteCount: Int64(bytes)) + } + public var compressWithGzip: Bool = false public var batchSize: Int = 500 public var excludeAutoIncrementValue: Bool = true @@ -64,7 +79,7 @@ public struct SQLExportOptions: Equatable, Codable { /// `CustomStringConvertible` as well as `LocalizedError`, because the warning the export writes /// into the dump interpolates the error value itself. Without it the comment read /// `failed to fetch DDL for table orders: emptyDefinition`, the case name. -internal enum SQLExportObjectError: LocalizedError, CustomStringConvertible { +nonisolated internal enum SQLExportObjectError: LocalizedError, CustomStringConvertible { case emptyDefinition internal var errorDescription: String? { diff --git a/apple/Plugins/SQLExportPlugin/SQLExportOptionsView.swift b/apple/Plugins/SQLExportPlugin/SQLExportOptionsView.swift index 9ce160c422..d1d7110a58 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportOptionsView.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportOptionsView.swift @@ -8,14 +8,6 @@ import SwiftUI struct SQLExportOptionsView: View { @ObservedObject var plugin: SQLExportPlugin - private static let batchSizeOptions = [1, 100, 500, 1_000] - - private static let splitSizeOptions = [0, 8, 32, 128, 512] - - /// Every value but the first is a `max_allowed_packet` default some MySQL or MariaDB release has - /// shipped with, so a dump restores into an untuned server of that generation. - private static let statementSizeOptions = [0, 262_144, 1_048_576, 4_194_304, 16_777_216, 67_108_864] - private static let statementSizeHelp = String( localized: """ Closes the INSERT at this size or the row count, whichever comes first. MySQL and MariaDB \ @@ -25,14 +17,6 @@ struct SQLExportOptionsView: View { bundle: .main ) - private static func statementSizeLabel(_ bytes: Int) -> String { - guard bytes > 0 else { return String(localized: "No limit", bundle: .main) } - let formatter = ByteCountFormatter() - formatter.countStyle = .binary - formatter.allowedUnits = [.useKB, .useMB] - return formatter.string(fromByteCount: Int64(bytes)) - } - private static let insertModeHelp = String( localized: """ What an INSERT does when the row already exists. MySQL, MariaDB, PostgreSQL and SQLite \ @@ -81,7 +65,7 @@ struct SQLExportOptionsView: View { Spacer() Picker(String(localized: "Rows per INSERT", bundle: .main), selection: $plugin.settings.batchSize) { - ForEach(Self.batchSizeOptions, id: \.self) { size in + ForEach(SQLExportOptions.batchSizeChoices, id: \.self) { size in Text(size == 1 ? String(localized: "1 (no batching)", bundle: .main) : "\(size)") .tag(size) } @@ -102,8 +86,8 @@ struct SQLExportOptionsView: View { String(localized: "Max INSERT size", bundle: .main), selection: $plugin.settings.maxStatementBytes ) { - ForEach(Self.statementSizeOptions, id: \.self) { size in - Text(Self.statementSizeLabel(size)).tag(size) + ForEach(SQLExportOptions.statementSizeChoices, id: \.self) { size in + Text(SQLExportOptions.statementSizeLabel(size)).tag(size) } } .pickerStyle(.menu) @@ -136,7 +120,7 @@ struct SQLExportOptionsView: View { Spacer() Picker(String(localized: "Split every", bundle: .main), selection: $plugin.settings.splitSizeMegabytes) { - ForEach(Self.splitSizeOptions, id: \.self) { size in + ForEach(SQLExportOptions.splitSizeChoices, id: \.self) { size in Text(size == 0 ? String(localized: "One file", bundle: .main) : "\(size) MB") .tag(size) } @@ -168,3 +152,10 @@ struct SQLExportOptionsView: View { } } } + +extension SQLExportPlugin { + @MainActor + func settingsView() -> AnyView? { + AnyView(SQLExportOptionsView(plugin: self)) + } +} diff --git a/apple/Plugins/SQLExportPlugin/SQLExportPlugin.swift b/apple/Plugins/SQLExportPlugin/SQLExportPlugin.swift index 8382c61ded..605c845b8e 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportPlugin.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportPlugin.swift @@ -6,10 +6,9 @@ import Combine import Foundation import os -import SwiftUI import TableProPluginKit -final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugin, @unchecked Sendable { +nonisolated final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugin, @unchecked Sendable { static let pluginName = "SQL Export" static let pluginVersion = "1.0.0" static let pluginDescription = "Export data to SQL format" @@ -45,50 +44,12 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi typealias Settings = SQLExportOptions static let settingsStorageId = "sql" - @Published var settings = SQLExportOptions() { + // A nonisolated class cannot hold an @Published property, so the change is announced by hand. + var settings = SQLExportOptions() { + willSet { objectWillChange.send() } didSet { saveSettings() } } - var ddlFailures: [String] = [] - - /// Kept apart from `ddlFailures` because the summary that reads it says "table structure", and - /// a table whose `CREATE TABLE` came back fine and only lost its indexes is a different thing - /// to tell the user about. - var indexFailures: [String] = [] - - /// Kept apart for the same reason `indexFailures` is: an object whose definition came back fine - /// and only lost its comments is a different thing to tell the user about. - var commentFailures: [String] = [] - - /// The tables whose dependent sequences or enum types could not be read. Kept apart from - /// `ddlFailures` for the same reason `indexFailures` is: the table's own `CREATE TABLE` came - /// back fine, and what is missing is an object that statement names. Measured on PostgreSQL - /// 17.11 with `SELECT` revoked on `pg_catalog.pg_enum`: the enum query fails, `fetchTableDDL` - /// still returns a `CREATE TABLE` declaring `status order_status`, and the restore stops at - /// `type "order_status" does not exist` with nothing in the dump explaining it. One entry per - /// table, however many of its two fetches failed. - private var dependentObjectFailures: [String] = [] - - var metadataWarnings: [String] = [] - - /// The tables a foreign key cycle left the ordering unable to place. They keep the order the - /// export tree gave them, which is the only order left once no parent-first one exists, and - /// the dump says so rather than reading as if it were restorable with the checks on. - var tablesUnorderedByCycle: [String] = [] - - /// Sequence names already written this export. A sequence can be reached twice, once as an - /// object the user ticked and once as a dependency of a table that defaults from it, and - /// `CREATE SEQUENCE` a second time fails the restore. - private var emittedSequenceNames: Set = [] - - /// A dump refers to its tables unqualified whenever every selected table lives in one - /// container, which is what makes it restorable into any database. Qualifying became necessary - /// only once an export could span two containers holding the same table name: unqualified there - /// means one schema's rows land in the other's table. The CREATE statements come back from the - /// driver verbatim and cannot be qualified without rewriting engine DDL, so a spanning export - /// says so rather than shipping a dump whose three phases disagree. - var exportSpansContainers = false - private static let logger = Logger(subsystem: "com.TablePro", category: "SQLExportPlugin") required init() { loadSettings() } @@ -109,16 +70,14 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi settings.compressWithGzip ? "sql.gz" : "sql" } - private func ddlRewriter(for dataSource: any PluginExportDataSource) -> SQLExportDDLRewriter { + private func ddlRewriter( + for dataSource: any PluginExportDataSource, + options: SQLExportOptions + ) -> SQLExportDDLRewriter { SQLExportDDLRewriter( dialect: SqlDialect.from(databaseTypeId: dataSource.databaseTypeId), - excludesAutoIncrementValue: settings.excludeAutoIncrementValue, - excludesDefiner: settings.excludeDefiner) - } - - @MainActor - func settingsView() -> AnyView? { - AnyView(SQLExportOptionsView(plugin: self)) + excludesAutoIncrementValue: options.excludeAutoIncrementValue, + excludesDefiner: options.excludeDefiner) } func resetSettingsToDefaults() { @@ -131,19 +90,9 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi destination: URL, progress: PluginExportProgress ) async throws -> ExportFormatResult { - ddlFailures = [] - indexFailures = [] - commentFailures = [] - dependentObjectFailures = [] - metadataWarnings = [] - exportSpansContainers = false - tablesUnorderedByCycle = [] - emittedSequenceNames = [] - - /// Read once, because `PluginManager` hands every window the same plugin instance and a - /// second window's options pane can write `settings` while this export is still running. A - /// cap re-read per table would let one export start at a mebibyte and finish unbounded. + // Read once: every window shares this plugin, and a cap re-read per table could start at 1 MiB and end unbounded. let options = settings + let run = ExportRun(options: options) var statementTally = SQLExportStatementTally() let actualDestination: URL @@ -159,11 +108,10 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi actualDestination = destination } - /// Compression runs over one file, so a compressed export never splits. Saying so beats - /// silently gzipping the first part and dropping the rest. + // Compression runs over one file, so a compressed export never splits, and says so. let splitSize = options.compressWithGzip ? 0 : options.splitSizeMegabytes if options.compressWithGzip, options.splitSizeMegabytes > 0 { - metadataWarnings.append(String(localized: + run.metadataWarnings.append(String(localized: "A compressed export is written as one file, so the split size was not applied.")) } var readScope = await SQLExportReadScope.open(on: dataSource, for: tables) @@ -199,57 +147,54 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let definitionObjects = tables.filter { !$0.kind.carriesRows } try writeHeader(to: writer, dataSource: dataSource) - let columnsByTable = await prefetchColumns(tables: rowObjects, dataSource: dataSource) - let fkMap = await prefetchForeignKeys(tables: rowObjects, dataSource: dataSource) - let sortedTables = topologicallySort(rowObjects, fkMap: fkMap) - noteContainerSpan(of: tables) - try writeDependencyCycleNote(to: writer) + let columnsByTable = await prefetchColumns(tables: rowObjects, dataSource: dataSource, run: run) + let fkMap = await prefetchForeignKeys(tables: rowObjects, dataSource: dataSource, run: run) + let sortedTables = topologicallySort(rowObjects, fkMap: fkMap, run: run) + noteContainerSpan(of: tables, run: run) + try writeDependencyCycleNote(to: writer, run: run) try writeDropPhase( sortedTables: sortedTables, definitionObjects: definitionObjects, - dataSource: dataSource, to: writer) + dataSource: dataSource, to: writer, run: run) try await writeObjectCreatePhase( objects: definitionObjects, kinds: [.userType, .sequence], - dataSource: dataSource, to: writer, progress: progress) + dataSource: dataSource, to: writer, progress: progress, run: run) try await writeDependentTypesAndSequences( - tables: rowObjects, dataSource: dataSource, to: writer) + tables: rowObjects, dataSource: dataSource, to: writer, run: run) try await writeCreatePhase( - sortedTables: sortedTables, dataSource: dataSource, to: writer, progress: progress) + sortedTables: sortedTables, dataSource: dataSource, to: writer, progress: progress, run: run) statementTally = try await writeDataPhase( - sortedTables: sortedTables, columnsByTable: columnsByTable, options: options, - dataSource: dataSource, to: writer, progress: progress) + sortedTables: sortedTables, columnsByTable: columnsByTable, + dataSource: dataSource, to: writer, progress: progress, run: run) + let finishing = FinishingSteps( + progress: progress, + total: finishingStepCount(sortedTables: sortedTables, definitionObjects: definitionObjects)) try await writeFinalizationPhase( sortedTables: sortedTables, fkMap: fkMap, columnsByTable: columnsByTable, - dataSource: dataSource, to: writer, progress: progress) + dataSource: dataSource, to: writer, progress: progress, run: run, finishing: finishing) try await writeObjectCreatePhase( objects: definitionObjects, - kinds: [.view, .materializedView, .routine, .trigger, .event], - dataSource: dataSource, to: writer, progress: progress) - /// A materialized view carries its own indexes, and PostgreSQL needs a unique one - /// before `REFRESH MATERIALIZED VIEW CONCURRENTLY` will run. It holds no rows the - /// export streams, so it is not in `sortedTables` and gets its own pass here, once the - /// phase above has created it. + kinds: Self.definitionKindsAfterRows, + dataSource: dataSource, to: writer, progress: progress, run: run, finishing: finishing) + // PostgreSQL needs a materialized view's unique index for `REFRESH ... CONCURRENTLY`; it is not in `sortedTables`. if try await writeIndexPhase( objects: definitionObjects.filter { $0.kind == .materializedView }, - dataSource: dataSource, to: writer, progress: progress) { + dataSource: dataSource, to: writer, progress: progress, run: run, finishing: finishing) { try writer.write("\n") } try await writeGrantPhase( - objects: definitionObjects, dataSource: dataSource, to: writer) + objects: definitionObjects, dataSource: dataSource, to: writer, run: run, finishing: finishing) if let snapshot { await snapshot.end(on: dataSource) } await readScope.close(on: dataSource) - /// The publication point. Every phase above can suspend on a driver call, and the - /// grant phase makes no cancellation check of its own, so a Stop arriving in one of - /// them used to resume and replace the user's file anyway. Nothing is published past - /// this line without the stop having been seen. + // The publication point: a Stop during any phase above, the grant phase included, must not replace the file. try progress.checkCancellation() let writtenParts = try writer.commit() committed = true if writer.didSplit, let first = writtenParts.first, let last = writtenParts.last { - metadataWarnings.append(String( + run.metadataWarnings.append(String( format: String(localized: """ The dump was written as %1$lld numbered parts, %2$@ through %3$@. Restore \ them in order. @@ -276,28 +221,28 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi progress.finalizeTable() var warnings: [String] = [] - if !ddlFailures.isEmpty { - let failedTables = ddlFailures.joined(separator: ", ") + if !run.ddlFailures.isEmpty { + let failedTables = run.ddlFailures.joined(separator: ", ") warnings.append(String( format: String(localized: "Could not fetch table structure for: %@"), failedTables)) } - if !dependentObjectFailures.isEmpty { + if !run.dependentObjectFailures.isEmpty { warnings.append(String( format: String(localized: """ Could not fetch the types and sequences needed by: %@. A CREATE TABLE that \ names one of them will not restore. """), - dependentObjectFailures.joined(separator: ", "))) + run.dependentObjectFailures.joined(separator: ", "))) } - if !indexFailures.isEmpty { + if !run.indexFailures.isEmpty { warnings.append(String( format: String(localized: "Could not fetch indexes for: %@"), - indexFailures.joined(separator: ", "))) + run.indexFailures.joined(separator: ", "))) } - if !commentFailures.isEmpty { + if !run.commentFailures.isEmpty { warnings.append(String( format: String(localized: "Could not fetch comments for: %@"), - commentFailures.joined(separator: ", "))) + run.commentFailures.joined(separator: ", "))) } if let oversized = Self.oversizedRowWarning(tally: statementTally) { warnings.append(oversized) @@ -305,10 +250,70 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi if let unrepresentable = Self.unrepresentableValueWarning(tally: statementTally) { warnings.append(unrepresentable) } - warnings.append(contentsOf: metadataWarnings) + warnings.append(contentsOf: run.metadataWarnings) return ExportFormatResult(warnings: warnings, notes: Self.statementSizeNotes(tally: statementTally)) } + private static let definitionKindsAfterRows: [PluginExportObjectKind] = [ + .view, .materializedView, .routine, .trigger, .event + ] + + // What one export records. Every window and scene shares the plugin, so none of it lives on the instance. + private final class ExportRun { + let options: SQLExportOptions + var ddlFailures: [String] = [] + // Apart from ddlFailures, whose summary says "table structure": these tables lost only their indexes. + var indexFailures: [String] = [] + // Apart for the same reason: the definition came back and only its comments were lost. + var commentFailures: [String] = [] + // One entry per table. PostgreSQL 17.11 with SELECT revoked on pg_enum still returns the CREATE TABLE, + // and the restore then stops at `type "order_status" does not exist`. + var dependentObjectFailures: [String] = [] + var metadataWarnings: [String] = [] + // Kept in the export tree's order, the only one left once no parent-first order exists. + var tablesUnorderedByCycle: [String] = [] + // A sequence ticked and also a table's default is reached twice, and a second CREATE SEQUENCE fails the restore. + var emittedSequenceNames: Set = [] + // Unqualified names across two containers send one schema's rows into the other's table. + var exportSpansContainers = false + + init(options: SQLExportOptions) { + self.options = options + } + + func noteDependentObjectFailure(_ sanitizedTableName: String) { + guard !dependentObjectFailures.contains(sanitizedTableName) else { return } + dependentObjectFailures.append(sanitizedTableName) + } + } + + // The phases after the rows read none, so they move the status instead: iOS expires a background task that stalls. + private final class FinishingSteps { + private let progress: PluginExportProgress + private let total: Int + private var done = 0 + + init(progress: PluginExportProgress, total: Int) { + self.progress = progress + self.total = total + } + + func advance() { + done += 1 + progress.setStatus(String( + format: String(localized: "Writing definitions: %1$lld of %2$lld"), Int64(done), Int64(total))) + } + } + + // One step per object the index, definition and grant phases below fetch, with the filters those loops use. + private func finishingStepCount(sortedTables: [PluginExportTable], definitionObjects: [PluginExportTable]) -> Int { + let afterRows = Set(Self.definitionKindsAfterRows) + let wanted = definitionObjects.filter { optionValue($0, at: 0) } + return sortedTables.filter { optionValue($0, at: 0) }.count + + wanted.filter { afterRows.contains($0.kind) || $0.kind == .grant }.count + + wanted.filter { $0.kind == .materializedView }.count + } + /// The size a byte limit should be judged against, in the same binary units the limit's own menu /// names, so a statement that hit a 1 MB limit reads as 1 MB rather than 1.05 MB. private static func formatted(bytes: Int) -> String { @@ -391,7 +396,8 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi private func prefetchForeignKeys( tables: [PluginExportTable], - dataSource: any PluginExportDataSource + dataSource: any PluginExportDataSource, + run: ExportRun ) async -> [String: [PluginForeignKeyInfo]] { var merged: [String: [PluginForeignKeyInfo]] = [:] var anyGroupFailed = false @@ -407,7 +413,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } } if anyGroupFailed { - metadataWarnings.append(String(localized: + run.metadataWarnings.append(String(localized: "Could not fetch foreign keys, so foreign key constraints may be missing from the export.")) } return merged @@ -415,7 +421,8 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi private func prefetchColumns( tables: [PluginExportTable], - dataSource: any PluginExportDataSource + dataSource: any PluginExportDataSource, + run: ExportRun ) async -> [String: [PluginColumnInfo]] { var merged: [String: [PluginColumnInfo]] = [:] var anyGroupFailed = false @@ -431,7 +438,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } } if anyGroupFailed { - metadataWarnings.append(String(localized: + run.metadataWarnings.append(String(localized: "Could not fetch column metadata, so identity and generated columns may not round-trip correctly.")) } return merged @@ -439,24 +446,22 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi private func topologicallySort( _ tables: [PluginExportTable], - fkMap: [String: [PluginForeignKeyInfo]] + fkMap: [String: [PluginForeignKeyInfo]], + run: ExportRun ) -> [PluginExportTable] { let byIdentifier = Dictionary( tables.map { (node(for: $0).identifier, $0) }, uniquingKeysWith: { first, _ in first }) let ordering = ForeignKeyTopologicalSort.order(tables.map { node(for: $0) }, foreignKeysByTable: fkMap) - tablesUnorderedByCycle = ordering.unorderedByCycle.map { $0.identifier } + run.tablesUnorderedByCycle = ordering.unorderedByCycle.map { $0.identifier } return ordering.tables.compactMap { byIdentifier[$0.identifier] } } - /// The warning states what the file is rather than prescribing a remedy, because the remedy is - /// not the same everywhere: `foreignKeyDisableStatements` is nil on SQL Server, Oracle, - /// Snowflake and DuckDB, so telling every user to import with the checks off would be wrong on - /// the engines that cannot turn them off. - private func writeDependencyCycleNote(to writer: SQLExportFileWriter) throws { - guard !tablesUnorderedByCycle.isEmpty else { return } - let names = tablesUnorderedByCycle.joined(separator: ", ") - metadataWarnings.append(String( + // States what the file is, not a remedy: foreignKeyDisableStatements is nil on SQL Server, Oracle, Snowflake and DuckDB. + private func writeDependencyCycleNote(to writer: SQLExportFileWriter, run: ExportRun) throws { + guard !run.tablesUnorderedByCycle.isEmpty else { return } + let names = run.tablesUnorderedByCycle.joined(separator: ", ") + run.metadataWarnings.append(String( format: String(localized: """ Foreign keys between %@ reference each other, so no order puts every parent before \ its children. Those tables are written in the order they were listed, and the dump \ @@ -468,14 +473,13 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi try writer.write(note) } - /// Drops run in the reverse of the order the objects are created in, so a dependent goes before - /// what it depends on: triggers and routines first, then views, then the tables in reverse - /// topological order, then the sequences and types those tables referenced. + // Reverse creation order, so a dependent drops before what it depends on. private func writeDropPhase( sortedTables: [PluginExportTable], definitionObjects: [PluginExportTable], dataSource: any PluginExportDataSource, - to writer: SQLExportFileWriter + to writer: SQLExportFileWriter, + run: ExportRun ) throws { let afterTables = definitionObjects .filter { $0.kind.dumpOrder > PluginExportObjectKind.table.dumpOrder } @@ -487,7 +491,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi .filter { optionValue($0, at: 1) && $0.kind != .grant } guard !dropTargets.isEmpty else { return } for object in dropTargets { - guard let statement = dropStatement(for: object, dataSource: dataSource) else { continue } + guard let statement = dropStatement(for: object, dataSource: dataSource, run: run) else { continue } try writer.write(statement + dataSource.dumpStatementEnd) } try writer.write("\n") @@ -499,39 +503,36 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi dataSource.supportsCascadeDrop ? " CASCADE" : "" } - /// The engine spells its own DROP for the kinds where dialects disagree: PostgreSQL's - /// `DROP TRIGGER` takes an `ON ` clause where MySQL's does not, and MySQL has no - /// `DROP ROUTINE` at all. Only the table-shaped kinds, which every SQL engine spells the same - /// way, fall through to the generic form here. + // The engine spells its own DROP where dialects disagree, such as PostgreSQL's `DROP TRIGGER ... ON
`. private func dropStatement( for object: PluginExportTable, - dataSource: any PluginExportDataSource + dataSource: any PluginExportDataSource, + run: ExportRun ) -> String? { if let driverStatement = dataSource.dropStatement(for: object) { return dataSource.scriptText(for: driverStatement) } + // Only PostgreSQL has `DROP ROUTINE`, and MySQL, SQL Server, Oracle and DuckDB stop the restore at one. + guard object.kind != .routine else { return nil } let keyword = object.kind.dropKeyword guard !keyword.isEmpty else { return nil } let ref = qualifiedRef( - schema: object.databaseName, table: object.name, dataSource: dataSource) + schema: object.databaseName, table: object.name, dataSource: dataSource, run: run) switch object.kind { - case .trigger, .event, .routine: + case .trigger, .event: return "\(keyword) IF EXISTS \(dataSource.quoteIdentifier(object.name));" default: return "\(keyword) IF EXISTS \(ref)\(cascadeClause(dataSource));" } } - /// The sequences and enum types the selected tables depend on, created before the tables that - /// reference them. - /// - /// Their `DROP` follows the table's own Drop column. It used to be written unconditionally, so - /// a dump meant to be appended to a live database opened with `DROP TYPE ... CASCADE`, which - /// takes the enum column out of every other table using that type. + // The DROP follows the table's Drop column: an unconditional DROP TYPE ... CASCADE took the enum column out of every + // other table using that type when the dump was appended to a live database. private func writeDependentTypesAndSequences( tables: [PluginExportTable], dataSource: any PluginExportDataSource, - to writer: SQLExportFileWriter + to writer: SQLExportFileWriter, + run: ExportRun ) async throws { var emittedTypeNames: Set = [] let structureTables = tables.filter { optionValue($0, at: 0) } @@ -541,8 +542,8 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi do { let sequences = try await dataSource.fetchDependentSequences( table: table.name, databaseName: table.databaseName) - for seq in sequences where !emittedSequenceNames.contains(seq.name) { - emittedSequenceNames.insert(seq.name) + for seq in sequences where !run.emittedSequenceNames.contains(seq.name) { + run.emittedSequenceNames.insert(seq.name) let quotedName = dataSource.quoteIdentifier(seq.name) if optionValue(table, at: 1) { try writer.write( @@ -552,7 +553,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } } catch { let sanitizedName = PluginExportUtilities.sanitizeForSQLComment(table.name) - noteDependentObjectFailure(sanitizedName) + run.noteDependentObjectFailure(sanitizedName) Self.logger.warning("Failed to fetch dependent sequences for table \(table.name): \(error)") let warning = "Warning: failed to fetch dependent sequences for \(sanitizedName): \(error)" try writer.write("-- \(PluginExportUtilities.sanitizeForSQLComment(warning))\n\n") @@ -574,7 +575,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } } catch { let sanitizedName = PluginExportUtilities.sanitizeForSQLComment(table.name) - noteDependentObjectFailure(sanitizedName) + run.noteDependentObjectFailure(sanitizedName) Self.logger.warning("Failed to fetch dependent types for table \(table.name): \(error)") let warning = "Warning: failed to fetch dependent types for \(sanitizedName): \(error)" try writer.write("-- \(PluginExportUtilities.sanitizeForSQLComment(warning))\n\n") @@ -582,18 +583,14 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } } - private func noteDependentObjectFailure(_ sanitizedTableName: String) { - guard !dependentObjectFailures.contains(sanitizedTableName) else { return } - dependentObjectFailures.append(sanitizedTableName) - } - private func writeCreatePhase( sortedTables: [PluginExportTable], dataSource: any PluginExportDataSource, to writer: SQLExportFileWriter, - progress: PluginExportProgress + progress: PluginExportProgress, + run: ExportRun ) async throws { - let rewriter = ddlRewriter(for: dataSource) + let rewriter = ddlRewriter(for: dataSource, options: run.options) for (index, table) in sortedTables.enumerated() where optionValue(table, at: 0) { try progress.checkCancellation() progress.setCurrentTable(table.qualifiedName, index: index + 1) @@ -611,13 +608,13 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi try writer.write(dataSource.scriptText(for: ddl) + dataSource.dumpStatementEnd) try writer.write("\n") } catch { - ddlFailures.append(sanitizedName) + run.ddlFailures.append(sanitizedName) let ddlWarning = "Warning: failed to fetch DDL for table \(sanitizedName): \(error)" Self.logger.warning("Failed to fetch DDL for table \(sanitizedName): \(error)") try writer.write("-- \(PluginExportUtilities.sanitizeForSQLComment(ddlWarning))\n\n") continue } - try await writeComments(for: table, dataSource: dataSource, to: writer) + try await writeComments(for: table, dataSource: dataSource, to: writer, run: run) } } @@ -633,16 +630,13 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi .table, .view, .materializedView ] - /// Writes an object's comments directly after its own `CREATE`, which is where `pg_dump` puts - /// them, so a comment travels with the object it belongs to rather than with a later phase. - /// - /// Only ever called on the success branch: a `COMMENT` on an object whose `CREATE` was not - /// written fails the restore. An unreadable comment list is recorded and commented into the file - /// rather than failing the export, exactly as the index phase does. + // Right after the object's own CREATE, as pg_dump writes them. Never for an object whose CREATE was not written: + // that COMMENT fails the restore. private func writeComments( for object: PluginExportTable, dataSource: any PluginExportDataSource, - to writer: SQLExportFileWriter + to writer: SQLExportFileWriter, + run: ExportRun ) async throws { guard Self.commentedKinds.contains(object.kind) else { return } let sanitizedName = PluginExportUtilities.sanitizeForSQLComment(object.name) @@ -655,23 +649,22 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } try writer.write("\n") } catch { - commentFailures.append(sanitizedName) + run.commentFailures.append(sanitizedName) Self.logger.warning("Failed to fetch comments for \(sanitizedName): \(error)") let warning = "Warning: failed to fetch comments for \(sanitizedName): \(error)" try writer.write("-- \(PluginExportUtilities.sanitizeForSQLComment(warning))\n\n") } } - /// Writes the definition of every object of the named kinds, in dump order within the group so - /// a view that another view selects from is created first. A kind the driver cannot produce a - /// definition for is recorded as a failure and commented into the file rather than aborting the - /// export: one unreadable routine must not cost the user the whole dump. + // Dump order within the group creates a view before one that selects from it; an unreadable object is noted, not fatal. private func writeObjectCreatePhase( objects: [PluginExportTable], kinds: [PluginExportObjectKind], dataSource: any PluginExportDataSource, to writer: SQLExportFileWriter, - progress: PluginExportProgress + progress: PluginExportProgress, + run: ExportRun, + finishing: FinishingSteps? = nil ) async throws { let wanted = Set(kinds) let targets = objects @@ -681,42 +674,44 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi for object in targets { try progress.checkCancellation() + finishing?.advance() let sanitizedName = PluginExportUtilities.sanitizeForSQLComment(object.name) let label = objectCommentLabel(for: object.kind) try writer.write("-- --------------------------------------------------------\n") try writer.write("-- \(label): \(sanitizedName)\n") try writer.write("-- --------------------------------------------------------\n\n") if object.kind == .sequence { - guard emittedSequenceNames.insert(object.name).inserted else { continue } + guard run.emittedSequenceNames.insert(object.name).inserted else { continue } } do { - let ddl = ddlRewriter(for: dataSource).rewrite(try await dataSource.fetchObjectDDL(object)) + let ddl = ddlRewriter(for: dataSource, options: run.options) + .rewrite(try await dataSource.fetchObjectDDL(object)) guard !ddl.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw SQLExportObjectError.emptyDefinition } - let scope = try await creationScope(for: object, dataSource: dataSource, to: writer) + let scope = try await creationScope(for: object, dataSource: dataSource, to: writer, run: run) try writer.write(creationScript(ddl: ddl, scope: scope, dataSource: dataSource)) try writer.write("\n") } catch { - ddlFailures.append(sanitizedName) + run.ddlFailures.append(sanitizedName) Self.logger.warning("Failed to fetch DDL for \(sanitizedName): \(error)") let warning = "Warning: failed to fetch definition for \(label.lowercased()) \(sanitizedName): \(error)" try writer.write("-- \(PluginExportUtilities.sanitizeForSQLComment(warning))\n\n") continue } - try await writeComments(for: object, dataSource: dataSource, to: writer) + try await writeComments(for: object, dataSource: dataSource, to: writer, run: run) } } /// The kinds a server stores the creating session's settings with, such as MySQL's `sql_mode`. private static let creationScopedKinds: Set = [.routine, .trigger, .event] - /// An unreadable scope still lets the definition be written, so it is recorded and commented into the file - /// rather than costing the object. + // An unreadable scope is noted in the file and still lets the definition be written. private func creationScope( for object: PluginExportTable, dataSource: any PluginExportDataSource, - to writer: SQLExportFileWriter + to writer: SQLExportFileWriter, + run: ExportRun ) async throws -> PluginStatementScope? { guard Self.creationScopedKinds.contains(object.kind) else { return nil } do { @@ -724,7 +719,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } catch { let sanitizedName = PluginExportUtilities.sanitizeForSQLComment(object.name) Self.logger.warning("Failed to fetch the creation settings for \(sanitizedName): \(error)") - metadataWarnings.append(String( + run.metadataWarnings.append(String( format: String(localized: "Could not read the settings %@ was created with, so it restores under the dump's SQL mode."), object.name)) let warning = "Warning: failed to fetch the creation settings for \(sanitizedName): \(error)" @@ -745,11 +740,13 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi .joined() } - /// Grants come last, because every object they name has to exist first. + // Grants come last, because every object they name has to exist first. private func writeGrantPhase( objects: [PluginExportTable], dataSource: any PluginExportDataSource, - to writer: SQLExportFileWriter + to writer: SQLExportFileWriter, + run: ExportRun, + finishing: FinishingSteps ) async throws { let principals = objects .filter { $0.kind == .grant && optionValue($0, at: 0) } @@ -761,6 +758,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi try writer.write("-- --------------------------------------------------------\n\n") for principal in principals { + finishing.advance() do { let statements = try await dataSource.fetchGrantStatements( principal: principal.name, host: principal.identity) @@ -770,7 +768,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } } catch { let sanitized = PluginExportUtilities.sanitizeForSQLComment(principal.name) - ddlFailures.append(sanitized) + run.ddlFailures.append(sanitized) Self.logger.warning("Failed to fetch grants for \(sanitized): \(error)") let warning = "Warning: failed to fetch privileges for \(sanitized): \(error)" try writer.write("-- \(PluginExportUtilities.sanitizeForSQLComment(warning))\n") @@ -797,10 +795,10 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi private func writeDataPhase( sortedTables: [PluginExportTable], columnsByTable: [String: [PluginColumnInfo]], - options: SQLExportOptions, dataSource: any PluginExportDataSource, to writer: SQLExportFileWriter, - progress: PluginExportProgress + progress: PluginExportProgress, + run: ExportRun ) async throws -> SQLExportStatementTally { var tally = SQLExportStatementTally() for table in sortedTables where optionValue(table, at: 2) && table.kind.carriesRows { @@ -808,10 +806,10 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi tally.merge(try await writeTableData( table: table, columnInfo: columnsByTable[node(for: table).identifier] ?? [], - options: options, dataSource: dataSource, to: writer, - progress: progress)) + progress: progress, + run: run)) } return tally } @@ -822,35 +820,32 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi columnsByTable: [String: [PluginColumnInfo]], dataSource: any PluginExportDataSource, to writer: SQLExportFileWriter, - progress: PluginExportProgress + progress: PluginExportProgress, + run: ExportRun, + finishing: FinishingSteps ) async throws { var emittedAnything = try await writeIndexPhase( - objects: sortedTables, dataSource: dataSource, to: writer, progress: progress) - /// A driver that hands back the server's own CREATE statement has already declared these - /// constraints inline, so adding them again names each one twice: MySQL and SQL Server - /// reject the duplicate, and SQLite has no ADD CONSTRAINT to reject it with. The phase - /// exists for the drivers whose DDL leaves foreign keys out, PostgreSQL and Oracle. + objects: sortedTables, dataSource: dataSource, to: writer, progress: progress, run: run, finishing: finishing) + // Only for DDL that leaves foreign keys out (PostgreSQL, Oracle): MySQL and SQL Server reject a duplicate one. if !dataSource.tableDDLIncludesForeignKeys { for table in sortedTables where optionValue(table, at: 0) { let fks = fkMap[node(for: table).identifier] ?? [] let grouped = groupForeignKeysByConstraint(fks) for group in grouped { - let alter = renderAddConstraintFK(table: table, group: group, dataSource: dataSource) + let alter = renderAddConstraintFK(table: table, group: group, dataSource: dataSource, run: run) try writer.write(alter + dataSource.dumpStatementEnd) emittedAnything = true } } } - /// `setval` and `pg_get_serial_sequence` are PostgreSQL's own, so the sequence is only - /// rewound on PostgreSQL. Every other engine reports its identity columns the same way and - /// would take the statement as a syntax error. + // `setval` and `pg_get_serial_sequence` are PostgreSQL's own; any other engine reads them as a syntax error. if SqlDialect.from(databaseTypeId: dataSource.databaseTypeId) == .postgres { for table in sortedTables where optionValue(table, at: 2) && table.kind.carriesRows { let columns = columnsByTable[node(for: table).identifier] ?? [] for column in columns where column.isIdentity { let setval = renderIdentitySetval( - table: table, columnName: column.name, dataSource: dataSource) + table: table, columnName: column.name, dataSource: dataSource, run: run) try writer.write(setval + dataSource.dumpStatementEnd) emittedAnything = true } @@ -862,26 +857,20 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } } - /// Writes each object's `CREATE INDEX` statements, after its rows and before the deferred - /// foreign keys. - /// - /// That is where every engine's own dump tool puts them, and the reason is that a bulk load - /// into an indexed table pays to maintain an index it is about to have rebuilt anyway. - /// A driver whose `CREATE TABLE` already declares its indexes answers nothing here, so a dump - /// never creates one twice. - /// - /// A driver that cannot read them is recorded and commented into the file rather than failing - /// the export: one unreadable table must not cost the user a dump that is otherwise correct. + // After the rows, as every engine's own dump tool does, so the bulk load does not maintain the index. @discardableResult private func writeIndexPhase( objects: [PluginExportTable], dataSource: any PluginExportDataSource, to writer: SQLExportFileWriter, - progress: PluginExportProgress + progress: PluginExportProgress, + run: ExportRun, + finishing: FinishingSteps ) async throws -> Bool { var emittedAnything = false for object in objects where optionValue(object, at: 0) { try progress.checkCancellation() + finishing.advance() let sanitizedName = PluginExportUtilities.sanitizeForSQLComment(object.name) do { let statements = try await dataSource.fetchIndexDDL( @@ -891,7 +880,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi emittedAnything = true } } catch { - indexFailures.append(sanitizedName) + run.indexFailures.append(sanitizedName) Self.logger.warning("Failed to fetch indexes for \(sanitizedName): \(error)") let warning = "Warning: failed to fetch indexes for \(sanitizedName): \(error)" try writer.write("-- \(PluginExportUtilities.sanitizeForSQLComment(warning))\n") @@ -904,10 +893,11 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi private func renderIdentitySetval( table: PluginExportTable, columnName: String, - dataSource: any PluginExportDataSource + dataSource: any PluginExportDataSource, + run: ExportRun ) -> String { let tableRef = qualifiedRef( - schema: table.databaseName, table: table.name, dataSource: dataSource) + schema: table.databaseName, table: table.name, dataSource: dataSource, run: run) let columnRef = dataSource.quoteIdentifier(columnName) let tableLiteral = dataSource.escapeStringLiteral(tableRef) let columnLiteral = dataSource.escapeStringLiteral(columnName) @@ -931,11 +921,12 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi return orderedNames.compactMap { groups[$0] } } - private func noteContainerSpan(of tables: [PluginExportTable]) { + // CREATE TABLE comes back from the driver verbatim, so a spanning export says it cannot qualify that statement. + private func noteContainerSpan(of tables: [PluginExportTable], run: ExportRun) { let containers = Set(tables.map { $0.containerName ?? "" }) - exportSpansContainers = containers.count > 1 - guard exportSpansContainers else { return } - metadataWarnings.append(String( + run.exportSpansContainers = containers.count > 1 + guard run.exportSpansContainers else { return } + run.metadataWarnings.append(String( format: String(localized: """ This export spans %lld databases or schemas. Table references are qualified, but \ CREATE TABLE comes from the server unqualified, so restore it into the matching \ @@ -947,26 +938,28 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi private func qualifiedRef( schema: String, table: String, - dataSource: any PluginExportDataSource + dataSource: any PluginExportDataSource, + run: ExportRun ) -> String { let quotedTable = dataSource.quoteIdentifier(table) - guard exportSpansContainers, !schema.isEmpty else { return quotedTable } + guard run.exportSpansContainers, !schema.isEmpty else { return quotedTable } return "\(dataSource.quoteIdentifier(schema)).\(quotedTable)" } private func renderAddConstraintFK( table: PluginExportTable, group: [PluginForeignKeyInfo], - dataSource: any PluginExportDataSource + dataSource: any PluginExportDataSource, + run: ExportRun ) -> String { let tableRef = qualifiedRef( - schema: table.databaseName, table: table.name, dataSource: dataSource) + schema: table.databaseName, table: table.name, dataSource: dataSource, run: run) let constraintName = dataSource.quoteIdentifier(group[0].name) let cols = group.map { dataSource.quoteIdentifier($0.column) }.joined(separator: ", ") let refCols = group.map { dataSource.quoteIdentifier($0.referencedColumn) }.joined(separator: ", ") let refSchema = (group[0].referencedSchema?.isEmpty == false ? group[0].referencedSchema : nil) ?? table.databaseName let refTable = qualifiedRef( - schema: refSchema, table: group[0].referencedTable, dataSource: dataSource) + schema: refSchema, table: group[0].referencedTable, dataSource: dataSource, run: run) let onDelete = group[0].onDelete.uppercased() let onUpdate = group[0].onUpdate.uppercased() var alter = "ALTER TABLE \(tableRef) ADD CONSTRAINT \(constraintName) FOREIGN KEY (\(cols)) REFERENCES \(refTable) (\(refCols))" @@ -985,10 +978,10 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi private func writeTableData( table: PluginExportTable, columnInfo: [PluginColumnInfo], - options: SQLExportOptions, dataSource: any PluginExportDataSource, to writer: SQLExportFileWriter, - progress: PluginExportProgress + progress: PluginExportProgress, + run: ExportRun ) async throws -> SQLExportStatementTally { var wroteAnyRows = false var tally = SQLExportStatementTally() @@ -998,13 +991,9 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let usesOverridingSystemValue = SqlDialect.from(databaseTypeId: dataSource.databaseTypeId) == .postgres && columnInfo.contains { $0.identityKind == .always } let tableRef = qualifiedRef( - schema: table.databaseName, table: table.name, dataSource: dataSource) - /// SQL Server refuses an explicit value for an IDENTITY column unless the table is opened - /// for it first. The rows are exported with their keys, so without this the dump restores - /// nothing: every INSERT for the table is rejected while the export itself reported success. - /// The writer owns it as a scope so a rotation inside the table's rows carries it into the - /// next part: as two plain statements, one part ended on an unmatched `ON` and the next - /// opened with rows and none (#2533). + schema: table.databaseName, table: table.name, dataSource: dataSource, run: run) + // SQL Server rejects an explicit IDENTITY value unless the table is opened for it. A writer scope, so a part + // rotation inside the rows carries it into the next part (#2533). let needsIdentityInsert = dataSource.databaseTypeId == "SQL Server" && columnInfo.contains(where: \.isIdentity) let identityInsert = needsIdentityInsert @@ -1016,7 +1005,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi try writer.write("-- Rows narrowed to: \(scopeNote)\n") } if table.rowScope.hasRejectedFilter { - metadataWarnings.append(String( + run.metadataWarnings.append(String( format: String(localized: "The row filter on %@ was not a single expression, so every row was exported."), table.name)) @@ -1024,20 +1013,15 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi var encoder: SQLExportRowValueEncoder? var accumulator: SQLExportStatementAccumulator? - /// The insert mode the engine cannot spell is only worth reporting once a row is actually - /// written under it. Held until then, because a table whose scope selected nothing still - /// produces a header, and a warning raised from that alone brands a clean export a failed - /// one: `warnings` is what retitles the summary alert and takes away its suppression. + // Raised only once a row is written under it: a table whose scope selected nothing still sends a header, and + // any warning retitles the summary alert as a failure. var pendingModeWarning: String? - /// `SET IDENTITY_INSERT` opens the table and has to precede the first statement that carries - /// a key, so it is written from whatever hands one over rather than from a row count. The - /// byte budget can hold rows well past `batchSize` before closing a statement, so a - /// row-count trigger would write it in the wrong place or not at all. + // The byte budget can hold rows past batchSize, so IDENTITY_INSERT opens on the first statement, not a row count. func emit(_ statement: String) throws { if let warning = pendingModeWarning { - if !metadataWarnings.contains(warning) { - metadataWarnings.append(warning) + if !run.metadataWarnings.contains(warning) { + run.metadataWarnings.append(warning) } pendingModeWarning = nil } @@ -1055,15 +1039,12 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi switch element { case .header(let header): - /// A second header describes different columns, so the statement built under the - /// first one is closed before anything is rendered against the new prefix. + // A second header describes different columns: close the statement built under the first. if let statement = accumulator?.finish() { try emit(statement) } if let accumulator { tally.merge(accumulator.tally) } - /// The outgoing encoder's count goes with it. Only the last encoder used to reach - /// the tally, so a second header after rows dropped every unrepresentable value the - /// first segment rendered, and an export whose last segment held none reported clean. + // The outgoing encoder's count too, or values a first segment could not represent go unreported. if let encoder { tally.unrepresentableValues += encoder.unrepresentableValues.total } let built = SQLExportRowValueEncoder( columns: header.columns, @@ -1084,7 +1065,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi encoder: built, primaryKeyColumns: primaryKeyColumns, usesOverridingSystemValue: usesOverridingSystemValue, - options: options, + options: run.options, dataSource: dataSource) accumulator = statementWriter.accumulator pendingModeWarning = statementWriter.modeWarning @@ -1100,10 +1081,7 @@ final class SQLExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } } - /// Stop can land between the last row and the end of the stream, where the loop's own checks - /// no longer run. Without this, a cancelled export writes its last statement, commits the - /// file and reports success; the batch path this replaced checked cancellation per row and - /// so threw instead, which is what makes the writer roll the whole dump back. + // A Stop between the last row and the end of the stream would otherwise commit the dump and report success. try progress.checkCancellation() if let statement = accumulator?.finish() { diff --git a/apple/Plugins/SQLExportPlugin/SQLExportReadScope.swift b/apple/Plugins/SQLExportPlugin/SQLExportReadScope.swift index a42419289e..4b57a761d4 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportReadScope.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportReadScope.swift @@ -9,7 +9,7 @@ import TableProPluginKit /// The driver pins the read session, such as MySQL's `time_zone`, so the dump's values read back the same on the /// server that loads it. The export reads on the user's own session, so every exit puts it back. -internal struct SQLExportReadScope { +nonisolated internal struct SQLExportReadScope { private static let logger = Logger(subsystem: "com.TablePro", category: "SQLExportReadScope") private var scope: PluginSessionScope? diff --git a/apple/Plugins/SQLExportPlugin/SQLExportRowValueEncoder.swift b/apple/Plugins/SQLExportPlugin/SQLExportRowValueEncoder.swift index c900e4cf98..b54f25f978 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportRowValueEncoder.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportRowValueEncoder.swift @@ -15,7 +15,7 @@ import TableProPluginKit /// /// Separate from `SQLExportInsertRenderer`, which owns the statement's prefix and suffix for an /// insert mode. This owns the rows between them. -internal struct SQLExportRowValueEncoder { +nonisolated internal struct SQLExportRowValueEncoder { internal let includedColumnIndices: [Int] /// The values this encoder rendered that the engine cannot carry in one statement whatever diff --git a/apple/Plugins/SQLExportPlugin/SQLExportSessionDeclaration.swift b/apple/Plugins/SQLExportPlugin/SQLExportSessionDeclaration.swift index d1117fd69f..96f1452ecc 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportSessionDeclaration.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportSessionDeclaration.swift @@ -6,7 +6,7 @@ import Foundation import TableProPluginKit -internal struct SQLExportSessionDeclaration: Equatable { +nonisolated internal struct SQLExportSessionDeclaration: Equatable { static let empty = SQLExportSessionDeclaration(prologue: "", epilogue: "") let prologue: String diff --git a/apple/Plugins/SQLExportPlugin/SQLExportSessionScope.swift b/apple/Plugins/SQLExportPlugin/SQLExportSessionScope.swift index a0d5dc52e6..8b88d6c3cc 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportSessionScope.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportSessionScope.swift @@ -12,7 +12,7 @@ import Foundation /// one, but not in one session, so session state cannot be assumed to cross a part boundary. A /// scope therefore belongs to every part it spans rather than to the file: the writer re-opens it /// at the top of each new part and closes it at the bottom of each finished one. -internal struct SQLExportSessionScope: Equatable { +nonisolated internal struct SQLExportSessionScope: Equatable { internal let opener: String internal let closer: String diff --git a/apple/Plugins/SQLExportPlugin/SQLExportSnapshot.swift b/apple/Plugins/SQLExportPlugin/SQLExportSnapshot.swift index 8fe7abd84b..b6a1e09094 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportSnapshot.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportSnapshot.swift @@ -14,7 +14,7 @@ import TableProPluginKit /// is not portable: MySQL takes a modifier on `START TRANSACTION`, PostgreSQL sets the isolation /// level on `BEGIN`, and SQLite gets the same guarantee from a plain deferred transaction. An /// engine with no spelling for it opens nothing rather than sending a statement it would reject. -internal struct SQLExportSnapshot { +nonisolated internal struct SQLExportSnapshot { private static let logger = Logger(subsystem: "com.TablePro", category: "SQLExportSnapshot") private let dialect: SqlDialect diff --git a/apple/Plugins/SQLExportPlugin/SQLExportStatementBudget.swift b/apple/Plugins/SQLExportPlugin/SQLExportStatementBudget.swift index 06c77f5d07..5d0010172f 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportStatementBudget.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportStatementBudget.swift @@ -14,7 +14,7 @@ import Foundation /// the byte ceiling off and reproduces a row-count-only dump. /// /// `maxRows` is a syntax ceiling, not a size one: see `SQLMultiRowInsert`. -internal struct SQLExportStatementBudget: Equatable { +nonisolated internal struct SQLExportStatementBudget: Equatable { internal let maxRows: Int internal let maxBytes: Int @@ -30,7 +30,7 @@ internal struct SQLExportStatementBudget: Equatable { /// figure accumulated on that instance belongs to whichever export wrote it last. It also carries the /// limit it was measured against, because the setting behind that limit is mutable and a second /// window's options pane can move it while this export is still running. -internal struct SQLExportStatementTally: Equatable { +nonisolated internal struct SQLExportStatementTally: Equatable { internal var largestStatementBytes = 0 internal var largestStatementRows = 0 internal var oversizedRowCount = 0 @@ -67,7 +67,7 @@ internal struct SQLExportStatementTally: Equatable { /// A class because it is a running buffer whose identity outlives the chunk of rows being fed to it: /// a statement spans as many driver chunks as the budget allows, so nothing may reset it at a chunk /// boundary. -internal final class SQLExportStatementAccumulator { +nonisolated internal final class SQLExportStatementAccumulator { /// What separates two rows, and what terminates a statement. Both are counted as the UTF-8 bytes /// they are, because the file is written as UTF-8 whatever encoding its prologue declares to the /// server. A terminator can carry a `GO` line, which ends each statement's batch on SQL Server. diff --git a/apple/Plugins/SQLExportPlugin/SQLExportStatementEnd.swift b/apple/Plugins/SQLExportPlugin/SQLExportStatementEnd.swift index a18a9a15e1..2580d3d7e0 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportStatementEnd.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportStatementEnd.swift @@ -6,7 +6,7 @@ import Foundation import TableProPluginKit -extension PluginExportDataSource { +nonisolated extension PluginExportDataSource { /// What ends every statement of a dump of this source: a line break, and on an engine whose client runs a script in /// batches cut at `GO` lines, that line too. SQL Server refuses a view, a routine or a trigger that is not the first /// statement of its batch, and a routine's body runs to the end of its batch, so each statement is written as a diff --git a/apple/Plugins/SQLImportPlugin/SQLImportCopyRows.swift b/apple/Plugins/SQLImportPlugin/SQLImportCopyRows.swift index 1b6034c571..164294958c 100644 --- a/apple/Plugins/SQLImportPlugin/SQLImportCopyRows.swift +++ b/apple/Plugins/SQLImportPlugin/SQLImportCopyRows.swift @@ -7,7 +7,7 @@ import Foundation import TableProPluginKit // The driver pulls a COPY's rows on its own thread, so the check for a Stop and the progress note ride on each pull. -internal final class SQLImportCopyRows: @unchecked Sendable { +nonisolated internal final class SQLImportCopyRows: @unchecked Sendable { private static let lineFeed = UInt8(ascii: "\n") private static let linesPerNote = 10_000 diff --git a/apple/Plugins/SQLImportPlugin/SQLImportFailure.swift b/apple/Plugins/SQLImportPlugin/SQLImportFailure.swift index 4fdd896c43..249a8d82a7 100644 --- a/apple/Plugins/SQLImportPlugin/SQLImportFailure.swift +++ b/apple/Plugins/SQLImportPlugin/SQLImportFailure.swift @@ -6,7 +6,7 @@ import Foundation import TableProPluginKit -internal enum SQLImportFailure { +nonisolated internal enum SQLImportFailure { internal static func error(cause: any Error, rollbackError: (any Error)?) -> any Error { if cause is PluginImportCancellationError || cause is CancellationError { return cause @@ -35,7 +35,7 @@ internal enum SQLImportFailure { } } -internal struct RollbackAlsoFailedError: LocalizedError { +nonisolated internal struct RollbackAlsoFailedError: LocalizedError { internal let cause: any Error internal let rollbackError: any Error diff --git a/apple/Plugins/SQLImportPlugin/SQLImportOptions.swift b/apple/Plugins/SQLImportPlugin/SQLImportOptions.swift index d314df8020..d8f2816ffa 100644 --- a/apple/Plugins/SQLImportPlugin/SQLImportOptions.swift +++ b/apple/Plugins/SQLImportPlugin/SQLImportOptions.swift @@ -6,7 +6,7 @@ import Foundation import TableProPluginKit -struct SQLImportOptions: Equatable, Codable { +nonisolated struct SQLImportOptions: Equatable, Codable { var errorHandling: ImportErrorHandling = .stopAndRollback var wrapInTransaction: Bool = true var disableForeignKeyChecks: Bool = true diff --git a/apple/Plugins/SQLImportPlugin/SQLImportOptionsView.swift b/apple/Plugins/SQLImportPlugin/SQLImportOptionsView.swift index 129421264e..33aef2fefa 100644 --- a/apple/Plugins/SQLImportPlugin/SQLImportOptionsView.swift +++ b/apple/Plugins/SQLImportPlugin/SQLImportOptionsView.swift @@ -36,3 +36,10 @@ struct SQLImportOptionsView: View { } } } + +extension SQLImportPlugin { + @MainActor + func settingsView() -> AnyView? { + AnyView(SQLImportOptionsView(plugin: self)) + } +} diff --git a/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift b/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift index bdff60a382..a5b7266c1a 100644 --- a/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift +++ b/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift @@ -6,10 +6,9 @@ import Combine import Foundation import os -import SwiftUI import TableProPluginKit -final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugin, @unchecked Sendable { +nonisolated final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugin, @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "SQLImportPlugin") static let pluginName = "SQL Import" @@ -24,17 +23,14 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi typealias Settings = SQLImportOptions static let settingsStorageId = "sql-import" - @Published var settings = SQLImportOptions() { + // A nonisolated class cannot hold an @Published property, so the change is announced by hand. + var settings = SQLImportOptions() { + willSet { objectWillChange.send() } didSet { saveSettings() } } required init() { loadSettings() } - @MainActor - func settingsView() -> AnyView? { - AnyView(SQLImportOptionsView(plugin: self)) - } - func resetSettingsToDefaults() { settings = SQLImportOptions() } diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLiteCatalogReads.swift b/apple/Plugins/SQLiteDriverPlugin/SQLiteCatalogReads.swift new file mode 100644 index 0000000000..c6fd358f0a --- /dev/null +++ b/apple/Plugins/SQLiteDriverPlugin/SQLiteCatalogReads.swift @@ -0,0 +1,206 @@ +// +// SQLiteCatalogReads.swift +// SQLiteDriverPlugin +// + +import Foundation +import TableProPluginKit + +// The catalog reads the macOS plugin and the iOS driver run over their own connection, so both apps export one schema. +nonisolated enum SQLiteCatalogReads { + typealias Run = @Sendable (String) async throws -> PluginQueryResult + + static let tableDDLIncludesForeignKeys = true + + static let foreignKeyDisableStatements = ["PRAGMA foreign_keys = OFF"] + static let foreignKeyEnableStatements = ["PRAGMA foreign_keys = ON"] + + static func columns(table: String, schema: String?, run: Run) async throws -> [PluginColumnInfo] { + let safeTable = SQLiteMasterQueries.escapeLiteral(table) + // table_info omits generated columns, which every write path reading this list has to see. + let query = "SELECT * FROM pragma_table_xinfo('\(safeTable)'\(SQLiteMasterQueries.pragmaSchemaArgument(schema)))" + let result = try await run(query) + let generationExpressions = SQLiteCheckConstraintParser.generationExpressions( + inCreateStatement: try await createTableStatement(table: table, schema: schema, run: run) ?? "" + ) + + return result.rows.compactMap { row in + guard row.count >= 7, + let name = row[1].asText, + let dataType = row[2].asText else { + return nil + } + + // hidden: 0 normal, 1 a virtual table's hidden column, 2 VIRTUAL generated, 3 STORED generated. + let hidden = row[6].asText.flatMap { Int($0) } ?? 0 + guard hidden != 1 else { return nil } + + let isNullable = row[3].asText == "0" + // pk: 0 not in the key, 1+ the position in a composite key. + let pkText = row[5].asText + let isPrimaryKey = pkText != nil && pkText != "0" + let defaultValue = sqliteDefaultValueFromCatalog(row[4].asText) + let generationKind: GenerationKind? = hidden == 2 ? .virtual : (hidden == 3 ? .stored : nil) + + return PluginColumnInfo( + name: name, + dataType: dataType, + isNullable: isNullable, + isPrimaryKey: isPrimaryKey, + defaultValue: defaultValue, + isGenerated: generationKind != nil, + generationExpression: generationKind == nil ? nil : generationExpressions[name], + generationKind: generationKind + ) + } + } + + static func createTableStatement(table: String, schema: String?, run: Run) async throws -> String? { + let query = SQLiteTemporarySchema.definitionQuery(name: table, types: ["table"], schema: schema) + let result = try await run(query) + return result.rows.first?[safe: 1]?.asText + } + + static func checkConstraints(table: String, schema: String?, run: Run) async throws -> [PluginCheckConstraintInfo] { + guard let statement = try await createTableStatement(table: table, schema: schema, run: run) else { return [] } + return SQLiteCheckConstraintParser.constraints(inCreateStatement: statement).map { parsed in + PluginCheckConstraintInfo(name: parsed.name, expression: parsed.expression) + } + } + + // `m.sql` rides along so the generation expressions come from the CREATE statement without a read per table. + static func allColumns(schema: String?, run: Run) async throws -> [String: [PluginColumnInfo]] { + let query = """ + SELECT m.name AS tbl, p.cid, p.name, p.type, p."notnull", p.dflt_value, p.pk, + p.hidden, m.sql + FROM \(SQLiteMasterQueries.catalogTable(schema: schema)) m, + pragma_table_xinfo(m.name\(SQLiteMasterQueries.pragmaSchemaArgument(schema))) p + WHERE m.type = 'table' AND m.name NOT GLOB 'sqlite_*' + ORDER BY m.name, p.cid + """ + let result = try await run(query) + + var allColumns: [String: [PluginColumnInfo]] = [:] + var expressionsByTable: [String: [String: String]] = [:] + + for row in result.rows { + guard row.count >= 9, + let tableName = row[0].asText, + let columnName = row[2].asText, + let dataType = row[3].asText else { + continue + } + + // hidden: 0 normal, 1 a virtual table's hidden column, 2 VIRTUAL generated, 3 STORED generated. + let hidden = row[7].asText.flatMap { Int($0) } ?? 0 + guard hidden != 1 else { continue } + + let isNullable = row[4].asText == "0" + let defaultValue = sqliteDefaultValueFromCatalog(row[5].asText) + // pk: 0 not in the key, 1+ the position in a composite key. + let pkText = row[6].asText + let isPrimaryKey = pkText != nil && pkText != "0" + let generationKind: GenerationKind? = hidden == 2 ? .virtual : (hidden == 3 ? .stored : nil) + + if generationKind != nil, expressionsByTable[tableName] == nil { + expressionsByTable[tableName] = SQLiteCheckConstraintParser.generationExpressions( + inCreateStatement: row[8].asText ?? "" + ) + } + + let column = PluginColumnInfo( + name: columnName, + dataType: dataType, + isNullable: isNullable, + isPrimaryKey: isPrimaryKey, + defaultValue: defaultValue, + isGenerated: generationKind != nil, + generationExpression: generationKind == nil ? nil : expressionsByTable[tableName]?[columnName], + generationKind: generationKind + ) + + allColumns[tableName, default: []].append(column) + } + + return allColumns + } + + static func allForeignKeys(schema: String?, run: Run) async throws -> [String: [PluginForeignKeyInfo]] { + let result = try await run( + SQLiteForeignKeyGrouping.schemaForeignKeysQuery(schema: schema, excludingTableGlob: nil) + ) + return SQLiteForeignKeyGrouping.foreignKeysByTable(schema: schema, fromRows: result.rows) + } + + // A view goes back as written: SQLite keeps a trailing comment in it, which an appended `;` would land inside. + static func tableDDL(table: String, schema: String?, run: Run) async throws -> String { + let query = SQLiteTemporarySchema.definitionQuery(name: table, types: ["table", "view"], schema: schema) + let result = try await run(query) + + guard let firstRow = result.rows.first, + let ddl = firstRow[safe: 1]?.asText else { + throw SQLiteCatalogReadError.missingTableDefinition(table) + } + + guard firstRow[safe: 0]?.asText != "view" else { return ddl } + return ddl.hasSuffix(";") ? ddl : ddl + ";" + } + + static func viewDefinition(view: String, schema: String?, run: Run) async throws -> String { + let safeView = SQLiteMasterQueries.escapeLiteral(view) + let query = """ + SELECT sql FROM \(SQLiteMasterQueries.catalogTable(schema: schema)) + WHERE type = 'view' AND name = '\(safeView)' + """ + let result = try await run(query) + + guard let firstRow = result.rows.first, + let ddl = firstRow[0].asText else { + throw SQLiteCatalogReadError.missingViewDefinition(view) + } + + return ddl + } + + // A null `sql` is an index SQLite made for a UNIQUE or PRIMARY KEY, which comes back inside `CREATE TABLE`. + static func indexDDL(table: String, schema: String?, run: Run) async throws -> [String] { + let result = try await run(SQLiteTemporarySchema.indexDefinitionsQuery(table: table, schema: schema)) + return result.rows.compactMap { $0[safe: 0]?.asText } + } + + static func triggers(table: String?, schema: String?, run: Run) async throws -> [PluginTriggerInfo] { + let query = SQLiteTemporarySchema.triggerListQuery(table: table, schema: schema) + let result = try await run(query) + return result.rows.compactMap { row -> PluginTriggerInfo? in + guard row.count >= 3, let name = row[0].asText, let sql = row[2].asText else { return nil } + return SQLiteMasterQueries.trigger(name: name, table: row[1].asText, schema: schema, sql: sql) + } + } + + static func triggerDDL(_ trigger: PluginTriggerInfo, run: Run) async throws -> String { + if let definition = trigger.definition, !definition.isEmpty { return definition } + let listed = try await triggers(table: trigger.table, schema: trigger.schema, run: run) + guard let definition = listed.first(where: { $0.name == trigger.name })?.definition, + !definition.isEmpty + else { + throw PluginObjectSourceError.notFound(trigger.name) + } + return definition + } + + static func dropTriggerStatement(name: String, schema: String?) -> String { + "DROP TRIGGER IF EXISTS \(SQLiteMasterQueries.qualifiedTable(name, schema: schema, quote: sqliteQuoteIdentifier))" + } +} + +nonisolated enum SQLiteCatalogReadError: Error, PluginDriverError { + case missingTableDefinition(String) + case missingViewDefinition(String) + + var pluginErrorMessage: String { + switch self { + case .missingTableDefinition(let table): "Failed to fetch DDL for table '\(table)'" + case .missingViewDefinition(let view): "Failed to fetch definition for view '\(view)'" + } + } +} diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLiteCheckConstraintParser.swift b/apple/Plugins/SQLiteDriverPlugin/SQLiteCheckConstraintParser.swift index 24340eea38..85c8b648c9 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLiteCheckConstraintParser.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLiteCheckConstraintParser.swift @@ -7,7 +7,7 @@ import Foundation import TableProPluginKit // SQLite keeps constraints only in `sqlite_master.sql`. Only named table-level checks are read: DROP CONSTRAINT needs a name. -enum SQLiteCheckConstraintParser { +nonisolated enum SQLiteCheckConstraintParser { struct ParsedConstraint: Equatable { let name: String let expression: String diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLiteCreateTableDDL.swift b/apple/Plugins/SQLiteDriverPlugin/SQLiteCreateTableDDL.swift index d862b11674..14c59dae5e 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLiteCreateTableDDL.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLiteCreateTableDDL.swift @@ -8,19 +8,29 @@ import TableProPluginKit import TableProSQLGrammar import TableProSQLQuoting +// The id `SQLitePlugin.databaseTypeId` holds, for the files iOS compiles without the plugin class. +nonisolated internal let sqliteDatabaseTypeId = "SQLite" + // Free functions, so the test target exercises the DDL without a connection. -internal func sqliteQuoteIdentifier(_ name: String) -> String { +nonisolated internal func sqliteQuoteIdentifier(_ name: String) -> String { SQLQuoting.identifier(name, quote: .backtick) } +// SQLite reads statement text only up to a NUL, so a NUL in a value goes in as char(0) between two literals. +nonisolated internal func sqliteStringLiteralBody(_ value: String) -> String { + value.split(separator: "\0", omittingEmptySubsequences: false) + .map { SQLQuoting.stringLiteralBody(String($0)) } + .joined(separator: "'||char(0)||'") +} + // Type, default and generation text, from the catalog or the user, goes into DDL only as one type or expression. -internal func sqliteColumnRefusal( +nonisolated internal func sqliteColumnRefusal( name: String, type: String?, defaultValue: String?, generationExpression: String? = nil ) -> String? { - let typeId = SQLitePlugin.databaseTypeId + let typeId = sqliteDatabaseTypeId if let type, !type.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, !SQLExpressionFragment.isSingleType(type, databaseTypeId: typeId) { return String(format: String(localized: "The type of column %@ cannot be written as SQL."), name) @@ -36,7 +46,7 @@ internal func sqliteColumnRefusal( return nil } -internal func sqliteColumnRefusal(_ column: PluginColumnDefinition) -> String? { +nonisolated internal func sqliteColumnRefusal(_ column: PluginColumnDefinition) -> String? { sqliteColumnRefusal( name: column.name, type: column.dataType, @@ -45,7 +55,7 @@ internal func sqliteColumnRefusal(_ column: PluginColumnDefinition) -> String? { ) } -internal func sqliteRespecificationRefusal(_ respecification: PluginTableRespecification) -> String? { +nonisolated internal func sqliteRespecificationRefusal(_ respecification: PluginTableRespecification) -> String? { respecification.addedColumns.lazy.compactMap { sqliteColumnRefusal($0) }.first ?? respecification.alteredColumns.lazy.compactMap { sqliteColumnRefusal(name: $0.column, type: $0.type, defaultValue: $0.defaultValue) @@ -53,8 +63,8 @@ internal func sqliteRespecificationRefusal(_ respecification: PluginTableRespeci } // The partial-index predicate and the key list come back from `sqlite_master` and are spliced as written. -internal func sqliteIndexRefusal(_ index: PluginIndexDefinition) -> String? { - let typeId = SQLitePlugin.databaseTypeId +nonisolated internal func sqliteIndexRefusal(_ index: PluginIndexDefinition) -> String? { + let typeId = sqliteDatabaseTypeId let keys = index.ddlMethodAndKeys?.nilIfEmpty.map { [$0] } ?? (index.expressions ?? []) guard keys.allSatisfy({ SQLExpressionFragment.isSingleExpression($0, databaseTypeId: typeId) }) else { return String(format: String(localized: "Index %@ has a key part that is not one SQL expression."), index.name) @@ -65,7 +75,7 @@ internal func sqliteIndexRefusal(_ index: PluginIndexDefinition) -> String? { return nil } -internal func sqliteCreateTableSQL(definition: PluginCreateTableDefinition) -> String? { +nonisolated internal func sqliteCreateTableSQL(definition: PluginCreateTableDefinition) -> String? { guard !definition.columns.isEmpty, definition.columns.allSatisfy({ sqliteColumnRefusal($0) == nil }) else { return nil } let declaredKey = Set(definition.primaryKeyColumns) @@ -89,10 +99,8 @@ internal func sqliteCreateTableSQL(definition: PluginCreateTableDefinition) -> S + "\n);" } -/// `isInlinePrimaryKey` says whether to declare `PRIMARY KEY` on this column, so a key named only by -/// `definition.primaryKeyColumns` still reaches the DDL along with the `AUTOINCREMENT` that only -/// this arm writes. -internal func sqliteColumnDefinitionSQL( +// Only the inline key arm writes `AUTOINCREMENT`, so a key named by `primaryKeyColumns` alone is declared here too. +nonisolated internal func sqliteColumnDefinitionSQL( _ column: PluginColumnDefinition, isInlinePrimaryKey: Bool ) -> String { @@ -117,14 +125,8 @@ internal func sqliteColumnDefinitionSQL( return definition } -/// `CONSTRAINT name` is optional in SQLite's table-constraint grammar, so it is written when there -/// is a name and left out when there is not. This used to drop the name unconditionally, so a name -/// the user typed never reached the database. -/// -/// The referenced column list is omitted rather than emitted empty: `REFERENCES "t" ()` is a syntax -/// error, while `REFERENCES "t"` means the parent's primary key, which is what an empty list asks -/// for. SQLite has no cross-schema foreign key, so `referencedSchema` is deliberately not written. -internal func sqliteForeignKeyDefinitionSQL(_ foreignKey: PluginForeignKeyDefinition) -> String { +// `REFERENCES "t" ()` is a syntax error and bare `REFERENCES "t"` means the parent's key; SQLite has no cross-schema key. +nonisolated internal func sqliteForeignKeyDefinitionSQL(_ foreignKey: PluginForeignKeyDefinition) -> String { let columns = foreignKey.columns.map(sqliteQuoteIdentifier).joined(separator: ", ") let constraint = foreignKey.name.isEmpty ? "" diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLiteDefaultValue.swift b/apple/Plugins/SQLiteDriverPlugin/SQLiteDefaultValue.swift index 0eb8257171..b2413bdd98 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLiteDefaultValue.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLiteDefaultValue.swift @@ -7,7 +7,7 @@ import Foundation import TableProSQLQuoting // The pragma drops a default's parentheses and a bare word's quotes (3.54.0); only an expression has a call or an operator. -internal func sqliteDefaultValueFromCatalog(_ value: String?) -> String? { +nonisolated internal func sqliteDefaultValueFromCatalog(_ value: String?) -> String? { guard let value, !value.isEmpty else { return nil } let upper = value.uppercased() if upper == "NULL" || upper == "TRUE" || upper == "FALSE" @@ -20,8 +20,8 @@ internal func sqliteDefaultValueFromCatalog(_ value: String?) -> String? { return sqliteLooksLikeExpression(value) ? "(\(value))" : SQLQuoting.stringLiteral(value) } -private let sqliteExpressionScalars: Set = ["(", ")", "|", "+", "-", "*", "/", "%", "<", ">", "=", "!", "~", "&", " ", ","] +nonisolated private let sqliteExpressionScalars: Set = ["(", ")", "|", "+", "-", "*", "/", "%", "<", ">", "=", "!", "~", "&", " ", ","] -internal func sqliteLooksLikeExpression(_ value: String) -> Bool { +nonisolated internal func sqliteLooksLikeExpression(_ value: String) -> Bool { value.unicodeScalars.contains { sqliteExpressionScalars.contains($0) } } diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift b/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift index c4175d87dc..9e3057137d 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift @@ -160,6 +160,10 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { sqliteQuoteIdentifier(name) } + func escapeStringLiteral(_ value: String) -> String { + sqliteStringLiteralBody(value) + } + convenience init(config: DriverConnectionConfig) { self.init(config: config, backend: Self.makeBackend(config: config)) } @@ -230,6 +234,10 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { Self.queryResult(try await backend.executeQuery(query)) } + var catalogRun: SQLiteCatalogReads.Run { + { [self] query in try await self.execute(query: query) } + } + func execute(query: String, context: PluginStatementContext) async throws -> PluginQueryResult { let rules = SQLiteStatementRules(context) guard rules != .unrestricted else { return try await execute(query: query) } @@ -315,11 +323,11 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Foreign Key Checks func foreignKeyDisableStatements() -> [String]? { - ["PRAGMA foreign_keys = OFF"] + SQLiteCatalogReads.foreignKeyDisableStatements } func foreignKeyEnableStatements() -> [String]? { - ["PRAGMA foreign_keys = ON"] + SQLiteCatalogReads.foreignKeyEnableStatements } // MARK: - Session Scope @@ -454,134 +462,25 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { - let safeTable = escapeStringLiteral(table) - // table_xinfo rather than table_info: table_info omits generated columns entirely, so they - // were invisible to the structure editor and to every write path that reads this list. - let query = "SELECT * FROM pragma_table_xinfo('\(safeTable)'\(SQLiteMasterQueries.pragmaSchemaArgument(schema)))" - let result = try await execute(query: query) - let generationExpressions = SQLiteCheckConstraintParser.generationExpressions( - inCreateStatement: try await createStatement(forTable: table, schema: schema) ?? "" - ) - - return result.rows.compactMap { row in - guard row.count >= 7, - let name = row[1].asText, - let dataType = row[2].asText else { - return nil - } - - // hidden: 0 normal, 1 a virtual table's hidden column, 2 VIRTUAL generated, - // 3 STORED generated. - let hidden = row[6].asText.flatMap { Int($0) } ?? 0 - guard hidden != 1 else { return nil } - - let isNullable = row[3].asText == "0" - // PRAGMA pk column: 0 = not PK, 1+ = position in composite PK - let pkText = row[5].asText - let isPrimaryKey = pkText != nil && pkText != "0" - let defaultValue = sqliteDefaultValueFromCatalog(row[4].asText) - let generationKind: GenerationKind? = hidden == 2 ? .virtual : (hidden == 3 ? .stored : nil) - - return PluginColumnInfo( - name: name, - dataType: dataType, - isNullable: isNullable, - isPrimaryKey: isPrimaryKey, - defaultValue: defaultValue, - isGenerated: generationKind != nil, - generationExpression: generationKind == nil ? nil : generationExpressions[name], - generationKind: generationKind - ) - } - } - - private func createStatement(forTable table: String, schema: String?) async throws -> String? { - let query = SQLiteTemporarySchema.definitionQuery(name: table, types: ["table"], schema: schema) - let result = try await execute(query: query) - return result.rows.first?[safe: 1]?.asText + try await SQLiteCatalogReads.columns(table: table, schema: schema, run: catalogRun) } func fetchCheckConstraints(table: String, schema: String?) async throws -> [PluginCheckConstraintInfo] { - guard let statement = try await createStatement(forTable: table, schema: schema) else { return [] } - return SQLiteCheckConstraintParser.constraints(inCreateStatement: statement).map { parsed in - PluginCheckConstraintInfo(name: parsed.name, expression: parsed.expression) - } + try await SQLiteCatalogReads.checkConstraints(table: table, schema: schema, run: catalogRun) } var providesBulkColumnFetch: Bool { true } - /// `pragma_table_xinfo`, not `pragma_table_info`, for the same reason `fetchColumns` uses it: - /// `table_info` omits generated columns entirely, so the bulk read used to answer with a - /// shorter column list than the per-table read for the same table. A caller comparing two - /// schemas through the bulk read saw neither side's generated columns and reported them as - /// matching. `m.sql` rides along so the generation expressions are parsed from the CREATE - /// statement without a second round trip per table. func fetchAllColumns(schema: String?) async throws -> [String: [PluginColumnInfo]] { - let query = """ - SELECT m.name AS tbl, p.cid, p.name, p.type, p."notnull", p.dflt_value, p.pk, - p.hidden, m.sql - FROM \(SQLiteMasterQueries.catalogTable(schema: schema)) m, - pragma_table_xinfo(m.name\(SQLiteMasterQueries.pragmaSchemaArgument(schema))) p - WHERE m.type = 'table' AND m.name NOT GLOB 'sqlite_*' - ORDER BY m.name, p.cid - """ - let result = try await execute(query: query) - - var allColumns: [String: [PluginColumnInfo]] = [:] - var expressionsByTable: [String: [String: String]] = [:] - - for row in result.rows { - guard row.count >= 9, - let tableName = row[0].asText, - let columnName = row[2].asText, - let dataType = row[3].asText else { - continue - } - - // hidden: 0 normal, 1 a virtual table's hidden column, 2 VIRTUAL generated, - // 3 STORED generated. - let hidden = row[7].asText.flatMap { Int($0) } ?? 0 - guard hidden != 1 else { continue } - - let isNullable = row[4].asText == "0" - let defaultValue = sqliteDefaultValueFromCatalog(row[5].asText) - // PRAGMA table_xinfo pk column: 0 = not PK, 1+ = position in composite PK - let pkText = row[6].asText - let isPrimaryKey = pkText != nil && pkText != "0" - let generationKind: GenerationKind? = hidden == 2 ? .virtual : (hidden == 3 ? .stored : nil) - - if generationKind != nil, expressionsByTable[tableName] == nil { - expressionsByTable[tableName] = SQLiteCheckConstraintParser.generationExpressions( - inCreateStatement: row[8].asText ?? "" - ) - } - - let column = PluginColumnInfo( - name: columnName, - dataType: dataType, - isNullable: isNullable, - isPrimaryKey: isPrimaryKey, - defaultValue: defaultValue, - isGenerated: generationKind != nil, - generationExpression: generationKind == nil ? nil : expressionsByTable[tableName]?[columnName], - generationKind: generationKind - ) - - allColumns[tableName, default: []].append(column) - } - - return allColumns + try await SQLiteCatalogReads.allColumns(schema: schema, run: catalogRun) } var providesBulkForeignKeyFetch: Bool { true } - var tableDDLIncludesForeignKeys: Bool { true } + var tableDDLIncludesForeignKeys: Bool { SQLiteCatalogReads.tableDDLIncludesForeignKeys } func fetchAllForeignKeys(schema: String?) async throws -> [String: [PluginForeignKeyInfo]] { - let result = try await execute( - query: SQLiteForeignKeyGrouping.schemaForeignKeysQuery(schema: schema, excludingTableGlob: nil) - ) - return SQLiteForeignKeyGrouping.foreignKeysByTable(schema: schema, fromRows: result.rows) + try await SQLiteCatalogReads.allForeignKeys(schema: schema, run: catalogRun) } func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { @@ -595,7 +494,7 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func fetchTriggers(table: String, schema: String?) async throws -> [PluginTriggerInfo] { - try await sqliteTriggerList(table: table, schema: schema) + try await SQLiteCatalogReads.triggers(table: table, schema: schema, run: catalogRun) } var supportsTransactionalDDL: Bool { true } @@ -613,51 +512,19 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func generateDropTriggerSQL(name: String, table: String, schema: String?) -> String? { - "DROP TRIGGER IF EXISTS \(SQLiteMasterQueries.qualifiedTable(name, schema: schema, quote: quoteIdentifier))" + SQLiteCatalogReads.dropTriggerStatement(name: name, schema: schema) } - /// A view's `CREATE VIEW` sits in `sqlite_master` beside the tables, so one read answers both. - /// The view goes back exactly as it was written: SQLite keeps a trailing comment in it, which a - /// `;` appended here would land inside. func fetchTableDDL(table: String, schema: String?) async throws -> String { - let query = SQLiteTemporarySchema.definitionQuery(name: table, types: ["table", "view"], schema: schema) - let result = try await execute(query: query) - - guard let firstRow = result.rows.first, - let ddl = firstRow[safe: 1]?.asText else { - throw SQLitePluginError.queryFailed("Failed to fetch DDL for table '\(table)'") - } - - guard firstRow[safe: 0]?.asText != "view" else { return ddl } - return ddl.hasSuffix(";") ? ddl : ddl + ";" + try await SQLiteCatalogReads.tableDDL(table: table, schema: schema, run: catalogRun) } - /// `sqlite_master` stores each index's own `CREATE INDEX` text, which is what `sqlite3 .dump` - /// replays and which carries a partial predicate, an expression key, a collation and a sort - /// direction exactly as written. An index SQLite created for itself to back a UNIQUE or PRIMARY - /// KEY constraint has a null `sql`, so testing for that is what keeps `sqlite_autoindex_*` out - /// of the dump: those come back with the constraint inside `CREATE TABLE`. func fetchIndexDDL(table: String, schema: String?) async throws -> [String] { - let result = try await execute( - query: SQLiteTemporarySchema.indexDefinitionsQuery(table: table, schema: schema) - ) - return result.rows.compactMap { $0[safe: 0]?.asText } + try await SQLiteCatalogReads.indexDDL(table: table, schema: schema, run: catalogRun) } func fetchViewDefinition(view: String, schema: String?) async throws -> String { - let safeView = escapeStringLiteral(view) - let query = """ - SELECT sql FROM \(SQLiteMasterQueries.catalogTable(schema: schema)) - WHERE type = 'view' AND name = '\(safeView)' - """ - let result = try await execute(query: query) - - guard let firstRow = result.rows.first, - let ddl = firstRow[0].asText else { - throw SQLitePluginError.queryFailed("Failed to fetch definition for view '\(view)'") - } - - return ddl + try await SQLiteCatalogReads.viewDefinition(view: view, schema: schema, run: catalogRun) } func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLitePluginDriver+Triggers.swift b/apple/Plugins/SQLiteDriverPlugin/SQLitePluginDriver+Triggers.swift index b2429d888f..e7bcd0fa96 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLitePluginDriver+Triggers.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLitePluginDriver+Triggers.swift @@ -10,26 +10,10 @@ extension SQLitePluginDriver { var providesBulkTriggerFetch: Bool { true } func fetchAllTriggers(schema: String?) async throws -> [PluginTriggerInfo] { - try await sqliteTriggerList(table: nil, schema: schema) + try await SQLiteCatalogReads.triggers(table: nil, schema: schema, run: catalogRun) } func fetchTriggerDDL(_ trigger: PluginTriggerInfo) async throws -> String { - if let definition = trigger.definition, !definition.isEmpty { return definition } - let listed = try await sqliteTriggerList(table: trigger.table, schema: trigger.schema) - guard let definition = listed.first(where: { $0.name == trigger.name })?.definition, - !definition.isEmpty - else { - throw PluginObjectSourceError.notFound(trigger.name) - } - return definition - } - - func sqliteTriggerList(table: String?, schema: String?) async throws -> [PluginTriggerInfo] { - let query = SQLiteTemporarySchema.triggerListQuery(table: table, schema: schema) - let result = try await execute(query: query) - return result.rows.compactMap { row -> PluginTriggerInfo? in - guard row.count >= 3, let name = row[0].asText, let sql = row[2].asText else { return nil } - return SQLiteMasterQueries.trigger(name: name, table: row[1].asText, schema: schema, sql: sql) - } + try await SQLiteCatalogReads.triggerDDL(trigger, run: catalogRun) } } diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLiteTemporarySchema.swift b/apple/Plugins/SQLiteDriverPlugin/SQLiteTemporarySchema.swift index 53dd2fdc7a..99e895ac3f 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLiteTemporarySchema.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLiteTemporarySchema.swift @@ -9,7 +9,7 @@ import TableProPluginKit // A connection's `temp` schema is its own: another connection to the same file sees none of it. SQLite resolves a bare // name in `temp` before `main`, so a catalog read that looks at `sqlite_master` alone misses a temporary table, or // answers with the main table it hides. -enum SQLiteTemporarySchema { +nonisolated enum SQLiteTemporarySchema { static let tablesQuery = """ SELECT name, type FROM sqlite_temp_master WHERE type IN ('table', 'view') diff --git a/apple/Plugins/SQLiteDriverTests/SQLiteCatalogReadsTests.swift b/apple/Plugins/SQLiteDriverTests/SQLiteCatalogReadsTests.swift new file mode 100644 index 0000000000..8d79ca9004 --- /dev/null +++ b/apple/Plugins/SQLiteDriverTests/SQLiteCatalogReadsTests.swift @@ -0,0 +1,245 @@ +// +// SQLiteCatalogReadsTests.swift +// SQLiteDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +private actor RecordingRunner { + private(set) var sent: [String] = [] + private var answers: [[[PluginCellValue]]] + + init(_ answers: [[[PluginCellValue]]] = []) { + self.answers = answers + } + + func answer(_ sql: String) -> PluginQueryResult { + sent.append(sql) + let rows = answers.isEmpty ? [] : answers.removeFirst() + return PluginQueryResult(columns: [], columnTypeNames: [], rows: rows, rowsAffected: 0, executionTime: 0) + } + + nonisolated var run: SQLiteCatalogReads.Run { + { await self.answer($0) } + } +} + +struct SQLiteCatalogReadsTests { + @Test("A table's columns come from table_xinfo, with generation expressions from its CREATE statement") + func columnsReadTableXinfoAndTheCreateStatement() async throws { + let runner = RecordingRunner([ + [ + ["0", "id", "INTEGER", "1", nil, "1", "0"], + ["1", "doubled", "INT", "0", nil, "0", "2"], + ["2", "rowid_alias", "INT", "0", nil, "0", "1"], + ["3", "n", "INT", "0", "0", "0", "0"], + ], + [["table", "CREATE TABLE t (id INTEGER PRIMARY KEY, doubled INT GENERATED ALWAYS AS (id * 2) VIRTUAL, n INT DEFAULT 0)"]], + ]) + + let columns = try await SQLiteCatalogReads.columns(table: "t", schema: "auth", run: runner.run) + + #expect(await runner.sent == [ + "SELECT * FROM pragma_table_xinfo('t', 'auth')", + SQLiteTemporarySchema.definitionQuery(name: "t", types: ["table"], schema: "auth"), + ]) + #expect(columns.map { $0.name } == ["id", "doubled", "n"]) + #expect(columns[0].isPrimaryKey) + #expect(!columns[0].isNullable) + #expect(columns[1].generationKind == .virtual) + #expect(columns[1].generationExpression == "id * 2") + #expect(columns[2].isNullable) + #expect(columns[2].defaultValue == "0") + #expect(columns[2].generationExpression == nil) + } + + @Test("Check constraints are parsed from the table's own CREATE statement") + func checkConstraintsReadTheCreateStatement() async throws { + let runner = RecordingRunner([ + [["table", "CREATE TABLE t (a INT, CONSTRAINT positive CHECK (a > 0))"]], + ]) + + let constraints = try await SQLiteCatalogReads.checkConstraints(table: "t", schema: nil, run: runner.run) + + #expect(await runner.sent == [SQLiteTemporarySchema.definitionQuery(name: "t", types: ["table"], schema: nil)]) + #expect(constraints.map { $0.name } == ["positive"]) + #expect(constraints.map { $0.expression } == ["a > 0"]) + } + + @Test("All columns are one read joined with table_xinfo, grouped by table") + func allColumnsIsOneRead() async throws { + let createA = "CREATE TABLE a (id INTEGER PRIMARY KEY, g INT GENERATED ALWAYS AS (id + 1) STORED)" + let runner = RecordingRunner([ + [ + ["a", "0", "id", "INTEGER", "1", nil, "1", "0", .text(createA)], + ["a", "1", "g", "INT", "0", nil, "0", "3", .text(createA)], + ["a", "2", "hidden", "INT", "0", nil, "0", "1", .text(createA)], + ["b", "0", "label", "TEXT", "0", "'none'", "0", "0", "CREATE TABLE b (label TEXT DEFAULT 'none')"], + ], + ]) + + let columns = try await SQLiteCatalogReads.allColumns(schema: nil, run: runner.run) + + #expect(await runner.sent == [ + """ + SELECT m.name AS tbl, p.cid, p.name, p.type, p."notnull", p.dflt_value, p.pk, + p.hidden, m.sql + FROM sqlite_master m, + pragma_table_xinfo(m.name) p + WHERE m.type = 'table' AND m.name NOT GLOB 'sqlite_*' + ORDER BY m.name, p.cid + """, + ]) + #expect(columns["a"]?.map { $0.name } == ["id", "g"]) + #expect(columns["a"]?.last?.generationKind == .stored) + #expect(columns["a"]?.last?.generationExpression == "id + 1") + #expect(columns["b"]?.first?.defaultValue == "'none'") + #expect(columns["b"]?.first?.isNullable == true) + } + + @Test("All foreign keys are the shared grouping over one read, naming the attached database") + func allForeignKeysUseTheSharedGrouping() async throws { + let runner = RecordingRunner([ + [[ + "orders", "0", "0", "customers", "customer_id", "id", "NO ACTION", "CASCADE", "NONE", + "CREATE TABLE orders (customer_id INT REFERENCES customers (id) ON DELETE CASCADE)", + ]], + ]) + + let keys = try await SQLiteCatalogReads.allForeignKeys(schema: "auth", run: runner.run) + + #expect(await runner.sent == [SQLiteForeignKeyGrouping.schemaForeignKeysQuery(schema: "auth", excludingTableGlob: nil)]) + let key = try #require(keys["orders"]?.first) + #expect(key.column == "customer_id") + #expect(key.referencedTable == "customers") + #expect(key.referencedColumn == "id") + #expect(key.referencedSchema == "auth") + #expect(key.onDelete == "CASCADE") + #expect(key.onUpdate == "NO ACTION") + } + + @Test("A table's DDL ends with a semicolon and a view's comes back as stored") + func tableDDLTerminatesTablesOnly() async throws { + let runner = RecordingRunner([ + [["table", "CREATE TABLE t (a INT)"]], + [["view", "CREATE VIEW v AS SELECT 1 -- kept"]], + ]) + + #expect(try await SQLiteCatalogReads.tableDDL(table: "t", schema: nil, run: runner.run) == "CREATE TABLE t (a INT);") + #expect(try await SQLiteCatalogReads.tableDDL(table: "v", schema: nil, run: runner.run) == "CREATE VIEW v AS SELECT 1 -- kept") + #expect(await runner.sent == [ + SQLiteTemporarySchema.definitionQuery(name: "t", types: ["table", "view"], schema: nil), + SQLiteTemporarySchema.definitionQuery(name: "v", types: ["table", "view"], schema: nil), + ]) + } + + @Test("A table with no stored definition is named in the error") + func missingTableDDLThrows() async { + let runner = RecordingRunner() + do { + _ = try await SQLiteCatalogReads.tableDDL(table: "gone", schema: nil, run: runner.run) + Issue.record("Expected an error") + } catch let error as SQLiteCatalogReadError { + #expect(error.pluginErrorMessage == "Failed to fetch DDL for table 'gone'") + } catch { + Issue.record("Unexpected error \(error)") + } + } + + @Test("A view's definition is read from the catalog of its own database") + func viewDefinitionReadsTheAttachedCatalog() async throws { + let runner = RecordingRunner([[["CREATE VIEW v AS SELECT 1"]]]) + + let ddl = try await SQLiteCatalogReads.viewDefinition(view: "v", schema: "auth", run: runner.run) + + #expect(ddl == "CREATE VIEW v AS SELECT 1") + #expect(await runner.sent == [ + """ + SELECT sql FROM "auth".sqlite_master + WHERE type = 'view' AND name = 'v' + """, + ]) + } + + @Test("A missing view is named in the error") + func missingViewDefinitionThrows() async { + let runner = RecordingRunner() + do { + _ = try await SQLiteCatalogReads.viewDefinition(view: "gone", schema: nil, run: runner.run) + Issue.record("Expected an error") + } catch let error as SQLiteCatalogReadError { + #expect(error.pluginErrorMessage == "Failed to fetch definition for view 'gone'") + } catch { + Issue.record("Unexpected error \(error)") + } + } + + @Test("Index DDL is each index's stored text, without the indexes SQLite made itself") + func indexDDLKeepsStoredText() async throws { + let runner = RecordingRunner([[["CREATE INDEX i ON t (a)", "i"], [nil, "sqlite_autoindex_t_1"]]]) + + let ddl = try await SQLiteCatalogReads.indexDDL(table: "t", schema: nil, run: runner.run) + + #expect(ddl == ["CREATE INDEX i ON t (a)"]) + #expect(await runner.sent == [SQLiteTemporarySchema.indexDefinitionsQuery(table: "t", schema: nil)]) + } + + @Test("A schema's triggers carry their table, schema and stored text") + func triggersMapTheCatalogRows() async throws { + let sql = "CREATE TRIGGER trg AFTER INSERT ON t BEGIN SELECT 1; END" + let runner = RecordingRunner([[[.text("trg"), "t", .text(sql)], ["short", "t"]]]) + + let triggers = try await SQLiteCatalogReads.triggers(table: nil, schema: "auth", run: runner.run) + + #expect(await runner.sent == [SQLiteMasterQueries.triggerList(table: nil, schema: "auth", excludeNameGlob: nil)]) + #expect(triggers.map { $0.name } == ["trg"]) + #expect(triggers.first?.table == "t") + #expect(triggers.first?.schema == "auth") + #expect(triggers.first?.definition == sql) + } + + @Test("A trigger that carries its definition is answered without a read") + func triggerDDLUsesTheListedDefinition() async throws { + let runner = RecordingRunner() + let trigger = PluginTriggerInfo( + name: "trg", table: "t", timing: "AFTER", event: "INSERT", statement: "", definition: "CREATE TRIGGER trg" + ) + + #expect(try await SQLiteCatalogReads.triggerDDL(trigger, run: runner.run) == "CREATE TRIGGER trg") + #expect(await runner.sent.isEmpty) + } + + @Test("A trigger without a definition is looked up among its table's triggers") + func triggerDDLReadsItsTable() async throws { + let sql = "CREATE TRIGGER trg AFTER INSERT ON t BEGIN SELECT 1; END" + let runner = RecordingRunner([[[.text("trg"), "t", .text(sql)]], []]) + let trigger = PluginTriggerInfo(name: "trg", table: "t", timing: "AFTER", event: "INSERT", statement: "") + + #expect(try await SQLiteCatalogReads.triggerDDL(trigger, run: runner.run) == sql) + #expect(await runner.sent == [SQLiteTemporarySchema.triggerListQuery(table: "t", schema: nil)]) + + let missing = PluginTriggerInfo(name: "other", table: "t", timing: "AFTER", event: "INSERT", statement: "") + await #expect(throws: PluginObjectSourceError.self) { + try await SQLiteCatalogReads.triggerDDL(missing, run: runner.run) + } + } + + @Test("A trigger drop names the attached database it lives in") + func dropTriggerStatementIsQualified() { + #expect(SQLiteCatalogReads.dropTriggerStatement(name: "trg", schema: "auth") == "DROP TRIGGER IF EXISTS `auth`.`trg`") + #expect(SQLiteCatalogReads.dropTriggerStatement(name: "trg", schema: nil) == "DROP TRIGGER IF EXISTS `trg`") + } + + @Test("The driver turns foreign key checks off and on with the shared statements") + func foreignKeyStatementsAreShared() { + #expect(SQLiteCatalogReads.foreignKeyDisableStatements == ["PRAGMA foreign_keys = OFF"]) + #expect(SQLiteCatalogReads.foreignKeyEnableStatements == ["PRAGMA foreign_keys = ON"]) + let driver = SQLitePluginDriver(config: DriverConnectionConfig(host: "", port: 0, username: "", password: "", database: ":memory:")) + #expect(driver.foreignKeyDisableStatements() == SQLiteCatalogReads.foreignKeyDisableStatements) + #expect(driver.foreignKeyEnableStatements() == SQLiteCatalogReads.foreignKeyEnableStatements) + #expect(driver.tableDDLIncludesForeignKeys) + #expect(driver.generateDropTriggerSQL(name: "trg", table: "t", schema: "auth") == "DROP TRIGGER IF EXISTS `auth`.`trg`") + } +} diff --git a/apple/Plugins/SQLiteDriverTests/SQLiteNulTextTests.swift b/apple/Plugins/SQLiteDriverTests/SQLiteNulTextTests.swift index 9fea368cb2..2bcad83756 100644 --- a/apple/Plugins/SQLiteDriverTests/SQLiteNulTextTests.swift +++ b/apple/Plugins/SQLiteDriverTests/SQLiteNulTextTests.swift @@ -80,4 +80,18 @@ struct SQLiteNulTextTests { let after = try await driver.execute(query: "SELECT id, body FROM notes ORDER BY id") #expect(texts(after.rows) == [["k", "other"], [Self.key, "x\u{0}y"]]) } + + @Test("A literal the driver escapes keeps a NUL, so a SQL dump restores the text whole") + func escapedLiteralKeepsNul() async throws { + let (driver, url) = try await connectedDriver() + defer { + driver.disconnect() + try? FileManager.default.removeItem(at: url) + } + let literal = driver.escapeStringLiteral(Self.value) + #expect(literal == "a'||char(0)||'b") + _ = try await driver.execute(query: "INSERT INTO notes VALUES ('k', '\(literal)')") + let result = try await driver.execute(query: "SELECT hex(body) FROM notes WHERE id = 'k'") + #expect(texts(result.rows) == [["610062"]]) + } } diff --git a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index 6b35fd0965..a9fb9a0aee 100644 --- a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -769,6 +769,9 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { func fetchTriggerDefinition(name: String, table: String, schema: String?) async throws -> String? func generateDropTriggerSQL(name: String, table: String, schema: String?) -> String? func generateDropRoutineSQL(name: String, signature: String?, schema: String?, isFunction: Bool) -> String? + // A dump's drop of a table, view or materialized view, which succeeds when the relation is missing. Nil keeps the + // export's own `DROP ... IF EXISTS`. + func generateDropRelationSQL(name: String, kind: PluginExportObjectKind, schema: String?) -> String? var triggerEditUsesReplace: Bool { get } /// Whether an object whose definition this driver reports as `CREATE OR REPLACE` is replaced by running that @@ -1000,6 +1003,7 @@ public extension PluginDatabaseDriver { schema: String?, isFunction: Bool ) -> String? { nil } + func generateDropRelationSQL(name: String, kind: PluginExportObjectKind, schema: String?) -> String? { nil } var triggerEditUsesReplace: Bool { false } var replacesDefinitionsInPlace: Bool { false } var supportsTransactionalDDL: Bool { false } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Routines.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Routines.swift index 680fd3fda5..dc2caa2f23 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Routines.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Routines.swift @@ -55,18 +55,14 @@ extension TeradataPluginDriver { .compactMap { cellText($0, 0) } .joined(separator: "\n") .trimmingCharacters(in: .whitespacesAndNewlines) - /// A procedure created without SPL retention has no stored text, which the server reports - /// as an empty answer rather than an error. + // A procedure created without SPL retention has no stored text, and the server answers empty, not an error. guard !text.isEmpty else { throw PluginObjectSourceError.insufficientPrivilege(routine.name) } return text } - /// False until the whole-schema scope is checked against a live server. This driver has no - /// per-table trigger read, so the protocol default answers with nothing and a comparison has - /// never listed its triggers; opting in here would change what is compared rather than only - /// how fast it is read. + // Unchecked on a live server: with no per-table read, opting in would change what Compare lists, not its speed. var providesBulkTriggerFetch: Bool { false } func fetchAllTriggers(schema: String?) async throws -> [PluginTriggerInfo] { @@ -84,6 +80,10 @@ extension TeradataPluginDriver { return definition } + func generateDropTriggerSQL(name: String, table: String, schema: String?) -> String? { + TeradataObjectQueries.dropTriggerStatement(name: name, database: routineDatabase(schema)) + } + func teradataTriggerList(schema: String?, table: String?) async throws -> [PluginTriggerInfo] { guard let database = routineDatabase(schema) else { return [] } let query = TeradataObjectQueries.triggerList(database: database, table: table) diff --git a/apple/Plugins/XLSXExportPlugin/XLSXExportModels.swift b/apple/Plugins/XLSXExportPlugin/XLSXExportModels.swift index e8242b3650..d68d54e402 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXExportModels.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXExportModels.swift @@ -5,7 +5,7 @@ import Foundation -public struct XLSXExportOptions: Equatable, Codable { +nonisolated public struct XLSXExportOptions: Equatable, Codable { public var includeHeaderRow: Bool = true public var convertNullToEmpty: Bool = true diff --git a/apple/Plugins/XLSXExportPlugin/XLSXExportOptionsView.swift b/apple/Plugins/XLSXExportPlugin/XLSXExportOptionsView.swift index 39aafdb7bf..abe0e6dade 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXExportOptionsView.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXExportOptionsView.swift @@ -19,3 +19,10 @@ struct XLSXExportOptionsView: View { .font(.system(size: 13)) } } + +extension XLSXExportPlugin { + @MainActor + func settingsView() -> AnyView? { + AnyView(XLSXExportOptionsView(plugin: self)) + } +} diff --git a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift index 6ed1863eac..d2212c8d27 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift @@ -5,10 +5,9 @@ import Combine import Foundation -import SwiftUI import TableProPluginKit -final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugin, @unchecked Sendable { +nonisolated final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugin, @unchecked Sendable { static let pluginName = "XLSX Export" static let pluginVersion = "1.0.0" static let pluginDescription = "Export data to Excel format" @@ -20,17 +19,14 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug typealias Settings = XLSXExportOptions static let settingsStorageId = "xlsx" - @Published var settings = XLSXExportOptions() { + // A nonisolated class cannot hold an @Published property, so the change is announced by hand. + var settings = XLSXExportOptions() { + willSet { objectWillChange.send() } didSet { saveSettings() } } required init() { loadSettings() } - @MainActor - func settingsView() -> AnyView? { - AnyView(XLSXExportOptionsView(plugin: self)) - } - func resetSettingsToDefaults() { settings = XLSXExportOptions() } @@ -106,6 +102,8 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug } } + // The stream's own checks end with its last element, and the batch left over can be 4,999 rows. + try progress.checkCancellation() if !rowBatch.isEmpty { try writeBatch() } @@ -123,7 +121,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug } } - try await writer.write(to: destination) + try await writer.write(to: destination, progress: progress) progress.finalizeTable() diff --git a/apple/Plugins/XLSXExportPlugin/XLSXSheetRows.swift b/apple/Plugins/XLSXExportPlugin/XLSXSheetRows.swift index 2e3827dbe0..76a128a3ae 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXSheetRows.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXSheetRows.swift @@ -7,7 +7,7 @@ import Foundation // Splits incoming rows across sheets of at most `limit` rows, the header included. A batch can be a whole in-memory // result, so one overflow can need several new sheets. -struct XLSXSheetRows { +nonisolated struct XLSXSheetRows { struct Chunk: Equatable { let rows: Range let startsNewSheet: Bool diff --git a/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift b/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift index ca8d0fe4a6..64cb9a00e3 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift @@ -11,7 +11,7 @@ import zlib // Each sheet's XML goes to its own file as rows arrive, and `write(to:)` copies those files into a // stored ZIP, so no sheet is ever held in memory whole. Strings are inline (`t="inlineStr"`), so // there is no shared string table to grow with the data either. -final class XLSXWriter { +nonisolated final class XLSXWriter { static let defaultFlushThreshold = 1 << 20 static let partsDirectoryPrefix = "TablePro-XLSX-" @@ -209,15 +209,22 @@ final class XLSXWriter { ) } - func write(to url: URL) async throws { + func write(to url: URL, progress: PluginExportProgress? = nil) async throws { if let part = sheets.last, part.isOpen { try closeSheet(part) } let entries = archiveEntries() let maximumSize = maximumArchiveSize - try await Task.detached(priority: .userInitiated) { - try ZipBuilder.write(entries, to: url, maximumSize: maximumSize) - }.value + let archive = Task.detached(priority: .userInitiated) { + var meter = ZipBuilder.Meter(progress: progress, totalBytes: entries.reduce(0) { $0 + $1.size }) + try ZipBuilder.write(entries, to: url, maximumSize: maximumSize, meter: &meter) + } + // A detached task inherits no cancellation, so the caller's is handed on to the copy. + try await withTaskCancellationHandler { + try await archive.value + } onCancel: { + archive.cancel() + } } private func archiveEntries() -> [ZipFileEntry] { @@ -328,10 +335,11 @@ final class XLSXWriter { switch cell { case .string(let value): + let space = Self.keepsOuterWhitespace(value) ? " xml:space=\"preserve\"" : "" if isHeader { - data.appendUTF8("") + data.appendUTF8("") } else { - data.appendUTF8("") + data.appendUTF8("") } data.appendXMLEscaped(value) data.appendUTF8("") @@ -347,6 +355,11 @@ final class XLSXWriter { data.appendUTF8("") } + // Excel trims a ``'s outer whitespace without it; XlsxWriter's `_xml_inline_string` adds it only in this case. + static func keepsOuterWhitespace(_ text: String) -> Bool { + text.first?.isWhitespace == true || text.last?.isWhitespace == true + } + // MARK: - XML Generation private func contentTypesXML() -> Data { @@ -463,7 +476,7 @@ final class XLSXWriter { // MARK: - Data XML Helpers -private extension Data { +nonisolated private extension Data { mutating func appendUTF8(_ string: String) { string.utf8.withContiguousStorageIfAvailable { buffer in if let baseAddress = buffer.baseAddress { @@ -500,7 +513,7 @@ private extension Data { // MARK: - Sheet Part -private final class XLSXSheetPart { +nonisolated private final class XLSXSheetPart { let name: String let fileURL: URL var pending = Data() @@ -546,7 +559,7 @@ private final class XLSXSheetPart { // MARK: - ZIP File Builder // A stored (uncompressed) ZIP, written entry by entry. A sheet's bytes are copied from its file. -private struct ZipFileEntry { +nonisolated private struct ZipFileEntry { enum Contents { case data(Data) case file(URL) @@ -572,7 +585,7 @@ private struct ZipFileEntry { } } -private enum ZipBuilder { +nonisolated private enum ZipBuilder { enum ZipError: LocalizedError { case fileTooLarge case tooManySheets @@ -589,11 +602,44 @@ private enum ZipBuilder { private static let copyChunkSize = 1 << 20 + // The copy reads no rows, so it checks Stop and reports its bytes itself: iOS expires a background task that stalls. + struct Meter { + static let reportInterval = 4 << 20 + + let progress: PluginExportProgress? + let totalBytes: Int + private var writtenBytes = 0 + private var reportedBytes = 0 + + init(progress: PluginExportProgress?, totalBytes: Int) { + self.progress = progress + self.totalBytes = totalBytes + } + + func checkCancellation() throws { + if let progress { + try progress.checkCancellation() + } else { + try Task.checkCancellation() + } + } + + mutating func didWrite(_ count: Int) { + writtenBytes += count + guard let progress, writtenBytes - reportedBytes >= Self.reportInterval else { return } + reportedBytes = writtenBytes + progress.setStatus(String( + format: String(localized: "Writing the workbook: %1$@ of %2$@"), + ByteCountFormatter.string(fromByteCount: Int64(writtenBytes), countStyle: .file), + ByteCountFormatter.string(fromByteCount: Int64(totalBytes), countStyle: .file))) + } + } + static func localHeaderSize(path: String) -> Int { 30 + path.utf8.count } - static func write(_ entries: [ZipFileEntry], to destination: URL, maximumSize: Int) throws { + static func write(_ entries: [ZipFileEntry], to destination: URL, maximumSize: Int, meter: inout Meter) throws { var offsets: [Int] = [] offsets.reserveCapacity(entries.count) var offset = 0 @@ -610,12 +656,14 @@ private enum ZipBuilder { guard entries.count <= Int(UInt16.max) else { throw ZipError.tooManySheets } + try meter.checkCancellation() let (handle, temporaryURL) = try PluginExportUtilities.beginAtomicWrite(for: destination) var isOpen = true do { var centralDirectory = Data() for (entry, entryOffset) in zip(entries, offsets) { + try meter.checkCancellation() let pathData = Data(entry.path.utf8) var header = Data() @@ -632,7 +680,7 @@ private enum ZipBuilder { header.appendUInt16(0) header.append(pathData) try handle.write(contentsOf: header) - try writeContents(of: entry, to: handle) + try writeContents(of: entry, to: handle, meter: &meter) centralDirectory.append(contentsOf: [0x50, 0x4B, 0x01, 0x02]) centralDirectory.appendUInt16(20) @@ -668,6 +716,7 @@ private enum ZipBuilder { try handle.write(contentsOf: end) isOpen = false try handle.close() + try meter.checkCancellation() try PluginExportUtilities.commitAtomicWrite(from: temporaryURL, to: destination) } catch { if isOpen { @@ -678,19 +727,22 @@ private enum ZipBuilder { } } - private static func writeContents(of entry: ZipFileEntry, to handle: FileHandle) throws { + private static func writeContents(of entry: ZipFileEntry, to handle: FileHandle, meter: inout Meter) throws { switch entry.contents { case .data(let data): try handle.write(contentsOf: data) + meter.didWrite(data.count) case .file(let url): let reader = try FileHandle(forReadingFrom: url) defer { try? reader.close() } var copied = 0 while true { + try meter.checkCancellation() let chunk = try autoreleasepool { try reader.read(upToCount: copyChunkSize) } ?? Data() guard !chunk.isEmpty else { break } try handle.write(contentsOf: chunk) copied += chunk.count + meter.didWrite(chunk.count) } // The header already promised this size and CRC. guard copied == entry.size else { @@ -709,7 +761,7 @@ private enum ZipBuilder { // MARK: - Data Extensions for ZIP -private extension Data { +nonisolated private extension Data { mutating func appendUInt16(_ value: UInt16) { var val = value.littleEndian Swift.withUnsafeBytes(of: &val) { append(contentsOf: $0) } diff --git a/apple/Plugins/XLSXImportPlugin/XLSXImportOptions.swift b/apple/Plugins/XLSXImportPlugin/XLSXImportOptions.swift index 3821abe81d..afbd9711c4 100644 --- a/apple/Plugins/XLSXImportPlugin/XLSXImportOptions.swift +++ b/apple/Plugins/XLSXImportPlugin/XLSXImportOptions.swift @@ -6,7 +6,7 @@ import Foundation import TableProPluginKit -struct XLSXImportOptions: Equatable, Codable { +nonisolated struct XLSXImportOptions: Equatable, Codable { var hasHeaderRow: Bool = true var trimWhitespace: Bool = false var emptyAsNull: Bool = true diff --git a/apple/Plugins/XLSXImportPlugin/XLSXImportOptionsView.swift b/apple/Plugins/XLSXImportPlugin/XLSXImportOptionsView.swift index 84df672a5c..925854edd8 100644 --- a/apple/Plugins/XLSXImportPlugin/XLSXImportOptionsView.swift +++ b/apple/Plugins/XLSXImportPlugin/XLSXImportOptionsView.swift @@ -37,3 +37,10 @@ struct XLSXImportOptionsView: View { .font(.system(size: 13)) } } + +extension XLSXImportPlugin { + @MainActor + func settingsView() -> AnyView? { + AnyView(XLSXImportOptionsView(plugin: self)) + } +} diff --git a/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift b/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift index d74bafbfba..dd8f3303e4 100644 --- a/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift +++ b/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift @@ -6,11 +6,10 @@ import Combine import Foundation import os -import SwiftUI import TableProPluginKit import TableProTabularIO -final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugin, @unchecked Sendable { +nonisolated final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugin, @unchecked Sendable { static let pluginName = "XLSX Import" static let pluginVersion = "1.0.0" static let pluginDescription = "Import data from Excel workbooks" @@ -29,7 +28,9 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug // The whole sheet is already in memory, so this bounds only the type guess for the mapping sheet. private static let detectionSampleRows = 200 - @Published var settings = XLSXImportOptions() { + // A nonisolated class cannot hold an @Published property, so the change is announced by hand. + var settings = XLSXImportOptions() { + willSet { objectWillChange.send() } didSet { saveSettings() } } @@ -49,11 +50,6 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug ) } - @MainActor - func settingsView() -> AnyView? { - AnyView(XLSXImportOptionsView(plugin: self)) - } - func resetSettingsToDefaults() { settings = XLSXImportOptions() } @@ -61,7 +57,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug func detectSourceFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { let sheet = try XLSXImportSheet.read(at: url, hasHeaderRow: settings.hasHeaderRow, isCancelled: { Task.isCancelled }) guard let header = sheet.header else { return [] } - let sample = sheet.rows(0 ..< Self.detectionSampleRows).map(\.values) + let sample = try sheet.rows(0 ..< Self.detectionSampleRows).map(\.values) return header.enumerated().map { index, name in let values = sample.compactMap { $0[safe: index] ?? nil } return PluginImportField( @@ -104,7 +100,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug ) { guard cursor < sheet.rowCount else { return nil } let end = min(cursor + Self.batchSize, sheet.rowCount) - let rows = sheet.rows(cursor ..< end) + let rows = try sheet.rows(cursor ..< end) cursor = end let filled = rows.filter { $0.values.contains { $0 != nil } } if filled.count < rows.count { @@ -156,7 +152,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug } } -private extension Array { +nonisolated private extension Array { subscript(safe index: Int) -> Element? { indices.contains(index) ? self[index] : nil } diff --git a/apple/Plugins/XLSXImportPlugin/XLSXImportSheet.swift b/apple/Plugins/XLSXImportPlugin/XLSXImportSheet.swift index 6cbd595b09..5be62c31c0 100644 --- a/apple/Plugins/XLSXImportPlugin/XLSXImportSheet.swift +++ b/apple/Plugins/XLSXImportPlugin/XLSXImportSheet.swift @@ -7,7 +7,7 @@ import Foundation import TableProPluginKit import TableProTabularIO -struct XLSXImportSheet { +nonisolated struct XLSXImportSheet { enum ReadError: LocalizedError { case noWorksheet @@ -19,12 +19,18 @@ struct XLSXImportSheet { } } + // Excel stores at most 32,767 characters in a cell and Google Sheets 50,000; 1 MiB of UTF-8 holds either. + static let cellByteLimit = 1 << 20 + + // One read is the header, a batch or the detection sample. Capped at what the reader buffers for one workbook part. + static let readByteLimit = ZipArchive.bufferedEntryLimit + let header: [String]? private let source: XLSXSheetSource private let columns: [Int] private let dataRows: [Int] - init(source: XLSXSheetSource, hasHeaderRow: Bool) { + init(source: XLSXSheetSource, hasHeaderRow: Bool) throws { let columns = Array(0.. [String] { @@ -63,15 +69,16 @@ struct XLSXImportSheet { var rowCount: Int { dataRows.count } - func rows(_ range: Range) -> [(line: Int, values: [String?])] { - Self.rows(dataRows[range.clamped(to: dataRows.indices)], of: source, columns: columns) + func rows(_ range: Range) throws -> [(line: Int, values: [String?])] { + try Self.rows(dataRows[range.clamped(to: dataRows.indices)], of: source, columns: columns) } private static func rows( _ picked: some Collection, of source: XLSXSheetSource, columns: [Int] - ) -> [(line: Int, values: [String?])] { + ) throws -> [(line: Int, values: [String?])] { + try checkTextSize(of: picked, in: source, columns: columns) var result: [(line: Int, values: [String?])] = [] result.reserveCapacity(picked.count) source.scan(columns: columns, rows: picked, absentKind: .missing) { row, cells in @@ -81,6 +88,42 @@ struct XLSXImportSheet { return result } + // Measured before any cell becomes a String: one shared string can back every cell, so a small file can expand to gigabytes. + private static func checkTextSize( + of picked: some Collection, + in source: XLSXSheetSource, + columns: [Int] + ) throws { + var total = 0 + var failure: PluginImportError? + source.scan(columns: columns, rows: picked, absentKind: .missing) { row, cells in + for index in 0.. String { + ByteCountFormatter.string(fromByteCount: Int64(bytes), countStyle: .binary) + } + private static func values(of cells: TabularRowCells) -> [String?] { (0.. String? { - switch SqlDialect.from(databaseTypeId: databaseType.rawValue) { - case .postgres, .sqlite: - return "INSERT INTO \(qualifiedTable) DEFAULT VALUES" - case .mysql: - return "INSERT INTO \(qualifiedTable) () VALUES ()" - default: - break - } - switch databaseType { - case .mssql: - return "INSERT INTO \(qualifiedTable) DEFAULT VALUES" - case .databend, .oracle: - guard let firstWritableColumn else { return nil } - return "INSERT INTO \(qualifiedTable) (\(firstWritableColumn)) VALUES (DEFAULT)" - default: - return nil - } - } -} diff --git a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift index 6cec2f9225..b93f2b5c6c 100644 --- a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift +++ b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift @@ -1,11 +1,3 @@ -// -// SQLStatementGenerator.swift -// TablePro -// -// Generates parameterized SQL statements (INSERT, UPDATE, DELETE) from tracked changes. -// Uses prepared statements instead of string escaping to prevent SQL injection. -// - import Foundation import os import TableProPluginKit @@ -25,6 +17,16 @@ struct ParameterizedStatement: @unchecked Sendable { } } +extension ParameterizedStatement { + init(_ statement: SQLImportStatement) { + self.init( + sql: statement.sql, + parameters: statement.parameters.map(\.asAny), + parameterTypeNames: statement.parameterTypeNames + ) + } +} + // The rows it should touch hold the server to a count; the changes it writes tell a written change from a dropped one. struct AttributedStatement: @unchecked Sendable { let statement: ParameterizedStatement @@ -36,33 +38,26 @@ struct AttributedStatement: @unchecked Sendable { var rowCount: Int { rowIDs.count } } -/// Generates SQL statements from data changes struct SQLStatementGenerator { private static let logger = Logger(subsystem: "com.TablePro", category: "SQLStatementGenerator") let tableName: String - /// Written into every statement when set, so a generator can address a table outside whatever - /// schema the connection happens to be on. A grid edit leaves it nil and keeps the unqualified - /// name it has always produced; a copy between two databases sets it, because the table it - /// writes is not the one the driver is pointed at. + // Nil for a grid edit, which keeps the unqualified name; set by a copy between databases, whose target is not the + // database the driver is pointed at. let schemaName: String? let columns: [String] let primaryKeyColumns: [String] - /// Server-computed columns. They reject any written value, so they are - /// dropped from every INSERT and UPDATE this generator produces. + // Server-computed columns reject any written value, so every INSERT and UPDATE leaves them out. let generatedColumns: Set let rowMatchPolicy: RowMatchPolicy let databaseType: DatabaseType let parameterStyle: ParameterStyle - /// Set only by a restore that writes a row back with the identity it had: PostgreSQL refuses an explicit value - /// for a `GENERATED ALWAYS` column unless the INSERT says so. + // Set only by a restore that writes a row back with the identity it had: PostgreSQL refuses an explicit value for a + // `GENERATED ALWAYS` column unless the INSERT says so. let insertOverridesSystemValue: Bool - private let columnTypeNames: [String: String] - // BIT columns of a family that reads them as a number. Text bound to one is stored and compared as its bytes. - private let numberBitColumns: Set + // The import statements, and the binding every statement here goes through. + let importStatements: SQLImportStatements private let quoteIdentifierFn: (String) -> String - private let rowUpdateFn: ((_ table: String, _ assignments: String, _ predicate: String) -> String?)? - private let rowDeleteFn: ((_ table: String, _ predicate: String?) -> String?)? init( tableName: String, @@ -80,32 +75,42 @@ struct SQLStatementGenerator { rowUpdateStatement: ((_ table: String, _ assignments: String, _ predicate: String) -> String?)? = nil, rowDeleteStatement: ((_ table: String, _ predicate: String?) -> String?)? = nil ) throws { + let quote: (String) -> String + if let quoteIdentifier { + quote = quoteIdentifier + } else { + let resolvedDialect = try resolveSQLDialect(for: databaseType, explicit: dialect) + quote = quoteIdentifierFromDialect(resolvedDialect, databaseType: databaseType) + } + let explicitSchema = SchemaQualifiedName.explicitSchema(schemaName, databaseType: databaseType) + let style = parameterStyle ?? Self.defaultParameterStyle(for: databaseType) self.tableName = tableName - self.rowUpdateFn = rowUpdateStatement - self.rowDeleteFn = rowDeleteStatement - self.schemaName = SchemaQualifiedName.explicitSchema(schemaName, databaseType: databaseType) + self.schemaName = explicitSchema self.columns = columns self.primaryKeyColumns = primaryKeyColumns self.generatedColumns = generatedColumns self.rowMatchPolicy = rowMatchPolicy self.databaseType = databaseType - self.parameterStyle = parameterStyle ?? Self.defaultParameterStyle(for: databaseType) + self.parameterStyle = style self.insertOverridesSystemValue = insertOverridesSystemValue - self.columnTypeNames = columnTypeNames - self.numberBitColumns = Self.numberBitColumns(in: columnTypeNames, databaseType: databaseType) - if let quoteIdentifier { - self.quoteIdentifierFn = quoteIdentifier - } else { - let resolvedDialect = try resolveSQLDialect(for: databaseType, explicit: dialect) - self.quoteIdentifierFn = quoteIdentifierFromDialect(resolvedDialect, databaseType: databaseType) - } + self.quoteIdentifierFn = quote + self.importStatements = SQLImportStatements( + databaseTypeId: databaseType.rawValue, + qualifiedTableName: SchemaQualifiedName.render( + name: tableName, schema: explicitSchema, databaseType: databaseType, quote: quote + ), + parameterStyle: style, + quoteIdentifier: quote, + columnTypeNames: columnTypeNames, + numberBitColumns: Self.numberBitColumns(in: columnTypeNames, databaseType: databaseType), + rowUpdateStatement: rowUpdateStatement, + rowDeleteStatement: rowDeleteStatement + ) } - /// The table as every statement spells it. + // The table as every statement spells it. var qualifiedTableName: String { - SchemaQualifiedName.render( - name: tableName, schema: schemaName, databaseType: databaseType, quote: quoteIdentifierFn - ) + importStatements.qualifiedTableName } private static func defaultParameterStyle(for databaseType: DatabaseType) -> ParameterStyle { @@ -114,13 +119,6 @@ struct SQLStatementGenerator { // MARK: - Public API - /// Generate all parameterized SQL statements from changes - /// - Parameters: - /// - changes: Array of row changes to process - /// - insertedRowData: Lazy storage for inserted row values - /// - deletedRowIDs: Rows still marked as deleted - /// - insertedRowIDs: Rows still marked as inserted - /// - Returns: Array of parameterized SQL statements func generateStatements( from changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]], @@ -182,15 +180,6 @@ struct SQLStatementGenerator { return statements } - private func placeholder(at index: Int) -> String { - switch parameterStyle { - case .dollar: - return "$\(index + 1)" - case .questionMark: - return "?" - } - } - private static func numberBitColumns(in typeNames: [String: String], databaseType: DatabaseType) -> Set { let family = SQLTypeFamily.of(databaseType) guard family.readsBitsAsDecimal else { return [] } @@ -201,69 +190,18 @@ struct SQLStatementGenerator { return Set(bitColumns.keys) } - private struct Bindings { - var values: [Any?] = [] - var typeNames: [String?] = [] - - var count: Int { values.count } - - mutating func append(_ value: Any?, typeName: String?) { - values.append(value) - typeNames.append(typeName) - } - } + private typealias Bindings = SQLImportStatements.Bindings private func statement(_ sql: String, _ bindings: Bindings) -> ParameterizedStatement { - ParameterizedStatement( - sql: sql, - parameters: bindings.values, - parameterTypeNames: bindings.typeNames.contains { $0 != nil } ? bindings.typeNames : nil - ) + ParameterizedStatement(importStatements.statement(sql, bindings)) } - private func typeName(of column: String?) -> String? { - column.flatMap { columnTypeNames[$0] } - } - - // Blank text goes as '', which the server stores as 0: cast, strict mode refuses it as a truncated integer, and a - // space bound as it is would be its byte, 32. private func bind(_ value: PluginCellValue, toColumn column: String?, into parameters: inout Bindings) -> String { - guard let column, numberBitColumns.contains(column), case .text(let text) = value else { - parameters.append(value.asAny, typeName: typeName(of: column)) - return placeholder(at: parameters.count - 1) - } - guard !text.trimmingCharacters(in: .whitespaces).isEmpty else { - parameters.append("", typeName: typeName(of: column)) - return placeholder(at: parameters.count - 1) - } - // The value is the CAST's operand, not the column's. - parameters.append(Self.decimalBitText(text), typeName: nil) - return "CAST(\(placeholder(at: parameters.count - 1)) AS UNSIGNED)" + importStatements.bind(value, toColumn: column, into: ¶meters) } - // b'101' and 0b101 are SQL, not values, and a boolean word is no number: cast as text, each is 0 or an error. static func decimalBitText(_ text: String) -> String { - let trimmed = text.trimmingCharacters(in: .whitespaces) - switch PluginSQLLiteral.booleanSynonym(for: trimmed) { - case .isTrue: return "1" - case .isFalse: return "0" - case nil: break - @unknown default: break - } - switch trimmed.lowercased() { - case "t": return "1" - case "f": return "0" - default: break - } - let digits: Substring - if trimmed.count > 3, trimmed.hasPrefix("b'") || trimmed.hasPrefix("B'"), trimmed.hasSuffix("'") { - digits = trimmed.dropFirst(2).dropLast() - } else if trimmed.hasPrefix("0b") { - digits = trimmed.dropFirst(2) - } else { - return text - } - return UInt64(digits, radix: 2).map { String($0) } ?? text + SQLImportStatements.decimalBitText(text) } // MARK: - INSERT Generation @@ -277,8 +215,7 @@ struct SQLStatementGenerator { return generateInsertSQLFromCellChanges(for: change) } - /// A cell is written as SQL only when its change says so. Text that merely reads like a function, typed or - /// stored, is bound like any other value. + // A cell is written as SQL only when its change says so. Text that merely reads like a function is bound. private func generateInsertSQLFromStoredData(values: [PluginCellValue], expressionColumns: Set) -> ParameterizedStatement? { @@ -320,64 +257,20 @@ struct SQLStatementGenerator { allDefaultsInsertStatement(firstColumn: columns.first { !generatedColumns.contains($0) }) } - // For a generator built without the table's columns, the caller names the one Oracle needs. func allDefaultsInsertStatement(firstColumn: String?) -> ParameterizedStatement? { - AllDefaultsInsert.sql( - into: qualifiedTableName, databaseType: databaseType, firstWritableColumn: firstColumn.map(quoteIdentifierFn) - ).map { ParameterizedStatement(sql: $0, parameters: []) } + importStatements.allDefaultsInsertStatement(firstColumn: firstColumn).map(ParameterizedStatement.init) } - func insertStatement(columns insertColumns: [String], values: [PluginCellValue]) - -> ParameterizedStatement? - { - guard !insertColumns.isEmpty, insertColumns.count == values.count else { return nil } - - var bindParameters = Bindings() - let columnList = insertColumns.map(quoteIdentifierFn).joined(separator: ", ") - let placeholders = zip(insertColumns, values).map { column, value in - bind(value, toColumn: column, into: &bindParameters) - }.joined(separator: ", ") - - let sql = - "INSERT INTO \(qualifiedTableName) (\(columnList)) VALUES (\(placeholders))" - - return statement(sql, bindParameters) + func insertStatement(columns insertColumns: [String], values: [PluginCellValue]) -> ParameterizedStatement? { + importStatements.insertStatement(columns: insertColumns, values: values).map(ParameterizedStatement.init) } - func insertStatement(columns insertColumns: [String], rows: [[PluginCellValue]]) - -> ParameterizedStatement? - { - guard !insertColumns.isEmpty, !rows.isEmpty, - rows.allSatisfy({ $0.count == insertColumns.count }) else { return nil } - - var bindParameters = Bindings() - let columnList = insertColumns.map(quoteIdentifierFn).joined(separator: ", ") - let rowTuples = rows.map { values -> String in - let placeholders = zip(insertColumns, values).map { column, value in - bind(value, toColumn: column, into: &bindParameters) - }.joined(separator: ", ") - return "(\(placeholders))" - }.joined(separator: ", ") - - let sql = - "INSERT INTO \(qualifiedTableName) (\(columnList)) VALUES \(rowTuples)" - - return statement(sql, bindParameters) + func insertStatement(columns insertColumns: [String], rows: [[PluginCellValue]]) -> ParameterizedStatement? { + importStatements.insertStatement(columns: insertColumns, rows: rows).map(ParameterizedStatement.init) } - // The server compares, so its casting and collation decide a match, as they do for the UPDATE that follows. func keyMatchCountStatement(keyColumns: [String], keys: [[PluginCellValue]]) -> ParameterizedStatement? { - guard !keyColumns.isEmpty, !keys.isEmpty, keys.allSatisfy({ $0.count == keyColumns.count }) else { return nil } - - var bindParameters = Bindings() - let selects = keys.enumerated().map { position, values -> String in - let conditions = zip(keyColumns, values).map { column, value -> String in - "\(quoteIdentifierFn(column)) = \(bind(value, toColumn: column, into: &bindParameters))" - }.joined(separator: " AND ") - return "SELECT \(position), COUNT(*) FROM \(qualifiedTableName) WHERE \(conditions)" - } - - return statement(selects.joined(separator: " UNION ALL "), bindParameters) + importStatements.keyMatchCountStatement(keyColumns: keyColumns, keys: keys).map(ParameterizedStatement.init) } func keyedUpdateStatement( @@ -386,32 +279,17 @@ struct SQLStatementGenerator { keyColumns: [String], keyValues: [PluginCellValue] ) -> ParameterizedStatement? { - guard !setColumns.isEmpty, setColumns.count == values.count, - !keyColumns.isEmpty, keyColumns.count == keyValues.count else { return nil } - - var bindParameters = Bindings() - func bound(_ column: String, _ value: PluginCellValue) -> String { - "\(quoteIdentifierFn(column)) = \(bind(value, toColumn: column, into: &bindParameters))" - } - let assignments = zip(setColumns, values).map(bound).joined(separator: ", ") - let conditions = zip(keyColumns, keyValues).map(bound).joined(separator: " AND ") - let sql = rowUpdateFn?(qualifiedTableName, assignments, conditions) - ?? "UPDATE \(qualifiedTableName) SET \(assignments) WHERE \(conditions)" - - return statement(sql, bindParameters) + importStatements.keyedUpdateStatement( + columns: setColumns, values: values, keyColumns: keyColumns, keyValues: keyValues + ).map(ParameterizedStatement.init) } var maxBindParameters: Int { - switch databaseType { - case .sqlite, .libsql, .turso: 32_766 - case .mssql: 2_100 - case .cloudflareD1: 100 - default: 65_535 - } + importStatements.maxBindParameters } func deleteAllRowsStatement() -> String { - rowDeleteFn?(qualifiedTableName, nil) ?? "DELETE FROM \(qualifiedTableName)" + importStatements.deleteAllRowsStatement() } private func generateInsertSQLFromCellChanges(for change: RowChange) -> ParameterizedStatement? @@ -537,10 +415,9 @@ struct SQLStatementGenerator { rowMatchPolicy.textColumns.contains(column) ? nil : column } - /// The conditions that pick one row. SQLite keeps the number 5 and the text '5' apart in a column with no - /// affinity, and the grid read either one as `5`, so a number the grid read matches both, held by CAST to the - /// text SQLite reads it back as so a REAL 5.0 is not matched too; a typed column converts either literal to its - /// own type, so the form is exact there too. When both rows exist the statement itself refuses them. + // SQLite keeps the number 5 and the text '5' apart in a column with no affinity, and the grid read either as `5`, + // so a number the grid read matches both, held by a CAST to text so a REAL 5.0 is not matched too. When both rows + // exist the statement itself refuses them. private func rowCondition(_ terms: [RowMatchTerm], parameters: inout Bindings) -> String { let clause = conditions(terms, parameters: ¶meters) guard terms.contains(where: { matchesEitherWay($0.value) }) else { return clause } @@ -554,11 +431,11 @@ struct SQLStatementGenerator { guard matchesEitherWay(value), case .text(let text) = value else { return "\(term.matched) = \(bind(value, toColumn: term.column, into: ¶meters))" } - parameters.append(value.asAny, typeName: typeName(of: term.column)) - let bound = placeholder(at: parameters.count - 1) + parameters.append(value, typeName: importStatements.typeName(of: term.column)) + let bound = importStatements.placeholder(at: parameters.count - 1) // Compared with the column's text, not the column. - parameters.append(value.asAny, typeName: nil) - let read = placeholder(at: parameters.count - 1) + parameters.append(value, typeName: nil) + let read = importStatements.placeholder(at: parameters.count - 1) return "(\(term.matched) IN (\(text), \(bound)) AND CAST(\(term.matched) AS TEXT) = \(read))" }.joined(separator: " AND ") } diff --git a/apple/TablePro/Core/ChangeTracking/SQLWriteBatchBudget.swift b/apple/TablePro/Core/ChangeTracking/SQLWriteBatchBudget.swift deleted file mode 100644 index 60fffb9e70..0000000000 --- a/apple/TablePro/Core/ChangeTracking/SQLWriteBatchBudget.swift +++ /dev/null @@ -1,155 +0,0 @@ -// -// SQLWriteBatchBudget.swift -// TablePro -// - -import Foundation -import TableProPluginKit - -/// How much one parameterized `INSERT` may carry, in the two units that can reject it. -/// -/// A bind-parameter count is not a size. `executeParameterized` reaches MySQL as -/// `mysql_stmt_prepare` plus `mysql_stmt_execute`, one packet each, and a packet over -/// `max_allowed_packet` answers `ERROR 1153: Got a packet bigger than 'max_allowed_packet' bytes`, -/// or `ERROR 2006: Server has gone away` once it is large enough. So a batch bounded by parameters -/// alone is unbounded in bytes: 500 rows of three 1 MB values sit well inside a three-column -/// table's 21,845-row parameter ceiling and go out as one 500 MB statement the server refuses, -/// which fails the first batch and leaves nothing written at all. (#2533) -/// -/// `maximumBytes` is the mebibyte `mysqldump` settles on for `net_buffer_length` and the SQL export -/// defaults to: under every `max_allowed_packet` a release has shipped with, and far above a batch -/// of ordinary rows, so nothing narrow writes more statements than before. Not a user preference -/// here, because this statement goes to a server the app is connected to rather than into a file. -internal struct SQLWriteBatchBudget: Sendable, Equatable { - /// However narrow the table is: 65,535 one-column rows in one statement parse slowly on every - /// engine, cannot be cancelled part-way, and save nothing measurable past a thousand. - internal static let maximumRows = 1_000 - internal static let maximumBytes = 1_048_576 - - internal let maxRows: Int - internal let maxBytes: Int - - /// Whether the target's driver sends a value as a bind parameter or renders it into the SQL. - /// Databend has no binary protocol here: `MariaDBPluginConnection` routes it through - /// `DatabendLiteral.inline`, so a value crosses as a literal, binary doubles as hex and text - /// grows by whatever escaping it needs. Charging it the parameter rate made the cap no bound at - /// all: two 400,000-byte values are 800,024 bytes of parameters but over 1.6 MiB of SQL. - internal let rendersValuesAsLiterals: Bool - - internal init( - maxRows: Int, - maxBytes: Int = SQLWriteBatchBudget.maximumBytes, - rendersValuesAsLiterals: Bool = false - ) { - self.maxRows = max(1, maxRows) - self.maxBytes = max(1, maxBytes) - self.rendersValuesAsLiterals = rendersValuesAsLiterals - } - - // ClickHouse writes the values of a statement too long for the URL into it as escaped literals, binary as hex (#3777). - internal static func rendersValuesAsLiterals(_ databaseType: DatabaseType) -> Bool { - databaseType == .databend || databaseType == .clickhouse - } - - /// The engine's bind-parameter ceiling over the row's width, clamped by its multi-row `VALUES` - /// ceiling and by the flat cap. `SQLMultiRowInsert` owns that middle term, so the SQL export - /// reads the same rule and Oracle keeps its one row per statement. - internal init( - columnCount: Int, - generator: SQLStatementGenerator, - maxBytes: Int = SQLWriteBatchBudget.maximumBytes - ) { - self.init( - maxRows: min( - Self.maximumRows, - SQLMultiRowInsert.maximumRowsPerStatement( - forDatabaseTypeId: generator.databaseType.rawValue), - generator.maxBindParameters / max(1, columnCount)), - maxBytes: maxBytes, - rendersValuesAsLiterals: Self.rendersValuesAsLiterals(generator.databaseType)) - } - - /// A batch holding nothing takes the row whatever it weighs: a row cannot be split across two - /// statements, so refusing it would write nothing at all. It goes out alone, which is what - /// `mysqldump` does with a row larger than its own buffer. - internal func hasRoom(for rowBytes: Int, inBatchOf rows: Int, bytes: Int) -> Bool { - guard rows > 0 else { return true } - guard rows < maxRows else { return false } - return bytes + rowBytes <= maxBytes - } - - /// What a row costs on the wire, as an upper bound. - /// - /// Measured against MariaDB 12.3.3 through the binary protocol, the per-value overhead settles - /// at about 6 bytes: 1,500 values of 100 bytes crossed as 158,650 against 150,000 of payload. - /// The documented `COM_STMT_EXECUTE` layout says why, and bounds it: 2 bytes of the type array - /// per parameter, a length-encoded prefix of at most 4 bytes for any value under 16 MB, and one - /// null-bitmap bit. Twelve covers every size including the 9-byte prefix a value could - /// theoretically carry, and over-charging many small values is safe where under-charging a few - /// large ones is exactly what fails. - internal func byteCount(of values: Values) -> Int - where Values.Element == PluginCellValue { - var total = 0 - for value in values { - switch value { - case .null: - total += Self.valueOverheadBytes - case .text(let text): - total += Self.valueOverheadBytes + charged(text) - case .bytes(let data): - total += Self.valueOverheadBytes + charged(data.count) - } - } - return total - } - - /// A driver that inlines is charged double, which is exactly what hex costs a binary value and - /// the worst case for escaping a text one, but for NUL: Databend writes it as the four bytes `\x00`. - private func charged(_ bytes: Int) -> Int { - rendersValuesAsLiterals ? bytes * 2 : bytes - } - - private func charged(_ text: String) -> Int { - guard rendersValuesAsLiterals else { return text.utf8.count } - return text.utf8.reduce(into: text.utf8.count * 2) { total, byte in - if byte == 0 { total += 2 } - } - } - - private static let valueOverheadBytes = 12 -} - -/// Fills one batch at a time under a budget, so no call site holds a byte counter beside its row -/// array and has to keep the two in step. The same shape as `SQLExportStatementAccumulator` and the -/// same rule: a batch is closed *before* the row that would cross the budget, never after, so what -/// is sent stays under the limit. -internal struct SQLWriteBatchFiller { - private let budget: SQLWriteBatchBudget - private var rows: [Row] = [] - private var bytes = 0 - - internal init(budget: SQLWriteBatchBudget) { - self.budget = budget - } - - /// Hands back the batch this row closed, if it closed one. The row itself is always held, never - /// handed back inside a batch it did not fit. - internal mutating func append(_ row: Row, bytes rowBytes: Int) -> [Row]? { - var completed: [Row]? - if !budget.hasRoom(for: rowBytes, inBatchOf: rows.count, bytes: bytes) { - completed = take() - } - rows.append(row) - bytes += rowBytes - return completed - } - - /// Closes whatever is held, and answers nil when nothing is. - internal mutating func take() -> [Row]? { - guard !rows.isEmpty else { return nil } - let batch = rows - rows.removeAll(keepingCapacity: true) - bytes = 0 - return batch - } -} diff --git a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift index 6306730dd1..fba13fb820 100644 --- a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift +++ b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift @@ -16,10 +16,8 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable private static let logger = Logger(subsystem: "com.TablePro", category: "ExportDataSourceAdapter") - /// The same capability the sidebar's drop prompt reads, so the engines whose dumps carry a - /// `CASCADE` are exactly the engines that offer the user a Cascade checkbox. Resolved once at - /// construction, on the main actor, because the registry lives there and this is asked for from - /// the export plugin's own thread. + // The sidebar drop prompt's capability, so a dump carries `CASCADE` exactly where the user is offered one. Read + // at construction on the main actor, where the registry lives; the export plugin asks from its own thread. let supportsCascadeDrop: Bool let lexicalFeatures: SQLLexicalFeatures private let implicitSchemaName: String? @@ -101,8 +99,8 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable return streamLeadingRows(query: query, table: table) } - /// An engine that caps its rows answers a statement with no LIMIT with a smaller default of its - /// own, so every read here states a limit, and a limit past the ceiling is lowered to it. + // An engine that caps its rows answers a statement with no LIMIT with a smaller default of its own, so every read + // states a limit, lowered to the ceiling. private func leadingRowsQuery( columns: String, from source: String, @@ -124,8 +122,7 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable requested.map(pagination.clampedRowCount) ?? pagination.maximumRows } - /// Streams through the adapter rather than the plugin, so the statement text is validated the - /// way every other statement is: a row scope carries a filter the user typed. + // Through the adapter, so the text is validated like every other statement: a row scope carries a typed filter. private func streamLeadingRows( query: String, table: String @@ -186,15 +183,10 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable guard let context = driver.readOnlyRunContext(safeModeIsReadOnly: safeModeIsReadOnly) else { return mapToPluginResult(try await driver.execute(query: query)) } - let keyword = QueryClassifier.leadingKeyword(of: query, grammar: dbType.lexicalGrammar) - guard !Self.transactionKeywords.contains(keyword) else { - return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) - } + guard !ExportSnapshotStatements.isTransactionStatement(query) else { return ExportSnapshotStatements.noResult } return mapToPluginResult(try await driver.execute(query: query, context: context)) } - private static let transactionKeywords: Set = ["BEGIN", "START", "COMMIT", "END", "ROLLBACK"] - func quoteIdentifier(_ identifier: String) -> String { driver.quoteIdentifier(identifier) } @@ -291,9 +283,8 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable // MARK: - Object DDL - /// A driver addresses a routine, trigger or type through the info object it handed out, which - /// carries an opaque identity the export item cannot reproduce. So the list is fetched once per - /// database and the item is matched back onto its own info object rather than a rebuilt one. + // A driver addresses a routine, trigger or type through the info object it handed out, whose identity the export + // item cannot rebuild, so the list is read once per database and the item matched back onto it. func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { guard let pluginDriver else { return try await driver.fetchTableDDL(table: object.name) @@ -305,11 +296,8 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable case .view: return try await pluginDriver.fetchViewDefinition(view: object.name, schema: schema) case .materializedView: - /// No fallback to `fetchTableDDL`. It succeeds on a materialized view on several - /// engines and returns a `CREATE TABLE`, so the dump carried a `DROP MATERIALIZED - /// VIEW` followed by a table definition, reported success, and restored an empty - /// ordinary table where the view had been. A driver that cannot produce the definition - /// now says so and the export names the object it could not write. + // No fallback to `fetchTableDDL`: on several engines it answers a materialized view with a `CREATE TABLE`, + // and the dump restored an empty table where the view had been. return try await pluginDriver.fetchViewDefinition(view: object.name, schema: schema) case .routine: let routine = try await matchedRoutine(object, schema: schema) @@ -343,8 +331,7 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable } } - /// Matched to its info object the way `fetchObjectDDL` matches it, so the settings wrap the definition the dump - /// writes. + // Matched the way `fetchObjectDDL` matches, so the settings wrap the definition the dump writes. func fetchCreationScope(_ object: PluginExportTable) async throws -> PluginStatementScope? { guard let pluginDriver else { return nil } let schema = exportSchema(for: object.databaseName) @@ -363,9 +350,15 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable } } + // The kind is part of the match: MySQL keeps functions and procedures apart, so `f()` can name one of each. private func matchedRoutine(_ object: PluginExportTable, schema: String?) async throws -> PluginRoutineInfo { + let kind = Self.routineKind(of: object) guard let routine = try await cachedRoutines(schema: schema, databaseName: object.databaseName) - .first(where: { $0.name == object.name && ($0.argumentSignature ?? "") == (object.identity ?? "") }) + .first(where: { candidate in + candidate.name == object.name + && (candidate.argumentSignature ?? "") == (object.identity ?? "") + && (kind == nil || candidate.kind == kind) + }) else { throw PluginObjectSourceError.unsupported(object.name) } @@ -400,9 +393,8 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable try await driver.closeSessionScope(scope) } - /// The engine renders its own GRANT text, because only the driver knows how it spells a - /// grantee and a privilege target. `principal` is the name and `host` the MySQL-style host - /// part, which is what separates two principals that share a name. + // Only the driver knows how it spells a grantee and a privilege target. `host` is the MySQL-style host part that + // separates two principals of one name. func fetchGrantStatements(principal: String, host: String?) async throws -> [String] { guard let management = pluginDriver as? any PluginPrincipalManagement else { return [] } let ref = PluginPrincipalRef(name: principal, host: host) @@ -413,8 +405,8 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable ) ?? [] } - /// Through the same owner Compare & Sync saves its scripts with, so a dump ends an Oracle unit on a `/` line and - /// puts a MySQL routine in a `DELIMITER` block exactly as a saved sync script does. + // The owner Compare & Sync saves its scripts with, so a dump ends an Oracle unit on a `/` line and wraps a MySQL + // routine in a `DELIMITER` block as a saved sync script does. func scriptText(for ddl: String) -> String { SQLScriptText( databaseType: dbType, @@ -422,28 +414,35 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable ).scriptText(forDriverText: ddl) } - /// `tableType` carries the routine's own kind for a `.routine`, because `DROP FUNCTION` and - /// `DROP PROCEDURE` are different statements on every engine that has both and MySQL has no - /// `DROP ROUTINE` to fall back on. func dropStatement(for object: PluginExportTable) -> String? { guard let pluginDriver else { return nil } let schema = exportSchema(for: object.databaseName) switch object.kind { + case .table, .view, .materializedView: + return pluginDriver.generateDropRelationSQL(name: object.name, kind: object.kind, schema: schema) case .trigger: guard let parent = object.parentTable else { return nil } return pluginDriver.generateDropTriggerSQL(name: object.name, table: parent, schema: schema) case .routine: + // No guess without the kind: PostgreSQL refuses `DROP FUNCTION` on a procedure. + guard let kind = Self.routineKind(of: object) else { return nil } return pluginDriver.generateDropRoutineSQL( name: object.name, signature: object.identity, schema: schema, - isFunction: object.tableType.lowercased() != "procedure" + isFunction: kind == .function ) default: return nil } } + // A routine's `tableType` is its `PluginRoutineKind` raw value; nil when the caller passed the export kind instead. + static func routineKind(of object: PluginExportTable) -> PluginRoutineKind? { + guard object.kind == .routine else { return nil } + return PluginRoutineKind(rawValue: object.tableType.lowercased()) + } + private func cachedEvents(schema: String?, databaseName: String) async throws -> [PluginEventInfo] { try await objectCache.events(forDatabase: databaseName) { [pluginDriver] in try await pluginDriver?.fetchEvents(schema: schema) ?? [] @@ -476,13 +475,9 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable // MARK: - Helpers - /// The export tree names every group after a schema on a schema-aware engine and after a - /// database everywhere else. Either way the name is the container the driver has to read in, - /// which is what `schema:` means to a driver with no schema layer of its own: withholding it - /// there left a MySQL dump reading its DDL and column metadata from whichever database the - /// connection happened to be on while `streamRows` qualified the rows by the name the export - /// actually named. An empty name means the engine's implicit schema where it has one, and the - /// driver's own container everywhere else. + // A group's name is the container the driver reads in, which is what `schema:` means to a driver without schemas: + // withheld, a MySQL dump read its DDL from whichever database the connection was on. An empty name is the engine's + // implicit schema where it has one, else the driver's own container. func exportSchema(for databaseName: String) -> String? { guard let pluginDriver else { return nil } guard !databaseName.isEmpty else { diff --git a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift index adb9b50c60..4ce287d2dc 100644 --- a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift +++ b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift @@ -1,11 +1,7 @@ -// -// ImportDataSinkAdapter.swift -// TablePro -// - import Foundation import os import TableProPluginKit +import TableProSQLFile import TableProSQLGrammar internal enum ImportTransactionOutcome: Equatable, Sendable { @@ -25,16 +21,11 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { private let importControl: ImportSinkRun? private let driver: DatabaseDriver private let databaseType: DatabaseType - private let exactMapping: [String: String] - private let foldedMapping: [String: String] - private let sourceFields: Set - private let sourceFieldsPerFoldedName: [String: Int] private let rowGenerator: SQLStatementGenerator? private let typedRowGenerator = OSAllocatedUnfairLock(uncheckedState: nil) private let targetValues = OSAllocatedUnfairLock(initialState: nil) private let binaryText: ImportBinaryText? - private let rowWriting: ImportRowWriting - private let tally: OSAllocatedUnfairLock + private let rowWriter: ImportRowWriter private let immutableColumns = OSAllocatedUnfairLock?>(initialState: nil) private let isCancelled: @Sendable () -> Bool @@ -90,9 +81,13 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { } self.commitMode = commitMode self.isCancelled = isCancelled - self.rowWriting = rowWriting self.binaryText = binaryText - self.tally = OSAllocatedUnfairLock(initialState: ImportRowTally(mode: rowWriting.mode)) + self.rowWriter = ImportRowWriter( + databaseTypeId: databaseType.rawValue, + columnMapping: columnMapping, + sourceFields: sourceFields, + rowWriting: rowWriting + ) self.joinsTransaction = joinsTransaction self.driver = driver self.databaseType = databaseType @@ -100,11 +95,6 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { self.databaseTypeId = databaseType.rawValue self.targetTable = targetTable self.targetSchema = targetSchema - self.exactMapping = columnMapping - self.sourceFields = sourceFields - self.sourceFieldsPerFoldedName = Dictionary(sourceFields.map { ($0.lowercased(), 1) }, uniquingKeysWith: +) - self.foldedMapping = Dictionary(grouping: columnMapping, by: { $0.key.lowercased() }) - .compactMapValues { $0.count == 1 ? $0.first?.value : nil } if let targetTable, Self.canWriteRows(into: databaseType) { self.rowGenerator = Self.makeRowGenerator( table: targetTable, schema: targetSchema, databaseType: databaseType, driver: driver @@ -179,12 +169,48 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { return values } + private func writingTarget() async throws -> ImportRowWriter.Target? { + guard let targetTable, let generator = try await writingGenerator() else { return nil } + return ImportRowWriter.Target( + tableName: targetTable, statements: generator.importStatements, values: try await targetColumnValues() + ) + } + + private var writerHost: ImportRowWriter.Host { + ImportRowWriter.Host( + execute: { statement, purpose in try await self.runWriterStatement(statement, purpose) }, + columnsUpdateCannotSet: { try await self.columnsUpdateCannotSet() }, + isCancelled: isCancelled + ) + } + + private func runWriterStatement( + _ statement: SQLImportStatement, + _ purpose: ImportRowWriter.Purpose + ) async throws -> [[String?]] { + let parameterized = ParameterizedStatement(statement) + let driver = self.driver + func sendStatement() async throws -> QueryResult { + try await driver.executeParameterized( + query: parameterized.sql, parameters: parameterized.parameters, context: parameterized.context + ) + } + switch purpose { + case .matchProbe: + let result = try await importedData { try await sendStatement() } + return result.rows.map { $0.map(\.asText) } + case .write(let rows): + _ = try await importedData(rows: rows, writes: true) { try await sendStatement() } + return [] + } + } + func execute(statement: String) async throws { try await execute(statement: statement, line: 1) } var rowTally: ImportRowTally { - tally.withLock { $0 } + rowWriter.tally } var transactionOutcome: ImportTransactionOutcome { @@ -266,105 +292,20 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { guard let targetTable else { throw PluginImportError.importFailed("No target table configured for row import") } - guard let rowGenerator = try await writingGenerator() else { + guard let target = try await writingTarget() else { throw PluginImportError.importFailed("Could not resolve SQL dialect for \(targetTable)") } - - let row = try await writableRows([values])[0] - guard !rowWriting.mode.matchesExistingRows else { - try await writeMatching([row], generator: rowGenerator) - return - } - guard !row.columns.isEmpty else { - try await insertDefaultRows(1, generator: rowGenerator) - return - } - guard let statement = rowGenerator.insertStatement(columns: row.columns, values: row.values) else { - throw PluginImportError.importFailed( - String(format: String(localized: "Could not build an INSERT for %@"), targetTable) - ) - } - - try await importedData(rows: 1, writes: true) { - _ = try await self.driver.executeParameterized( - query: statement.sql, parameters: statement.parameters, context: statement.context - ) - self.tally.withLock { $0.inserted += 1 } - } + try await rowWriter.insertRow(values, into: target, host: writerHost) } func insertRows(_ rows: [[String: PluginCellValue]]) async throws { guard targetTable != nil else { throw PluginImportError.importFailed("No target table configured for row import") } - guard let rowGenerator = try await writingGenerator() else { + guard let target = try await writingTarget() else { throw PluginImportError.importFailed("Could not resolve SQL dialect for row import") } - - let writable = try await writableRows(rows) - - if rowWriting.mode.matchesExistingRows { - try await writeMatching(writable, generator: rowGenerator) - } else { - try await insertGrouped(writable, generator: rowGenerator) - } - } - - private struct MappedRow { - let columns: [String] - let values: [PluginCellValue] - } - - // A row with no field, such as JSON's `{}` for an all-NULL row, is a row of defaults; a row whose fields all - // missed the mapping is an error. - private func writableRows(_ rows: [[String: PluginCellValue]]) async throws -> [MappedRow] { - let target = try await targetColumnValues() - return try rows.map { values in - let (columns, mappedValues) = mappedColumnsAndValues(values) - if columns.isEmpty, !values.isEmpty { - throw PluginImportError.importFailed(Self.unmappedRowMessage) - } - return MappedRow(columns: columns, values: try target.values(mappedValues, columns: columns)) - } - } - - private func insertGrouped(_ rows: [MappedRow], generator: SQLStatementGenerator) async throws { - var index = 0 - while index < rows.count { - let columns = rows[index].columns - var groupValues: [[PluginCellValue]] = [rows[index].values] - var next = index + 1 - while next < rows.count, rows[next].columns == columns { - groupValues.append(rows[next].values) - next += 1 - } - index = next - - if columns.isEmpty { - try await insertDefaultRows(groupValues.count, generator: generator) - } else { - try await insertGroup(groupValues, columns: columns, generator: generator) - } - } - } - - private func insertDefaultRows(_ count: Int, generator: SQLStatementGenerator) async throws { - let firstColumn = try await targetColumnValues().firstWritableColumn - guard let statement = generator.allDefaultsInsertStatement(firstColumn: firstColumn) else { - throw PluginImportError.importFailed(String( - format: String(localized: "This row has no values, and %@ has no way to insert a row of only defaults."), - databaseType.rawValue - )) - } - for _ in 0.. Void = { _ in } - ) async throws { - let budget = SQLWriteBatchBudget(columnCount: columns.count, generator: generator) - var filler = SQLWriteBatchFiller<[PluginCellValue]>(budget: budget) - for row in groupValues { - guard let batch = filler.append(row, bytes: budget.byteCount(of: row)) else { - continue - } - try await write(batch, columns: columns, generator: generator) - onWrite(batch.count) - } - guard let batch = filler.take() else { return } - try await write(batch, columns: columns, generator: generator) - onWrite(batch.count) - } - - private func write( - _ batch: [[PluginCellValue]], - columns: [String], - generator: SQLStatementGenerator - ) async throws { - guard !isCancelled() else { throw PluginImportCancellationError() } - guard let statement = generator.insertStatement(columns: columns, rows: batch) else { - throw PluginImportError.importFailed( - String(localized: "Could not build an INSERT for the mapped columns") - ) - } - try await importedData(rows: batch.count, writes: true) { - _ = try await self.driver.executeParameterized( - query: statement.sql, parameters: statement.parameters, context: statement.context - ) - self.tally.withLock { $0.inserted += batch.count } - } - } - - // MARK: - Matching existing rows - - private struct KeyedRow { - let row: MappedRow - let key: [PluginCellValue] - } - - // Not the engines' upsert forms: they refuse a NOT NULL column the file leaves out even when the row exists. - private func writeMatching(_ rows: [MappedRow], generator: SQLStatementGenerator) async throws { - var run: [KeyedRow] = [] - var keysInRun = Set<[String]>() - for row in rows { - let key = try keyValues(of: row) - // A repeated key has to see the write of its first occurrence, so the run ends before it. - let folded = key.contains(where: \.isNull) ? nil : key.map(Self.foldedKeyValue) - if let folded, keysInRun.contains(folded) { - try await writeMatchedRun(run, generator: generator) - run.removeAll(keepingCapacity: true) - keysInRun.removeAll(keepingCapacity: true) - } - if let folded { keysInRun.insert(folded) } - run.append(KeyedRow(row: row, key: key)) - } - try await writeMatchedRun(run, generator: generator) + guard !columns.isEmpty, !rows.isEmpty, let target = try await writingTarget() else { return } + try await rowWriter.insertRows(columns: columns, values: rows, into: target, host: writerHost, onWrite: onWrite) } - // Folds more than a collation or a numeric comparison does: two keys the database takes as one must not share a run. static func foldedKeyValue(_ value: PluginCellValue) -> String { - switch value { - case .null: - return "null" - case .bytes(let data): - return "bytes:" + data.base64EncodedString() - case .text(let text): - let folded = text.trimmingCharacters(in: .whitespacesAndNewlines) - .folding(options: [.caseInsensitive, .diacriticInsensitive, .widthInsensitive], locale: nil) - // A Double, not a Decimal: a REAL column holds 0.1 and 0.100000000000000001 as one value. - guard folded.contains(where: { $0.isASCII && $0.isNumber }), - folded.allSatisfy({ numberCharacters.contains($0) }), - let number = Double(folded) else { - return "text:" + folded - } - return "number:\(number == 0 ? 0 : number)" - } - } - - private static let numberCharacters = Set("0123456789+-.e") - - private func keyValues(of row: MappedRow) throws -> [PluginCellValue] { - try rowWriting.keyColumns.map { column in - guard let index = row.columns.firstIndex(of: column) else { - throw PluginImportError.importFailed(String( - format: String(localized: "This row has no value for the key column %@, so it cannot be matched."), - column - )) - } - return row.values[index] - } - } - - private func writeMatchedRun(_ run: [KeyedRow], generator: SQLStatementGenerator) async throws { - guard !run.isEmpty else { return } - let counts = try await matchCounts(of: run.map(\.key), generator: generator) - if let ambiguous = zip(run, counts).first(where: { $0.1 > 1 }) { - throw PluginImportError.importFailed(String( - format: String(localized: "The key %1$@ matched %2$@ rows, so the row was not written."), - keyDescription(ambiguous.0.key), - ambiguous.1.formatted() - )) - } - - // Writes go out in file order, so a new parent row lands before an update that points at it. - var inserts: [MappedRow] = [] - for (entry, count) in zip(run, counts) { - switch (count, rowWriting.mode) { - case (0, .update): - tally.withLock { $0.unmatched += 1 } - case (0, _): - inserts.append(entry.row) - case (_, .update), (_, .upsert): - try await insertGrouped(inserts, generator: generator) - inserts.removeAll(keepingCapacity: true) - try await update(entry, generator: generator) - default: - tally.withLock { $0.existing += 1 } - } - } - try await insertGrouped(inserts, generator: generator) - } - - // A column the engine never lets an UPDATE assign keeps its value, and is written only for an inserted row. - private func update(_ entry: KeyedRow, generator: SQLStatementGenerator) async throws { - let kept = Set(rowWriting.keyColumns).union(try await columnsUpdateCannotSet()) - let assignments = zip(entry.row.columns, entry.row.values).filter { !kept.contains($0.0) } - guard !assignments.isEmpty else { - tally.withLock { $0.existing += 1 } - return - } - guard !isCancelled() else { throw PluginImportCancellationError() } - guard let statement = generator.keyedUpdateStatement( - columns: assignments.map(\.0), - values: assignments.map(\.1), - keyColumns: rowWriting.keyColumns, - keyValues: entry.key - ) else { - throw PluginImportError.importFailed(String(localized: "Could not build an UPDATE for the mapped columns")) - } - try await importedData(rows: 1, writes: true) { - _ = try await self.driver.executeParameterized( - query: statement.sql, parameters: statement.parameters, context: statement.context - ) - self.tally.withLock { $0.updated += 1 } - } + ImportRowWriter.foldedKeyValue(value) } private func columnsUpdateCannotSet() async throws -> Set { @@ -544,95 +335,17 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { return fetched } - private static let keysPerProbe = 100 - - private func matchCounts(of keys: [[PluginCellValue]], generator: SQLStatementGenerator) async throws -> [Int] { - let keyWidth = max(1, rowWriting.keyColumns.count) - let perStatement = max(1, min(Self.keysPerProbe, generator.maxBindParameters / keyWidth)) - var counts: [Int] = [] - counts.reserveCapacity(keys.count) - var start = 0 - while start < keys.count { - let chunk = Array(keys[start.. [Int] { - var counts = [Int?](repeating: nil, count: expected) - for row in result.rows { - guard row.count >= 2, let position = integer(row[0]), counts.indices.contains(position), - let count = integer(row[1]) else { continue } - counts[position] = count - } - let read = counts.compactMap { $0 } - guard read.count == expected else { - throw PluginImportError.importFailed(String(localized: "Could not read which rows are already in the table")) - } - return read - } - - private static func integer(_ value: PluginCellValue) -> Int? { - guard let text = value.asText?.trimmingCharacters(in: .whitespaces) else { return nil } - return Int(text) ?? Double(text).flatMap { $0.rounded() == $0 ? Int(exactly: $0) : nil } - } - - private func keyDescription(_ key: [PluginCellValue]) -> String { - zip(rowWriting.keyColumns, key) - .map { "\($0.0) = \($0.1.isNull ? "NULL" : $0.1.sortKey)" } - .joined(separator: ", ") - } - - private static var matchingRefusesClearingMessage: String { - String(localized: """ - Delete existing rows before import empties the table, so no row would match. Turn it off, or import as \ - Insert. - """) - } - - private static var unmappedRowMessage: String { - String(localized: "No values in this row matched the column mapping") - } - func mappedColumnsAndValues(_ values: [String: PluginCellValue]) -> ([String], [PluginCellValue]) { - var pairs: [(column: String, value: PluginCellValue)] = [] - var matchedNames = Set() - for (field, value) in values { - guard let column = exactMapping[field] else { continue } - pairs.append((column, value)) - matchedNames.insert(field.lowercased()) - } - let unmatched = values.filter { exactMapping[$0.key] == nil && !sourceFields.contains($0.key) } - for (folded, group) in Dictionary(grouping: unmatched, by: { $0.key.lowercased() }) { - guard group.count == 1, let entry = group.first, !matchedNames.contains(folded), - sourceFieldsPerFoldedName[folded, default: 0] <= 1, - let column = foldedMapping[folded] else { continue } - pairs.append((column, entry.value)) - } - pairs.sort { $0.column < $1.column } - return (pairs.map(\.column), pairs.map(\.value)) + rowWriter.mappedColumnsAndValues(values) } func deleteAllRowsFromTargetTable() async throws { guard targetTable != nil, let rowGenerator else { throw PluginImportError.importFailed("No target table configured for row import") } - guard !rowWriting.mode.matchesExistingRows else { - throw PluginImportError.importFailed(Self.matchingRefusesClearingMessage) - } + let statement = try rowWriter.clearingStatement(using: rowGenerator.importStatements) try await importedData(writes: true, deletes: true) { - _ = try await self.driver.execute(query: rowGenerator.deleteAllRowsStatement()) + _ = try await self.driver.execute(query: statement) } } diff --git a/apple/TablePro/Core/Plugins/PluginManager+Registration.swift b/apple/TablePro/Core/Plugins/PluginManager+Registration.swift index 3d53726531..ece0dd7a7d 100644 --- a/apple/TablePro/Core/Plugins/PluginManager+Registration.swift +++ b/apple/TablePro/Core/Plugins/PluginManager+Registration.swift @@ -250,15 +250,9 @@ extension PluginManager { func importPlugins(for databaseType: DatabaseType) -> [any ImportFormatPlugin] { guard supportsImport(for: databaseType) else { return [] } - let typeId = databaseType.rawValue + let writesRows = ImportDataSinkAdapter.canWriteRows(into: databaseType) return allImportPlugins() - .filter { plugin in - let supported = type(of: plugin).supportedDatabaseTypeIds - let excluded = type(of: plugin).excludedDatabaseTypeIds - if !supported.isEmpty && !supported.contains(typeId) { return false } - if excluded.contains(typeId) { return false } - return true - } + .filter { Self.offers(type(of: $0), to: databaseType, writesRows: writesRows) } .sorted { lhs, rhs in let lhsRowBased = type(of: lhs).requiresTargetTable let rhsRowBased = type(of: rhs).requiresTargetTable @@ -267,6 +261,19 @@ extension PluginManager { } } + // A row format writes through the import sink, which has no statement to write without a SQL dialect (MongoDB). + static func offers( + _ format: any ImportFormatPlugin.Type, + to databaseType: DatabaseType, + writesRows: Bool + ) -> Bool { + let typeId = databaseType.rawValue + let supported = format.supportedDatabaseTypeIds + if !supported.isEmpty && !supported.contains(typeId) { return false } + if format.excludedDatabaseTypeIds.contains(typeId) { return false } + return !format.requiresTargetTable || writesRows + } + func importFormatOptions(for databaseType: DatabaseType) -> [ImportFormatOption] { importPlugins(for: databaseType).map { ImportFormatOption( diff --git a/apple/TablePro/Core/Plugins/QueryResultExportDataSource.swift b/apple/TablePro/Core/Plugins/QueryResultExportDataSource.swift index 8247482ecf..4e8c8ca4be 100644 --- a/apple/TablePro/Core/Plugins/QueryResultExportDataSource.swift +++ b/apple/TablePro/Core/Plugins/QueryResultExportDataSource.swift @@ -234,8 +234,12 @@ final class QueryResultExportDataSource: PluginExportDataSource, @unchecked Send "" } + // Held rows have no session to snapshot, so a format's BEGIN or COMMIT passes as a no-op. func execute(query: String) async throws -> PluginQueryResult { - throw ExportError.exportFailed("Execute is not supported for in-memory query result export") + guard ExportSnapshotStatements.isTransactionStatement(query) else { + throw ExportError.exportFailed("Execute is not supported for in-memory query result export") + } + return ExportSnapshotStatements.noResult } func fetchDependentSequences(table: String, databaseName: String) async throws -> [PluginSequenceInfo] { diff --git a/apple/TablePro/Core/Plugins/SqlFileImportSource.swift b/apple/TablePro/Core/Plugins/SqlFileImportSource.swift index 12c2abeef3..bd85d6e30f 100644 --- a/apple/TablePro/Core/Plugins/SqlFileImportSource.swift +++ b/apple/TablePro/Core/Plugins/SqlFileImportSource.swift @@ -6,8 +6,8 @@ import Foundation import os import TableProPluginKit +import TableProSQLFile import TableProSQLGrammar -import TableProUtilities final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "SqlFileImportSource") @@ -69,37 +69,30 @@ final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { let grammar = self.grammar.withLock { $0 } let onRead = readReporter.withLock { $0 } let parsed = parser.parseFile(url: fileURL, encoding: encoding, grammar: grammar, onRead: onRead) - guard !sendsFileBytesUnchanged else { return parsed } - let rewriting = TranscodedDumpStatements(parsed, rewriter: rewriter(grammar: grammar)) - return AsyncThrowingStream(unfolding: { - try await rewriting.next() - }) + guard SQLDumpStatementRewriter.rewrites(fileDecodedFrom: encoding) else { return parsed } + return rewriter(grammar: grammar).rewriting(parsed) } func elements() async throws -> AsyncThrowingStream { let fileURL = try await resolveURL() let grammar = self.grammar.withLock { $0 } let onRead = readReporter.withLock { $0 } - let parsed = parser.parseElements(url: fileURL, encoding: encoding, grammar: grammar, onRead: onRead) - guard !sendsFileBytesUnchanged else { return parsed } - let rewriting = TranscodedDumpElements(parsed, rewriter: rewriter(grammar: grammar)) + var parsed = parser.parseElements(url: fileURL, encoding: encoding, grammar: grammar, onRead: onRead) + if SQLDumpStatementRewriter.rewrites(fileDecodedFrom: encoding) { + parsed = rewriter(grammar: grammar).rewriting(parsed) + } + let elements = PluginElements(parsed) return AsyncThrowingStream(unfolding: { - try await rewriting.next() + try await elements.next() }) } - private var sendsFileBytesUnchanged: Bool { - encoding == .utf8 || encoding == .ascii + private func rewriter(grammar: @escaping @Sendable () -> SQLLexicalGrammar) -> SQLDumpStatementRewriter { + SQLDumpStatementRewriter(encoding: encoding, family: family.dumpFamily, grammar: grammar) } - private func rewriter(grammar: @escaping @Sendable () -> SQLLexicalGrammar) -> TranscodedStatementRewriter { - TranscodedStatementRewriter(encoding: encoding, family: family, grammar: grammar) - } - - /// `ownsDecompressedFile` answers only whether the caller handed over a file to delete. A file - /// this source decompressed itself is always its own to remove, and gating that on the same - /// flag leaked the whole expanded dump every time an import of a `.gz` was retried, because a - /// retry arrives with no caller-supplied file and therefore with the flag off. + // A file this source decompressed is always its own to remove: a retry arrives with the flag off, and gating on it + // leaked the whole expanded dump. func cleanup() { let tempURL = _decompressedURL.withLock { let url = $0 @@ -156,81 +149,34 @@ final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { } } -internal enum SqlFileImportError: LocalizedError, Equatable { - case unrecoverableBinaryLiteral(line: Int, encoding: String) - - var errorDescription: String? { +extension TransactionEngineFamily { + var dumpFamily: SQLDumpFamily { switch self { - case .unrecoverableBinaryLiteral(let line, let encoding): - return String( - format: String(localized: "Line %@ holds binary data that cannot be read back exactly from a %@ file. Export the dump with --hex-blob, then import it again."), - line.formatted(), - encoding - ) + case .mysql: + return .mysql + case .postgres, .redshift, .cockroach: + return .postgres + case .sqlite, .duckdb, .sqlServer, .oracle, .redis, .teradata, .other: + return .other } } } -private final class TranscodedDumpStatements: @unchecked Sendable { - private var statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>.AsyncIterator - private let rewriter: TranscodedStatementRewriter - - init(_ statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>, rewriter: TranscodedStatementRewriter) { - self.statements = statements.makeAsyncIterator() - self.rewriter = rewriter - } - - func next() async throws -> (statement: String, lineNumber: Int)? { - guard let (statement, lineNumber) = try await statements.next() else { return nil } - return (try rewriter.rewritten(statement, at: lineNumber), lineNumber) - } -} - -// COPY rows pass unchanged: the parser hands them out as UTF-8 already. -private final class TranscodedDumpElements: @unchecked Sendable { - private var elements: AsyncThrowingStream.AsyncIterator - private let rewriter: TranscodedStatementRewriter +private final class PluginElements: @unchecked Sendable { + private var elements: AsyncThrowingStream.AsyncIterator - init(_ elements: AsyncThrowingStream, rewriter: TranscodedStatementRewriter) { + init(_ elements: AsyncThrowingStream) { self.elements = elements.makeAsyncIterator() - self.rewriter = rewriter } func next() async throws -> PluginImportElement? { - guard let element = try await elements.next() else { return nil } - guard case let .statement(statement, lineNumber) = element else { return element } - return .statement(try rewriter.rewritten(statement, at: lineNumber), lineNumber: lineNumber) - } -} - -private struct TranscodedStatementRewriter: Sendable { - private static let logger = Logger(subsystem: "com.TablePro", category: "SqlFileImportSource") - - let encoding: String.Encoding - let family: TransactionEngineFamily - let grammar: @Sendable () -> SQLLexicalGrammar - - func rewritten(_ statement: String, at lineNumber: Int) throws -> String { - let grammar = self.grammar() - let literals = try rewrittenLiterals(of: statement, at: lineNumber, grammar: grammar) ?? statement - guard let rewritten = SQLClientEncodingRewriter.rewritten(literals, family: family, grammar: grammar) else { - return literals - } - Self.logger.info("Kept the session's client encoding at line \(lineNumber, privacy: .public), since the import sends UTF-8") - return rewritten - } - - private func rewrittenLiterals(of statement: String, at lineNumber: Int, grammar: SQLLexicalGrammar) throws -> String? { - guard family == .mysql else { return nil } - do { - return try SQLTranscodedLiteralRewriter.rewritten( - statement, - decodedFrom: encoding, - backslashEscapesAreOn: grammar.contains(.backslashEscapesInSingleQuotes), - grammar: grammar - ) - } catch SQLTranscodedLiteralError.unrecoverableBinaryLiteral { - throw SqlFileImportError.unrecoverableBinaryLiteral(line: lineNumber, encoding: String.localizedName(of: encoding)) + switch try await elements.next() { + case let .statement(statement, line)?: + return .statement(statement, lineNumber: line) + case let .copyFromStandardInput(statement, line, readRows)?: + return .copyFromStandardInput(statement: statement, lineNumber: line, rows: PluginCopyData(read: readRows)) + case nil: + return nil } } } diff --git a/apple/TablePro/Core/Services/ColumnTypeClassifier.swift b/apple/TablePro/Core/Services/ColumnTypeClassifier.swift index becc872c02..e3550023af 100644 --- a/apple/TablePro/Core/Services/ColumnTypeClassifier.swift +++ b/apple/TablePro/Core/Services/ColumnTypeClassifier.swift @@ -1,12 +1,3 @@ -// -// ColumnTypeClassifier.swift -// TablePro -// -// Maps raw database type strings to semantic ColumnType values. -// Handles type wrappers (Nullable, LowCardinality), parameterized types, -// and database-specific conventions across all supported databases. -// - import Foundation struct ColumnTypeClassifier { @@ -211,11 +202,7 @@ struct ColumnTypeClassifier { map[key] = { .json(rawType: $0) } } - // Oracle's `LONG RAW` holds bytes, and no pattern arm below reads the name as binary. - for key in [ - "BLOB", "BYTEA", "BINARY", "VARBINARY", "RAW", "IMAGE", - "TINYBLOB", "MEDIUMBLOB", "LONGBLOB", "LONG RAW" - ] { + for key in ColumnTypeTraits.byteTypeNames { map[key] = { .blob(rawType: $0) } } diff --git a/apple/TablePro/Core/Services/Export/ExportFormatCatalog.swift b/apple/TablePro/Core/Services/Export/ExportFormatCatalog.swift index ca1c45abdc..616819e8e1 100644 --- a/apple/TablePro/Core/Services/Export/ExportFormatCatalog.swift +++ b/apple/TablePro/Core/Services/Export/ExportFormatCatalog.swift @@ -8,11 +8,7 @@ internal struct InstallableExportFormat: Equatable, Sendable { } internal enum ExportFormatCatalog { - internal static let dataDictionaryFormatId = "datadict" - - private static let displayOrder = [ - "csv", "json", "sql", "xlsx", "md", "parquet", "mql", dataDictionaryFormatId - ] + internal static let dataDictionaryFormatId = ExportFormatOffer.dataDictionaryFormatId internal static func installable(in manifest: RegistryManifest?) -> [InstallableExportFormat] { (manifest?.plugins ?? []).flatMap { plugin in @@ -37,53 +33,21 @@ internal enum ExportFormatCatalog { listsDatabaseObjects: Bool, columnsAreSampled: Bool ) -> [any ExportFormatPlugin] { - sorted(plugins.filter { plugin in - let pluginType = type(of: plugin) - if writesStructureOnly(plugin), !listsDatabaseObjects || columnsAreSampled { - return false - } - return accepts( - databaseTypeId: databaseTypeId, - supportedDatabaseTypeIds: pluginType.supportedDatabaseTypeIds, - excludedDatabaseTypeIds: pluginType.excludedDatabaseTypeIds - ) - }) + ExportFormatOffer.offered( + plugins.filter { !writesStructureOnly($0) || (listsDatabaseObjects && !columnsAreSampled) }, + forDatabaseTypeId: databaseTypeId + ) } internal static func writesStructureOnly(_ plugin: any ExportFormatPlugin) -> Bool { !plugin.readsRows(optionValues: plugin.defaultTableOptionValues(), kind: .table) } - internal static func accepts( - databaseTypeId: String, - supportedDatabaseTypeIds: [String], - excludedDatabaseTypeIds: [String] - ) -> Bool { - guard supportedDatabaseTypeIds.isEmpty else { - return supportedDatabaseTypeIds.contains(databaseTypeId) - } - return !excludedDatabaseTypeIds.contains(databaseTypeId) - } - - internal static func sorted(_ plugins: [any ExportFormatPlugin]) -> [any ExportFormatPlugin] { - plugins.sorted { first, second in - let firstRank = rank(of: type(of: first).formatId) - let secondRank = rank(of: type(of: second).formatId) - guard firstRank == secondRank else { return firstRank < secondRank } - return type(of: first).formatDisplayName - .localizedCaseInsensitiveCompare(type(of: second).formatDisplayName) == .orderedAscending - } - } - internal static func description(for plugin: any ExportFormatPlugin) -> String { let pluginType = type(of: plugin) return curatedDescription(for: pluginType.formatId) ?? pluginType.pluginDescription } - private static func rank(of formatId: String) -> Int { - displayOrder.firstIndex(of: formatId) ?? displayOrder.count - } - private static func curatedDescription(for formatId: String) -> String? { switch formatId { case "csv": String(localized: "Comma-separated values. Compatible with Excel and most tools.") diff --git a/apple/TablePro/Core/Services/Export/ExportObjectLoader.swift b/apple/TablePro/Core/Services/Export/ExportObjectLoader.swift index 6b4ecdcff1..ed72cb0d73 100644 --- a/apple/TablePro/Core/Services/Export/ExportObjectLoader.swift +++ b/apple/TablePro/Core/Services/Export/ExportObjectLoader.swift @@ -7,17 +7,10 @@ import Foundation import os import TableProPluginKit -/// Everything one container of the export tree can offer, read from the driver. -/// -/// The export dialog asks per container, and only for the kinds the chosen format says it can -/// write, so picking CSV never pays for a routine list nobody will export. A kind the driver -/// cannot answer comes back empty rather than failing the load: one engine without routines must -/// not cost the user their table list. +// A kind the driver cannot answer comes back empty: one engine without routines must not cost the user their table list. internal enum ExportObjectLoader { private static let logger = Logger(subsystem: "com.TablePro", category: "ExportObjectLoader") - /// The kinds this loader knows how to read. A driver that has no answer for one returns empty, - /// so a kind listed here costs nothing on an engine that lacks it. internal static let loadableKinds: Set = [ .table, .view, .materializedView, .foreignTable, .routine, .trigger, .event, .sequence, .userType, .grant @@ -86,7 +79,8 @@ internal enum ExportObjectLoader { name: routine.name, databaseName: container, kind: .routine, - identity: routine.argumentSignature + identity: routine.argumentSignature, + routineKind: routine.kind ) } } catch { @@ -95,14 +89,13 @@ internal enum ExportObjectLoader { } } - /// Only a driver that answers a schema-wide trigger list is asked. Falling back to a read per - /// table would be one round trip per table just to fill a tree the user may never open. + // Not gated on `providesBulkTriggerFetch`: that is Compare's scope rule, and Oracle answers this list with it false. + // A driver without a schema-wide list answers empty without a read, and none is asked per table. private static func loadTriggers( schema: String?, container: String, driver: any PluginDatabaseDriver ) async -> [ExportObjectItem] { - guard driver.providesBulkTriggerFetch else { return [] } do { return try await driver.fetchAllTriggers(schema: schema).map { trigger in ExportObjectItem( @@ -133,9 +126,7 @@ internal enum ExportObjectLoader { } } - /// Every sequence in the container, including the ones a table already owns. The SQL export - /// emits each name once, so a sequence reached by both this list and `fetchDependentSequences` - /// is written a single time. + // Includes sequences a table owns; the SQL export writes a name reached twice only once. private static func loadSequences( schema: String?, container: String, @@ -166,9 +157,7 @@ internal enum ExportObjectLoader { } } - /// Principals are server-wide rather than per container, so they are listed only under the - /// container the dialog opened on. Listing them under every schema would offer the same GRANT - /// statements several times over. + // Principals are server-wide, so they are listed only under the container the dialog opened on. private static func loadPrincipals( container: String, driver: any PluginDatabaseDriver diff --git a/apple/TablePro/Core/Services/Export/ExportService.swift b/apple/TablePro/Core/Services/Export/ExportService.swift index cbb422ff52..cce1bb82e0 100644 --- a/apple/TablePro/Core/Services/Export/ExportService.swift +++ b/apple/TablePro/Core/Services/Export/ExportService.swift @@ -265,7 +265,8 @@ final class ExportService: ObservableObject { PluginExportTable( name: object.name, databaseName: dataSource.pluginDatabaseName(for: object.databaseName), - tableType: object.kind.rawValue, + // A routine's own kind, which picks DROP FUNCTION or DROP PROCEDURE. + tableType: object.routineKind?.rawValue ?? object.kind.rawValue, optionValues: object.optionValues, schema: dataSource.exportSchema(for: object.databaseName), kind: object.kind, diff --git a/apple/TablePro/Core/Services/Export/NewTableImportPlanner.swift b/apple/TablePro/Core/Services/Export/NewTableImportPlanner.swift deleted file mode 100644 index 639de788c2..0000000000 --- a/apple/TablePro/Core/Services/Export/NewTableImportPlanner.swift +++ /dev/null @@ -1,49 +0,0 @@ -// -// NewTableImportPlanner.swift -// TablePro -// - -import Foundation - -enum NewTableImportPlan: Equatable { - case create - case reuseAfterClearing - case nameTakenWithDifferentColumns -} - -/// What a row import into a new table should do about the table itself. -/// -/// A failed import leaves the table it created behind, so the next attempt finds the name taken. It -/// cannot simply create again, and it cannot simply carry on either: Stop and Commit keeps the batch -/// it managed to write, and so does any run with transaction wrapping turned off, so importing into -/// the table as it stands would write those rows a second time. The table is only ever reused when -/// this sheet created it in this session with exactly the columns being asked for now, which is the -/// one case where clearing it can lose nothing but the sheet's own failed attempt. -/// -/// A table is known by its database and schema as well as its name. Keyed by name alone, a table the -/// sheet made in one database vouched for a same-named table in another, and the retry cleared that -/// one with `DELETE FROM`. -struct NewTableImportPlanner { - private var createStatements: [TableScope: String] = [:] - - func plan(forTable table: TableScope, createTableSQL: String) -> NewTableImportPlan { - guard let previousSQL = createStatements[table] else { - return .create - } - return previousSQL == createTableSQL ? .reuseAfterClearing : .nameTakenWithDifferentColumns - } - - func created(_ table: TableScope) -> Bool { - createStatements[table] != nil - } - - func createdTableNames(in scope: DatabaseScope) -> [String] { - createStatements.keys - .filter { $0 == TableScope(table: $0.table, in: scope) } - .map(\.table) - } - - mutating func recordCreated(_ table: TableScope, createTableSQL: String) { - createStatements[table] = createTableSQL - } -} diff --git a/apple/TablePro/Core/Services/Export/QueryExportOptions.swift b/apple/TablePro/Core/Services/Export/QueryExportOptions.swift deleted file mode 100644 index 0ab1553b68..0000000000 --- a/apple/TablePro/Core/Services/Export/QueryExportOptions.swift +++ /dev/null @@ -1,39 +0,0 @@ -// -// QueryExportOptions.swift -// TablePro -// - -import Foundation -import TableProPluginKit - -/// Which per-object options a query export may ask for. -/// -/// A result set has no schema behind it. `fetchTableDDL` returns an empty string on both query data -/// sources, so asking for structure produces no `CREATE` at all, and leaving drop on beside it -/// wrote `DROP TABLE ` into a dump that then never recreated it. The name is the -/// real table's whenever the export came from a table tab, so the dump destroyed what it claimed to -/// copy. -internal enum QueryExportOptions { - /// The option columns a source with no DDL must not be asked for. - private static let schemaColumnIds: Set = ["structure", "drop"] - - /// Cleared by column id rather than by position. The ids are per format and the positions do - /// not line up: SQL export declares `[structure, drop, data]` while MQL export declares - /// `[drop, indexes, data]`, so clearing index 0 and 1 would turn off MQL's indexes and leave - /// its drop on, which is the opposite of what is wanted. - /// - /// A defaults array that is not the length of the column list is discarded whole rather than - /// read element-wise, because a mismatched array is exactly the misalignment this is here to - /// avoid. `ExportObjectItem.normalized(forOptionColumnCount:defaultOptionValues:)` already - /// treats one that way. - internal static func dataOnly( - columns: [PluginExportOptionColumn], - defaults: [Bool] - ) -> [Bool] { - let aligned = defaults.count == columns.count ? defaults : [] - return columns.enumerated().map { index, column -> Bool in - guard !schemaColumnIds.contains(column.id) else { return false } - return aligned.indices.contains(index) ? aligned[index] : column.defaultValue - } - } -} diff --git a/apple/TablePro/Core/Services/Export/RowImportEligibility.swift b/apple/TablePro/Core/Services/Export/RowImportEligibility.swift index ebdda3e9fc..548620fd4c 100644 --- a/apple/TablePro/Core/Services/Export/RowImportEligibility.swift +++ b/apple/TablePro/Core/Services/Export/RowImportEligibility.swift @@ -9,6 +9,7 @@ import TableProPluginKit internal enum RowImportRefusal: LocalizedError, Equatable { case readOnly(connectionName: String) case importNotSupported(DatabaseType) + case rowsNotWritable(DatabaseType) case formatNotOffered(formatId: String) case sheetAlreadyPresented(connectionName: String) @@ -18,6 +19,8 @@ internal enum RowImportRefusal: LocalizedError, Equatable { return String(format: String(localized: "%@ is read-only, so nothing can be written to it."), connectionName) case .importNotSupported(let databaseType): return String(format: String(localized: "Import is not supported for %@ connections."), databaseType.rawValue) + case .rowsNotWritable(let databaseType): + return String(format: String(localized: "%@ cannot take rows from a file."), databaseType.rawValue) case .formatNotOffered(let formatId): return String( format: String(localized: "%@ files cannot be imported into this connection."), @@ -37,6 +40,20 @@ internal struct ImportFormatLookup { internal var supportsImport: (DatabaseType) -> Bool internal var offeredFormats: (DatabaseType) -> [ImportFormatOption] internal var requiresTargetTable: (String) -> Bool? + internal var writesRows: (DatabaseType) -> Bool + + internal init( + supportsImport: @escaping (DatabaseType) -> Bool, + offeredFormats: @escaping (DatabaseType) -> [ImportFormatOption], + requiresTargetTable: @escaping (String) -> Bool?, + // The sink's own test, so a file is refused before it is read rather than at the first batch. + writesRows: @escaping (DatabaseType) -> Bool = ImportDataSinkAdapter.canWriteRows(into:) + ) { + self.supportsImport = supportsImport + self.offeredFormats = offeredFormats + self.requiresTargetTable = requiresTargetTable + self.writesRows = writesRows + } internal static let live = ImportFormatLookup( supportsImport: { PluginManager.shared.supportsImport(for: $0) }, @@ -63,8 +80,12 @@ internal enum RowImportEligibility { guard lookup.supportsImport(databaseType) else { return .importNotSupported(databaseType) } - guard lookup.offeredFormats(databaseType).contains(where: { $0.id == formatId }), - lookup.requiresTargetTable(formatId) == true else { + let readsRows = lookup.requiresTargetTable(formatId) == true + // Asked before the offer, which leaves a row format out for the same reason and would only say "not offered". + guard !readsRows || lookup.writesRows(databaseType) else { + return .rowsNotWritable(databaseType) + } + guard lookup.offeredFormats(databaseType).contains(where: { $0.id == formatId }), readsRows else { return .formatNotOffered(formatId: formatId) } guard !isPresentingSheet else { diff --git a/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift b/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift index 056d832464..fc84131ccd 100644 --- a/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift +++ b/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift @@ -1,10 +1,3 @@ -// -// BlobFormattingService.swift -// TablePro -// -// Centralized BLOB formatting service for binary data display. -// - import Foundation import TableProPluginKit @@ -74,37 +67,11 @@ final class BlobFormattingService: Sendable { } static func hasHexPrefix(_ text: String) -> Bool { - text.hasPrefix("0x") || text.hasPrefix("0X") + PrefixedHexBytes.hasPrefix(text) } - // The form a copy, a CSV export and an XLSX export write bytes in, read strictly: a copy cut short with `…` - // or text that only starts like hex is not bytes. func bytes(fromPrefixedHex text: String) -> Data? { - guard Self.hasHexPrefix(text) else { return nil } - let digits = text.utf8.dropFirst(2) - guard digits.count.isMultiple(of: 2) else { return nil } - var bytes = [UInt8]() - bytes.reserveCapacity(digits.count / 2) - var high: UInt8? - for character in digits { - guard let nibble = Self.nibble(character) else { return nil } - if let pending = high { - bytes.append(pending << 4 | nibble) - high = nil - } else { - high = nibble - } - } - return Data(bytes) - } - - private static func nibble(_ character: UInt8) -> UInt8? { - switch character { - case UInt8(ascii: "0")...UInt8(ascii: "9"): return character - UInt8(ascii: "0") - case UInt8(ascii: "a")...UInt8(ascii: "f"): return character - UInt8(ascii: "a") + 10 - case UInt8(ascii: "A")...UInt8(ascii: "F"): return character - UInt8(ascii: "A") + 10 - default: return nil - } + PrefixedHexBytes.bytes(from: text) } func requiresFormatting(columnType: ColumnType) -> Bool { diff --git a/apple/TablePro/Core/Transfer/ImportMatchKey+Catalog.swift b/apple/TablePro/Core/Transfer/ImportMatchKey+Catalog.swift new file mode 100644 index 0000000000..2a6cce4a6f --- /dev/null +++ b/apple/TablePro/Core/Transfer/ImportMatchKey+Catalog.swift @@ -0,0 +1,20 @@ +import Foundation + +extension ImportMatchKey { + static func options(columns: [ColumnInfo], indexes: [IndexInfo]) -> [ImportMatchKey] { + options( + columns: columns.map { ImportMatchKey.Column(name: $0.name, isPrimaryKey: $0.isPrimaryKey) }, + indexes: indexes.map { index in + ImportMatchKey.Index( + name: index.name, + columns: index.columns, + isUnique: index.isUnique, + isPrimary: index.isPrimary, + isValid: index.isValid, + isPartial: index.whereClause != nil, + hasExpressions: !(index.expressions ?? []).isEmpty + ) + } + ) + } +} diff --git a/apple/TablePro/Core/Transfer/ImportTargetValues+Classification.swift b/apple/TablePro/Core/Transfer/ImportTargetValues+Classification.swift new file mode 100644 index 0000000000..14a120a172 --- /dev/null +++ b/apple/TablePro/Core/Transfer/ImportTargetValues+Classification.swift @@ -0,0 +1,19 @@ +import Foundation +import TableProPluginKit + +extension ImportTargetValues { + init( + columns: [(name: String, typeName: String, isGenerated: Bool)], + databaseType: DatabaseType, + binaryText: ImportBinaryText? + ) { + // MySQL strict mode refuses `true` for an integer flag, SQL Server refuses `yes` for a bit, and SQLite keeps either as text. + let writesNumericBooleans = (try? resolveSQLDialect(for: databaseType))?.booleanLiteralStyle == .numeric + self.init( + columns: columns, + databaseTypeId: databaseType.rawValue, + writesNumericBooleans: writesNumericBooleans, + binaryText: binaryText + ) + } +} diff --git a/apple/TablePro/Core/Transfer/ImportTypeMapper+DatabaseType.swift b/apple/TablePro/Core/Transfer/ImportTypeMapper+DatabaseType.swift new file mode 100644 index 0000000000..4efefcd549 --- /dev/null +++ b/apple/TablePro/Core/Transfer/ImportTypeMapper+DatabaseType.swift @@ -0,0 +1,18 @@ +import Foundation +import TableProPluginKit + +extension ImportTypeMapper { + static func sqlType( + for type: PluginImportFieldType, + databaseType: DatabaseType, + serverVersion: String? = nil + ) -> String { + sqlType(for: type, databaseTypeId: databaseType.rawValue, serverVersion: serverVersion) + } +} + +extension PostgreSQLServerVersion { + static func jsonColumnType(for databaseType: DatabaseType, serverVersion: String?) -> JSONColumnType { + ImportTypeMapper.postgresJSONColumnType(databaseTypeId: databaseType.rawValue, serverVersion: serverVersion) + } +} diff --git a/apple/TablePro/Core/Transfer/NewTableImportPlanner+TableScope.swift b/apple/TablePro/Core/Transfer/NewTableImportPlanner+TableScope.swift new file mode 100644 index 0000000000..f42fa0134e --- /dev/null +++ b/apple/TablePro/Core/Transfer/NewTableImportPlanner+TableScope.swift @@ -0,0 +1,25 @@ +import Foundation + +extension NewTableImportPlanner.Table { + init(_ scope: TableScope) { + self.init(connectionId: scope.connectionId, database: scope.database, schema: scope.schema, name: scope.table) + } +} + +extension NewTableImportPlanner { + func plan(forTable table: TableScope, createTableSQL: String) -> NewTableImportPlan { + plan(for: NewTableImportPlanner.Table(table), createTableSQL: createTableSQL) + } + + func created(_ table: TableScope) -> Bool { + created(NewTableImportPlanner.Table(table)) + } + + func createdTableNames(in scope: DatabaseScope) -> [String] { + createdTableNames(connectionId: scope.connectionId, database: scope.database, schema: scope.schema) + } + + mutating func recordCreated(_ table: TableScope, createTableSQL: String) { + recordCreated(NewTableImportPlanner.Table(table), createTableSQL: createTableSQL) + } +} diff --git a/apple/TablePro/Core/Transfer/SQLScriptText+DatabaseType.swift b/apple/TablePro/Core/Transfer/SQLScriptText+DatabaseType.swift new file mode 100644 index 0000000000..a1ed659ea9 --- /dev/null +++ b/apple/TablePro/Core/Transfer/SQLScriptText+DatabaseType.swift @@ -0,0 +1,13 @@ +import Foundation +import TableProSQLGrammar + +// The grammar comes from the registry, so a plugin engine TablePro does not curate lexes the way its plugin declares. +extension SQLScriptText { + init(databaseType: DatabaseType) { + self.init(databaseType: databaseType, grammar: databaseType.lexicalGrammar) + } + + init(databaseType: DatabaseType, grammar: SQLLexicalGrammar) { + self.init(databaseTypeId: databaseType.rawValue, grammar: grammar, readings: databaseType.lexicalReadings) + } +} diff --git a/apple/TablePro/Core/Transfer/SQLWriteBatchBudget+Generator.swift b/apple/TablePro/Core/Transfer/SQLWriteBatchBudget+Generator.swift new file mode 100644 index 0000000000..49ac1e54bf --- /dev/null +++ b/apple/TablePro/Core/Transfer/SQLWriteBatchBudget+Generator.swift @@ -0,0 +1,11 @@ +import Foundation + +extension SQLWriteBatchBudget { + init( + columnCount: Int, + generator: SQLStatementGenerator, + maxBytes: Int = SQLWriteBatchBudget.maximumBytes + ) { + self.init(columnCount: columnCount, statements: generator.importStatements, maxBytes: maxBytes) + } +} diff --git a/apple/TablePro/Core/Transfer/Shared/AllDefaultsInsert.swift b/apple/TablePro/Core/Transfer/Shared/AllDefaultsInsert.swift new file mode 100644 index 0000000000..6384949a65 --- /dev/null +++ b/apple/TablePro/Core/Transfer/Shared/AllDefaultsInsert.swift @@ -0,0 +1,27 @@ +import Foundation +import TableProPluginKit + +// A row whose every column the server fills in. Returning nothing dropped such a row from a save that still reported +// success. +nonisolated enum AllDefaultsInsert { + // Nil where the engine has no such spelling, or needs a column to name and the table has none left. + static func sql(into qualifiedTable: String, databaseTypeId: String, firstWritableColumn: String?) -> String? { + switch SqlDialect.from(databaseTypeId: databaseTypeId) { + case .postgres, .sqlite: + return "INSERT INTO \(qualifiedTable) DEFAULT VALUES" + case .mysql: + return "INSERT INTO \(qualifiedTable) () VALUES ()" + default: + break + } + switch databaseTypeId { + case "SQL Server": + return "INSERT INTO \(qualifiedTable) DEFAULT VALUES" + case "Databend", "Oracle": + guard let firstWritableColumn else { return nil } + return "INSERT INTO \(qualifiedTable) (\(firstWritableColumn)) VALUES (DEFAULT)" + default: + return nil + } + } +} diff --git a/apple/TablePro/Core/Transfer/Shared/ColumnTypeTraits.swift b/apple/TablePro/Core/Transfer/Shared/ColumnTypeTraits.swift new file mode 100644 index 0000000000..7a3f3620f3 --- /dev/null +++ b/apple/TablePro/Core/Transfer/Shared/ColumnTypeTraits.swift @@ -0,0 +1,116 @@ +import Foundation +import TableProPluginKit + +// The two readings of a column type name an import needs, made as macOS `ColumnTypeClassifier` makes them for +// `.blob` and `.boolean`. `ColumnTypeTraitsTests` runs both over the same names. +nonisolated enum ColumnTypeTraits { + // Oracle's `LONG RAW` holds bytes, and no pattern reads the name as binary. + static let byteTypeNames: Set = [ + "BLOB", "BYTEA", "BINARY", "VARBINARY", "RAW", "IMAGE", "TINYBLOB", "MEDIUMBLOB", "LONGBLOB", "LONG RAW" + ] + + private static let caseSensitiveBytesTypeName = "BYTES" + + static func holdsBytes(_ typeName: String) -> Bool { + guard let shape = Shape(typeName) else { return false } + if shape.base == caseSensitiveBytesTypeName { return true } + let upper = shape.base.uppercased() + if byteTypeNames.contains(upper) { return true } + // The classifier tries these arms before it looks for BLOB inside a name. + guard upper.contains("BLOB") else { return false } + return !upper.contains("BOOL") && !upper.hasSuffix("SERIAL") && !upper.hasSuffix("INT") + && !upper.hasPrefix("TIME") && !upper.hasSuffix("TEXT") && !upper.hasSuffix("CHAR") + } + + // MySQL's `TINYINT(1)` is a flag on every engine, as the classifier reads it. + static func isBooleanFlag(_ typeName: String, databaseTypeId: String) -> Bool { + guard let shape = Shape(typeName) else { return false } + let upper = shape.base.uppercased() + if upper == "TINYINT", shape.params == "1" { return true } + if upper == "BIT" { return bitIsFlag(shape, databaseTypeId: databaseTypeId) } + return upper.contains("BOOL") + } + + // A PostgreSQL or DuckDB `bit` holds digits like `101`, and a MySQL `BIT(8)` reads back as the number it holds. + // The families are `SQLTypeFamily`'s: its PostgreSQL and MySQL ids are `SqlDialect`'s, and DuckDB is its own. + private static func bitIsFlag(_ shape: Shape, databaseTypeId: String) -> Bool { + let dialect = SqlDialect.from(databaseTypeId: databaseTypeId) + if dialect == .postgres || databaseTypeId == "DuckDB" { + let spelling = shape.suffix.isEmpty ? "BIT" : "BIT " + shape.suffix.uppercased() + let bitStrings: Set = dialect == .postgres ? ["BIT", "VARBIT", "BIT VARYING"] : ["BIT", "BITSTRING"] + if bitStrings.contains(spelling) { return false } + } + if dialect == .mysql, let width = shape.params.flatMap({ Int($0) }), width > 1 { return false } + return true + } + + // The name as the classifier splits it; nil for an array or a composite, which is neither. + private struct Shape { + let base: String + let params: String? + let suffix: String + + init?(_ typeName: String) { + let stripped = Self.strippingAttributes(Self.strippingWrappers(typeName)) + let upper = stripped.uppercased() + if upper.hasPrefix("ARRAY<") || upper.hasPrefix("STRUCT<") || stripped.hasSuffix("[]") { return nil } + if let open = stripped.firstIndex(of: "("), let close = stripped.lastIndex(of: ")"), open < close { + base = String(stripped[.. String { + for prefix in ["Nullable(", "LowCardinality("] where value.hasPrefix(prefix) && value.hasSuffix(")") { + return strippingWrappers(String(value.dropFirst(prefix.count).dropLast())) + } + return value + } + + private static func strippingAttributes(_ value: String) -> String { + var stripped = value + while let attribute = ["UNSIGNED", "SIGNED", "ZEROFILL"].first(where: { stripped.uppercased().hasSuffix(" \($0)") }) { + stripped = String(stripped.dropLast(attribute.count)).trimmingCharacters(in: .whitespaces) + } + return stripped + } + } +} + +nonisolated extension ImportTargetValues.ColumnTraits { + // An engine whose dialect writes `true` and `false` takes the words; one that writes 1 and 0 is sent digits. + init(typeName: String, databaseTypeId: String, writesNumericBooleans: Bool) { + self.init( + holdsBytes: ColumnTypeTraits.holdsBytes(typeName), + takesNumericBooleans: writesNumericBooleans + && ColumnTypeTraits.isBooleanFlag(typeName, databaseTypeId: databaseTypeId) + ) + } +} + +nonisolated extension ImportTargetValues { + // `writesNumericBooleans` is the dialect's `booleanLiteralStyle == .numeric`, which only the host can read. + init( + columns: [(name: String, typeName: String, isGenerated: Bool)], + databaseTypeId: String, + writesNumericBooleans: Bool, + binaryText: ImportBinaryText? + ) { + self.init( + columns: columns, + binaryText: binaryText, + prefixedHex: { PrefixedHex(reading: $0) }, + traits: { typeName in + ColumnTraits( + typeName: typeName, databaseTypeId: databaseTypeId, writesNumericBooleans: writesNumericBooleans + ) + } + ) + } +} diff --git a/apple/TablePro/Core/Transfer/Shared/ExportFormatOffer.swift b/apple/TablePro/Core/Transfer/Shared/ExportFormatOffer.swift new file mode 100644 index 0000000000..e5b17c37c1 --- /dev/null +++ b/apple/TablePro/Core/Transfer/Shared/ExportFormatOffer.swift @@ -0,0 +1,50 @@ +import Foundation +import TableProPluginKit + +// Which export formats an engine is offered, and in what order, for both apps' export sheets. +nonisolated enum ExportFormatOffer { + static let dataDictionaryFormatId = "datadict" + + private static let displayOrder = [ + "csv", "json", "sql", "xlsx", "md", "parquet", "mql", dataDictionaryFormatId + ] + + static func offered(_ plugins: [any ExportFormatPlugin], forDatabaseTypeId databaseTypeId: String) -> [any ExportFormatPlugin] { + sorted(plugins.filter { accepts($0, databaseTypeId: databaseTypeId) }) + } + + static func accepts(_ plugin: any ExportFormatPlugin, databaseTypeId: String) -> Bool { + let pluginType = type(of: plugin) + return accepts( + databaseTypeId: databaseTypeId, + supportedDatabaseTypeIds: pluginType.supportedDatabaseTypeIds, + excludedDatabaseTypeIds: pluginType.excludedDatabaseTypeIds + ) + } + + static func accepts( + databaseTypeId: String, + supportedDatabaseTypeIds: [String], + excludedDatabaseTypeIds: [String] + ) -> Bool { + guard supportedDatabaseTypeIds.isEmpty else { + return supportedDatabaseTypeIds.contains(databaseTypeId) + } + return !excludedDatabaseTypeIds.contains(databaseTypeId) + } + + // A format missing from the curated order, such as a registry one, sorts after it by display name. + static func sorted(_ plugins: [any ExportFormatPlugin]) -> [any ExportFormatPlugin] { + plugins.sorted { first, second in + let firstRank = rank(of: type(of: first).formatId) + let secondRank = rank(of: type(of: second).formatId) + guard firstRank == secondRank else { return firstRank < secondRank } + return type(of: first).formatDisplayName + .localizedCaseInsensitiveCompare(type(of: second).formatDisplayName) == .orderedAscending + } + } + + private static func rank(of formatId: String) -> Int { + displayOrder.firstIndex(of: formatId) ?? displayOrder.count + } +} diff --git a/apple/TablePro/Core/Transfer/Shared/ExportObjectKindLabel.swift b/apple/TablePro/Core/Transfer/Shared/ExportObjectKindLabel.swift new file mode 100644 index 0000000000..86001bb0fc --- /dev/null +++ b/apple/TablePro/Core/Transfer/Shared/ExportObjectKindLabel.swift @@ -0,0 +1,36 @@ +import Foundation +import TableProPluginKit + +// How each object kind names and draws itself in both apps' export lists. One table rather than a switch per call site, +// so a kind added later cannot pick up a different icon in one of them. +nonisolated enum ExportObjectKindLabel { + static func title(for kind: PluginExportObjectKind) -> String { + switch kind { + case .table: return String(localized: "Tables") + case .view: return String(localized: "Views") + case .materializedView: return String(localized: "Materialized Views") + case .foreignTable: return String(localized: "Foreign Tables") + case .sequence: return String(localized: "Sequences") + case .userType: return String(localized: "Types") + case .routine: return String(localized: "Routines") + case .trigger: return String(localized: "Triggers") + case .event: return String(localized: "Events") + case .grant: return String(localized: "Privileges") + @unknown default: return String(localized: "Objects") + } + } + + static func systemImage(for kind: PluginExportObjectKind) -> String { + switch kind { + case .table, .foreignTable: return "tablecells" + case .view, .materializedView: return "eye" + case .sequence: return "number" + case .userType: return "curlybraces" + case .routine: return "function" + case .trigger: return "bolt" + case .event: return "clock" + case .grant: return "key" + @unknown default: return "shippingbox" + } + } +} diff --git a/apple/TablePro/Core/Transfer/Shared/ExportOptionMask.swift b/apple/TablePro/Core/Transfer/Shared/ExportOptionMask.swift new file mode 100644 index 0000000000..20f92d5665 --- /dev/null +++ b/apple/TablePro/Core/Transfer/Shared/ExportOptionMask.swift @@ -0,0 +1,18 @@ +import Foundation +import TableProPluginKit + +// Clears every option a format says a kind does not take, so a routine never carries a Data flag that would make it +// look exportable for a phase it has no rows for. Values that do not line up with the columns are left alone. +nonisolated enum ExportOptionMask { + static func masked( + _ values: [Bool], + columns: [PluginExportOptionColumn], + kind: PluginExportObjectKind, + supports: (String, PluginExportObjectKind) -> Bool + ) -> [Bool] { + guard values.count == columns.count else { return values } + return zip(columns, values).map { column, value in + supports(column.id, kind) ? value : false + } + } +} diff --git a/apple/TablePro/Core/Transfer/Shared/ExportSnapshotStatements.swift b/apple/TablePro/Core/Transfer/Shared/ExportSnapshotStatements.swift new file mode 100644 index 0000000000..076eceda70 --- /dev/null +++ b/apple/TablePro/Core/Transfer/Shared/ExportSnapshotStatements.swift @@ -0,0 +1,30 @@ +import Foundation +import TableProPluginKit + +// The transaction statements a format sends around a consistent snapshot (`BEGIN`, `START TRANSACTION`, `COMMIT`). +// A source that cannot run them, because it holds rows already read or each read takes its own read-only transaction, +// passes them as no-ops. +nonisolated enum ExportSnapshotStatements { + static let transactionKeywords: Set = ["BEGIN", "START", "COMMIT", "END", "ROLLBACK"] + + static let noResult = PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + + static func isTransactionStatement(_ statement: String) -> Bool { + transactionKeywords.contains(leadingKeyword(of: statement)) + } + + static func leadingKeyword(of statement: String) -> String { + var text = Substring(statement) + while true { + text = text.drop { $0.isWhitespace } + if text.hasPrefix("--") { + text = text.drop { $0 != "\n" } + } else if text.hasPrefix("/*"), let end = text.range(of: "*/") { + text = text[end.upperBound...] + } else { + break + } + } + return text.prefix { $0.isLetter }.uppercased() + } +} diff --git a/apple/TablePro/Core/Services/Export/ImportColumnMatcher.swift b/apple/TablePro/Core/Transfer/Shared/ImportColumnMatcher.swift similarity index 83% rename from apple/TablePro/Core/Services/Export/ImportColumnMatcher.swift rename to apple/TablePro/Core/Transfer/Shared/ImportColumnMatcher.swift index 7c9ca833a5..53b5e0284c 100644 --- a/apple/TablePro/Core/Services/Export/ImportColumnMatcher.swift +++ b/apple/TablePro/Core/Transfer/Shared/ImportColumnMatcher.swift @@ -1,12 +1,7 @@ -// -// ImportColumnMatcher.swift -// TablePro -// - import Foundation -/// A row can be unticked while still naming a column, so the two are kept apart. -internal struct ImportFieldChoice: Equatable, Sendable { +// A row can be unticked while still naming a column, so the two are kept apart. +nonisolated struct ImportFieldChoice: Equatable, Sendable { var include: Bool var column: String? @@ -17,9 +12,8 @@ internal struct ImportFieldChoice: Equatable, Sendable { } } -/// Only departures from the name match are stored, so a column the table gains later is still found -/// by name instead of staying skipped. -internal enum ImportMappingOverride: Codable, Equatable, Sendable { +// Only departures from the name match are stored, so a column the table gains later is still found by name. +nonisolated enum ImportMappingOverride: Codable, Equatable, Sendable { case column(String) case skip @@ -41,9 +35,9 @@ internal enum ImportMappingOverride: Codable, Equatable, Sendable { } } -/// Unlike `TableColumnMatcher`, exact matches come before case-insensitive ones and a column goes to -/// one field at most, because the sheet refuses two fields mapped to one column. -internal enum ImportColumnMatcher { +// Unlike `TableColumnMatcher`, exact matches come first and a column goes to one field at most: the sheet refuses +// two fields mapped to one column. +nonisolated enum ImportColumnMatcher { static func byName(fields: [String], columns: [String]) -> [ImportFieldChoice] { resolve(fields: fields, columns: columns, preferring: []) } @@ -55,8 +49,8 @@ internal enum ImportColumnMatcher { } } - /// Each tier is settled for every field before the next starts, so a higher tier wins a column - /// whatever the field order. A choice that can no longer be honoured falls through to the name match. + // Each tier is settled for every field before the next starts, so a higher tier wins a column whatever the field + // order. A choice that can no longer be honoured falls through to the name match. static func resolve( fields: [String], columns: [String], @@ -112,7 +106,7 @@ internal enum ImportColumnMatcher { resolve(fields: fields, columns: columns, preferring: [overrides.mapValues(\.choice)]) } - /// The fields whose current choice is a remembered override the name match would not have made. + // The fields whose current choice is a remembered override the name match would not have made. static func restoredFields( _ overrides: [String: ImportMappingOverride], fields: [String], @@ -146,8 +140,8 @@ internal enum ImportColumnMatcher { return overrides } - /// Entries for fields this file does not hold are kept, so several layouts into one table all come - /// back. A header respelled only by case takes over its one differently-cased entry. + // Entries for fields this file does not hold are kept, so several layouts into one table all come back. A header + // respelled only by case takes over its one differently-cased entry. static func merging( _ overrides: [String: ImportMappingOverride], forFields fields: [String], @@ -167,8 +161,8 @@ internal enum ImportColumnMatcher { return merged } - /// A differently-cased entry is used only when that is unambiguous on both sides: `Email` and `email` - /// can be two fields of one file. + // A differently-cased entry is used only when that is unambiguous on both sides: `Email` and `email` can be two + // fields of one file. private static func choice(for field: String, among fields: [String], in prior: [String: Value]) -> Value? { if let exact = prior[field] { return exact } let folded = normalized(field) diff --git a/apple/TablePro/Core/Plugins/ImportFieldDetection.swift b/apple/TablePro/Core/Transfer/Shared/ImportFieldDetection.swift similarity index 52% rename from apple/TablePro/Core/Plugins/ImportFieldDetection.swift rename to apple/TablePro/Core/Transfer/Shared/ImportFieldDetection.swift index 391461d814..abfd516ae3 100644 --- a/apple/TablePro/Core/Plugins/ImportFieldDetection.swift +++ b/apple/TablePro/Core/Transfer/Shared/ImportFieldDetection.swift @@ -1,16 +1,10 @@ -// -// ImportFieldDetection.swift -// TablePro -// - import Foundation import TableProPluginKit -/// `detectSourceFields` is synchronous and reads the file: the XLSX plugin materialises the whole -/// workbook and the JSON one reads every row. The read runs off the main actor, and cancelling the -/// caller cancels the read with it. The import sheet's `.task(id:)` cancels its caller when the -/// sheet closes or the read it answers changes, such as another table picked. -enum ImportFieldDetection { +// `detectSourceFields` is synchronous and reads the file: the XLSX plugin materialises the whole workbook and the JSON +// one reads every row. It runs off the main actor, and cancelling the caller cancels the read, so a sheet's +// `.task(id:)` drops a read that another table picked or the sheet closing made stale. +nonisolated enum ImportFieldDetection { static func detectFields( plugin: any ImportFormatPlugin, at url: URL, diff --git a/apple/TablePro/Core/Transfer/Shared/ImportRowWriter.swift b/apple/TablePro/Core/Transfer/Shared/ImportRowWriter.swift new file mode 100644 index 0000000000..0f0b80b81e --- /dev/null +++ b/apple/TablePro/Core/Transfer/Shared/ImportRowWriter.swift @@ -0,0 +1,387 @@ +import Foundation +import TableProPluginKit + +// The write half of a row import: mapping, batching, matching and the mode decision. The host's sink owns the +// connection, the transaction, foreign keys, COPY and cancellation, and runs every statement through `Host.execute`. +nonisolated final class ImportRowWriter: @unchecked Sendable { + nonisolated enum Purpose: Equatable, Sendable { + case matchProbe + case write(rows: Int) + } + + // What one table takes, read once per import by the host. + nonisolated struct Target { + let tableName: String + let statements: SQLImportStatements + let values: ImportTargetValues + } + + nonisolated struct Host { + // A probe answers its rows as text; a write's answer is ignored. + let execute: (_ statement: SQLImportStatement, _ purpose: Purpose) async throws -> [[String?]] + let columnsUpdateCannotSet: () async throws -> Set + let isCancelled: () -> Bool + } + + let rowWriting: ImportRowWriting + private let databaseTypeId: String + private let exactMapping: [String: String] + private let foldedMapping: [String: String] + private let sourceFields: Set + private let sourceFieldsPerFoldedName: [String: Int] + private let tallyLock = NSLock() + private var currentTally: ImportRowTally + + init( + databaseTypeId: String, + columnMapping: [String: String], + sourceFields: Set, + rowWriting: ImportRowWriting + ) { + self.databaseTypeId = databaseTypeId + self.rowWriting = rowWriting + self.currentTally = ImportRowTally(mode: rowWriting.mode) + self.exactMapping = columnMapping + self.sourceFields = sourceFields + self.sourceFieldsPerFoldedName = Dictionary(sourceFields.map { ($0.lowercased(), 1) }, uniquingKeysWith: +) + self.foldedMapping = Dictionary(grouping: columnMapping, by: { $0.key.lowercased() }) + .compactMapValues { $0.count == 1 ? $0.first?.value : nil } + } + + var tally: ImportRowTally { + tallyLock.lock() + defer { tallyLock.unlock() } + return currentTally + } + + private func recordTally(_ change: (inout ImportRowTally) -> Void) { + tallyLock.lock() + defer { tallyLock.unlock() } + change(¤tTally) + } + + // MARK: - Writing + + func insertRow(_ values: [String: PluginCellValue], into target: Target, host: Host) async throws { + let row = try writableRows([values], target: target)[0] + guard !rowWriting.mode.matchesExistingRows else { + try await writeMatching([row], target: target, host: host) + return + } + guard !row.columns.isEmpty else { + try await insertDefaultRows(1, target: target, host: host) + return + } + guard let statement = target.statements.insertStatement(columns: row.columns, values: row.values) else { + throw PluginImportError.importFailed( + String(format: String(localized: "Could not build an INSERT for %@"), target.tableName) + ) + } + _ = try await host.execute(statement, .write(rows: 1)) + recordTally { $0.inserted += 1 } + } + + func insertRows(_ rows: [[String: PluginCellValue]], into target: Target, host: Host) async throws { + let writable = try writableRows(rows, target: target) + if rowWriting.mode.matchesExistingRows { + try await writeMatching(writable, target: target, host: host) + } else { + try await insertGrouped(writable, target: target, host: host) + } + } + + // Rows already in column order, written in the same budgeted batches as a mapped import. + func insertRows( + columns: [String], + values rows: [[PluginCellValue]], + into target: Target, + host: Host, + onWrite: (Int) -> Void = { _ in } + ) async throws { + try await insertGroup(rows, columns: columns, target: target, host: host, onWrite: onWrite) + } + + // Clearing first would leave no row to match. + func clearingStatement(using statements: SQLImportStatements) throws -> String { + guard !rowWriting.mode.matchesExistingRows else { + throw PluginImportError.importFailed(Self.matchingRefusesClearingMessage) + } + return statements.deleteAllRowsStatement() + } + + nonisolated private struct MappedRow { + let columns: [String] + let values: [PluginCellValue] + } + + // A row with no field, such as JSON's `{}` for an all-NULL row, is a row of defaults; a row whose fields all missed + // the mapping is an error. + private func writableRows(_ rows: [[String: PluginCellValue]], target: Target) throws -> [MappedRow] { + try rows.map { values in + let (columns, mappedValues) = mappedColumnsAndValues(values) + if columns.isEmpty, !values.isEmpty { + throw PluginImportError.importFailed(Self.unmappedRowMessage) + } + return MappedRow(columns: columns, values: try target.values.values(mappedValues, columns: columns)) + } + } + + private func insertGrouped(_ rows: [MappedRow], target: Target, host: Host) async throws { + var index = 0 + while index < rows.count { + let columns = rows[index].columns + var groupValues: [[PluginCellValue]] = [rows[index].values] + var next = index + 1 + while next < rows.count, rows[next].columns == columns { + groupValues.append(rows[next].values) + next += 1 + } + index = next + + if columns.isEmpty { + try await insertDefaultRows(groupValues.count, target: target, host: host) + } else { + try await insertGroup(groupValues, columns: columns, target: target, host: host) + } + } + } + + private func insertDefaultRows(_ rowCount: Int, target: Target, host: Host) async throws { + guard let statement = target.statements.allDefaultsInsertStatement( + firstColumn: target.values.firstWritableColumn + ) else { + throw PluginImportError.importFailed(String( + format: String(localized: "This row has no values, and %@ has no way to insert a row of only defaults."), + databaseTypeId + )) + } + for _ in 0.. Void = { _ in } + ) async throws { + let budget = SQLWriteBatchBudget(columnCount: columns.count, statements: target.statements) + var filler = SQLWriteBatchFiller<[PluginCellValue]>(budget: budget) + for row in groupValues { + guard let batch = filler.append(row, bytes: budget.byteCount(of: row)) else { + continue + } + try await write(batch, columns: columns, target: target, host: host) + onWrite(batch.count) + } + guard let batch = filler.take() else { return } + try await write(batch, columns: columns, target: target, host: host) + onWrite(batch.count) + } + + private func write(_ batch: [[PluginCellValue]], columns: [String], target: Target, host: Host) async throws { + guard !host.isCancelled() else { throw PluginImportCancellationError() } + guard let statement = target.statements.insertStatement(columns: columns, rows: batch) else { + throw PluginImportError.importFailed( + String(localized: "Could not build an INSERT for the mapped columns") + ) + } + _ = try await host.execute(statement, .write(rows: batch.count)) + recordTally { $0.inserted += batch.count } + } + + // MARK: - Matching existing rows + + nonisolated private struct KeyedRow { + let row: MappedRow + let key: [PluginCellValue] + } + + // Not the engines' upsert forms: they refuse a NOT NULL column the file leaves out even when the row exists. + private func writeMatching(_ rows: [MappedRow], target: Target, host: Host) async throws { + var run: [KeyedRow] = [] + var keysInRun = Set<[String]>() + for row in rows { + let key = try keyValues(of: row) + // A repeated key has to see the write of its first occurrence, so the run ends before it. + let folded = key.contains(where: \.isNull) ? nil : key.map(Self.foldedKeyValue) + if let folded, keysInRun.contains(folded) { + try await writeMatchedRun(run, target: target, host: host) + run.removeAll(keepingCapacity: true) + keysInRun.removeAll(keepingCapacity: true) + } + if let folded { keysInRun.insert(folded) } + run.append(KeyedRow(row: row, key: key)) + } + try await writeMatchedRun(run, target: target, host: host) + } + + // Folds more than a collation or a numeric comparison does: two keys the database takes as one must not share a run. + static func foldedKeyValue(_ value: PluginCellValue) -> String { + switch value { + case .null: + return "null" + case .bytes(let data): + return "bytes:" + data.base64EncodedString() + case .text(let text): + let folded = text.trimmingCharacters(in: .whitespacesAndNewlines) + .folding(options: [.caseInsensitive, .diacriticInsensitive, .widthInsensitive], locale: nil) + // A Double, not a Decimal: a REAL column holds 0.1 and 0.100000000000000001 as one value. + guard folded.contains(where: { $0.isASCII && $0.isNumber }), + folded.allSatisfy({ numberCharacters.contains($0) }), + let number = Double(folded) else { + return "text:" + folded + } + return "number:\(number == 0 ? 0 : number)" + } + } + + private static let numberCharacters = Set("0123456789+-.e") + + private func keyValues(of row: MappedRow) throws -> [PluginCellValue] { + try rowWriting.keyColumns.map { column in + guard let index = row.columns.firstIndex(of: column) else { + throw PluginImportError.importFailed(String( + format: String(localized: "This row has no value for the key column %@, so it cannot be matched."), + column + )) + } + return row.values[index] + } + } + + private func writeMatchedRun(_ run: [KeyedRow], target: Target, host: Host) async throws { + guard !run.isEmpty else { return } + let counts = try await matchCounts(of: run.map(\.key), target: target, host: host) + if let ambiguous = zip(run, counts).first(where: { $0.1 > 1 }) { + throw PluginImportError.importFailed(String( + format: String(localized: "The key %1$@ matched %2$@ rows, so the row was not written."), + keyDescription(ambiguous.0.key), + ambiguous.1.formatted() + )) + } + + // Writes go out in file order, so a new parent row lands before an update that points at it. + var inserts: [MappedRow] = [] + for (entry, matches) in zip(run, counts) { + switch (matches, rowWriting.mode) { + case (0, .update): + recordTally { $0.unmatched += 1 } + case (0, _): + inserts.append(entry.row) + case (_, .update), (_, .upsert): + try await insertGrouped(inserts, target: target, host: host) + inserts.removeAll(keepingCapacity: true) + try await update(entry, target: target, host: host) + default: + recordTally { $0.existing += 1 } + } + } + try await insertGrouped(inserts, target: target, host: host) + } + + // A column the engine never lets an UPDATE assign keeps its value, and is written only for an inserted row. + private func update(_ entry: KeyedRow, target: Target, host: Host) async throws { + let kept = Set(rowWriting.keyColumns).union(try await host.columnsUpdateCannotSet()) + let assignments = zip(entry.row.columns, entry.row.values).filter { !kept.contains($0.0) } + guard !assignments.isEmpty else { + recordTally { $0.existing += 1 } + return + } + guard !host.isCancelled() else { throw PluginImportCancellationError() } + guard let statement = target.statements.keyedUpdateStatement( + columns: assignments.map(\.0), + values: assignments.map(\.1), + keyColumns: rowWriting.keyColumns, + keyValues: entry.key + ) else { + throw PluginImportError.importFailed(String(localized: "Could not build an UPDATE for the mapped columns")) + } + _ = try await host.execute(statement, .write(rows: 1)) + recordTally { $0.updated += 1 } + } + + private static let keysPerProbe = 100 + + private func matchCounts(of keys: [[PluginCellValue]], target: Target, host: Host) async throws -> [Int] { + let keyWidth = max(1, rowWriting.keyColumns.count) + let perStatement = max(1, min(Self.keysPerProbe, target.statements.maxBindParameters / keyWidth)) + var counts: [Int] = [] + counts.reserveCapacity(keys.count) + var start = 0 + while start < keys.count { + let chunk = Array(keys[start.. [Int] { + var counts = [Int?](repeating: nil, count: expected) + for row in rows { + guard row.count >= 2, let position = integer(row[0]), counts.indices.contains(position), + let count = integer(row[1]) else { continue } + counts[position] = count + } + let read = counts.compactMap { $0 } + guard read.count == expected else { + throw PluginImportError.importFailed(String(localized: "Could not read which rows are already in the table")) + } + return read + } + + private static func integer(_ value: String?) -> Int? { + guard let text = value?.trimmingCharacters(in: .whitespaces) else { return nil } + return Int(text) ?? Double(text).flatMap { $0.rounded() == $0 ? Int(exactly: $0) : nil } + } + + private func keyDescription(_ key: [PluginCellValue]) -> String { + zip(rowWriting.keyColumns, key) + .map { "\($0.0) = \($0.1.isNull ? "NULL" : $0.1.sortKey)" } + .joined(separator: ", ") + } + + // MARK: - Mapping + + func mappedColumnsAndValues(_ values: [String: PluginCellValue]) -> ([String], [PluginCellValue]) { + var pairs: [(column: String, value: PluginCellValue)] = [] + var matchedNames = Set() + for (field, value) in values { + guard let column = exactMapping[field] else { continue } + pairs.append((column, value)) + matchedNames.insert(field.lowercased()) + } + let unmatched = values.filter { exactMapping[$0.key] == nil && !sourceFields.contains($0.key) } + for (folded, group) in Dictionary(grouping: unmatched, by: { $0.key.lowercased() }) { + guard group.count == 1, let entry = group.first, !matchedNames.contains(folded), + sourceFieldsPerFoldedName[folded, default: 0] <= 1, + let column = foldedMapping[folded] else { continue } + pairs.append((column, entry.value)) + } + pairs.sort { $0.column < $1.column } + return (pairs.map(\.column), pairs.map(\.value)) + } + + private static var matchingRefusesClearingMessage: String { + String(localized: """ + Delete existing rows before import empties the table, so no row would match. Turn it off, or import as \ + Insert. + """) + } + + private static var unmappedRowMessage: String { + String(localized: "No values in this row matched the column mapping") + } +} diff --git a/apple/TablePro/Core/Services/Export/ImportRowWriting.swift b/apple/TablePro/Core/Transfer/Shared/ImportRowWriting.swift similarity index 65% rename from apple/TablePro/Core/Services/Export/ImportRowWriting.swift rename to apple/TablePro/Core/Transfer/Shared/ImportRowWriting.swift index 243f0d6192..e9c1a2a31f 100644 --- a/apple/TablePro/Core/Services/Export/ImportRowWriting.swift +++ b/apple/TablePro/Core/Transfer/Shared/ImportRowWriting.swift @@ -1,19 +1,14 @@ -// -// ImportRowWriting.swift -// TablePro -// - import Foundation -internal enum ImportWriteMode: String, CaseIterable, Identifiable, Sendable { +nonisolated enum ImportWriteMode: String, CaseIterable, Identifiable, Sendable { case insert case update case upsert case insertSkippingExisting - internal var id: String { rawValue } + var id: String { rawValue } - internal var label: String { + var label: String { switch self { case .insert: return String(localized: "Insert") case .update: return String(localized: "Update") @@ -22,24 +17,58 @@ internal enum ImportWriteMode: String, CaseIterable, Identifiable, Sendable { } } - internal var matchesExistingRows: Bool { + var matchesExistingRows: Bool { self != .insert } - internal var updatesMatchedRows: Bool { + var updatesMatchedRows: Bool { self == .update || self == .upsert } } -internal struct ImportMatchKey: Hashable, Identifiable, Sendable { - internal let indexName: String? - internal let columns: [String] +nonisolated struct ImportMatchKey: Hashable, Identifiable, Sendable { + nonisolated struct Column: Equatable, Sendable { + let name: String + let isPrimaryKey: Bool + } + + // A catalog that cannot say whether an index is valid, partial or over expressions reads as a plain valid one. + nonisolated struct Index: Equatable, Sendable { + let name: String + let columns: [String] + let isUnique: Bool + let isPrimary: Bool + let isValid: Bool + let isPartial: Bool + let hasExpressions: Bool + + init( + name: String, + columns: [String], + isUnique: Bool, + isPrimary: Bool, + isValid: Bool = true, + isPartial: Bool = false, + hasExpressions: Bool = false + ) { + self.name = name + self.columns = columns + self.isUnique = isUnique + self.isPrimary = isPrimary + self.isValid = isValid + self.isPartial = isPartial + self.hasExpressions = hasExpressions + } + } + + let indexName: String? + let columns: [String] - internal var id: [String] { columns } + var id: [String] { columns } - internal var isPrimaryKey: Bool { indexName == nil } + var isPrimaryKey: Bool { indexName == nil } - internal var label: String { + var label: String { let list = columns.joined(separator: ", ") guard let indexName else { return String(format: String(localized: "Primary Key (%@)"), list) @@ -48,7 +77,7 @@ internal struct ImportMatchKey: Hashable, Identifiable, Sendable { } // A partial or expression index leaves the plain columns free to repeat, so it cannot pick out one row. - internal static func options(columns: [ColumnInfo], indexes: [IndexInfo]) -> [ImportMatchKey] { + static func options(columns: [Column], indexes: [Index]) -> [ImportMatchKey] { let exact = Set(columns.map(\.name)) // A spelling two columns share once case is folded names neither: PostgreSQL keeps "ID" and "id" apart. let folded = Dictionary(grouping: columns.map(\.name), by: { $0.lowercased() }) @@ -64,8 +93,8 @@ internal struct ImportMatchKey: Hashable, Identifiable, Sendable { ?? (flagged.isEmpty ? nil : flagged) { keys.append(ImportMatchKey(indexName: nil, columns: primary)) } - for index in indexes where index.isUnique && !index.isPrimary && index.isValid && index.whereClause == nil { - guard (index.expressions ?? []).isEmpty, + for index in indexes where index.isUnique && !index.isPrimary && index.isValid && !index.isPartial { + guard !index.hasExpressions, let keyColumns = tableColumns(index.columns), !keys.contains(where: { Set($0.columns) == Set(keyColumns) }) else { continue } keys.append(ImportMatchKey(indexName: index.name, columns: keyColumns)) @@ -74,19 +103,19 @@ internal struct ImportMatchKey: Hashable, Identifiable, Sendable { } } -internal struct ImportRowWriting: Equatable, Sendable { - internal let mode: ImportWriteMode - internal let keyColumns: [String] +nonisolated struct ImportRowWriting: Equatable, Sendable { + let mode: ImportWriteMode + let keyColumns: [String] - internal static let insert = ImportRowWriting(mode: .insert, keyColumns: []) + static let insert = ImportRowWriting(mode: .insert, keyColumns: []) } -internal enum ImportRowWritingProblem: Equatable { +nonisolated enum ImportRowWritingProblem: Equatable { case noKey case unmappedKeyColumns([String]) case nothingToUpdate - internal static func of( + static func of( mode: ImportWriteMode, key: ImportMatchKey?, mappedColumns: [String] @@ -102,7 +131,7 @@ internal enum ImportRowWritingProblem: Equatable { return nil } - internal var message: String { + var message: String { switch self { case .noKey: return String(localized: "This table has no primary key or unique index to match rows on.") @@ -118,23 +147,23 @@ internal enum ImportRowWritingProblem: Equatable { } // Counted per row as it lands, so a row the database refused is in none of them. -internal struct ImportRowTally: Equatable, Sendable { - internal let mode: ImportWriteMode - internal var inserted = 0 - internal var updated = 0 +nonisolated struct ImportRowTally: Equatable, Sendable { + let mode: ImportWriteMode + var inserted = 0 + var updated = 0 // Matched no row, so Update left it out. - internal var unmatched = 0 + var unmatched = 0 // Matched a row left as it was: Insert, Skip Existing, or a row with no value outside the key. - internal var existing = 0 + var existing = 0 - internal init(mode: ImportWriteMode) { + init(mode: ImportWriteMode) { self.mode = mode } - internal var matched: Int { updated + existing } - internal var skipped: Int { unmatched + existing } + var matched: Int { updated + existing } + var skipped: Int { unmatched + existing } - internal func summaryLines(failed: Int) -> [String] { + func summaryLines(failed: Int) -> [String] { var lines = [String(format: String(localized: "Inserted: %@"), inserted.formatted())] if mode.matchesExistingRows { lines = [ diff --git a/apple/TablePro/Core/Plugins/ImportTargetValues.swift b/apple/TablePro/Core/Transfer/Shared/ImportTargetValues.swift similarity index 71% rename from apple/TablePro/Core/Plugins/ImportTargetValues.swift rename to apple/TablePro/Core/Transfer/Shared/ImportTargetValues.swift index fdf052f14c..ef0c08d369 100644 --- a/apple/TablePro/Core/Plugins/ImportTargetValues.swift +++ b/apple/TablePro/Core/Transfer/Shared/ImportTargetValues.swift @@ -1,13 +1,8 @@ -// -// ImportTargetValues.swift -// TablePro -// - import Foundation import TableProPluginKit // How a format's export writes bytes as text, so its import can read them back. -enum ImportBinaryText: Sendable, Equatable { +nonisolated enum ImportBinaryText: Sendable, Equatable { case prefixedHex case base64 @@ -21,32 +16,44 @@ enum ImportBinaryText: Sendable, Equatable { } // A file holds text; the target column's type decides the value it stands for. -struct ImportTargetValues: Sendable { +nonisolated struct ImportTargetValues: Sendable { + nonisolated struct ColumnTraits: Equatable, Sendable { + let holdsBytes: Bool + let takesNumericBooleans: Bool + } + + nonisolated enum PrefixedHex: Equatable, Sendable { + case notHex + case bytes(Data) + case malformed + } + let typeNames: [String: String] // A row of defaults on Oracle names one column, and a virtual column refuses even DEFAULT. let firstWritableColumn: String? private let binaryColumns: Set private let numericBooleanColumns: Set private let binaryText: ImportBinaryText? + private let prefixedHex: @Sendable (String) -> PrefixedHex + // `traits` reads one column type; the host's classifier says which hold bytes and which take 1 and 0 for booleans. init( columns: [(name: String, typeName: String, isGenerated: Bool)], - databaseType: DatabaseType, - binaryText: ImportBinaryText? + binaryText: ImportBinaryText?, + prefixedHex: @escaping @Sendable (String) -> PrefixedHex, + traits: (_ typeName: String) -> ColumnTraits ) { var typeNames: [String: String] = [:] for column in columns where typeNames[column.name] == nil { typeNames[column.name] = column.typeName } - let classifier = ColumnTypeClassifier(family: SQLTypeFamily.of(databaseType)) - let types = typeNames.mapValues { classifier.classify(rawTypeName: $0) } - // MySQL strict mode refuses `true` for an integer flag, SQL Server refuses `yes` for a bit, and SQLite keeps either as text. - let writesNumericBooleans = (try? resolveSQLDialect(for: databaseType))?.booleanLiteralStyle == .numeric + let columnTraits = typeNames.mapValues(traits) self.typeNames = typeNames self.firstWritableColumn = columns.first { !$0.isGenerated }?.name - self.binaryColumns = Set(types.filter { $0.value.isBlobType }.keys) - self.numericBooleanColumns = writesNumericBooleans ? Set(types.filter { $0.value.isBooleanType }.keys) : [] + self.binaryColumns = Set(columnTraits.filter { $0.value.holdsBytes }.keys) + self.numericBooleanColumns = Set(columnTraits.filter { $0.value.takesNumericBooleans }.keys) self.binaryText = binaryText + self.prefixedHex = prefixedHex } func values(_ values: [PluginCellValue], columns: [String]) throws -> [PluginCellValue] { @@ -67,14 +74,17 @@ struct ImportTargetValues: Sendable { private func binaryValue(_ text: String, column: String) throws -> PluginCellValue { switch binaryText { case .prefixedHex: - guard BlobFormattingService.hasHexPrefix(text) else { return .text(text) } - guard let bytes = BlobFormattingService.shared.bytes(fromPrefixedHex: text) else { + switch prefixedHex(text) { + case .notHex: + return .text(text) + case .bytes(let bytes): + return .bytes(bytes) + case .malformed: throw PluginImportError.importFailed(String( format: String(localized: "%@ is a binary column, and this value is not 0x followed by hex digits."), column )) } - return .bytes(bytes) case .base64: return Data(base64Encoded: text).map { .bytes($0) } ?? .text(text) case nil: diff --git a/apple/TablePro/Core/Plugins/ImportTypeMapper.swift b/apple/TablePro/Core/Transfer/Shared/ImportTypeMapper.swift similarity index 67% rename from apple/TablePro/Core/Plugins/ImportTypeMapper.swift rename to apple/TablePro/Core/Transfer/Shared/ImportTypeMapper.swift index c9bc57bd03..7f3c8f2bcd 100644 --- a/apple/TablePro/Core/Plugins/ImportTypeMapper.swift +++ b/apple/TablePro/Core/Transfer/Shared/ImportTypeMapper.swift @@ -1,38 +1,41 @@ -// -// ImportTypeMapper.swift -// TablePro -// - import Foundation import TableProPluginKit -enum ImportTypeMapper { +nonisolated enum ImportTypeMapper { static func sqlType( for type: PluginImportFieldType, - databaseType: DatabaseType, + databaseTypeId: String, serverVersion: String? = nil ) -> String { - switch databaseType { - case .postgresql, .redshift, .cockroachdb, .kingbasees: + switch databaseTypeId { + case "PostgreSQL", "Redshift", "CockroachDB", "KingbaseES": return postgresType( type, - jsonColumnType: PostgreSQLServerVersion.jsonColumnType( - for: databaseType, serverVersion: serverVersion - ) + jsonColumnType: postgresJSONColumnType(databaseTypeId: databaseTypeId, serverVersion: serverVersion) ) - case .mysql, .mariadb, .tidb, .oceanbase: + case "MySQL", "MariaDB", "TiDB", "OceanBase": return mysqlType(type) - case .sqlite: + case "SQLite": return sqliteType(type) - case .mssql: + case "SQL Server": return mssqlType(type) - case .oracle: + case "Oracle": return oracleType(type, majorRelease: oracleMajorRelease(in: serverVersion)) default: return genericType(type) } } + // Only PostgreSQL itself is read from its version: Redshift reports 8.0.2 and CockroachDB 13.0.0, and neither number + // says what JSON support the engine has. + static func postgresJSONColumnType(databaseTypeId: String, serverVersion: String?) -> PostgreSQLJSONColumnType { + guard databaseTypeId == "PostgreSQL", let version = PostgreSQLServerVersion(serverVersion) else { + return .jsonb + } + if version >= .jsonbType { return .jsonb } + return version >= .jsonType ? .json : .text + } + private static func postgresType(_ type: PluginImportFieldType, jsonColumnType: PostgreSQLJSONColumnType) -> String { switch type { case .integer: return "BIGINT" @@ -55,8 +58,7 @@ enum ImportTypeMapper { } } - // SQLite stores a boolean as the integer 1 or 0. `BOOLEAN` has NUMERIC affinity and tells the import and the grid - // that the column is one. + // SQLite stores a boolean as 1 or 0. `BOOLEAN` has NUMERIC affinity and tells the import and the grid the column is one. private static func sqliteType(_ type: PluginImportFieldType) -> String { switch type { case .integer: return "INTEGER" @@ -77,9 +79,8 @@ enum ImportTypeMapper { } } - /// Oracle has no `TEXT` (ORA-00902), and `BOOLEAN` only from 23ai. Text gets the widest `VARCHAR2` rather than a - /// `CLOB`, which Oracle cannot compare or sort by, and a boolean before 23ai keeps the file's own words, which a - /// `NUMBER(1)` refuses (ORA-01722). JSON lands in a `CLOB`, the type every release holds it in. + // Oracle has no TEXT (ORA-00902) and BOOLEAN only from 23ai. Text takes the widest VARCHAR2, since a CLOB cannot be + // compared or sorted; a boolean before 23ai keeps the file's words, which NUMBER(1) refuses (ORA-01722). private static func oracleType(_ type: PluginImportFieldType, majorRelease: Int?) -> String { switch type { case .integer: return "NUMBER(19)" @@ -91,7 +92,7 @@ enum ImportTypeMapper { } } - /// The release in an Oracle banner, `Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production`. + // `Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production` is release 19. static func oracleMajorRelease(in banner: String?) -> Int? { guard let banner, let range = banner.range(of: #"\bRelease\s+(\d+)"#, options: .regularExpression) else { return nil diff --git a/apple/TablePro/Views/Import/NewTableDraft.swift b/apple/TablePro/Core/Transfer/Shared/NewTableDraft.swift similarity index 58% rename from apple/TablePro/Views/Import/NewTableDraft.swift rename to apple/TablePro/Core/Transfer/Shared/NewTableDraft.swift index b62208d4be..9bd97aafa9 100644 --- a/apple/TablePro/Views/Import/NewTableDraft.swift +++ b/apple/TablePro/Core/Transfer/Shared/NewTableDraft.swift @@ -1,23 +1,16 @@ -// -// NewTableDraft.swift -// TablePro -// - import Foundation import TableProPluginKit -/// What a row import makes of one field of the file in the table it creates. -internal struct NewTableColumnSettings: Equatable { - internal var include: Bool - internal var name: String - internal var type: String - internal var isPrimaryKey: Bool - internal var isNullable: Bool - internal var defaultValue: String - - /// Every field, under its own name and with the type its values suggest, nullable, with no key - /// and no default. - internal static func proposed(name: String, type: String) -> NewTableColumnSettings { +// What a row import makes of one field of the file in the table it creates. +nonisolated struct NewTableColumnSettings: Equatable { + var include: Bool + var name: String + var type: String + var isPrimaryKey: Bool + var isNullable: Bool + var defaultValue: String + + static func proposed(name: String, type: String) -> NewTableColumnSettings { NewTableColumnSettings( include: true, name: name, @@ -29,21 +22,21 @@ internal struct NewTableColumnSettings: Equatable { } } -/// What the user set on one column, each setting nil until a write changes it. A read never touches -/// it, so a setting stays the user's even when a later read proposes the same value. -internal struct NewTableColumnEdits: Equatable { - internal var include: Bool? - internal var name: String? - internal var type: String? - internal var isPrimaryKey: Bool? - internal var isNullable: Bool? - internal var defaultValue: String? - - internal var isEmpty: Bool { +// What the user set on one column, each setting nil until a write changes it. A read never touches it, so a setting +// stays the user's even when a later read proposes the same value. +nonisolated struct NewTableColumnEdits: Equatable { + var include: Bool? + var name: String? + var type: String? + var isPrimaryKey: Bool? + var isNullable: Bool? + var defaultValue: String? + + var isEmpty: Bool { self == NewTableColumnEdits() } - internal func applied(to proposal: NewTableColumnSettings) -> NewTableColumnSettings { + func applied(to proposal: NewTableColumnSettings) -> NewTableColumnSettings { NewTableColumnSettings( include: include ?? proposal.include, name: name ?? proposal.name, @@ -54,9 +47,9 @@ internal struct NewTableColumnEdits: Equatable { ) } - /// A type differing only in case is not a change: the type menu offers the dialect's own - /// spelling of the type on show, and choosing it again writes that spelling back. - internal func recording( + // A type differing only in case is not a change: the type menu offers the dialect's own spelling of the type on + // show, and choosing it again writes that spelling back. + func recording( _ settings: NewTableColumnSettings, over shown: NewTableColumnSettings ) -> NewTableColumnEdits { @@ -71,15 +64,13 @@ internal struct NewTableColumnEdits: Equatable { } } -/// One field of the file and the column the user is making of it. -internal struct NewTableColumn: Identifiable { - internal let field: PluginImportField +nonisolated struct NewTableColumn: Identifiable { + let field: PluginImportField + // What the latest read proposes for this field. + let proposal: NewTableColumnSettings + private(set) var edits: NewTableColumnEdits - /// What the latest read proposes for this field. - internal let proposal: NewTableColumnSettings - internal private(set) var edits: NewTableColumnEdits - - internal init( + init( field: PluginImportField, proposal: NewTableColumnSettings, edits: NewTableColumnEdits = NewTableColumnEdits() @@ -89,33 +80,38 @@ internal struct NewTableColumn: Identifiable { self.edits = edits } - internal var id: String { field.name } + var id: String { field.name } - internal var settings: NewTableColumnSettings { + var settings: NewTableColumnSettings { get { edits.applied(to: proposal) } set { edits = edits.recording(newValue, over: settings) } } } -internal enum NewTableColumnProblem: Equatable { +nonisolated enum NewTableColumnProblem: Equatable { case unnamedColumn case duplicateName + + var message: String { + switch self { + case .unnamedColumn: + return String(localized: "Every included column needs a name.") + case .duplicateName: + return String(localized: "Column names must be unique.") + } + } } -/// The table a row import creates: the fields of the file, what the sheet proposed for each, and -/// what the user changed. -/// -/// Changing a parsing option reads the file again, and rebuilding the columns from that read threw -/// away every rename, type, key, nullability, default and exclusion the user had set. Each field -/// keeps the settings the user set, and each setting left alone follows the read, the way the -/// sheet's table name keeps what the user typed over its own suggestion. -internal struct NewTableDraft { - internal var columns: [NewTableColumn] = [] - - /// A wrong delimiter reads other fields for one read, and correcting it brings these back. +// Changing a parsing option reads the file again, and rebuilding the columns from that read threw away every rename, +// type, key, nullability, default and exclusion the user had set. Each field keeps the settings the user set, and each +// setting left alone follows the read, the way the sheet's table name keeps what the user typed over its suggestion. +nonisolated struct NewTableDraft { + var columns: [NewTableColumn] = [] + + // A wrong delimiter reads other fields for one read, and correcting it brings these back. private var setAsideEdits: [String: NewTableColumnEdits] = [:] - internal mutating func load( + mutating func load( fields: [PluginImportField], proposingType proposedType: (PluginImportFieldType) -> String ) { @@ -133,23 +129,23 @@ internal struct NewTableDraft { setAsideEdits = edits.filter { !$0.value.isEmpty } } - internal var includesEveryColumn: Bool { + var includesEveryColumn: Bool { !columns.isEmpty && columns.allSatisfy(\.settings.include) } - internal mutating func setAllIncluded(_ include: Bool) { + mutating func setAllIncluded(_ include: Bool) { for index in columns.indices { columns[index].settings.include = include } } - /// Every field of the file in file order, the ones left out of the table included. - internal var fields: [String] { + // Every field of the file in file order, the ones left out of the table included. + var fields: [String] { columns.map(\.field.name) } - /// The column each field is written to, for every included field whose column has a name. - internal var columnMapping: [String: String] { + // The column each field is written to, for every included field whose column has a name. + var columnMapping: [String: String] { var mapping: [String: String] = [:] for column in namedColumns { mapping[column.field.name] = column.settings.name @@ -157,11 +153,11 @@ internal struct NewTableDraft { return mapping } - internal var hasNamedColumn: Bool { + var hasNamedColumn: Bool { !namedColumns.isEmpty } - internal var problem: NewTableColumnProblem? { + var problem: NewTableColumnProblem? { let names = columns .filter(\.settings.include) .map { $0.settings.name.trimmingCharacters(in: .whitespaces).lowercased() } @@ -171,8 +167,8 @@ internal struct NewTableDraft { return Set(names).count == names.count ? nil : .duplicateName } - /// Nil when no included column has both a name and a type, which leaves nothing to create. - internal func definition(tableName: String) -> PluginCreateTableDefinition? { + // Nil when no included column has both a name and a type, which leaves nothing to create. + func definition(tableName: String) -> PluginCreateTableDefinition? { let included = namedColumns .map(\.settings) .filter { !$0.type.trimmingCharacters(in: .whitespaces).isEmpty } diff --git a/apple/TablePro/Core/Transfer/Shared/NewTableImportPlanner.swift b/apple/TablePro/Core/Transfer/Shared/NewTableImportPlanner.swift new file mode 100644 index 0000000000..b0e61d37b3 --- /dev/null +++ b/apple/TablePro/Core/Transfer/Shared/NewTableImportPlanner.swift @@ -0,0 +1,42 @@ +import Foundation + +nonisolated enum NewTableImportPlan: Equatable { + case create + case reuseAfterClearing + case nameTakenWithDifferentColumns +} + +// A failed import leaves its table behind, and a retry may neither create it again nor write the rows it kept a second +// time. So a table is reused, after clearing, only when this sheet made it with the same DDL. A table is known by its +// database and schema too: keyed by name alone, a retry cleared a same-named table in another database. +nonisolated struct NewTableImportPlanner { + nonisolated struct Table: Hashable, Sendable { + let connectionId: UUID + let database: String? + let schema: String? + let name: String + } + + private var createStatements: [Table: String] = [:] + + func plan(for table: Table, createTableSQL: String) -> NewTableImportPlan { + guard let previousSQL = createStatements[table] else { + return .create + } + return previousSQL == createTableSQL ? .reuseAfterClearing : .nameTakenWithDifferentColumns + } + + func created(_ table: Table) -> Bool { + createStatements[table] != nil + } + + func createdTableNames(connectionId: UUID, database: String?, schema: String?) -> [String] { + createStatements.keys + .filter { $0.connectionId == connectionId && $0.database == database && $0.schema == schema } + .map(\.name) + } + + mutating func recordCreated(_ table: Table, createTableSQL: String) { + createStatements[table] = createTableSQL + } +} diff --git a/apple/TablePro/Core/Plugins/PlainFileImportSource.swift b/apple/TablePro/Core/Transfer/Shared/PlainFileImportSource.swift similarity index 74% rename from apple/TablePro/Core/Plugins/PlainFileImportSource.swift rename to apple/TablePro/Core/Transfer/Shared/PlainFileImportSource.swift index 142f332f0f..ad8b751907 100644 --- a/apple/TablePro/Core/Plugins/PlainFileImportSource.swift +++ b/apple/TablePro/Core/Transfer/Shared/PlainFileImportSource.swift @@ -1,13 +1,9 @@ -// -// PlainFileImportSource.swift -// TablePro -// - import Foundation import os import TableProPluginKit -final class PlainFileImportSource: PluginImportSource, @unchecked Sendable { +// A row format reads the file itself; the source only names it. +nonisolated final class PlainFileImportSource: PluginImportSource, @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "PlainFileImportSource") private let url: URL @@ -32,6 +28,6 @@ final class PlainFileImportSource: PluginImportSource, @unchecked Sendable { } func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { - throw PluginImportError.importFailed("This import format does not produce SQL statements") + throw PluginImportError.importFailed(String(localized: "This import format does not produce SQL statements")) } } diff --git a/apple/TablePro/Core/Database/PostgreSQLServerVersion.swift b/apple/TablePro/Core/Transfer/Shared/PostgreSQLServerVersion.swift similarity index 74% rename from apple/TablePro/Core/Database/PostgreSQLServerVersion.swift rename to apple/TablePro/Core/Transfer/Shared/PostgreSQLServerVersion.swift index 8d6b40db15..e1eca345d5 100644 --- a/apple/TablePro/Core/Database/PostgreSQLServerVersion.swift +++ b/apple/TablePro/Core/Transfer/Shared/PostgreSQLServerVersion.swift @@ -1,17 +1,12 @@ -// -// PostgreSQLServerVersion.swift -// TablePro -// - import Foundation -enum PostgreSQLJSONColumnType: String, Sendable { +nonisolated enum PostgreSQLJSONColumnType: String, Sendable { case jsonb = "JSONB" case json = "JSON" case text = "TEXT" } -struct PostgreSQLServerVersion: Comparable, Hashable, Sendable { +nonisolated struct PostgreSQLServerVersion: Comparable, Hashable, Sendable { typealias JSONColumnType = PostgreSQLJSONColumnType static let stateColumns = PostgreSQLServerVersion(number: 90_200) @@ -55,18 +50,6 @@ struct PostgreSQLServerVersion: Comparable, Hashable, Sendable { return "\(major).\((number / 100) % 100).\(number % 100)" } - /// The richest JSON column type a target holds. - /// - /// Only `.postgresql` is read from the reported version. Redshift reports 8.0.2 and CockroachDB - /// reports 13.0.0, and neither number describes the JSON support that engine actually has. - static func jsonColumnType(for databaseType: DatabaseType, serverVersion: String?) -> JSONColumnType { - guard databaseType == .postgresql, let version = PostgreSQLServerVersion(serverVersion) else { - return .jsonb - } - if version >= .jsonbType { return .jsonb } - return version >= .jsonType ? .json : .text - } - static func < (lhs: PostgreSQLServerVersion, rhs: PostgreSQLServerVersion) -> Bool { lhs.number < rhs.number } diff --git a/apple/TablePro/Core/Transfer/Shared/PrefixedHexBytes.swift b/apple/TablePro/Core/Transfer/Shared/PrefixedHexBytes.swift new file mode 100644 index 0000000000..d6cc54075a --- /dev/null +++ b/apple/TablePro/Core/Transfer/Shared/PrefixedHexBytes.swift @@ -0,0 +1,47 @@ +import Foundation + +// The form a copy, a CSV export and an XLSX export write bytes in, read strictly: a copy cut short with `…` +// or text that only starts like hex is not bytes. +nonisolated enum PrefixedHexBytes { + static func hasPrefix(_ text: String) -> Bool { + text.hasPrefix("0x") || text.hasPrefix("0X") + } + + static func bytes(from text: String) -> Data? { + guard hasPrefix(text) else { return nil } + let digits = text.utf8.dropFirst(2) + guard digits.count.isMultiple(of: 2) else { return nil } + var bytes = [UInt8]() + bytes.reserveCapacity(digits.count / 2) + var high: UInt8? + for character in digits { + guard let nibble = nibble(character) else { return nil } + if let pending = high { + bytes.append(pending << 4 | nibble) + high = nil + } else { + high = nibble + } + } + return Data(bytes) + } + + private static func nibble(_ character: UInt8) -> UInt8? { + switch character { + case UInt8(ascii: "0")...UInt8(ascii: "9"): return character - UInt8(ascii: "0") + case UInt8(ascii: "a")...UInt8(ascii: "f"): return character - UInt8(ascii: "a") + 10 + case UInt8(ascii: "A")...UInt8(ascii: "F"): return character - UInt8(ascii: "A") + 10 + default: return nil + } + } +} + +nonisolated extension ImportTargetValues.PrefixedHex { + init(reading text: String) { + guard PrefixedHexBytes.hasPrefix(text) else { + self = .notHex + return + } + self = PrefixedHexBytes.bytes(from: text).map(Self.bytes) ?? .malformed + } +} diff --git a/apple/TablePro/Core/Transfer/Shared/QueryExportOptions.swift b/apple/TablePro/Core/Transfer/Shared/QueryExportOptions.swift new file mode 100644 index 0000000000..eb3993a346 --- /dev/null +++ b/apple/TablePro/Core/Transfer/Shared/QueryExportOptions.swift @@ -0,0 +1,21 @@ +import Foundation +import TableProPluginKit + +// A result set has no schema behind it, so a query export never asks for structure or drop: drop alone wrote +// `DROP TABLE ` into a dump that never recreated it. +nonisolated enum QueryExportOptions { + private static let schemaColumnIds: Set = ["structure", "drop"] + + // Cleared by column id, not position: SQL declares [structure, drop, data] and MQL [drop, indexes, data]. + // A defaults array of the wrong length is dropped whole, as `ExportObjectItem.normalized` does. + static func dataOnly( + columns: [PluginExportOptionColumn], + defaults: [Bool] + ) -> [Bool] { + let aligned = defaults.count == columns.count ? defaults : [] + return columns.enumerated().map { index, column -> Bool in + guard !schemaColumnIds.contains(column.id) else { return false } + return aligned.indices.contains(index) ? aligned[index] : column.defaultValue + } + } +} diff --git a/apple/TablePro/Core/Transfer/Shared/SQLImportStatements.swift b/apple/TablePro/Core/Transfer/Shared/SQLImportStatements.swift new file mode 100644 index 0000000000..370e00df8d --- /dev/null +++ b/apple/TablePro/Core/Transfer/Shared/SQLImportStatements.swift @@ -0,0 +1,216 @@ +import Foundation +import TableProPluginKit + +nonisolated struct SQLImportStatement: Equatable, Sendable { + let sql: String + let parameters: [PluginCellValue] + // By position, the type of the column each parameter is written to or compared with. Nil when none is known. + let parameterTypeNames: [String?]? +} + +// The statements a row import sends to one table, every value bound. What the host knows of its engine and session +// (placeholders, quoting, BIT columns, the table as statements spell it) comes in; the rest is keyed by type id. +nonisolated struct SQLImportStatements { + let databaseTypeId: String + let qualifiedTableName: String + let parameterStyle: ParameterStyle + let maxBindParameters: Int + let maxRowsPerInsert: Int + let rendersValuesAsLiterals: Bool + private let columnTypeNames: [String: String] + // BIT columns of a family that reads them as a number. Text bound to one is stored and compared as its bytes. + private let numberBitColumns: Set + private let quoteIdentifier: (String) -> String + private let rowUpdate: ((_ table: String, _ assignments: String, _ predicate: String) -> String?)? + private let rowDelete: ((_ table: String, _ predicate: String?) -> String?)? + + init( + databaseTypeId: String, + qualifiedTableName: String, + parameterStyle: ParameterStyle, + quoteIdentifier: @escaping (String) -> String, + columnTypeNames: [String: String] = [:], + numberBitColumns: Set = [], + rowUpdateStatement: ((_ table: String, _ assignments: String, _ predicate: String) -> String?)? = nil, + rowDeleteStatement: ((_ table: String, _ predicate: String?) -> String?)? = nil, + maxBindParameters: Int? = nil, + maxRowsPerInsert: Int? = nil, + rendersValuesAsLiterals: Bool? = nil + ) { + self.databaseTypeId = databaseTypeId + self.qualifiedTableName = qualifiedTableName + self.parameterStyle = parameterStyle + self.quoteIdentifier = quoteIdentifier + self.columnTypeNames = columnTypeNames + self.numberBitColumns = numberBitColumns + self.rowUpdate = rowUpdateStatement + self.rowDelete = rowDeleteStatement + self.maxBindParameters = maxBindParameters ?? Self.maxBindParameters(forDatabaseTypeId: databaseTypeId) + self.maxRowsPerInsert = maxRowsPerInsert + ?? SQLMultiRowInsert.maximumRowsPerStatement(forDatabaseTypeId: databaseTypeId) + self.rendersValuesAsLiterals = rendersValuesAsLiterals + ?? SQLWriteBatchBudget.rendersValuesAsLiterals(databaseTypeId: databaseTypeId) + } + + static func maxBindParameters(forDatabaseTypeId databaseTypeId: String) -> Int { + switch databaseTypeId { + case "SQLite", "libSQL", "Turso": 32_766 + case "SQL Server": 2_100 + case "Cloudflare D1": 100 + default: 65_535 + } + } + + // MARK: - Statements + + // For statements built without the table's columns, the caller names the one Oracle needs. + func allDefaultsInsertStatement(firstColumn: String?) -> SQLImportStatement? { + AllDefaultsInsert.sql( + into: qualifiedTableName, databaseTypeId: databaseTypeId, firstWritableColumn: firstColumn.map(quoteIdentifier) + ).map { SQLImportStatement(sql: $0, parameters: [], parameterTypeNames: nil) } + } + + func insertStatement(columns insertColumns: [String], values: [PluginCellValue]) -> SQLImportStatement? { + guard !insertColumns.isEmpty, insertColumns.count == values.count else { return nil } + + var bindParameters = Bindings() + let columnList = insertColumns.map(quoteIdentifier).joined(separator: ", ") + let placeholders = zip(insertColumns, values).map { column, value in + bind(value, toColumn: column, into: &bindParameters) + }.joined(separator: ", ") + + return statement("INSERT INTO \(qualifiedTableName) (\(columnList)) VALUES (\(placeholders))", bindParameters) + } + + func insertStatement(columns insertColumns: [String], rows: [[PluginCellValue]]) -> SQLImportStatement? { + guard !insertColumns.isEmpty, !rows.isEmpty, + rows.allSatisfy({ $0.count == insertColumns.count }) else { return nil } + + var bindParameters = Bindings() + let columnList = insertColumns.map(quoteIdentifier).joined(separator: ", ") + let rowTuples = rows.map { values -> String in + let placeholders = zip(insertColumns, values).map { column, value in + bind(value, toColumn: column, into: &bindParameters) + }.joined(separator: ", ") + return "(\(placeholders))" + }.joined(separator: ", ") + + return statement("INSERT INTO \(qualifiedTableName) (\(columnList)) VALUES \(rowTuples)", bindParameters) + } + + // The server compares, so its casting and collation decide a match, as they do for the UPDATE that follows. + func keyMatchCountStatement(keyColumns: [String], keys: [[PluginCellValue]]) -> SQLImportStatement? { + guard !keyColumns.isEmpty, !keys.isEmpty, keys.allSatisfy({ $0.count == keyColumns.count }) else { return nil } + + var bindParameters = Bindings() + let selects = keys.enumerated().map { position, values -> String in + let conditions = zip(keyColumns, values).map { column, value -> String in + "\(quoteIdentifier(column)) = \(bind(value, toColumn: column, into: &bindParameters))" + }.joined(separator: " AND ") + return "SELECT \(position), COUNT(*) FROM \(qualifiedTableName) WHERE \(conditions)" + } + + return statement(selects.joined(separator: " UNION ALL "), bindParameters) + } + + func keyedUpdateStatement( + columns setColumns: [String], + values: [PluginCellValue], + keyColumns: [String], + keyValues: [PluginCellValue] + ) -> SQLImportStatement? { + guard !setColumns.isEmpty, setColumns.count == values.count, + !keyColumns.isEmpty, keyColumns.count == keyValues.count else { return nil } + + var bindParameters = Bindings() + func bound(_ column: String, _ value: PluginCellValue) -> String { + "\(quoteIdentifier(column)) = \(bind(value, toColumn: column, into: &bindParameters))" + } + let assignments = zip(setColumns, values).map(bound).joined(separator: ", ") + let conditions = zip(keyColumns, keyValues).map(bound).joined(separator: " AND ") + let sql = rowUpdate?(qualifiedTableName, assignments, conditions) + ?? "UPDATE \(qualifiedTableName) SET \(assignments) WHERE \(conditions)" + + return statement(sql, bindParameters) + } + + func deleteAllRowsStatement() -> String { + rowDelete?(qualifiedTableName, nil) ?? "DELETE FROM \(qualifiedTableName)" + } + + // MARK: - Binding + + nonisolated struct Bindings { + private(set) var values: [PluginCellValue] = [] + private(set) var typeNames: [String?] = [] + + var count: Int { values.count } + + mutating func append(_ value: PluginCellValue, typeName: String?) { + values.append(value) + typeNames.append(typeName) + } + } + + func statement(_ sql: String, _ bindings: Bindings) -> SQLImportStatement { + SQLImportStatement( + sql: sql, + parameters: bindings.values, + parameterTypeNames: bindings.typeNames.contains { $0 != nil } ? bindings.typeNames : nil + ) + } + + func placeholder(at index: Int) -> String { + switch parameterStyle { + case .dollar: + return "$\(index + 1)" + case .questionMark: + return "?" + } + } + + func typeName(of column: String?) -> String? { + column.flatMap { columnTypeNames[$0] } + } + + // Blank text goes as '', which the server stores as 0: cast, strict mode refuses it as a truncated integer, and a + // space bound as it is would be its byte, 32. + func bind(_ value: PluginCellValue, toColumn column: String?, into parameters: inout Bindings) -> String { + guard let column, numberBitColumns.contains(column), case .text(let text) = value else { + parameters.append(value, typeName: typeName(of: column)) + return placeholder(at: parameters.count - 1) + } + guard !text.trimmingCharacters(in: .whitespaces).isEmpty else { + parameters.append(.text(""), typeName: typeName(of: column)) + return placeholder(at: parameters.count - 1) + } + // The value is the CAST's operand, not the column's. + parameters.append(.text(Self.decimalBitText(text)), typeName: nil) + return "CAST(\(placeholder(at: parameters.count - 1)) AS UNSIGNED)" + } + + // b'101' and 0b101 are SQL, not values, and a boolean word is no number: cast as text, each is 0 or an error. + static func decimalBitText(_ text: String) -> String { + let trimmed = text.trimmingCharacters(in: .whitespaces) + switch PluginSQLLiteral.booleanSynonym(for: trimmed) { + case .isTrue: return "1" + case .isFalse: return "0" + case nil: break + @unknown default: break + } + switch trimmed.lowercased() { + case "t": return "1" + case "f": return "0" + default: break + } + let digits: Substring + if trimmed.count > 3, trimmed.hasPrefix("b'") || trimmed.hasPrefix("B'"), trimmed.hasSuffix("'") { + digits = trimmed.dropFirst(2).dropLast() + } else if trimmed.hasPrefix("0b") { + digits = trimmed.dropFirst(2) + } else { + return text + } + return UInt64(digits, radix: 2).map { String($0) } ?? text + } +} diff --git a/apple/TablePro/Core/Transfer/Shared/SQLLexicalFeaturesBridge.swift b/apple/TablePro/Core/Transfer/Shared/SQLLexicalFeaturesBridge.swift new file mode 100644 index 0000000000..f28b82883b --- /dev/null +++ b/apple/TablePro/Core/Transfer/Shared/SQLLexicalFeaturesBridge.swift @@ -0,0 +1,56 @@ +import Foundation +import TableProPluginKit +import TableProSQLGrammar + +nonisolated extension SQLLexicalGrammar { + // The kit's feature for each grammar fact; `SQLLexicalFeatureMappingTests` holds every fact to a partner. A fact only + // the curated table sets, such as `cqlBatches`, has none: no plugin can declare it, and it never reaches one. + static let pluginFeaturePairs: [(SQLLexicalFeatures, SQLLexicalGrammar)] = [ + (.backslashEscapesInSingleQuotes, .backslashEscapesInSingleQuotes), + (.backslashEscapesInDoubleQuotes, .backslashEscapesInDoubleQuotes), + (.backslashEscapesInBackticks, .backslashEscapesInBackticks), + (.backtickQuotes, .backtickQuotes), + (.bracketQuotedIdentifiers, .bracketQuotedIdentifiers), + (.tripleQuotedStrings, .tripleQuotedStrings), + (.escapeStringPrefix, .escapeStringPrefix), + (.alternativeQuoting, .alternativeQuoting), + (.untaggedDollarQuotes, .untaggedDollarQuotes), + (.taggedDollarQuotes, .taggedDollarQuotes), + (.nestedBlockComments, .nestedBlockComments), + (.hashLineComments, .hashLineComments), + (.doubleSlashLineComments, .doubleSlashLineComments), + (.executableComments, .executableComments), + (.slashLineTerminators, .slashLineTerminators), + (.dollarAndHashInIdentifiers, .dollarAndHashInIdentifiers), + (.plsqlBlocks, .plsqlBlocks), + (.delimiterDirective, .delimiterDirective), + (.dashCommentsNeedWhitespace, .dashCommentsNeedWhitespace), + (.parenthesizedParameterNames, .parenthesizedParameterNames), + (.doubledClosingBracketEscapes, .doubledClosingBracketEscapes), + (.carriageReturnEndsLineComments, .carriageReturnEndsLineComments), + (.batchSeparatorLines, .batchSeparatorLines), + (.unterminatedStatements, .unterminatedStatements), + (.terminatedMergeStatements, .terminatedMergeStatements), + ] + + init(pluginFeatures: SQLLexicalFeatures) { + self = Self.pluginFeaturePairs.reduce(into: []) { grammar, pair in + if pluginFeatures.contains(pair.0) { grammar.insert(pair.1) } + } + } + + var pluginFeatures: SQLLexicalFeatures { + Self.pluginFeaturePairs.reduce(into: []) { features, pair in + if contains(pair.1) { features.insert(pair.0) } + } + } +} + +nonisolated extension SQLSessionLexicalFacts { + init(pluginState: PluginSessionLexicalState) { + self.init( + determined: SQLLexicalGrammar(pluginFeatures: pluginState.determined), + enabled: SQLLexicalGrammar(pluginFeatures: pluginState.enabled) + ) + } +} diff --git a/apple/TablePro/Core/Utilities/SQL/SQLScriptText.swift b/apple/TablePro/Core/Transfer/Shared/SQLScriptText.swift similarity index 66% rename from apple/TablePro/Core/Utilities/SQL/SQLScriptText.swift rename to apple/TablePro/Core/Transfer/Shared/SQLScriptText.swift index 5dfdb45bb2..3028fccf9a 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLScriptText.swift +++ b/apple/TablePro/Core/Transfer/Shared/SQLScriptText.swift @@ -1,50 +1,41 @@ -// -// SQLScriptText.swift -// TablePro -// - import Foundation import TableProPluginKit import TableProSQLGrammar -/// The one owner of how SQL text meets an engine: what each statement is sent as, and how a list of statements is -/// written into a script that engine's own client runs. -/// -/// The two are different texts, and one string serving as both is how a saved sync script came to run only its first -/// `DROP`. A driver takes one statement per call with no separator after it, and on Oracle the `;` after a unit's -/// `END` belongs to the unit while the one after a `CALL` trigger body stores the trigger INVALID. A script client -/// needs every statement ended its own way: SQL*Plus reads a PL/SQL unit until a line holding only `/`, the mysql -/// client ends a statement at its first `;` unless a `DELIMITER` block moves the delimiter, and SQL Server's tools -/// need a routine, a view or a trigger alone in its batch. -/// -/// Text is divided into statements only where the app tracks the engine's grammar: Oracle, MySQL, PostgreSQL and -/// SQLite. Every other engine's text is sent whole, as it always was, because the generic grammar cuts a T-SQL body -/// with no `BEGIN`, a Dameng declaration section and a Snowflake `$$` body into pieces the server rejects. -internal struct SQLScriptText { +// The one owner of how SQL text meets an engine: each statement as a driver takes it in one call, and a list of them as +// a script the engine's own client runs. One string serving as both made a saved sync script run only its first DROP. +// Only Oracle, MySQL, PostgreSQL and SQLite text is cut into statements: the generic grammar cuts a T-SQL body with no +// BEGIN, a Dameng declaration section and a Snowflake `$$` body into pieces the server rejects. +nonisolated struct SQLScriptText { private static let mysqlScriptDelimiterUnit = "/" // MySQL connects without CLIENT_MULTI_STATEMENTS, DuckDB prepares every text and SQLite refuses a tail. - private static let enginesRefusingASecondStatement: Set = [.sqlite, .duckdb] + private static let enginesRefusingASecondStatement: Set = ["SQLite", "DuckDB"] + + private static let oracleGrammar = SQLLexicalReadings.resolve( + databaseTypeId: "Oracle", declared: nil, session: nil + ).execution - internal let databaseType: DatabaseType + let databaseTypeId: String private let dialect: SqlDialect private let grammar: SQLLexicalGrammar private let readings: SQLLexicalReadings - internal init(databaseType: DatabaseType) { - self.init(databaseType: databaseType, grammar: databaseType.lexicalGrammar) + // `readings` is every way the engine could lex a text; its execution grammar splits the statements. + init(databaseTypeId: String, readings: SQLLexicalReadings) { + self.init(databaseTypeId: databaseTypeId, grammar: readings.execution, readings: readings) } // `grammar` is the session's execution grammar, which a server setting such as KingbaseES Oracle mode changes. - internal init(databaseType: DatabaseType, grammar: SQLLexicalGrammar) { - self.databaseType = databaseType - self.dialect = SqlDialect.from(databaseTypeId: databaseType.rawValue) + init(databaseTypeId: String, grammar: SQLLexicalGrammar, readings: SQLLexicalReadings) { + self.databaseTypeId = databaseTypeId + self.dialect = SqlDialect.from(databaseTypeId: databaseTypeId) self.grammar = grammar - self.readings = SQLLexicalReadings(execution: grammar, all: databaseType.lexicalReadings.all) + self.readings = SQLLexicalReadings(execution: grammar, all: readings.all) } - /// The statements `text` holds, each exactly as the driver receives it in one call, in order. - internal func sendableStatements(_ text: String) -> [String] { + // The statements `text` holds, each exactly as the driver receives it in one call, in order. + func sendableStatements(_ text: String) -> [String] { guard tracksGrammar else { let whole = StatementBlank.trimming(text) return whole.isEmpty ? [] : [whole] @@ -53,7 +44,7 @@ internal struct SQLScriptText { } // Nil when text the app did not write reads as several statements where the driver may run a second one. - internal func sendableStatements(ofForeignText text: String) -> [String]? { + func sendableStatements(ofForeignText text: String) -> [String]? { switch foreignTextSending { case .split: return sendableStatements(text) @@ -66,7 +57,7 @@ internal struct SQLScriptText { } } - private enum ForeignTextSending { + nonisolated private enum ForeignTextSending { case split // The driver refuses a second statement in one call, so the server decides where the text ends. case whole @@ -78,7 +69,7 @@ internal struct SQLScriptText { case .mysql: return .whole case .sqlite: - return Self.enginesRefusingASecondStatement.contains(databaseType) ? .whole : .oneStatement + return Self.enginesRefusingASecondStatement.contains(databaseTypeId) ? .whole : .oneStatement default: return .split } @@ -90,12 +81,9 @@ internal struct SQLScriptText { return StatementBlank.trimming(String(trimmed.unicodeScalars.dropLast())) } - /// The text that decides whether two definitions create the same object. - /// - /// Where the grammar is tracked that is the text that would run, which keeps a unit's own `;`, because Oracle - /// stores a procedure sent without it INVALID and one sent with it VALID. An engine whose grammar is not tracked - /// accepts a definition with or without a trailing `;`, so a difference in it is not one there. - internal func comparableText(_ definition: String) -> String { + // The text that decides whether two definitions create the same object. A tracked grammar keeps a unit's own `;`, + // since Oracle stores a procedure sent without it INVALID; any other engine takes a definition with or without one. + func comparableText(_ definition: String) -> String { guard tracksGrammar else { var text = StatementBlank.trimming(definition) while text.hasSuffix(";") { @@ -106,7 +94,7 @@ internal struct SQLScriptText { return sendableStatements(definition).joined(separator: ";\n") } - internal func leadingKeyword(of statement: String) -> String? { + func leadingKeyword(of statement: String) -> String? { let text = statement as NSString let length = text.length var index = 0 @@ -125,8 +113,8 @@ internal struct SQLScriptText { return nil } - /// Every statement in `statements`, which are sendable texts, written as one script for this engine's client. - internal func script(_ statements: [String]) -> String { + // Every statement in `statements`, which are sendable texts, written as one script for this engine's client. + func script(_ statements: [String]) -> String { let ended = statements .map { StatementBlank.trimming($0) } .filter { !$0.isEmpty } @@ -137,8 +125,8 @@ internal struct SQLScriptText { return ended.map { "\($0)\nGO" }.joined(separator: "\n") } - /// `ddl`, text a driver produced that may hold several statements, as script text for this engine's client. - internal func scriptText(forDriverText ddl: String) -> String { + // `ddl`, text a driver produced that may hold several statements, as script text for this engine's client. + func scriptText(forDriverText ddl: String) -> String { // A MySQL definition was recorded under its own sql_mode, so the export never cuts it with the session's. let statements = dialect == .mysql ? (sendableStatements(ofForeignText: ddl) ?? []) : sendableStatements(ddl) return statements.map(terminated).joined(separator: "\n") @@ -155,16 +143,15 @@ internal struct SQLScriptText { } } - /// Whether the engine's client reads a unit until a line holding only `/`. SQL*Plus does, and so does Dameng's - /// DISQL: measured on DM8, a script ending a procedure with `END;` ran the statement before it and then reported - /// "The script file is not complete" without running anything after it. + // SQL*Plus reads a unit until a line holding only `/`, and so does Dameng's DISQL: on DM8 a script ending a + // procedure with `END;` ran the statement before it, then reported "The script file is not complete". private var endsUnitsWithSlashLines: Bool { - dialect == .oracle || Self.slashLineClientEngines.contains(databaseType) + dialect == .oracle || Self.slashLineClientEngines.contains(databaseTypeId) } - private static let slashLineClientEngines: Set = [.dameng] + private static let slashLineClientEngines: Set = ["Dameng"] - /// One sendable statement, ended the way this engine's client ends it. + // One sendable statement, ended the way this engine's client ends it. private func terminated(_ statement: String) -> String { if endsUnitsWithSlashLines { return runsAsPLSQLUnit(statement) ? "\(statement)\n/" : endedBySemicolon(statement) @@ -175,14 +162,11 @@ internal struct SQLScriptText { return endedBySemicolon(statement) } - /// Whether the client reads `statement` in PL/SQL mode, where only a `/` line runs it. - /// - /// A definition takes no bind parameters and a block keeps its own `;`, and between them they are every - /// statement SQL*Plus buffers: a stored unit including a `CALL` trigger, `CREATE JAVA`, and an anonymous block. A - /// `CALL` trigger is the case a `;` cannot end, because Oracle stores it INVALID with one. The kind is read from - /// the statement's first words, which Dameng spells the way Oracle does. + // Whether the client reads `statement` in PL/SQL mode, where only a `/` line runs it: a definition, which takes no + // bind parameters, or a block, which keeps its `;`. A CALL trigger is the case a `;` cannot end: Oracle stores it + // INVALID. Dameng spells the first words the way Oracle does. private func runsAsPLSQLUnit(_ statement: String) -> Bool { - guard let located = SQLStatementScanner.executableStatements(in: statement, grammar: DatabaseType.oracle.lexicalGrammar).first else { + guard let located = SQLStatementScanner.executableStatements(in: statement, grammar: Self.oracleGrammar).first else { return false } return !located.acceptsBindParameters || located.sql.hasSuffix(";") @@ -212,7 +196,7 @@ internal struct SQLScriptText { // A comment the text leaves open would run over the terminator, even one the text already ends with: SQLite stores // `STRICT /* open` and `-- note` after a definition as written. - internal func endedBySemicolon(_ statement: String) -> String { + func endedBySemicolon(_ statement: String) -> String { let shapes = readings.distinct(for: statement).map { codeShape(of: statement, grammar: $0) } if shapes.contains(where: \.endsInOpenBlockComment) { return "\(statement) */;" @@ -227,7 +211,7 @@ internal struct SQLScriptText { readings.distinct(for: statement).contains { codeShape(of: statement, grammar: $0).endsInLineComment } } - private struct CodeShape { + nonisolated private struct CodeShape { var holdsSemicolon = false var endsInLineComment = false var endsInOpenBlockComment = false diff --git a/apple/TablePro/Core/Transfer/Shared/SQLWriteBatchBudget.swift b/apple/TablePro/Core/Transfer/Shared/SQLWriteBatchBudget.swift new file mode 100644 index 0000000000..ea9010f8c7 --- /dev/null +++ b/apple/TablePro/Core/Transfer/Shared/SQLWriteBatchBudget.swift @@ -0,0 +1,120 @@ +import Foundation +import TableProPluginKit + +// How much one parameterized INSERT may carry, in rows and in bytes. A parameter count is no size: MySQL sends a +// prepared statement as one packet, and one over `max_allowed_packet` fails the first batch (#2533). The byte cap is the +// mebibyte `mysqldump` uses for `net_buffer_length`, under every shipped `max_allowed_packet`. +nonisolated struct SQLWriteBatchBudget: Sendable, Equatable { + // However narrow the table: 65,535 one-column rows parse slowly, cannot be cancelled part-way, and save nothing past + // a thousand. + static let maximumRows = 1_000 + static let maximumBytes = 1_048_576 + + let maxRows: Int + let maxBytes: Int + + // A driver that renders values into the SQL text, as Databend does through `DatabendLiteral.inline`, pays for hex + // and escaping: two 400,000-byte values are 800,024 bytes of parameters but over 1.6 MiB of SQL. + let rendersValuesAsLiterals: Bool + + init( + maxRows: Int, + maxBytes: Int = SQLWriteBatchBudget.maximumBytes, + rendersValuesAsLiterals: Bool = false + ) { + self.maxRows = max(1, maxRows) + self.maxBytes = max(1, maxBytes) + self.rendersValuesAsLiterals = rendersValuesAsLiterals + } + + // The bind ceiling over the row's width, clamped by the multi-row VALUES ceiling, so Oracle keeps one row per + // statement. + init( + columnCount: Int, + statements: SQLImportStatements, + maxBytes: Int = SQLWriteBatchBudget.maximumBytes + ) { + self.init( + maxRows: min( + Self.maximumRows, + statements.maxRowsPerInsert, + statements.maxBindParameters / max(1, columnCount)), + maxBytes: maxBytes, + rendersValuesAsLiterals: statements.rendersValuesAsLiterals) + } + + // ClickHouse writes the values of a statement too long for the URL into it as escaped literals, binary as hex (#3777). + static func rendersValuesAsLiterals(databaseTypeId: String) -> Bool { + databaseTypeId == "Databend" || databaseTypeId == "ClickHouse" + } + + // An empty batch takes the row whatever it weighs: a row cannot be split, so it goes out alone, as `mysqldump` + // sends a row larger than its buffer. + func hasRoom(for rowBytes: Int, inBatchOf rows: Int, bytes: Int) -> Bool { + guard rows > 0 else { return true } + guard rows < maxRows else { return false } + return bytes + rowBytes <= maxBytes + } + + // An upper bound. Measured on MariaDB 12.3.3, a value costs about 6 bytes beyond its payload; `COM_STMT_EXECUTE` + // bounds it at 2 type bytes, a length prefix of at most 9 and a null bit, so 12 covers every size. + func byteCount(of values: Values) -> Int + where Values.Element == PluginCellValue { + var total = 0 + for value in values { + switch value { + case .null: + total += Self.valueOverheadBytes + case .text(let text): + total += Self.valueOverheadBytes + charged(text) + case .bytes(let data): + total += Self.valueOverheadBytes + charged(data.count) + } + } + return total + } + + // Double is what hex costs a binary value and the worst escaping of a text one, but for NUL: Databend writes `\x00`. + private func charged(_ bytes: Int) -> Int { + rendersValuesAsLiterals ? bytes * 2 : bytes + } + + private func charged(_ text: String) -> Int { + guard rendersValuesAsLiterals else { return text.utf8.count } + return text.utf8.reduce(into: text.utf8.count * 2) { total, byte in + if byte == 0 { total += 2 } + } + } + + private static let valueOverheadBytes = 12 +} + +// A batch closes before the row that would cross the budget, never after, so what is sent stays under it. +nonisolated struct SQLWriteBatchFiller { + private let budget: SQLWriteBatchBudget + private var rows: [Row] = [] + private var bytes = 0 + + init(budget: SQLWriteBatchBudget) { + self.budget = budget + } + + // The batch this row closed, if any. The row itself is always held for the next one. + mutating func append(_ row: Row, bytes rowBytes: Int) -> [Row]? { + var completed: [Row]? + if !budget.hasRoom(for: rowBytes, inBatchOf: rows.count, bytes: bytes) { + completed = take() + } + rows.append(row) + bytes += rowBytes + return completed + } + + mutating func take() -> [Row]? { + guard !rows.isEmpty else { return nil } + let batch = rows + rows.removeAll(keepingCapacity: true) + bytes = 0 + return batch + } +} diff --git a/apple/TablePro/Core/Utilities/File/FileDecompressor.swift b/apple/TablePro/Core/Utilities/File/FileDecompressor.swift index 1575d7d57e..3635477186 100644 --- a/apple/TablePro/Core/Utilities/File/FileDecompressor.swift +++ b/apple/TablePro/Core/Utilities/File/FileDecompressor.swift @@ -6,20 +6,6 @@ import Foundation import TableProGzip -enum DecompressionError: LocalizedError { - case decompressFailed - case fileReadFailed(String) - - var errorDescription: String? { - switch self { - case .decompressFailed: - return String(localized: "Failed to decompress .gz file") - case .fileReadFailed(let message): - return String(format: String(localized: "Failed to read file: %@"), message) - } - } -} - enum FileDecompressor { private static func innerExtension(for url: URL) -> String { let name = url.deletingPathExtension().pathExtension.lowercased() diff --git a/apple/TablePro/Core/Utilities/SQL/SQLDumpEncodingChoice.swift b/apple/TablePro/Core/Utilities/SQL/SQLDumpEncodingChoice.swift deleted file mode 100644 index 0f93059e52..0000000000 --- a/apple/TablePro/Core/Utilities/SQL/SQLDumpEncodingChoice.swift +++ /dev/null @@ -1,78 +0,0 @@ -// -// SQLDumpEncodingChoice.swift -// TablePro -// - -import Foundation -import TableProSQLGrammar -import TableProTabularIO -import TableProUtilities - -internal enum SQLDumpEncodingChoice { - private static let declarationScanLength = 65_536 - private static let declarationPattern = try? NSRegularExpression(pattern: [ - #"(?i)\A(?:\s+|--[^\n]*(?:\n|\z)|#[^\n]*(?:\n|\z)|/\*(?!M?!)[\s\S]*?\*/)*"#, - #"(?:/\*M?!\d*\s*)?SET\s+(?:(?:SESSION|LOCAL)\s+)?"#, - #"(?:NAMES|CHARACTER\s+SET|CHARSET|character_set_client\s*=|client_encoding\s*(?:=|TO))"#, - #"\s*['"]?([A-Za-z0-9_-]+)"# - ].joined()) - - private static let declaredNames: [String: ImportEncoding] = [ - "UTF8": .utf8, "UTF8MB4": .utf8, "UTF8MB3": .utf8, "UNICODE": .utf8, - "CP1252": .windows1252, "WIN1252": .windows1252, "WINDOWS1252": .windows1252, - "LATIN1MYSQL": .windows1252, "LATIN1": .latin1, "ISO88591": .latin1, - "ASCII": .ascii, "SQLASCII": .ascii, - "CP932": .shiftJIS, "SJIS": .shiftJIS, "SHIFTJIS": .shiftJIS, "SHIFTJIS2004": .shiftJIS, - "UJIS": .eucJP, "EUCJPMS": .eucJP, "EUCJP": .eucJP, - "GBK": .gb18030, "GB2312": .gb18030, "GB18030": .gb18030, "EUCCN": .gb18030, - "BIG5": .big5, "CP950": .big5, - "EUCKR": .eucKR, "UHC": .eucKR, "CP949": .eucKR - ] - - internal static func encoding( - replacing selected: ImportEncoding, - forPreview preview: Data, - isWholeFile: Bool, - family: TransactionEngineFamily, - grammar: SQLLexicalGrammar - ) -> ImportEncoding? { - if let declared = declaredEncoding(in: preview, family: family, grammar: grammar) { - guard declared != selected, decodes(preview, as: declared) else { return nil } - return declared - } - let sniff = TabularEncodingDetector.sniff(preview, isWholeFile: isWholeFile) - guard let detected = ImportEncoding(detected: sniff.encoding), detected != selected, - !detected.canHideABackslashInsideACharacter, decodes(preview, as: detected) else { - return nil - } - return detected - } - - internal static func declaredEncoding( - in preview: Data, - family: TransactionEngineFamily, - grammar: SQLLexicalGrammar - ) -> ImportEncoding? { - guard let declarationPattern, - let head = String(data: preview.prefix(declarationScanLength), encoding: .isoLatin1) else { - return nil - } - for statement in SQLStatementScanner.allStatements(in: head, grammar: grammar) { - let text = statement as NSString - guard let match = declarationPattern.firstMatch(in: statement, range: NSRange(location: 0, length: text.length)) else { - continue - } - let name = text.substring(with: match.range(at: 1)).uppercased().filter { $0.isLetter || $0.isNumber } - let key = name == "LATIN1" && family == .mysql ? "LATIN1MYSQL" : name - if let encoding = declaredNames[key] { - return encoding - } - } - return nil - } - - private static func decodes(_ preview: Data, as encoding: ImportEncoding) -> Bool { - var decoder = SQLChunkDecoder(encoding: encoding.encoding) - return decoder.decode(preview) != nil - } -} diff --git a/apple/TablePro/Core/Utilities/SQL/SQLLexicalResolver.swift b/apple/TablePro/Core/Utilities/SQL/SQLLexicalResolver.swift index a578d945dc..376eccca82 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLLexicalResolver.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLLexicalResolver.swift @@ -1,19 +1,11 @@ -// -// SQLLexicalResolver.swift -// TablePro -// - import Foundation import TableProPluginKit import TableProSQLGrammar import TableProSQLQuoting -/// The one place a connection is matched to the grammar its engine lexes with. -/// -/// Three sources, in order: the curated table in `TableProSQLGrammar`, which both apps read and which wins for every -/// engine TablePro ships; what a plugin for any other engine declares on its `SQLDialectDescriptor`; and what the -/// connected driver read from its own session. The last only chooses how a script is split for execution. The gates -/// read ``SQLLexicalReadings/all`` whatever the session says. +// The one place a connection is matched to its grammar: the curated table, which wins for every engine TablePro ships; +// else what the plugin declares; then what the driver read from its session, which only chooses how a script is split +// for execution. The gates read every reading whatever the session says. enum SQLLexicalResolver { static func readings( for databaseType: DatabaseType, @@ -26,14 +18,14 @@ enum SQLLexicalResolver { ) } - /// The grammar a script is split with for execution on `connectionId`, with the driver's session facts applied. + // The grammar a script is split with for execution on `connectionId`, with the driver's session facts applied. @MainActor static func executionGrammar(for databaseType: DatabaseType, connectionId: UUID) -> SQLLexicalGrammar { readings(for: databaseType, session: sessionState(for: connectionId)).execution } - /// The grammar for text bound to `scope`, read from the driver holding that database: a connection that keeps one - /// session per database can hold different session facts on each. + // Read from the driver holding that database: a connection that keeps one session per database can hold different + // session facts on each. @MainActor static func executionGrammar(for databaseType: DatabaseType, scope: DatabaseScope) -> SQLLexicalGrammar { readings(for: databaseType, session: DatabaseManager.shared.sessionDriver(holding: scope)?.sessionLexicalState) @@ -92,7 +84,7 @@ enum SQLLexicalResolver { stringEscapes(for: databaseType, session: sessionState(for: connectionId)) } - /// What the connected driver read from its session, or `nil` while `connectionId` has none. + // Nil while `connectionId` has no driver. @MainActor static func sessionState(for connectionId: UUID) -> PluginSessionLexicalState? { DatabaseManager.shared.driver(for: connectionId)?.sessionLexicalState @@ -105,67 +97,12 @@ enum SQLLexicalResolver { } extension DatabaseType { - /// Every way this engine could lex a text, for a gate. var lexicalReadings: SQLLexicalReadings { SQLLexicalResolver.readings(for: self) } - /// The grammar this engine is split with when no session says otherwise. + // The grammar this engine is split with when no session says otherwise. var lexicalGrammar: SQLLexicalGrammar { lexicalReadings.execution } } - -extension SQLLexicalGrammar { - /// The kit's feature for each grammar fact. The two sets carry the same bits by design; this table is what says - /// which is which, and `SQLLexicalFeatureMappingTests` holds every fact to a partner. A fact only the curated - /// table sets, such as `cqlBatches`, has none: no plugin can declare it, and it never reaches one. - static let pluginFeaturePairs: [(SQLLexicalFeatures, SQLLexicalGrammar)] = [ - (.backslashEscapesInSingleQuotes, .backslashEscapesInSingleQuotes), - (.backslashEscapesInDoubleQuotes, .backslashEscapesInDoubleQuotes), - (.backslashEscapesInBackticks, .backslashEscapesInBackticks), - (.backtickQuotes, .backtickQuotes), - (.bracketQuotedIdentifiers, .bracketQuotedIdentifiers), - (.tripleQuotedStrings, .tripleQuotedStrings), - (.escapeStringPrefix, .escapeStringPrefix), - (.alternativeQuoting, .alternativeQuoting), - (.untaggedDollarQuotes, .untaggedDollarQuotes), - (.taggedDollarQuotes, .taggedDollarQuotes), - (.nestedBlockComments, .nestedBlockComments), - (.hashLineComments, .hashLineComments), - (.doubleSlashLineComments, .doubleSlashLineComments), - (.executableComments, .executableComments), - (.slashLineTerminators, .slashLineTerminators), - (.dollarAndHashInIdentifiers, .dollarAndHashInIdentifiers), - (.plsqlBlocks, .plsqlBlocks), - (.delimiterDirective, .delimiterDirective), - (.dashCommentsNeedWhitespace, .dashCommentsNeedWhitespace), - (.parenthesizedParameterNames, .parenthesizedParameterNames), - (.doubledClosingBracketEscapes, .doubledClosingBracketEscapes), - (.carriageReturnEndsLineComments, .carriageReturnEndsLineComments), - (.batchSeparatorLines, .batchSeparatorLines), - (.unterminatedStatements, .unterminatedStatements), - (.terminatedMergeStatements, .terminatedMergeStatements), - ] - - init(pluginFeatures: SQLLexicalFeatures) { - self = Self.pluginFeaturePairs.reduce(into: []) { grammar, pair in - if pluginFeatures.contains(pair.0) { grammar.insert(pair.1) } - } - } - - var pluginFeatures: SQLLexicalFeatures { - Self.pluginFeaturePairs.reduce(into: []) { features, pair in - if contains(pair.1) { features.insert(pair.0) } - } - } -} - -extension SQLSessionLexicalFacts { - init(pluginState: PluginSessionLexicalState) { - self.init( - determined: SQLLexicalGrammar(pluginFeatures: pluginState.determined), - enabled: SQLLexicalGrammar(pluginFeatures: pluginState.enabled) - ) - } -} diff --git a/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift b/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift index acadf5177a..9042dba723 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift @@ -1,22 +1,13 @@ -// -// SQLRowToStatementConverter.swift -// TablePro - import Foundation import TableProAutocomplete import TableProPluginKit -/// Writes grid rows as INSERT and UPDATE text for the user to run, with the values written in. -/// -/// The statements follow the rules the grid's own save does: a column the server owns is never assigned, every key -/// column identifies the row, a row with no key is matched on every column it has through the same row-match policy, -/// and the WHERE clause names the row as it is stored, not as it has been edited. +// Grid rows as INSERT and UPDATE text to run, by the rules the grid's own save follows: a server-owned column is never +// assigned, a row with no key is matched on every column through the row-match policy, and the WHERE clause names the +// row as stored, not as edited. internal struct SQLRowToStatementConverter { - /// One row of the grid, as it is shown and as the server has it. - /// - /// `defaultedColumns` names the cells the grid staged as left to the server's default. The grid stages that as the - /// keyword DEFAULT marked as SQL, and a stored value can read the same, so only a cell named here is the default. - /// `expressionColumns` names the cells staged as SQL, which go out unquoted; text anywhere else is a literal. + // The grid stages a default as the keyword DEFAULT marked as SQL, and a stored value can read the same, so only a + // cell in `defaultedColumns` is the default. A cell in `expressionColumns` goes out unquoted; other text is a literal. internal struct SourceRow { internal let values: [PluginCellValue] internal let storedValues: [PluginCellValue] @@ -109,8 +100,7 @@ internal struct SQLRowToStatementConverter { rows.prefix(Self.maxRows).compactMap(insertStatement).joined(separator: "\n") } - /// A column the server owns, or one the row leaves to its default, stays out of the list, so the copied statement - /// inserts what a save of the same row would. + // A column the server owns, or one the row leaves to its default, stays out, so the copy inserts what a save would. private func insertStatement(row: SourceRow) -> String? { let written = zip(columns.indices, zip(columns, row.values)).filter { index, cell in !unwritableColumns.contains(cell.0) && !row.defaultedColumns.contains(index) @@ -118,7 +108,7 @@ internal struct SQLRowToStatementConverter { guard !written.isEmpty else { let firstWritable = columns.first { !unwritableColumns.contains($0) }.map(quoteIdentifierFn) return AllDefaultsInsert.sql( - into: qualifiedTable, databaseType: databaseType, firstWritableColumn: firstWritable + into: qualifiedTable, databaseTypeId: databaseType.rawValue, firstWritableColumn: firstWritable ).map { "\($0);" } } let columnList = written.map { quoteIdentifierFn($0.1.0) }.joined(separator: ", ") @@ -152,9 +142,8 @@ internal struct SQLRowToStatementConverter { rows.prefix(Self.maxRows).compactMap(updateStatement).joined(separator: "\n") } - /// Nothing for a row the table does not hold yet, for a key that is NULL (which identifies no row), for a row with - /// nothing left to assign, and for a table without a key that has a column no match can compare: leaving that - /// column out would let the statement change rows that differ only in it. + // Nothing for a row the table does not hold yet, a NULL key, a row with nothing to assign, or a keyless table with a + // column no match can compare: leaving it out would change rows that differ only in it. private func updateStatement(row: SourceRow) -> String? { guard !row.isPendingInsert, let whereClause = rowMatch(storedValues: row.storedValues) else { return nil } let assignments = assignments(for: row).joined(separator: ", ") @@ -168,8 +157,7 @@ internal struct SQLRowToStatementConverter { !primaryKeyColumns.isEmpty && primaryKeyColumns.allSatisfy(columns.contains) } - /// A key column is assigned only when its value was edited, so a changed natural key is carried and an unchanged - /// one is not restated. + // A key column is assigned only when its value was edited, so an unchanged natural key is not restated. private func assignments(for row: SourceRow) -> [String] { let keys = keyColumnsPresent ? Set(primaryKeyColumns) : [] return (settableColumns ?? columns).compactMap { column -> String? in @@ -208,7 +196,7 @@ internal struct SQLRowToStatementConverter { // MARK: - Literals - /// A marked cell is written raw only when the engine's vocabulary holds its text exactly. + // A marked cell is written raw only when the engine's vocabulary holds its text exactly. private func formatValue(_ value: PluginCellValue, isExpression: Bool) -> String { if isExpression, case .text(let sql) = value, CellExpressionVocabulary.isCanonical(sql, for: databaseType) { return sql diff --git a/apple/TablePro/Models/Export/ExportModels.swift b/apple/TablePro/Models/Export/ExportModels.swift index 288bdb1e30..c8c9d118d2 100644 --- a/apple/TablePro/Models/Export/ExportModels.swift +++ b/apple/TablePro/Models/Export/ExportModels.swift @@ -221,6 +221,9 @@ struct ExportObjectItem: Identifiable, Hashable { /// The table a trigger fires for. Nil for every other kind. let parentTable: String? + // Function or procedure, so the drop names the right statement. Nil for every other kind. + let routineKind: PluginRoutineKind? + var isSelected: Bool = false var optionValues: [Bool] = [] @@ -234,6 +237,7 @@ struct ExportObjectItem: Identifiable, Hashable { kind: PluginExportObjectKind, identity: String? = nil, parentTable: String? = nil, + routineKind: PluginRoutineKind? = nil, isSelected: Bool = false, optionValues: [Bool] = [], rowScope: PluginExportRowScope = .unrestricted @@ -244,6 +248,7 @@ struct ExportObjectItem: Identifiable, Hashable { self.kind = kind self.identity = identity self.parentTable = parentTable + self.routineKind = routineKind self.isSelected = isSelected self.optionValues = optionValues self.rowScope = rowScope @@ -491,17 +496,12 @@ extension ExportObjectItem { return normalizedItem } - /// Clears every option the format says this kind does not support, so a routine never carries a - /// `Data` flag that would make it look exportable for a phase it has no rows for. func maskingUnsupportedOptions( columns: [PluginExportOptionColumn], supports: (String, PluginExportObjectKind) -> Bool ) -> ExportObjectItem { - guard optionValues.count == columns.count else { return self } var masked = self - masked.optionValues = zip(columns, optionValues).map { column, value in - supports(column.id, kind) ? value : false - } + masked.optionValues = ExportOptionMask.masked(optionValues, columns: columns, kind: kind, supports: supports) return masked } diff --git a/apple/TablePro/Models/Export/ImportModels.swift b/apple/TablePro/Models/Export/ImportModels.swift index e13ac772fa..be86c5c983 100644 --- a/apple/TablePro/Models/Export/ImportModels.swift +++ b/apple/TablePro/Models/Export/ImportModels.swift @@ -10,12 +10,8 @@ import TableProTabularIO // MARK: - Import Encoding Options -/// The text encodings the SQL import dialog offers. -/// -/// The raw value is the key the last choice is stored under, so it stays put; `label` is what the -/// menu shows. Latin-1 and Windows-1252 are both here because they disagree over 0x80 to 0x9F, -/// which is where a dump written by MySQL keeps its curly quotes, en dashes and euro sign: read -/// as Latin-1 they arrive as C1 control characters instead. +// The raw value is the key the last choice is stored under. Latin-1 and Windows-1252 disagree over 0x80 to 0x9F, where +// a MySQL dump keeps its curly quotes, en dashes and euro sign. enum ImportEncoding: String, CaseIterable, Identifiable { case utf8 = "UTF-8" case utf16 = "UTF-16" @@ -37,15 +33,6 @@ enum ImportEncoding: String, CaseIterable, Identifiable { self = match } - var canHideABackslashInsideACharacter: Bool { - switch self { - case .shiftJIS, .gb18030, .big5: - return true - case .utf8, .utf16, .utf16LittleEndian, .utf16BigEndian, .latin1, .windows1252, .ascii, .eucJP, .eucKR: - return false - } - } - var label: String { switch self { case .utf8: return "UTF-8" diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index d69b4216d5..dad7f01cdf 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -3081,6 +3081,9 @@ } } } + }, + "%@ cannot take rows from a file." : { + }, "%@ cannot take rows from this file, so no table was created." : { "localizations" : { @@ -25960,6 +25963,10 @@ } } }, + "Another task needs this connection alone. Try again when it ends." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Answered with %d tools." : { "localizations" : { "fr" : { @@ -36107,6 +36114,10 @@ } } }, + "Cell %1$@ holds more than %2$@ of text, so the workbook was not read." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Cell selection cleared" : { "localizations" : { "fr" : { @@ -47481,6 +47492,10 @@ } } }, + "Compressing… %1$@ of %2$@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Compression" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -105075,6 +105090,8 @@ } }, "Line %@ holds binary data that cannot be read back exactly from a %@ file. Export the dump with --hex-blob, then import it again." : { + "extractionState" : "manual", + "generatesSymbol" : false, "localizations" : { "fr" : { "stringUnit" : { @@ -105178,7 +105195,8 @@ } }, "Line %lld runs COPY FROM STDIN, and this connection cannot load its rows. Run the file with psql, or dump it again with --inserts." : { - + "extractionState" : "manual", + "generatesSymbol" : false }, "Line %lld runs COPY FROM STDIN, whose rows TablePro cannot send. Run the file with psql, or dump it again with --inserts." : { @@ -158041,6 +158059,10 @@ } } }, + "Rows read together hold more than %1$@ of text by row %2$@, so the workbook was not read." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Rows sent" : { "localizations" : { "fr" : { @@ -192049,6 +192071,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The data could not be compressed." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The data could not be read: %@" : { }, @@ -199045,7 +199071,8 @@ } }, "The string or comment that starts on line %lld is longer than the import reads." : { - + "extractionState" : "manual", + "generatesSymbol" : false }, "The structure of a table in an attached database cannot be changed here." : { "extractionState" : "manual", @@ -201755,6 +201782,18 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "This connection cannot read its catalog for an export." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "This connection cannot send statements with bound values." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "This connection cannot stream rows to a file." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This connection comes from a file in a linked folder, so its author chose the server and how it signs in." : { }, @@ -204335,6 +204374,9 @@ } } } + }, + "This import format does not produce SQL statements" : { + }, "This import runs each statement on its own, so nothing can wait for Commit. Turn on Wrap in transaction or turn off Manual Commit." : { @@ -223551,6 +223593,14 @@ } } }, + "Writing definitions: %1$lld of %2$lld" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Writing the workbook: %1$@ of %2$@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Writing to %@" : { "localizations" : { "fr" : { diff --git a/apple/TablePro/Theme/MotionAccessibility.swift b/apple/TablePro/Theme/MotionAccessibility.swift index 95b99b4b4a..b76b7d5746 100644 --- a/apple/TablePro/Theme/MotionAccessibility.swift +++ b/apple/TablePro/Theme/MotionAccessibility.swift @@ -1,21 +1,26 @@ -// -// MotionAccessibility.swift -// TablePro -// - +#if canImport(AppKit) import AppKit +#else +import UIKit +#endif import SwiftUI +// Shared with the iOS app. internal enum MotionAccessibility { - /// `withAnimation(nil)` still performs the change, it just does not animate it, so a gated - /// call site never needs a second branch for the reduced case. + // `withAnimation(nil)` still performs the change without animating it, so a gated call site needs no second branch. internal static func animation(_ animation: Animation?, reduceMotion: Bool) -> Animation? { reduceMotion ? nil : animation } + #if canImport(AppKit) internal static var systemReduceMotion: Bool { NSWorkspace.shared.accessibilityDisplayShouldReduceMotion } + #else + @MainActor internal static var systemReduceMotion: Bool { + UIAccessibility.isReduceMotionEnabled + } + #endif } private struct MotionAnimation: ViewModifier { @@ -38,8 +43,7 @@ internal extension View { } } -/// For imperative call sites, where there is no environment to read. `withAnimation` captures the -/// animation at the call, so the setting has to be read here rather than passed down. +// For imperative call sites with no environment to read: `withAnimation` captures the animation at the call. internal func withMotion( _ animation: Animation? = .default, _ body: () throws -> Result diff --git a/apple/TablePro/ViewModels/LinkedFolderCommitModel.swift b/apple/TablePro/ViewModels/LinkedFolderCommitModel.swift index ac17409b3f..6d08fa6863 100644 --- a/apple/TablePro/ViewModels/LinkedFolderCommitModel.swift +++ b/apple/TablePro/ViewModels/LinkedFolderCommitModel.swift @@ -6,6 +6,7 @@ import Combine import Foundation import os +import TableProTabularIO import TableProUtilities internal enum LinkedFolderCommitDiff: Equatable, Sendable { diff --git a/apple/TablePro/Views/Export/ExportObjectRows.swift b/apple/TablePro/Views/Export/ExportObjectRows.swift index bef475a196..e0a8677cac 100644 --- a/apple/TablePro/Views/Export/ExportObjectRows.swift +++ b/apple/TablePro/Views/Export/ExportObjectRows.swift @@ -6,39 +6,8 @@ import SwiftUI import TableProPluginKit -/// How each object kind names and draws itself in the export tree. One table rather than a switch -/// per call site, so a kind added later cannot pick up a different icon in one of them. +// The kind's title and symbol are `ExportObjectKindLabel`'s; this adds what only the Mac's tree draws. internal enum ExportObjectKindPresentation { - internal static func groupTitle(for kind: PluginExportObjectKind) -> String { - switch kind { - case .table: return String(localized: "Tables") - case .view: return String(localized: "Views") - case .materializedView: return String(localized: "Materialized Views") - case .foreignTable: return String(localized: "Foreign Tables") - case .sequence: return String(localized: "Sequences") - case .userType: return String(localized: "Types") - case .routine: return String(localized: "Routines") - case .trigger: return String(localized: "Triggers") - case .event: return String(localized: "Events") - case .grant: return String(localized: "Privileges") - @unknown default: return String(localized: "Objects") - } - } - - internal static func iconName(for kind: PluginExportObjectKind) -> String { - switch kind { - case .table, .foreignTable: return "tablecells" - case .view, .materializedView: return "eye" - case .sequence: return "number" - case .userType: return "curlybraces" - case .routine: return "function" - case .trigger: return "bolt" - case .event: return "clock" - case .grant: return "key" - @unknown default: return "shippingbox" - } - } - internal static func checkboxValue(for state: TristateCheckbox.State) -> String { switch state { case .checked: String(localized: "Selected") @@ -114,7 +83,7 @@ internal struct ExportTreeObjectRow: View { .labelsHidden() .frame(width: 18) - Image(systemName: ExportObjectKindPresentation.iconName(for: object.kind)) + Image(systemName: ExportObjectKindLabel.systemImage(for: object.kind)) .foregroundStyle(ExportObjectKindPresentation.iconColor(for: object.kind)) .font(.body) .accessibilityHidden(true) diff --git a/apple/TablePro/Views/Export/ExportObjectTreeView.swift b/apple/TablePro/Views/Export/ExportObjectTreeView.swift index a2129390d7..70b555fafe 100644 --- a/apple/TablePro/Views/Export/ExportObjectTreeView.swift +++ b/apple/TablePro/Views/Export/ExportObjectTreeView.swift @@ -379,8 +379,8 @@ internal final class ExportObjectTreeCoordinator: NSObject, NSOutlineViewDataSou case .group(_, let objectKind): return AnyView( ExportTreeContainerRow( - title: ExportObjectKindPresentation.groupTitle(for: objectKind), - iconName: ExportObjectKindPresentation.iconName(for: objectKind), + title: ExportObjectKindLabel.title(for: objectKind), + iconName: ExportObjectKindLabel.systemImage(for: objectKind), iconColor: ExportObjectKindPresentation.iconColor(for: objectKind), state: containerState(for: node), toggle: { [weak self] in self?.toggleContainer(node) } @@ -559,7 +559,7 @@ internal final class ExportObjectTreeCoordinator: NSObject, NSOutlineViewDataSou case .database(let databaseID): return databases.first(where: { $0.id == databaseID })?.name ?? "" case .group(_, let objectKind): - return ExportObjectKindPresentation.groupTitle(for: objectKind) + return ExportObjectKindLabel.title(for: objectKind) case .object, .pending: return "" } diff --git a/apple/TablePro/Views/Export/ExportTreeBuilder.swift b/apple/TablePro/Views/Export/ExportTreeBuilder.swift index aa5608c184..3069542a14 100644 --- a/apple/TablePro/Views/Export/ExportTreeBuilder.swift +++ b/apple/TablePro/Views/Export/ExportTreeBuilder.swift @@ -423,7 +423,7 @@ struct ExportTreeBuilder { return await reader.loadObjects(request: request, tables: tables) } - /// Carries a row's checkboxes across a reload, falling back to what the preselection asked for. + // Carries a row's checkboxes across a reload, falling back to what the preselection asked for. private func restoring( _ object: ExportObjectItem, priorRows: [String: ExportRowSnapshot], @@ -439,6 +439,7 @@ struct ExportTreeBuilder { kind: object.kind, identity: object.identity, parentTable: object.parentTable, + routineKind: object.routineKind, isSelected: priorRow?.isSelected ?? preselection.selects( object: object.name, kind: object.kind, diff --git a/apple/TablePro/Views/Import/ImportDialog.swift b/apple/TablePro/Views/Import/ImportDialog.swift index 51662e3b6d..caa8fc3732 100644 --- a/apple/TablePro/Views/Import/ImportDialog.swift +++ b/apple/TablePro/Views/Import/ImportDialog.swift @@ -8,6 +8,7 @@ import Combine import os import SwiftUI import TableProPluginKit +import TableProSQLFile import TableProSQLGrammar import TableProUtilities @@ -439,17 +440,18 @@ struct ImportDialog: View { let maxPreviewSize = 5 * 1_024 * 1_024 let previewData = try handle.read(upToCount: maxPreviewSize) ?? Data() - var decoder = SQLChunkDecoder(encoding: selectedEncoding.encoding) - if let preview = decoder.decode(previewData) { + if let preview = SQLDumpEncodingChoice.decodedPreview( + previewData, as: selectedEncoding.encoding, isWholeFile: previewData.count < maxPreviewSize + ) { filePreview = preview hasPreviewError = false } else if let detected = SQLDumpEncodingChoice.encoding( - replacing: selectedEncoding, + replacing: selectedEncoding.encoding, forPreview: previewData, isWholeFile: previewData.count < maxPreviewSize, - family: TransactionEngineFamily.of(connection.type), + family: TransactionEngineFamily.of(connection.type).dumpFamily, grammar: sessionGrammar - ) { + ).flatMap({ found in ImportEncoding.allCases.first { $0.encoding == found } }) { Self.logger.info("Preview did not decode as \(selectedEncoding.rawValue, privacy: .public); switching to the detected \(detected.rawValue, privacy: .public)") detectedEncoding = detected selectedEncoding = detected diff --git a/apple/TablePro/Views/Import/RowImportTarget.swift b/apple/TablePro/Views/Import/RowImportTarget.swift index f6220239ff..e3917f0ad6 100644 --- a/apple/TablePro/Views/Import/RowImportTarget.swift +++ b/apple/TablePro/Views/Import/RowImportTarget.swift @@ -125,14 +125,3 @@ internal extension NewTableNameProblem { } } } - -internal extension NewTableColumnProblem { - var message: String { - switch self { - case .unnamedColumn: - return String(localized: "Every included column needs a name.") - case .duplicateName: - return String(localized: "Column names must be unique.") - } - } -} diff --git a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinator.swift b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinator.swift index 7e72b5f3a9..09a104d987 100644 --- a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinator.swift +++ b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinator.swift @@ -338,17 +338,24 @@ final class ConnectionCoordinator { appState.sessionTransactions.token(for: connection.id, on: session?.driver) } - // `reading` work is a read the app built, so a session lost under it cannot have committed anything. + // `reading` work is a read the app built, so a session lost under it cannot have committed anything. Throws + // `SessionHeldByImport` at once, before anything runs, while an import holds the session; that refusal checks no + // session, since nothing reached it. Other work is a write that leaves the session as a fresh connection would not + // have it, unless `marksSession` is off for work that reads its own statements (a query run). func track( reading: Bool = false, + marksSession: Bool = true, failed: (T) -> Bool = { _ in false }, rolledBack: (T) -> Bool = { _ in false }, _ work: () async throws -> T - ) async rethrows -> T { + ) async throws -> T { let heldBefore = transactionHold != nil let loss: SessionLoss = reading ? .whileIdle : .duringWork + let marks = !reading && marksSession + let driver = session?.driver do { let value = try await appState.sessionTransactions.track(connection.id, reading: reading, work) + if marks, let driver { noteSessionChanged(on: driver) } var lossSettled = false if failed(value) { lossSettled = await restoreSession(after: loss) @@ -360,12 +367,59 @@ final class ConnectionCoordinator { } await model.reloadMetadataIfReleased(from: self) return value + } catch let refusal as SessionHeldByImport { + throw refusal } catch { + if marks, let driver { noteSessionChanged(on: driver) } await restoreSession(after: loss) throw error } } + // Export Results runs the statement again on a connection of its own, which reads what the session read only while + // it is still that session, still where it read, and has run nothing since it connected that leaves state a fresh + // connection lacks: a write, a `SET`, a `USE`, a temporary table or a transaction statement. + func reproducesOnFreshConnection(_ placement: TablePlacement?) -> Bool { + guard let placement, placement == tablePlacement, let driver = session?.driver else { return false } + return !model.hasChangedSession(driver) + } + + func noteSessionChanged(on driver: any DatabaseDriver) { + model.noteSessionChanged(on: driver) + } + + // An import holds the session for its whole run. It is admitted only when nothing else runs on the session and + // nothing moves it, and the driver and the Safe Mode level it runs under are read in the turn that sets the mark, + // so a switch or a redial cannot hand it another session and a Safe Mode change cannot pass it by. + func trackImport( + failed: @Sendable (T) -> Bool, + _ work: @Sendable (ImportAdmission) async throws -> T + ) async throws -> T { + let admission = ImportAdmission(driver: try writableSession.get().driver, safeModeLevel: connection.safeModeLevel) + let value: T + do { + value = try await appState.sessionTransactions.track( + connection.id, importing: true, sessionMoving: isSwitching || model.isMovingSession + ) { + try await work(admission) + } + } catch let refusal as SessionHeldByImport { + throw refusal + } catch let refusal as SessionBusyForImport { + throw refusal + } catch { + noteSessionChanged(on: admission.driver) + await restoreSession(after: .duringWork) + throw error + } + noteSessionChanged(on: admission.driver) + if failed(value) { + await restoreSession(after: .duringWork) + } + await model.reloadMetadataIfReleased(from: self) + return value + } + // A list reload reading the session would end the page with its next statement, so the page waits for it; nothing // suspends between that wait and `track` marking the session busy. A page that fails on a dead session redials, // and the browser reloads on the new one. @@ -378,12 +432,13 @@ final class ConnectionCoordinator { await read() return } - await self.track(reading: true, failed: { _ in failed() }) { await read() } + try await self.track(reading: true, failed: { _ in failed() }) { await read() } } } // The token is what the indicator showed when the user asked, so a swapped session or newer work refuses. func commitTransaction(as token: SessionTransactionMonitor.HoldToken) async throws { + noteSessionChanged(on: token.driver) do { try await appState.sessionTransactions.commit(connection.id, as: token) } catch { @@ -398,6 +453,7 @@ final class ConnectionCoordinator { } func rollBackTransaction(as token: SessionTransactionMonitor.HoldToken) async throws { + noteSessionChanged(on: token.driver) do { try await appState.sessionTransactions.rollBack(connection.id, as: token) } catch { @@ -427,6 +483,11 @@ final class ConnectionCoordinator { var schema = false } + // What a run on `driver` left there that the catalog connection never sees: a role, a search path, a temporary table. + func noteSessionEffect(_ effect: SessionLocalStatements.Effect, on driver: any DatabaseDriver) { + appState.catalogChanges.noteSessionEffect(effect, for: connection.id, driver: ObjectIdentifier(driver)) + } + // A read that costs a statement replaces ROW_COUNT() and the warnings, so it is sent only after a run that can have // moved the session. The work runs in a task of its own, since Stop may have cancelled the run that asks. func followSessionPlacement(after move: SessionMove) async { @@ -434,14 +495,15 @@ final class ConnectionCoordinator { if move.schema { await followSessionSchema() } } + // An import holding the session keeps it where it is; the next run that can move it reads it again. private func followSessionDatabase(after move: SessionMove) async { - guard supportsDatabaseSwitching, let driver = session?.driver as? any SessionDatabaseReporting, + guard supportsDatabaseSwitching, !isImportRunning, let driver = session?.driver as? any SessionDatabaseReporting, move.database || driver.reportsDatabaseWithoutStatement else { return } await Task { let used = try? await driver.sessionDatabase() // With autocommit off, a MySQL read opens a transaction. if !driver.reportsDatabaseWithoutStatement { await refreshTransaction() } - guard let used, !used.isEmpty, used != activeDatabase, session?.driver === driver else { return } + guard let used, !used.isEmpty, used != activeDatabase, session?.driver === driver, !isImportRunning else { return } // An unsaved edit holds the session where it was made, as it holds the picker. if isHeldByEditor || anotherWindowHoldsEditor, !activeDatabase.isEmpty { let back = activeDatabase @@ -483,7 +545,7 @@ final class ConnectionCoordinator { // The Tables tab names its schema, so only completion and the picker read where the session is. private func followSessionSchema() async { - guard supportsSchemas, let driver = session?.driver as? any SessionSchemaReporting else { return } + guard supportsSchemas, !isImportRunning, let driver = session?.driver as? any SessionSchemaReporting else { return } await driver.settleCurrentSchema() guard session?.driver === driver, let current = driver.currentSchema, current != activeSchema else { return } appState.bumpSessionRevision(connection.id) @@ -507,6 +569,25 @@ final class ConnectionCoordinator { var isDatabaseMoving: Bool { model.isDatabaseMoving } + // Set in every window on the connection while an import holds its session. + var isImportRunning: Bool { appState.sessionTransactions.isHeldByImport(connection.id) } + + // Why a typed run cannot start now. `track` refuses a run an import holds the session against anyway. + var runRefusal: AppError? { + if isDatabaseMoving { return Self.databaseMovingRefusal } + return isImportRunning ? SessionHeldByImport().appError : nil + } + + // `work` runs once the import holding the session ends, or at once when none does. + func afterImport(_ work: @escaping @MainActor () -> Void) { + appState.sessionTransactions.afterImport(connection.id, work) + } + + // The message a refused `track` call shows, else what the error itself says. + static func refusal(for error: any Error) -> AppError? { + (error as? SessionHeldByImport)?.appError + } + static var databaseMovingRefusal: AppError { AppError( category: .query, @@ -524,6 +605,10 @@ final class ConnectionCoordinator { // `open` is the open this switch puts back where the connection was; the switch runs as a step of it. func switchDatabase(to name: String, inside open: ConnectionAttempt?) async { guard isAttached, session != nil || model.isSwitchingDatabase, name != activeDatabase, !isSwitching else { return } + guard !isImportRunning else { + presentFailure(title: Self.switchDatabaseFailedTitle, message: SessionHeldByImport().localizedDescription) + return + } guard !anotherWindowHoldsEditor else { presentFailure(title: Self.switchDatabaseFailedTitle, message: Self.editorHeldElsewhereMessage) return @@ -538,6 +623,10 @@ final class ConnectionCoordinator { func switchSchema(to name: String) async { guard let session, name != activeSchema, !isSwitching else { return } + guard !isImportRunning else { + presentFailure(title: Self.switchSchemaFailedTitle, message: SessionHeldByImport().localizedDescription) + return + } let failure = await whileSwitching(to: name) { await model.switchSchema(to: name, on: session) } @@ -563,6 +652,11 @@ final class ConnectionCoordinator { // A failure on a dead session redials first. The alert then says why, unless the session is lost, which its own // screen says. func report(_ failure: ConnectionFailure, operation: String) async { + // Refused before it reached the session, which is in use by an import, so there is nothing to check. + if failure.error is SessionHeldByImport { + presentFailure(title: failure.title, message: failure.error.localizedDescription) + return + } let failedOn = session?.driver if await restoreSession(after: .whileIdle) { return } guard !isLost, !failure.redialRepeats || session?.driver === failedOn else { return } diff --git a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinatorStore.swift b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinatorStore.swift index 1368652e4e..ef919aba9d 100644 --- a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinatorStore.swift +++ b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinatorStore.swift @@ -76,18 +76,29 @@ final class ConnectionCoordinatorStore { retired?.retire() generations[id, default: 0] += 1 guard droppingSession else { return } - retired?.dropSession() - disconnect(id) + dropSession(of: retired, id) } private func remove(_ id: UUID) { heldRebuilds[id] = nil - if let retired = coordinators.removeValue(forKey: id) { + let retired = coordinators.removeValue(forKey: id) + if let retired { removedRecords[id] = retired.connection retired.retire() + } + dropSession(of: retired, id) + } + + // An import holding the session ends its own transaction first; the edit's drop lands once it has. + private func dropSession(of retired: ConnectionCoordinator?, _ id: UUID) { + guard let retired else { + disconnect(id) + return + } + retired.afterImport { [weak self] in retired.dropSession() + self?.disconnect(id) } - disconnect(id) } private func disconnect(_ id: UUID) { diff --git a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionModel.swift b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionModel.swift index 1291979f5d..2f675fb09c 100644 --- a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionModel.swift +++ b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionModel.swift @@ -590,6 +590,8 @@ final class ConnectionSessionModel { // True when the redial dropped a held transaction and the outcome is already settled for the user. @discardableResult func restoreSession(after loss: SessionLoss, from scene: ConnectionCoordinator) async -> Bool { + // The check's ping would land between an import's batches; the import's own end checks the session. + guard !appState.sessionTransactions.isHeldByImport(connectionId) else { return false } // One check per connection, and a statement that failed while it ran makes the loss ambiguous. if let running = state.check, let task = running.task { if loss == .duringWork { running.ambiguous = true } @@ -893,6 +895,24 @@ final class ConnectionSessionModel { var isDatabaseMoving: Bool { databaseMoves > 0 } + // Sessions that ran something a fresh connection would not have: a write, a `SET`, a `USE`, a temporary table, a + // transaction statement. A redial lands another driver, which starts clean. + @ObservationIgnored private var changedSessions: Set = [] + + func noteSessionChanged(on driver: any DatabaseDriver) { + changedSessions.insert(ObjectIdentifier(driver)) + } + + func hasChangedSession(_ driver: any DatabaseDriver) -> Bool { + changedSessions.contains(ObjectIdentifier(driver)) + } + + // A switch, a run with a `USE`, a dial or a session check in flight: an import admitted now could be handed a + // session that is about to move or be replaced. + var isMovingSession: Bool { + switchesInFlight > 0 || databaseMoves > 0 || state.check != nil || state.isReconnecting || state.isOpening + } + func beginDatabaseMove() { databaseMoves += 1 } @@ -906,11 +926,19 @@ final class ConnectionSessionModel { return placementEpoch } + // Every door that moves the session refuses while an import holds it: a `USE` would land between its batches, and a + // PostgreSQL redial would tear its session down. + private var importRefusal: ConnectionFailure? { + guard appState.sessionTransactions.isHeldByImport(connectionId) else { return nil } + return ConnectionFailure(error: SessionHeldByImport(), title: ConnectionCoordinator.switchDatabaseFailedTitle, redialRepeats: false) + } + func switchDatabase( to name: String, from scene: ConnectionCoordinator, inside open: ConnectionAttempt? ) async -> ConnectionFailure? { + if let importRefusal { return importRefusal } let failure: ConnectionFailure? if redialsToSwitchDatabase { await reconnectWithDatabase(name, from: scene, inside: open) @@ -1050,6 +1078,9 @@ final class ConnectionSessionModel { } func switchSchema(to name: String, on session: ConnectionSession) async -> ConnectionFailure? { + guard !appState.sessionTransactions.isHeldByImport(connectionId) else { + return ConnectionFailure(error: SessionHeldByImport(), title: ConnectionCoordinator.switchSchemaFailedTitle, redialRepeats: false) + } let request = beginPlacementRequest() do { try await session.driver.switchSchema(to: name) diff --git a/apple/TableProMobile/TableProMobile/Coordinators/ScenePresenter.swift b/apple/TableProMobile/TableProMobile/Coordinators/ScenePresenter.swift index 2cad4279df..f3a7f6d2f0 100644 --- a/apple/TableProMobile/TableProMobile/Coordinators/ScenePresenter.swift +++ b/apple/TableProMobile/TableProMobile/Coordinators/ScenePresenter.swift @@ -14,6 +14,7 @@ enum SceneSheet: Identifiable { case settings case importFile(URL) case export + case importData(URL) var id: String { switch self { @@ -28,6 +29,7 @@ enum SceneSheet: Identifiable { case .settings: "settings" case .importFile(let url): "importFile-\(url.absoluteString)" case .export: "export" + case .importData(let url): "importData-\(url.absoluteString)" } } @@ -44,6 +46,11 @@ nonisolated struct PendingTableRequest: Hashable, Sendable { let tableName: String } +nonisolated struct PendingDataImport: Hashable, Sendable { + let connectionId: UUID + let fileURL: URL +} + @MainActor @Observable final class ScenePresenter { /// Why the stored connection is not being restored yet. Each reason is inserted and removed by @@ -60,6 +67,7 @@ final class ScenePresenter { private(set) var editingConnectionId: UUID? private(set) var pendingIntent: SceneIntent? private(set) var pendingTable: PendingTableRequest? + private(set) var pendingDataImport: PendingDataImport? private(set) var restoreHolds: Set = [] private(set) var editorHolds: Set = [] private(set) var heldImportFile: URL? @@ -215,4 +223,30 @@ final class ScenePresenter { self.pendingTable = nil return pendingTable.tableName } + + // An opened file goes to the connection on screen, which keeps its session; the import sheet over it asks for + // the table and offers another connection. + func requestDataImport(_ url: URL, in connectionId: UUID) { + pendingDataImport = PendingDataImport(connectionId: connectionId, fileURL: url) + } + + func takeDataImport(for connectionId: UUID) -> URL? { + guard let pendingDataImport, pendingDataImport.connectionId == connectionId else { return nil } + self.pendingDataImport = nil + return pendingDataImport.fileURL + } + + // A file waiting for a connection that is no longer the one on screen was never asked about, so it is dropped + // rather than shown the next time that connection opens. + func discardDataImport(unlessFor connectionId: UUID?) { + guard let pendingDataImport, pendingDataImport.connectionId != connectionId else { return } + self.pendingDataImport = nil + } + + // Choosing another connection for an opened file: the file waits for that connection, which opens the way a + // link to it does. + func moveDataImport(_ url: URL, to connectionId: UUID) { + requestDataImport(url, in: connectionId) + receive(.openConnection(connectionId, table: nil)) + } } diff --git a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Schema.swift b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Schema.swift index 9ba944ab2f..dc62281163 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Schema.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Schema.swift @@ -13,7 +13,7 @@ nonisolated extension DuckDBDriver { WHERE table_schema = '\(escapeLiteral(schemaName))' ORDER BY table_name """ - let result = try await actor.query(query) + let result = try await actor.query(query, lane: transferLane) return result.rows.compactMap { row in guard row.count > 0, let name = row[0] else { return nil } let typeString = (row.count > 1 ? row[1] : nil) ?? "BASE TABLE" @@ -31,7 +31,7 @@ nonisolated extension DuckDBDriver { AND table_name = '\(escapeLiteral(table))' ORDER BY ordinal_position """ - let result = try await actor.query(query) + let result = try await actor.query(query, lane: transferLane) let primaryKeys = try await fetchPrimaryKeyColumns(table: table, schema: schemaName) return result.rows.enumerated().compactMap { index, row in @@ -55,7 +55,7 @@ nonisolated extension DuckDBDriver { WHERE schema_name = '\(escapeLiteral(schemaName))' AND table_name = '\(escapeLiteral(table))' """ - let result = try await actor.query(query) + let result = try await actor.query(query, lane: transferLane) return result.rows.compactMap { row in guard row.count > 0, let name = row[0] else { return nil } let sql = row.count > 2 ? row[2] : nil @@ -94,7 +94,7 @@ nonisolated extension DuckDBDriver { AND kcu.table_name = '\(escapeLiteral(table))' """ do { - let result = try await actor.query(query) + let result = try await actor.query(query, lane: transferLane) return result.rows.compactMap { row in guard row.count >= 4, let name = row[0], @@ -121,7 +121,7 @@ nonisolated extension DuckDBDriver { func fetchSchemas() async throws -> [String] { let query = "SELECT schema_name FROM information_schema.schemata ORDER BY schema_name" - let result = try await actor.query(query) + let result = try await actor.query(query, lane: transferLane) return result.rows.compactMap { $0.first ?? nil } } @@ -130,7 +130,7 @@ nonisolated extension DuckDBDriver { } func switchSchema(to name: String) async throws { - _ = try await actor.query("SET schema = \(SQLQuoting.identifier(name))") + _ = try await actor.query("SET schema = \(SQLQuoting.identifier(name))", lane: transferLane) setCurrentSchema(name) } @@ -147,7 +147,7 @@ nonisolated extension DuckDBDriver { AND tc.table_schema = '\(escapeLiteral(schema))' AND tc.table_name = '\(escapeLiteral(table))' """ - let result = try await actor.query(query) + let result = try await actor.query(query, lane: transferLane) return Set(result.rows.compactMap { $0.first ?? nil }) } diff --git a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Streaming.swift b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Streaming.swift index c5e7f64c31..c21c9f53a9 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Streaming.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Streaming.swift @@ -72,6 +72,10 @@ nonisolated extension DuckDBDriver { } func executeStreaming(query: String, options: StreamOptions) -> AsyncThrowingStream { + // A transfer connection reads with `exportRows`; anything else it runs is read whole. + guard transferLane == nil else { + return QueryResultStreaming.stream(options: options) { try await self.execute(query: query) } + } let actor = self.actor let interruptHandle = actor.interruptHandle let token = DuckDBStreamToken() @@ -271,7 +275,7 @@ extension DuckDBActor { // MARK: - Column Planning - private func planColumns(connection: duckdb_connection, query: String) throws -> [DuckDBStreamColumn] { + func planColumns(connection: duckdb_connection, query: String) throws -> [DuckDBStreamColumn] { var probe = duckdb_result() guard duckdb_query(connection, Self.zeroRowQuery(for: query), &probe) != DuckDBError else { let message = duckdb_result_error(&probe).map { String(cString: $0) } ?? "Unknown DuckDB error" diff --git a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Transaction.swift b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Transaction.swift index aaf8769843..2d2b8dc614 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Transaction.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Transaction.swift @@ -43,29 +43,28 @@ nonisolated enum DuckDBTransactionProbe { nonisolated extension DuckDBDriver { func sessionTransactionState() async -> DriverTransactionState { - await actor.transactionState() + await actor.transactionState(lane: transferLane) } } extension DuckDBActor { // One actor turn for every reading, so no other statement lands between the two transaction ids. - func transactionState() -> DriverTransactionState { - guard connection != nil else { return .unknown } + func transactionState(lane: UInt64? = nil) -> DriverTransactionState { + guard let connection = connection(for: lane) else { return .unknown } // A statement would end the open stream inside DuckDB. Its consumer's own work reads the state once it ends. - guard streamResult == nil else { return .unknown } - let catalogType = probeReading(DuckDBTransactionProbe.catalogTypeQuery) + guard lane == nil ? streamResult == nil : !holdsExportResult(lane: lane) else { return .unknown } + let catalogType = probeReading(DuckDBTransactionProbe.catalogTypeQuery, on: connection) if let decided = DuckDBTransactionProbe.state(catalogType: catalogType) { return decided } return DuckDBTransactionProbe.state( - first: probeReading(DuckDBTransactionProbe.transactionIdQuery), - second: probeReading(DuckDBTransactionProbe.transactionIdQuery) + first: probeReading(DuckDBTransactionProbe.transactionIdQuery, on: connection), + second: probeReading(DuckDBTransactionProbe.transactionIdQuery, on: connection) ) } // duckdb_query is the one path that reports the error type; the message text is no contract. - private func probeReading(_ sql: String) -> DuckDBTransactionProbe.Reading { - guard let connection else { return .unreadable } + private func probeReading(_ sql: String, on connection: duckdb_connection) -> DuckDBTransactionProbe.Reading { var result = duckdb_result() defer { duckdb_destroy_result(&result) } guard duckdb_query(connection, sql, &result) != DuckDBError else { diff --git a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Transfer.swift b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Transfer.swift new file mode 100644 index 0000000000..e8bd97bab2 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Transfer.swift @@ -0,0 +1,403 @@ +import CDuckDB +import Foundation +import os +import TableProDatabase +import TableProModels +import TableProPluginKit +import TableProSQLQuoting + +// One extra `duckdb_connection` on the session's database. DuckDB gives it its own transaction, so an export reads one +// snapshot while the session writes, and the grid's next statement never ends its streamed result. +nonisolated struct DuckDBTransferLane { + let connection: duckdb_connection + let interruptHandle: DuckDBInterruptHandle + var exportResult: duckdb_result? + var exportColumns: [DuckDBStreamColumn] = [] + var exportGeneration: UInt64 = 0 +} + +nonisolated extension DuckDBDriver { + func catalogQuery(_ sql: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + try await actor.catalogRows(sql, parameters: parameters, lane: transferLane) + } + + // Bound as the Mac plugin binds: text as VARCHAR, bytes as a BLOB. + func executeParameterized(query: String, parameters: [PluginCellValue], typeNames: [String?]?) async throws + -> QueryResult { + let raw = try await actor.executeBound(query, parameters: parameters, lane: transferLane) + return QueryResult( + columns: raw.columnNames.enumerated().map { index, name in + ColumnInfo(name: name, typeName: raw.columnTypeNames[safe: index] ?? "", ordinalPosition: index) + }, + rows: raw.rows, + rowsAffected: raw.rowsAffected, + executionTime: raw.executionTime + ) + } + + // The session reads through a connection of the stream's own, so a grid statement cannot end the read. + func exportRows(query: String, readOnly: Bool) -> AsyncThrowingStream { + let cursor = DuckDBExportCursor( + actor: actor, + query: query, + lane: transferLane.map { ($0, laneInterrupt) }, + schema: transferLane == nil ? currentSchema : nil + ) + return AsyncThrowingStream(unfolding: { try await cursor.next() }) + } + + // Every close path of the session ends this connection before the database (`DuckDBActor.close`). + func openTransferDriver() async throws -> any DatabaseDriver { + guard transferLane == nil else { return self } + let lane = try await actor.openTransferLane(schema: currentSchema) + let driver = DuckDBDriver(transferOn: actor, lane: lane.id, interrupt: lane.interruptHandle, source: .inMemory) + driver.setCurrentSchema(currentSchema ?? "main") + return driver + } +} + +extension DuckDBActor { + private static let wholeText = StreamOptions(textTruncationBytes: .max) + + func connection(for lane: UInt64?) -> duckdb_connection? { + guard let lane else { return connection } + return transferLanes[lane]?.connection + } + + // Opened on the session's schema, so an unqualified name reads the table the session would. + func openTransferLane(schema: String?) throws -> (id: UInt64, interruptHandle: DuckDBInterruptHandle) { + guard let database else { throw DuckDBDriverError.notConnected } + var opened: duckdb_connection? + guard duckdb_connect(database, &opened) != DuckDBError, let opened else { + throw DuckDBDriverError.connectionFailed("Failed to create a DuckDB connection") + } + let handle = DuckDBInterruptHandle() + handle.attach(opened) + lastTransferLane &+= 1 + let id = lastTransferLane + transferLanes[id] = DuckDBTransferLane(connection: opened, interruptHandle: handle) + if let schema, schema != "main" { + do { + _ = try query("SET schema = \(SQLQuoting.identifier(schema))", lane: id) + } catch { + closeTransferLane(id) + throw error + } + } + return (id, handle) + } + + func closeTransferLane(_ id: UInt64) { + guard var lane = transferLanes.removeValue(forKey: id) else { return } + lane.interruptHandle.interrupt() + lane.interruptHandle.detach() + if var result = lane.exportResult { + duckdb_destroy_result(&result) + lane.exportResult = nil + } + var connection: duckdb_connection? = lane.connection + duckdb_disconnect(&connection) + } + + func closeTransferLanes() { + for id in Array(transferLanes.keys) { + closeTransferLane(id) + } + } + + func closeExportResult(lane id: UInt64) { + guard var result = transferLanes[id]?.exportResult else { return } + duckdb_destroy_result(&result) + transferLanes[id]?.exportResult = nil + transferLanes[id]?.exportColumns = [] + } + + func holdsExportResult(lane id: UInt64?) -> Bool { + guard let id else { return false } + return transferLanes[id]?.exportResult != nil + } + + // MARK: - Bound statements + + // Every row, bytes kept. A column of a type read only as text is read again cast to VARCHAR, as `query` does. + func catalogRows(_ sql: String, parameters: [PluginCellValue], lane: UInt64?) throws -> PluginQueryResult { + let start = Date() + var result = try run(sql, parameters: parameters, lane: lane) + let plan = Self.plan(of: &result) + if plan.contains(where: { $0.castToText }) { + duckdb_destroy_result(&result) + result = try run(Self.castedQuery(originalQuery: sql, columns: plan), parameters: parameters, lane: lane) + } + defer { duckdb_destroy_result(&result) } + let rowsChanged = Int(duckdb_rows_changed(&result)) + var rows: [[PluginCellValue]] = [] + while true { + var chunk = duckdb_fetch_chunk(result) + guard chunk != nil else { break } + defer { duckdb_destroy_data_chunk(&chunk) } + rows += Self.exportValues(chunk: chunk, plan: plan) + } + return PluginQueryResult( + columns: plan.map(\.name), + columnTypeNames: plan.map(\.typeName), + rows: rows, + rowsAffected: plan.isEmpty ? rowsChanged : 0, + executionTime: Date().timeIntervalSince(start) + ) + } + + // Run once: an import's statement writes, so it is never run again cast. A value of a type read only as text comes + // back nil. + func executeBound(_ sql: String, parameters: [PluginCellValue], lane: UInt64?) throws -> DuckDBRawResult { + let start = Date() + var result = try run(sql, parameters: parameters, lane: lane) + defer { duckdb_destroy_result(&result) } + let plan = Self.plan(of: &result) + var rows: [[String?]] = [] + while true { + var chunk = duckdb_fetch_chunk(result) + guard chunk != nil else { break } + defer { duckdb_destroy_data_chunk(&chunk) } + rows += Self.exportValues(chunk: chunk, plan: plan).map { row in + zip(row, plan).map { value, column in + guard !column.castToText else { return nil } + switch value { + case .null: return nil + case .text(let text): return text + case .bytes(let data): return data.base64EncodedString() + } + } + } + } + return DuckDBRawResult( + columnNames: plan.map(\.name), + columnTypeNames: plan.map(\.typeName), + rows: rows, + rowsAffected: Int(duckdb_rows_changed(&result)), + executionTime: Date().timeIntervalSince(start), + isTruncated: false + ) + } + + // With no values the text runs unprepared, as the Mac plugin's catalog reads do. + private func run(_ sql: String, parameters: [PluginCellValue], lane: UInt64?) throws -> duckdb_result { + guard let connection = connection(for: lane) else { throw DuckDBDriverError.notConnected } + if let lane { + closeExportResult(lane: lane) + } else { + closeStreamResult() + } + var result = duckdb_result() + guard !parameters.isEmpty else { + if duckdb_query(connection, sql, &result) == DuckDBError { + let message = duckdb_result_error(&result).map { String(cString: $0) } ?? "Unknown DuckDB error" + duckdb_destroy_result(&result) + throw DuckDBDriverError.queryFailed(message) + } + return result + } + var statement: duckdb_prepared_statement? + guard duckdb_prepare(connection, sql, &statement) != DuckDBError, let prepared = statement else { + let message = duckdb_prepare_error(statement).map { String(cString: $0) } ?? "Failed to prepare statement" + duckdb_destroy_prepare(&statement) + throw DuckDBDriverError.queryFailed(message) + } + defer { duckdb_destroy_prepare(&statement) } + try Self.bind(parameters, to: prepared) + if duckdb_execute_prepared(prepared, &result) == DuckDBError { + let message = duckdb_result_error(&result).map { String(cString: $0) } ?? "Unknown DuckDB error" + duckdb_destroy_result(&result) + throw DuckDBDriverError.queryFailed(message) + } + return result + } + + private static func bind(_ parameters: [PluginCellValue], to statement: duckdb_prepared_statement) throws { + guard Int(duckdb_nparams(statement)) == parameters.count else { + throw DuckDBDriverError.queryFailed("The statement does not take the values it was given") + } + for (index, parameter) in parameters.enumerated() { + let position = idx_t(index + 1) + let state: duckdb_state + switch parameter { + case .null: + state = duckdb_bind_null(statement, position) + case .text(let value): + state = duckdb_bind_varchar_length(statement, position, value, idx_t(value.utf8.count)) + case .bytes(let data): + // An empty Data has no base address, and a null pointer would bind NULL rather than an empty blob. + let bytes = data.isEmpty ? Data([0]) : data + state = bytes.withUnsafeBytes { buffer in + duckdb_bind_blob(statement, position, buffer.baseAddress, idx_t(data.count)) + } + } + if state == DuckDBError { + throw DuckDBDriverError.queryFailed("Failed to bind parameter at index \(index)") + } + } + } + + private static func plan(of result: inout duckdb_result) -> [DuckDBStreamColumn] { + (0.. [[PluginCellValue]] { + let size = duckdb_data_chunk_get_size(chunk) + guard size > 0 else { return [] } + let vectors = plan.indices.map { duckdb_data_chunk_get_vector(chunk, idx_t($0)) } + return (0.. PluginCellValue { + if let validity = duckdb_vector_get_validity(vector), !duckdb_validity_row_is_valid(validity, row) { + return .null + } + if !column.castToText, column.type == DUCKDB_TYPE_BLOB { + guard let data = duckdb_vector_get_data(vector) else { return .null } + var value = data.assumingMemoryBound(to: duckdb_string_t.self)[Int(row)] + let length = Int(duckdb_string_t_length(value)) + guard length > 0, let pointer = duckdb_string_t_data(&value) else { return .bytes(Data()) } + return .bytes(Data(bytes: pointer, count: length)) + } + switch decodeCell(vector: vector, row: row, column: column, options: wholeText) { + case .null, .binary: + return .null + case .text(let text): + return .text(text) + case .truncatedText(let prefix, _, _): + return .text(prefix) + } + } + + // MARK: - Export stream + + func beginExport(query: String, lane id: UInt64) throws -> (header: PluginStreamHeader, generation: UInt64) { + guard let lane = transferLanes[id] else { throw DuckDBDriverError.notConnected } + try Task.checkCancellation() + closeExportResult(lane: id) + let plan = try planColumns(connection: lane.connection, query: query) + let streamQuery = plan.contains(where: { $0.castToText }) ? Self.castedQuery(originalQuery: query, columns: plan) : query + + var statement: duckdb_prepared_statement? + guard duckdb_prepare(lane.connection, streamQuery, &statement) != DuckDBError, let prepared = statement else { + let message = duckdb_prepare_error(statement).map { String(cString: $0) } ?? "Failed to prepare statement" + duckdb_destroy_prepare(&statement) + throw DuckDBDriverError.queryFailed(message) + } + defer { duckdb_destroy_prepare(&statement) } + var pending: duckdb_pending_result? + guard duckdb_pending_prepared_streaming(prepared, &pending) != DuckDBError, pending != nil else { + let message = duckdb_pending_error(pending).map { String(cString: $0) } ?? "Failed to execute query" + duckdb_destroy_pending(&pending) + throw DuckDBDriverError.queryFailed(message) + } + defer { duckdb_destroy_pending(&pending) } + + var result = duckdb_result() + guard duckdb_execute_pending(pending, &result) != DuckDBError else { + let message = duckdb_result_error(&result).map { String(cString: $0) } ?? "Failed to execute query" + duckdb_destroy_result(&result) + throw DuckDBDriverError.queryFailed(message) + } + let generation = lane.exportGeneration &+ 1 + transferLanes[id]?.exportGeneration = generation + transferLanes[id]?.exportResult = result + transferLanes[id]?.exportColumns = plan + return (PluginStreamHeader(columns: plan.map(\.name), columnTypeNames: plan.map(\.typeName)), generation) + } + + // Nil at the end, which releases the result. + func nextExportChunk(lane id: UInt64, generation: UInt64) throws -> [[PluginCellValue]]? { + guard let lane = transferLanes[id], lane.exportGeneration == generation, var result = lane.exportResult else { + throw CancellationError() + } + try Task.checkCancellation() + var chunk = duckdb_stream_fetch_chunk(result) + guard chunk != nil else { + let failure = duckdb_result_error(&result).map { String(cString: $0) } + closeExportResult(lane: id) + if let failure { throw DuckDBDriverError.queryFailed(failure) } + return nil + } + defer { duckdb_destroy_data_chunk(&chunk) } + return Self.exportValues(chunk: chunk, plan: lane.exportColumns) + } + + func endExport(lane id: UInt64, generation: UInt64) { + guard transferLanes[id]?.exportGeneration == generation else { return } + closeExportResult(lane: id) + } +} + +// The pull side of one export read. A stream read through the session opens a connection of its own and ends it with +// the read. +nonisolated private final class DuckDBExportCursor: @unchecked Sendable { + private let actor: DuckDBActor + private let query: String + private let schema: String? + private let lock = NSLock() + private var lane: UInt64? + private var interruptHandle: DuckDBInterruptHandle? + private let ownsLane: Bool + private var generation: UInt64? + + init(actor: DuckDBActor, query: String, lane: (id: UInt64, interruptHandle: DuckDBInterruptHandle?)?, schema: String?) { + self.actor = actor + self.query = query + self.schema = schema + self.lane = lane?.id + self.interruptHandle = lane?.interruptHandle + self.ownsLane = lane == nil + } + + deinit { + let actor = actor + let state = lock.withLock { (lane, generation) } + let ownsLane = ownsLane + guard let lane = state.0 else { return } + Task { + if let generation = state.1 { + await actor.endExport(lane: lane, generation: generation) + } + if ownsLane { + await actor.closeTransferLane(lane) + } + } + } + + func next() async throws -> PluginStreamElement? { + let actor = actor + return try await withTaskCancellationHandler { + let lane = try await resolvedLane() + if let generation = lock.withLock({ self.generation }) { + guard let rows = try await actor.nextExportChunk(lane: lane, generation: generation) else { return nil } + return .rows(rows) + } + let begun = try await actor.beginExport(query: query, lane: lane) + lock.withLock { generation = begun.generation } + return .header(begun.header) + } onCancel: { + lock.withLock { interruptHandle }?.interrupt() + } + } + + private func resolvedLane() async throws -> UInt64 { + if let lane = lock.withLock({ lane }) { return lane } + let opened = try await actor.openTransferLane(schema: schema) + lock.withLock { + lane = opened.id + interruptHandle = opened.interruptHandle + } + return opened.id + } +} diff --git a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver.swift index 081f7c8430..0d7ba34d26 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver.swift @@ -5,8 +5,12 @@ import TableProDatabase import TableProModels import TableProSQLQuoting -nonisolated final class DuckDBDriver: DatabaseDriver, @unchecked Sendable { - let actor = DuckDBActor() +nonisolated final class DuckDBDriver: DatabaseDriver, CatalogQuerying, ExportRowStreaming, + DedicatedTransferConnection, @unchecked Sendable { + let actor: DuckDBActor + // A second connection on the session's database, which a transfer reads through; nil for the session itself. + let transferLane: UInt64? + let laneInterrupt: DuckDBInterruptHandle? private let source: LocalDatabaseFileSource private let openMode: LocalDatabaseOpenMode private let fileAccess = LocalDatabaseFileAccess() @@ -16,7 +20,8 @@ nonisolated final class DuckDBDriver: DatabaseDriver, @unchecked Sendable { var supportsSchemas: Bool { true } var supportsTransactions: Bool { true } var serverVersion: String? { String(cString: duckdb_library_version()) } - var holdsSuspensionBlockingResource: Bool { source != .inMemory } + // The session holds the file; a transfer connection lives and ends inside it. + var holdsSuspensionBlockingResource: Bool { transferLane == nil && source != .inMemory } // DuckDB ends statement text at a NUL, so a NUL in a value goes in as chr(0) between two literals. func escapeStringLiteral(_ value: String) -> String { @@ -32,13 +37,25 @@ nonisolated final class DuckDBDriver: DatabaseDriver, @unchecked Sendable { } init(source: LocalDatabaseFileSource, openMode: LocalDatabaseOpenMode = .existingOnly) { + self.actor = DuckDBActor() + self.transferLane = nil + self.laneInterrupt = nil self.source = source self.openMode = openMode } + init(transferOn actor: DuckDBActor, lane: UInt64, interrupt: DuckDBInterruptHandle, source: LocalDatabaseFileSource) { + self.actor = actor + self.transferLane = lane + self.laneInterrupt = interrupt + self.source = source + self.openMode = .existingOnly + } + // MARK: - Connection func connect() async throws { + guard transferLane == nil else { return } let path = try fileAccess.begin(source, openMode: openMode) do { try await actor.open(path: path) @@ -49,20 +66,24 @@ nonisolated final class DuckDBDriver: DatabaseDriver, @unchecked Sendable { } func disconnect() async throws { + if let transferLane { + await actor.closeTransferLane(transferLane) + return + } await actor.close() fileAccess.end() } // In-process, so an open handle is a live session; an aborted transaction refuses every statement until ROLLBACK. func ping() async throws -> Bool { - guard await actor.isOpen else { throw DuckDBDriverError.notConnected } + guard await actor.isOpen(lane: transferLane) else { throw DuckDBDriverError.notConnected } return true } // MARK: - Query Execution func execute(query: String) async throws -> QueryResult { - let raw = try await actor.query(query) + let raw = try await actor.query(query, lane: transferLane) let columns = raw.columnNames.enumerated().map { index, name in ColumnInfo( name: name, @@ -81,21 +102,21 @@ nonisolated final class DuckDBDriver: DatabaseDriver, @unchecked Sendable { /// Runs off the actor: the actor is busy inside the query for as long as it runs. func cancelCurrentQuery() async throws { - actor.interruptHandle.interrupt() + (laneInterrupt ?? actor.interruptHandle).interrupt() } // MARK: - Transactions func beginTransaction() async throws { - _ = try await actor.query("BEGIN TRANSACTION") + _ = try await actor.query("BEGIN TRANSACTION", lane: transferLane) } func commitTransaction() async throws { - _ = try await actor.query("COMMIT") + _ = try await actor.query("COMMIT", lane: transferLane) } func rollbackTransaction() async throws { - _ = try await actor.query("ROLLBACK") + _ = try await actor.query("ROLLBACK", lane: transferLane) } // MARK: - State @@ -173,8 +194,11 @@ actor DuckDBActor { nonisolated var unownedExecutor: UnownedSerialExecutor { executor.asUnownedSerialExecutor() } nonisolated let interruptHandle = DuckDBInterruptHandle() - private var database: duckdb_database? + private(set) var database: duckdb_database? var connection: duckdb_connection? + // The transfer connections on `database`, which `close()` ends before the database. + var transferLanes: [UInt64: DuckDBTransferLane] = [:] + var lastTransferLane: UInt64 = 0 /// One result streams at a time. Each `beginStream` gets a new generation, so the producer of a stream that a /// later one replaced cannot fetch from or destroy the later result. @@ -237,7 +261,14 @@ actor DuckDBActor { var isOpen: Bool { database != nil && connection != nil } + func isOpen(lane: UInt64?) -> Bool { + guard let lane else { return isOpen } + return database != nil && transferLanes[lane] != nil + } + + // A connection left open after `duckdb_close` keeps reading the file, so each transfer connection goes first. func close() { + closeTransferLanes() interruptHandle.detach() closeStreamResult() if connection != nil { @@ -250,11 +281,15 @@ actor DuckDBActor { } } - func query(_ sql: String) throws -> DuckDBRawResult { - guard let connection else { throw DuckDBDriverError.notConnected } + func query(_ sql: String, lane: UInt64? = nil) throws -> DuckDBRawResult { + guard let connection = connection(for: lane) else { throw DuckDBDriverError.notConnected } // Another statement closes an open streaming result inside DuckDB, and its next fetch would fail with an // internal error. Closing it here ends that stream as cancelled. - closeStreamResult() + if let lane { + closeExportResult(lane: lane) + } else { + closeStreamResult() + } let startTime = Date() var result = duckdb_result() diff --git a/apple/TableProMobile/TableProMobile/Drivers/MSSQLDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/MSSQLDriver.swift index e00048133e..10e75b7e4e 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/MSSQLDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/MSSQLDriver.swift @@ -18,6 +18,18 @@ nonisolated private extension MSSQLRawResult { isTruncated: isTruncated ) } + + // Bytes stay bytes, as the Mac's plugin bridges them. + nonisolated func toPluginResult(executionTime: TimeInterval) -> PluginQueryResult { + PluginQueryResult( + columns: columns.map(\.name), + columnTypeNames: columns.map(\.type.canonicalName), + rows: rows.map { $0.map(\.pluginCell) }, + rowsAffected: affectedRows, + executionTime: executionTime, + isTruncated: isTruncated + ) + } } nonisolated final class MSSQLDriver: DatabaseDriver, SessionDatabaseReporting, @unchecked Sendable { @@ -47,6 +59,9 @@ nonisolated final class MSSQLDriver: DatabaseDriver, SessionDatabaseReporting, @ } nonisolated(unsafe) private(set) var serverVersion: String? + // What an export learned of the server's principals, for the life of the connection, as on the Mac. + let principalState = OSAllocatedUnfairLock(initialState: MSSQLPrincipalState()) + // This form collects no CA for FreeTDS, so a verify mode from the Mac or an import cannot be honoured here. static let verifyModeUnavailableMessage = String(localized: """ SQL Server on this device cannot check the server's certificate. Set SSL Mode to Required in the \ @@ -160,6 +175,75 @@ nonisolated final class MSSQLDriver: DatabaseDriver, SessionDatabaseReporting, @ } } + private func pluginResult(_ query: String) async throws -> PluginQueryResult { + let startTime = Date() + do { + let raw = try await conn.executeQuery(query) + return raw.toPluginResult(executionTime: Date().timeIntervalSince(startTime)) + } catch let error as MSSQLCoreError { + throw mapToConnectionError(error) + } + } + + func executeParameterized(query: String, parameters: [PluginCellValue], typeNames: [String?]?) async throws + -> QueryResult { + try await runQuery(Self.boundStatement(query, parameters: parameters)) + } + + // The Mac's binding: `?` placeholders become `sp_executesql` parameters typed by their values. + static func boundStatement(_ query: String, parameters: [PluginCellValue]) -> String { + guard !parameters.isEmpty else { return query } + let statement = MSSQLParameterBatch.spExecuteSql(query: query, parameters: parameters.map(MSSQLParameter.init)) + return statement.isEmpty ? query : statement.executeSqlText + } + + // In `database`, or where the app last put the session when it names none. + func catalogCells(_ statement: String, parameters: [PluginCellValue] = [], in database: String?) async throws + -> PluginQueryResult { + try await pluginResult(catalogRead(Self.boundStatement(statement, parameters: parameters), in: database)) + } + + // The first result set's rows, a FreeTDS batch at a time. A consumer that stops or lets the stream go ends the + // request on the server. + func exportRows(query: String, readOnly: Bool) -> AsyncThrowingStream { + let conn = self.conn + let abort = MSSQLStreamAbort() + let coreStream = AsyncThrowingStream( + bufferingPolicy: .bufferingOldest(PluginRowStream.boundedBufferBatches) + ) { continuation in + continuation.onTermination = { @Sendable _ in abort.abort() } + Task { + do { + try await conn.streamQuery(query, isAborted: { abort.isAborted }, continuation: continuation) + } catch { + continuation.finish(throwing: error) + } + } + } + let source = MSSQLPulledElements(coreStream) + return AsyncThrowingStream(unfolding: { [self] in + do { + while let element = try await source.next() { + switch element { + case .header(let columns): + return .header(PluginStreamHeader( + columns: columns.map(\.name), columnTypeNames: columns.map(\.type.canonicalName) + )) + case .rows(let batch): + return .rows(batch.map { $0.map(\.pluginCell) }) + case .affectedRows: + continue + } + } + // A cancelled reader's iterator ends as if the rows had, which must not read as the whole table. + try Task.checkCancellation() + return nil + } catch let error as MSSQLCoreError { + throw mapToConnectionError(error) + } + }) + } + func cancelCurrentQuery() async throws { conn.cancelCurrentQuery() } @@ -315,7 +399,12 @@ nonisolated final class MSSQLDriver: DatabaseDriver, SessionDatabaseReporting, @ // MARK: - Table Metadata - private var effectiveSchema: String { currentSchema ?? "dbo" } + var effectiveSchema: String { currentSchema ?? "dbo" } + + // An empty schema is the current one, as on the Mac, never a schema named "". + private func schemaName(_ schema: String?) -> String { + MSSQLExportCatalog.effectiveSchema(schema, current: effectiveSchema) + } // A system procedure called through a database name runs in that database and leaves the session where it is. static func catalogStatement(_ statement: String, in database: String?) -> String { @@ -337,7 +426,7 @@ nonisolated final class MSSQLDriver: DatabaseDriver, SessionDatabaseReporting, @ func fetchTables(schema: String?) async throws -> [TableInfo] { let result = try await runQuery( - catalogRead(MSSQLSchemaQueries.tables(schema: schema ?? effectiveSchema), in: nil) + catalogRead(MSSQLSchemaQueries.tables(schema: schemaName(schema)), in: nil) ) return result.rows.compactMap { row in MSSQLSchemaQueries.parseTableRow(row).map { @@ -352,7 +441,7 @@ nonisolated final class MSSQLDriver: DatabaseDriver, SessionDatabaseReporting, @ func fetchColumns(table: String, schema: String?, database: String?) async throws -> [ColumnInfo] { let result = try await runQuery( - catalogRead(MSSQLSchemaQueries.columns(schema: schema ?? effectiveSchema, table: table), in: database) + catalogRead(MSSQLSchemaQueries.columns(schema: schemaName(schema), table: table), in: database) ) return result.rows.enumerated().compactMap { idx, row in MSSQLSchemaQueries.parseColumnRow(row).map { parsed in @@ -378,7 +467,7 @@ nonisolated final class MSSQLDriver: DatabaseDriver, SessionDatabaseReporting, @ func fetchIndexes(table: String, schema: String?, database: String?) async throws -> [IndexInfo] { let result = try await runQuery( - catalogRead(MSSQLSchemaQueries.indexes(schema: schema ?? effectiveSchema, table: table), in: database) + catalogRead(MSSQLSchemaQueries.indexes(schema: schemaName(schema), table: table), in: database) ) var byName: [String: (unique: Bool, primary: Bool, cols: [String])] = [:] for row in result.rows { @@ -399,7 +488,7 @@ nonisolated final class MSSQLDriver: DatabaseDriver, SessionDatabaseReporting, @ func fetchForeignKeys(table: String, schema: String?, database: String?) async throws -> [ForeignKeyInfo] { let result = try await runQuery( - catalogRead(MSSQLSchemaQueries.foreignKeys(schema: schema ?? effectiveSchema, table: table), in: database) + catalogRead(MSSQLSchemaQueries.foreignKeys(schema: schemaName(schema), table: table), in: database) ) return result.rows.compactMap { row in MSSQLSchemaQueries.parseForeignKeyRow(row).map { @@ -445,6 +534,37 @@ nonisolated private extension MSSQLRawCell { guard case .string(let value) = self else { return nil } return value } + + var pluginCell: PluginCellValue { + switch self { + case .null: return .null + case .string(let value): return .text(value) + case .bytes(let data): return .bytes(data) + } + } +} + +nonisolated private extension MSSQLParameter { + init(_ value: PluginCellValue) { + switch value { + case .null: self = .null + case .text(let text): self = .text(text) + case .bytes(let data): self = .bytes(data) + } + } +} + +// One consumer pulls the export stream, so one call at a time reaches the iterator. +nonisolated private final class MSSQLPulledElements: @unchecked Sendable { + private var iterator: AsyncThrowingStream.AsyncIterator + + init(_ stream: AsyncThrowingStream) { + iterator = stream.makeAsyncIterator() + } + + func next() async throws -> MSSQLStreamElement? { + try await iterator.next() + } } nonisolated private final class MSSQLStreamAbort: @unchecked Sendable { diff --git a/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift index 23c17277fa..9576c4ae35 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/MongoDBBackedDriver.swift @@ -32,6 +32,15 @@ nonisolated final class MongoDBBackedDriver: PluginRowWritingDriver, DriverQuery } func pageQuery(_ request: TableReadRequest, limit: Int, offset: Int) -> String { + browseQuery(request, limit: limit, offset: offset) + } + + // Every document the browse shows, in its order: what an export of a filtered or sorted collection reads. + func exportQuery(_ request: TableReadRequest) -> String { + browseQuery(request, limit: nil, offset: 0) + } + + private func browseQuery(_ request: TableReadRequest, limit: Int?, offset: Int) -> String { let builder = mongo.filterQueryBuilder(for: request.table, database: pinnedDatabase) let sorted = request.sortState.columns return builder.buildQuery( diff --git a/apple/TableProMobile/TableProMobile/Drivers/MySQLDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/MySQLDriver.swift index 68fb65dfca..f2aded1912 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/MySQLDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/MySQLDriver.swift @@ -10,7 +10,8 @@ import TableProPluginKit import TableProSQLGrammar import TableProSQLQuoting -nonisolated final class MySQLDriver: DatabaseDriver, SessionDatabaseReporting, @unchecked Sendable { +nonisolated final class MySQLDriver: DatabaseDriver, SessionDatabaseReporting, CatalogQuerying, ExportRowStreaming, + @unchecked Sendable { private let actor = MySQLActor() private let host: String private let port: Int @@ -41,6 +42,7 @@ nonisolated final class MySQLDriver: DatabaseDriver, SessionDatabaseReporting, @ } private let settledFacts = OSAllocatedUnfairLock(initialState: nil) + private let schemaNameCase = OSAllocatedUnfairLock(initialState: nil) var settledLexicalFacts: SQLSessionLexicalFacts? { settledFacts.withLock { $0 } @@ -183,7 +185,11 @@ nonisolated final class MySQLDriver: DatabaseDriver, SessionDatabaseReporting, @ func execute(query: String) async throws -> QueryResult { let raw = try await actor.execute(query) await settleLexicalFacts() - return QueryResult( + return Self.queryResult(raw) + } + + private static func queryResult(_ raw: RawMySQLResult) -> QueryResult { + QueryResult( columns: raw.columns.enumerated().map { i, name in ColumnInfo( name: name, @@ -292,6 +298,15 @@ nonisolated final class MySQLDriver: DatabaseDriver, SessionDatabaseReporting, @ return MySQLTableListing.tables(fromShowTables: raw.rows, databaseType: databaseType) } + // A MySQL schema is a database, so a read that names the database reads there whatever the session's `USE` says. + func fetchColumns(table: String, schema: String?, database: String?) async throws -> [ColumnInfo] { + try await fetchColumns(table: table, schema: schema ?? database) + } + + func fetchIndexes(table: String, schema: String?, database: String?) async throws -> [IndexInfo] { + try await fetchIndexes(table: table, schema: schema ?? database) + } + func fetchColumns(table: String, schema: String?) async throws -> [ColumnInfo] { let raw = try await actor.execute(Self.showFullColumnsStatement(table: table, database: schema)) let source = MySQLColumnListing.defaultSource( @@ -357,14 +372,38 @@ nonisolated final class MySQLDriver: DatabaseDriver, SessionDatabaseReporting, @ // Below 5.1.10 there is no REFERENTIAL_CONSTRAINTS. Failing there would also drop the columns and // indexes loaded with the keys, so those servers read SHOW CREATE TABLE, as on the Mac. + // The Mac's two single-catalog reads, merged here: ShardingSphere-Proxy answers a join of two + // `information_schema` tables with no columns. func fetchForeignKeys(table: String, schema: String?) async throws -> [ForeignKeyInfo] { guard Self.readsForeignKeyCatalog(for: databaseType, banner: serverVersion) else { return try await declaredForeignKeys(table: table, database: schema) } - let raw = try await actor.execute( - Self.foreignKeyCatalogQuery(table: table, database: schema, escapes: literalEscapes) - ) - return Self.foreignKeys(fromCatalogRows: raw.rows) + let named = schema.flatMap(\.nilIfEmpty) + let resolved: String? + if let named { + resolved = named + } else { + resolved = try await sessionDatabase() + } + guard let database = resolved, !database.isEmpty else { return [] } + let grouped = try await exportCatalogReader(activeDatabase: database) + .catalogForeignKeys(database: database, table: table) + return Self.foreignKeys(MySQLForeignKeyCatalog.keys(for: table, in: grouped), database: database) + } + + // A key into its own database names no schema, as a declared key does. + static func foreignKeys(_ keys: [PluginForeignKeyInfo], database: String) -> [ForeignKeyInfo] { + keys.map { key in + ForeignKeyInfo( + name: key.name, + column: key.column, + referencedTable: key.referencedTable, + referencedColumn: key.referencedColumn, + referencedSchema: key.referencedSchema == database ? nil : key.referencedSchema, + onDelete: key.onDelete, + onUpdate: key.onUpdate + ) + } } private func declaredForeignKeys(table: String, database: String?) async throws -> [ForeignKeyInfo] { @@ -449,6 +488,138 @@ nonisolated final class MySQLDriver: DatabaseDriver, SessionDatabaseReporting, @ static func keptTextEscapes(for databaseType: DatabaseType, banner: String?) -> SQLStringEscapes { stringEscapes(for: databaseType, session: nil, isDatabend: serverFlavor(for: databaseType, banner: banner).isDatabend) } + + // MARK: - Transfer + + var serverFlavor: MySQLServerFlavor { + Self.serverFlavor(for: databaseType, banner: serverVersion) + } + + // Every row, bytes kept, re-spelled for the session as the Mac's `execute(ownStatement:)` does. + func catalogQuery(_ sql: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + let result = try await actor.rawQuery(sql, parameters: parameters, observesFootprint: true) + await settleLexicalFacts() + return result + } + + // A session scope's own reads and settings, which the footprint leaves out as the Mac's `executeSetup` does. + func runSetup(_ sql: String) async throws -> PluginQueryResult { + let result = try await actor.rawQuery(sql, parameters: [], observesFootprint: false) + await settleLexicalFacts() + return result + } + + func adjustFootprint(_ change: (inout MySQLSessionFootprint) -> Void) { + actor.footprint.withLockUnchecked { change(&$0) } + } + + // Inlined rather than bound: text as hex under the character set the driver set at connect, bytes as a hex + // literal, which store what a server-side bind of MYSQL_TYPE_STRING or MYSQL_TYPE_LONG_BLOB stores. + func executeParameterized(query: String, parameters: [PluginCellValue], typeNames: [String?]?) async throws + -> QueryResult { + let raw = try await actor.executeInlined(query, parameters: parameters) + await settleLexicalFacts() + return Self.queryResult(raw) + } + + func foreignKeyCheckStatements(enabled: Bool) -> [String] { + serverFlavor.foreignKeyCheckStatements(enabled: enabled) ?? [] + } + + // Every row, bytes kept, a batch per pull. A reader that leaves stops the statement on the server. + func exportRows(query: String, readOnly: Bool) -> AsyncThrowingStream { + let cursor = MySQLExportCursor( + actor: actor, query: query, inReadOnlyTransaction: readOnly && enforcesReadOnlyMode + ) + return AsyncThrowingStream(unfolding: { try await cursor.next() }) + } + + func exportCatalogReader(activeDatabase: String) -> MySQLExportCatalog { + MySQLExportCatalog( + banner: serverVersion, + flavor: serverFlavor, + activeDatabase: activeDatabase, + quote: { mysqlQuoteIdentifier($0) }, + run: { [self] sql in try await self.catalogQuery(sql) }, + schemaNamesAreCaseSensitive: { [self] in await self.schemaNamesAreCaseSensitive() }, + settlesBlindness: { MySQLStatementFailure.settlesBlindness($0) } + ) + } + + // A server that will not answer keeps the plain comparison, for this call only, as on the Mac. + func schemaNamesAreCaseSensitive() async -> Bool { + if let known = schemaNameCase.withLock({ $0 }) { return known } + let run: MySQLExportCatalog.Run = { [self] sql in try await self.catalogQuery(sql) } + guard let sensitive = await MySQLExportCatalog.schemaNameCaseSensitivity(run: run) else { return false } + schemaNameCase.withLock { $0 = sensitive } + return sensitive + } +} + +// A statement the server refused, with its code, so a catalog read can tell a server that cannot answer from a lost +// connection. +nonisolated struct MySQLStatementFailure: Error, LocalizedError, Equatable { + let code: UInt32 + let message: String + + var errorDescription: String? { "MySQL query failed: \(message)" } + + // Code 0 is the driver's own, and 2000 to 2999 the client library's, which say nothing about the catalog. + static func settlesBlindness(_ error: any Error) -> Bool { + guard let failure = error as? MySQLStatementFailure else { return false } + return failure.code != 0 && !(2_000...2_999).contains(failure.code) + } +} + +// A value never goes through escaping: under a multibyte character set the session can switch to (`SET NAMES gbk`), a +// backslash escape's bytes read as part of one character and the quote after them ends the literal. Text is its UTF-8 +// bytes in hex behind the introducer of the character set the driver set at connect, so the server reads the same +// characters whatever the session's character set is later. +nonisolated enum MySQLInlineLiterals { + static let defaultCharacterSet = MySQLConnectionEncoding.sessionCharacterSetName + + static func literal(_ value: PluginCellValue, characterSet: String) -> String { + switch value { + case .null: + return "NULL" + case .text(let text): + return "_\(characterSetName(characterSet)) " + hexLiteral(Data(text.utf8)) + case .bytes(let data): + return hexLiteral(data) + } + } + + // The name goes into the statement as written, so anything but a plain name falls back to utf8mb4. + static func characterSetName(_ reported: String?) -> String { + guard let reported, !reported.isEmpty, + reported.unicodeScalars.allSatisfy({ $0.isASCII && (CharacterSet.alphanumerics.contains($0) || $0 == "_") }) + else { return defaultCharacterSet } + return reported + } + + private static func hexLiteral(_ data: Data) -> String { + var hex = "X'" + hex.reserveCapacity(3 + data.count * 2) + for byte in data { + hex += String(format: "%02X", byte) + } + return hex + "'" + } + + // A placeholder counts only in code, as the session lexes it. + static func bind( + _ sql: String, + parameters: [PluginCellValue], + characterSet: String, + features: SQLLexicalFeatures + ) throws -> String { + try PluginParameterBinder.bind( + sql, + literals: parameters.map { literal($0, characterSet: characterSet) }, + style: .questionMark, + features: features + ) + } } // The sign-in went through and the database was refused as a USE there is (scripts/probes/check-mysql-dial-database-refusal.sh). @@ -603,47 +774,6 @@ nonisolated extension MySQLDriver { return SQLQuoting.stringLiteral(database, escapes: escapes) } - static func foreignKeyCatalogQuery(table: String, database: String?, escapes: SQLStringEscapes = .mysql) -> String { - """ - SELECT - kcu.CONSTRAINT_NAME, - kcu.COLUMN_NAME, - kcu.REFERENCED_TABLE_NAME, - kcu.REFERENCED_COLUMN_NAME, - rc.DELETE_RULE, - rc.UPDATE_RULE, - kcu.TABLE_SCHEMA, - kcu.REFERENCED_TABLE_SCHEMA - FROM information_schema.KEY_COLUMN_USAGE kcu - JOIN information_schema.REFERENTIAL_CONSTRAINTS rc - ON kcu.CONSTRAINT_NAME = rc.CONSTRAINT_NAME - AND kcu.CONSTRAINT_SCHEMA = rc.CONSTRAINT_SCHEMA - WHERE kcu.TABLE_SCHEMA = \(databaseLiteral(database, escapes: escapes)) - AND kcu.TABLE_NAME = \(SQLQuoting.stringLiteral(table, escapes: escapes)) - AND kcu.REFERENCED_TABLE_NAME IS NOT NULL - ORDER BY kcu.CONSTRAINT_NAME, kcu.ORDINAL_POSITION - """ - } - - static func foreignKeys(fromCatalogRows rows: [[String?]]) -> [ForeignKeyInfo] { - rows.compactMap { row in - guard row.count >= 8, - let name = row[0], - let column = row[1], - let refTable = row[2], - let refColumn = row[3] else { return nil } - return ForeignKeyInfo( - name: name, - column: column, - referencedTable: refTable, - referencedColumn: refColumn, - referencedSchema: row[7] == row[6] ? nil : row[7], - onDelete: row[4] ?? "NO ACTION", - onUpdate: row[5] ?? "NO ACTION" - ) - } - } - // The parser fills the source database into every unqualified key. Nil keeps a same-database key // the same as the catalog read gives it. static func foreignKeys(declaredIn createTable: String, database: String?, omittedAction: String) -> [ForeignKeyInfo] { @@ -699,6 +829,9 @@ private actor MySQLActor { ] private var encoding: MySQLConnectionEncoding = .utf8 + // `character_set_client` right after the session was set up: utf8mb4, utf8 on a server without it, latin1 for + // UTF-8 via Latin 1. Inlined text names it, so a later `SET NAMES` cannot change how the value reads. + private var literalCharacterSet = MySQLInlineLiterals.defaultCharacterSet private var errorLanguage: String.Encoding? private var showTextIsBinary = false private var tlsServerName: MySQLTLSServerName? @@ -774,6 +907,9 @@ private actor MySQLActor { mysql_close(handle) throw MySQLError.connectionFailed(msg) } + literalCharacterSet = MySQLInlineLiterals.characterSetName( + Self.clientCharacterSet(on: handle) ?? mysql_character_set_name(handle).map { String(cString: $0) } + ) let banner = mysql_get_server_info(handle).map { String(cString: $0) } let sessionFlavor = MySQLDriver.serverFlavor(for: databaseType, banner: banner) @@ -899,7 +1035,7 @@ private actor MySQLActor { func close() { killer.detach() holdsReadOnlyTransaction = false - footprint.reset() + footprint.withLock { $0.reset() } guard let mysql else { return } if let result = streamingResult { // An open result points at the handle `mysql_close` frees, so it goes first; shutting the socket @@ -942,6 +1078,17 @@ private actor MySQLActor { private static let hexPreviewBytes = 256 + // Asked once at connect, before any statement of the user's can move it. + private static func clientCharacterSet(on mysql: UnsafeMutablePointer) -> String? { + let statement = "SELECT @@character_set_client" + guard statement.withCString({ mysql_real_query(mysql, $0, UInt(strlen($0))) }) == 0, + let result = mysql_store_result(mysql) + else { return nil } + defer { mysql_free_result(result) } + guard mysql_num_fields(result) >= 1, let row = mysql_fetch_row(result), let value = row[0] else { return nil } + return String(cString: value) + } + private func message(from mysql: UnsafeMutablePointer) -> String { guard let text = mysql_error(mysql) else { return "" } return MySQLErrorText.decode(cString: text, language: errorLanguage, encoding: encoding) @@ -990,9 +1137,9 @@ private actor MySQLActor { let start = Date() let features = lexicalFeatures(of: mysql) - footprint.observe(query, lexicalFeatures: features) + footprint.withLock { $0.observe(query, lexicalFeatures: features) } guard mysql_real_query(mysql, query, UInt(query.utf8.count)) == 0 else { - footprint.observeFailure(of: query, lexicalFeatures: features) + footprint.withLock { $0.observeFailure(of: query, lexicalFeatures: features) } throw MySQLError.queryFailed(message(from: mysql)) } @@ -1048,6 +1195,112 @@ private actor MySQLActor { ) } + // MARK: - Transfer + + private func noBackslashEscapes(of mysql: UnsafeMutablePointer) -> Bool? { + var serverStatus: UInt32 = 0 + guard mariadb_get_info(mysql, MARIADB_CONNECTION_SERVER_STATUS, &serverStatus) == 0 else { return nil } + return serverStatus & UInt32(SERVER_STATUS_NO_BACKSLASH_ESCAPES) != 0 + } + + // Read from the last reply's status flags, as `mysql_real_escape_string` reads them. `respells` is for text the + // shared catalogs wrote with backslash escapes; an import statement holds no literal of its own. + private func spelled( + _ sql: String, + parameters: [PluginCellValue], + respells: Bool, + on mysql: UnsafeMutablePointer + ) throws -> String { + let spelling = MySQLLiteralSpelling(noBackslashEscapes: noBackslashEscapes(of: mysql)) + let template = respells ? spelling.respelled(sql) : sql + guard !parameters.isEmpty else { return template } + return try MySQLInlineLiterals.bind( + template, parameters: parameters, characterSet: literalCharacterSet, features: lexicalFeatures(of: mysql) + ) + } + + func executeInlined(_ sql: String, parameters: [PluginCellValue]) throws -> RawMySQLResult { + guard let mysql else { throw MySQLError.notConnected } + return try execute(try spelled(sql, parameters: parameters, respells: false, on: mysql)) + } + + private func statementFailure(of mysql: UnsafeMutablePointer) -> MySQLStatementFailure { + MySQLStatementFailure(code: mysql_errno(mysql), message: message(from: mysql)) + } + + // No row cap and no display form: every value as `MySQLColumnDecoding` reads it, bytes as bytes. + func rawQuery(_ sql: String, parameters: [PluginCellValue], observesFootprint: Bool) throws -> PluginQueryResult { + guard let mysql else { throw MySQLError.notConnected } + endLeftoverStream(of: mysql) + let query = try spelled(sql, parameters: parameters, respells: true, on: mysql) + let sent = send(killer.reserve(), on: mysql) + defer { finish(sent, on: mysql) } + let start = Date() + let features = lexicalFeatures(of: mysql) + if observesFootprint { + footprint.withLock { $0.observe(query, lexicalFeatures: features) } + } + guard mysql_real_query(mysql, query, UInt(query.utf8.count)) == 0 else { + if observesFootprint { + footprint.withLock { $0.observeFailure(of: query, lexicalFeatures: features) } + } + throw statementFailure(of: mysql) + } + guard let result = mysql_store_result(mysql) else { + if mysql_field_count(mysql) != 0 { throw statementFailure(of: mysql) } + let raw = mysql_affected_rows(mysql) + try consumeTrailingResults(of: mysql) + return PluginQueryResult( + columns: [], columnTypeNames: [], rows: [], rowsAffected: raw == .max ? 0 : Int(clamping: raw), + executionTime: Date().timeIntervalSince(start) + ) + } + let described = MariaDBCharacterSet.describeColumns( + of: mysql_fetch_fields(result), count: Int(mysql_num_fields(result)), encoding: encoding, + binaryStringsAreText: binaryStringsAreText(in: query) + ) + var rows: [[PluginCellValue]] = [] + while let row = mysql_fetch_row(result) { + let lengths = mysql_fetch_lengths(result) + rows.append(described.row(encoding: encoding) { index in + guard let value = row[index] else { return nil } + return UnsafeRawBufferPointer(start: value, count: Int(clamping: lengths?[index] ?? 0)) + }) + } + mysql_free_result(result) + try consumeTrailingResults(of: mysql) + return PluginQueryResult( + columns: described.names, columnTypeNames: described.typeNames, rows: rows, rowsAffected: 0, + executionTime: Date().timeIntervalSince(start) + ) + } + + // Sent as written, as the Mac sends an export's SELECT: a filter in it is the user's, spelled for the session. + func beginExport(query: String, statement: UInt64, inReadOnlyTransaction: Bool) throws -> MySQLBeginStreamResult { + try beginStream(query: query, statement: statement, inReadOnlyTransaction: inReadOnlyTransaction) + } + + // Nil once the result has no rows left. + func fetchExportRows(_ stream: MySQLStreamID, limit: Int) throws -> [[PluginCellValue]]? { + guard let mysql, stream.generation == streamGeneration, let result = streamingResult else { + throw CancellationError() + } + var rows: [[PluginCellValue]] = [] + while rows.count < limit { + let errorBefore = mysql_errno(mysql) + guard let row = mysql_fetch_row(result) else { + if let failure = fetchFailure(of: mysql, errorBefore: errorBefore) { throw failure } + return rows.isEmpty ? nil : rows + } + let lengths = mysql_fetch_lengths(result) + rows.append(streamingDecoding.row(encoding: encoding) { index in + guard let value = row[index] else { return nil } + return UnsafeRawBufferPointer(start: value, count: Int(clamping: lengths?[index] ?? 0)) + }) + } + return rows + } + // MARK: - Streaming private static let logger = Logger(subsystem: "com.TablePro", category: "MySQLDriver") @@ -1061,7 +1314,7 @@ private actor MySQLActor { // Set from START TRANSACTION READ ONLY until the ROLLBACK after the statement's last result. private var holdsReadOnlyTransaction = false // The status flags never show a LOCK TABLES, which a START TRANSACTION releases, so the statements say it. - private var footprint = MySQLSessionFootprint() + nonisolated let footprint = OSAllocatedUnfairLock(initialState: MySQLSessionFootprint()) // With `inReadOnlyTransaction` the state check, the transaction and the statement go out in this one actor call, // so nothing else on the session runs between them. The session is opened without CLIENT_MULTI_STATEMENTS, so no @@ -1086,9 +1339,9 @@ private actor MySQLActor { } let features = lexicalFeatures(of: mysql) - footprint.observe(query, lexicalFeatures: features) + footprint.withLock { $0.observe(query, lexicalFeatures: features) } guard mysql_real_query(mysql, query, UInt(query.utf8.count)) == 0 else { - footprint.observeFailure(of: query, lexicalFeatures: features) + footprint.withLock { $0.observeFailure(of: query, lexicalFeatures: features) } let failure = queryFailure(of: mysql) finish(sent, on: mysql) try? endReadOnlyTransaction() @@ -1238,7 +1491,7 @@ private actor MySQLActor { } func takesReadOnlyTransaction() -> Bool { - mysql != nil && transactionState() == .idle && !footprint.hasLockedTables + mysql != nil && transactionState() == .idle && !footprint.withLock { $0.hasLockedTables } } private func lexicalFeatures(of mysql: UnsafeMutablePointer) -> SQLLexicalFeatures { @@ -1356,6 +1609,72 @@ nonisolated private struct MySQLStreamID: Sendable { let generation: UInt64 } +// The pull side of one export read: one batch per pull, and a reader that leaves early kills the statement before the +// rest of its result is drained. +nonisolated private final class MySQLExportCursor: @unchecked Sendable { + private static let batchSize = 500 + + private let actor: MySQLActor + private let query: String + private let inReadOnlyTransaction: Bool + private let statement: UInt64 + private let lock = NSLock() + private var started = false + private var openStream: MySQLStreamID? + + init(actor: MySQLActor, query: String, inReadOnlyTransaction: Bool) { + self.actor = actor + self.query = query + self.inReadOnlyTransaction = inReadOnlyTransaction + self.statement = actor.killer.reserve() + } + + deinit { + guard let stream = openStream else { return } + actor.killer.readerLeft(statement) + let actor = actor + Task { try? await actor.endStream(stream) } + } + + func next() async throws -> PluginStreamElement? { + let actor = actor + let killer = actor.killer + let statement = statement + return try await withTaskCancellationHandler { + let isFirst = lock.withLock { + defer { started = true } + return !started + } + if isFirst { + switch try await actor.beginExport( + query: query, statement: statement, inReadOnlyTransaction: inReadOnlyTransaction + ) { + case .noResult: + return .header(PluginStreamHeader(columns: [], columnTypeNames: [])) + case .rowSet(let stream, let columns): + lock.withLock { openStream = stream } + return .header(PluginStreamHeader(columns: columns.map(\.name), columnTypeNames: columns.map(\.typeName))) + } + } + guard let stream = lock.withLock({ openStream }) else { return nil } + do { + if let rows = try await actor.fetchExportRows(stream, limit: Self.batchSize) { + return .rows(rows) + } + lock.withLock { openStream = nil } + try await actor.endStream(stream) + return nil + } catch { + lock.withLock { openStream = nil } + try? await actor.endStream(stream) + throw error + } + } onCancel: { + killer.readerLeft(statement) + } + } +} + nonisolated private enum MySQLBeginStreamResult: Sendable { case rowSet(stream: MySQLStreamID, columns: [ColumnInfo]) case noResult(affectedRows: Int) diff --git a/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift index 8e7d52ce96..5a67f80335 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift @@ -1,7 +1,9 @@ import Foundation +import os import TableProDatabase import TableProModels import TableProOracleCore +import TableProPluginKit nonisolated private extension OracleRawResult { nonisolated func toQueryResult(executionTime: TimeInterval) -> QueryResult { @@ -15,13 +17,27 @@ nonisolated private extension OracleRawResult { isTruncated: isTruncated ) } + + // Bytes stay bytes, as the Mac's plugin bridges them. + nonisolated func toPluginResult(executionTime: TimeInterval) -> PluginQueryResult { + PluginQueryResult( + columns: columns.map(\.name), + columnTypeNames: columns.map(\.typeName), + rows: rows.map { $0.map(\.asPluginCell) }, + rowsAffected: affectedRows, + executionTime: executionTime, + isTruncated: isTruncated + ) + } } nonisolated final class OracleDriver: DatabaseDriver, @unchecked Sendable { private let core: OracleCoreConnection private let host: String let ssl: DriverSSLConfiguration - private let fallbackSchema: String + let fallbackSchema: String + // What an export learned of the account's dictionary access and roles, for the life of the connection, as on the Mac. + let principalState = OSAllocatedUnfairLock(initialState: OraclePrincipalState()) var supportsSchemas: Bool { true } var supportsTransactions: Bool { true } @@ -117,6 +133,93 @@ nonisolated final class OracleDriver: DatabaseDriver, @unchecked Sendable { } } + // Values bind to `:1` through `:n` in order. + func catalogCells(_ query: String, binds: [PluginCellValue] = []) async throws -> PluginQueryResult { + let startTime = Date() + do { + let raw = binds.isEmpty + ? try await core.executeQuery(query) + : try await core.executeQuery(query, binds: binds.map(\.oracleBindValue)) + return raw.toPluginResult(executionTime: Date().timeIntervalSince(startTime)) + } catch let error as OracleCoreError { + throw Self.connectionError(for: error) + } + } + + // The shared catalog's read keeps the server's error, which tells a dictionary view the account cannot read apart. + func catalogResult(_ query: String) async throws -> OracleRawResult { + do { + return try await core.executeQuery(query) + } catch OracleCoreError.cancelled { + throw CancellationError() + } + } + + var release: OracleServerRelease? { core.serverRelease } + + func executeParameterized(query: String, parameters: [PluginCellValue], typeNames: [String?]?) async throws + -> QueryResult { + guard !parameters.isEmpty else { return try await runQuery(query) } + let statement = try Self.boundStatement( + query, parameters: parameters, typeNames: typeNames, writesBooleans: release?.hasBooleanType ?? false + ) + guard !statement.binds.isEmpty else { return try await runQuery(statement.sql) } + let startTime = Date() + do { + let raw = try await core.executeQuery(statement.sql, binds: statement.binds.map(\.oracleBindValue)) + return raw.toQueryResult(executionTime: Date().timeIntervalSince(startTime)) + } catch let error as OracleCoreError { + throw Self.connectionError(for: error) + } + } + + // The Mac's binding: each value written for its column's type, and only bytes and text past the literal limit + // bound as `:n`. + static func boundStatement( + _ query: String, + parameters: [PluginCellValue], + typeNames: [String?]?, + writesBooleans: Bool + ) throws -> (sql: String, binds: [PluginCellValue]) { + try OracleBindPlaceholders.substituting( + query, parameters: parameters, typeNames: typeNames, writesBooleans: writesBooleans + ) + } + + // A batch of rows at a time, as the Mac streams them. A consumer that stops or lets the stream go cancels the read. + func exportRows(query: String, readOnly: Bool) -> AsyncThrowingStream { + let core = self.core + return AsyncThrowingStream(bufferingPolicy: .bufferingOldest(PluginRowStream.boundedBufferBatches)) { continuation in + // Not a child task, so only this cancel stops oracle-nio once the reader stops. + let task = Task { + do { + try await core.streamQuery( + query, + batching: OracleRowBatcher(rowsPerBatch: OracleRowBatcher.batchRows), + onElement: { element in + switch element { + case .header(let columns): + _ = await PluginRowStream.deliver(.header(PluginStreamHeader( + columns: columns.map(\.name), columnTypeNames: columns.map(\.typeName) + )), to: continuation) + case .rows(let batch): + _ = await PluginRowStream.deliver(.rows(batch.map { $0.map(\.asPluginCell) }), to: continuation) + case .affectedRows: + break + } + } + ) + continuation.finish() + } catch let error as OracleCoreError { + continuation.finish(throwing: Self.connectionError(for: error)) + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { @Sendable _ in task.cancel() } + } + } + func cancelCurrentQuery() async throws { core.cancelCurrentQuery() } diff --git a/apple/TableProMobile/TableProMobile/Drivers/PluginBackedDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/PluginBackedDriver.swift index 61c8b7b681..d4224570a0 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/PluginBackedDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/PluginBackedDriver.swift @@ -7,7 +7,7 @@ import TableProPluginKit // Runs a driver the Mac ships as a plugin, compiled into this app, the way the Mac's PluginDriverAdapter does. Its // rows are written with SQLBuilder's statements, as the Mac writes them for a plugin that generates none of its own. -nonisolated class PluginBackedDriver: DatabaseDriver, @unchecked Sendable { +nonisolated class PluginBackedDriver: DatabaseDriver, CatalogQuerying, ExportRowStreaming, @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "PluginBackedDriver") let plugin: any PluginDatabaseDriver @@ -318,6 +318,54 @@ nonisolated class PluginBackedDriver: DatabaseDriver, @unchecked Sendable { var currentSchema: String? { plugin.currentSchema } + // MARK: - Transfer + + // The plugin's own result, bytes kept and no row cap, taking its turn like any other request. + func catalogQuery(_ sql: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + let plugin = plugin + var catalogContext = statementContext(sql) + catalogContext.readsCatalog = true + let context = catalogContext + return try await inSessionDatabase { + guard !parameters.isEmpty else { return try await plugin.execute(query: sql, context: context) } + return try await plugin.executeParameterized(query: sql, parameters: parameters, context: context) + } + } + + // The plugin's stream, which reads a batch only when the export asks for one. It skips the request gate, so a + // transfer runs it on a driver of its own (`ConnectionManager.openDedicatedDriver`). + func exportRows(query: String, readOnly: Bool) -> AsyncThrowingStream { + var context = statementContext(query) + // A file export only reads, whatever the statement classifies as, as on the Mac. + context.readOnly = true + context.enforcesReadOnly = readOnly + return plugin.streamRows(query: query, context: context) + } + + // A query export with the plugin's typed documents beside its rows, which MQL writes from (MongoDB). + func exportDocuments(query: String, readOnly: Bool) -> AsyncThrowingStream { + var context = statementContext(query) + context.readOnly = true + context.enforcesReadOnly = readOnly + return plugin.streamExportRows(query: query, context: context) + } + + // The column types reach the plugin as the Mac's import hands them over. + func executeParameterized(query: String, parameters: [PluginCellValue], typeNames: [String?]?) async throws + -> QueryResult { + let plugin = plugin + var typedContext = statementContext(query) + typedContext.parameterTypeNames = typeNames + let context = typedContext + return QueryResult(from: try await inSessionDatabase { + try await plugin.executeParameterized(query: query, parameters: parameters, context: context) + }) + } + + func foreignKeyCheckStatements(enabled: Bool) -> [String] { + (enabled ? plugin.foreignKeyEnableStatements() : plugin.foreignKeyDisableStatements()) ?? [] + } + // MARK: - Transactions var supportsTransactions: Bool { plugin.supportsTransactions } diff --git a/apple/TableProMobile/TableProMobile/Drivers/PostgreSQLDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/PostgreSQLDriver.swift index 8408576e5d..6dafdef431 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/PostgreSQLDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/PostgreSQLDriver.swift @@ -54,6 +54,9 @@ nonisolated final class PostgreSQLDriver: DatabaseDriver, SessionSchemaReporting var effectiveSchema: String { currentSchema ?? "public" } + // Redshift's svv_external_* views carry rows for every database on the cluster. + var connectedDatabase: String { database } + init( host: String, port: Int, @@ -154,6 +157,71 @@ nonisolated final class PostgreSQLDriver: DatabaseDriver, SessionSchemaReporting Self.queryResult(from: try await actor.executeCatalogRead(statement, pinsSearchPath: databaseType != .redshift)) } + // What the shared catalog and an export read: every row, bytes kept. `schema` puts that schema after pg_catalog on + // the path, for a read the server deparses names in. + func catalogCells( + _ statement: String, + parameters: [PluginCellValue] = [], + namesRelativeTo schema: String? = nil + ) async throws -> PluginQueryResult { + let path = databaseType == .redshift + ? nil + : schema.map(PostgreSQLCatalogSearchPath.relative(to:)) ?? PostgreSQLCatalogSearchPath.pinned + return try await actor.catalogCells(statement, parameters: parameters, searchPath: path).pluginResult + } + + func executeParameterized(query: String, parameters: [PluginCellValue], typeNames: [String?]?) async throws + -> QueryResult { + let raw = try await actor.executeParameterized(query, parameters: parameters) + await settleLexicalFacts() + return Self.queryResult(from: raw) + } + + func foreignKeyCheckStatements(enabled: Bool) -> [String] { + guard databaseType == .postgresql else { return [] } + return enabled ? PostgreSQLForeignKeyChecks.enableStatements : PostgreSQLForeignKeyChecks.disableStatements + } + + // An export's session-scope statement, as the Mac's `executeSessionSetting` sends it. + func executeSessionSetting( + _ statement: String, + parameters: [PluginCellValue], + requiresIdleSession: Bool + ) async throws -> [[PluginCellValue]] { + let rows = try await actor.executeSessionSetting( + statement, parameters: parameters, requiresIdleSession: requiresIdleSession + ) + await settleLexicalFacts() + return rows + } + + // Redshift's COPY reads only its cloud sources, as on the Mac. + func copyFromStandardInput(statement: String, rows: PluginCopyData) async throws -> QueryResult { + guard databaseType == .postgresql else { throw PluginDriverUnsupportedOperation.copyFromStandardInput } + let actor = self.actor + let cancelHandle = actor.cancelHandle + let id = cancelHandle.reserveQuery() + let raw = try await withTaskCancellationHandler { + try await actor.copyFromStandardInput(statement, rows: rows, id: id) + } onCancel: { + cancelHandle.cancel(id) + } + await settleLexicalFacts() + return Self.queryResult(from: raw) + } + + // Pulled a batch at a time; a consumer that cancels stops the statement on the server, and one that lets the + // stream go cancels the rows it never read. + func exportRows(query: String, readOnly: Bool) -> AsyncThrowingStream { + let cursor = PostgreSQLExportCursor( + actor: actor, + query: query, + readsOnly: !SQLWriteClassifier.isWriteQuery(query, databaseType: databaseType), + inReadOnlyTransaction: readOnly && enforcesReadOnlyMode + ) + return AsyncThrowingStream(unfolding: { try await cursor.next() }) + } + private static func queryResult(from raw: RawPGResult) -> QueryResult { QueryResult( columns: raw.columns.enumerated().map { i, name in @@ -537,109 +605,263 @@ private actor PostgreSQLActor { return "\(major).\(minor).\(patch)" } - /// The search path lasts only as long as the read: outside a transaction block the pair is one implicit - /// transaction, and inside one a savepoint rolled back straight after scopes it. One actor call, so no other - /// statement on this session runs between the savepoint and its rollback. func executeCatalogRead(_ statement: String, pinsSearchPath: Bool) throws -> RawPGResult { guard pinsSearchPath else { return try execute(statement) } - endLeftoverStream() - let scoped = PostgreSQLCatalogSearchPath.pinned + statement - guard let conn, PostgreSQLSessionTransaction.isInsideBlock(PQtransactionStatus(conn)) else { - return try execute(scoped) + return try scopingSearchPath(PostgreSQLCatalogSearchPath.pinned, sendsPathAlone: false) { path in + try execute(path + statement) } - _ = try execute(Self.catalogReadSavepoint) - let result: RawPGResult + } + + func catalogCells(_ statement: String, parameters: [PluginCellValue], searchPath: String?) throws -> PGCellResult { + let bound = parameters.isEmpty ? nil : parameters + guard let searchPath else { return try cells(statement, parameters: bound) } + return try scopingSearchPath(searchPath, sendsPathAlone: bound != nil) { path in + try cells(path + statement, parameters: bound) + } + } + + /// The search path lasts only as long as the read: outside a transaction block the pair is one implicit + /// transaction, and inside one a savepoint rolled back straight after scopes it. One actor call, so no other + /// statement on this session runs between the savepoint and its rollback. The extended protocol takes one + /// statement, so a read with parameters sets the path on its own first, in a block it opens when there is none. + private func scopingSearchPath( + _ path: String, + sendsPathAlone: Bool, + _ read: (_ pathPrefix: String) throws -> T + ) throws -> T { + endLeftoverStream() + guard let conn else { throw PostgreSQLError.notConnected } + let insideBlock = PostgreSQLSessionTransaction.isInsideBlock(PQtransactionStatus(conn)) + guard insideBlock || sendsPathAlone else { return try read(path) } + _ = try execute(insideBlock ? Self.catalogReadSavepoint : Self.catalogReadBegin) + let closing = insideBlock ? Self.catalogReadRollback : Self.catalogReadEnd + let result: T do { - result = try execute(scoped) + if sendsPathAlone { _ = try execute(path) } + result = try read(sendsPathAlone ? "" : path) } catch { - _ = try? execute(Self.catalogReadRollback) + _ = try? execute(closing) throw error } - _ = try execute(Self.catalogReadRollback) + _ = try execute(closing) return result } private static let catalogReadSavepoint = "SAVEPOINT tablepro_catalog_read" private static let catalogReadRollback = "ROLLBACK TO SAVEPOINT tablepro_catalog_read; RELEASE SAVEPOINT tablepro_catalog_read" + private static let catalogReadBegin = "BEGIN" + private static let catalogReadEnd = "ROLLBACK" func execute(_ query: String) throws -> RawPGResult { + let start = Date() + let result = try sendStatement(query, parameters: nil) + defer { PQclear(result) } + return Self.textResult(of: result, since: start) + } + + func executeParameterized(_ query: String, parameters: [PluginCellValue]) throws -> RawPGResult { + let start = Date() + let result = try sendStatement(query, parameters: parameters) + defer { PQclear(result) } + return Self.textResult(of: result, since: start) + } + + // Every row, unlike `execute`, and each value as the Mac's driver reads it. + func cells(_ query: String, parameters: [PluginCellValue]?) throws -> PGCellResult { + let start = Date() + let result = try sendStatement(query, parameters: parameters) + defer { PQclear(result) } + return Self.cellResult(of: result, since: start) + } + + // Inside a transaction a savepoint holds the statement, so a refusal leaves the user's transaction open. The idle + // check and the statement go out in this one call, so no transaction opens between them. + func executeSessionSetting( + _ statement: String, + parameters: [PluginCellValue], + requiresIdleSession: Bool + ) throws -> [[PluginCellValue]] { + endLeftoverStream() + guard let conn else { throw PostgreSQLError.notConnected } + let status = PQtransactionStatus(conn) + if requiresIdleSession, status != PQTRANS_IDLE { throw PostgreSQLSessionScopeError.sessionNotIdle } + guard status == PQTRANS_INTRANS else { return try cells(statement, parameters: parameters).rows } + _ = try execute(PostgreSQLSessionScopeSQL.settingSavepoint) + let rows: [[PluginCellValue]] + do { + rows = try cells(statement, parameters: parameters).rows + } catch { + _ = try? execute(PostgreSQLSessionScopeSQL.settingRollback) + throw error + } + _ = try execute(PostgreSQLSessionScopeSQL.settingRelease) + return rows + } + + /// The statement's result once it succeeded: a COPY is refused and a failure thrown here. The caller clears it. + /// With `parameters` the statement goes through the extended protocol, which takes one statement. + private func sendStatement(_ query: String, parameters: [PluginCellValue]?) throws -> OpaquePointer { endLeftoverStream() guard let conn else { throw PostgreSQLError.notConnected } - let start = Date() let cancelsOutput = cancelsAbandonedOutput(conn) let id = cancelHandle.reserveQuery() cancelHandle.waitForPendingCancels() cancelHandle.querySent(id) defer { cancelHandle.queryFinished(id) } - let result = PQexec(conn, query) - defer { - if result != nil { PQclear(result) } + let result: OpaquePointer? + if let parameters { + result = try LibPQParameterBuffers.with(parameters) { values, lengths, formats in + PQexecParams(conn, query, Int32(parameters.count), nil, values, lengths, formats, 0) + } + } else { + result = PQexec(conn, query) } + return try succeeded(result, conn: conn, cancelsOutput: cancelsOutput, id: id) + } - let status = PQresultStatus(result) - + private func succeeded( + _ result: OpaquePointer?, + conn: OpaquePointer, + cancelsOutput: Bool, + id: UInt64 + ) throws -> OpaquePointer { if let copy = result.flatMap(LibPQCopyState.copy(of:)) { + PQclear(result) _ = finishPendingResults(conn, cancellingOutput: cancelsOutput) throw PostgreSQLError.unsupported(copy.direction.mobileUnsupportedMessage) } - - if status == PGRES_FATAL_ERROR { - if cancelHandle.isCancelled(id) { throw CancellationError() } - let msg = result.flatMap { String(cString: PQresultErrorMessage($0)) } ?? "Unknown error" - throw PostgreSQLError.queryFailed(msg) + let status = PQresultStatus(result) + guard let result, status == PGRES_COMMAND_OK || status == PGRES_TUPLES_OK else { + let message = result.map { String(cString: PQresultErrorMessage($0)) } + if result != nil { PQclear(result) } + if status == PGRES_FATAL_ERROR, cancelHandle.isCancelled(id) { throw CancellationError() } + throw PostgreSQLError.queryFailed( + message ?? (status == PGRES_FATAL_ERROR ? "Unknown error" : "Unexpected result status") + ) } + return result + } - if status == PGRES_COMMAND_OK { - let affectedStr = result.flatMap { String(cString: PQcmdTuples($0)) } ?? "0" - let affected = Int(affectedStr) ?? 0 + private static func textResult(of result: OpaquePointer, since start: Date) -> RawPGResult { + guard PQresultStatus(result) == PGRES_TUPLES_OK else { return RawPGResult( columns: [], columnTypes: [], rows: [], - rowsAffected: affected, executionTime: Date().timeIntervalSince(start), isTruncated: false + rowsAffected: Int(String(cString: PQcmdTuples(result))) ?? 0, + executionTime: Date().timeIntervalSince(start), isTruncated: false ) } - guard status == PGRES_TUPLES_OK else { - let msg = result.flatMap { String(cString: PQresultErrorMessage($0)) } ?? "Unexpected result status" - throw PostgreSQLError.queryFailed(msg) - } - let rowCount = Int(PQntuples(result)) - let colCount = Int(PQnfields(result)) + let colCount = Int32(PQnfields(result)) + let oids = (0.. 100_000 - - for r in 0.. String? in + guard PQgetisnull(result, r, c) == 0, let value = PQgetvalue(result, r, c) else { return nil } + let bytes = UnsafeRawBufferPointer(start: value, count: Int(PQgetlength(result, r, c))) + guard !keepsText[Int(c)] else { return LibPQCellDecoding.text(from: bytes) } + return LibPQCellDecoding.value(from: bytes, oid: oids[Int(c)]).asText } - rows.append(rowData) } return RawPGResult( columns: columns, columnTypes: columnTypes, rows: rows, - rowsAffected: 0, executionTime: Date().timeIntervalSince(start), isTruncated: isTruncated + rowsAffected: 0, executionTime: Date().timeIntervalSince(start), isTruncated: rowCount > 100_000 + ) + } + + private static func cellResult(of result: OpaquePointer, since start: Date) -> PGCellResult { + guard PQresultStatus(result) == PGRES_TUPLES_OK else { + return PGCellResult( + columns: [], columnTypeNames: [], rows: [], + rowsAffected: Int(String(cString: PQcmdTuples(result))) ?? 0, + executionTime: Date().timeIntervalSince(start) + ) + } + let colCount = Int32(PQnfields(result)) + let oids = (0.. PluginCellValue { + guard PQgetisnull(result, row, column) == 0, let value = PQgetvalue(result, row, column) else { return .null } + let bytes = UnsafeRawBufferPointer(start: value, count: Int(PQgetlength(result, row, column))) + return LibPQCellDecoding.value(from: bytes, oid: oid) + } + + // MARK: - COPY FROM STDIN + + /// One actor call holds the connection from the COPY to its last result, as on the Mac: a statement sent between + /// them would end the copy. `id` is reserved by the caller, so a Stop reaches the copy while this call runs. + func copyFromStandardInput(_ statement: String, rows: PluginCopyData, id: UInt64) throws -> RawPGResult { + endLeftoverStream() + guard let conn else { throw PostgreSQLError.notConnected } + + let start = Date() + let cancelsOutput = cancelsAbandonedOutput(conn) + let handle = cancelHandle + handle.waitForPendingCancels() + handle.querySent(id) + defer { handle.queryFinished(id) } + let stop = LibPQCopyIn.Stop( + requested: { handle.isCancelled(id) }, + closing: { false }, + sendCancelRequest: { Self.cancelAndWait(conn) } ) + if Task.isCancelled || handle.isCancelled(id) { throw CancellationError() } + guard let begun = LibPQCopyIn.begin(statement, conn: conn, stop: stop) else { + throw PostgreSQLError.queryFailed(Self.message(from: conn)) + } + // A statement that is not a copy-in, the server's refusal of the COPY among them, reads as any other result. + guard let copy = LibPQCopyState.copy(of: begun), copy.direction == .copyIn else { + let result = try succeeded(begun, conn: conn, cancelsOutput: cancelsOutput, id: id) + defer { PQclear(result) } + return Self.textResult(of: result, since: start) + } + PQclear(begun) + + let sent = LibPQCopyIn.send(rows, format: copy.format, conn: conn, stop: stop) + guard let result = sent.result else { throw PostgreSQLError.queryFailed(Self.message(from: conn)) } + defer { PQclear(result) } + switch sent.interruption { + case .cancelled: + throw CancellationError() + case .source(let error): + throw error + case .binaryFormat: + throw PostgreSQLError.unsupported(LibPQCopyIn.binaryRefusal) + case .connection, nil: + break + } + guard PQresultStatus(result) == PGRES_COMMAND_OK else { + if cancelHandle.isCancelled(id) { throw CancellationError() } + throw PostgreSQLError.queryFailed(String(cString: PQresultErrorMessage(result))) + } + return Self.textResult(of: result, since: start) + } + + // From the copy's own thread, so the cancel has reached the server before the copy's results are read. + private static func cancelAndWait(_ conn: OpaquePointer) { + guard let cancel = PQgetCancel(conn) else { return } + defer { PQfreeCancel(cancel) } + var errbuf = [CChar](repeating: 0, count: 256) + PQcancel(cancel, &errbuf, Int32(errbuf.count)) } // MARK: - Streaming @@ -790,6 +1012,29 @@ private actor PostgreSQLActor { return cells(of: result, options: options, columns: columns) } + // Up to `limit` rows of an export stream, nil at its end. A failure behind the rows ends the stream with it. + func fetchExportRows(stream id: UInt64, limit: Int) throws -> [[PluginCellValue]]? { + guard let state = stream, state.id == id else { throw CancellationError() } + guard let conn else { throw PostgreSQLError.notConnected } + var rows: [[PluginCellValue]] = [] + while rows.count < limit, !state.isDrained { + let readAhead = state.pendingResult + state.pendingResult = nil + guard let result = readAhead ?? PQgetResult(conn) else { + markDrained(state) + break + } + guard PQresultStatus(result) == PGRES_SINGLE_TUPLE else { + state.pendingResult = result + if let error = drainFailure(state) { throw error } + break + } + rows.append((0.. [Cell] { - var cells: [Cell] = [] - cells.reserveCapacity(columns.count) - for c in 0.. Cell in let col = Int32(c) - if PQgetisnull(result, 0, col) == 1 { - cells.append(.null) - } else if let value = PQgetvalue(result, 0, col) { - let str = String(cString: value) - let ref = makeCellRef(column: columns[c].name, columns: columns, result: result, options: options) - cells.append(Cell.from( - legacyValue: str, - columnTypeName: columns[c].typeName, - options: options, - ref: ref - )) - } else { - cells.append(.null) - } + let value = Self.cell(in: result, row: 0, column: col, oid: PQftype(result, col)) + guard !value.isNull else { return .null } + let ref = makeCellRef(column: columns[c].name, columns: columns, result: result, options: options) + return PostgreSQLDriver.cell(from: value, columnTypeName: columns[c].typeName, options: options, ref: ref) } - return cells } private func finishPendingResults(_ conn: OpaquePointer, cancellingOutput: Bool) -> LibPQDrainOutcome { @@ -961,7 +1193,7 @@ private actor PostgreSQLActor { let oid = PQftype(result, Int32(i)) cols.append(ColumnInfo( name: name, - typeName: pgOidToTypeName(oid), + typeName: pgTypeName(oid), isPrimaryKey: false, isNullable: true, defaultValue: nil, @@ -1247,38 +1479,8 @@ nonisolated enum PostgreSQLStreamDrain { } } -// MARK: - PostgreSQL OID Type Names - -nonisolated private func pgOidToTypeName(_ oid: UInt32) -> String { - switch oid { - case 16: return "boolean" - case 17: return "bytea" - case 18: return "char" - case 19: return "name" - case 20: return "bigint" - case 21: return "smallint" - case 23: return "integer" - case 25: return "text" - case 26: return "oid" - case 114: return "json" - case 142: return "xml" - case 700: return "real" - case 701: return "double precision" - case 869: return "inet" - // PostgreSQL OID constants: separators would obscure the wire-protocol values - // swiftlint:disable number_separator - case 1042: return "char" - case 1043: return "varchar" - case 1082: return "date" - case 1083: return "time" - case 1114: return "timestamp" - case 1184: return "timestamptz" - case 1700: return "numeric" - case 2950: return "uuid" - case 3802: return "jsonb" - // swiftlint:enable number_separator - default: return "unknown" - } +nonisolated private func pgTypeName(_ oid: UInt32) -> String { + PostgreSQLCatalogTypeNames.builtinTypeName(for: oid) ?? PostgreSQLCatalogTypeNames.unresolved } nonisolated private struct RawPGResult: Sendable { @@ -1290,6 +1492,118 @@ nonisolated private struct RawPGResult: Sendable { let isTruncated: Bool } +nonisolated private struct PGCellResult: Sendable { + let columns: [String] + let columnTypeNames: [String] + let rows: [[PluginCellValue]] + let rowsAffected: Int + let executionTime: TimeInterval + + var pluginResult: PluginQueryResult { + PluginQueryResult( + columns: columns, + columnTypeNames: columnTypeNames, + rows: rows, + rowsAffected: rowsAffected, + executionTime: executionTime + ) + } +} + +nonisolated extension PostgreSQLDriver { + // The grid's cell for a value decoded as the Mac decodes it: a boolean reads `true`, a bytea is its bytes. + static func cell(from value: PluginCellValue, columnTypeName: String, options: StreamOptions, ref: CellRef?) -> Cell { + switch value { + case .null: + return .null + case .bytes(let data): + return .binary(byteCount: data.count, ref: ref) + case .text(let text): + return Cell.from(legacyValue: text, columnTypeName: columnTypeName, options: options, ref: ref) + } + } +} + +// One consumer pulls it, so one call at a time reaches it. +nonisolated private final class PostgreSQLExportCursor: @unchecked Sendable { + private enum Phase { + case pending + case streaming + case finished + } + + // Rows per actor call: single-row mode hands over one row per result. + private static let rowsPerBatch = 1_000 + + private let actor: PostgreSQLActor + private let query: String + private let readsOnly: Bool + private let inReadOnlyTransaction: Bool + private let id: UInt64 + private var phase = Phase.pending + + init(actor: PostgreSQLActor, query: String, readsOnly: Bool, inReadOnlyTransaction: Bool) { + self.actor = actor + self.query = query + self.readsOnly = readsOnly + self.inReadOnlyTransaction = inReadOnlyTransaction + self.id = actor.cancelHandle.reserveQuery() + } + + // A stream let go before its end: the rows it never read are cancelled where that cannot abort a transaction. + deinit { + guard phase == .streaming else { return } + let actor = actor + let id = id + Task { try? await actor.endStreamAtRowCap(id) } + } + + func next() async throws -> PluginStreamElement? { + do { + switch phase { + case .finished: + return nil + case .pending: + let begun = try await cancellingOnStop { + try await $0.beginStream( + query: self.query, + id: self.id, + readsOnly: self.readsOnly, + inReadOnlyTransaction: self.inReadOnlyTransaction + ) + } + guard case .tuples(let columns) = begun else { + phase = .finished + return .header(PluginStreamHeader(columns: [], columnTypeNames: [])) + } + phase = .streaming + return .header(PluginStreamHeader(columns: columns.map(\.name), columnTypeNames: columns.map(\.typeName))) + case .streaming: + if let rows = try await cancellingOnStop({ try await $0.fetchExportRows(stream: self.id, limit: Self.rowsPerBatch) }) { + return .rows(rows) + } + phase = .finished + try await actor.endStream(id) + return nil + } + } catch { + phase = .finished + try? await actor.endStream(id) + throw error + } + } + + private func cancellingOnStop(_ body: @Sendable (PostgreSQLActor) async throws -> T) async throws -> T { + let actor = actor + let id = id + return try await withTaskCancellationHandler { + try await body(actor) + } onCancel: { + actor.cancelHandle.cancel(id) + } + } +} + // MARK: - Errors nonisolated enum PostgreSQLError: Error, LocalizedError { diff --git a/apple/TableProMobile/TableProMobile/Drivers/RedisDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/RedisDriver.swift index ba5713ea0d..97c917bbe6 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/RedisDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/RedisDriver.swift @@ -3,10 +3,12 @@ import Foundation import os import TableProDatabase import TableProModels +import TableProPluginKit import TableProRedisCore -nonisolated final class RedisDriver: DatabaseDriver, KeyContentsBrowsing, @unchecked Sendable { +nonisolated final class RedisDriver: DatabaseDriver, KeyContentsBrowsing, ExportRowStreaming, @unchecked Sendable { private let actor = RedisActor() + private let selected: OSAllocatedUnfairLock private let host: String private let port: Int private let username: String? @@ -35,6 +37,7 @@ nonisolated final class RedisDriver: DatabaseDriver, KeyContentsBrowsing, @unche self.password = password self.database = database self.ssl = ssl + self.selected = OSAllocatedUnfairLock(initialState: database) } // MARK: - Connection @@ -145,6 +148,36 @@ nonisolated final class RedisDriver: DatabaseDriver, KeyContentsBrowsing, @unche } try await send(["SELECT", dbNum]).throwIfError().throwIfQueued("SELECT") + if let index = Int(dbNum) { + selected.withLock { $0 = index } + } + } + + // MARK: - Transfer + + // The database the last SELECT moved the session to. + var selectedDatabase: Int { + selected.withLock { $0 } + } + + func selectDatabase(_ index: Int) async throws { + try await send(["SELECT", String(index)]).throwIfError().throwIfQueued("SELECT") + selected.withLock { $0 = index } + } + + func command(_ arguments: [String]) async throws -> RedisReplyValue { + try await send(arguments) + } + + // A bulk string's own bytes, which a value that is not UTF-8 text needs whole. + func bulkBytes(_ arguments: [String]) async throws -> Data? { + try await actor.bulkBytes(arguments.map { Data($0.utf8) }) + } + + // The Mac's KEYBROWSE export rows (`RedisExportRows`), a batch per pull. + func exportRows(query: String, readOnly: Bool) -> AsyncThrowingStream { + let cursor = RedisExportCursor(driver: self, query: query) + return AsyncThrowingStream(unfolding: { try await cursor.next() }) } func switchSchema(to name: String) async throws { @@ -287,6 +320,25 @@ private actor RedisActor { try executeCommand(args) } + // Nil for anything but a bulk string: a missing key, an error, a queued command. + func bulkBytes(_ args: [Data]) throws -> Data? { + guard let ctx else { throw RedisError.notConnected } + let rawReply = RedisCommandArgv.withPointers(args) { argv, argvlen in + redisCommandArgv(ctx, Int32(args.count), argv, argvlen) + } + guard let rawReply else { + if ctx.pointee.err != 0 { + let msg = withUnsafePointer(to: &ctx.pointee.errstr.0) { String(cString: $0) } + throw RedisError.connectionFailed(msg) + } + throw RedisError.connectionFailed("No reply from server") + } + defer { HiredisReply.release(rawReply) } + let reply = rawReply.assumingMemoryBound(to: redisReply.self) + guard reply.pointee.type == REDIS_REPLY_STRING, let str = reply.pointee.str else { return nil } + return Data(bytes: str, count: reply.pointee.len) + } + func fetchServerVersion() throws -> String? { let reply = try executeCommand(["INFO", "server"]) guard case .string(let info) = reply else { return nil } diff --git a/apple/TableProMobile/TableProMobile/Drivers/RedisExportRows.swift b/apple/TableProMobile/TableProMobile/Drivers/RedisExportRows.swift new file mode 100644 index 0000000000..c04298a5b4 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Drivers/RedisExportRows.swift @@ -0,0 +1,245 @@ +import Foundation +import TableProModels +import TableProPluginKit +import TableProRedisCore + +// The rows the Mac's `KEYBROWSE` export streams for one database (`RedisPluginDriver.streamScanRows`): every key SCAN +// lists, with its type, TTL, length and value, the value whole for a string and the Mac's preview for a collection. +nonisolated enum RedisExportRows { + static let columns = ["Key", "Type", "TTL", "Length", "Value"] + static let columnTypeNames = ["String", "RedisType", "RedisInt", "Int64", "RedisRaw"] + static let batchSize = 200 + + typealias Send = @Sendable ([String]) async throws -> RedisReplyValue + typealias ReadBytes = @Sendable ([String]) async throws -> Data? + + enum Request: Equatable { + case keyBrowse(database: Int?) + case command + } + + // The export statement the catalog hands out (`RedisQueryBuilder.buildExportQuery`), or any other command. + static func request(for query: String) -> Request { + let words = query.split(whereSeparator: \.isWhitespace).map(String.init) + guard words.first?.uppercased() == "KEYBROWSE" else { return .command } + guard words.count >= 3, words[1].uppercased() == "DB" else { return .keyBrowse(database: nil) } + return .keyBrowse(database: RedisDatabaseIndex.parse(words[2])) + } + + // A reply the server refused, which an ACL user gets for a key outside its patterns, is an empty cell, as on the Mac. + static func rows(keys: [String], send: Send, readBytes: ReadBytes) async throws -> [PluginRow] { + var rows: [PluginRow] = [] + rows.reserveCapacity(keys.count) + for key in keys { + try Task.checkCancellation() + rows.append(try await row(key: key, send: send, readBytes: readBytes)) + } + return rows + } + + static func row(key: String, send: Send, readBytes: ReadBytes) async throws -> PluginRow { + let typeName = answered(try await send(["TYPE", key])).flatMap(text) + let ttl = answered(try await send(["TTL", key])).flatMap(integer) + var length: PluginCellValue = .null + var value: PluginCellValue = .null + if let typeName, let kind = RedisKeyKind(typeName: typeName) { + length = .fromOptional(answered(try await send(RedisKeySummary.lengthCommand(for: kind, key: key))) + .flatMap(integer).map(String.init)) + if kind == .string { + value = try await readBytes(["GET", key]).map(stringCell) ?? .null + } else if let preview = answered(try await send(RedisKeySummary.previewCommand(for: kind, key: key))) { + value = previewCell(preview, kind: kind) + } + } + return [ + .text(key), + .fromOptional(typeName?.uppercased()), + .fromOptional(ttl.map(String.init)), + length, + value, + ] + } + + static func stringCell(_ bytes: Data) -> PluginCellValue { + RedisWireText.decode(bytes).map(PluginCellValue.text) ?? .bytes(bytes) + } + + static func previewCell(_ reply: RedisReplyValue, kind: RedisKeyKind) -> PluginCellValue { + switch kind { + case .string: + return .fromOptional(reply.stringRepresentation) + case .hash: + return .fromOptional(RedisKeySummary.jsonObject(flatPairs: scanElements(of: reply).map(displayText))) + case .list: + return .fromOptional(RedisKeySummary.jsonArray(elements: elements(of: reply).map(displayText))) + case .set: + return .fromOptional(RedisKeySummary.jsonArray(elements: scanElements(of: reply).map(displayText))) + case .zset: + return .fromOptional(RedisKeySummary.jsonScorePairs(flatPairs: elements(of: reply).map(displayText))) + case .stream: + return .fromOptional(RedisKeySummary.jsonStreamEntries(streamEntries(of: reply))) + } + } + + // The Mac's `RedisReply.displayText`. + static func displayText(_ reply: RedisReplyValue) -> String { + switch reply { + case .string(let text), .status(let text): return text + case .error(let message): return "(error) \(message)" + case .integer(let value): return String(value) + case .array(let items): return "[\(items.map(displayText).joined(separator: ", "))]" + case .null: return "(nil)" + } + } + + private static func answered(_ reply: RedisReplyValue) -> RedisReplyValue? { + switch reply { + case .error, .null: return nil + default: return reply.isQueued ? nil : reply + } + } + + private static func text(_ reply: RedisReplyValue) -> String? { + switch reply { + case .string(let value), .status(let value): return value + default: return nil + } + } + + private static func integer(_ reply: RedisReplyValue) -> Int64? { + guard case .integer(let value) = reply else { return nil } + return value + } + + private static func elements(of reply: RedisReplyValue) -> [RedisReplyValue] { + guard case .array(let items) = reply else { return [] } + return items + } + + // HSCAN and SSCAN answer `[cursor, [elements]]`. + private static func scanElements(of reply: RedisReplyValue) -> [RedisReplyValue] { + if case .array(let parts) = reply, parts.count == 2, case .array(let items) = parts[1] { + return items + } + return elements(of: reply) + } + + private static func streamEntries(of reply: RedisReplyValue) -> [(id: String, flatFields: [String])] { + elements(of: reply).compactMap { entry in + guard case .array(let parts) = entry, parts.count >= 2, case .array(let fields) = parts[1] else { return nil } + return (id: displayText(parts[0]), flatFields: fields.map(displayText)) + } + } +} + +// The pull side of one export read: a SCAN page at a time, a batch of rows per pull. A read of another database selects +// it for the read and goes back to the session's after. +nonisolated final class RedisExportCursor: @unchecked Sendable { + private let driver: RedisDriver + private let query: String + private let lock = NSLock() + private var started = false + private var finished = false + private var scanCursor = RedisScanPage.startCursor + private var scanned = false + private var pending: [String] = [] + private var restoreDatabase: Int? + private var commandRows: [PluginRow]? + + init(driver: RedisDriver, query: String) { + self.driver = driver + self.query = query + } + + deinit { + guard let restore = restoreDatabase, !finished else { return } + let driver = driver + Task { try? await driver.selectDatabase(restore) } + } + + func next() async throws -> PluginStreamElement? { + try Task.checkCancellation() + switch RedisExportRows.request(for: query) { + case .command: + return try await nextCommandElement() + case .keyBrowse(let database): + return try await nextKeyBrowseElement(database: database) + } + } + + // Any other command answers whole: its header on the first pull, its rows on the next. + private func nextCommandElement() async throws -> PluginStreamElement? { + if let rows = lock.withLock({ () -> [PluginRow]? in + defer { commandRows = nil } + return commandRows + }) { + lock.withLock { finished = true } + return .rows(rows) + } + guard lock.withLock({ !started && !finished }) else { return nil } + lock.withLock { started = true } + let result = try await driver.execute(query: query) + let rows = result.rows.map { $0.map(PluginCellValue.fromOptional) } + lock.withLock { + if rows.isEmpty { + finished = true + } else { + commandRows = rows + } + } + return .header(PluginStreamHeader(columns: result.columns.map(\.name), columnTypeNames: result.columns.map(\.typeName))) + } + + private func nextKeyBrowseElement(database: Int?) async throws -> PluginStreamElement? { + guard lock.withLock({ !finished }) else { return nil } + if lock.withLock({ !started }) { + lock.withLock { started = true } + let session = driver.selectedDatabase + if let database, database != session { + try await driver.selectDatabase(database) + lock.withLock { restoreDatabase = session } + } + return .header(PluginStreamHeader( + columns: RedisExportRows.columns, columnTypeNames: RedisExportRows.columnTypeNames + )) + } + while true { + try Task.checkCancellation() + let batch: [String] = lock.withLock { + let taken = Array(pending.prefix(RedisExportRows.batchSize)) + pending.removeFirst(taken.count) + return taken + } + if !batch.isEmpty { + let driver = driver + return .rows(try await RedisExportRows.rows( + keys: batch, + send: { try await driver.command($0) }, + readBytes: { try await driver.bulkBytes($0) } + )) + } + let (cursor, wasScanned) = lock.withLock { (scanCursor, scanned) } + if wasScanned, cursor == RedisScanPage.startCursor { + try await finish() + return nil + } + let reply = try await driver.command(RedisKeyspaceReads.scanArguments(cursor: cursor)) + let page = try RedisScanPage(reply: reply, command: "SCAN") + lock.withLock { + scanned = true + scanCursor = page.cursor + pending += page.elements + } + } + } + + private func finish() async throws { + let restore = lock.withLock { () -> Int? in + finished = true + return restoreDatabase + } + if let restore { + try await driver.selectDatabase(restore) + } + } +} diff --git a/apple/TableProMobile/TableProMobile/Drivers/SQLiteDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/SQLiteDriver.swift index c411752fdd..1ab2ed4bdf 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/SQLiteDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/SQLiteDriver.swift @@ -1,13 +1,18 @@ import Foundation +import os import SQLite3 import TableProDatabase import TableProModels import TableProPluginKit import TableProSQLQuoting -nonisolated final class SQLiteDriver: DatabaseDriver, @unchecked Sendable { +nonisolated final class SQLiteDriver: DatabaseDriver, CatalogQuerying, ExportRowStreaming, + DedicatedTransferConnection, @unchecked Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "SQLiteDriver") + private let source: LocalDatabaseFileSource private let openMode: LocalDatabaseOpenMode + private let access: SQLiteAccess private let fileAccess = LocalDatabaseFileAccess() private let actor = SQLiteActor() @@ -16,16 +21,22 @@ nonisolated final class SQLiteDriver: DatabaseDriver, @unchecked Sendable { var supportsTransactions: Bool { true } var serverVersion: String? { String(cString: sqlite3_libversion()) } - // SQLite ends statement text at a NUL, so a NUL in a value goes in as char(0) between two literals. func escapeStringLiteral(_ value: String) -> String { - value.split(separator: "\0", omittingEmptySubsequences: false) - .map { SQLQuoting.stringLiteralBody(String($0)) } - .joined(separator: "'||char(0)||'") + Self.literalBody(value) + } + + static func literalBody(_ value: String) -> String { + sqliteStringLiteralBody(value) } - init(source: LocalDatabaseFileSource, openMode: LocalDatabaseOpenMode = .existingOnly) { + init( + source: LocalDatabaseFileSource, + openMode: LocalDatabaseOpenMode = .existingOnly, + access: SQLiteAccess = .readWrite + ) { self.source = source self.openMode = openMode + self.access = access } // MARK: - Connection @@ -43,9 +54,13 @@ nonisolated final class SQLiteDriver: DatabaseDriver, @unchecked Sendable { func disconnect() async throws { await actor.close() fileAccess.end() + if access == .snapshot, case .file(let url) = source { + try? FileManager.default.removeItem(at: url) + } } private var openFlags: Int32 { + guard access == .readWrite else { return SQLITE_OPEN_READONLY } guard openMode == .createNew || source == .inMemory else { return SQLITE_OPEN_READWRITE } return SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE } @@ -58,7 +73,13 @@ nonisolated final class SQLiteDriver: DatabaseDriver, @unchecked Sendable { // MARK: - Query Execution func execute(query: String) async throws -> QueryResult { - let raw = try await actor.execute(query) + try await executeParameterized(query: query, parameters: [], typeNames: nil) + } + + // Bound the way the Mac plugin binds: text as UTF-8 text, bytes as a blob, nothing else rewritten. + func executeParameterized(query: String, parameters: [PluginCellValue], typeNames: [String?]?) async throws + -> QueryResult { + let raw = try await actor.execute(query, parameters: parameters) return QueryResult( columns: raw.columns.enumerated().map { i, name in ColumnInfo( @@ -139,6 +160,59 @@ nonisolated final class SQLiteDriver: DatabaseDriver, @unchecked Sendable { await actor.transactionState() } + // MARK: - Transfer + + func catalogQuery(_ sql: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + try await actor.catalogRows(sql, parameters: parameters) + } + + // Every row, a blob as its bytes, one batch per pull. A cancel interrupts the step only while this read is the one + // stepping, so the grid's statement on the same connection runs on. + func exportRows(query: String, readOnly: Bool) -> AsyncThrowingStream { + let cursor = SQLiteExportCursor(actor: actor, query: query) + return AsyncThrowingStream(unfolding: { try await cursor.next() }) + } + + func foreignKeyCheckStatements(enabled: Bool) -> [String] { + enabled ? SQLiteCatalogReads.foreignKeyEnableStatements : SQLiteCatalogReads.foreignKeyDisableStatements + } + + // A second reader in rollback-journal mode makes the user's COMMIT fail with SQLITE_BUSY, so outside WAL the + // transfer reads a copy, made only while the session is idle and the copy fits. Every other case reads the session. + func openTransferDriver() async throws -> any DatabaseDriver { + guard access == .readWrite, source != .inMemory, + await actor.transactionState() == .idle else { return self } + if await actor.journalMode() == "wal" { + let reader = SQLiteDriver(source: source, access: .readOnly) + do { + try await reader.connect() + return reader + } catch { + Self.logger.warning("A second SQLite reader did not open: \(error.localizedDescription, privacy: .private)") + return self + } + } + guard let copy = await actor.snapshot(into: Self.snapshotDirectory) else { return self } + let snapshot = SQLiteDriver(source: .file(copy), access: .snapshot) + do { + try await snapshot.connect() + return snapshot + } catch { + try? FileManager.default.removeItem(at: copy) + Self.logger.warning("A SQLite snapshot did not open: \(error.localizedDescription, privacy: .private)") + return self + } + } + + static var snapshotDirectory: URL { + FileManager.default.temporaryDirectory.appendingPathComponent("TransferSnapshots", isDirectory: true) + } + + // A snapshot a closed or killed app left behind; nothing reads one across launches. + static func removeLeftoverSnapshots() { + try? FileManager.default.removeItem(at: snapshotDirectory) + } + // MARK: - Schema func fetchTables(schema: String?) async throws -> [TableInfo] { @@ -357,6 +431,10 @@ private actor SQLiteActor { func close() { interruptHandle.detach() finalizeStreamingStatement() + for statement in exportStatements.values { + sqlite3_finalize(statement.handle) + } + exportStatements.removeAll() if let db { sqlite3_close(db) self.db = nil @@ -370,7 +448,7 @@ private actor SQLiteActor { return sqlite3_get_autocommit(db) == 0 ? .explicitTransaction : .idle } - func execute(_ query: String) throws -> RawResult { + func execute(_ query: String, parameters: [PluginCellValue] = []) throws -> RawResult { guard let db else { throw SQLiteError.notConnected } let start = Date() @@ -382,9 +460,11 @@ private actor SQLiteActor { defer { sqlite3_finalize(stmt) } // Text with only whitespace or comments prepares to no statement, and stepping nothing is a misuse. guard stmt != nil else { + guard parameters.isEmpty else { throw SQLiteError.queryFailed(Self.parameterCountMessage) } return RawResult(columns: [], columnTypes: [], rows: [], rowsAffected: 0, executionTime: Date().timeIntervalSince(start), isTruncated: false) } + try Self.bind(parameters, to: stmt, db: db) let colCount = sqlite3_column_count(stmt) var columns: [String] = [] @@ -455,6 +535,199 @@ private actor SQLiteActor { return Int(sqlite3_changes64(db)) } + // MARK: - Binding + + private static let parameterCountMessage = "The statement does not take the values it was given" + + private static func bind(_ parameters: [PluginCellValue], to statement: OpaquePointer?, db: OpaquePointer) throws { + // A statement run with no values leaves any placeholder NULL, as it always has. + guard !parameters.isEmpty else { return } + guard Int(sqlite3_bind_parameter_count(statement)) == parameters.count else { + throw SQLiteError.queryFailed(parameterCountMessage) + } + let transient = unsafeBitCast(-1, to: sqlite3_destructor_type.self) + for (index, parameter) in parameters.enumerated() { + let position = Int32(index + 1) + let result: Int32 + switch parameter { + case .null: + result = sqlite3_bind_null(statement, position) + case .text(let text): + result = sqlite3_bind_text64( + statement, position, text, sqlite3_uint64(text.utf8.count), transient, UInt8(SQLITE_UTF8) + ) + case .bytes(let data): + // An empty Data has no base address, and a null pointer would bind NULL rather than an empty blob. + let bytes = data.isEmpty ? Data([0]) : data + result = bytes.withUnsafeBytes { buffer in + sqlite3_bind_blob64(statement, position, buffer.baseAddress, sqlite3_uint64(data.count), transient) + } + } + guard result == SQLITE_OK else { + throw SQLiteError.queryFailed(String(cString: sqlite3_errmsg(db))) + } + } + } + + // MARK: - Transfer + + // Every row the statement returns, each value as SQLite holds it. + func catalogRows(_ query: String, parameters: [PluginCellValue]) throws -> PluginQueryResult { + guard let db else { throw SQLiteError.notConnected } + let start = Date() + var stmt: OpaquePointer? + guard sqlite3_prepare_v2(db, query, -1, &stmt, nil) == SQLITE_OK else { + throw SQLiteError.queryFailed(String(cString: sqlite3_errmsg(db))) + } + defer { sqlite3_finalize(stmt) } + guard let stmt else { + return PluginQueryResult( + columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0 + ) + } + try Self.bind(parameters, to: stmt, db: db) + let header = Self.header(of: stmt) + var rows: [[PluginCellValue]] = [] + let totalBefore = sqlite3_total_changes64(db) + var stepResult = sqlite3_step(stmt) + while stepResult == SQLITE_ROW { + rows.append(Self.values(of: stmt, count: header.columns.count)) + stepResult = sqlite3_step(stmt) + } + guard stepResult == SQLITE_DONE else { + throw SQLiteError.queryFailed(String(cString: sqlite3_errmsg(db))) + } + return PluginQueryResult( + columns: header.columns, + columnTypeNames: header.columnTypeNames, + rows: rows, + rowsAffected: header.columns.isEmpty ? changes(of: db, sinceTotal: totalBefore) : 0, + executionTime: Date().timeIntervalSince(start) + ) + } + + private static func header(of stmt: OpaquePointer) -> PluginStreamHeader { + let count = Int(sqlite3_column_count(stmt)) + let names = (0.. [PluginCellValue] { + (0.. 0, let pointer = sqlite3_column_blob(stmt, column) else { return .bytes(Data()) } + return .bytes(Data(bytes: pointer, count: byteCount)) + default: + return text(of: stmt, column: column).map(PluginCellValue.text) ?? .null + } + } + } + + // Kept apart from `streamingStmt`, so the grid's next page never ends a transfer reading the same connection. + private var exportStatements: [ObjectIdentifier: (handle: OpaquePointer, columnCount: Int)] = [:] + // A read whose last batch carried rows: its statement is already finalized, and the next pull ends the stream. + private var finishedExports: Set = [] + + func beginExport(query: String, stream: SQLiteStreamToken) throws -> PluginStreamHeader { + guard let db else { throw SQLiteError.notConnected } + try Task.checkCancellation() + var stmt: OpaquePointer? + guard sqlite3_prepare_v2(db, query, -1, &stmt, nil) == SQLITE_OK else { + throw SQLiteError.queryFailed(String(cString: sqlite3_errmsg(db))) + } + guard let stmt else { throw SQLiteError.queryFailed("Failed to prepare statement") } + let header = Self.header(of: stmt) + exportStatements[ObjectIdentifier(stream)] = (stmt, header.columns.count) + return header + } + + // Nil once the statement has no rows left, and the statement is then finalized. + func nextExportRows(stream: SQLiteStreamToken, limit: Int) throws -> [[PluginCellValue]]? { + guard let statement = exportStatements[ObjectIdentifier(stream)] else { + if finishedExports.remove(ObjectIdentifier(stream)) != nil { return nil } + throw CancellationError() + } + var rows: [[PluginCellValue]] = [] + while rows.count < limit { + let outcome: (result: Int32, interrupted: Bool) + do { + outcome = try step(statement.handle, of: stream) + } catch { + endExport(stream) + throw error + } + if outcome.interrupted { + endExport(stream) + throw CancellationError() + } + switch outcome.result { + case SQLITE_ROW: + rows.append(Self.values(of: statement.handle, count: statement.columnCount)) + case SQLITE_DONE: + endExport(stream) + guard !rows.isEmpty else { return nil } + finishedExports.insert(ObjectIdentifier(stream)) + return rows + default: + let message = String(cString: sqlite3_errmsg(sqlite3_db_handle(statement.handle))) + endExport(stream) + throw SQLiteError.queryFailed(message) + } + } + return rows + } + + func endExport(_ stream: SQLiteStreamToken) { + finishedExports.remove(ObjectIdentifier(stream)) + guard let statement = exportStatements.removeValue(forKey: ObjectIdentifier(stream)) else { return } + sqlite3_finalize(statement.handle) + } + + func journalMode() -> String? { + pragmaValue("journal_mode")?.lowercased() + } + + private func pragmaValue(_ name: String) -> String? { + guard let result = try? execute("PRAGMA \(name)"), let row = result.rows.first, let value = row.first else { + return nil + } + return value + } + + // Nil when the session holds a transaction, the copy would not fit, or SQLite refuses it; the partial copy is + // removed. VACUUM INTO copies the main database only, rowids included, as SQLite 3.27 and later make it. + func snapshot(into directory: URL) -> URL? { + guard let db, sqlite3_get_autocommit(db) != 0 else { return nil } + guard let pageCount = pragmaValue("page_count").flatMap(Int64.init), + let pageSize = pragmaValue("page_size").flatMap(Int64.init) else { return nil } + do { + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + } catch { + return nil + } + let free = try? directory.resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]) + .volumeAvailableCapacityForImportantUsage + guard let free, free > pageCount * pageSize else { return nil } + let destination = directory.appendingPathComponent("\(UUID().uuidString).sqlite") + do { + _ = try execute("VACUUM INTO \(SQLQuoting.stringLiteral(destination.path))") + return destination + } catch { + try? FileManager.default.removeItem(at: destination) + return nil + } + } + // MARK: - Streaming /// One statement streams at a time, held with the token of the stream that began it, so the @@ -599,6 +872,54 @@ private actor SQLiteActor { } } +nonisolated enum SQLiteAccess: Sendable, Equatable { + case readWrite + case readOnly + // A copy made for one transfer, removed when the driver disconnects. + case snapshot +} + +// The pull side of one export read. A reader that stops before the end finalizes the statement when it lets go. +nonisolated private final class SQLiteExportCursor: @unchecked Sendable { + private static let batchSize = 500 + + private let actor: SQLiteActor + private let query: String + private let stream = SQLiteStreamToken() + private let lock = NSLock() + private var started = false + + init(actor: SQLiteActor, query: String) { + self.actor = actor + self.query = query + } + + deinit { + let actor = actor + let stream = stream + Task { await actor.endExport(stream) } + } + + func next() async throws -> PluginStreamElement? { + let actor = actor + let stream = stream + let interruptHandle = actor.interruptHandle + let isFirst = lock.withLock { + defer { started = true } + return !started + } + return try await withTaskCancellationHandler { + if isFirst { + return .header(try await actor.beginExport(query: query, stream: stream)) + } + guard let rows = try await actor.nextExportRows(stream: stream, limit: Self.batchSize) else { return nil } + return .rows(rows) + } onCancel: { + interruptHandle.interrupt(stream) + } + } +} + nonisolated enum SQLiteBeginStreamResult: Sendable { case rowSet([ColumnInfo]) case commandOk(affectedRows: Int) diff --git a/apple/TableProMobile/TableProMobile/Helpers/ClipboardExporter.swift b/apple/TableProMobile/TableProMobile/Helpers/ClipboardExporter.swift index f92dfca54b..87a7c46a48 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/ClipboardExporter.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/ClipboardExporter.swift @@ -2,6 +2,7 @@ import TableProMSSQLCore import Foundation import TableProDatabase import TableProModels +import TableProPluginKit import TableProSQLQuoting import TableProTabularIO import UIKit @@ -53,18 +54,21 @@ nonisolated enum ClipboardExporter { incompleteColumns(columns: columns, rows: [cells]) } + // `columnDetails` is the table's catalog read, which alone says which columns the server fills in itself. static func exportRow( columns: [ColumnInfo], cells: [Cell], format: ExportFormat, tableName: String? = nil, databaseType: DatabaseType, - driver: (any DatabaseDriver)? + driver: (any DatabaseDriver)?, + columnDetails: [ColumnInfo] = [] ) throws(IncompleteValues) -> String { let values = try completeValues(columns: columns, rows: [cells]) return exportRow( columns: columns, row: values[0], - format: format, tableName: tableName, databaseType: databaseType, driver: driver + format: format, tableName: tableName, databaseType: databaseType, driver: driver, + columnDetails: columnDetails ) } @@ -74,12 +78,14 @@ nonisolated enum ClipboardExporter { format: ExportFormat, tableName: String? = nil, databaseType: DatabaseType, - driver: (any DatabaseDriver)? + driver: (any DatabaseDriver)?, + columnDetails: [ColumnInfo] = [] ) throws(IncompleteValues) -> String { let values = try completeValues(columns: columns, rows: rows) return exportRows( columns: columns, rows: values, - format: format, tableName: tableName, databaseType: databaseType, driver: driver + format: format, tableName: tableName, databaseType: databaseType, driver: driver, + columnDetails: columnDetails ) } @@ -89,7 +95,8 @@ nonisolated enum ClipboardExporter { format: ExportFormat, tableName: String? = nil, databaseType: DatabaseType, - driver: (any DatabaseDriver)? + driver: (any DatabaseDriver)?, + columnDetails: [ColumnInfo] = [] ) -> String { switch format { case .json: @@ -97,10 +104,10 @@ nonisolated enum ClipboardExporter { case .csv: return rowToCsv(columns: columns, row: row, includeHeader: true) case .sqlInsert: - return rowToInsert( - columns: columns, row: row, tableName: tableName ?? "table", + return InsertWriter( + columns: columns, columnDetails: columnDetails, tableName: tableName ?? "table", databaseType: databaseType, driver: driver - ) + ).statement(for: row) } } @@ -110,7 +117,8 @@ nonisolated enum ClipboardExporter { format: ExportFormat, tableName: String? = nil, databaseType: DatabaseType, - driver: (any DatabaseDriver)? + driver: (any DatabaseDriver)?, + columnDetails: [ColumnInfo] = [] ) -> String { switch format { case .json: @@ -125,13 +133,11 @@ nonisolated enum ClipboardExporter { } return ([header] + dataLines).joined(separator: "\n") case .sqlInsert: - let name = tableName ?? "table" - return rows.map { - rowToInsert( - columns: columns, row: $0, tableName: name, - databaseType: databaseType, driver: driver - ) - }.joined(separator: "\n") + let writer = InsertWriter( + columns: columns, columnDetails: columnDetails, tableName: tableName ?? "table", + databaseType: databaseType, driver: driver + ) + return rows.map { writer.statement(for: $0) }.joined(separator: "\n") } } @@ -154,7 +160,7 @@ nonisolated enum ClipboardExporter { // MARK: - Private - private static func incompleteColumns(columns: [ColumnInfo], rows: [[Cell]]) -> [String] { + static func incompleteColumns(columns: [ColumnInfo], rows: [[Cell]]) -> [String] { columns.indices.compactMap { index in let isIncomplete = rows.contains { cells in guard index < cells.count else { return false } @@ -188,49 +194,9 @@ nonisolated enum ClipboardExporter { return "{\n" + pairs.joined(separator: ",\n") + "\n}" } - // The column's type decides, as in the Mac export: text that reads like a number or `true` stays text. A column with - // no type, as SQLite reports for an expression, is read by its value. + // The macOS JSON export's own rule, so a row copied here and a table exported there agree on every value. private static func jsonValue(_ value: String, typeName: String) -> String { - if typeName.trimmingCharacters(in: .whitespaces).isEmpty { - let bare = isJsonNumber(value) || value == "true" || value == "false" - return bare ? value : JSONText.stringLiteral(value) - } - if isBooleanType(typeName), let flag = booleanLiteral(value) { - return flag - } - if isNumberType(typeName), isJsonNumber(value) { - return value - } - return JSONText.stringLiteral(value) - } - - // PostgreSQL's text format sends `t` or `f`, and SQLite keeps a boolean as 1 or 0. - private static func booleanLiteral(_ value: String) -> String? { - switch value.lowercased() { - case "true", "t", "1": "true" - case "false", "f", "0": "false" - default: nil - } - } - - // SQL Server documents `bit` as an integer type that holds 1, 0 or NULL. - private static func isNumberType(_ typeName: String) -> Bool { - ColumnInfo.isNumericType(typeName: typeName) || unwrappedTypeName(typeName) == "bit" - } - - private static func isBooleanType(_ typeName: String) -> Bool { - let base = unwrappedTypeName(typeName) - return base == "bool" || base == "boolean" - } - - // ClickHouse wraps the type, as in `Nullable(Bool)`. - private static func unwrappedTypeName(_ typeName: String) -> String { - var base = typeName.lowercased().trimmingCharacters(in: .whitespaces) - while base.hasSuffix(")"), let open = base.firstIndex(of: "("), - ["nullable", "lowcardinality"].contains(String(base[.. String { @@ -245,27 +211,73 @@ nonisolated enum ClipboardExporter { return lines.joined(separator: "\n") } - private static func rowToInsert( - columns: [ColumnInfo], - row: [String?], - tableName: String, - databaseType: DatabaseType, - driver: (any DatabaseDriver)? - ) -> String { - let quotedTable = SQLBuilder.quoteIdentifier(tableName, for: databaseType) - let cols = columns - .map { SQLBuilder.quoteIdentifier($0.name, for: databaseType) } - .joined(separator: ", ") - let vals = columns.indices.map { i in - let value = i < row.count ? row[i] : nil - guard let value else { return "NULL" } - if databaseType == .mssql { return MSSQLStringLiteral.quoted(value) } - return "'\(escapeLiteral(value, databaseType: databaseType, driver: driver))'" - }.joined(separator: ", ") - return "INSERT INTO \(quotedTable) (\(cols)) VALUES (\(vals));" + // The rules the macOS SQL export writes a row by: a column the server fills in is left out, and a MySQL BIT value goes + // bare, since a quoted '5' is stored as the byte 53. + nonisolated private struct InsertWriter { + let columns: [ColumnInfo] + let tableName: String + let databaseType: DatabaseType + let driver: (any DatabaseDriver)? + private let written: [Int] + private let bitColumns: Set + private let firstWritableColumn: String? + + init( + columns: [ColumnInfo], + columnDetails: [ColumnInfo], + tableName: String, + databaseType: DatabaseType, + driver: (any DatabaseDriver)? + ) { + self.columns = columns + self.tableName = tableName + self.databaseType = databaseType + self.driver = driver + let serverOwned = Set(columnDetails.filter(\.isServerOwned).map(\.name)) + let declaredTypes = Dictionary(columnDetails.map { ($0.name, $0.typeName) }, uniquingKeysWith: { first, _ in first }) + written = columns.indices.filter { !serverOwned.contains(columns[$0].name) } + let readsBitAsNumber = SQLBuilder.speaksMySQLDialect(databaseType) + bitColumns = Set(written.filter { index in + readsBitAsNumber && ClipboardExporter.isBitType(declaredTypes[columns[index].name] ?? columns[index].typeName) + }) + firstWritableColumn = columnDetails.first { !$0.isServerOwned }?.name + } + + func statement(for row: [String?]) -> String { + let quotedTable = SQLBuilder.quoteIdentifier(tableName, for: databaseType) + let first = firstWritableColumn.map { SQLBuilder.quoteIdentifier($0, for: databaseType) } + if written.isEmpty, let defaults = AllDefaultsInsert.sql( + into: quotedTable, databaseTypeId: databaseType.rawValue, firstWritableColumn: first + ) { + return "\(defaults);" + } + // An engine with no all-defaults spelling gets every column, which at least shows what the row held. + let written = written.isEmpty ? Array(columns.indices) : written + let cols = written + .map { SQLBuilder.quoteIdentifier(columns[$0].name, for: databaseType) } + .joined(separator: ", ") + let vals = written.map { index -> String in + guard index < row.count, let value = row[index] else { return "NULL" } + if bitColumns.contains(index), ClipboardExporter.isUnsignedInteger(value) { return value } + if databaseType == .mssql { return MSSQLStringLiteral.quoted(value) } + return "'\(ClipboardExporter.literalBody(value, databaseType: databaseType, driver: driver))'" + }.joined(separator: ", ") + return "INSERT INTO \(quotedTable) (\(cols)) VALUES (\(vals));" + } + } + + // StarRocks' BITMAP is not this type. + static func isBitType(_ typeName: String) -> Bool { + let upper = typeName.uppercased().trimmingCharacters(in: .whitespaces) + return upper == "BIT" || upper.hasPrefix("BIT(") + } + + static func isUnsignedInteger(_ value: String) -> Bool { + !value.isEmpty && value.utf8.allSatisfy { $0 >= UInt8(ascii: "0") && $0 <= UInt8(ascii: "9") } } - private static func escapeLiteral( + // The copy declares no sql_mode, so a MySQL literal keeps the escapes every MySQL session reads. + static func literalBody( _ value: String, databaseType: DatabaseType, driver: (any DatabaseDriver)? @@ -284,53 +296,6 @@ nonisolated enum ClipboardExporter { return escapeCsvField(value) } - // ASCII digits only, so "01234", "+5" or Arabic-Indic digits stay quoted strings. - private static func isJsonNumber(_ value: String) -> Bool { - var characters = Substring(value) - if characters.first == "-" { characters = characters.dropFirst() } - guard let first = characters.first, isAsciiDigit(first) else { return false } - if first == "0", characters.count > 1, characters.dropFirst().first.map(isAsciiDigit) == true { - return false - } - - var sawDigit = false - var index = characters.startIndex - while index < characters.endIndex, isAsciiDigit(characters[index]) { - sawDigit = true - index = characters.index(after: index) - } - guard sawDigit else { return false } - - if index < characters.endIndex, characters[index] == "." { - index = characters.index(after: index) - var sawFraction = false - while index < characters.endIndex, isAsciiDigit(characters[index]) { - sawFraction = true - index = characters.index(after: index) - } - guard sawFraction else { return false } - } - - if index < characters.endIndex, characters[index] == "e" || characters[index] == "E" { - index = characters.index(after: index) - if index < characters.endIndex, characters[index] == "+" || characters[index] == "-" { - index = characters.index(after: index) - } - var sawExponent = false - while index < characters.endIndex, isAsciiDigit(characters[index]) { - sawExponent = true - index = characters.index(after: index) - } - guard sawExponent else { return false } - } - - return index == characters.endIndex - } - - private static func isAsciiDigit(_ character: Character) -> Bool { - character.isASCII && character.isNumber - } - private static func escapeCsvField(_ field: String) -> String { if field.unicodeScalars.contains(where: { $0 == "," || $0 == "\"" || $0 == "\n" || $0 == "\r" }) { return "\"" + field.replacingOccurrences(of: "\"", with: "\"\"", options: .literal) + "\"" diff --git a/apple/TableProMobile/TableProMobile/Helpers/RowInserter.swift b/apple/TableProMobile/TableProMobile/Helpers/RowInserter.swift index 2ec25c6f66..d74cf3d026 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/RowInserter.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/RowInserter.swift @@ -26,7 +26,8 @@ nonisolated enum RowInsertPlanner { type: type, driver: driver, columns: values.map(\.column), - values: values.map(\.value) + values: values.map(\.value), + firstWritableColumn: columns.first { !$0.isServerOwned }?.name ) } } diff --git a/apple/TableProMobile/TableProMobile/Helpers/SQLBuilder.swift b/apple/TableProMobile/TableProMobile/Helpers/SQLBuilder.swift index 4fbe342211..1358ee60b1 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/SQLBuilder.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/SQLBuilder.swift @@ -182,6 +182,7 @@ nonisolated enum SQLBuilder { return "UPDATE \(quotedTable) SET \(assignments) WHERE \(predicate)" } + // `firstWritableColumn` is the column Oracle names in a row of only defaults. static func buildInsert( table: String, schema: String?, @@ -189,11 +190,12 @@ nonisolated enum SQLBuilder { type: DatabaseType, driver: any DatabaseDriver, columns: [String], - values: [String?] + values: [String?], + firstWritableColumn: String? = nil ) -> String? { let qualifiedTable = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) guard !columns.isEmpty else { - return buildAllDefaultsInsert(qualifiedTable: qualifiedTable, for: type) + return buildAllDefaultsInsert(qualifiedTable: qualifiedTable, for: type, firstWritableColumn: firstWritableColumn) } let cols = columns.map { quoteIdentifier($0, for: type) }.joined(separator: ", ") let vals = values.map { val in @@ -203,13 +205,17 @@ nonisolated enum SQLBuilder { return "INSERT INTO \(qualifiedTable) (\(cols)) VALUES (\(vals))" } - // Oracle accepts neither `() VALUES ()` nor `DEFAULT VALUES`, so it gets no statement. - static func buildAllDefaultsInsert(qualifiedTable: String, for type: DatabaseType) -> String? { - if speaksMySQLDialect(type) { - return "INSERT INTO \(qualifiedTable) () VALUES ()" - } - let takesDefaultValues = type.isSQLiteFamily || [.postgresql, .redshift, .mssql, .duckdb].contains(type) - return takesDefaultValues ? "INSERT INTO \(qualifiedTable) DEFAULT VALUES" : nil + // The macOS spelling through the shared `AllDefaultsInsert`: Oracle names one column with `VALUES (DEFAULT)`. + static func buildAllDefaultsInsert( + qualifiedTable: String, + for type: DatabaseType, + firstWritableColumn: String? = nil + ) -> String? { + AllDefaultsInsert.sql( + into: qualifiedTable, + databaseTypeId: type.rawValue, + firstWritableColumn: firstWritableColumn.map { quoteIdentifier($0, for: type) } + ) } // SQLite has no TRUNCATE statement; a DELETE with no WHERE is its truncate. @@ -218,8 +224,9 @@ nonisolated enum SQLBuilder { return type.isSQLiteFamily ? "DELETE FROM \(quoted)" : "TRUNCATE TABLE \(quoted)" } + // Given a column to name, which every table with a column the server does not fill in has. static func supportsAllDefaultsInsert(_ type: DatabaseType) -> Bool { - buildAllDefaultsInsert(qualifiedTable: "t", for: type) != nil + buildAllDefaultsInsert(qualifiedTable: "t", for: type, firstWritableColumn: "c") != nil } // SQL Server resolves `[schema].[table]` in the session's database, which a `USE` run in the Query tab moves, so @@ -301,6 +308,54 @@ nonisolated enum SQLBuilder { return "SELECT COUNT(*) FROM \(quoted) \(whereClause)" } + // MARK: - Export + + // The data browser's rows with its filter, search and sort, every page of them: the table export a filtered view + // runs, as macOS exports `baseQueryForMore`. + static func buildExportSelect( + table: String, + schema: String?, + database: String? = nil, + type: DatabaseType, + filters: [TableFilter] = [], + logicMode: FilterLogicMode = .and, + columns: [ColumnInfo] = [], + session: SQLSessionLexicalFacts? = nil, + searchText: String? = nil, + searchColumns: [ColumnInfo] = [], + sortState: SortState = SortState() + ) -> String { + let quoted = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) + let whereClause = buildSearchWhereClause( + searchText: searchText ?? "", searchColumns: searchColumns, + filters: filters, logicMode: logicMode, columns: columns, type: type, session: session + ) + let orderBy = buildOrderByClause(sortState, for: type) + var sql = "SELECT * FROM \(quoted)" + if !whereClause.isEmpty { sql += " \(whereClause)" } + if !orderBy.isEmpty { sql += " \(orderBy)" } + return sql + } + + // Rows picked in the browser, read again by their keys so the file holds whole values and bytes, not the page's + // previews. + static func buildRowsSelect( + table: String, + schema: String?, + database: String? = nil, + type: DatabaseType, + keys: [[(column: String, value: String)]], + columns: [ColumnInfo], + driver: any DatabaseDriver + ) -> String { + let quoted = qualifiedIdentifier(table: table, schema: schema, database: database, for: type) + let predicates = keys.filter { !$0.isEmpty }.map { key in + "(\(keyPredicate(key, columns: columns, type: type, driver: driver)))" + } + let condition = predicates.isEmpty ? "1 = 0" : predicates.joined(separator: " OR ") + return "SELECT * FROM \(quoted) WHERE \(condition)" + } + // MARK: - Search static func buildSearchSelect( diff --git a/apple/TableProMobile/TableProMobile/Helpers/SceneIntent.swift b/apple/TableProMobile/TableProMobile/Helpers/SceneIntent.swift index 73084c0a0f..87a748b73e 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/SceneIntent.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/SceneIntent.swift @@ -4,6 +4,7 @@ import Foundation nonisolated enum SceneIntent: Hashable, Sendable { case openConnection(UUID, table: String?) case importConnections(URL) + case importData(URL) static let viewConnectionActivity = "com.TablePro.viewConnection" static let viewTableActivity = "com.TablePro.viewTable" @@ -11,8 +12,11 @@ nonisolated enum SceneIntent: Hashable, Sendable { static func parse(url: URL) -> SceneIntent? { if url.isFileURL { - guard url.pathExtension.lowercased() == connectionFileExtension else { return nil } - return .importConnections(url) + if url.pathExtension.lowercased() == connectionFileExtension { + return .importConnections(url) + } + guard MobileImportFormats.acceptedFileExtensions.contains(url.pathExtension.lowercased()) else { return nil } + return .importData(url) } guard url.scheme?.lowercased() == "tablepro", url.host(percentEncoded: false)?.lowercased() == "connect" else { return nil } diff --git a/apple/TableProMobile/TableProMobile/Helpers/SessionLocalStatements.swift b/apple/TableProMobile/TableProMobile/Helpers/SessionLocalStatements.swift index fb746b4caa..fb389b5b7a 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/SessionLocalStatements.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/SessionLocalStatements.swift @@ -1,5 +1,6 @@ import Foundation import TableProModels +import TableProQuery import TableProSQLGrammar // What a run leaves on its own session that a catalog connection beside it never sees. Read from every statement the @@ -57,6 +58,31 @@ nonisolated enum SessionLocalStatements { return SQLCatalogEffectClassifier.effect(of: text, grammar: grammar).runsUnseenStatements } + // A statement that leaves the session as a fresh connection would not have it: a write (the classifier counts SET, + // USE, CREATE TEMPORARY, CALL and DECLARE as writes), a transaction statement, or anything `effect` sees. + static func changesSession(_ statements: [String], databaseType: DatabaseType) -> Bool { + statements.contains { statement in + SQLWriteClassifier.isWriteQuery(statement, databaseType: databaseType) + || sessionKeywords.contains(ExportSnapshotStatements.leadingKeyword(of: statement)) + || effect(of: [statement], databaseType: databaseType) != Effect() + || assignsSessionState(statement, databaseType: databaseType) + } + } + + // A read can still set session state: MySQL's `SELECT @v := 7`, PostgreSQL's `SELECT set_config(...)`. + private static func assignsSessionState(_ statement: String, databaseType: DatabaseType) -> Bool { + let grammar = SQLLexicalReadings.resolve( + databaseTypeId: databaseType.rawValue, declared: nil, session: nil + ).execution + let code = SQLCodeProjection.code(of: statement, grammar: grammar, revealingExecutableComments: true) + return code.range(of: ":=", options: .literal) != nil + || code.range(of: "set_config", options: [.caseInsensitive, .literal]) != nil + } + + private static let sessionKeywords: Set = [ + "BEGIN", "START", "COMMIT", "END", "ROLLBACK", "SAVEPOINT", "RELEASE", "SET", "USE", "RESET" + ] + static func effect(of statements: [String], databaseType: DatabaseType) -> Effect { let grammar = SQLLexicalReadings.resolve( databaseTypeId: databaseType.rawValue, declared: nil, session: nil diff --git a/apple/TableProMobile/TableProMobile/Info.plist b/apple/TableProMobile/TableProMobile/Info.plist index 88d57f4c64..38f8f73b41 100644 --- a/apple/TableProMobile/TableProMobile/Info.plist +++ b/apple/TableProMobile/TableProMobile/Info.plist @@ -62,6 +62,7 @@ BGTaskSchedulerPermittedIdentifiers com.TablePro.sync + com.TablePro.TableProMobile.transfer.* UTExportedTypeDeclarations @@ -83,6 +84,28 @@ + UTImportedTypeDeclarations + + + UTTypeIdentifier + org.iso.sql + UTTypeDescription + SQL File + UTTypeConformsTo + + public.source-code + + UTTypeTagSpecification + + public.filename-extension + + sql + psql + pgsql + + + + LSSupportsOpeningDocumentsInPlace CFBundleDocumentTypes @@ -99,6 +122,66 @@ com.tablepro.connection-share + + CFBundleTypeName + CSV Document + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + LSItemContentTypes + + public.comma-separated-values-text + + + + CFBundleTypeName + TSV Document + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + LSItemContentTypes + + public.tab-separated-values-text + + + + CFBundleTypeName + JSON Document + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + LSItemContentTypes + + public.json + + + + CFBundleTypeName + Excel Workbook + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + LSItemContentTypes + + org.openxmlformats.spreadsheetml.sheet + + + + CFBundleTypeName + SQL File + CFBundleTypeRole + Viewer + LSHandlerRank + Alternate + LSItemContentTypes + + org.iso.sql + + diff --git a/apple/TableProMobile/TableProMobile/Localizable.xcstrings b/apple/TableProMobile/TableProMobile/Localizable.xcstrings index d4c8c46296..d54c0294ae 100644 --- a/apple/TableProMobile/TableProMobile/Localizable.xcstrings +++ b/apple/TableProMobile/TableProMobile/Localizable.xcstrings @@ -1,44 +1,6 @@ { "sourceLanguage" : "en", "strings" : { - "A document could not be read" : { - - }, - "A document field could not be read" : { - - }, - "A random identifier for this install" : { - - }, - "Help decide what to improve next. Nothing is sent unless you choose Share Usage Data." : { - - }, - "Off until you turn it on. Sent to PostHog with a random ID: app and iOS versions, device model, screen size, language, time zone, network type, database types, connection count, and first connection and query. PostHog derives an approximate location from your IP address. Never hostnames, credentials, queries, or rows." : { - - }, - "PostHog derives an approximate location from your IP address. Nothing sent contains a hostname, username, password, query, or any data from your databases. You can change this in Settings." : { - - }, - "Sent to PostHog:" : { - - }, - "The app and iOS versions, device model, screen size, language, time zone, and network type" : { - - }, - "The result has no stored documents" : { - - }, - "Export documents do not match their columns and rows" : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "The column stream cannot represent every document field" : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "TablePro does not have this row's stored _id, so it cannot tell which document it is." : { - - }, "" : { "localizations" : { "ko" : { @@ -67,217 +29,23 @@ } } }, - "%1$@: %2$@" : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "'%@' is not a Cloudflare D1 database name or UUID" : { - - }, - "%1$@ has type %2$@, which MongoDB does not have. Choose a type from the list." : { - - }, - "%1$@ is a capped collection, and a longer field name makes MongoDB delete its oldest documents. Choose a name no longer than %2$@." : { - - }, - "%1$@ is a collection of type %2$@. Its fields cannot be renamed or removed." : { - - }, - "%@ cannot be the primary key. MongoDB keys every document by _id." : { - - }, - "%@ changed after this save was prepared, so nothing was changed. Review the save and save again." : { - - }, - "%@ changed while its documents were being checked, so nothing was changed. Save again." : { - - }, - "%@ holds values that cannot be compared, so the row cannot be found again." : { - - }, - "Account ID" : { - - }, - "Account ID is required" : { - - }, - "API request failed" : { - - }, - "API Token" : { - - }, - "API Token is required" : { - - }, - "Authentication failed. Check your API token and Account ID." : { - - }, - "Can't Load Databases" : { - - }, - "Check the API token and account of this connection." : { - - }, - "Cloudflare D1 cannot bind a binary value through its API." : { - - }, - "Connect" : { - - }, - "connected" : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "Connecting" : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "Connection lost" : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "Copy the Account ID from the Cloudflare dashboard. The API token needs the D1 Edit permission." : { - - }, - "Couldn't connect" : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "Database '%@' not found" : { - - }, - "Database '%@' not found in account" : { - - }, - "Database name or UUID is required" : { - - }, - "Empty response from Cloudflare D1" : { - - }, - "Enter the Account ID and API Token to choose a database." : { - - }, - "Enter the Account ID and API Token to list the databases in this account." : { - - }, - "Failed to create database" : { - - }, - "Failed to fetch database details" : { - - }, - "Failed to list databases" : { - - }, - "Index %@ has a key part that is not one SQL expression." : { - - }, - "Invalid Account ID" : { - - }, - "Invalid Account ID or database identifier" : { - - }, - "Name or UUID" : { - - }, - "No Account" : { - - }, - "No D1 databases are available to this API token." : { - - }, - "No Databases" : { - - }, - "not connected" : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "Rate limited by Cloudflare. Please try again later." : { - - }, - "Rate limited by Cloudflare. Retry after %@ seconds." : { - - }, - "Reconnect" : { - - }, - "Reconnecting" : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "Reconnecting (%lld of %lld)" : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "Switching to %@…" : { - - }, - "The %@ setting holds a NUL character, which a PostgreSQL connection cannot carry." : { - - }, - "The condition of index %@ cannot be written as SQL." : { - - }, - "The default of column %@ cannot be written as SQL." : { - - }, - "The SSH host %@ contains characters TablePro does not allow in a host name." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "The SSH host is empty." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "The SSH host keys saved on this device could not be read, so this server's key could not be checked." : { - - }, - "The SSH port %lld is not between 1 and 65535." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "The SSH user %@ contains characters TablePro does not allow in a user name." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "The row has no loaded values to be found by." : { - - }, - "The row's loaded values are missing, so it cannot be found again." : { - - }, - "The statement's placeholders do not match the %d values given." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "Refused %@: this statement was not approved to drop or delete data." : { + "_id cannot be renamed, removed or used as a new name. MongoDB keys every document by it." : { }, - "Refused %@: this statement was not approved to run server-side code, $out or $merge." : { + "_id must be an objectId. MongoDB generates it for every new document." : { }, - "The type of column %@ cannot be written as SQL." : { + ".%@() cannot be set once the cursor has started." : { }, - "This request is read only, and the statement is not a read." : { + ".%@() takes a whole number." : { }, - "the SSH host, user or port of “%@” is not one TablePro allows" : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "When you first connect and first run a query" : { + "'%@' is not a Cloudflare D1 database name or UUID" : { }, "“%@” has an open transaction. Deleting the connection rolls it back and discards its uncommitted changes." : { - }, - "%@ holds a document or an array, and this text is not valid JSON." : { - }, "\"%@\" is a name the database keeps for itself. Choose another name." : { "extractionState" : "manual", @@ -440,86 +208,86 @@ "extractionState" : "manual", "generatesSymbol" : false }, - "\"%@\" will be permanently deleted." : { + "“%@” is not on any connection." : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "\"%@\" 항목이 영구적으로 삭제됩니다." + "value" : "“%@” 태그가 지정된 연결이 없습니다." } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "\"%@\" sẽ bị xoá vĩnh viễn." + "value" : "Không có kết nối nào gắn nhãn “%@”." } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "\"%@\" 将被永久删除。" + "value" : "没有连接使用标签“%@”。" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "「%@」將會永久刪除。" + "value" : "沒有連線使用標籤「%@」。" } } } }, - "“%@” is not on any connection." : { + "“%@” isn't available on this device." : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "“%@” 태그가 지정된 연결이 없습니다." + "value" : "이 기기에서 “%@”을(를) 사용할 수 없습니다." } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "Không có kết nối nào gắn nhãn “%@”." + "value" : "“%@” không có trên thiết bị này." } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "没有连接使用标签“%@”。" + "value" : "“%@”在此设备上不可用。" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "沒有連線使用標籤「%@」。" + "value" : "「%@」在此裝置上無法使用。" } } } }, - "“%@” isn't available on this device." : { + "\"%@\" will be permanently deleted." : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "이 기기에서 “%@”을(를) 사용할 수 없습니다." + "value" : "\"%@\" 항목이 영구적으로 삭제됩니다." } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "“%@” không có trên thiết bị này." + "value" : "\"%@\" sẽ bị xoá vĩnh viễn." } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "“%@”在此设备上不可用。" + "value" : "\"%@\" 将被永久删除。" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "「%@」在此裝置上無法使用。" + "value" : "「%@」將會永久刪除。" } } } @@ -589,9 +357,6 @@ "“%1$@” refers to “%2$@”, which is not in the file" : { "extractionState" : "manual", "generatesSymbol" : false - }, - "%1$@ on socket %2$@" : { - }, "%@" : { "localizations" : { @@ -689,6 +454,24 @@ } } } + }, + "%@ cannot be narrowed to some rows or columns, because this database has no SQL." : { + + }, + "%@ cannot be the primary key. MongoDB keys every document by _id." : { + + }, + "%@ cannot export to a file." : { + + }, + "%@ cannot take rows from a file." : { + + }, + "%@ changed after this save was prepared, so nothing was changed. Review the save and save again." : { + + }, + "%@ changed while its documents were being checked, so nothing was changed. Save again." : { + }, "%@ connections do not support adding rows from Shortcuts." : { "localizations" : { @@ -745,6 +528,9 @@ } } } + }, + "%@ files cannot be imported into this connection." : { + }, "%@ has no column named %@." : { "localizations" : { @@ -779,10 +565,43 @@ } } } + }, + "%@ holds a document or an array, and this text is not valid JSON." : { + + }, + "%@ holds an empty key, which the MongoDB client library will not insert. Set it to NULL, then set it on the saved document." : { + + }, + "%@ holds binary data. Edit it as bytes, or set it to NULL." : { + + }, + "%@ holds text in some rows and documents or arrays in others, so this value could be either. Change it with a query." : { + + }, + "%@ holds values that cannot be compared, so the row cannot be found again." : { + }, "%@ in the workbook expands to far more data than it holds, so it was not read." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "%@ is a binary column, and this value is not 0x followed by hex digits." : { + + }, + "%@ is a system collection. Its fields cannot be renamed or removed." : { + + }, + "%@ is a time series collection. MongoDB cannot rename or remove its fields." : { + + }, + "%@ is a view. Rename or remove the field in the collection it reads." : { + + }, + "%@ is an encrypted field. MongoDB cannot rename or remove it." : { + + }, + "%@ is changed twice in this save. Save one change to it at a time." : { + }, "%@ is missing its closing parenthesis." : { "extractionState" : "manual", @@ -813,10 +632,16 @@ } } } + }, + "%@ is not a CSV, JSON, Excel or SQL file." : { + }, "%@ is not on the server this connection uses, so TablePro did not send the credentials there." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "%@ is read-only, so nothing can be written to it." : { + }, "%@ is read-only, so rows cannot be added." : { "localizations" : { @@ -875,6 +700,15 @@ } } } + }, + "%@ rows" : { + + }, + "%@ rows written." : { + + }, + "%@ statements" : { + }, "%@ was given arguments it cannot use." : { "extractionState" : "manual", @@ -1142,6 +976,9 @@ } } } + }, + "%1$@ (%2$lld of %3$lld)" : { + }, "%1$@ (ORA-%2$ld)." : { "extractionState" : "manual", @@ -1176,9 +1013,36 @@ "%1$@ %2$@ was created with compilation errors:" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "%1$@ could not represent %2$@ and other characters. Each was written as \"?\"." : { + + }, + "%1$@ could not represent %2$@. Each was written as \"?\"." : { + + }, + "%1$@ has type %2$@, which MongoDB does not have. Choose a type from the list." : { + + }, + "%1$@ is a capped collection, and a longer field name makes MongoDB delete its oldest documents. Choose a name no longer than %2$@." : { + + }, + "%1$@ is a collection of type %2$@. Its fields cannot be renamed or removed." : { + + }, + "%1$@ of %2$@" : { + + }, + "%1$@ of %2$@ rows" : { + + }, + "%1$@ of %2$@ statements" : { + }, "%1$@ on Mac socket %2$@" : { + }, + "%1$@ on socket %2$@" : { + }, "%1$@ via %2$@" : { "localizations" : { @@ -1208,6 +1072,10 @@ } } }, + "%1$@: %2$@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%1$lld-%2$lld of %3$lld" : { "localizations" : { "ko" : { @@ -1236,28 +1104,10 @@ } } }, - "%@ holds an empty key, which the MongoDB client library will not insert. Set it to NULL, then set it on the saved document." : { + "%2$lld binary values are larger than the %1$@ this engine allows in one statement, so their INSERTs may not restore." : { }, - "%@ holds binary data. Edit it as bytes, or set it to NULL." : { - - }, - "%@ holds text in some rows and documents or arrays in others, so this value could be either. Change it with a query." : { - - }, - "%@ is a system collection. Its fields cannot be renamed or removed." : { - - }, - "%@ is a time series collection. MongoDB cannot rename or remove its fields." : { - - }, - "%@ is a view. Rename or remove the field in the collection it reads." : { - - }, - "%@ is an encrypted field. MongoDB cannot rename or remove it." : { - - }, - "%@ is changed twice in this save. Save one change to it at a time." : { + "%2$lld rows do not fit a %1$@ INSERT on their own, so their statements pass that size." : { }, "%d connections imported." : { @@ -1529,6 +1379,18 @@ } } } + }, + "^[%lld Row](inflect: true) Selected" : { + + }, + "^[%lld row](inflect: true) selected." : { + + }, + "^[Delete %lld Row](inflect: true)" : { + + }, + "^[Delete %lld Row](inflect: true)?" : { + }, "+%lld more columns" : { "localizations" : { @@ -1557,15 +1419,6 @@ } } } - }, - ".%@() cannot be set once the cursor has started." : { - - }, - ".%@() takes a whole number." : { - - }, - "A database name cannot contain %@." : { - }, "≤" : { "localizations" : { @@ -1622,6 +1475,12 @@ } } } + }, + "1 (no batching)" : { + + }, + "1 binary value is larger than the %1$@ this engine allows in one statement, so its INSERT may not restore." : { + }, "1 change could not be uploaded to iCloud. It stays on this device and goes again on the next sync." : { @@ -1745,7 +1604,7 @@ } } }, - "A client certificate needs its client key. On the connection's Network tab, choose the client key, or clear the client certificate." : { + "1 row does not fit a %1$@ INSERT on its own, so its statement passes that size." : { }, "1 saved query that belongs to these connections, with its folder and keyword." : { @@ -1770,10 +1629,7 @@ "extractionState" : "manual", "generatesSymbol" : false }, - "_id cannot be renamed, removed or used as a new name. MongoDB keys every document by it." : { - - }, - "_id must be an objectId. MongoDB generates it for every new document." : { + "A client certificate needs its client key. On the connection's Network tab, choose the client key, or clear the client certificate." : { }, "A collection name cannot contain %@." : { @@ -1781,12 +1637,18 @@ }, "A collection name cannot start with %@." : { + }, + "A compressed export is written as one file, so the split size was not applied." : { + }, "A cursor has no .%@() modifier." : { }, "A damaged history file from every connection was kept on this device. Clear All History deletes it." : { + }, + "A database name cannot contain %@." : { + }, "A database name must be shorter than %ld bytes." : { @@ -1818,6 +1680,12 @@ } } } + }, + "A document could not be read" : { + + }, + "A document field could not be read" : { + }, "A document is one JSON object, written between { and }." : { @@ -1930,6 +1798,9 @@ }, "A query that switches the database is running on this connection. Try again once it ends." : { + }, + "A random identifier for this install" : { + }, "A renamed column's foreign key is rewritten, so its MATCH and DEFERRABLE clauses are not carried over." : { "extractionState" : "manual", @@ -2021,6 +1892,9 @@ }, "A saved query with this name exists." : { + }, + "A script cannot hold binary COPY data, so no rows were sent. Dump the rows as text or CSV." : { + }, "A string cannot hold a line break, tab or other control character." : { @@ -2083,6 +1957,15 @@ } } } + }, + "A table named “%@” already exists." : { + + }, + "A table whose columns are all part of its key has nothing to update, so its rows are written as plain inserts." : { + + }, + "A table with no primary key has no conflict target, so its rows are written as plain inserts." : { + }, "A text index cannot be unique." : { @@ -2271,6 +2154,12 @@ "Account" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Account ID" : { + + }, + "Account ID is required" : { + }, "Acknowledgements" : { "localizations" : { @@ -2445,6 +2334,9 @@ } } } + }, + "Add a connection, then open the file again." : { + }, "Add a database connection to get started." : { "extractionState" : "stale", @@ -2735,9 +2627,6 @@ } } } - }, - "Add the CA certificate that signed the server's certificate, or set SSL Mode to Required." : { - }, "Add Rows to Table" : { "localizations" : { @@ -2794,6 +2683,9 @@ } } } + }, + "Add the CA certificate that signed the server's certificate, or set SSL Mode to Required." : { + }, "Add to Favorites" : { "localizations" : { @@ -3039,6 +2931,9 @@ } } } + }, + "Align columns" : { + }, "All" : { "extractionState" : "stale", @@ -3068,13 +2963,13 @@ } } } - }, - "All connections" : { - }, "All columns" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "All connections" : { + }, "All data in \"%@\" will be permanently deleted." : { "localizations" : { @@ -3131,6 +3026,9 @@ } } } + }, + "Allows NULL" : { + }, "Already exists" : { "localizations" : { @@ -3159,13 +3057,25 @@ } } } + }, + "already saved" : { + }, "Also include global saved queries" : { + }, + "Always" : { + }, "Americas" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "An earlier import made a table named %@ with other columns. Choose another name." : { + + }, + "An empty Markdown cell and a cell holding an empty string render the same, so a null needs its own text" : { + }, "An extension in the list has no file." : { "extractionState" : "manual", @@ -3196,6 +3106,12 @@ } } } + }, + "An extra value that should be imported as NULL, for example \\N." : { + + }, + "An import is running on this connection." : { + }, "an item has an empty reference" : { "extractionState" : "manual", @@ -3289,6 +3205,9 @@ } } } + }, + "Another connection closes this one and opens that one with the file." : { + }, "Another connection to %@ with other settings is still logging in. Try again once it finishes." : { "extractionState" : "manual", @@ -3297,10 +3216,14 @@ "Another row holds the same value in every column, so this row cannot be changed on its own. Change these rows with a query." : { }, - "Another window has unsaved changes on this connection. Save or discard them before switching database." : { + "Another selected row adds the same query." : { }, - "Another selected row adds the same query." : { + "Another task needs this connection alone. Try again when it ends." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Another window has unsaved changes on this connection. Save or discard them before switching database." : { }, "Answers the server's password prompt. TablePro asks you for anything else when you connect." : { @@ -3312,6 +3235,15 @@ }, "Any statement in the batch that ran stays applied." : { + }, + "API request failed" : { + + }, + "API Token" : { + + }, + "API Token is required" : { + }, "App" : { "extractionState" : "manual", @@ -3502,6 +3434,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Ask the SSH server's administrator to turn on a method TablePro supports." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Assigned by the database" : { "localizations" : { "ko" : { @@ -3530,10 +3466,6 @@ } } }, - "Ask the SSH server's administrator to turn on a method TablePro supports." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "Atom" : { "extractionState" : "manual", "generatesSymbol" : false @@ -3728,6 +3660,9 @@ }, "Authentication failed for user '%1$@' against database '%2$@'. Check the username, password, and Auth Database in the connection settings." : { + }, + "Authentication failed. Check your API token and Account ID." : { + }, "Auto" : { "localizations" : { @@ -3757,6 +3692,36 @@ } } }, + "Auto Generate" : { + "extractionState" : "manual", + "generatesSymbol" : false, + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "자동 생성" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Tự động tạo" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "自动生成" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "自動產生" + } + } + } + }, "Auto Inc" : { "extractionState" : "manual", "generatesSymbol" : false, @@ -3786,6 +3751,12 @@ } } } + }, + "AUTO_INCREMENT and DEFINER apply to MySQL and MariaDB." : { + + }, + "Auto-detect" : { + }, "auto-increment" : { "localizations" : { @@ -3843,6 +3814,41 @@ } } }, + "Automatic (%@)" : { + + }, + "Automatic does not work through a tunnel to %@. DuckDB would use HTTPS and check the certificate against 127.0.0.1, not against the server's name. Set SSL Mode to Verify Fingerprint or Required (skip verify), or to Disabled if the server serves plain HTTP." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Available in all connections" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "모든 연결에서 사용 가능" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Có trong mọi kết nối" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "在所有连接中可用" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "在所有連線中可用" + } + } + } + }, "AWS Access Key Needed" : { }, @@ -3977,6 +3983,12 @@ } } } + }, + "AWS Region Needed" : { + + }, + "AWS Sign-In Failed" : { + }, "AWS SSO returned an unexpected response." : { "extractionState" : "manual", @@ -4007,6 +4019,9 @@ } } } + }, + "AWS SSO Sign-In Failed" : { + }, "AWS SSO sign-in failed: %@" : { "extractionState" : "manual", @@ -4037,76 +4052,9 @@ } } } - }, - "Auto Generate" : { - "extractionState" : "manual", - "generatesSymbol" : false, - "localizations" : { - "ko" : { - "stringUnit" : { - "state" : "translated", - "value" : "자동 생성" - } - }, - "vi" : { - "stringUnit" : { - "state" : "translated", - "value" : "Tự động tạo" - } - }, - "zh-Hans" : { - "stringUnit" : { - "state" : "translated", - "value" : "自动生成" - } - }, - "zh-Hant" : { - "stringUnit" : { - "state" : "translated", - "value" : "自動產生" - } - } - } - }, - "AWS Region Needed" : { - - }, - "AWS Sign-In Failed" : { - - }, - "AWS SSO Sign-In Failed" : { - }, "AWS SSO Sign-In Required" : { - }, - "Available in all connections" : { - "localizations" : { - "ko" : { - "stringUnit" : { - "state" : "translated", - "value" : "모든 연결에서 사용 가능" - } - }, - "vi" : { - "stringUnit" : { - "state" : "translated", - "value" : "Có trong mọi kết nối" - } - }, - "zh-Hans" : { - "stringUnit" : { - "state" : "translated", - "value" : "在所有连接中可用" - } - }, - "zh-Hant" : { - "stringUnit" : { - "state" : "translated", - "value" : "在所有連線中可用" - } - } - } }, "Bag" : { "extractionState" : "manual", @@ -4213,6 +4161,9 @@ "Briefcase" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "BRIN indexes need PostgreSQL 9.5 or later." : { + }, "Broadcast" : { "extractionState" : "manual", @@ -4413,6 +4364,9 @@ } } } + }, + "Can't Load Databases" : { + }, "Can't Reach Google" : { @@ -4457,6 +4411,9 @@ }, "Cannot Export" : { + }, + "Cannot Import This File" : { + }, "Cannot reach iCloud. Changes sync when this device is back online." : { @@ -4520,6 +4477,9 @@ } } } + }, + "Cannot read this file" : { + }, "Cannot Save" : { "localizations" : { @@ -4559,6 +4519,9 @@ "Cart" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Cell %1$@ holds more than %2$@ of text, so the workbook was not read." : { + }, "Certificate" : { "localizations" : { @@ -4682,6 +4645,9 @@ }, "Change Not Saved" : { + }, + "Change Options" : { + }, "Changes from iCloud could not be saved on this device. They download again on the next sync." : { @@ -4816,9 +4782,6 @@ } } } - }, - "Check the connection's TOTP secret, and that this device sets its clock automatically." : { - }, "Check that the server accepts TLS connections." : { @@ -4828,6 +4791,12 @@ }, "Check that these AWS credentials are allowed to assume the role." : { + }, + "Check the API token and account of this connection." : { + + }, + "Check the connection's TOTP secret, and that this device sets its clock automatically." : { + }, "Check the date and time on this device." : { @@ -5145,7 +5114,12 @@ } } }, - "Choose another SSL mode in the connection's settings, or open it on the Mac." : {}, + "Choose a destination table to map fields." : { + + }, + "Choose another SSL mode in the connection's settings, or open it on the Mac." : { + + }, "Choose File" : { "localizations" : { "ko" : { @@ -5599,6 +5573,9 @@ "Cloud and Sun" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Cloudflare D1 cannot bind a binary value through its API." : { + }, "Code" : { "extractionState" : "manual", @@ -5699,6 +5676,15 @@ } } } + }, + "Column %@ has a type, default, generation expression or check that is not one SQL expression." : { + + }, + "Column %@ is a generated column, which needs PostgreSQL 12 or later." : { + + }, + "Column names must be unique." : { + }, "Columns" : { "localizations" : { @@ -5727,6 +5713,9 @@ } } } + }, + "Comma (,)" : { + }, "Command" : { "extractionState" : "manual", @@ -5762,10 +5751,22 @@ } } }, - "Commit" : { + "Commit" : { + + }, + "Commit or roll back the transaction open on this connection before importing into a new table." : { + + }, + "Commit the Transaction?" : { + + }, + "Compress the file using Gzip" : { }, - "Commit the Transaction?" : { + "Compressing…" : { + + }, + "Compressing… %1$@ of %2$@" : { }, "Computed by the database" : { @@ -5892,6 +5893,9 @@ "connect" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Connect" : { + }, "Connect Anyway" : { "localizations" : { @@ -5977,6 +5981,10 @@ } } }, + "connected" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Connected" : { "localizations" : { "ko" : { @@ -6005,6 +6013,10 @@ } } }, + "Connecting" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Connecting to %@..." : { "localizations" : { "ko" : { @@ -6259,6 +6271,10 @@ } } }, + "Connection lost" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Connection Not Saved" : { "localizations" : { "ko" : { @@ -6594,6 +6610,15 @@ } } } + }, + "Convert line break to space" : { + + }, + "Convert NULL to empty" : { + + }, + "Convert NULL to EMPTY" : { + }, "Copies" : { "extractionState" : "manual", @@ -6601,6 +6626,16 @@ }, "Copy" : { + }, + "COPY at line %@: %@ lines sent" : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "COPY at line %1$@: %2$@ lines sent" + } + } + } }, "Copy Column Name" : { "localizations" : { @@ -6667,6 +6702,9 @@ } } } + }, + "Copy Page" : { + }, "Copy Query" : { "localizations" : { @@ -6751,6 +6789,9 @@ } } } + }, + "Copy the Account ID from the Cloudflare dashboard. The API token needs the D1 Edit permission." : { + }, "Copy to Clipboard" : { "localizations" : { @@ -6871,6 +6912,15 @@ "Could not attach \"%@\": %@" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Could not build an INSERT for %@" : { + + }, + "Could not build an INSERT for the mapped columns" : { + + }, + "Could not build an UPDATE for the mapped columns" : { + }, "Could not build the %1$@ endpoint for the AWS region \"%2$@\"." : { "extractionState" : "manual", @@ -6901,6 +6951,12 @@ } } } + }, + "Could not build the CREATE TABLE statement" : { + + }, + "Could not build the query that matches rows" : { + }, "Could not complete Oracle native network encryption with this server. It may require an encryption or checksum algorithm the driver does not support." : { "extractionState" : "manual", @@ -7045,6 +7101,24 @@ } } } + }, + "Could not fetch column metadata, so identity and generated columns may not round-trip correctly." : { + + }, + "Could not fetch comments for: %@" : { + + }, + "Could not fetch foreign keys, so foreign key constraints may be missing from the export." : { + + }, + "Could not fetch indexes for: %@" : { + + }, + "Could not fetch table structure for: %@" : { + + }, + "Could not fetch the types and sequences needed by: %@. A CREATE TABLE that names one of them will not restore." : { + }, "Could not install the sample database: %@" : { "localizations" : { @@ -7223,10 +7297,16 @@ } } } + }, + "Could not read the settings %@ was created with, so it restores under the dump's SQL mode." : { + }, "Could not read this Mac's trusted certificates to verify the server. Choose the server's CA certificate under SSL, then connect again." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Could not read which rows are already in the table" : { + }, "Could not run the credential_process command for profile \"%@\": %@" : { "extractionState" : "manual", @@ -7285,6 +7365,15 @@ } } } + }, + "Could not switch to schema %1$@: %2$@" : { + + }, + "Could not write %1$@. %2$@" : { + + }, + "Could not write %1$@. %2$@ These parts were written and are still on disk: %3$@" : { + }, "Could not write this text as %@." : { "extractionState" : "manual", @@ -7318,10 +7407,17 @@ }, "Couldn't check %1$@ before changing its documents: %2$@" : { + }, + "Couldn't connect" : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Count all rows" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Counting rows…" : { + }, "Create" : { "localizations" : { @@ -7505,7 +7601,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, - "DNS Lookup Failed" : { + "CSV" : { + + }, + "Data" : { }, "Data & Analytics" : { @@ -7573,6 +7672,12 @@ } } } + }, + "Database '%@' not found" : { + + }, + "Database '%@' not found in account" : { + }, "Database & Storage" : { "extractionState" : "manual", @@ -7689,6 +7794,9 @@ } } } + }, + "Database name or UUID is required" : { + }, "Database names can't be blank, contain a slash, or start with a period." : { "localizations" : { @@ -7717,6 +7825,9 @@ } } } + }, + "Database Objects" : { + }, "Database or Schema" : { "localizations" : { @@ -7804,6 +7915,9 @@ } } } + }, + "Decimal" : { + }, "Decrypt" : { "localizations" : { @@ -8116,6 +8230,12 @@ } } } + }, + "Delete existing rows before import empties the table, so no row would match. Turn it off, or import as Insert." : { + + }, + "Delete existing rows first" : { + }, "Delete Folder" : { "localizations" : { @@ -8316,6 +8436,9 @@ } } } + }, + "Delimiter" : { + }, "Descending" : { "localizations" : { @@ -8344,10 +8467,16 @@ } } } + }, + "Deselect All" : { + }, "Desktop" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Destination" : { + }, "Development" : { "extractionState" : "manual", @@ -8397,6 +8526,9 @@ } } } + }, + "Disable foreign key checks" : { + }, "Disabled" : { "extractionState" : "manual", @@ -8515,6 +8647,12 @@ "Display" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "DNS Lookup Failed" : { + + }, + "Do not import" : { + }, "Document" : { "extractionState" : "manual", @@ -8579,6 +8717,9 @@ } } } + }, + "Double quote (\")" : { + }, "Downtrend" : { "extractionState" : "manual", @@ -8670,6 +8811,26 @@ } } }, + "DuckDB could not install its %@ extension. It downloads the extension from extensions.duckdb.org the first time it connects to a remote database." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "DuckDB could not load its %@ extension." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "DuckDB could not read the CA Certificate. It reads PEM files only." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "DuckDB could not verify the server's certificate. Either no CA it trusts signed it, or it was not issued for this host name. DuckDB checks certificates against /etc/ssl/cert.pem and the CA Certificate, and does not read the macOS keychain, so a CA added in Keychain Access is not trusted." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "DuckDB reported: %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Duplicate" : { "localizations" : { "ko" : { @@ -8697,6 +8858,12 @@ } } } + }, + "Each array element must be a JSON object" : { + + }, + "Each column can be mapped from only one field." : { + }, "Edit" : { "localizations" : { @@ -8782,10 +8949,10 @@ } } }, - "Edit Query" : { + "Edit JSON" : { }, - "Edit JSON" : { + "Edit Query" : { }, "Edit Tag" : { @@ -8855,9 +9022,6 @@ }, "Edit the connection and set a key for this jump host under Jump Hosts." : { - }, - "Editing a document needs MongoDB 4.0 or later." : { - }, "Edit…" : { "localizations" : { @@ -8886,10 +9050,16 @@ } } } + }, + "Editing a document needs MongoDB 4.0 or later." : { + }, "ElastiCache IAM authentication requires TLS. Enable SSL in the connection's SSL settings." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Empty response from Cloudflare D1" : { + }, "Empty String" : { "localizations" : { @@ -9165,6 +9335,12 @@ }, "Enter the access key in this connection's Authentication section, or turn on Sync Passwords on the Mac and save the connection there again." : { + }, + "Enter the Account ID and API Token to choose a database." : { + + }, + "Enter the Account ID and API Token to list the databases in this account." : { + }, "Enter the code %@ in the browser to finish signing in. It is on your clipboard." : { "localizations" : { @@ -9356,34 +9532,6 @@ } } }, - "global" : { - "localizations" : { - "ko" : { - "stringUnit" : { - "state" : "translated", - "value" : "전역" - } - }, - "vi" : { - "stringUnit" : { - "state" : "translated", - "value" : "toàn cục" - } - }, - "zh-Hans" : { - "stringUnit" : { - "state" : "translated", - "value" : "全局" - } - }, - "zh-Hant" : { - "stringUnit" : { - "state" : "translated", - "value" : "全域" - } - } - } - }, "Error" : { "localizations" : { "ko" : { @@ -9415,6 +9563,9 @@ "Europe and Africa" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Events" : { + }, "Every change you made to the sample database is replaced with the original data." : { "localizations" : { @@ -9443,9 +9594,30 @@ } } } + }, + "Every included column needs a name." : { + + }, + "Every row in “%@” is deleted before the file’s rows are inserted." : { + + }, + "Every row that matches the search, filter and sort, not only this page." : { + }, "Everything selected was already in your library." : { + }, + "Excel" : { + + }, + "Excel on Windows needs this to read UTF-8. Other readers show it as part of the first value." : { + + }, + "Exclude DEFINER clauses" : { + + }, + "Exclude the AUTO_INCREMENT counter" : { + }, "Execute" : { "localizations" : { @@ -9558,6 +9730,9 @@ } } } + }, + "Existing table" : { + }, "Expected '.method()' after the collection reference" : { "extractionState" : "manual", @@ -9740,6 +9915,40 @@ } } } + }, + "Export Database" : { + + }, + "Export Database…" : { + + }, + "Export documents do not match their columns and rows" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Export Failed" : { + + }, + "Export Results" : { + + }, + "Export Results…" : { + + }, + "Export Rows" : { + + }, + "Export Table" : { + + }, + "Export…" : { + + }, + "Exporting %@" : { + + }, + "Exports the %d rows shown." : { + }, "Expression" : { "extractionState" : "manual", @@ -9905,6 +10114,9 @@ }, "Failed to configure the connection timeout" : { + }, + "Failed to create database" : { + }, "Failed to decode the SSO portal response for profile \"%@\"." : { "extractionState" : "manual", @@ -10024,6 +10236,12 @@ } } } + }, + "Failed to fetch database details" : { + + }, + "Failed to list databases" : { + }, "Failed to load more rows" : { "extractionState" : "stale", @@ -10115,7 +10333,8 @@ } }, "Failed to read file: %@" : { - "extractionState" : "stale", + "extractionState" : "manual", + "generatesSymbol" : false, "localizations" : { "ko" : { "stringUnit" : { @@ -10283,6 +10502,9 @@ } } } + }, + "Failed: %@" : { + }, "Favorite" : { "localizations" : { @@ -10367,6 +10589,12 @@ } } } + }, + "File Name" : { + + }, + "File on the Mac" : { + }, "File path or paste JSON" : { "extractionState" : "manual", @@ -10397,9 +10625,6 @@ } } } - }, - "File on the Mac" : { - }, "Fill in at least one column. This database cannot insert a row made only of defaults." : { "localizations" : { @@ -10520,6 +10745,9 @@ } } } + }, + "Filtered and Sorted" : { + }, "Filters" : { "localizations" : { @@ -10576,6 +10804,9 @@ } } } + }, + "Finishing…" : { + }, "Firewall" : { "extractionState" : "manual", @@ -10583,6 +10814,12 @@ }, "First Collection" : { + }, + "First row holds column names" : { + + }, + "First row is a header" : { + }, "Fish" : { "extractionState" : "manual", @@ -10630,6 +10867,9 @@ }, "Folder Deleted" : { + }, + "For a file opened in a spreadsheet. Values that would run as formulas, such as those starting with =, +, - or @, get a leading single quote, which stays in the value for anything that imports the file." : { + }, "Foreign key enforcement could not be put back while a transaction is open." : { "extractionState" : "manual", @@ -10662,9 +10902,15 @@ } } } + }, + "Foreign keys between %@ reference each other, so no order puts every parent before its children. Those tables are written in the order they were listed, and the dump cannot be restored while foreign keys are enforced." : { + }, "Foreign Table" : { + }, + "Foreign Tables" : { + }, "Fork and Knife" : { "extractionState" : "manual", @@ -10823,6 +11069,34 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "global" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "전역" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "toàn cục" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "全局" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "全域" + } + } + } + }, "Global" : { "localizations" : { "ko" : { @@ -11356,6 +11630,9 @@ } } } + }, + "Help decide what to improve next. Nothing is sent unless you choose Share Usage Data." : { + }, "Help improve TablePro by sharing anonymous usage statistics (no personal data or queries)." : { "extractionState" : "stale", @@ -11598,18 +11875,16 @@ } } }, + "Host is required for a remote DuckDB connection" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Hosts" : { }, "House" : { "extractionState" : "manual", "generatesSymbol" : false - }, - "If the SSH server asks for a verification code, TablePro asks you for it when you connect." : { - - }, - "The SSH server asked “%@”, which TablePro can answer only while it is open." : { - }, "IAM Identity Center" : { @@ -11884,6 +12159,9 @@ } } } + }, + "If the SSH server asks for a verification code, TablePro asks you for it when you connect." : { + }, "If this server uses Redis 6 or later ACL users, fill in the Username field." : { "extractionState" : "manual", @@ -12000,6 +12278,12 @@ } } } + }, + "Import as" : { + + }, + "Import Blocked" : { + }, "Import Complete" : { "localizations" : { @@ -12056,9 +12340,6 @@ } } } - }, - "Import Failed" : { - }, "Import connections from your Mac" : { "extractionState" : "stale", @@ -12088,6 +12369,9 @@ } } } + }, + "Import Failed" : { + }, "Import File" : { "localizations" : { @@ -12119,6 +12403,15 @@ }, "Import Incomplete" : { + }, + "Import into" : { + + }, + "Import Into" : { + + }, + "Import is not supported for %@ connections." : { + }, "Import PKCS#12" : { "localizations" : { @@ -12147,6 +12440,33 @@ } } } + }, + "Import Rows" : { + + }, + "Import Rows?" : { + + }, + "Import Rows…" : { + + }, + "Import Running" : { + + }, + "Import Stopped" : { + + }, + "Import stopped when TablePro moved to the background." : { + + }, + "Import this field" : { + + }, + "Import…" : { + + }, + "Importing %@" : { + }, "in" : { "localizations" : { @@ -12260,7 +12580,16 @@ } } }, - "Include Saved Queries" : { + "Include byte order mark" : { + + }, + "Include column headers" : { + + }, + "INCLUDE columns need PostgreSQL 11 or later." : { + + }, + "Include NULL values" : { }, "Include passwords" : { @@ -12290,6 +12619,9 @@ } } } + }, + "Include Saved Queries" : { + }, "Incorrect passphrase" : { "extractionState" : "manual", @@ -12321,11 +12653,14 @@ } } }, - "Index %1$@ uses %2$@. Drop it first by running %3$@ in a query tab, then save again." : { + "Index %@ has a key part that is not one SQL expression." : { }, "Index %@ has a WHERE clause, an expression, included columns or a prefix length. MongoDB indexes take none of these." : { + }, + "Index %1$@ uses %2$@. Drop it first by running %3$@ in a query tab, then save again." : { + }, "Indexes" : { "localizations" : { @@ -12470,6 +12805,12 @@ } } } + }, + "Insert all rows in a single transaction. If any row fails, all changes are rolled back." : { + + }, + "Insert or Update" : { + }, "Insert Row" : { "localizations" : { @@ -12526,6 +12867,15 @@ } } } + }, + "Insert, skip existing" : { + + }, + "Insert, Skip Existing" : { + + }, + "Inserted: %@" : { + }, "Institution" : { "extractionState" : "manual", @@ -12590,6 +12940,12 @@ } } } + }, + "Invalid Account ID" : { + + }, + "Invalid Account ID or database identifier" : { + }, "Invalid escape sequence." : { @@ -12716,34 +13072,6 @@ } } }, - "keyword: %@" : { - "localizations" : { - "ko" : { - "stringUnit" : { - "state" : "translated", - "value" : "키워드: %@" - } - }, - "vi" : { - "stringUnit" : { - "state" : "translated", - "value" : "từ khoá: %@" - } - }, - "zh-Hans" : { - "stringUnit" : { - "state" : "translated", - "value" : "关键字:%@" - } - }, - "zh-Hant" : { - "stringUnit" : { - "state" : "translated", - "value" : "關鍵字:%@" - } - } - } - }, "It never contains a hostname, username, password, query, or any data from your databases. You can change this in Settings." : { "localizations" : { "ko" : { @@ -12776,10 +13104,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, - "Its connection is not imported." : { + "Its changes are saved to the database." : { }, - "Its changes are saved to the database." : { + "Its connection is not imported." : { }, "Its subgroups are deleted too. Their connections move to Ungrouped." : { @@ -12822,18 +13150,15 @@ }, "JSON Too Large" : { - }, - "Jump Hosts" : { - - }, - "Jump host %1$@ failed: %2$@" : { - }, "Jump host %@ has no key on this device." : { }, "Jump host %@ uses the SSH server's key, which is not on this device." : { + }, + "Jump host %1$@ failed: %2$@" : { + }, "Jump host key not found: %@" : { "extractionState" : "manual", @@ -12864,6 +13189,9 @@ } } } + }, + "Jump Hosts" : { + }, "Jump hosts are connected in order before reaching the SSH server above." : { @@ -12951,6 +13279,9 @@ } } } + }, + "Keep leading zeros in ZIP codes, phone numbers, and IDs by outputting all values as strings" : { + }, "Keep your connections, groups, and tags the same on your iPhone, iPad, and Mac." : { "extractionState" : "stale", @@ -13029,10 +13360,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, - "Key Matches Several Rows" : { + "Key for This Device" : { }, - "Key for This Device" : { + "Key Matches Several Rows" : { }, "Key Not Found" : { @@ -13105,6 +13436,15 @@ } } } + }, + "Keyword “%@”" : { + + }, + "Keyword “%@” is in use, so it imports without one." : { + + }, + "Keyword “%@” is not valid, so it imports without one." : { + }, "Keyword cannot contain spaces" : { "localizations" : { @@ -13134,18 +13474,43 @@ } } }, - "Keyword “%@”" : { - - }, - "Keyword “%@” is in use, so it imports without one." : { - - }, - "Keyword “%@” is not valid, so it imports without one." : { - + "keyword: %@" : { + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "키워드: %@" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "từ khoá: %@" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "关键字:%@" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "關鍵字:%@" + } + } + } }, "Laptop" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Largest INSERT written: %1$@ (%2$lld rows)." : { + + }, + "Largest INSERT written: %1$@ (1 row)." : { + }, "Last Connected" : { "localizations" : { @@ -13281,9 +13646,6 @@ "Light Bulb" : { "extractionState" : "manual", "generatesSymbol" : false - }, - "The SSH server asks for more to finish signing in." : { - }, "like" : { "localizations" : { @@ -13313,6 +13675,20 @@ } } }, + "Line %@ holds binary data that cannot be read back exactly from a %@ file. Export the dump with --hex-blob, then import it again." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Line %@ is not valid %@ text. Choose the file's encoding in the import options, then import again." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Line %1$@ is not valid %2$@ text. Choose the file's encoding in the import options, then import again." + } + } + } + }, "Line %1$lld, column %2$lld: %3$@" : { "extractionState" : "manual", "generatesSymbol" : false @@ -13345,6 +13721,10 @@ } } }, + "Line %lld runs COPY FROM STDIN, and this connection cannot load its rows. Run the file with psql, or dump it again with --inserts." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Line %lld, column %lld: %@" : { "localizations" : { "en" : { @@ -13354,6 +13734,9 @@ } } } + }, + "Line break" : { + }, "Line Chart" : { "extractionState" : "manual", @@ -13738,18 +14121,6 @@ } } } - }, - "The SSH server closed the connection during the handshake." : { - - }, - "The SSH server ended the connection during the handshake: %@" : { - - }, - "TOTP" : { - - }, - "The SSH server does not accept keyboard-interactive sign-in. It accepts %@." : { - }, "lowercase" : { "extractionState" : "manual", @@ -13912,10 +14283,19 @@ "Map" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Map a field to %@ to match rows on it." : { + + }, + "Map a field to a column outside the key to have something to update." : { + }, "Map Pin" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Markdown" : { + }, "Match All Tags" : { "localizations" : { @@ -13944,6 +14324,12 @@ } } } + }, + "Match by Name" : { + + }, + "Match by Position" : { + }, "Match Case" : { "localizations" : { @@ -13972,6 +14358,12 @@ } } } + }, + "Match Columns" : { + + }, + "Match on" : { + }, "Match what I type, otherwise lowercase" : { "extractionState" : "manual", @@ -13980,6 +14372,9 @@ "Match what I type, otherwise UPPERCASE" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Matched: %@" : { + }, "Matches “%@”" : { "localizations" : { @@ -14009,7 +14404,16 @@ } } }, - "Materialized View" : { + "Materialized View" : { + + }, + "Materialized Views" : { + + }, + "Max INSERT size" : { + + }, + "Maximum documents per batch. Large documents use smaller batches to fit the server limit." : { }, "Medical Case" : { @@ -14027,6 +14431,9 @@ "Mesh" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Method" : { + }, "Microsoft Entra ID needs you to sign in again." : { "extractionState" : "manual", @@ -14057,6 +14464,9 @@ } } } + }, + "Microsoft Entra ID Not Set Up" : { + }, "Microsoft Entra ID returned an unexpected response." : { "extractionState" : "manual", @@ -14087,6 +14497,9 @@ } } } + }, + "Microsoft Entra ID Sign-In Failed" : { + }, "Microsoft Entra ID sign-in failed: %@" : { "extractionState" : "manual", @@ -14117,15 +14530,6 @@ } } } - }, - "Method" : { - - }, - "Microsoft Entra ID Not Set Up" : { - - }, - "Microsoft Entra ID Sign-In Failed" : { - }, "Microsoft Entra ID Sign-In Required" : { "localizations" : { @@ -14232,10 +14636,10 @@ "MongoDB cannot make this change to a document." : { }, - "MongoDB cannot read this document." : { + "MongoDB cannot read this document: %@" : { }, - "MongoDB cannot read this document: %@" : { + "MongoDB cannot read this document." : { }, "MongoDB did not return a cursor for this query." : { @@ -14459,13 +14863,13 @@ }, "Moves the cursor to the error." : { + }, + "MQL" : { + }, "Music" : { "extractionState" : "manual", "generatesSymbol" : false - }, - "Password (optional)" : { - }, "my-cache" : { "extractionState" : "manual", @@ -14554,6 +14958,9 @@ } } } + }, + "Name or UUID" : { + }, "Network" : { "extractionState" : "manual", @@ -14850,6 +15257,9 @@ } } } + }, + "New table" : { + }, "New Tag" : { "localizations" : { @@ -14938,6 +15348,9 @@ } } } + }, + "No Account" : { + }, "No active database session." : { "localizations" : { @@ -14966,6 +15379,9 @@ } } } + }, + "No columns found in the file." : { + }, "No Connections" : { "localizations" : { @@ -15023,6 +15439,9 @@ } } } + }, + "No D1 databases are available to this API token." : { + }, "No Data" : { "localizations" : { @@ -15080,7 +15499,10 @@ } } }, - "No key on this device" : { + "No Databases" : { + + }, + "No fields found in the file." : { }, "No Foreign Keys" : { @@ -15195,6 +15617,9 @@ } } } + }, + "No key on this device" : { + }, "No License Information" : { "localizations" : { @@ -15223,6 +15648,9 @@ } } } + }, + "No limit" : { + }, "No Matching Connections" : { "extractionState" : "stale", @@ -15365,80 +15793,80 @@ } } }, - "No Saved Queries" : { - - }, - "No SSH username is set for %@." : { - - }, - "No Table Selected" : { + "No row found in %@ where %@ = '%@'" : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "선택된 테이블 없음" + "value" : "%@에서 %@ = '%@'인 행을 찾을 수 없음" } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "Chưa chọn bảng" + "value" : "Không tìm thấy dòng trong %@ với %@ = '%@'" } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "未选择表" + "value" : "在%@中未找到%@ = '%@'的行" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "未選擇資料表" + "value" : "在 %@ 中找不到 %@ = '%@' 的列" } } } }, - "No row found in %@ where %@ = '%@'" : { + "No row found in %@ where %@." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "No row found in %1$@ where %2$@." + } + } + } + }, + "No Rows on This Page" : { + + }, + "No Saved Queries" : { + + }, + "No SSH username is set for %@." : { + + }, + "No Table Selected" : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "%@에서 %@ = '%@'인 행을 찾을 수 없음" + "value" : "선택된 테이블 없음" } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "Không tìm thấy dòng trong %@ với %@ = '%@'" + "value" : "Chưa chọn bảng" } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "在%@中未找到%@ = '%@'的行" + "value" : "未选择表" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "在 %@ 中找不到 %@ = '%@' 的列" - } - } - } - }, - "No row found in %@ where %@." : { - "localizations" : { - "en" : { - "stringUnit" : { - "state" : "new", - "value" : "No row found in %1$@ where %2$@." + "value" : "未選擇資料表" } } } - }, - "No Rows on This Page" : { - }, "No Tables" : { "localizations" : { @@ -15495,6 +15923,15 @@ } } } + }, + "No target table configured for row import" : { + + }, + "No typed documents. Update the database driver, save changes and reload before exporting MQL." : { + + }, + "No values in this row matched the column mapping" : { + }, "No view named %@ in this database" : { @@ -15557,6 +15994,13 @@ } } }, + "Not available in skip-and-continue mode" : { + + }, + "not connected" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Not Connected" : { "localizations" : { "ko" : { @@ -15648,39 +16092,6 @@ } } }, - "Open this connection in TablePro and connect from there to answer the question." : { - - }, - "Prompt at Connect" : { - "extractionState" : "manual", - "generatesSymbol" : false, - "localizations" : { - "ko" : { - "stringUnit" : { - "state" : "translated", - "value" : "연결 시 입력 요청" - } - }, - "vi" : { - "stringUnit" : { - "state" : "translated", - "value" : "Nhắc khi kết nối" - } - }, - "zh-Hans" : { - "stringUnit" : { - "state" : "translated", - "value" : "连接时提示" - } - }, - "zh-Hant" : { - "stringUnit" : { - "state" : "translated", - "value" : "連線時提示" - } - } - } - }, "not equals" : { "localizations" : { "ko" : { @@ -15792,6 +16203,9 @@ } } } + }, + "Not readable with your permissions" : { + }, "Not set" : { "localizations" : { @@ -15820,9 +16234,6 @@ } } } - }, - "Nothing Imported" : { - }, "Not Signed In to iCloud" : { "extractionState" : "manual", @@ -15833,6 +16244,12 @@ }, "Not valid JSON." : { + }, + "Nothing here can be exported in this format." : { + + }, + "Nothing Imported" : { + }, "Nothing to Run" : { @@ -15864,6 +16281,12 @@ } } } + }, + "NULL shows as" : { + + }, + "NULL text" : { + }, "Nullable" : { "extractionState" : "manual", @@ -15926,6 +16349,9 @@ "Object" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Objects" : { + }, "Off" : { "extractionState" : "manual", @@ -15956,6 +16382,12 @@ } } } + }, + "Off names the columns column1, column2 and so on" : { + + }, + "Off until you turn it on. Sent to PostHog with a random ID: app and iOS versions, device model, screen size, language, time zone, network type, database types, connection count, and first connection and query. PostHog derives an approximate location from your IP address. Never hostnames, credentials, queries, or rows." : { + }, "Office Buildings" : { "extractionState" : "manual", @@ -16020,6 +16452,12 @@ } } } + }, + "On error" : { + + }, + "On existing rows" : { + }, "On the connection's Network tab, choose the server's CA certificate under Verify Identity or Verify CA. Required (skip verify) also connects, but does not check the certificate." : { "extractionState" : "manual", @@ -16171,6 +16609,12 @@ }, "Once this save updates the validator, it would reject the document with _id %@. Fix the document or the validator first." : { + }, + "One file" : { + + }, + "One JSON object" : { + }, "One report a day: hashed device ID, versions, language, database types, first-use dates. Never hostnames, credentials, queries, or data." : { "localizations" : { @@ -16199,6 +16643,9 @@ } } } + }, + "One row per line (NDJSON)" : { + }, "Only folders available in all connections can hold a global query." : { @@ -16315,86 +16762,86 @@ } } }, - "Open in Editor" : { + "Open Connection" : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "편집기에서 열기" + "value" : "연결 열기" } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "Mở trong trình soạn thảo" + "value" : "Mở kết nối" } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "在编辑器中打开" + "value" : "打开连接" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "在編輯器中開啟" + "value" : "開啟連線" } } } }, - "Open Connection" : { + "Open Database File" : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "연결 열기" + "value" : "데이터베이스 파일 열기" } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "Mở kết nối" + "value" : "Mở tệp cơ sở dữ liệu" } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "打开连接" + "value" : "打开数据库文件" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "開啟連線" + "value" : "開啟資料庫檔案" } } } }, - "Open Database File" : { + "Open in Editor" : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "데이터베이스 파일 열기" + "value" : "편집기에서 열기" } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "Mở tệp cơ sở dữ liệu" + "value" : "Mở trong trình soạn thảo" } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "打开数据库文件" + "value" : "在编辑器中打开" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "開啟資料庫檔案" + "value" : "在編輯器中開啟" } } } @@ -16470,6 +16917,9 @@ }, "Open Table" : { + }, + "Open this connection in TablePro and connect from there to answer the question." : { + }, "Open this connection in TablePro on the Mac, or edit it and choose a private key that is not on a security key." : { @@ -16682,6 +17132,9 @@ } } } + }, + "Options" : { + }, "OR" : { "localizations" : { @@ -16738,10 +17191,16 @@ } } } + }, + "Oracle returned no definition for view '%@'." : { + }, "Oracle returned no plan for this statement." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Oracle returns only the first 4000 bytes of this view's definition, so the full text cannot be read." : { + }, "OracleNIO has no TLS fallback. Preferred connects in plain TCP. Use Required for TCPS to Oracle Autonomous Database." : { @@ -16830,12 +17289,18 @@ } } }, - "Own key" : { + "Other work is running on this connection. Start the import again once it ends." : { + + }, + "Other work ran on this connection before you confirmed. Check the transaction and try again." : { }, "out-of-band check" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "output" : { + }, "Output" : { "extractionState" : "manual", @@ -16867,7 +17332,7 @@ } } }, - "output" : { + "Own key" : { }, "Package" : { @@ -17055,6 +17520,9 @@ } } } + }, + "Password (optional)" : { + }, "Passwords are excluded by default. To include them, set a passphrase. The file is encrypted with it." : { "localizations" : { @@ -17427,9 +17895,6 @@ "Pin" : { "extractionState" : "manual", "generatesSymbol" : false - }, - "Other work ran on this connection before you confirmed. Check the transaction and try again." : { - }, "PING failed: %@" : { "extractionState" : "manual", @@ -17488,6 +17953,9 @@ } } } + }, + "Pipe (|)" : { + }, "Play" : { "extractionState" : "manual", @@ -17520,10 +17988,29 @@ } } } + }, + "Port must be a number between 1 and 65535" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "PostgreSQL has no %@ index type." : { + + }, + "PostHog derives an approximate location from your IP address. Nothing sent contains a hostname, username, password, query, or any data from your databases. You can change this in Settings." : { + }, "Preferred" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Preparing…" : { + + }, + "Preserve all values as strings" : { + + }, + "Pretty print (formatted output)" : { + }, "Previous Page" : { "localizations" : { @@ -17609,6 +18096,9 @@ } } } + }, + "PRIMARY is the primary key's name, and no other index can take it. Rename the index, or change the key on the Columns tab." : { + }, "primary key" : { "localizations" : { @@ -17667,6 +18157,9 @@ } } } + }, + "Primary Key (%@)" : { + }, "Privacy" : { "localizations" : { @@ -17779,6 +18272,9 @@ } } } + }, + "Privileges" : { + }, "Processor" : { "extractionState" : "manual", @@ -18084,6 +18580,36 @@ } } }, + "Prompt at Connect" : { + "extractionState" : "manual", + "generatesSymbol" : false, + "localizations" : { + "ko" : { + "stringUnit" : { + "state" : "translated", + "value" : "연결 시 입력 요청" + } + }, + "vi" : { + "stringUnit" : { + "state" : "translated", + "value" : "Nhắc khi kết nối" + } + }, + "zh-Hans" : { + "stringUnit" : { + "state" : "translated", + "value" : "连接时提示" + } + }, + "zh-Hant" : { + "stringUnit" : { + "state" : "translated", + "value" : "連線時提示" + } + } + } + }, "protocol negotiation" : { "extractionState" : "manual", "generatesSymbol" : false @@ -18150,6 +18676,9 @@ } } } + }, + "Put field names in the first row" : { + }, "Puzzle Piece" : { "extractionState" : "manual", @@ -18336,6 +18865,12 @@ } } } + }, + "Query Result" : { + + }, + "Query Results" : { + }, "Query text and results will be cleared." : { "localizations" : { @@ -18454,6 +18989,15 @@ } } } + }, + "Quote" : { + + }, + "Quote character" : { + + }, + "Quote if needed" : { + }, "Raised Hand" : { "extractionState" : "manual", @@ -18464,6 +19008,12 @@ }, "Ran %d statements" : { + }, + "Rate limited by Cloudflare. Please try again later." : { + + }, + "Rate limited by Cloudflare. Retry after %@ seconds." : { + }, "RDS Endpoint" : { "extractionState" : "manual", @@ -18506,6 +19056,9 @@ }, "RDS Endpoint Needed" : { + }, + "Read every table at one snapshot" : { + }, "Read it with a command in Query." : { @@ -18597,6 +19150,15 @@ } } } + }, + "Reading objects…" : { + + }, + "Reading the file…" : { + + }, + "Ready to Share" : { + }, "Recent" : { "localizations" : { @@ -18625,6 +19187,9 @@ } } } + }, + "Reconnect" : { + }, "Reconnect and Roll Back?" : { @@ -18657,6 +19222,14 @@ } } }, + "Reconnecting" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Reconnecting (%lld of %lld)" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Reconnecting..." : { "localizations" : { "ko" : { @@ -18810,6 +19383,9 @@ } } } + }, + "Redis driver has no TLS fallback. Preferred and Required both force TLS. Use Required for Redis Cloud, Upstash, and AWS ElastiCache encrypted endpoints." : { + }, "Redis queued %@ instead of running it." : { "localizations" : { @@ -18838,9 +19414,6 @@ } } } - }, - "Redis driver has no TLS fallback. Preferred and Required both force TLS. Use Required for Redis Cloud, Upstash, and AWS ElastiCache encrypted endpoints." : { - }, "Refresh from iCloud" : { "extractionState" : "stale", @@ -18902,6 +19475,12 @@ }, "Refused %@: this statement ran as a read." : { + }, + "Refused %@: this statement was not approved to drop or delete data." : { + + }, + "Refused %@: this statement was not approved to run server-side code, $out or $merge." : { + }, "Refused a pipeline with $out or $merge: an export only reads." : { @@ -19142,6 +19721,9 @@ }, "Rename Folder" : { + }, + "Renaming a check constraint needs PostgreSQL 9.2 or later." : { + }, "Renaming would take the document with _id %@ past MongoDB's 16 MB limit. Choose shorter names, or shrink that document first." : { @@ -19173,6 +19755,9 @@ } } } + }, + "Replace existing" : { + }, "Replaces the editor's text with this query." : { @@ -19412,6 +19997,9 @@ } } } + }, + "Restored the mapping saved for %@." : { + }, "result" : { @@ -19599,6 +20187,9 @@ }, "Roll Back the Transaction?" : { + }, + "Roll back to remove the rows written so far, or commit to keep them." : { + }, "Roll it back to continue." : { @@ -19613,6 +20204,9 @@ }, "Route from Mac" : { + }, + "Routines" : { + }, "row" : { "localizations" : { @@ -19669,12 +20263,6 @@ } } } - }, - "Row Not Deleted" : { - - }, - "Row Not Found" : { - }, "Row %d of %d" : { "localizations" : { @@ -19703,6 +20291,12 @@ } } } + }, + "Row Not Deleted" : { + + }, + "Row Not Found" : { + }, "Row updated" : { "localizations" : { @@ -19787,6 +20381,12 @@ } } } + }, + "Rows per batch" : { + + }, + "Rows per INSERT" : { + }, "Rows per Page" : { "localizations" : { @@ -19815,6 +20415,9 @@ } } } + }, + "Rows read together hold more than %1$@ of text by row %2$@, so the workbook was not read." : { + }, "Run" : { "localizations" : { @@ -19905,6 +20508,15 @@ }, "Run Selection" : { + }, + "Run SQL File" : { + + }, + "Run SQL File?" : { + + }, + "Run SQL File…" : { + }, "Run Statement" : { @@ -19971,6 +20583,12 @@ }, "Running statement %1$d of %2$d" : { + }, + "Runs every statement in one transaction, so a failure rolls back what the import changed. A database that commits CREATE, ALTER and DROP as they run, such as MySQL, keeps those statements and everything before them." : { + + }, + "Runs the query again and exports every row it returns." : { + }, "Runs the statement at the cursor, or the selected text." : { @@ -20009,11 +20627,14 @@ "Safe Mode is set to Read-Only for this connection, and the database refused the statement." : { }, - "Same Key as SSH Server" : { + "Safe Mode on this connection changed. Start the import again." : { }, "Same key as SSH server" : { + }, + "Same Key as SSH Server" : { + }, "Sample Database" : { "localizations" : { @@ -20070,6 +20691,9 @@ } } } + }, + "Sanitize formula-like values" : { + }, "Save" : { "localizations" : { @@ -20129,9 +20753,6 @@ } } } - }, - "Saved" : { - }, "Save Query" : { @@ -20139,74 +20760,68 @@ "Save Query…" : { }, - "Saved Queries" : { + "Saved" : { + + }, + "Saved credentials for these connections will be permanently removed." : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "저장된 쿼리" + "value" : "이 연결들에 저장된 자격 증명이 영구적으로 제거됩니다." } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "Truy vấn đã lưu" + "value" : "Thông tin đăng nhập đã lưu của các kết nối này sẽ bị xóa vĩnh viễn." } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "已保存的查询" + "value" : "这些连接已保存的凭据将被永久移除。" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "已儲存的查詢" + "value" : "這些連線已儲存的憑證將被永久移除。" } } } }, - "Saved Queries Unavailable" : { - - }, - "Saved Query Deleted" : { - + "Saved passwords can only be exported to an encrypted file." : { + "extractionState" : "manual", + "generatesSymbol" : false }, - "Saved credentials for these connections will be permanently removed." : { + "Saved Queries" : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "이 연결들에 저장된 자격 증명이 영구적으로 제거됩니다." + "value" : "저장된 쿼리" } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "Thông tin đăng nhập đã lưu của các kết nối này sẽ bị xóa vĩnh viễn." + "value" : "Truy vấn đã lưu" } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "这些连接已保存的凭据将被永久移除。" + "value" : "已保存的查询" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "這些連線已儲存的憑證將被永久移除。" + "value" : "已儲存的查詢" } } } - }, - "Saved passwords can only be exported to an encrypted file." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "Saving reconnects “%@”, which has an open transaction. Reconnecting rolls it back and discards its uncommitted changes." : { - }, "Saved queries are limited to %@ so they fit in iCloud." : { @@ -20216,6 +20831,15 @@ }, "Saved queries can't be read right now, so they are not imported." : { + }, + "Saved Queries Unavailable" : { + + }, + "Saved Query Deleted" : { + + }, + "Saving reconnects “%@”, which has an open transaction. Reconnecting rolls it back and discards its uncommitted changes." : { + }, "Scatter Plot" : { "extractionState" : "manual", @@ -20553,6 +21177,12 @@ } } } + }, + "Select a table…" : { + + }, + "Select All" : { + }, "Select Private Key" : { "localizations" : { @@ -20581,6 +21211,18 @@ } } } + }, + "Select Rows" : { + + }, + "Selected Rows" : { + + }, + "Semicolon (;)" : { + + }, + "Sent to PostHog:" : { + }, "Sentinel resolved to another Sentinel rather than a data node. Check the primary group name in your sentinel configuration." : { "extractionState" : "manual", @@ -20591,6 +21233,9 @@ }, "Sequence" : { + }, + "Sequences" : { + }, "Server" : { "extractionState" : "manual", @@ -20621,6 +21266,12 @@ } } } + }, + "Server Has No TLS" : { + + }, + "Server Requires TLS" : { + }, "Server response: %@" : { "extractionState" : "manual", @@ -20651,12 +21302,6 @@ } } } - }, - "Server Has No TLS" : { - - }, - "Server Requires TLS" : { - }, "Servers & Network" : { "extractionState" : "manual", @@ -20796,9 +21441,6 @@ }, "Session Token (optional)" : { - }, - "Set SSL Mode to Preferred or Required." : { - }, "Set a passphrase to include passwords." : { "localizations" : { @@ -20827,6 +21469,27 @@ } } } + }, + "Set SSL Mode to Preferred or Required." : { + + }, + "Set SSL Mode to Verify Fingerprint, or to Required (skip verify) to encrypt without checking the certificate." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Set SSL Mode to Verify Identity and choose the CA that signed the certificate in CA Certificate, or set SSL Mode to Verify Fingerprint and enter the server's Certificate Fingerprint. Required (skip verify) also connects, but does not check the certificate." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Set SSL Mode to Verify Identity, or to Verify Fingerprint for a server with a self-signed certificate." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Set the application (client) ID for this connection on the Mac." : { + + }, + "Set the AWS Region in this connection's Authentication section." : { + }, "Set the Microsoft Entra ID application (client) ID for this connection." : { "extractionState" : "manual", @@ -20857,12 +21520,6 @@ } } } - }, - "Set the application (client) ID for this connection on the Mac." : { - - }, - "Set the AWS Region in this connection's Authentication section." : { - }, "Set to SSH Agent on the Mac. iPhone and iPad sign in with a private key." : { @@ -20983,6 +21640,9 @@ } } } + }, + "Share Again" : { + }, "Share anonymous usage data" : { "extractionState" : "stale", @@ -21368,8 +22028,12 @@ } } }, - "Signs in with the SSH profile's password from iCloud Keychain." : { + "Sign In with IAM Identity Center" : { + }, + "Sign in with your Google account to use this connection." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Sign it again with: codesign --force --sign - %@" : { "extractionState" : "manual", @@ -21401,13 +22065,6 @@ } } }, - "Sign In with IAM Identity Center" : { - - }, - "Sign in with your Google account to use this connection." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "Sign-In Denied" : { }, @@ -21417,6 +22074,15 @@ "Signing the service account token failed." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Signs in with the SSH profile's password from iCloud Keychain." : { + + }, + "Single quote (')" : { + + }, + "Size" : { + }, "Skip" : { "extractionState" : "stale", @@ -21446,6 +22112,12 @@ } } } + }, + "Skip and Continue" : { + + }, + "Skipped: %@" : { + }, "Snowflake" : { "extractionState" : "manual", @@ -21497,6 +22169,9 @@ }, "Some long values are shown in part to stay within memory." : { + }, + "Some of the file does not read as %@. Choose the encoding it was saved in." : { + }, "Sort" : { "localizations" : { @@ -21553,6 +22228,12 @@ } } } + }, + "Source" : { + + }, + "SP-GiST indexes need PostgreSQL 9.2 or later." : { + }, "Sparkles" : { "extractionState" : "manual", @@ -21561,6 +22242,9 @@ "Speedometer" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Split every" : { + }, "SQL" : { "localizations" : { @@ -21778,9 +22462,6 @@ } } } - }, - "SSH Sign-In" : { - }, "SSH private key not found: %@" : { "extractionState" : "manual", @@ -21839,6 +22520,9 @@ } } } + }, + "SSH Sign-In" : { + }, "SSH Sign-In Unavailable" : { @@ -22019,7 +22703,9 @@ "extractionState" : "manual", "generatesSymbol" : false }, - "SSL Mode Unavailable" : {}, + "SSL Mode Unavailable" : { + + }, "SSO portal returned HTTP %lld for profile \"%@\"." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -22049,6 +22735,9 @@ } } } + }, + "SSO Region" : { + }, "SSO role credentials for profile \"%@\" were already expired. Run 'aws sso login --profile %@' to refresh." : { "extractionState" : "manual", @@ -22229,9 +22918,6 @@ } } } - }, - "SSO Region" : { - }, "Stack" : { "extractionState" : "manual", @@ -22278,6 +22964,15 @@ }, "Statement %1$d/%2$d failed: %3$@" : { + }, + "Statements" : { + + }, + "Statements run: %@" : { + + }, + "Statements skipped: %@" : { + }, "Stats" : { "localizations" : { @@ -22392,6 +23087,18 @@ } } } + }, + "Stop and Commit" : { + + }, + "Stop and Rollback" : { + + }, + "Stop Import" : { + + }, + "Stop Import?" : { + }, "Stop Sign" : { "extractionState" : "manual", @@ -22520,6 +23227,12 @@ } } } + }, + "Structure" : { + + }, + "Suggested Types" : { + }, "Sum" : { "extractionState" : "manual", @@ -22565,6 +23278,9 @@ }, "Switched to %@" : { + }, + "Switching to %@…" : { + }, "Switching..." : { "extractionState" : "stale", @@ -22902,90 +23618,6 @@ }, "System Table" : { - }, - "The shell would reorder or drop a field named %@. Choose another name." : { - - }, - "The shell would reorder or drop a key inside %@: it lists number-like keys first and drops __proto__. Change this field with a query." : { - - }, - "The top-level field “%@” holds Timestamp(0, 0), which MongoDB replaces with the current time on every save." : { - - }, - "The validator already declares %@. Remove that declaration first, then save again." : { - - }, - "The validator could not be read, so the change was not checked against it." : { - - }, - "The validator has a key named %@ that the shell would reorder or drop. Change the validator from a query tab instead." : { - - }, - "The validator would reject the document with _id %1$@ once %2$@ is removed. Fix the document or the validator first." : { - - }, - "The validator would reject the document with _id %1$@ once %2$@ is renamed to %3$@. Fix the document or the validator first." : { - - }, - "The value in %@ is shortened for display, so saving it would store only the part shown. Change this field with a query." : { - - }, - "The write was applied, but the servers did not confirm it as the write concern asks: %@" : { - - }, - "This cursor has already been closed." : { - - }, - "This device cannot check a SQL Server certificate, so it does not connect in this mode. Choose Required to connect without the check." : { - - }, - "This document has no _id, so it cannot be edited here." : { - - }, - "This document holds a value text cannot write back exactly, such as a subdocument like {\"$numberInt\": \"5\"}." : { - - }, - "This document holds a value that cannot be edited as text." : { - - }, - "This document holds JavaScript code with a scope. MongoDB compares a scope by value, so a save could not tell whether someone else changed it." : { - - }, - "This document is too large to edit here. Saving it would send more than 16 MB to the server." : { - - }, - "This filter is not a document MongoDB can read: %@" : { - - }, - "This is a time-series collection, which cannot replace a single document, so its documents cannot be edited here." : { - - }, - "This is not a document MongoDB can read: %@" : { - - }, - "This is not valid JSON at line %1$d, column %2$d." : { - - }, - "This pipeline is not something MongoDB can read: %@" : { - - }, - "This query calls a function that can change data, which Read-Only does not run. Switch Safe Mode to %@ to run it." : { - - }, - "This row has no _id, so TablePro cannot tell which document to change." : { - - }, - "This row's _id is binary data whose subtype is not known, so TablePro cannot tell which document it names." : { - - }, - "This server runs Atlas Search but cannot list its search indexes, so one that uses the field cannot be ruled out. Check the collection's search indexes in Atlas, then change the field from a query tab." : { - - }, - "TOTP Secret" : { - - }, - "TLS Handshake Failed" : { - }, "Tab" : { "extractionState" : "stale", @@ -23150,10 +23782,6 @@ "extractionState" : "manual", "generatesSymbol" : false }, - "TablePro signs RSA keys with rsa-sha2-256 or rsa-sha2-512, not with ssh-rsa, which uses SHA-1." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "TablePro can't open “%@”." : { "localizations" : { "ko" : { @@ -23181,6 +23809,9 @@ } } } + }, + "TablePro cannot rebuild the definition of %@ from what the server reported." : { + }, "TablePro cannot receive the output of COPY TO STDOUT, so it was discarded. Run a SELECT instead." : { "localizations" : { @@ -23337,6 +23968,10 @@ } } }, + "TablePro could not read the CA Certificate at %@: %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "TablePro could not read your connection library, so nothing was imported." : { "localizations" : { "ko" : { @@ -23395,6 +24030,10 @@ }, "TablePro could not read your saved queries. Nothing on this device has been changed." : { + }, + "TablePro could not write the CA bundle DuckDB reads: %@" : { + "extractionState" : "manual", + "generatesSymbol" : false }, "TablePro could not write your saved queries, so this change was not saved." : { @@ -23402,6 +24041,9 @@ "TablePro Data Was Removed from iCloud" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "TablePro does not have this row's stored _id, so it cannot tell which document it is." : { + }, "TablePro doesn't support “%@” connections" : { "extractionState" : "manual", @@ -23523,18 +24165,24 @@ } } }, - "TablePro is locked. Open TablePro and unlock it, then try again." : {}, - "TablePro on iPhone and iPad can't connect to %1$@ with the SSL mode %2$@." : { - "extractionState" : "manual", - "generatesSymbol" : false + "TablePro is locked. Open TablePro and unlock it, then try again." : { + }, "TablePro is reading or writing rows on this connection with settings of its own." : { + }, + "TablePro on iPhone and iPad can't connect to %1$@ with the SSL mode %2$@." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "TablePro on iPhone and iPad can't open %@ connections." : { "extractionState" : "manual", "generatesSymbol" : false }, + "TablePro signs RSA keys with rsa-sha2-256 or rsa-sha2-512, not with ssh-rsa, which uses SHA-1." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "TablePro supports these ciphers: %@." : { "extractionState" : "manual", "generatesSymbol" : false @@ -24063,10 +24711,16 @@ } } } + }, + "The _id cannot change. Put the original _id back, or insert a new document instead." : { + }, "The %@ plugin is not installed. TablePro offers to install it on connect." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "The %@ setting holds a NUL character, which a PostgreSQL connection cannot carry." : { + }, "The %d batches before it stay applied." : { @@ -24074,7 +24728,7 @@ "The %d statements before it stay applied." : { }, - "The _id cannot change. Put the original _id back, or insert a new document instead." : { + "The action of foreign key %@ cannot be written as SQL." : { }, "The actual plan reads V$SESSION, V$PARAMETER and V$SQL_PLAN_STATISTICS_ALL, and this account cannot. Ask for SELECT on V_$SESSION, V_$PARAMETER and V_$SQL_PLAN_STATISTICS_ALL, or for SELECT_CATALOG_ROLE. The estimated plan needs no grant." : { @@ -24108,6 +24762,9 @@ } } } + }, + "The app and iOS versions, device model, screen size, language, time zone, and network type" : { + }, "The attached database \"%@\" has no file." : { "extractionState" : "manual", @@ -24188,6 +24845,10 @@ }, "The CA certificate at %@ could not be read as a PEM or DER certificate." : { + }, + "The CA Certificate at %@ is not a PEM certificate. DuckDB reads PEM files only. Convert a DER file with: openssl x509 -inform der -in ca.der -out ca.pem" : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The CA certificate for this connection is missing. Import it again in the connection's SSL settings." : { "localizations" : { @@ -24220,12 +24881,6 @@ "The CA certificate path cannot be passed to FreeTDS. Use a shorter path, without “;”, “#” or repeated spaces." : { "extractionState" : "manual", "generatesSymbol" : false - }, - "The client certificate and key could not be written to a temporary file." : { - - }, - "The SSH server did not accept the verification code." : { - }, "The certificate could not be saved to this device's keychain." : { "localizations" : { @@ -24282,6 +24937,13 @@ } } } + }, + "The Certificate Fingerprint is not a SHA-256 fingerprint. Enter the 64 hex digits that quack_server_list() shows on the server, with or without colons." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The client certificate and key could not be written to a temporary file." : { + }, "The client certificate at %@ could not be read as a PEM or DER certificate." : { "extractionState" : "manual", @@ -24457,9 +25119,23 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The column stream cannot represent every document field" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The compressed data expands to far more than it holds, so it was not read." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The computed expression of column %@ is not one SQL expression, so it cannot be written." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "The condition of check constraint %@ cannot be written as SQL." : { + + }, + "The condition of index %@ cannot be written as SQL." : { + }, "The connection attempt was cancelled." : { "localizations" : { @@ -24494,6 +25170,9 @@ }, "The connection moved to another database or schema. Choose the table again." : { + }, + "The connection moved to another database. Open the SQL file again from that database." : { + }, "The connection to %@ was lost before the server answered, so its open transaction may have been committed or rolled back. It has been reconnected; check which of its changes were saved before running them again." : { @@ -24505,18 +25184,6 @@ "extractionState" : "manual", "generatesSymbol" : false }, - "The credentials file can't be read." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "The credentials file has no %@ field." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "The credentials point at %@, which is not a Google endpoint." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "The connections could not be saved." : { "localizations" : { "ko" : { @@ -24695,6 +25362,18 @@ } } }, + "The credentials file can't be read." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The credentials file has no %@ field." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The credentials point at %@, which is not a Google endpoint." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The CSV data has no header row." : { "localizations" : { "ko" : { @@ -24725,6 +25404,10 @@ }, "The cursor is not in a statement." : { + }, + "The data could not be compressed." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The data could not be read: %@" : { "localizations" : { @@ -24782,80 +25465,96 @@ } } }, - "The database limited the result. Showing %d rows." : { + "The database “%@” has an open transaction. Switching rolls it back and discards its uncommitted changes." : { + + }, + "The database and collection names together must be %ld bytes or fewer." : { + + }, + "The database assigns this column, so it is never written." : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "데이터베이스에서 결과를 제한했습니다. %d개 행을 표시합니다." + "value" : "데이터베이스가 이 열을 할당하므로 이 열은 기록되지 않습니다." } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "Cơ sở dữ liệu đã giới hạn kết quả. Đang hiển thị %d dòng." + "value" : "Cơ sở dữ liệu gán giá trị cho cột này, nên cột này không bao giờ được ghi." } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "数据库限制了结果。正在显示 %d 行。" + "value" : "此列由数据库分配,因此永远不会写入。" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "資料庫限制了結果。正在顯示 %d 列。" + "value" : "此欄位由資料庫指派,因此永遠不會寫入。" } } } }, - "The database name is longer than the 128 bytes FreeTDS takes." : { - - }, - "The default of column %@ is not one SQL expression, so it cannot be written." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "The database “%@” has an open transaction. Switching rolls it back and discards its uncommitted changes." : { - - }, - "The database and collection names together must be %ld bytes or fewer." : { + "The database had already ended the import's transaction, so none of its rows were kept." : { }, - "The database assigns this column, so it is never written." : { + "The database limited the result. Showing %d rows." : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "데이터베이스가 이 열을 할당하므로 이 열은 기록되지 않습니다." + "value" : "데이터베이스에서 결과를 제한했습니다. %d개 행을 표시합니다." } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "Cơ sở dữ liệu gán giá trị cho cột này, nên cột này không bao giờ được ghi." + "value" : "Cơ sở dữ liệu đã giới hạn kết quả. Đang hiển thị %d dòng." } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "此列由数据库分配,因此永远不会写入。" + "value" : "数据库限制了结果。正在显示 %d 行。" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "此欄位由資料庫指派,因此永遠不會寫入。" + "value" : "資料庫限制了結果。正在顯示 %d 列。" } } } }, + "The database name is longer than the 128 bytes FreeTDS takes." : { + + }, + "The decompressed file needs more free disk space than this Mac has." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The default is SQL, written after DEFAULT as it is: quote a text value." : { + + }, + "The default of column %@ cannot be written as SQL." : { + + }, + "The default of column %@ is not one SQL expression, so it cannot be written." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The document changed on the server after it was opened, so nothing was saved. Copy your text, then open it again." : { }, "The document is nested more than %d levels deep." : { + }, + "The dump was written as %1$lld numbered parts, %2$@ through %3$@. Restore them in order." : { + }, "The encrypted file is corrupt or incomplete" : { "extractionState" : "manual", @@ -24886,6 +25585,12 @@ } } } + }, + "The export was cancelled. No file was kept." : { + + }, + "The expression of generated column %@ cannot be written as SQL." : { + }, "The extension failed to start: %@" : { "extractionState" : "manual", @@ -24917,74 +25622,83 @@ } } }, - "The folder \"%@\" will be deleted. Items inside will be moved to the parent level." : { + "The failed transaction was rolled back. None of its changes were kept." : { + + }, + "The field “%@” appears more than once." : { + + }, + "The file does not read as %1$@, so it is read as %2$@." : { + + }, + "The file ended the import's transaction early, so the rows written before that stay." : { + + }, + "The file has no function named %@." : { + "extractionState" : "manual", + "generatesSymbol" : false, "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "\"%@\" 폴더가 삭제됩니다. 폴더 안의 항목은 상위 단계로 이동됩니다." + "value" : "파일에 %@(이)라는 함수가 없습니다." } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "Thư mục \"%@\" sẽ bị xoá. Các mục bên trong sẽ được chuyển lên cấp cha." + "value" : "Tệp không có hàm nào tên %@." } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "文件夹 \"%@\" 将被删除。其中的项目将移至上一级。" + "value" : "该文件中没有名为 %@ 的函数。" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "資料夾「%@」將被刪除。其中的項目將移至上一層。" + "value" : "此檔案中沒有名為 %@ 的函式。" } } } }, - "The failed transaction was rolled back. None of its changes were kept." : { - + "The filter of index %@ is not one SQL expression, so it cannot be written." : { + "extractionState" : "manual", + "generatesSymbol" : false }, - "The field “%@” appears more than once." : { + "The first sheet has no header row to map columns from." : { }, - "The file has no function named %@." : { - "extractionState" : "manual", - "generatesSymbol" : false, + "The folder \"%@\" will be deleted. Items inside will be moved to the parent level." : { "localizations" : { "ko" : { "stringUnit" : { "state" : "translated", - "value" : "파일에 %@(이)라는 함수가 없습니다." + "value" : "\"%@\" 폴더가 삭제됩니다. 폴더 안의 항목은 상위 단계로 이동됩니다." } }, "vi" : { "stringUnit" : { "state" : "translated", - "value" : "Tệp không có hàm nào tên %@." + "value" : "Thư mục \"%@\" sẽ bị xoá. Các mục bên trong sẽ được chuyển lên cấp cha." } }, "zh-Hans" : { "stringUnit" : { "state" : "translated", - "value" : "该文件中没有名为 %@ 的函数。" + "value" : "文件夹 \"%@\" 将被删除。其中的项目将移至上一级。" } }, "zh-Hant" : { "stringUnit" : { "state" : "translated", - "value" : "此檔案中沒有名為 %@ 的函式。" + "value" : "資料夾「%@」將被刪除。其中的項目將移至上一層。" } } } }, - "The filter of index %@ is not one SQL expression, so it cannot be written." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "The FreeTDS configuration could not be written: %@" : { "extractionState" : "manual", "generatesSymbol" : false @@ -24994,6 +25708,14 @@ }, "The full values on this page need more memory than this device has free. Choose a smaller page size and try again." : { + }, + "The gzip data ends before its last block." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The gzip data is corrupt." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The host cannot be passed to FreeTDS. Use a host name or an IP address, without spaces, brackets, “=”, “;” or “#”." : { "extractionState" : "manual", @@ -25040,13 +25762,13 @@ "extractionState" : "manual", "generatesSymbol" : false }, - "The insert stopped at a document larger than MongoDB accepts." : { + "The import was cancelled. No rows were changed." : { }, - "The jump host keys on this device could not be read. Close the form and open it again to change them." : { + "The import's transaction was rolled back, so none of its rows were kept." : { }, - "The jump host keys on this device could not be read: %@" : { + "The insert stopped at a document larger than MongoDB accepts." : { }, "The JSON array must contain only objects." : { @@ -25079,6 +25801,21 @@ }, "The JSON ends before it is complete." : { + }, + "The JSON file ends before its last value is complete." : { + + }, + "The JSON file has text that is not valid in its encoding." : { + + }, + "The JSON file is not valid at byte %@." : { + + }, + "The jump host keys on this device could not be read: %@" : { + + }, + "The jump host keys on this device could not be read. Close the form and open it again to change them." : { + }, "The Kerberos service principal name for this server cannot be passed to FreeTDS. Connect through a shorter host name." : { "extractionState" : "manual", @@ -25086,6 +25823,9 @@ }, "The key %@ no longer exists." : { + }, + "The key %1$@ matched %2$@ rows, so the row was not written." : { + }, "The license text is missing from this build." : { "localizations" : { @@ -25616,15 +26356,6 @@ } } } - }, - "The script did not finish within %d seconds and was abandoned." : { - - }, - "The script was cancelled." : { - - }, - "The SSH server does not sign in with a private key, so this jump host needs its own key." : { - }, "The password is longer than the 128 bytes FreeTDS takes." : { @@ -25633,10 +26364,6 @@ "extractionState" : "manual", "generatesSymbol" : false }, - "The plan setting could not be turned off again, so the connection was closed. Reconnect to go on." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, "The path \"%@\" is too long." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -25697,6 +26424,10 @@ } } }, + "The plan setting could not be turned off again, so the connection was closed. Reconnect to go on." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The previous session has not finished closing." : { "localizations" : { "ko" : { @@ -25724,13 +26455,13 @@ } } } - }, - "The profile is read from files on the Mac. A key added here signs in on this device only. The Mac's connection does not change." : { - }, "The previous session is still closing" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "The profile is read from files on the Mac. A key added here signs in on this device only. The Mac's connection does not change." : { + }, "The query did not finish within the configured timeout, so the connection was reset. Run the query again." : { "extractionState" : "manual", @@ -25831,6 +26562,10 @@ }, "The query timed out after %d seconds. Sorting or filtering on a field with no index makes MongoDB read the whole collection, even when you only ask for one page. Add an index for that field, or raise this connection's query timeout in Options." : { + }, + "The RDS Endpoint \"%1$@\" is not the database at \"%2$@\", where the connection goes. Clear the RDS Endpoint unless you connect through a port forward." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "the reference “%@” is inside itself" : { "extractionState" : "manual", @@ -25875,9 +26610,42 @@ } } } + }, + "The result has no stored documents" : { + + }, + "The row filter on %@ was not a single expression, so every row was exported." : { + + }, + "The row has %1$lld fields where %2$lld are expected. A field that holds the delimiter must be quoted." : { + + }, + "The row has no loaded values to be found by." : { + }, "The row is still in the table. A trigger, rule or row security policy may have stopped the delete." : { + }, + "The row's loaded values are missing, so it cannot be found again." : { + + }, + "The row's primary key is not loaded, so the row cannot be found." : { + + }, + "The rows were read on a connection of the export's own, so changes in the transaction open on this connection are not in the file." : { + + }, + "The rows written before the import ended stay in the table." : { + + }, + "The rows written before the import ended were committed." : { + + }, + "The rows written so far stay in the table." : { + + }, + "The rows written stay in the transaction that is open on this connection. Commit or roll it back." : { + }, "The sample database is missing from the app." : { "localizations" : { @@ -25991,17 +26759,6 @@ } } } - }, - "The server answered with something other than HTTP." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "The server asked for the password in plain text, which TablePro sends only over TLS. Set SSL Mode to Required, or turn TLS on for the server." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "The server did not encrypt the connection, so TablePro closed it. An IAM sign-in needs TLS." : { - }, "The saved queries could not be read. Nothing was exported." : { @@ -26033,6 +26790,12 @@ } } } + }, + "The script did not finish within %d seconds and was abandoned." : { + + }, + "The script was cancelled." : { + }, "The server and TablePro could not agree on a TLS cipher or protocol version." : { "extractionState" : "manual", @@ -26067,6 +26830,21 @@ "The server answered HTTP %lld with a redirect to %@, which is not the server this connection uses. TablePro did not follow it." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "The server answered with something other than HTTP." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The server asked for the password in plain text, which TablePro sends only over TLS. Set SSL Mode to Required, or turn TLS on for the server." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The server did not answer the TLS handshake. If it serves plain HTTP, set SSL Mode to Disabled." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The server did not encrypt the connection, so TablePro closed it. An IAM sign-in needs TLS." : { + }, "The server does not accept encrypted connections but TablePro is configured to require TLS." : { "extractionState" : "manual", @@ -26097,6 +26875,9 @@ } } } + }, + "The server fills in %@, so it cannot be given a value." : { + }, "The server is not responding. Check the host and port." : { "localizations" : { @@ -26126,6 +26907,22 @@ } } }, + "The server offers only ciphers TablePro no longer supports: %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The server offers only host key types TablePro no longer supports: %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The server offers only key exchange methods TablePro no longer supports: %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The server offers only MAC algorithms TablePro no longer supports: %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The server requires a client certificate for TLS mutual authentication." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -26186,21 +26983,8 @@ } } }, - "The server offers only ciphers TablePro no longer supports: %@." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "The server offers only host key types TablePro no longer supports: %@." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "The server offers only key exchange methods TablePro no longer supports: %@." : { - "extractionState" : "manual", - "generatesSymbol" : false - }, - "The server offers only MAC algorithms TablePro no longer supports: %@." : { - "extractionState" : "manual", - "generatesSymbol" : false + "the server returned no definition" : { + }, "The server runs %@." : { "extractionState" : "manual", @@ -26236,6 +27020,14 @@ } } }, + "The server uses HTTPS, and this connection sent plain HTTP." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The server's certificate does not match the Certificate Fingerprint. If the server has a new certificate, copy its fingerprint again from quack_server_list() on the server." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The server's host key has changed." : { "localizations" : { "ko" : { @@ -26291,9 +27083,6 @@ } } } - }, - "The session that held this transaction has ended." : { - }, "The server's SSH host key has changed. Open this connection in TablePro to check it." : { "localizations" : { @@ -26403,9 +27192,18 @@ } } } + }, + "The session that held this transaction has ended." : { + }, "The shell drops a field named __proto__ from a new document. Remove the field, then set it on the saved document." : { + }, + "The shell would reorder or drop a field named %@. Choose another name." : { + + }, + "The shell would reorder or drop a key inside %@: it lists number-like keys first and drops __proto__. Change this field with a query." : { + }, "The socket path contains a control or invisible character." : { @@ -26422,10 +27220,6 @@ }, "The socket path must start with /." : { - }, - "The RDS Endpoint \"%1$@\" is not the database at \"%2$@\", where the connection goes. Clear the RDS Endpoint unless you connect through a port forward." : { - "extractionState" : "manual", - "generatesSymbol" : false }, "The SQLite extensions for this connection could not be loaded." : { "extractionState" : "manual", @@ -26457,7 +27251,38 @@ } } }, - "The SSH server accepted the first sign-in step and asks for another (%@), which TablePro on iPhone and iPad cannot complete." : { + "The SSH host %@ contains characters TablePro does not allow in a host name." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The SSH host is empty." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The SSH host keys saved on this device could not be read, so this server's key could not be checked." : { + + }, + "the SSH host, user or port of “%@” is not one TablePro allows" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The SSH port %lld is not between 1 and 65535." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The SSH server accepted the first sign-in step and asks for another (%@), which TablePro on iPhone and iPad cannot complete." : { + + }, + "The SSH server asked “%@”, which TablePro can answer only while it is open." : { + + }, + "The SSH server asks for more to finish signing in." : { + + }, + "The SSH server closed the connection during the handshake." : { + + }, + "The SSH server did not accept the verification code." : { }, "The SSH server did not answer in time." : { @@ -26468,6 +27293,15 @@ }, "The SSH server does not accept a private key. It accepts %@." : { + }, + "The SSH server does not accept keyboard-interactive sign-in. It accepts %@." : { + + }, + "The SSH server does not sign in with a private key, so this jump host needs its own key." : { + + }, + "The SSH server ended the connection during the handshake: %@" : { + }, "The SSH server may be unreachable or running a different protocol." : { "localizations" : { @@ -26528,6 +27362,10 @@ } } }, + "The SSH user %@ contains characters TablePro does not allow in a user name." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The statement before it stays applied." : { }, @@ -26535,6 +27373,14 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The statement's placeholders do not match the %d values given." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The string or comment that starts on line %lld is longer than the import reads." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The table \"%@\" and all its data will be permanently deleted." : { "localizations" : { "ko" : { @@ -26562,6 +27408,9 @@ } } } + }, + "The table has no primary key, so this row cannot be identified." : { + }, "The table or column does not exist." : { "localizations" : { @@ -26624,6 +27473,9 @@ } } } + }, + "The top-level field “%@” holds Timestamp(0, 0), which MongoDB replaces with the current time on every save." : { + }, "The top-level field “%@” starts with $, which MongoDB refuses in a replaced document." : { @@ -26633,6 +27485,9 @@ }, "The transaction on this connection is still open. Commit or roll it back." : { + }, + "The type of column %@ cannot be written as SQL." : { + }, "The type of column %@ is not one SQL Server type: %@" : { "extractionState" : "manual", @@ -26640,12 +27495,36 @@ }, "The user name is longer than the 128 bytes FreeTDS takes." : { + }, + "The validator already declares %@. Remove that declaration first, then save again." : { + + }, + "The validator could not be read, so the change was not checked against it." : { + + }, + "The validator has a key named %@ that the shell would reorder or drop. Change the validator from a query tab instead." : { + }, "The validator uses %@ in a way that cannot be updated with the field. Change the validator first, then save again." : { + }, + "The validator would reject the document with _id %1$@ once %2$@ is removed. Fix the document or the validator first." : { + + }, + "The validator would reject the document with _id %1$@ once %2$@ is renamed to %3$@. Fix the document or the validator first." : { + + }, + "The value in %@ is shortened for display, so saving it would store only the part shown. Change this field with a query." : { + }, "The value of %@ is not valid JSON." : { + }, + "The workbook has more sheets than an XLSX file can hold." : { + + }, + "The workbook has no worksheet to read." : { + }, "The workbook is missing %@." : { "extractionState" : "manual", @@ -26657,6 +27536,9 @@ }, "The write did not finish within %d seconds, so MongoDB stopped it. Raise this connection's query timeout in Options if it needs longer." : { + }, + "The write was applied, but the servers did not confirm it as the write concern asks: %@" : { + }, "Theater Masks" : { "extractionState" : "manual", @@ -26697,9 +27579,15 @@ }, "These connections have open transactions: %@. Deleting them rolls the transactions back and discards their uncommitted changes." : { + }, + "These rows are not read from the database again." : { + }, "These settings stay on this device. The Mac reads them from its AWS profile." : { + }, + "This _id reads as more than one value type in this collection, such as text and an ObjectId, so TablePro cannot tell which document the row names. Use a query." : { + }, "This account signs in with Kerberos. Set Authentication to Kerberos (GSSAPI), then reconnect." : { "extractionState" : "manual", @@ -26734,12 +27622,21 @@ } } } + }, + "This action cannot be undone." : { + }, "This also commits the open transaction, keeping its uncommitted changes." : { }, "This bracket does not close the one that is open." : { + }, + "This BSON date is outside the shell's supported date range" : { + + }, + "This BSON value cannot be represented by the shell's Extended JSON reader" : { + }, "This build has no driver for that database type." : { "localizations" : { @@ -26824,9 +27721,6 @@ } } } - }, - "This _id reads as more than one value type in this collection, such as text and an ObjectId, so TablePro cannot tell which document the row names. Use a query." : { - }, "This column is generated, so it is never written." : { "localizations" : { @@ -26863,6 +27757,21 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "This connection cannot read its catalog for an export." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "This connection cannot run the query again read-only, so it was not run. Export the rows shown instead." : { + + }, + "This connection cannot send statements with bound values." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "This connection cannot stream rows to a file." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This connection goes through an SSH jump host, which TablePro on iPhone and iPad cannot dial. Open it in TablePro on the Mac." : { "localizations" : { "ko" : { @@ -27154,6 +28063,9 @@ }, "This connection's Redis user is not allowed to run this command or reach this key. Ask an administrator to grant it in the user's ACL." : { + }, + "This cursor has already been closed." : { + }, "This database cannot be renamed" : { "extractionState" : "manual", @@ -27369,12 +28281,30 @@ } } } + }, + "This device cannot check a SQL Server certificate, so it does not connect in this mode. Choose Required to connect without the check." : { + + }, + "This document has no _id, so it cannot be edited here." : { + }, "This document holds a NaN. MongoDB treats every NaN as the same value, so a save could not tell whether someone else changed it." : { + }, + "This document holds a value text cannot write back exactly, such as a subdocument like {\"$numberInt\": \"5\"}." : { + + }, + "This document holds a value that cannot be edited as text." : { + + }, + "This document holds JavaScript code with a scope. MongoDB compares a scope by value, so a save could not tell whether someone else changed it." : { + }, "This document is nested more than %d levels deep, which is too deep to edit here." : { + }, + "This document is too large to edit here. Saving it would send more than 16 MB to the server." : { + }, "This driver cannot explain a statement" : { "extractionState" : "manual", @@ -27387,6 +28317,12 @@ "This driver cannot send rows to COPY FROM STDIN." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "This engine has no conflict handling for INSERT, so the rows are written as plain inserts." : { + + }, + "This export spans %lld databases or schemas. Table references are qualified, but CREATE TABLE comes from the server unqualified, so restore it into the matching database or schema." : { + }, "This file also holds saved queries, which import only on a Mac." : { "localizations" : { @@ -27565,6 +28501,9 @@ } } } + }, + "This filter is not a document MongoDB can read: %@" : { + }, "This folder no longer exists. It may have been removed from another device." : { @@ -27630,13 +28569,31 @@ }, "This hop comes from the Mac's ~/.ssh/config. Change it there." : { + }, + "This import format does not produce SQL statements" : { + + }, + "This import will modify data. Are you sure you want to continue?" : { + }, "This is a Redis Sentinel, not a data node, so it only answers SENTINEL commands. Set Connection Mode to Sentinel and list this address under Sentinel Nodes." : { "extractionState" : "manual", "generatesSymbol" : false + }, + "This is a time-series collection, which cannot replace a single document, so its documents cannot be edited here." : { + }, "This is a view, so its documents cannot be edited. Edit them in the collection the view reads." : { + }, + "This is not a document MongoDB can read: %@" : { + + }, + "This is not valid JSON at line %1$d, column %2$d." : { + + }, + "This JSON document is too large for tree view. Use text mode instead." : { + }, "This keyword is already in use" : { "localizations" : { @@ -27665,9 +28622,6 @@ } } } - }, - "This JSON document is too large for tree view. Use text mode instead." : { - }, "This Oracle server is older than release 11.1, which the database driver does not support." : { "extractionState" : "manual", @@ -27705,6 +28659,12 @@ }, "This page is past the last row." : { + }, + "This pipeline is not something MongoDB can read: %@" : { + + }, + "This query calls a function that can change data, which Read-Only does not run. Switch Safe Mode to %@ to run it." : { + }, "This query will modify data. Are you sure you want to continue?" : { "localizations" : { @@ -27733,15 +28693,30 @@ } } } + }, + "This request is read only, and the statement is not a read." : { + }, "This row does not name a stored document." : { + }, + "This row has no _id, so TablePro cannot tell which document to change." : { + + }, + "This row has no value for the key column %@, so it cannot be matched." : { + + }, + "This row has no values, and %@ has no way to insert a row of only defaults." : { + }, "This row holds a value too long or binary to match exactly, so it cannot identify the row." : { }, "This row is no longer in the table. Another session may have changed its key or deleted it." : { + }, + "This row's _id is binary data whose subtype is not known, so TablePro cannot tell which document it names." : { + }, "This row's primary key is not plain text, so it cannot identify the row." : { @@ -27846,6 +28821,9 @@ } } } + }, + "This server runs Atlas Search but cannot list its search indexes, so one that uses the field cannot be ruled out. Check the collection's search indexes in Atlas, then change the field from a query tab." : { + }, "This SSH private key belongs to a security key (FIDO), which TablePro on iPhone and iPad cannot sign with." : { @@ -27908,6 +28886,12 @@ } } } + }, + "This table has no primary key or unique index to match rows on." : { + + }, + "This table has no primary key, and %@ cannot be compared to find the row." : { + }, "This table is empty." : { "localizations" : { @@ -28179,6 +29163,9 @@ } } } + }, + "TLS Handshake Failed" : { + }, "TLS handshake failed: %@" : { "extractionState" : "manual", @@ -28251,6 +29238,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "Token is required. Enter the token the server was started with, from quack_serve." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Too large to save (%@)" : { }, @@ -28289,6 +29280,12 @@ "Tortoise" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "TOTP" : { + + }, + "TOTP Secret" : { + }, "Touch and hold a connection to rename, duplicate, or move it to a group." : { "localizations" : { @@ -28321,23 +29318,23 @@ "Transaction Failed" : { }, - "Transaction Open" : { + "Transaction failed on %@. Roll it back to continue." : { }, "Transaction failed on %1$@, %2$@. Roll it back to continue." : { - }, - "Transaction failed on %@. Roll it back to continue." : { - }, "Transaction open" : { }, - "Transaction open on %1$@, %2$@" : { + "Transaction Open" : { }, "Transaction open on %@" : { + }, + "Transaction open on %1$@, %2$@" : { + }, "Transactions are not supported by Cloudflare D1" : { @@ -28345,6 +29342,12 @@ "Transfer" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Treat empty cells as NULL" : { + + }, + "Treat empty values as NULL" : { + }, "Tree" : { @@ -28352,6 +29355,15 @@ "Triangle" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Triggers" : { + + }, + "Trim leading and trailing spaces" : { + + }, + "Trim whitespace" : { + }, "Trophy" : { "extractionState" : "manual", @@ -28477,6 +29489,9 @@ }, "Try again in a moment." : { + }, + "Try again once it ends." : { + }, "Try again or check your connection." : { "localizations" : { @@ -28540,9 +29555,6 @@ } } } - }, - "Turn on TLS on the server, or set SSL Mode to Preferred to connect without it." : { - }, "Turn On iCloud Sync" : { "localizations" : { @@ -28571,6 +29583,9 @@ } } } + }, + "Turn on TLS on the server, or set SSL Mode to Preferred to connect without it." : { + }, "Two People" : { "extractionState" : "manual", @@ -28636,6 +29651,12 @@ } } } + }, + "Types" : { + + }, + "unbalanced quotes" : { + }, "Unexpected character." : { @@ -28669,9 +29690,6 @@ } } } - }, - "unbalanced quotes" : { - }, "Unfavorite" : { "localizations" : { @@ -28870,7 +29888,9 @@ } } }, - "Unlock TablePro to add rows to %@." : {}, + "Unlock TablePro to add rows to %@." : { + + }, "Unlocked" : { "extractionState" : "manual", "generatesSymbol" : false @@ -29004,6 +30024,15 @@ "Up to Date" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Update" : { + + }, + "Update changes the rows whose key is in the file. Insert or Update also adds the rest, and Insert, Skip Existing adds only rows whose key is not in the table. Columns not mapped keep their values." : { + + }, + "Update existing" : { + }, "Update Required" : { "extractionState" : "manual", @@ -29038,6 +30067,9 @@ } } } + }, + "Updated: %@" : { + }, "Upload Again" : { "extractionState" : "manual", @@ -29137,6 +30169,9 @@ } } } + }, + "Use Saved Mapping" : { + }, "Use Service Name Instead" : { "localizations" : { @@ -29424,6 +30459,12 @@ } } } + }, + "Values a spreadsheet would read as formulas, written with a leading single quote: %lld. Turn off Sanitize formula-like values to export them unchanged." : { + + }, + "Values of type %@ can't be written." : { + }, "Vault" : { "extractionState" : "manual", @@ -29470,6 +30511,10 @@ }, "Verify CA needs a CA certificate. On the connection's Network tab, choose the CA certificate that signed the server's certificate, or set SSL Mode to Verify Identity." : { + }, + "Verify Fingerprint needs the server's Certificate Fingerprint. Run quack_server_list() on the server to read it." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Verify Identity" : { "localizations" : { @@ -29499,6 +30544,10 @@ } } }, + "Verify Identity does not work through a tunnel. DuckDB checks the certificate against 127.0.0.1 there, not against the server's name. Set SSL Mode to Verify Fingerprint, or to Required (skip verify)." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Verify Identity through a tunnel checks the certificate against the server's host name, and %@ is an IP address. Enter the server's host name, or set SSL Mode to Verify CA." : { "extractionState" : "manual", "generatesSymbol" : false @@ -29635,6 +30684,9 @@ "Warning" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Warnings" : { + }, "Waveform" : { "extractionState" : "manual", @@ -29838,6 +30890,9 @@ } } } + }, + "When you first connect and first run a query" : { + }, "When you first connected and first ran a query" : { "localizations" : { @@ -29894,6 +30949,9 @@ } } } + }, + "Whole Table" : { + }, "Wi-Fi" : { "extractionState" : "manual", @@ -29934,6 +30992,15 @@ "Work & Places" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Working…" : { + + }, + "Wrap in transaction" : { + + }, + "Wrap in transaction (BEGIN/COMMIT)" : { + }, "Wrench" : { "extractionState" : "manual", @@ -29969,6 +31036,9 @@ } } } + }, + "Write each table's name as a heading" : { + }, "Write Query Blocked" : { "localizations" : { @@ -30025,6 +31095,15 @@ } } } + }, + "Writing definitions: %1$lld of %2$lld" : { + + }, + "Writing the workbook: %1$@ of %2$@" : { + + }, + "XLSX file exceeds 4 GB ZIP limit" : { + }, "Yellow" : { "localizations" : { @@ -30139,9 +31218,6 @@ } } } - }, - "already saved" : { - }, "Your Mac reaches the SSH server through these hosts from its ~/.ssh/config, and this device dials the same route with its own keys. Changing the SSH server or jump hosts above replaces it." : { diff --git a/apple/TableProMobile/TableProMobile/Platform/SessionTransactionMonitor.swift b/apple/TableProMobile/TableProMobile/Platform/SessionTransactionMonitor.swift index dc2d6ee94f..3d2a2c4acc 100644 --- a/apple/TableProMobile/TableProMobile/Platform/SessionTransactionMonitor.swift +++ b/apple/TableProMobile/TableProMobile/Platform/SessionTransactionMonitor.swift @@ -40,6 +40,9 @@ final class SessionTransactionMonitor { // Sessions whose last answer was idle, apart from those that could not say. @ObservationIgnored private var idleDrivers: [UUID: ObjectIdentifier] = [:] private var workCounts: [UUID: Int] = [:] + // Sessions an import holds: every other statement on them is refused until it ends. + private var heldByImport: Set = [] + @ObservationIgnored private var importEndHandlers: [UUID: [@MainActor () -> Void]] = [:] @ObservationIgnored private var statementCounts: [UUID: Int] = [:] private var rollBackCounts: [UUID: Int] = [:] @ObservationIgnored private var refreshGenerations: [UUID: Int] = [:] @@ -84,6 +87,19 @@ final class SessionTransactionMonitor { workCounts[connectionId, default: 0] > 0 } + func isHeldByImport(_ connectionId: UUID) -> Bool { + heldByImport.contains(connectionId) + } + + // Runs `handler` once the import holding the connection ends, or at once when none holds it. + func afterImport(_ connectionId: UUID, _ handler: @escaping @MainActor () -> Void) { + guard heldByImport.contains(connectionId) else { + handler() + return + } + importEndHandlers[connectionId, default: []].append(handler) + } + // A grid page is a read the app built, so a session lost under it cannot have committed anything. func isRunningStatements(_ connectionId: UUID) -> Bool { statementCounts[connectionId, default: 0] > 0 @@ -110,18 +126,34 @@ final class SessionTransactionMonitor { } // `reading` work keeps Commit and Roll Back off the session while it streams, and stales a dialog opened before it. - func track(_ connectionId: UUID, reading: Bool = false, _ work: () async throws -> T) async rethrows -> T { + // Admission is decided before the first suspension, so nothing can slip in between the check and the mark, and + // nothing ever waits: an import (`importing`) is refused while any other work runs on the session or the session + // is moving (`sessionMoving`: a switch, a `USE`, a dial or a check), and every other statement, reads included, is + // refused while an import holds it. A read inside an import's transaction is not harmless: a cancelled or failed + // one aborts a PostgreSQL transaction, and an interrupt rolls back SQLite's. + func track( + _ connectionId: UUID, + reading: Bool = false, + importing: Bool = false, + sessionMoving: Bool = false, + _ work: () async throws -> T + ) async throws -> T { + if heldByImport.contains(connectionId) { throw SessionHeldByImport() } + if importing { + if workCounts[connectionId, default: 0] > 0 || sessionMoving { throw SessionBusyForImport() } + heldByImport.insert(connectionId) + } workCounts[connectionId, default: 0] += 1 if !reading { statementCounts[connectionId, default: 0] += 1 } workGenerations[connectionId, default: 0] += 1 do { let value = try await work() await refresh(connectionId) - endWork(connectionId, reading: reading) + endWork(connectionId, reading: reading, importing: importing) return value } catch { await refresh(connectionId) - endWork(connectionId, reading: reading) + endWork(connectionId, reading: reading, importing: importing) throw error } } @@ -221,10 +253,16 @@ final class SessionTransactionMonitor { } // A read cannot end a transaction, so only its start, which a confirmation may race, stales a dialog. - private func endWork(_ connectionId: UUID, reading: Bool) { + private func endWork(_ connectionId: UUID, reading: Bool, importing: Bool) { if !reading { workGenerations[connectionId, default: 0] += 1 } let remaining = workCounts[connectionId, default: 1] - 1 workCounts[connectionId] = remaining > 0 ? remaining : nil + if importing { + heldByImport.remove(connectionId) + for handler in importEndHandlers.removeValue(forKey: connectionId) ?? [] { + handler() + } + } guard !reading else { return } let statements = statementCounts[connectionId, default: 1] - 1 statementCounts[connectionId] = statements > 0 ? statements : nil @@ -235,6 +273,30 @@ final class SessionTransactionMonitor { } } +// Any statement on a session an import holds, reads included, from any window. +nonisolated struct SessionHeldByImport: Error, LocalizedError, Equatable { + var errorDescription: String? { + String(localized: "An import is running on this connection.") + } + + var appError: AppError { + AppError( + category: .query, + title: String(localized: "Import Running"), + message: errorDescription ?? "", + recovery: String(localized: "Try again once it ends."), + underlying: self + ) + } +} + +// An import asked for a session that other work is using or moving; it starts only on one nothing else touches. +nonisolated struct SessionBusyForImport: Error, LocalizedError, Equatable { + var errorDescription: String? { + String(localized: "Other work is running on this connection. Start the import again once it ends.") + } +} + nonisolated enum TransactionActionError: Error, LocalizedError, Equatable { case cannotCommit case sessionReplaced diff --git a/apple/TableProMobile/TableProMobile/TableProMobileApp.swift b/apple/TableProMobile/TableProMobile/TableProMobileApp.swift index f7a93cad27..93f613759e 100644 --- a/apple/TableProMobile/TableProMobile/TableProMobileApp.swift +++ b/apple/TableProMobile/TableProMobile/TableProMobileApp.swift @@ -5,6 +5,7 @@ import SwiftUI import TableProDatabase import TableProModels import TableProSyncTransport +import UIKit @main struct TableProMobileApp: App { @@ -19,9 +20,20 @@ struct TableProMobileApp: App { init() { ConnectionListTips.configure() + // Nothing transfers at launch, so whatever tmp/Exports and tmp/Imports hold was left by a kill. + TransferFiles.sweep() + SQLiteDriver.removeLeftoverSnapshots() let appState = AppState() _appState = State(initialValue: appState) TableProAppDelegate.appState = appState + // A transfer holds its file database open through the background release; the last hold ending there + // releases it, or the file stays locked while the app is suspended. + appState.connectionManager.observeTransferHoldsEnded { _ in + Task { @MainActor in + guard UIApplication.shared.applicationState == .background else { return } + await appState.backgroundRelease.releaseForSuspension() + } + } // At launch, so a connection opened before the scene first turns active still sends its first-use event. if !TestRuntime.isActive { Self.startTelemetry(appState: appState) diff --git a/apple/TableProMobile/TableProMobile/Transfer/Background/TransferBackgroundRun.swift b/apple/TableProMobile/TableProMobile/Transfer/Background/TransferBackgroundRun.swift new file mode 100644 index 0000000000..be5e6f73f5 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Background/TransferBackgroundRun.swift @@ -0,0 +1,278 @@ +import BackgroundTasks +import Foundation +import os +import TableProDatabase +import TableProLogRedaction +import UIKit + +// Keeps the connection a transfer runs on out of the suspension release while the transfer runs. +@MainActor +protocol TransferHolding: AnyObject { + func release() +} + +extension TransferHold: TransferHolding {} + +@MainActor +protocol ContinuedProcessingTask: AnyObject { + var progress: Progress { get } + func updateTitle(_ title: String, subtitle: String) + func setTaskCompleted(success: Bool) + func onExpiration(_ handler: @escaping @MainActor () -> Void) +} + +@MainActor +protocol ContinuedProcessingScheduling: AnyObject { + // Registers `identifier` and submits a request for it; `launch` runs once the system starts the task. + func submit( + identifier: String, + title: String, + subtitle: String, + launch: @escaping @MainActor (any ContinuedProcessingTask) -> Void + ) throws +} + +// A long export or import keeps running when the user leaves the app: a `beginBackgroundTask` assertion on every +// release, and from iOS 26 a continued processing task the system shows with its progress. A run that cannot get the +// continued task carries on under the assertion alone. +@MainActor +final class TransferBackgroundRun { + private static let logger = Logger(subsystem: "com.TablePro", category: "TransferBackgroundRun") + static let identifierPrefix = "com.TablePro.TableProMobile.transfer." + + private let title: String + private let asserter: any BackgroundTaskAsserting + private let scheduler: (any ContinuedProcessingScheduling)? + private let holdProvider: (@MainActor () -> (any TransferHolding)?)? + private let expiryGrace: Duration + + private var assertion: UIBackgroundTaskIdentifier = .invalid + private var hold: (any TransferHolding)? + private var continuedTask: (any ContinuedProcessingTask)? + private var onExpiration: (@MainActor () -> Void)? + private var hasBegun = false + private var hasEnded = false + private var hasExpired = false + private var subtitle = "" + private(set) var submittedIdentifier: String? + + init( + title: String, + asserter: any BackgroundTaskAsserting, + scheduler: (any ContinuedProcessingScheduling)?, + hold: (@MainActor () -> (any TransferHolding)?)?, + expiryGrace: Duration = .seconds(2) + ) { + self.title = title + self.asserter = asserter + self.scheduler = scheduler + self.holdProvider = hold + self.expiryGrace = expiryGrace + } + + convenience init(title: String, connectionManager: ConnectionManager, connectionId: UUID) { + self.init( + title: title, + asserter: UIApplication.shared, + scheduler: Self.systemScheduler(), + hold: { connectionManager.holdForTransfer(connectionId) } + ) + } + + static func systemScheduler() -> (any ContinuedProcessingScheduling)? { + if #available(iOS 26.0, *) { + return SystemContinuedProcessingScheduler() + } + return nil + } + + var isRunningContinuedTask: Bool { continuedTask != nil } + + // Started while the app is in front, from the user's tap, which is when Apple requires a continued task to begin. + func begin(onExpiration: @escaping @MainActor () -> Void) { + guard !hasBegun else { return } + hasBegun = true + self.onExpiration = onExpiration + hold = holdProvider?() + assertion = asserter.beginBackgroundTask(name: title) { [weak self] in + self?.assertionExpired() + } + submitContinuedTask() + } + + func report(_ progress: TransferProgress) { + guard let continuedTask, !hasEnded else { return } + let mirrored = continuedTask.progress + if let fraction = progress.fractionCompleted { + mirrored.totalUnitCount = 1_000 + mirrored.completedUnitCount = Int64(fraction * 1_000) + } else { + // No total: the count still moves, which is what keeps the scheduler from calling the task stalled. + mirrored.totalUnitCount = Int64(progress.completedUnits + 1) + mirrored.completedUnitCount = Int64(progress.completedUnits) + } + let line = Self.subtitle(for: progress) + guard line != subtitle else { return } + subtitle = line + continuedTask.updateTitle(title, subtitle: line) + } + + func end(success: Bool) { + guard hasBegun, !hasEnded else { return } + hasEnded = true + continuedTask?.setTaskCompleted(success: success) + continuedTask = nil + hold?.release() + hold = nil + endAssertion() + } + + static func subtitle(for progress: TransferProgress) -> String { + if let status = progress.status, !status.isEmpty { return status } + switch progress.unit { + case .rows: + guard let total = progress.totalUnits else { + return String(format: String(localized: "%@ rows"), progress.completedUnits.formatted()) + } + return String( + format: String(localized: "%1$@ of %2$@ rows"), progress.completedUnits.formatted(), total.formatted() + ) + case .statements: + guard let total = progress.totalUnits else { + return String(format: String(localized: "%@ statements"), progress.completedUnits.formatted()) + } + return String( + format: String(localized: "%1$@ of %2$@ statements"), + progress.completedUnits.formatted(), + total.formatted() + ) + } + } + + // MARK: - Private + + private func submitContinuedTask() { + guard let scheduler else { return } + let identifier = Self.identifierPrefix + UUID().uuidString + do { + try scheduler.submit(identifier: identifier, title: title, subtitle: "") { [weak self] task in + self?.adopt(task) + } + submittedIdentifier = identifier + } catch { + // Apple's guidance: a continued task that does not start leaves the work running on the assertion. + Self.logger.info("Transfer runs without a continued task: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + } + } + + private func adopt(_ task: any ContinuedProcessingTask) { + guard !hasEnded else { + task.setTaskCompleted(success: true) + return + } + continuedTask = task + task.onExpiration { [weak self] in + self?.continuedTaskExpired() + } + } + + // The person cancelled from the system's progress UI, or the system needs the resources back. + private func continuedTaskExpired() { + expire() + } + + // While a continued task runs, the app keeps running past the assertion; without one, the run must stop now. + private func assertionExpired() { + guard continuedTask == nil else { + endAssertion() + return + } + expire() + let grace = expiryGrace + Task { [weak self] in + try? await Task.sleep(for: grace) + self?.endAssertion() + } + } + + private func expire() { + guard !hasExpired, !hasEnded else { return } + hasExpired = true + Self.logger.warning("Background time for a transfer ran out; the transfer stops") + onExpiration?() + } + + private func endAssertion() { + guard assertion != .invalid else { return } + asserter.endBackgroundTask(assertion) + assertion = .invalid + } +} + +@available(iOS 26.0, *) +@MainActor +final class SystemContinuedProcessingScheduler: ContinuedProcessingScheduling { + nonisolated enum SubmissionError: Error { + case registrationRefused + } + + // Registered for the job's own identifier right before it is submitted: a handler on the wildcard is refused, and a + // second registration of one identifier kills the app (BGTaskScheduler.h). The closures are `@Sendable` and hop to + // the main actor themselves, so the scheduler's queue never trips the main-actor check (critic B1). + func submit( + identifier: String, + title: String, + subtitle: String, + launch: @escaping @MainActor (any ContinuedProcessingTask) -> Void + ) throws { + let registered = BGTaskScheduler.shared.register(forTaskWithIdentifier: identifier, using: .main) { @Sendable task in + let box = UncheckedTask(task) + Task { @MainActor in + guard let continued = box.task as? BGContinuedProcessingTask else { + box.task.setTaskCompleted(success: false) + return + } + launch(SystemContinuedProcessingTask(continued)) + } + } + guard registered else { throw SubmissionError.registrationRefused } + let request = BGContinuedProcessingTaskRequest(identifier: identifier, title: title, subtitle: subtitle) + request.strategy = .fail + try BGTaskScheduler.shared.submit(request) + } +} + +@available(iOS 26.0, *) +@MainActor +final class SystemContinuedProcessingTask: ContinuedProcessingTask { + private let task: BGContinuedProcessingTask + + init(_ task: BGContinuedProcessingTask) { + self.task = task + } + + var progress: Progress { task.progress } + + func updateTitle(_ title: String, subtitle: String) { + task.updateTitle(title, subtitle: subtitle) + } + + func setTaskCompleted(success: Bool) { + task.setTaskCompleted(success: success) + } + + func onExpiration(_ handler: @escaping @MainActor () -> Void) { + task.expirationHandler = { @Sendable in + Task { @MainActor in handler() } + } + } +} + +// A BGTask crosses from the scheduler's callback to the main actor once, and is used only there. +nonisolated private final class UncheckedTask: @unchecked Sendable { + let task: BGTask + + init(_ task: BGTask) { + self.task = task + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Background/TransferProgress.swift b/apple/TableProMobile/TableProMobile/Transfer/Background/TransferProgress.swift new file mode 100644 index 0000000000..e918d8c95f --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Background/TransferProgress.swift @@ -0,0 +1,103 @@ +import Foundation + +nonisolated struct TransferProgress: Sendable, Equatable { + nonisolated enum Stage: Sendable, Equatable { + case preparing + case counting + case running + case finishing + } + + nonisolated enum Unit: Sendable, Equatable { + case rows + case statements + } + + var stage: Stage + var unit: Unit + var completedUnits: Int = 0 + var totalUnits: Int? + var objectName: String? + var objectIndex: Int = 0 + var objectCount: Int = 0 + var status: String? + var bytesRead: Int64? + var totalBytes: Int64? + + init(stage: Stage, unit: Unit = .rows) { + self.stage = stage + self.unit = unit + } + + // A total the work ran past is a guess, so it is never shown past the end. + var fractionCompleted: Double? { + if let totalUnits, totalUnits > 0 { + return min(1, Double(completedUnits) / Double(totalUnits)) + } + if let bytesRead, let totalBytes, totalBytes > 0 { + return min(1, Double(bytesRead) / Double(totalBytes)) + } + return nil + } +} + +// Reads a plugin's `Progress` on a timer: its own KVO fires every thousand rows and on threads the caller does not own. +nonisolated final class TransferProgressPoller: @unchecked Sendable { + private let lock = NSLock() + private var task: Task? + private var last: TransferProgress? + private var isStopped = false + + static let interval: Duration = .milliseconds(250) + + func start( + read: @escaping @Sendable () -> TransferProgress, + report: @escaping @Sendable (TransferProgress) -> Void + ) { + let poll = Task.detached { [weak self] in + while !Task.isCancelled { + guard let self else { return } + self.emit(read(), final: false, to: report) + try? await Task.sleep(for: Self.interval) + } + } + lock.withLock { task = poll } + } + + // Nothing the timer read before the stop is reported after the final value. + func stop(final progress: TransferProgress?, report: @Sendable (TransferProgress) -> Void) { + let running = lock.withLock { () -> Task? in + isStopped = true + defer { task = nil } + return task + } + running?.cancel() + if let progress { emit(progress, final: true, to: report) } + } + + private func emit(_ progress: TransferProgress, final: Bool, to report: @Sendable (TransferProgress) -> Void) { + let changed = lock.withLock { () -> Bool in + guard final || !isStopped, last != progress else { return false } + last = progress + return true + } + if changed { report(progress) } + } +} + +// Foundation's `Progress` is thread-safe; the box lets a poller read it from its own task. +nonisolated final class SharedProgress: @unchecked Sendable { + let progress: Progress + + init(_ progress: Progress) { + self.progress = progress + } + + var description: String? { + progress.localizedDescription.flatMap { $0.isEmpty ? nil : $0 } + } + + var status: String? { + progress.localizedAdditionalDescription.flatMap { $0.isEmpty ? nil : $0 } + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Contracts/MobileExportCatalog.swift b/apple/TableProMobile/TableProMobile/Transfer/Contracts/MobileExportCatalog.swift new file mode 100644 index 0000000000..06fcc431cf --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Contracts/MobileExportCatalog.swift @@ -0,0 +1,75 @@ +import Foundation +import TableProPluginKit + +// One engine's answers to everything an export asks besides the rows. The names and defaults are the +// `PluginExportDataSource` ones, so `MobileExportDataSource` forwards each call unchanged. +nonisolated protocol MobileExportCatalog: Sendable { + var databaseTypeId: String { get } + + // Every object the engine's macOS export lists for this database and schema, tables included, each kind + // filled the way macOS fills it (a routine's `tableType` is its `PluginRoutineKind.rawValue`). + func exportObjects(database: String, schema: String?) async throws -> [PluginExportTable] + + // The SELECT that reads an object's rows, or nil to let the host write `SELECT * FROM `. + func rowsQuery(for object: PluginExportTable) -> String? + + // An engine whose rows are not SQL rows (MongoDB documents) streams them itself; nil uses the driver's + // `ExportRowStreaming`. + func streamExportRows(for object: PluginExportTable) -> AsyncThrowingStream? + + func quoteIdentifier(_ identifier: String) -> String + func escapeStringLiteral(_ value: String) -> String + func fetchTableDDL(table: String, databaseName: String) async throws -> String + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? + func fetchDependentSequences(table: String, databaseName: String) async throws -> [PluginSequenceInfo] + func fetchDependentTypes(table: String, databaseName: String) async throws -> [PluginEnumTypeInfo] + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] + func fetchForeignKeys(table: String, databaseName: String) async throws -> [PluginForeignKeyInfo] + func fetchAllForeignKeys(databaseName: String) async throws -> [String: [PluginForeignKeyInfo]] + var tableDDLIncludesForeignKeys: Bool { get } + func fetchIndexDDL(table: String, databaseName: String) async throws -> [String] + func fetchCommentDDL(table: String, databaseName: String) async throws -> [String] + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String + func scriptText(for ddl: String) -> String + var lexicalFeatures: SQLLexicalFeatures { get } + func fetchGrantStatements(principal: String, host: String?) async throws -> [String] + var supportsCascadeDrop: Bool { get } + func dropStatement(for object: PluginExportTable) -> String? + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? + func closeSessionScope(_ scope: PluginSessionScope) async throws + func fetchCreationScope(_ object: PluginExportTable) async throws -> PluginStatementScope? + func fetchIndexes(table: String, databaseName: String) async throws -> [PluginIndexInfo] + func fetchObjectComment(_ object: PluginExportTable) async throws -> String? +} + +nonisolated extension MobileExportCatalog { + func rowsQuery(for object: PluginExportTable) -> String? { nil } + func streamExportRows(for object: PluginExportTable) + -> AsyncThrowingStream? { nil } + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { nil } + func fetchDependentSequences(table: String, databaseName: String) async throws -> [PluginSequenceInfo] { [] } + func fetchDependentTypes(table: String, databaseName: String) async throws -> [PluginEnumTypeInfo] { [] } + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { [] } + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] { [:] } + func fetchForeignKeys(table: String, databaseName: String) async throws -> [PluginForeignKeyInfo] { [] } + func fetchAllForeignKeys(databaseName: String) async throws -> [String: [PluginForeignKeyInfo]] { [:] } + var tableDDLIncludesForeignKeys: Bool { false } + func fetchIndexDDL(table: String, databaseName: String) async throws -> [String] { [] } + func fetchCommentDDL(table: String, databaseName: String) async throws -> [String] { [] } + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + try await fetchTableDDL(table: object.name, databaseName: object.databaseName) + } + func scriptText(for ddl: String) -> String { + ddl.hasSuffix(";") ? ddl : ddl + ";" + } + var lexicalFeatures: SQLLexicalFeatures { [] } + func fetchGrantStatements(principal: String, host: String?) async throws -> [String] { [] } + var supportsCascadeDrop: Bool { false } + func dropStatement(for object: PluginExportTable) -> String? { nil } + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { nil } + func closeSessionScope(_ scope: PluginSessionScope) async throws {} + func fetchCreationScope(_ object: PluginExportTable) async throws -> PluginStatementScope? { nil } + func fetchIndexes(table: String, databaseName: String) async throws -> [PluginIndexInfo] { [] } + func fetchObjectComment(_ object: PluginExportTable) async throws -> String? { nil } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Contracts/MobileTransferEngine.swift b/apple/TableProMobile/TableProMobile/Transfer/Contracts/MobileTransferEngine.swift new file mode 100644 index 0000000000..2ae5dbddd4 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Contracts/MobileTransferEngine.swift @@ -0,0 +1,28 @@ +import Foundation +import TableProDatabase +import TableProModels + +nonisolated struct MobileTransferContext: Sendable, Equatable { + let databaseType: DatabaseType + let database: String? + let schema: String? +} + +// A driver that can take part in an export or an import. Each engine conforms its own driver in its own file; +// a driver that does not conform offers neither. +nonisolated protocol MobileTransferEngine: DatabaseDriver { + func makeExportCatalog(context: MobileTransferContext) async throws -> any MobileExportCatalog + func makeWriteDialect(context: MobileTransferContext) async throws -> any MobileWriteDialect + // Whether each table `fetchTables` lists is an object of the export catalog, so it can be exported on its own. + var exportsListedTables: Bool { get } +} + +nonisolated extension MobileTransferEngine { + var exportsListedTables: Bool { true } + + func makeWriteDialect(context: MobileTransferContext) async throws -> any MobileWriteDialect { + throw MobileTransferRefusal(reason: String( + format: String(localized: "%@ cannot take rows from a file."), context.databaseType.mobileDisplayName + )) + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Contracts/MobileWriteDialect.swift b/apple/TableProMobile/TableProMobile/Transfer/Contracts/MobileWriteDialect.swift new file mode 100644 index 0000000000..8868197576 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Contracts/MobileWriteDialect.swift @@ -0,0 +1,33 @@ +import Foundation +import TableProModels +import TableProPluginKit + +// What the shared import writer (`TablePro/Core/Transfer/Shared`) needs that differs by engine and session. +nonisolated protocol MobileWriteDialect: Sendable { + var databaseTypeId: String { get } + + // The statement builder for one target table, with this engine's quoting, placeholders, BIT columns, + // ceilings and row-update hooks filled in. + func importStatements(table: String, schema: String?, columns: [ColumnInfo]) async throws -> SQLImportStatements + + // Columns an UPDATE may not set (ClickHouse sorting, primary and partition keys). + func columnsUpdateCannotSet(table: String, schema: String?) async throws -> Set + + // The CREATE TABLE statements for a table made from an import, or a throw naming why the engine refuses one. + func createTableStatements(definition: PluginCreateTableDefinition, schema: String?) throws -> [String] + + // Whether every row statement names the import's database, so they land there wherever the session has moved. + var namesDatabase: Bool { get } +} + +nonisolated extension MobileWriteDialect { + func columnsUpdateCannotSet(table: String, schema: String?) async throws -> Set { [] } + + var namesDatabase: Bool { false } +} + +nonisolated struct MobileTransferRefusal: Error, Equatable, Sendable, LocalizedError { + let reason: String + + var errorDescription: String? { reason } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Engines/DuckDBTransferEngine.swift b/apple/TableProMobile/TableProMobile/Transfer/Engines/DuckDBTransferEngine.swift new file mode 100644 index 0000000000..15aa5ee2bc --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Engines/DuckDBTransferEngine.swift @@ -0,0 +1,165 @@ +import Foundation +import TableProDatabase +import TableProDuckDBCore +import TableProModels +import TableProPluginKit +import TableProSQLGrammar +import TableProSQLQuoting + +nonisolated extension DuckDBDriver: MobileTransferEngine { + func makeExportCatalog(context: MobileTransferContext) async throws -> any MobileExportCatalog { + let catalog = try await catalogQuery("SELECT current_database()").rows.first?[safe: 0]?.asText + guard let catalog else { throw DuckDBDriverError.queryFailed("The database did not name its catalog") } + return DuckDBTransferCatalog( + driver: self, catalog: catalog, defaultSchema: context.schema ?? currentSchema ?? Self.defaultSchema + ) + } + + func makeWriteDialect(context: MobileTransferContext) async throws -> any MobileWriteDialect { + DuckDBWriteDialect(defaultSchema: context.schema ?? currentSchema ?? Self.defaultSchema) + } + + static let defaultSchema = "main" +} + +// The Mac plugin's export answers (`DuckDBPlugin`), read through the driver the transfer runs on. Every read names the +// catalog, because `duckdb_*()` spans every attached database; an object's container is its schema, as on the Mac. +nonisolated struct DuckDBTransferCatalog: MobileExportCatalog { + let driver: any CatalogQuerying + let catalog: String + let defaultSchema: String + + var databaseTypeId: String { duckdbDatabaseTypeId } + + private var run: DuckDBExportCatalog.Run { + { [driver] sql, parameters in try await driver.catalogQuery(sql, parameters: parameters) } + } + + private func schema(for databaseName: String) -> String { + databaseName.isEmpty ? defaultSchema : databaseName + } + + func exportObjects(database: String, schema: String?) async throws -> [PluginExportTable] { + let container = schema ?? defaultSchema + let tables = try await DuckDBExportCatalog.tables(catalog: catalog, schema: container, run: run) + let (catalog, run) = (catalog, run) + return tables.compactMap { ExportCatalogItems.relation($0, container: container) } + + (await ExportCatalogItems.listing("macros") { + ExportCatalogItems.routines( + try await DuckDBExportCatalog.routines(catalog: catalog, schema: container, run: run), + container: container + ) + }) + } + + func quoteIdentifier(_ identifier: String) -> String { + DuckDBSchemaQueries.quoteIdentifier(identifier) + } + + func escapeStringLiteral(_ value: String) -> String { + SQLQuoting.stringLiteralBody(value) + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + try await DuckDBExportCatalog.tableDDL(catalog: catalog, schema: schema(for: databaseName), table: table, run: run) + } + + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { + try await DuckDBExportCatalog.columns(catalog: catalog, schema: schema(for: databaseName), table: table, run: run) + } + + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] { + try await DuckDBExportCatalog.allColumns(catalog: catalog, schema: schema(for: databaseName), run: run) + } + + func fetchForeignKeys(table: String, databaseName: String) async throws -> [PluginForeignKeyInfo] { + try await DuckDBExportCatalog.foreignKeys( + catalog: catalog, schema: schema(for: databaseName), table: table, run: run + ) + } + + // The plugin has no schema-wide read, so PluginKit's default asks table by table; this does the same. + func fetchAllForeignKeys(databaseName: String) async throws -> [String: [PluginForeignKeyInfo]] { + let container = schema(for: databaseName) + var keys: [String: [PluginForeignKeyInfo]] = [:] + for table in try await DuckDBExportCatalog.tables(catalog: catalog, schema: container, run: run) { + try Task.checkCancellation() + keys[table.name] = try await DuckDBExportCatalog.foreignKeys( + catalog: catalog, schema: container, table: table.name, run: run + ) + } + return keys + } + + var tableDDLIncludesForeignKeys: Bool { DuckDBExportCatalog.tableDDLIncludesForeignKeys } + + func fetchIndexDDL(table: String, databaseName: String) async throws -> [String] { + try await DuckDBExportCatalog.indexDDL(catalog: catalog, schema: schema(for: databaseName), table: table, run: run) + } + + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + let container = schema(for: object.databaseName) + switch object.kind { + case .table, .foreignTable: + return try await DuckDBExportCatalog.tableDDL(catalog: catalog, schema: container, table: object.name, run: run) + case .view, .materializedView: + return try await DuckDBExportCatalog.viewDefinition( + catalog: catalog, schema: container, view: object.name, run: run + ) + case .routine: + let routines = try await DuckDBExportCatalog.routines(catalog: catalog, schema: container, run: run) + let routine = try ExportCatalogItems.routine(matching: object, in: routines) + return try await DuckDBExportCatalog.routineDDL(routine, catalog: catalog, run: run) + default: + throw PluginObjectSourceError.unsupported(object.name) + } + } + + // The identity is the shared listing's signature, which names a table macro's kind. + func dropStatement(for object: PluginExportTable) -> String? { + guard object.kind == .routine else { return nil } + return DuckDBObjectQueries.dropMacro( + name: object.name, + schema: schema(for: object.databaseName), + isTableMacro: DuckDBObjectQueries.namesTableMacro(object.identity) + ) + } + + private var script: ExportCatalogScript { + ExportCatalogScript(databaseTypeId: databaseTypeId, session: nil) + } + + var lexicalFeatures: SQLLexicalFeatures { script.lexicalFeatures } + + func scriptText(for ddl: String) -> String { + script.scriptText(for: ddl) + } +} + +nonisolated struct DuckDBWriteDialect: MobileWriteDialect { + let defaultSchema: String + + var databaseTypeId: String { duckdbDatabaseTypeId } + + func importStatements(table: String, schema: String?, columns: [ColumnInfo]) async throws -> SQLImportStatements { + let quote = DuckDBSchemaQueries.quoteIdentifier + return SQLImportStatements( + databaseTypeId: databaseTypeId, + qualifiedTableName: ImportTableNames.qualified(table, schema: schema, quote: quote), + parameterStyle: .dollar, + quoteIdentifier: quote, + columnTypeNames: ImportTableNames.columnTypeNames(columns) + ) + } + + func createTableStatements(definition: PluginCreateTableDefinition, schema: String?) throws -> [String] { + guard let statements = duckdbCreateTableStatements( + definition: definition, + schema: schema.flatMap(\.nilIfEmpty) ?? defaultSchema, + quote: DuckDBSchemaQueries.quoteIdentifier + ) else { + throw ImportTableNames.noStatement + } + return statements + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Engines/ExportCatalogSupport.swift b/apple/TableProMobile/TableProMobile/Transfer/Engines/ExportCatalogSupport.swift new file mode 100644 index 0000000000..8c9eccbf70 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Engines/ExportCatalogSupport.swift @@ -0,0 +1,163 @@ +import Foundation +import os +import TableProDatabase +import TableProModels +import TableProPluginKit +import TableProSQLGrammar + +// How a dump spells an engine's DDL and which lexical rules it declares: the session's readings, through the same owner +// the Mac's export adapter uses. Every iOS engine has a curated profile, so nothing is declared beside the session. +nonisolated struct ExportCatalogScript: Sendable { + let databaseTypeId: String + let readings: SQLLexicalReadings + + init(databaseTypeId: String, session: SQLSessionLexicalFacts?) { + self.databaseTypeId = databaseTypeId + readings = SQLLexicalReadings.resolve(databaseTypeId: databaseTypeId, declared: nil, session: session) + } + + var lexicalFeatures: SQLLexicalFeatures { + readings.execution.pluginFeatures + } + + func scriptText(for ddl: String) -> String { + SQLScriptText(databaseTypeId: databaseTypeId, readings: readings).scriptText(forDriverText: ddl) + } +} + +// A list read once per container and matched back by name, as the Mac's `ExportObjectCache` keeps it: a driver +// addresses a routine or trigger through the info object it handed out. +nonisolated final class ExportCatalogListCache: Sendable { + private let lists = OSAllocatedUnfairLock<[String: [Value]]>(initialState: [:]) + + func values(for container: String, load: () async throws -> [Value]) async throws -> [Value] { + if let cached = lists.withLock({ $0[container] }) { return cached } + let loaded = try await load() + lists.withLock { $0[container] = loaded } + return loaded + } +} + +nonisolated enum ExportCatalogItems { + private static let logger = Logger(subsystem: "com.TablePro", category: "ExportCatalogItems") + + // `container` is what the export groups the object under: the schema on these engines. A routine's `tableType` is + // its kind, which picks DROP FUNCTION or DROP PROCEDURE. + static func item( + _ name: String, + container: String, + kind: PluginExportObjectKind, + tableType: String? = nil, + identity: String? = nil, + parentTable: String? = nil + ) -> PluginExportTable { + PluginExportTable( + name: name, + databaseName: container, + tableType: tableType ?? kind.rawValue, + schema: container, + kind: kind, + identity: identity, + parentTable: parentTable + ) + } + + // The relation kinds a listing names, matched loosely as the Mac matches them. A sequence is listed on its own. + static func relation(_ table: PluginTableInfo, container: String) -> PluginExportTable? { + guard table.type.uppercased() != "SEQUENCE" else { return nil } + return item(table.name, container: container, kind: PluginExportObjectKind.from(tableType: table.type)) + } + + static func routines(_ routines: [PluginRoutineInfo], container: String) -> [PluginExportTable] { + routines.map { + item( + $0.name, container: container, kind: .routine, tableType: $0.kind.rawValue, identity: $0.argumentSignature + ) + } + } + + static func triggers(_ triggers: [PluginTriggerInfo], container: String) -> [PluginExportTable] { + triggers.map { item($0.name, container: container, kind: .trigger, parentTable: $0.table) } + } + + static func principals(_ principals: [PluginPrincipalInfo], container: String) -> [PluginExportTable] { + principals.map { item($0.ref.name, container: container, kind: .grant, identity: $0.ref.host) } + } + + // A kind the engine cannot answer comes back empty: one failed list must not cost the user the table list. + static func listing(_ kind: String, _ read: () async throws -> [PluginExportTable]) async -> [PluginExportTable] { + do { + return try await read() + } catch { + logger.warning("Failed to list \(kind, privacy: .public) for export: \(error.localizedDescription, privacy: .private)") + return [] + } + } + + // The kind is part of the match: `f()` can name a function and a procedure apart. + static func routine(matching object: PluginExportTable, in routines: [PluginRoutineInfo]) throws -> PluginRoutineInfo { + let kind = PluginRoutineKind(rawValue: object.tableType.lowercased()) + guard let routine = routines.first(where: { candidate in + candidate.name == object.name + && (candidate.argumentSignature ?? "") == (object.identity ?? "") + && (kind == nil || candidate.kind == kind) + }) else { + throw PluginObjectSourceError.unsupported(object.name) + } + return routine + } + + static func trigger(matching object: PluginExportTable, in triggers: [PluginTriggerInfo]) throws -> PluginTriggerInfo { + guard let trigger = triggers.first(where: { $0.name == object.name && $0.table == object.parentTable }) else { + throw PluginObjectSourceError.unsupported(object.name) + } + return trigger + } + + // The engine renders its own GRANT text: only it knows how it spells a grantee and a privilege target. + static func grantStatements( + principal: String, + host: String?, + fetchGrants: (PluginPrincipalRef) async throws -> [PluginGrantInfo], + generate: (PluginPrincipalChangeSet) -> [String]? + ) async throws -> [String] { + let ref = PluginPrincipalRef(name: principal, host: host) + let grants = try await fetchGrants(ref) + guard !grants.isEmpty else { return [] } + return generate(PluginPrincipalChangeSet(principal: ref, grantsToAdd: grants)) ?? [] + } + + // A routine's drop needs its kind: PostgreSQL refuses DROP FUNCTION on a procedure. + static func routineKind(of object: PluginExportTable) -> PluginRoutineKind? { + guard object.kind == .routine else { return nil } + return PluginRoutineKind(rawValue: object.tableType.lowercased()) + } +} + +nonisolated extension DriverTransactionState { + var pluginSessionState: PluginSessionTransactionState { + switch self { + case .idle: .idle + case .explicitTransaction, .implicitTransaction: .inTransaction + case .abortedTransaction: .abortedTransaction + case .unknown: .unknown + } + } +} + +nonisolated enum ImportTableNames { + // As the Mac's statement generator spells the target: schema-qualified when the import names a schema. + static func qualified(_ table: String, schema: String?, quote: (String) -> String) -> String { + guard let schema, !schema.isEmpty else { return quote(table) } + return "\(quote(schema)).\(quote(table))" + } + + static func columnTypeNames(_ columns: [ColumnInfo]) -> [String: String] { + Dictionary(columns.map { ($0.name, $0.typeName) }, uniquingKeysWith: { first, _ in first }) + } + + // The message the Mac's `RowImportTableMaker` gives when the engine builds no CREATE TABLE. + static var noStatement: MobileTransferRefusal { + MobileTransferRefusal(reason: String(localized: "Could not build the CREATE TABLE statement")) + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Engines/MSSQLTransferEngine.swift b/apple/TableProMobile/TableProMobile/Transfer/Engines/MSSQLTransferEngine.swift new file mode 100644 index 0000000000..53e8b7a7c5 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Engines/MSSQLTransferEngine.swift @@ -0,0 +1,233 @@ +import Foundation +import TableProDatabase +import TableProModels +import TableProMSSQLCore +import TableProPluginKit +import TableProSQLQuoting + +nonisolated extension MSSQLDriver: CatalogQuerying, ExportRowStreaming, MobileTransferEngine { + func catalogQuery(_ sql: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + try await catalogCells(sql, parameters: parameters, in: nil) + } + + // Every read runs in the export's database through `sp_executesql`, whatever a `USE` moved the session to. + func makeExportCatalog(context: MobileTransferContext) async throws -> any MobileExportCatalog { + let database = context.database + return MSSQLMobileExportCatalog( + catalog: MSSQLExportCatalog( + run: { [self] statement in try await catalogCells(statement, in: database) }, + currentSchema: effectiveSchema, + principalState: principalState + ), + script: ExportCatalogScript(databaseTypeId: DatabaseType.mssql.rawValue, session: settledLexicalFacts) + ) + } + + func makeWriteDialect(context: MobileTransferContext) async throws -> any MobileWriteDialect { + MSSQLMobileWriteDialect( + databaseTypeId: DatabaseType.mssql.rawValue, currentSchema: effectiveSchema, database: context.database + ) + } +} + +// The Mac's SQL Server export answers. `databaseName` is the schema the export groups an object under; empty is the +// session's own. +nonisolated struct MSSQLMobileExportCatalog: MobileExportCatalog { + let catalog: MSSQLExportCatalog + let script: ExportCatalogScript + private let routines = ExportCatalogListCache() + private let triggers = ExportCatalogListCache() + private let userTypes = ExportCatalogListCache() + + init(catalog: MSSQLExportCatalog, script: ExportCatalogScript) { + self.catalog = catalog + self.script = script + } + + var databaseTypeId: String { script.databaseTypeId } + + private func schema(_ databaseName: String) -> String { + catalog.effectiveSchema(databaseName) + } + + // MARK: - Listing + + func exportObjects(database: String, schema: String?) async throws -> [PluginExportTable] { + let container = catalog.effectiveSchema(schema) + var items = try await catalog.run(MSSQLSchemaQueries.tables(schema: container)).rows.compactMap { row in + MSSQLSchemaQueries.parseTableRow(row.map(\.asText)).map { + ExportCatalogItems.item($0.name, container: container, kind: $0.isView ? .view : .table) + } + } + items += await ExportCatalogItems.listing("routines") { + ExportCatalogItems.routines(try await cachedRoutines(container), container: container) + } + items += await ExportCatalogItems.listing("triggers") { + ExportCatalogItems.triggers(try await cachedTriggers(container), container: container) + } + items += await ExportCatalogItems.listing("user-defined types") { + try await cachedUserTypes(container).map { ExportCatalogItems.item($0.name, container: container, kind: .userType) } + } + items += await ExportCatalogItems.listing("principals") { + ExportCatalogItems.principals(try await catalog.fetchPrincipals(), container: container) + } + return items + } + + private func cachedRoutines(_ schema: String) async throws -> [PluginRoutineInfo] { + try await routines.values(for: schema) { try await catalog.fetchRoutines(schema: schema) } + } + + private func cachedTriggers(_ schema: String) async throws -> [PluginTriggerInfo] { + try await triggers.values(for: schema) { try await catalog.fetchAllTriggers(schema: schema) } + } + + private func cachedUserTypes(_ schema: String) async throws -> [PluginUserDefinedTypeInfo] { + try await userTypes.values(for: schema) { try await catalog.fetchUserDefinedTypes(schema: schema) } + } + + // MARK: - Text + + func quoteIdentifier(_ identifier: String) -> String { + SQLQuoting.identifier(identifier, quote: .bracket) + } + + func escapeStringLiteral(_ value: String) -> String { + MSSQLStringLiteral.escaped(value) + } + + func scriptText(for ddl: String) -> String { + script.scriptText(for: ddl) + } + + var lexicalFeatures: SQLLexicalFeatures { + script.lexicalFeatures + } + + var tableDDLIncludesForeignKeys: Bool { MSSQLExportCatalog.tableDDLIncludesForeignKeys } + + // MARK: - Definitions + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + try await catalog.fetchTableDDL(table: table, schema: schema(databaseName)) + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { + try await catalog.fetchApproximateRowCount(table: table, schema: schema(databaseName)) + } + + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { + try await catalog.fetchColumns(table: table, schema: schema(databaseName)) + } + + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] { + try await catalog.fetchAllColumns(schema: schema(databaseName)) + } + + func fetchForeignKeys(table: String, databaseName: String) async throws -> [PluginForeignKeyInfo] { + try await catalog.fetchForeignKeys(table: table, schema: schema(databaseName)) + } + + func fetchAllForeignKeys(databaseName: String) async throws -> [String: [PluginForeignKeyInfo]] { + try await catalog.fetchAllForeignKeys(schema: schema(databaseName)) + } + + func fetchIndexDDL(table: String, databaseName: String) async throws -> [String] { + try await catalog.fetchIndexDDL(table: table, schema: schema(databaseName)) + } + + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + let schema = schema(object.databaseName) + switch object.kind { + case .table: + return try await catalog.fetchTableDDL(table: object.name, schema: schema) + case .view: + return try await catalog.fetchViewDefinition(view: object.name, schema: schema) + case .routine: + let routine = try ExportCatalogItems.routine(matching: object, in: try await cachedRoutines(schema)) + return try await catalog.fetchRoutineDDL(routine) + case .trigger: + let trigger = try ExportCatalogItems.trigger(matching: object, in: try await cachedTriggers(schema)) + return try await catalog.fetchTriggerDDL(trigger) + case .userType: + guard let type = try await cachedUserTypes(schema).first(where: { $0.name == object.name }), + let definition = try await catalog.fetchUserDefinedType(type).definition, + !definition.isEmpty + else { + throw PluginObjectSourceError.unsupported(object.name) + } + return definition + default: + throw PluginObjectSourceError.unsupported(object.name) + } + } + + func dropStatement(for object: PluginExportTable) -> String? { + let schema = schema(object.databaseName) + switch object.kind { + case .trigger: + guard let parent = object.parentTable else { return nil } + return catalog.generateDropTriggerSQL(name: object.name, table: parent, schema: schema) + case .routine: + guard let kind = ExportCatalogItems.routineKind(of: object) else { return nil } + return catalog.generateDropRoutineSQL( + name: object.name, signature: object.identity, schema: schema, isFunction: kind == .function + ) + default: + return nil + } + } + + func fetchGrantStatements(principal: String, host: String?) async throws -> [String] { + try await ExportCatalogItems.grantStatements( + principal: principal, + host: host, + fetchGrants: { try await catalog.fetchGrants(for: $0) }, + generate: { catalog.generateGrantSQL(changeSet: $0) } + ) + } +} + +// Quoting, the bind and row ceilings of `SQLImportStatements`, and the Mac's CREATE TABLE writer. With a database the +// rows go to `[db].[schema].[table]`, so another window's switch or `USE` cannot send them to another database. +nonisolated struct MSSQLMobileWriteDialect: MobileWriteDialect { + let databaseTypeId: String + let currentSchema: String + var database: String? + + var namesDatabase: Bool { + database.map { !$0.isEmpty } ?? false + } + + func qualifiedTableName(_ table: String, schema: String?) -> String { + guard let database, !database.isEmpty else { + return ImportTableNames.qualified(table, schema: schema, quote: Self.quote) + } + let target = MSSQLExportCatalog.effectiveSchema(schema, current: currentSchema) + return [database, target, table].map(Self.quote).joined(separator: ".") + } + + func importStatements(table: String, schema: String?, columns: [ColumnInfo]) async throws -> SQLImportStatements { + SQLImportStatements( + databaseTypeId: databaseTypeId, + qualifiedTableName: qualifiedTableName(table, schema: schema), + parameterStyle: .questionMark, + quoteIdentifier: Self.quote, + columnTypeNames: ImportTableNames.columnTypeNames(columns) + ) + } + + // A column the writer cannot spell names its reason, as the Mac's schema editor refuses it. + func createTableStatements(definition: PluginCreateTableDefinition, schema: String?) throws -> [String] { + let target = MSSQLExportCatalog.effectiveSchema(schema, current: currentSchema) + guard let statements = MSSQLTableDefinitionSQL.createTableStatements(definition, schema: target) else { + let reason = definition.columns.lazy.compactMap { MSSQLTableDefinitionSQL.refusal(for: $0) }.first + throw reason.map { MobileTransferRefusal(reason: $0) } ?? ImportTableNames.noStatement + } + return statements + } + + private static func quote(_ name: String) -> String { + SQLQuoting.identifier(name, quote: .bracket) + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Engines/MySQLTransferEngine.swift b/apple/TableProMobile/TableProMobile/Transfer/Engines/MySQLTransferEngine.swift new file mode 100644 index 0000000000..626ff28dfe --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Engines/MySQLTransferEngine.swift @@ -0,0 +1,280 @@ +import Foundation +import TableProDatabase +import TableProModels +import TableProMySQLCore +import TableProPluginKit +import TableProSQLGrammar + +nonisolated extension MySQLDriver: MobileTransferEngine { + func makeExportCatalog(context: MobileTransferContext) async throws -> any MobileExportCatalog { + let active: String + if let database = context.database, !database.isEmpty { + active = database + } else { + active = try await sessionDatabase() ?? "" + } + return MySQLTransferCatalog(driver: self, reader: exportCatalogReader(activeDatabase: active), openedDatabase: active) + } + + func makeWriteDialect(context: MobileTransferContext) async throws -> any MobileWriteDialect { + MySQLWriteDialect( + databaseTypeId: databaseType.rawValue, + isMariaDB: serverFlavor.isMariaDB, + spelling: MySQLLiteralSpelling(noBackslashEscapes: Self.noBackslashEscapes(in: settledLexicalFacts)), + database: context.database + ) + } + + static func noBackslashEscapes(in facts: SQLSessionLexicalFacts?) -> Bool? { + guard let facts, facts.determined.contains(.backslashEscapesInSingleQuotes) else { return nil } + return !facts.enabled.contains(.backslashEscapesInSingleQuotes) + } +} + +// The Mac plugin's export answers, from the shared `MySQLExportCatalog` over the driver the transfer runs on. An object's +// container is its database, as on the Mac's tree; principals list under the database the export opened on. iOS has no +// SHOW fallback for a proxy whose catalog answers nothing. +nonisolated struct MySQLTransferCatalog: MobileExportCatalog { + let driver: MySQLDriver + let reader: MySQLExportCatalog + let openedDatabase: String + private let routineLists = ExportCatalogListCache() + private let triggerLists = ExportCatalogListCache() + private let eventLists = ExportCatalogListCache() + + init(driver: MySQLDriver, reader: MySQLExportCatalog, openedDatabase: String) { + self.driver = driver + self.reader = reader + self.openedDatabase = openedDatabase + } + + var databaseTypeId: String { driver.databaseType.rawValue } + + private func schema(for databaseName: String) -> String? { + databaseName.isEmpty ? nil : databaseName + } + + func exportObjects(database: String, schema: String?) async throws -> [PluginExportTable] { + let container = database.isEmpty ? openedDatabase : database + // A MariaDB sequence is written as the table SHOW CREATE TABLE gives, as on the Mac. + let tables = try await driver.fetchTables(schema: container.isEmpty ? nil : container).map { table in + ExportCatalogItems.item( + table.name, container: container, kind: PluginExportObjectKind.from(tableType: table.type.rawValue) + ) + } + let routines = await ExportCatalogItems.listing("routines") { + ExportCatalogItems.routines(try await listedRoutines(container), container: container) + } + let triggers = await ExportCatalogItems.listing("triggers") { + ExportCatalogItems.triggers(try await listedTriggers(container), container: container) + } + let events = await ExportCatalogItems.listing("events") { + try await listedEvents(container).map { ExportCatalogItems.item($0.name, container: container, kind: .event) } + } + // Principals are server-wide, so only the database the export opened on lists them. + let principals = container == openedDatabase + ? await ExportCatalogItems.listing("principals") { + ExportCatalogItems.principals(try await reader.principals(), container: container) + } + : [] + return tables + routines + triggers + events + principals + } + + private func listedRoutines(_ container: String) async throws -> [PluginRoutineInfo] { + try await routineLists.values(for: container) { try await reader.routines(schema: container) } + } + + private func listedTriggers(_ container: String) async throws -> [PluginTriggerInfo] { + try await triggerLists.values(for: container) { try await reader.allTriggers(schema: container) } + } + + private func listedEvents(_ container: String) async throws -> [PluginEventInfo] { + try await eventLists.values(for: container) { try await reader.events(schema: container) } + } + + func quoteIdentifier(_ identifier: String) -> String { + mysqlQuoteIdentifier(identifier) + } + + // The backslash form whatever the session's mode, as the Mac plugin writes a dump. + func escapeStringLiteral(_ value: String) -> String { + mysqlEscapeStringLiteral(value) + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + guard let ddl = try await reader.tableDDL(table: table, schema: schema(for: databaseName)) else { + throw MySQLStatementFailure(code: 0, message: "Failed to fetch DDL for table '\(table)'") + } + return ddl + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { + guard reader.hasInformationSchema else { return nil } + return try await reader.approximateRowCount(table: table, schema: schema(for: databaseName)) + } + + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { + guard reader.hasInformationSchema else { return [] } + let byTable = try await reader.informationSchemaColumns(schema: schema(for: databaseName), table: table) + return byTable[table] ?? (byTable.count == 1 ? byTable.values.first ?? [] : []) + } + + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] { + guard reader.hasInformationSchema else { return [:] } + return try await reader.informationSchemaColumns(schema: schema(for: databaseName), table: nil) + } + + func fetchForeignKeys(table: String, databaseName: String) async throws -> [PluginForeignKeyInfo] { + let database = reader.effectiveSchema(schema(for: databaseName)) + guard reader.flavor.hasMySQLObjectCatalogs, reader.hasForeignKeyCatalog, !database.isEmpty else { return [] } + let byTable = try await reader.catalogForeignKeys(database: database, table: table) + return MySQLForeignKeyCatalog.keys(for: table, in: byTable) + } + + func fetchAllForeignKeys(databaseName: String) async throws -> [String: [PluginForeignKeyInfo]] { + let database = reader.effectiveSchema(schema(for: databaseName)) + guard reader.flavor.hasMySQLObjectCatalogs, reader.hasForeignKeyCatalog, !database.isEmpty else { return [:] } + return try await reader.catalogForeignKeys(database: database, table: nil) + } + + var tableDDLIncludesForeignKeys: Bool { MySQLExportCatalog.tableDDLIncludesForeignKeys } + + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + let container = schema(for: object.databaseName) + switch object.kind { + case .table, .foreignTable: + return try await fetchTableDDL(table: object.name, databaseName: object.databaseName) + case .view, .materializedView: + guard let ddl = try await reader.viewDefinition(view: object.name, schema: container) else { + throw MySQLStatementFailure(code: 0, message: "Failed to fetch definition for view '\(object.name)'") + } + return ddl + case .routine: + return try await reader.routineDDL(try await matchedRoutine(object)) + case .trigger: + return try await reader.triggerDDL(try await matchedTrigger(object)) + case .event: + return try await reader.eventDDL(try await matchedEvent(object)) + default: + throw PluginObjectSourceError.unsupported(object.name) + } + } + + func fetchCreationScope(_ object: PluginExportTable) async throws -> PluginStatementScope? { + switch object.kind { + case .routine: + return try await reader.routineCreationScope(try await matchedRoutine(object)) + case .trigger: + return try await reader.triggerCreationScope(try await matchedTrigger(object)) + case .event: + return try await reader.eventCreationScope(try await matchedEvent(object)) + default: + return nil + } + } + + private func matchedRoutine(_ object: PluginExportTable) async throws -> PluginRoutineInfo { + try ExportCatalogItems.routine(matching: object, in: try await listedRoutines(object.databaseName)) + } + + private func matchedTrigger(_ object: PluginExportTable) async throws -> PluginTriggerInfo { + try ExportCatalogItems.trigger(matching: object, in: try await listedTriggers(object.databaseName)) + } + + private func matchedEvent(_ object: PluginExportTable) async throws -> PluginEventInfo { + guard let event = try await listedEvents(object.databaseName).first(where: { $0.name == object.name }) else { + throw PluginObjectSourceError.unsupported(object.name) + } + return event + } + + func dropStatement(for object: PluginExportTable) -> String? { + let container = schema(for: object.databaseName) + switch object.kind { + case .trigger: + guard object.parentTable != nil else { return nil } + return reader.dropTriggerStatement(name: object.name, schema: container) + case .routine: + guard let kind = ExportCatalogItems.routineKind(of: object) else { return nil } + return reader.dropRoutineStatement(name: object.name, schema: container, isFunction: kind == .function) + default: + return nil + } + } + + func fetchGrantStatements(principal: String, host: String?) async throws -> [String] { + let reader = reader + return try await ExportCatalogItems.grantStatements( + principal: principal, + host: host, + fetchGrants: { try await reader.grants(for: $0, privileges: try await reader.privilegeCatalog()) }, + generate: { reader.grantStatements(for: $0.principal, grants: $0.grantsToAdd) } + ) + } + + private var hooks: MySQLExportCatalog.SessionHooks { + let driver = driver + return MySQLExportCatalog.SessionHooks( + runSetup: { sql in try await driver.runSetup(sql) }, + transactionState: { await driver.sessionTransactionState().pluginSessionState }, + adjustFootprint: { change in driver.adjustFootprint(change) } + ) + } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + try await reader.openSessionScope(for: purpose, hooks: hooks) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + try await reader.closeSessionScope(scope, hooks: hooks) + } + + private var script: ExportCatalogScript { + ExportCatalogScript(databaseTypeId: databaseTypeId, session: driver.settledLexicalFacts) + } + + var lexicalFeatures: SQLLexicalFeatures { script.lexicalFeatures } + + func scriptText(for ddl: String) -> String { + script.scriptText(for: ddl) + } +} + +// With a database the rows go to `db`.`table`, so a `USE` another window ran cannot send them elsewhere. +nonisolated struct MySQLWriteDialect: MobileWriteDialect { + let databaseTypeId: String + let isMariaDB: Bool + let spelling: MySQLLiteralSpelling + var database: String? + + var namesDatabase: Bool { + database.map { !$0.isEmpty } ?? false + } + + // The family reads BIT as a number, so a value bound to one is cast (`SQLImportStatements.bind`). + static func isBitColumn(_ typeName: String) -> Bool { + typeName.prefix { $0 != "(" }.trimmingCharacters(in: .whitespaces).uppercased() == "BIT" + } + + func importStatements(table: String, schema: String?, columns: [ColumnInfo]) async throws -> SQLImportStatements { + SQLImportStatements( + databaseTypeId: databaseTypeId, + qualifiedTableName: ImportTableNames.qualified( + table, schema: schema ?? database, quote: { mysqlQuoteIdentifier($0) } + ), + parameterStyle: .questionMark, + quoteIdentifier: { mysqlQuoteIdentifier($0) }, + columnTypeNames: ImportTableNames.columnTypeNames(columns), + numberBitColumns: Set(columns.filter { Self.isBitColumn($0.typeName) }.map(\.name)), + rendersValuesAsLiterals: true + ) + } + + // Spelled for the session, as the Mac's `generateCreateTableSQL` spells it; run it as it is. + func createTableStatements(definition: PluginCreateTableDefinition, schema: String?) throws -> [String] { + guard let statement = mysqlCreateTableSQL(definition: definition, isMariaDB: isMariaDB) else { + throw ImportTableNames.noStatement + } + return [spelling.respelled(statement)] + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Engines/OracleTransferEngine.swift b/apple/TableProMobile/TableProMobile/Transfer/Engines/OracleTransferEngine.swift new file mode 100644 index 0000000000..32379cb69e --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Engines/OracleTransferEngine.swift @@ -0,0 +1,194 @@ +import Foundation +import os +import TableProDatabase +import TableProModels +import TableProOracleCore +import TableProPluginKit +import TableProSQLQuoting + +nonisolated extension OracleDriver: CatalogQuerying, ExportRowStreaming, MobileTransferEngine { + func catalogQuery(_ sql: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + try await catalogCells(sql, binds: parameters) + } + + func makeExportCatalog(context: MobileTransferContext) async throws -> any MobileExportCatalog { + guard let release else { throw ConnectionError.notConnected } + return OracleMobileExportCatalog( + catalog: OracleExportCatalog( + run: { [self] statement in try await catalogResult(statement) }, + currentSchema: currentSchema, + fallbackSchema: fallbackSchema, + release: release, + principalState: principalState, + coreError: { $0 as? OracleCoreError } + ), + script: ExportCatalogScript(databaseTypeId: DatabaseType.oracle.rawValue, session: settledLexicalFacts) + ) + } + + // An unknown release writes the identity, as on the Mac. + func makeWriteDialect(context: MobileTransferContext) async throws -> any MobileWriteDialect { + OracleMobileWriteDialect( + databaseTypeId: DatabaseType.oracle.rawValue, + createTableSchema: currentSchema ?? fallbackSchema, + supportsIdentity: release?.hasIdentityColumns ?? true + ) + } +} + +// The Mac's Oracle export answers, triggers listed (D14). A schema is what the connection lists as a database, so +// `databaseName` is the owner the export groups an object under; empty is the session's own. +nonisolated struct OracleMobileExportCatalog: MobileExportCatalog { + let catalog: OracleExportCatalog + let script: ExportCatalogScript + private let routines = ExportCatalogListCache() + private let triggers = ExportCatalogListCache() + + init(catalog: OracleExportCatalog, script: ExportCatalogScript) { + self.catalog = catalog + self.script = script + } + + var databaseTypeId: String { script.databaseTypeId } + + private func schema(_ databaseName: String) -> String { + catalog.effectiveSchema(databaseName.nilIfEmpty) + } + + // MARK: - Listing + + func exportObjects(database: String, schema: String?) async throws -> [PluginExportTable] { + let container = catalog.effectiveSchema(schema?.nilIfEmpty ?? database.nilIfEmpty) + var items = try await catalog.run(OracleSchemaQueries.tables(schema: container)).rows + .compactMap(OracleSchemaQueries.parseTableRow) + .map { ExportCatalogItems.item($0.name, container: container, kind: $0.isView ? .view : .table) } + items += await ExportCatalogItems.listing("routines") { + ExportCatalogItems.routines(try await cachedRoutines(container), container: container) + } + items += await ExportCatalogItems.listing("triggers") { + ExportCatalogItems.triggers(try await cachedTriggers(container), container: container) + } + items += await ExportCatalogItems.listing("principals") { + ExportCatalogItems.principals(try await catalog.fetchPrincipals(), container: container) + } + return items + } + + private func cachedRoutines(_ schema: String) async throws -> [PluginRoutineInfo] { + try await routines.values(for: schema) { try await catalog.fetchRoutines(schema: schema) } + } + + private func cachedTriggers(_ schema: String) async throws -> [PluginTriggerInfo] { + try await triggers.values(for: schema) { try await catalog.fetchAllTriggers(schema: schema) } + } + + // MARK: - Text + + func quoteIdentifier(_ identifier: String) -> String { + SQLQuoting.identifier(identifier) + } + + func escapeStringLiteral(_ value: String) -> String { + SQLQuoting.stringLiteralBody(value) + } + + func scriptText(for ddl: String) -> String { + script.scriptText(for: ddl) + } + + var lexicalFeatures: SQLLexicalFeatures { + script.lexicalFeatures + } + + var tableDDLIncludesForeignKeys: Bool { OracleExportCatalog.tableDDLIncludesForeignKeys } + + // MARK: - Definitions + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + try await catalog.fetchTableDDL(table: table, schema: schema(databaseName)) + } + + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { + try await catalog.fetchColumns(table: table, schema: schema(databaseName)) + } + + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] { + try await catalog.fetchAllColumns(schema: schema(databaseName)) + } + + func fetchForeignKeys(table: String, databaseName: String) async throws -> [PluginForeignKeyInfo] { + try await catalog.fetchForeignKeys(table: table, schema: schema(databaseName)) + } + + func fetchAllForeignKeys(databaseName: String) async throws -> [String: [PluginForeignKeyInfo]] { + try await catalog.fetchAllForeignKeys(schema: schema(databaseName)) + } + + func fetchIndexDDL(table: String, databaseName: String) async throws -> [String] { + try await catalog.fetchIndexDDL(table: table, schema: schema(databaseName)) + } + + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + let schema = schema(object.databaseName) + switch object.kind { + case .table: + return try await catalog.fetchTableDDL(table: object.name, schema: schema) + case .view: + return try await catalog.fetchViewDefinition(view: object.name, schema: schema) + case .routine: + let routine = try ExportCatalogItems.routine(matching: object, in: try await cachedRoutines(schema)) + return try await catalog.fetchRoutineDDL(routine) + case .trigger: + let trigger = try ExportCatalogItems.trigger(matching: object, in: try await cachedTriggers(schema)) + return try await catalog.fetchTriggerDDL(trigger) + default: + throw PluginObjectSourceError.unsupported(object.name) + } + } + + // Guarded PL/SQL before 23ai, where the export's own `DROP TABLE IF EXISTS` does not parse. + func dropStatement(for object: PluginExportTable) -> String? { + catalog.dropStatement(for: object, schema: schema(object.databaseName)) + } + + // The roles the principal listing found, which a grant spells without a privilege keyword. + func fetchGrantStatements(principal: String, host: String?) async throws -> [String] { + try await ExportCatalogItems.grantStatements( + principal: principal, + host: host, + fetchGrants: { try await catalog.fetchGrants(for: $0) }, + generate: { + OracleExportCatalog.generateGrantSQL(changeSet: $0, roles: catalog.principalState.withLock { $0.roles }) + } + ) + } +} + +// One statement per call and one row per INSERT, the ceilings `SQLImportStatements` keys by type id. The column types +// reach `executeParameterized`, where the Mac's binder writes each value for its column. +nonisolated struct OracleMobileWriteDialect: MobileWriteDialect { + let databaseTypeId: String + let createTableSchema: String + let supportsIdentity: Bool + + func importStatements(table: String, schema: String?, columns: [ColumnInfo]) async throws -> SQLImportStatements { + SQLImportStatements( + databaseTypeId: databaseTypeId, + qualifiedTableName: ImportTableNames.qualified(table, schema: schema) { SQLQuoting.identifier($0) }, + parameterStyle: .questionMark, + quoteIdentifier: { SQLQuoting.identifier($0) }, + columnTypeNames: ImportTableNames.columnTypeNames(columns) + ) + } + + func createTableStatements(definition: PluginCreateTableDefinition, schema: String?) throws -> [String] { + guard let statements = OracleCreateTableSQL.statements( + definition: definition, + schema: schema?.nilIfEmpty ?? createTableSchema, + supportsIdentity: supportsIdentity + ) else { + throw ImportTableNames.noStatement + } + return statements + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Engines/PluginBackedTransferEngine.swift b/apple/TableProMobile/TableProMobile/Transfer/Engines/PluginBackedTransferEngine.swift new file mode 100644 index 0000000000..d8386e99ad --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Engines/PluginBackedTransferEngine.swift @@ -0,0 +1,388 @@ +import Foundation +import TableProDatabase +import TableProModels +import TableProPluginKit +import TableProSQLGrammar + +// ClickHouse, Cloudflare D1 and MongoDB run their Mac plugin whole, so an export asks that plugin what the Mac's +// `ExportDataSourceAdapter` asks it. MongoDB takes no rows from a file, as on the Mac (D9). +nonisolated extension PluginBackedDriver: MobileTransferEngine { + func makeExportCatalog(context: MobileTransferContext) async throws -> any MobileExportCatalog { + PluginTransferCatalog( + driver: self, + databaseTypeId: context.databaseType.rawValue, + groupsByDatabase: context.databaseType == .clickhouse, + switchesDatabase: context.databaseType == .mongodb, + openedDatabase: context.database?.nilIfEmpty ?? currentDatabase + ) + } + + func makeWriteDialect(context: MobileTransferContext) async throws -> any MobileWriteDialect { + guard context.databaseType != .mongodb else { + throw MobileTransferRefusal(reason: String( + format: String(localized: "%@ cannot take rows from a file."), context.databaseType.mobileDisplayName + )) + } + return PluginWriteDialect(plugin: plugin, databaseTypeId: context.databaseType.rawValue) + } +} + +// `groupsByDatabase`: ClickHouse's tree names each database, as the Mac's does; D1 and MongoDB are flat, so an object +// carries no container. `switchesDatabase`: a MongoDB export reads the collection in its own database, so the +// transfer's own driver moves there first. +nonisolated struct PluginTransferCatalog: MobileExportCatalog { + let driver: PluginBackedDriver + let databaseTypeId: String + let groupsByDatabase: Bool + let switchesDatabase: Bool + let openedDatabase: String? + private let routineLists = ExportCatalogListCache() + private let triggerLists = ExportCatalogListCache() + private let eventLists = ExportCatalogListCache() + private let sequenceLists = ExportCatalogListCache() + private let userTypeLists = ExportCatalogListCache() + + init( + driver: PluginBackedDriver, + databaseTypeId: String, + groupsByDatabase: Bool, + switchesDatabase: Bool, + openedDatabase: String? + ) { + self.driver = driver + self.databaseTypeId = databaseTypeId + self.groupsByDatabase = groupsByDatabase + self.switchesDatabase = switchesDatabase + self.openedDatabase = openedDatabase + } + + private var plugin: any PluginDatabaseDriver { driver.plugin } + + // The adapter's `exportSchema(for:)`: an empty name is the driver's own container. + private func exportSchema(for databaseName: String) -> String? { + guard databaseName.isEmpty else { return databaseName } + return plugin.currentSchema + } + + // The database a flat object's rows and definition are read in. + private func database(of databaseName: String) -> String? { + databaseName.nilIfEmpty ?? openedDatabase + } + + private func moveIfNeeded(to database: String?) async throws { + guard switchesDatabase, let database, driver.currentDatabase != database else { return } + try await driver.switchDatabase(to: database) + } + + func exportObjects(database: String, schema: String?) async throws -> [PluginExportTable] { + let container: String + if groupsByDatabase { + container = database.nilIfEmpty ?? openedDatabase ?? "" + } else { + try await moveIfNeeded(to: self.database(of: database)) + container = database == openedDatabase ? "" : database + } + let readSchema = listSchema(container) + let plugin = plugin + let tables = try await driver.inSessionDatabase { try await plugin.fetchTables(schema: readSchema) } + var objects = tables.map { table in + ExportCatalogItems.item(table.name, container: container, kind: PluginExportObjectKind.from(tableType: table.type)) + } + objects += await ExportCatalogItems.listing("routines") { + ExportCatalogItems.routines(try await listedRoutines(container), container: container) + } + objects += await ExportCatalogItems.listing("triggers") { + ExportCatalogItems.triggers(try await listedTriggers(container), container: container) + } + objects += await ExportCatalogItems.listing("events") { + try await listedEvents(container).map { ExportCatalogItems.item($0.name, container: container, kind: .event) } + } + // The listing reported a sequence as a table too; one relation, written once as a sequence. + let sequences = await ExportCatalogItems.listing("sequences") { + try await listedSequences(container).map { + ExportCatalogItems.item($0.name, container: container, kind: .sequence) + } + } + let sequenceNames = Set(sequences.map(\.name)) + objects.removeAll { $0.kind == .table && sequenceNames.contains($0.name) } + objects += sequences + objects += await ExportCatalogItems.listing("user types") { + try await listedUserTypes(container).map { + ExportCatalogItems.item($0.name, container: container, kind: .userType) + } + } + // Principals are server-wide, so only the container the export opened on lists them. + guard container.isEmpty || container == openedDatabase, + let management = plugin as? any PluginPrincipalManagement else { return objects } + objects += await ExportCatalogItems.listing("principals") { + ExportCatalogItems.principals(try await management.fetchPrincipals(), container: container) + } + return objects + } + + // A driver's own export statement (MongoDB's `find`) streams typed documents for MQL, as the Mac's adapter streams + // them; a SQL engine's rows come through the driver's `exportRows`. + func streamExportRows(for object: PluginExportTable) -> AsyncThrowingStream? { + guard let query = plugin.defaultExportQuery(table: object.name, schema: exportSchema(for: object.databaseName)) else { + return nil + } + guard object.rowScope.isUnrestricted else { + let reason = String( + format: String(localized: "%@ cannot be narrowed to some rows or columns, because this database has no SQL."), + object.name + ) + return AsyncThrowingStream { $0.finish(throwing: PluginExportError.exportFailed(reason)) } + } + var readOnlyContext = PluginStatementContext() + readOnlyContext.readOnly = true + let context = readOnlyContext + let plugin = plugin + let target = database(of: object.databaseName) + let pull = PluginExportPull { [self] in + try await moveIfNeeded(to: target) + return plugin.streamExportRows(query: query, context: context) + } + return AsyncThrowingStream(unfolding: { try await pull.next() }) + } + + func quoteIdentifier(_ identifier: String) -> String { + plugin.quoteIdentifier(identifier) + } + + func escapeStringLiteral(_ value: String) -> String { + plugin.escapeStringLiteral(value) + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + try await moveIfNeeded(to: database(of: databaseName)) + return try await plugin.fetchTableDDL(table: table, schema: exportSchema(for: databaseName)) + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { + try await plugin.fetchApproximateRowCount(table: table, schema: exportSchema(for: databaseName)) + } + + func fetchDependentSequences(table: String, databaseName: String) async throws -> [PluginSequenceInfo] { + try await plugin.fetchDependentSequences(table: table, schema: exportSchema(for: databaseName)) + .map { PluginSequenceInfo(name: $0.name, ddl: $0.ddl) } + } + + func fetchDependentTypes(table: String, databaseName: String) async throws -> [PluginEnumTypeInfo] { + try await plugin.fetchDependentTypes(table: table, schema: exportSchema(for: databaseName)) + .map { PluginEnumTypeInfo(name: $0.name, labels: $0.labels) } + } + + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { + try await plugin.fetchColumns(table: table, schema: exportSchema(for: databaseName)) + } + + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] { + try await plugin.fetchAllColumns(schema: exportSchema(for: databaseName)) + } + + func fetchForeignKeys(table: String, databaseName: String) async throws -> [PluginForeignKeyInfo] { + try await plugin.fetchForeignKeys(table: table, schema: exportSchema(for: databaseName)) + } + + func fetchAllForeignKeys(databaseName: String) async throws -> [String: [PluginForeignKeyInfo]] { + try await plugin.fetchAllForeignKeys(schema: exportSchema(for: databaseName)) + } + + var tableDDLIncludesForeignKeys: Bool { plugin.tableDDLIncludesForeignKeys } + + func fetchIndexDDL(table: String, databaseName: String) async throws -> [String] { + try await plugin.fetchIndexDDL(table: table, schema: exportSchema(for: databaseName)) + } + + func fetchCommentDDL(table: String, databaseName: String) async throws -> [String] { + try await plugin.fetchCommentDDL(table: table, schema: exportSchema(for: databaseName)) + } + + func fetchIndexes(table: String, databaseName: String) async throws -> [PluginIndexInfo] { + try await plugin.fetchIndexes(table: table, schema: exportSchema(for: databaseName)) + } + + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + let schema = exportSchema(for: object.databaseName) + switch object.kind { + case .table, .foreignTable: + return try await fetchTableDDL(table: object.name, databaseName: object.databaseName) + case .view, .materializedView: + return try await plugin.fetchViewDefinition(view: object.name, schema: schema) + case .routine: + return try await plugin.fetchRoutineDDL(try await matchedRoutine(object)) + case .trigger: + return try await plugin.fetchTriggerDDL(try await matchedTrigger(object)) + case .event: + return try await plugin.fetchEventDDL(try await matchedEvent(object)) + case .sequence: + guard let sequence = try await listedSequences(object.databaseName).first(where: { $0.name == object.name }) else { + throw PluginObjectSourceError.unsupported(object.name) + } + return sequence.ddl + case .userType: + guard let type = try await listedUserTypes(object.databaseName).first(where: { $0.name == object.name }) else { + throw PluginObjectSourceError.unsupported(object.name) + } + let resolved = try await plugin.fetchUserDefinedType(type) + guard let definition = resolved.definition, !definition.isEmpty else { + throw PluginObjectSourceError.unsupported(object.name) + } + return definition + default: + throw PluginObjectSourceError.unsupported(object.name) + } + } + + func fetchCreationScope(_ object: PluginExportTable) async throws -> PluginStatementScope? { + switch object.kind { + case .routine: + return try await plugin.fetchRoutineCreationScope(try await matchedRoutine(object)) + case .trigger: + return try await plugin.fetchTriggerCreationScope(try await matchedTrigger(object)) + case .event: + return try await plugin.fetchEventCreationScope(try await matchedEvent(object)) + default: + return nil + } + } + + // What the listing passes as `schema:`: nothing for the database the export opened on, as the Mac's loader reads it. + private func listSchema(_ container: String) -> String? { + groupsByDatabase && container != openedDatabase ? container.nilIfEmpty : nil + } + + private func listedRoutines(_ container: String) async throws -> [PluginRoutineInfo] { + let (plugin, schema) = (plugin, listSchema(container)) + return try await routineLists.values(for: container) { try await plugin.fetchRoutines(schema: schema) } + } + + private func listedTriggers(_ container: String) async throws -> [PluginTriggerInfo] { + let (plugin, schema) = (plugin, listSchema(container)) + return try await triggerLists.values(for: container) { try await plugin.fetchAllTriggers(schema: schema) } + } + + private func listedEvents(_ container: String) async throws -> [PluginEventInfo] { + let (plugin, schema) = (plugin, listSchema(container)) + return try await eventLists.values(for: container) { try await plugin.fetchEvents(schema: schema) } + } + + private func listedSequences(_ container: String) async throws -> [PluginSequenceInfo] { + let (plugin, schema) = (plugin, listSchema(container)) + return try await sequenceLists.values(for: container) { try await plugin.fetchSequences(schema: schema) } + } + + private func listedUserTypes(_ container: String) async throws -> [PluginUserDefinedTypeInfo] { + let (plugin, schema) = (plugin, listSchema(container)) + return try await userTypeLists.values(for: container) { try await plugin.fetchUserDefinedTypes(schema: schema) } + } + + private func matchedRoutine(_ object: PluginExportTable) async throws -> PluginRoutineInfo { + try ExportCatalogItems.routine(matching: object, in: try await listedRoutines(object.databaseName)) + } + + private func matchedTrigger(_ object: PluginExportTable) async throws -> PluginTriggerInfo { + try ExportCatalogItems.trigger(matching: object, in: try await listedTriggers(object.databaseName)) + } + + private func matchedEvent(_ object: PluginExportTable) async throws -> PluginEventInfo { + guard let event = try await listedEvents(object.databaseName).first(where: { $0.name == object.name }) else { + throw PluginObjectSourceError.unsupported(object.name) + } + return event + } + + func dropStatement(for object: PluginExportTable) -> String? { + let schema = exportSchema(for: object.databaseName) + switch object.kind { + case .trigger: + guard let parent = object.parentTable else { return nil } + return plugin.generateDropTriggerSQL(name: object.name, table: parent, schema: schema) + case .routine: + guard let kind = ExportCatalogItems.routineKind(of: object) else { return nil } + return plugin.generateDropRoutineSQL( + name: object.name, signature: object.identity, schema: schema, isFunction: kind == .function + ) + default: + return nil + } + } + + func fetchGrantStatements(principal: String, host: String?) async throws -> [String] { + guard let management = plugin as? any PluginPrincipalManagement else { return [] } + return try await ExportCatalogItems.grantStatements( + principal: principal, + host: host, + fetchGrants: { try await management.fetchGrants(for: $0) }, + generate: { management.generateGrantSQL(changeSet: $0) } + ) + } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + try await plugin.openSessionScope(for: purpose) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + try await plugin.closeSessionScope(scope) + } + + private var script: ExportCatalogScript { + ExportCatalogScript(databaseTypeId: databaseTypeId, session: driver.settledLexicalFacts) + } + + var lexicalFeatures: SQLLexicalFeatures { script.lexicalFeatures } + + func scriptText(for ddl: String) -> String { + script.scriptText(for: ddl) + } +} + +// Opens the plugin's stream on the first pull, after the move it waits on, and reads it one element per pull. +nonisolated private final class PluginExportPull: @unchecked Sendable { + private let open: @Sendable () async throws -> AsyncThrowingStream + private var iterator: AsyncThrowingStream.AsyncIterator? + + init(open: @escaping @Sendable () async throws -> AsyncThrowingStream) { + self.open = open + } + + func next() async throws -> PluginExportStreamElement? { + if iterator == nil { + iterator = try await open().makeAsyncIterator() + } + return try await iterator?.next() + } +} + +nonisolated struct PluginWriteDialect: MobileWriteDialect { + let plugin: any PluginDatabaseDriver + let databaseTypeId: String + + func importStatements(table: String, schema: String?, columns: [ColumnInfo]) async throws -> SQLImportStatements { + let plugin = plugin + let quote: (String) -> String = { plugin.quoteIdentifier($0) } + return SQLImportStatements( + databaseTypeId: databaseTypeId, + qualifiedTableName: ImportTableNames.qualified(table, schema: schema, quote: quote), + parameterStyle: plugin.parameterStyle, + quoteIdentifier: quote, + columnTypeNames: ImportTableNames.columnTypeNames(columns), + rowUpdateStatement: { plugin.rowUpdateStatement(table: $0, assignments: $1, predicate: $2) }, + rowDeleteStatement: { plugin.rowDeleteStatement(table: $0, predicate: $1) } + ) + } + + // ClickHouse's sorting, primary and partition key columns. + func columnsUpdateCannotSet(table: String, schema: String?) async throws -> Set { + try await plugin.fetchImmutableColumns(table: table, schema: schema) + } + + func createTableStatements(definition: PluginCreateTableDefinition, schema: String?) throws -> [String] { + guard let statements = plugin.generateCreateTableStatements(definition: definition, schema: schema), + !statements.isEmpty else { + throw ImportTableNames.noStatement + } + return statements + } + +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Engines/PostgreSQLExportCatalogs.swift b/apple/TableProMobile/TableProMobile/Transfer/Engines/PostgreSQLExportCatalogs.swift new file mode 100644 index 0000000000..1290d51149 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Engines/PostgreSQLExportCatalogs.swift @@ -0,0 +1,317 @@ +import Foundation +import os +import TableProPluginKit +import TableProPostgreSQLCore +import TableProSQLQuoting + +// The Mac's PostgreSQL export answers, over the iOS session's raw reads. `databaseName` is the schema the export groups +// an object under; empty is the session's own. +nonisolated struct PostgreSQLMobileExportCatalog: MobileExportCatalog { + private static let logger = Logger(subsystem: "com.TablePro", category: "PostgreSQLExportCatalog") + + let catalog: PostgreSQLCatalogExport + let sessionScope: PostgreSQLSessionScope? + let script: ExportCatalogScript + let standardConformingStrings: Bool + private let routines = ExportCatalogListCache() + private let triggers = ExportCatalogListCache() + private let sequences = ExportCatalogListCache() + private let userTypes = ExportCatalogListCache() + + init( + catalog: PostgreSQLCatalogExport, + sessionScope: PostgreSQLSessionScope?, + script: ExportCatalogScript, + standardConformingStrings: Bool + ) { + self.catalog = catalog + self.sessionScope = sessionScope + self.script = script + self.standardConformingStrings = standardConformingStrings + } + + var databaseTypeId: String { script.databaseTypeId } + + private func schema(_ databaseName: String) -> String { + databaseName.isEmpty ? catalog.currentSchema : databaseName + } + + // MARK: - Listing + + func exportObjects(database: String, schema: String?) async throws -> [PluginExportTable] { + let container = schema?.nilIfEmpty ?? catalog.currentSchema + var items = try await relations(in: container) + items += await ExportCatalogItems.listing("routines") { + ExportCatalogItems.routines(try await cachedRoutines(container), container: container) + } + items += await ExportCatalogItems.listing("triggers") { + ExportCatalogItems.triggers(try await cachedTriggers(container), container: container) + } + items += await ExportCatalogItems.listing("sequences") { + try await cachedSequences(container).map { ExportCatalogItems.item($0.name, container: container, kind: .sequence) } + } + items += await ExportCatalogItems.listing("user-defined types") { + try await cachedUserTypes(container).map { ExportCatalogItems.item($0.name, container: container, kind: .userType) } + } + items += await ExportCatalogItems.listing("principals") { + ExportCatalogItems.principals(try await catalog.fetchPrincipals(), container: container) + } + return items + } + + // Partition-aware, so a partition is written once, inside its parent's definition. A server that refuses the + // catalog form is listed from information_schema, as the iOS sidebar falls back. + private func relations(in schema: String) async throws -> [PluginExportTable] { + let listing = PostgreSQLTableListing.query( + schema: schema, + includeMaterializedViews: catalog.includesMaterializedViews, + includeForeignTables: catalog.catalogPresence?.hasForeignTables ?? catalog.capabilities.hasForeignTablesCatalog, + includeComments: false, + includePartitionAwareness: true + ) + let rows: [[PluginCellValue]] + do { + rows = try await catalog.read(listing, nil) + } catch is CancellationError { + throw CancellationError() + } catch { + Self.logger.debug("Export listing fell back to information_schema: \(error.localizedDescription, privacy: .private)") + rows = try await catalog.read(PostgreSQLTableListing.informationSchemaQuery(in: .schema(schema)), nil) + } + return rows + .compactMap { PostgreSQLTableListing.table(fromRow: $0.map(\.asText)) } + .compactMap { ExportCatalogItems.relation($0, container: schema) } + } + + private func cachedRoutines(_ schema: String) async throws -> [PluginRoutineInfo] { + try await routines.values(for: schema) { try await catalog.fetchRoutines(schema: schema) } + } + + private func cachedTriggers(_ schema: String) async throws -> [PluginTriggerInfo] { + try await triggers.values(for: schema) { try await catalog.fetchAllTriggers(schema: schema) } + } + + private func cachedSequences(_ schema: String) async throws -> [PluginSequenceInfo] { + try await sequences.values(for: schema) { try await catalog.fetchSequences(schema: schema) } + } + + private func cachedUserTypes(_ schema: String) async throws -> [PluginUserDefinedTypeInfo] { + try await userTypes.values(for: schema) { try await catalog.fetchUserDefinedTypes(schema: schema) } + } + + // MARK: - Text + + func quoteIdentifier(_ identifier: String) -> String { + SQLQuoting.identifier(identifier) + } + + func escapeStringLiteral(_ value: String) -> String { + LibPQStringConformance.escape(value, standardConformingStrings: standardConformingStrings) + } + + func scriptText(for ddl: String) -> String { + script.scriptText(for: ddl) + } + + var lexicalFeatures: SQLLexicalFeatures { + script.lexicalFeatures + } + + var supportsCascadeDrop: Bool { true } + + var tableDDLIncludesForeignKeys: Bool { PostgreSQLCatalogExport.tableDDLIncludesForeignKeys } + + // MARK: - Definitions + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + try await catalog.fetchTableDDL(table: table, schema: schema(databaseName)) + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { + try await catalog.fetchApproximateRowCount(table: table, schema: schema(databaseName)) + } + + func fetchDependentSequences(table: String, databaseName: String) async throws -> [PluginSequenceInfo] { + try await catalog.fetchDependentSequences(table: table, schema: schema(databaseName)) + .map { PluginSequenceInfo(name: $0.name, ddl: $0.ddl) } + } + + func fetchDependentTypes(table: String, databaseName: String) async throws -> [PluginEnumTypeInfo] { + try await catalog.fetchDependentTypes(table: table, schema: schema(databaseName)) + .map { PluginEnumTypeInfo(name: $0.name, labels: $0.labels) } + } + + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { + try await catalog.fetchColumns(table: table, schema: schema(databaseName)) + } + + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] { + try await catalog.fetchAllColumns(schema: schema(databaseName)) + } + + func fetchForeignKeys(table: String, databaseName: String) async throws -> [PluginForeignKeyInfo] { + try await catalog.fetchForeignKeys(table: table, schema: schema(databaseName)) + } + + func fetchAllForeignKeys(databaseName: String) async throws -> [String: [PluginForeignKeyInfo]] { + try await catalog.fetchAllForeignKeys(schema: schema(databaseName)) + } + + func fetchIndexDDL(table: String, databaseName: String) async throws -> [String] { + try await catalog.fetchIndexDDL(table: table, schema: schema(databaseName)) + } + + func fetchCommentDDL(table: String, databaseName: String) async throws -> [String] { + try await catalog.fetchCommentDDL(table: table, schema: schema(databaseName)) + } + + // A materialized view never falls back to the table DDL, which would restore an empty table in its place. + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + let schema = schema(object.databaseName) + switch object.kind { + case .table, .foreignTable: + return try await catalog.fetchTableDDL(table: object.name, schema: schema) + case .view, .materializedView: + return try await catalog.fetchViewDefinition(view: object.name, schema: schema) + case .routine: + let routine = try ExportCatalogItems.routine(matching: object, in: try await cachedRoutines(schema)) + return try await catalog.fetchRoutineDDL(routine) + case .trigger: + let trigger = try ExportCatalogItems.trigger(matching: object, in: try await cachedTriggers(schema)) + return try await catalog.fetchTriggerDDL(trigger) + case .sequence: + guard let sequence = try await cachedSequences(schema).first(where: { $0.name == object.name }) else { + throw PluginObjectSourceError.unsupported(object.name) + } + return sequence.ddl + case .userType: + guard let type = try await cachedUserTypes(schema).first(where: { $0.name == object.name }), + let definition = try await catalog.fetchUserDefinedType(type).definition, + !definition.isEmpty + else { + throw PluginObjectSourceError.unsupported(object.name) + } + return definition + default: + throw PluginObjectSourceError.unsupported(object.name) + } + } + + func dropStatement(for object: PluginExportTable) -> String? { + let schema = schema(object.databaseName) + switch object.kind { + case .trigger: + guard let parent = object.parentTable else { return nil } + return catalog.generateDropTriggerSQL(name: object.name, table: parent, schema: schema) + case .routine: + guard let kind = ExportCatalogItems.routineKind(of: object) else { return nil } + return catalog.generateDropRoutineSQL( + name: object.name, signature: object.identity, schema: schema, isFunction: kind == .function + ) + default: + return nil + } + } + + func fetchGrantStatements(principal: String, host: String?) async throws -> [String] { + try await ExportCatalogItems.grantStatements( + principal: principal, + host: host, + fetchGrants: { try await catalog.fetchGrants(for: $0) }, + generate: { PostgreSQLCatalogExport.generateGrantSQL(changeSet: $0) } + ) + } + + // MARK: - Session + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + try await sessionScope?.open(for: purpose) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + try await sessionScope?.close(scope) + } +} + +// Redshift lists tables and views, external ones included, and exports nothing else, as the Mac's Redshift driver. +nonisolated struct RedshiftMobileExportCatalog: MobileExportCatalog { + let catalog: RedshiftCatalogExport + let script: ExportCatalogScript + let standardConformingStrings: Bool + + var databaseTypeId: String { script.databaseTypeId } + + private func schema(_ databaseName: String) -> String { + databaseName.isEmpty ? catalog.currentSchema : databaseName + } + + func exportObjects(database: String, schema: String?) async throws -> [PluginExportTable] { + let container = schema?.nilIfEmpty ?? catalog.currentSchema + return try await catalog.fetchTables(schema: container) + .compactMap { ExportCatalogItems.relation($0, container: container) } + } + + func quoteIdentifier(_ identifier: String) -> String { + SQLQuoting.identifier(identifier) + } + + func escapeStringLiteral(_ value: String) -> String { + LibPQStringConformance.escape(value, standardConformingStrings: standardConformingStrings) + } + + func scriptText(for ddl: String) -> String { + script.scriptText(for: ddl) + } + + var lexicalFeatures: SQLLexicalFeatures { + script.lexicalFeatures + } + + var supportsCascadeDrop: Bool { true } + + var tableDDLIncludesForeignKeys: Bool { RedshiftCatalogExport.tableDDLIncludesForeignKeys } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + try await catalog.fetchTableDDL(table: table, schema: schema(databaseName)) + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { + try await catalog.fetchApproximateRowCount(table: table, schema: schema(databaseName)) + } + + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { + try await catalog.fetchColumns(table: table, schema: schema(databaseName)) + } + + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] { + try await catalog.fetchAllColumns(schema: schema(databaseName)) + } + + func fetchForeignKeys(table: String, databaseName: String) async throws -> [PluginForeignKeyInfo] { + try await catalog.fetchForeignKeys(table: table, schema: schema(databaseName)) + } + + // The Mac's Redshift driver has no bulk read, so PluginKit's default asks table by table. + func fetchAllForeignKeys(databaseName: String) async throws -> [String: [PluginForeignKeyInfo]] { + let schema = schema(databaseName) + var grouped: [String: [PluginForeignKeyInfo]] = [:] + for table in try await catalog.fetchTables(schema: schema) { + try Task.checkCancellation() + let foreignKeys = try await catalog.fetchForeignKeys(table: table.name, schema: schema) + if !foreignKeys.isEmpty { grouped[table.name] = foreignKeys } + } + return grouped + } + + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + let schema = schema(object.databaseName) + switch object.kind { + case .table, .foreignTable: + return try await catalog.fetchTableDDL(table: object.name, schema: schema) + case .view, .materializedView: + return try await catalog.fetchViewDefinition(view: object.name, schema: schema) + default: + throw PluginObjectSourceError.unsupported(object.name) + } + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Engines/PostgreSQLTransferEngine.swift b/apple/TableProMobile/TableProMobile/Transfer/Engines/PostgreSQLTransferEngine.swift new file mode 100644 index 0000000000..f429581168 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Engines/PostgreSQLTransferEngine.swift @@ -0,0 +1,115 @@ +import Foundation +import os +import TableProDatabase +import TableProModels +import TableProPluginKit +import TableProPostgreSQLCore +import TableProSQLGrammar +import TableProSQLQuoting + +// A driver that takes the rows of a script's `COPY ... FROM STDIN`, handed over a piece at a time. +nonisolated protocol CopyFromStandardInputDriving: DatabaseDriver { + func copyFromStandardInput(statement: String, rows: PluginCopyData) async throws -> QueryResult +} + +nonisolated extension PostgreSQLDriver: CatalogQuerying, ExportRowStreaming, CopyFromStandardInputDriving, + MobileTransferEngine { + private static let transferLogger = Logger(subsystem: "com.TablePro", category: "PostgreSQLTransfer") + + func catalogQuery(_ sql: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + try await catalogCells(sql, parameters: parameters) + } + + func makeExportCatalog(context: MobileTransferContext) async throws -> any MobileExportCatalog { + let script = ExportCatalogScript(databaseTypeId: databaseType.rawValue, session: settledLexicalFacts) + let standardConformingStrings = !(settledLexicalFacts?.enabled.contains(.backslashEscapesInSingleQuotes) ?? false) + guard databaseType != .redshift else { + let read: RedshiftCatalogExport.Read = { [self] statement in try await catalogCells(statement).rows } + return RedshiftMobileExportCatalog( + catalog: RedshiftCatalogExport( + read: read, + currentSchema: effectiveSchema, + database: connectedDatabase, + serverVersionNumber: serverVersionNumber, + externalSchemas: try await Self.externalSchemas(read: read) + ), + script: script, + standardConformingStrings: standardConformingStrings + ) + } + let catalog = PostgreSQLCatalogExport( + read: { [self] statement, schema in try await catalogCells(statement, namesRelativeTo: schema).rows }, + currentSchema: effectiveSchema, + capabilities: PostgreSQLCapabilities(serverVersion: serverVersionNumber), + catalogCapabilities: .assumingModernWhenUnknown(serverVersionNumber), + catalogPresence: catalogPresence + ) + let sessionScope = PostgreSQLSessionScope( + setting: { [self] statement, requiresIdleSession in + try await executeSessionSetting( + statement.sql, parameters: statement.parameters, requiresIdleSession: requiresIdleSession + ) + }, + transactionState: { [self] in await sessionTransactionState().pluginSessionState } + ) + return PostgreSQLMobileExportCatalog( + catalog: catalog, + sessionScope: sessionScope, + script: script, + standardConformingStrings: standardConformingStrings + ) + } + + func makeWriteDialect(context: MobileTransferContext) async throws -> any MobileWriteDialect { + PostgreSQLMobileWriteDialect( + databaseTypeId: databaseType.rawValue, + currentSchema: effectiveSchema, + capabilities: PostgreSQLCapabilities(serverVersion: serverVersionNumber), + buildsTables: databaseType != .redshift + ) + } + + // A failed probe lists only the local tables of a schema, as on the Mac. + private static func externalSchemas(read: @escaping RedshiftCatalogExport.Read) async throws -> Set { + do { + return try await RedshiftCatalogExport.externalSchemaNames(read: read) + } catch is CancellationError { + throw CancellationError() + } catch { + transferLogger.warning("Could not read svv_external_schemas: \(error.localizedDescription, privacy: .private)") + return [] + } + } +} + +// Placeholders, quoting and the CREATE TABLE writer the Mac's PostgreSQL driver uses. Redshift has no writer on the +// Mac, so a new table is refused there too. +nonisolated struct PostgreSQLMobileWriteDialect: MobileWriteDialect { + let databaseTypeId: String + let currentSchema: String + let capabilities: PostgreSQLCapabilities + let buildsTables: Bool + + func importStatements(table: String, schema: String?, columns: [ColumnInfo]) async throws -> SQLImportStatements { + SQLImportStatements( + databaseTypeId: databaseTypeId, + qualifiedTableName: ImportTableNames.qualified(table, schema: schema) { SQLQuoting.identifier($0) }, + parameterStyle: .dollar, + quoteIdentifier: { SQLQuoting.identifier($0) }, + columnTypeNames: ImportTableNames.columnTypeNames(columns) + ) + } + + func createTableStatements(definition: PluginCreateTableDefinition, schema: String?) throws -> [String] { + guard buildsTables else { throw ImportTableNames.noStatement } + if let reason = PostgreSQLVersionedStatements.refusal(for: definition, capabilities: capabilities) { + throw MobileTransferRefusal(reason: reason) + } + guard let statements = PostgreSQLCreateTableSQL.statements( + definition, schema: schema?.nilIfEmpty ?? currentSchema, capabilities: capabilities + ) else { + throw ImportTableNames.noStatement + } + return statements + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Engines/RedisTransferEngine.swift b/apple/TableProMobile/TableProMobile/Transfer/Engines/RedisTransferEngine.swift new file mode 100644 index 0000000000..da8670eabb --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Engines/RedisTransferEngine.swift @@ -0,0 +1,57 @@ +import Foundation +import TableProDatabase +import TableProModels +import TableProPluginKit +import TableProRedisCore + +// Rows only, for CSV, JSON, XLSX and Markdown: SQL and MQL leave Redis out, as on the Mac. No import. +nonisolated extension RedisDriver: MobileTransferEngine { + func makeExportCatalog(context: MobileTransferContext) async throws -> any MobileExportCatalog { + RedisTransferCatalog(driver: self) + } + + // The list shows keys, but the export's objects are whole databases, as on the Mac: Export Database lists them. + var exportsListedTables: Bool { false } +} + +// Each database is one object, read whole with the Mac's `KEYBROWSE DB n` export statement, as the Mac's flat tree +// lists them. +nonisolated struct RedisTransferCatalog: MobileExportCatalog { + let driver: RedisDriver + + var databaseTypeId: String { DatabaseType.redis.rawValue } + + func exportObjects(database: String, schema: String?) async throws -> [PluginExportTable] { + try await driver.fetchDatabases().map { name in + PluginExportTable( + name: name, databaseName: "", tableType: PluginExportObjectKind.table.rawValue, schema: nil, kind: .table + ) + } + } + + func rowsQuery(for object: PluginExportTable) -> String? { + RedisQueryBuilder().buildExportQuery(database: RedisDatabaseIndex.parse(object.name)) + } + + func quoteIdentifier(_ identifier: String) -> String { + identifier + } + + func escapeStringLiteral(_ value: String) -> String { + driver.escapeStringLiteral(value) + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + throw PluginObjectSourceError.unsupported(table) + } + + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { + [ + PluginColumnInfo(name: "Key", dataType: "String", isNullable: false, isPrimaryKey: true), + PluginColumnInfo(name: "Type", dataType: "String", isNullable: true), + PluginColumnInfo(name: "TTL", dataType: "Int64", isNullable: true), + PluginColumnInfo(name: "Length", dataType: "Int64", isNullable: true, isGenerated: true), + PluginColumnInfo(name: "Value", dataType: "String", isNullable: true), + ] + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Engines/SQLiteTransferEngine.swift b/apple/TableProMobile/TableProMobile/Transfer/Engines/SQLiteTransferEngine.swift new file mode 100644 index 0000000000..86d5b103ab --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Engines/SQLiteTransferEngine.swift @@ -0,0 +1,140 @@ +import Foundation +import TableProDatabase +import TableProModels +import TableProPluginKit +import TableProSQLGrammar +import TableProSQLQuoting + +nonisolated extension SQLiteDriver: MobileTransferEngine { + func makeExportCatalog(context: MobileTransferContext) async throws -> any MobileExportCatalog { + SQLiteTransferCatalog(driver: self) + } + + func makeWriteDialect(context: MobileTransferContext) async throws -> any MobileWriteDialect { + SQLiteWriteDialect() + } +} + +// The Mac plugin's export answers (`SQLitePlugin`), read through the driver the transfer runs on. iOS attaches no +// databases, so every read is of `main` and an object carries no container, as on the Mac's flat tree. +nonisolated struct SQLiteTransferCatalog: MobileExportCatalog { + let driver: any CatalogQuerying + + var databaseTypeId: String { sqliteDatabaseTypeId } + + private var run: SQLiteCatalogReads.Run { + { [driver] sql in try await driver.catalogQuery(sql) } + } + + func exportObjects(database: String, schema: String?) async throws -> [PluginExportTable] { + let listed = try await run(""" + SELECT name, type FROM \(SQLiteMasterQueries.catalogTable(schema: nil)) + WHERE type IN ('table', 'view') + AND name NOT GLOB 'sqlite_*' + ORDER BY name + """) + let objects = listed.rows.compactMap { row -> PluginExportTable? in + guard let name = row[safe: 0]?.asText else { return nil } + let kind: PluginExportObjectKind = row[safe: 1]?.asText?.lowercased() == "view" ? .view : .table + return ExportCatalogItems.item(name, container: "", kind: kind) + } + let run = run + return objects + (await ExportCatalogItems.listing("triggers") { + ExportCatalogItems.triggers(try await SQLiteCatalogReads.triggers(table: nil, schema: nil, run: run), container: "") + }) + } + + func quoteIdentifier(_ identifier: String) -> String { + sqliteQuoteIdentifier(identifier) + } + + // The driver's own body, so a NUL in a value lands in the dump as char(0) rather than being dropped. + func escapeStringLiteral(_ value: String) -> String { + SQLiteDriver.literalBody(value) + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + try await SQLiteCatalogReads.tableDDL(table: table, schema: nil, run: run) + } + + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { + try await SQLiteCatalogReads.columns(table: table, schema: nil, run: run) + } + + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] { + try await SQLiteCatalogReads.allColumns(schema: nil, run: run) + } + + func fetchForeignKeys(table: String, databaseName: String) async throws -> [PluginForeignKeyInfo] { + let result = try await run(SQLiteForeignKeyGrouping.foreignKeysQuery(table: table, schema: nil)) + return SQLiteForeignKeyGrouping.foreignKeys(table: table, schema: nil, fromRows: result.rows) + } + + func fetchAllForeignKeys(databaseName: String) async throws -> [String: [PluginForeignKeyInfo]] { + try await SQLiteCatalogReads.allForeignKeys(schema: nil, run: run) + } + + var tableDDLIncludesForeignKeys: Bool { SQLiteCatalogReads.tableDDLIncludesForeignKeys } + + func fetchIndexDDL(table: String, databaseName: String) async throws -> [String] { + try await SQLiteCatalogReads.indexDDL(table: table, schema: nil, run: run) + } + + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + switch object.kind { + case .table: + return try await SQLiteCatalogReads.tableDDL(table: object.name, schema: nil, run: run) + case .view: + return try await SQLiteCatalogReads.viewDefinition(view: object.name, schema: nil, run: run) + case .trigger: + let triggers = try await SQLiteCatalogReads.triggers(table: object.parentTable, schema: nil, run: run) + return try await SQLiteCatalogReads.triggerDDL( + try ExportCatalogItems.trigger(matching: object, in: triggers), run: run + ) + default: + throw PluginObjectSourceError.unsupported(object.name) + } + } + + func dropStatement(for object: PluginExportTable) -> String? { + guard object.kind == .trigger, object.parentTable != nil else { return nil } + return SQLiteCatalogReads.dropTriggerStatement(name: object.name, schema: nil) + } + + private var script: ExportCatalogScript { + ExportCatalogScript(databaseTypeId: databaseTypeId, session: nil) + } + + var lexicalFeatures: SQLLexicalFeatures { script.lexicalFeatures } + + func scriptText(for ddl: String) -> String { + script.scriptText(for: ddl) + } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + try await SQLiteSessionScope.open(for: purpose, execute: run) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + try await SQLiteSessionScope.close(scope, execute: run) + } +} + +nonisolated struct SQLiteWriteDialect: MobileWriteDialect { + var databaseTypeId: String { sqliteDatabaseTypeId } + + func importStatements(table: String, schema: String?, columns: [ColumnInfo]) async throws -> SQLImportStatements { + SQLImportStatements( + databaseTypeId: databaseTypeId, + qualifiedTableName: SQLiteMasterQueries.qualifiedTable(table, schema: schema, quote: sqliteQuoteIdentifier), + parameterStyle: .questionMark, + quoteIdentifier: sqliteQuoteIdentifier, + columnTypeNames: ImportTableNames.columnTypeNames(columns) + ) + } + + func createTableStatements(definition: PluginCreateTableDefinition, schema: String?) throws -> [String] { + guard let statement = sqliteCreateTableSQL(definition: definition) else { throw ImportTableNames.noStatement } + return [statement] + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Export/ExportRun.swift b/apple/TableProMobile/TableProMobile/Transfer/Export/ExportRun.swift new file mode 100644 index 0000000000..304eee9471 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Export/ExportRun.swift @@ -0,0 +1,591 @@ +import Foundation +import os +import TableProDatabase +import TableProLogRedaction +import TableProModels +import TableProPluginKit + +// A read the app builds, written for the connection that runs it: an export reads on a connection of its own, whose +// literal rules (MySQL's NO_BACKSLASH_ESCAPES, PostgreSQL's standard_conforming_strings) can differ from the window's. +// The table a filtered or keyed read comes from, so a SQL file writes its INSERTs into that table and knows its +// generated and identity columns. +nonisolated struct ExportRelation: Sendable, Equatable { + let table: String + let schema: String? +} + +nonisolated struct ExportSelect: Sendable, ExpressibleByStringLiteral { + private let render: @Sendable (any DatabaseDriver) -> String + let relation: ExportRelation? + + init(relation: ExportRelation? = nil, render: @escaping @Sendable (any DatabaseDriver) -> String) { + self.relation = relation + self.render = render + } + + init(stringLiteral sql: String) { + render = { _ in sql } + relation = nil + } + + func statement(on driver: any DatabaseDriver) -> String { + render(driver) + } +} + +nonisolated enum ExportSource: Sendable { + // Objects of one database, from `ExportObjectListing`, with the option values the user chose. + case objects([PluginExportTable]) + // A read statement the app built, run again in full: a filtered or sorted table, or rows read again by key. + case query(ExportSelect) + // The user's statement run again for a query's whole result, only in the engine's read-only mode: the classifier + // calls `SELECT post_payment(42)` a read, and its function writes. + case rerun(String) + // Rows already held: selected rows with no key, or a result that cannot run again in read-only mode. + case rows(ExportRowSet) +} + +nonisolated struct ExportRequest: Sendable { + let formatId: String + let source: ExportSource + let fileName: String + let connectionId: UUID + let databaseType: DatabaseType + let database: String? + let schema: String? + let safeModeLevel: SafeModeLevel + // Spells the literals of a `.rows` export the way the session reads them. + let sessionDriver: (any DatabaseDriver)? + // Read when Export is tapped: a connection of the export's own cannot see that transaction's changes. + let sessionHoldsTransaction: Bool + + init( + formatId: String, + source: ExportSource, + fileName: String, + connectionId: UUID, + databaseType: DatabaseType, + database: String?, + schema: String?, + safeModeLevel: SafeModeLevel, + sessionDriver: (any DatabaseDriver)? = nil, + sessionHoldsTransaction: Bool = false + ) { + self.formatId = formatId + self.source = source + self.fileName = fileName + self.connectionId = connectionId + self.databaseType = databaseType + self.database = database + self.schema = schema + self.safeModeLevel = safeModeLevel + self.sessionDriver = sessionDriver + self.sessionHoldsTransaction = sessionHoldsTransaction + } +} + +nonisolated struct ExportOutcome: Sendable { + let folder: URL + let files: [URL] + let rowsWritten: Int + let warnings: [String] + let notes: [String] + + func discard() { + TransferFiles.remove(folder) + } +} + +nonisolated protocol ExportRunning: AnyObject, Sendable { + func run(onProgress: @escaping @Sendable (TransferProgress) -> Void) async throws -> ExportOutcome + func cancel() +} + +// The iOS counterpart of macOS `ExportService`: one format, one source, one file in a folder of its own. A stop, a +// failure or an expiry leaves no folder behind. +nonisolated final class ExportRun: ExportRunning, @unchecked Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "ExportRun") + + private let request: ExportRequest + private let plugin: any ExportFormatPlugin + private let drivers: any TransferDriverProvider + private let background: TransferBackgroundRun? + private let sessionHost: (any TransferSessionHost)? + private let exportsRoot: URL + private let stopGrace: Duration + + private let lock = NSLock() + private var stopRequested = false + private var pluginProgress: PluginExportProgress? + private var heldLease: TransferDriverLease? + private var writingFolder: URL? + private var work: Task? + private var ending: FirstResult? + + // `stopGrace` bounds how long a stop waits for the driver to stop its statement before the connection is closed + // anyway: a dedicated connection has no query timeout, and a driver like Redis cannot cancel at all. + init( + request: ExportRequest, + plugin: any ExportFormatPlugin, + drivers: any TransferDriverProvider, + background: TransferBackgroundRun? = nil, + sessionHost: (any TransferSessionHost)? = nil, + exportsRoot: URL = TransferFiles.exportsDirectory, + stopGrace: Duration = .seconds(2) + ) { + self.request = request + self.plugin = plugin + self.drivers = drivers + self.background = background + self.sessionHost = sessionHost + self.exportsRoot = exportsRoot + self.stopGrace = stopGrace + } + + // `sessionHost` tracks an export that ends up reading on the window's own session, as a SQLite file holding the + // user's transaction does, so the grid waits for it rather than ending its stream. + convenience init( + request: ExportRequest, + plugin: any ExportFormatPlugin, + connectionManager: ConnectionManager, + background: TransferBackgroundRun? = nil, + sessionHost: (any TransferSessionHost)? = nil + ) { + self.init( + request: request, + plugin: plugin, + drivers: DedicatedTransferDriverProvider( + connectionManager: connectionManager, connectionId: request.connectionId, database: request.database + ), + background: background, + sessionHost: sessionHost + ) + } + + // Latched, so a stop during the count or before the plugin starts is not lost. The run ends here and now, the + // background expiry included: the work is cancelled, the driver is asked to stop its statement, and the connection + // is released within `stopGrace` even when a read never returns. + func cancel() { + let stop = lock.withLock { () -> StopTargets in + stopRequested = true + defer { heldLease = nil } + return StopTargets( + progress: pluginProgress, work: work, ending: ending, lease: heldLease, folder: writingFolder + ) + } + stop.progress?.cancel() + stop.work?.cancel() + // Only a stop that ended the run removes the folder: a run that finished first hands its file to the caller. + if stop.ending?.settle(.failure(CancellationError())) == true, let folder = stop.folder { + TransferFiles.remove(folder) + } + // The window's own session is never closed, and a blind stop there could reach the user's next statement; the + // cancelled work ends its own stream on it. + if let lease = stop.lease, !lease.isSessionDriver { + stopAndRelease(lease) + } + } + + nonisolated private struct StopTargets { + let progress: PluginExportProgress? + let work: Task? + let ending: FirstResult? + let lease: TransferDriverLease? + let folder: URL? + } + + var isCancelled: Bool { + lock.withLock { stopRequested } + } + + private func throwIfStopped() throws { + if isCancelled || Task.isCancelled { throw CancellationError() } + } + + @concurrent + func run(onProgress: @escaping @Sendable (TransferProgress) -> Void) async throws -> ExportOutcome { + try throwIfStopped() + await background?.begin { [weak self] in self?.cancel() } + let report: @Sendable (TransferProgress) -> Void = { [background] progress in + onProgress(progress) + guard let background else { return } + Task { @MainActor in background.report(progress) } + } + do { + let outcome = try await performUntilStopped(report: report) + await background?.end(success: true) + return outcome + } catch { + await background?.end(success: false) + if error is CancellationError || error is PluginExportCancellationError || isCancelled { + throw CancellationError() + } + throw error + } + } + + // MARK: - Run + + // The work runs in a task of its own, so a stop returns at once even while a driver keeps reading; whatever the + // work finishes with after that is thrown away. + private func performUntilStopped( + report: @escaping @Sendable (TransferProgress) -> Void + ) async throws -> ExportOutcome { + let ending = FirstResult() + let work = Task { [self] in + do { + let outcome = try await perform(report: report) + if !ending.settle(.success(outcome)) { outcome.discard() } + } catch { + ending.settle(.failure(error)) + } + } + let stopped = lock.withLock { () -> Bool in + self.work = work + self.ending = ending + return stopRequested + } + if stopped { cancel() } + return try await withTaskCancellationHandler { + try await ending.value() + } onCancel: { [weak self] in + self?.cancel() + } + } + + private func perform(report: @escaping @Sendable (TransferProgress) -> Void) async throws -> ExportOutcome { + var preparing = TransferProgress(stage: .preparing) + preparing.objectCount = objectCount + report(preparing) + + let lease: TransferDriverLease? + switch request.source { + case .objects, .query, .rerun: + lease = try await hold(try await drivers.openTransferDriver()) + case .rows: + lease = nil + } + do { + try await lease?.settle(schema: request.schema) + let outcome: ExportOutcome + if let lease, lease.isSessionDriver, let sessionHost { + // Carried out as a value: an error thrown through the tracking would read as a session lost mid-write. + let result = try await sessionHost.trackTransfer { [self] () async throws -> Result in + do { + return .success(try await self.export(on: lease, report: report)) + } catch { + return .failure(error) + } + } + outcome = try result.get() + } else { + outcome = try await export(on: lease, report: report) + } + await releaseHeldLease() + return outcome + } catch { + await releaseHeldLease() + throw error + } + } + + // Kept where a stop can reach it. A stop that came while the connection opened closes it before anything reads. + private func hold(_ lease: TransferDriverLease) async throws -> TransferDriverLease { + let stopped = lock.withLock { () -> Bool in + if !stopRequested { heldLease = lease } + return stopRequested + } + guard !stopped else { + await lease.close() + throw CancellationError() + } + return lease + } + + // Closed once, by whichever of the run's end and a stop comes first. + private func releaseHeldLease() async { + let lease = lock.withLock { () -> TransferDriverLease? in + defer { heldLease = nil } + return heldLease + } + await lease?.close() + } + + private func stopAndRelease(_ lease: TransferDriverLease) { + let grace = stopGrace + let driver = lease.driver + let stopping = Task { _ = try? await driver.cancelCurrentQuery() } + Task { + let settled = FirstResult() + Task { + await stopping.value + settled.settle(.success(())) + } + Task { + try? await Task.sleep(for: grace) + settled.settle(.success(())) + } + try? await settled.value() + await lease.close() + } + } + + private var objectCount: Int { + if case .objects(let objects) = request.source { return objects.count } + return 1 + } + + @concurrent + private func export( + on lease: TransferDriverLease?, + report: @escaping @Sendable (TransferProgress) -> Void + ) async throws -> ExportOutcome { + let readOnly = request.safeModeLevel.blocksWrites + let (dataSource, tables, total) = try await prepare(on: lease, readOnly: readOnly, report: report) + try throwIfStopped() + + let folder = try openFolder() + do { + let destination = TransferFiles.fileURL( + in: folder, name: request.fileName, fileExtension: plugin.currentFileExtension + ) + let result = try await write(tables: tables, dataSource: dataSource, to: destination, total: total, report: report) + // A plugin that returns after a stop has still written the file the user said to throw away. + try throwIfStopped() + let missesTransaction = request.sessionHoldsTransaction && lease.map { !$0.isSessionDriver } == true + return ExportOutcome( + folder: folder, + files: TransferFiles.writtenFiles(in: folder, primary: destination), + rowsWritten: result.rows, + warnings: result.format.warnings + (missesTransaction ? [Self.missedTransactionWarning] : []), + notes: result.format.notes + ) + } catch { + TransferFiles.remove(folder) + throw error + } + } + + static var missedTransactionWarning: String { + String(localized: "The rows were read on a connection of the export's own, so changes in the transaction open on this connection are not in the file.") + } + + // Known to a stop, which removes it when it ends the run before the plugin returns. + private func openFolder() throws -> URL { + let folder = try TransferFiles.makeExportFolder(in: exportsRoot) + let stopped = lock.withLock { () -> Bool in + writingFolder = folder + return stopRequested + } + guard !stopped else { + TransferFiles.remove(folder) + throw CancellationError() + } + return folder + } + + private func prepare( + on lease: TransferDriverLease?, + readOnly: Bool, + report: @Sendable (TransferProgress) -> Void + ) async throws -> (any PluginExportDataSource, [PluginExportTable], Int?) { + switch request.source { + case .objects(let objects): + guard let lease else { throw CancellationError() } + let catalog = try await ExportObjectListing.catalog( + on: lease.driver, databaseType: request.databaseType, database: request.database, schema: request.schema + ) + let source = MobileExportDataSource( + catalog: catalog, + driver: lease.driver, + databaseType: request.databaseType, + readOnly: readOnly, + skipsSnapshotStatements: lease.isSessionDriver + ) + var counting = TransferProgress(stage: .counting) + counting.objectCount = objects.count + report(counting) + let total = await countRows(of: objects, in: source, driver: lease.driver, readOnly: readOnly) + return (source, objects, total) + case .query(let select): + guard let lease else { throw CancellationError() } + // A catalog that cannot be read leaves the rows to export; the format warns that it has no column metadata. + var columnCatalog: Result? + if let relation = select.relation { + do { + columnCatalog = .success(try await ExportObjectListing.catalog( + on: lease.driver, databaseType: request.databaseType, database: request.database, + schema: relation.schema + )) + } catch { + columnCatalog = .failure(error) + } + } + let source = MobileQueryExportDataSource( + query: select.statement(on: lease.driver), + driver: lease.driver, + databaseType: request.databaseType, + readOnly: readOnly, + skipsSnapshotStatements: lease.isSessionDriver, + columnCatalog: columnCatalog + ) + return (source, [select.relation.map(relationTable) ?? queryTable], nil) + case .rerun(let query): + guard let lease else { throw CancellationError() } + guard lease.driver.enforcesReadOnlyMode else { + throw MobileTransferRefusal(reason: String( + localized: "This connection cannot run the query again read-only, so it was not run. Export the rows shown instead." + )) + } + let source = MobileQueryExportDataSource( + query: query, + driver: lease.driver, + databaseType: request.databaseType, + readOnly: true, + skipsSnapshotStatements: lease.isSessionDriver + ) + return (source, [queryTable], nil) + case .rows(let rows): + let source = MobileRowsExportDataSource( + rows: rows, databaseType: request.databaseType, sessionDriver: request.sessionDriver + ) + return (source, [queryTable], rows.count) + } + } + + // A result has no structure behind it, so a query export asks only for data, as macOS `QueryExportOptions` does. + private func relationTable(_ relation: ExportRelation) -> PluginExportTable { + PluginExportTable( + name: relation.table, + databaseName: request.database ?? "", + tableType: "table", + optionValues: QueryExportOptions.dataOnly( + columns: type(of: plugin).perTableOptionColumns, + defaults: plugin.defaultTableOptionValues() + ), + schema: relation.schema, + kind: .table + ) + } + + private var queryTable: PluginExportTable { + PluginExportTable( + name: request.fileName, + databaseName: "", + tableType: "query", + optionValues: QueryExportOptions.dataOnly( + columns: type(of: plugin).perTableOptionColumns, + defaults: plugin.defaultTableOptionValues() + ), + schema: nil, + kind: .table + ) + } + + private func write( + tables: [PluginExportTable], + dataSource: any PluginExportDataSource, + to destination: URL, + total: Int?, + report: @escaping @Sendable (TransferProgress) -> Void + ) async throws -> (format: ExportFormatResult, rows: Int) { + let nsProgress = SharedProgress(Progress(totalUnitCount: Int64(total ?? 0))) + let progress = PluginExportProgress(progress: nsProgress.progress) + let stopped = lock.withLock { () -> Bool in + pluginProgress = progress + return stopRequested + } + if stopped { progress.cancel() } + defer { lock.withLock { pluginProgress = nil } } + + let objectCount = tables.count + let read: @Sendable () -> TransferProgress = { + var running = TransferProgress(stage: .running) + running.completedUnits = progress.processedRows + running.totalUnits = total + running.objectCount = objectCount + running.objectIndex = progress.currentTableIndex + running.objectName = nsProgress.description + running.status = nsProgress.status + return running + } + let poller = TransferProgressPoller() + poller.start(read: read, report: report) + do { + let result = try await plugin.export( + tables: tables, dataSource: dataSource, destination: destination, progress: progress + ) + var finished = read() + finished.stage = .finishing + poller.stop(final: finished, report: report) + return (result, progress.processedRows) + } catch { + poller.stop(final: nil, report: report) + throw error + } + } + + // A total the bar can show: COUNT(*) where the engine speaks SQL, the catalog's estimate elsewhere. An object that + // cannot be counted leaves the total out rather than showing a short one. + private func countRows( + of objects: [PluginExportTable], + in source: MobileExportDataSource, + driver: any DatabaseDriver, + readOnly: Bool + ) async -> Int? { + let counted = objects.filter { $0.kind.carriesRows && plugin.readsRows(optionValues: $0.optionValues, kind: $0.kind) } + guard !counted.isEmpty else { return nil } + var total = 0 + for object in counted { + guard !isCancelled else { return nil } + do { + let count: Int? + if let query = source.rowCountQuery(for: object) { + count = try await MobileExportStreams.count(query, on: driver, readOnly: readOnly) + } else { + count = try await source.fetchApproximateRowCount(table: object.name, databaseName: object.databaseName) + } + guard let count else { return nil } + total = total.addingReportingOverflow(count).overflow ? .max : total + count + } catch { + Self.logger.warning("Could not count the rows to export: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + return nil + } + } + return total + } +} + +// The first of several ways a wait can end wins; the others find it taken. `await task.value` alone cannot give up on +// a task that never finishes. +nonisolated final class FirstResult: @unchecked Sendable { + private let lock = NSLock() + private var result: Result? + private var waiters: [CheckedContinuation] = [] + + @discardableResult + func settle(_ result: Result) -> Bool { + let waiting = lock.withLock { () -> [CheckedContinuation]? in + guard self.result == nil else { return nil } + self.result = result + defer { waiters = [] } + return waiters + } + guard let waiting else { return false } + for waiter in waiting { + waiter.resume(with: result) + } + return true + } + + func value() async throws -> Value { + try await withCheckedThrowingContinuation { continuation in + let settled = lock.withLock { () -> Result? in + if let result { return result } + waiters.append(continuation) + return nil + } + if let settled { continuation.resume(with: settled) } + } + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Export/MobileExportDataSource.swift b/apple/TableProMobile/TableProMobile/Transfer/Export/MobileExportDataSource.swift new file mode 100644 index 0000000000..939dadbcf7 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Export/MobileExportDataSource.swift @@ -0,0 +1,217 @@ +import Foundation +import TableProDatabase +import TableProModels +import TableProPluginKit + +// The iOS counterpart of macOS `ExportDataSourceAdapter`: the engine's catalog answers every question about an object, +// and the driver streams its rows. +nonisolated final class MobileExportDataSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId: String + + private let catalog: any MobileExportCatalog + private let driver: any DatabaseDriver + private let databaseType: DatabaseType + private let readOnly: Bool + private let skipsSnapshotStatements: Bool + + // `readOnly`: the connection runs at Read-Only. `skipsSnapshotStatements`: the rows are read on the session that + // holds the user's transaction. + init( + catalog: any MobileExportCatalog, + driver: any DatabaseDriver, + databaseType: DatabaseType, + readOnly: Bool, + skipsSnapshotStatements: Bool = false + ) { + self.catalog = catalog + self.driver = driver + self.databaseType = databaseType + self.readOnly = readOnly + self.skipsSnapshotStatements = skipsSnapshotStatements + self.databaseTypeId = catalog.databaseTypeId + } + + // MARK: - Rows + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + PluginExportStreamElement.cells(streamExportRows(table: table, databaseName: databaseName)) + } + + func streamRows(for object: PluginExportTable) -> AsyncThrowingStream { + PluginExportStreamElement.cells(streamExportRows(for: object)) + } + + func streamExportRows(table: String, databaseName: String) -> AsyncThrowingStream { + streamExportRows(for: PluginExportTable( + name: table, databaseName: databaseName, tableType: "table", schema: nil, kind: .table + )) + } + + func streamExportRows(for object: PluginExportTable) -> AsyncThrowingStream { + if object.rowScope.isUnrestricted, let documents = catalog.streamExportRows(for: object) { + return documents + } + guard let rows = driver as? any ExportRowStreaming else { + return MobileExportStreams.failing(DatabaseDriverCapabilityError.unsupported(.exportRows)) + } + let query: String + do { + query = try rowsQuery(for: object) + } catch { + return MobileExportStreams.failing(error) + } + return PluginExportStreamElement.wrapping(rows.exportRows(query: query, readOnly: readOnly)) + } + + // The engine's own statement when it has one; otherwise every column of the object by its qualified name, narrowed + // to the row scope the way the macOS adapter narrows it. + func rowsQuery(for object: PluginExportTable) throws -> String { + let scope = object.rowScope + if scope.isUnrestricted, let custom = catalog.rowsQuery(for: object) { + return custom + } + guard databaseType.speaksSQL else { + throw PluginExportError.exportFailed(String( + format: String(localized: "%@ cannot be narrowed to some rows or columns, because this database has no SQL."), + object.name + )) + } + let projection = scope.columns.isEmpty ? "*" : scope.columns.map { catalog.quoteIdentifier($0) }.joined(separator: ", ") + var sql = "SELECT \(projection) FROM \(qualifiedName(of: object))" + if !scope.sanitizedFilter.isEmpty { + sql += " WHERE \(scope.sanitizedFilter)" + } + if let limit = scope.rowLimit { + sql += " " + SQLBuilder.paginationClause(orderBy: "", limit: max(0, limit), offset: 0, for: databaseType) + } + return sql + } + + // The container the catalog named the object in: its schema, or the database on an engine with no schema layer. + func qualifiedName(of object: PluginExportTable) -> String { + let container = [object.schema, object.databaseName].compactMap { $0 }.first { !$0.isEmpty } + let name = catalog.quoteIdentifier(object.name) + guard let container else { return name } + return "\(catalog.quoteIdentifier(container)).\(name)" + } + + // A statement that counts what `rowsQuery` reads, or nil where only the catalog can say. + func rowCountQuery(for object: PluginExportTable) -> String? { + guard databaseType.speaksSQL else { return nil } + let scope = object.rowScope + guard scope.isUnrestricted ? catalog.rowsQuery(for: object) == nil : scope.rowLimit == nil else { return nil } + var sql = "SELECT COUNT(*) FROM \(qualifiedName(of: object))" + if !scope.sanitizedFilter.isEmpty { + sql += " WHERE \(scope.sanitizedFilter)" + } + return sql + } + + func execute(query: String) async throws -> PluginQueryResult { + try await MobileExportStreams.execute( + query, on: driver, readOnly: readOnly, skipsTransactionStatements: skipsSnapshotStatements + ) + } + + // MARK: - Catalog + + func quoteIdentifier(_ identifier: String) -> String { + catalog.quoteIdentifier(identifier) + } + + func escapeStringLiteral(_ value: String) -> String { + catalog.escapeStringLiteral(value) + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + try await catalog.fetchTableDDL(table: table, databaseName: databaseName) + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { + try await catalog.fetchApproximateRowCount(table: table, databaseName: databaseName) + } + + func fetchDependentSequences(table: String, databaseName: String) async throws -> [PluginSequenceInfo] { + try await catalog.fetchDependentSequences(table: table, databaseName: databaseName) + } + + func fetchDependentTypes(table: String, databaseName: String) async throws -> [PluginEnumTypeInfo] { + try await catalog.fetchDependentTypes(table: table, databaseName: databaseName) + } + + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { + try await catalog.fetchColumns(table: table, databaseName: databaseName) + } + + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] { + try await catalog.fetchAllColumns(databaseName: databaseName) + } + + func fetchForeignKeys(table: String, databaseName: String) async throws -> [PluginForeignKeyInfo] { + try await catalog.fetchForeignKeys(table: table, databaseName: databaseName) + } + + func fetchAllForeignKeys(databaseName: String) async throws -> [String: [PluginForeignKeyInfo]] { + try await catalog.fetchAllForeignKeys(databaseName: databaseName) + } + + var tableDDLIncludesForeignKeys: Bool { + catalog.tableDDLIncludesForeignKeys + } + + func fetchIndexDDL(table: String, databaseName: String) async throws -> [String] { + try await catalog.fetchIndexDDL(table: table, databaseName: databaseName) + } + + func fetchCommentDDL(table: String, databaseName: String) async throws -> [String] { + try await catalog.fetchCommentDDL(table: table, databaseName: databaseName) + } + + func fetchObjectDDL(_ object: PluginExportTable) async throws -> String { + try await catalog.fetchObjectDDL(object) + } + + func scriptText(for ddl: String) -> String { + catalog.scriptText(for: ddl) + } + + var lexicalFeatures: SQLLexicalFeatures { + catalog.lexicalFeatures + } + + func fetchGrantStatements(principal: String, host: String?) async throws -> [String] { + try await catalog.fetchGrantStatements(principal: principal, host: host) + } + + var supportsCascadeDrop: Bool { + catalog.supportsCascadeDrop + } + + // No guess without the routine's kind: PostgreSQL refuses `DROP FUNCTION` on a procedure. + func dropStatement(for object: PluginExportTable) -> String? { + if object.kind == .routine, PluginRoutineKind(rawValue: object.tableType.lowercased()) == nil { + return nil + } + return catalog.dropStatement(for: object) + } + + func openSessionScope(for purpose: PluginSessionPurpose) async throws -> PluginSessionScope? { + try await catalog.openSessionScope(for: purpose) + } + + func closeSessionScope(_ scope: PluginSessionScope) async throws { + try await catalog.closeSessionScope(scope) + } + + func fetchCreationScope(_ object: PluginExportTable) async throws -> PluginStatementScope? { + try await catalog.fetchCreationScope(object) + } + + func fetchIndexes(table: String, databaseName: String) async throws -> [PluginIndexInfo] { + try await catalog.fetchIndexes(table: table, databaseName: databaseName) + } + + func fetchObjectComment(_ object: PluginExportTable) async throws -> String? { + try await catalog.fetchObjectComment(object) + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Export/MobileExportFormats.swift b/apple/TableProMobile/TableProMobile/Transfer/Export/MobileExportFormats.swift new file mode 100644 index 0000000000..06950386b0 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Export/MobileExportFormats.swift @@ -0,0 +1,42 @@ +import Foundation +import TableProModels +import TableProPluginKit + +// One instance of each format plugin for the app's life: a form edits its `settings`, and the run reads them once. +nonisolated enum MobileExportFormats { + static let csv = CSVExportPlugin() + static let json = JSONExportPlugin() + static let sql = SQLExportPlugin() + static let xlsx = XLSXExportPlugin() + static let markdown = MarkdownExportPlugin() + static let mql = MQLExportPlugin() + + static var all: [any ExportFormatPlugin] { + ExportFormatOffer.sorted([csv, json, sql, xlsx, markdown, mql]) + } + + static func available(for databaseType: DatabaseType) -> [any ExportFormatPlugin] { + ExportFormatOffer.offered(all, forDatabaseTypeId: databaseType.rawValue) + } + + static func plugin(formatId: String) -> (any ExportFormatPlugin)? { + all.first { type(of: $0).formatId == formatId } + } + + static func frozen(_ shared: any ExportFormatPlugin) -> any ExportFormatPlugin { + let fresh = type(of: shared).init() + RunPluginSettings.copy(from: shared, to: fresh) + return fresh + } +} + +// A run gets a plugin of its own holding the settings the shared one had when the user confirmed, so another window +// editing the shared form cannot change a run that is already decided. Restoring re-saves the same values as the +// defaults, which is harmless. +nonisolated enum RunPluginSettings { + static func copy(from shared: AnyObject, to fresh: AnyObject) { + guard let source = shared as? any SettablePluginDiscoverable, let data = source.snapshotSettingsData(), + let target = fresh as? any SettablePluginDiscoverable else { return } + target.restoreSettingsData(data) + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Export/MobileQueryExportDataSource.swift b/apple/TableProMobile/TableProMobile/Transfer/Export/MobileQueryExportDataSource.swift new file mode 100644 index 0000000000..b9f4d22593 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Export/MobileQueryExportDataSource.swift @@ -0,0 +1,139 @@ +import Foundation +import TableProDatabase +import TableProModels +import TableProPluginKit +import TableProSQLGrammar + +// A read statement run again for every row it returns, as macOS `StreamingQueryExportDataSource` does for a result +// larger than the grid holds. +nonisolated final class MobileQueryExportDataSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId: String + let lexicalFeatures: SQLLexicalFeatures + + private let query: String + private let driver: any DatabaseDriver + private let databaseType: DatabaseType + private let readOnly: Bool + private let skipsSnapshotStatements: Bool + private let columnCatalog: Result? + + init( + query: String, + driver: any DatabaseDriver, + databaseType: DatabaseType, + readOnly: Bool, + skipsSnapshotStatements: Bool = false, + columnCatalog: Result? = nil + ) { + self.columnCatalog = columnCatalog + self.query = query + self.driver = driver + self.databaseType = databaseType + self.readOnly = readOnly + self.skipsSnapshotStatements = skipsSnapshotStatements + self.databaseTypeId = databaseType.rawValue + self.lexicalFeatures = SQLLexicalReadings.resolve( + databaseTypeId: databaseType.rawValue, declared: nil, session: driver.settledLexicalFacts + ).execution.pluginFeatures + } + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + guard let rows = driver as? any ExportRowStreaming else { + return MobileExportStreams.failing(DatabaseDriverCapabilityError.unsupported(.exportRows)) + } + return rows.exportRows(query: query, readOnly: readOnly) + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { + nil + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + "" + } + + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { + guard let columnCatalog else { return [] } + return try await columnCatalog.get().fetchColumns(table: table, databaseName: databaseName) + } + + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] { + guard let columnCatalog else { return [:] } + return try await columnCatalog.get().fetchAllColumns(databaseName: databaseName) + } + + func execute(query: String) async throws -> PluginQueryResult { + try await MobileExportStreams.execute( + query, on: driver, readOnly: readOnly, skipsTransactionStatements: skipsSnapshotStatements + ) + } + + func quoteIdentifier(_ identifier: String) -> String { + SQLBuilder.quoteIdentifier(identifier, for: databaseType) + } + + func escapeStringLiteral(_ value: String) -> String { + ClipboardExporter.literalBody(value, databaseType: databaseType, driver: driver) + } +} + +nonisolated enum MobileExportStreams { + static func failing(_ error: any Error) -> AsyncThrowingStream { + AsyncThrowingStream { $0.finish(throwing: error) } + } + + // A snapshot statement is dropped where it cannot run: at Read-Only, and on a session that holds the user's own + // transaction, whose COMMIT would end that transaction. + static func execute( + _ query: String, + on driver: any DatabaseDriver, + readOnly: Bool, + skipsTransactionStatements: Bool + ) async throws -> PluginQueryResult { + if readOnly || skipsTransactionStatements, ExportSnapshotStatements.isTransactionStatement(query) { + return ExportSnapshotStatements.noResult + } + if let catalog = driver as? any CatalogQuerying, !readOnly { + return try await catalog.catalogQuery(query) + } + if let rows = driver as? any ExportRowStreaming { + return try await collect(rows.exportRows(query: query, readOnly: readOnly)) + } + let result = try await driver.execute(query: query) + return PluginQueryResult( + columns: result.columns.map(\.name), + columnTypeNames: result.columns.map(\.typeName), + rows: result.rows.map { $0.map(PluginCellValue.fromOptional) }, + rowsAffected: result.rowsAffected, + executionTime: result.executionTime + ) + } + + static func collect(_ stream: AsyncThrowingStream) async throws -> PluginQueryResult { + var columns: [String] = [] + var typeNames: [String] = [] + var rows: [[PluginCellValue]] = [] + for try await element in stream { + switch element { + case .header(let header): + columns = header.columns + typeNames = header.columnTypeNames + case .rows(let batch): + rows.append(contentsOf: batch) + } + } + return PluginQueryResult(columns: columns, columnTypeNames: typeNames, rows: rows, rowsAffected: 0, executionTime: 0) + } + + // The first cell of the first row as a count, from a statement read the way the export reads its rows. + static func count(_ query: String, on driver: any DatabaseDriver, readOnly: Bool) async throws -> Int? { + let result: PluginQueryResult + if let rows = driver as? any ExportRowStreaming { + result = try await collect(rows.exportRows(query: query, readOnly: readOnly)) + } else { + result = try await execute(query, on: driver, readOnly: readOnly, skipsTransactionStatements: false) + } + guard let cell = result.rows.first?.first?.asText?.trimmingCharacters(in: .whitespaces) else { return nil } + return Int(cell) ?? Double(cell).flatMap { Int(exactly: $0.rounded()) } + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Export/MobileRowsExportDataSource.swift b/apple/TableProMobile/TableProMobile/Transfer/Export/MobileRowsExportDataSource.swift new file mode 100644 index 0000000000..d203253691 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Export/MobileRowsExportDataSource.swift @@ -0,0 +1,132 @@ +import Foundation +import TableProDatabase +import TableProModels +import TableProPluginKit +import TableProSQLGrammar + +// Rows the app already holds whole: selected rows with no key to read them again by, or a result its statement cannot +// be run a second time for. +nonisolated struct ExportRowSet: Sendable { + let columns: [String] + let columnTypeNames: [String] + let rows: [[PluginCellValue]] + + init(columns: [String], columnTypeNames: [String], rows: [[PluginCellValue]]) { + self.columns = columns + self.columnTypeNames = columnTypeNames + self.rows = rows + } + + // A cut or binary cell holds only a preview here, and a file must not pass one off as the value. + init(columns: [ColumnInfo], cells: [[Cell]]) throws(ClipboardExporter.IncompleteValues) { + let incomplete = ClipboardExporter.incompleteColumns(columns: columns, rows: cells) + guard incomplete.isEmpty else { throw ClipboardExporter.IncompleteValues(columns: incomplete) } + self.columns = columns.map(\.name) + self.columnTypeNames = columns.map(\.typeName) + self.rows = cells.map { row in + columns.indices.map { index -> PluginCellValue in + guard index < row.count, case .text(let value) = row[index] else { return .null } + return .text(value) + } + } + } + + var count: Int { rows.count } +} + +nonisolated final class MobileRowsExportDataSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId: String + let lexicalFeatures: SQLLexicalFeatures + + private let rowSet: ExportRowSet + private let databaseType: DatabaseType + private let sessionDriver: (any DatabaseDriver)? + + private static let rowsPerElement = 500 + + init(rows: ExportRowSet, databaseType: DatabaseType, sessionDriver: (any DatabaseDriver)?) { + self.rowSet = rows + self.databaseType = databaseType + self.sessionDriver = sessionDriver + self.databaseTypeId = databaseType.rawValue + self.lexicalFeatures = SQLLexicalReadings.resolve( + databaseTypeId: databaseType.rawValue, declared: nil, session: sessionDriver?.settledLexicalFacts + ).execution.pluginFeatures + } + + // Pulled a slice at a time, so a writer that stops early never formats the rest. + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + let rowSet = rowSet + let cursor = RowSliceCursor(total: rowSet.rows.count, sliceSize: Self.rowsPerElement) + return AsyncThrowingStream(unfolding: { + switch cursor.next() { + case nil: + return nil + case .header: + return .header(PluginStreamHeader( + columns: rowSet.columns, + columnTypeNames: rowSet.columnTypeNames, + estimatedRowCount: rowSet.rows.count + )) + case .rows(let range): + return .rows(Array(rowSet.rows[range])) + } + }) + } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { + rowSet.rows.count + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { + "" + } + + // Held rows have no session to snapshot, so a format's BEGIN or COMMIT passes as a no-op. + func execute(query: String) async throws -> PluginQueryResult { + guard ExportSnapshotStatements.isTransactionStatement(query) else { + throw PluginExportError.exportFailed(String(localized: "These rows are not read from the database again.")) + } + return ExportSnapshotStatements.noResult + } + + func quoteIdentifier(_ identifier: String) -> String { + SQLBuilder.quoteIdentifier(identifier, for: databaseType) + } + + func escapeStringLiteral(_ value: String) -> String { + ClipboardExporter.literalBody(value, databaseType: databaseType, driver: sessionDriver) + } +} + +// Hands out the header first, then one row range per pull. +nonisolated final class RowSliceCursor: @unchecked Sendable { + nonisolated enum Pull: Equatable { + case header + case rows(Range) + } + + private let lock = NSLock() + private let total: Int + private let sliceSize: Int + private var sentHeader = false + private var start = 0 + + init(total: Int, sliceSize: Int) { + self.total = total + self.sliceSize = max(1, sliceSize) + } + + func next() -> Pull? { + lock.withLock { + guard sentHeader else { + sentHeader = true + return .header + } + guard start < total else { return nil } + let range = start.. Void +} + +nonisolated extension TransferDriverLease { + // A connection of the export's own opens on the connection's default schema. The schema the window shows is set on + // it before anything reads, so an unqualified name reads the table the user sees; the session already reads there. + // One that opened there already keeps the server's own search path. + func settle(schema: String?) async throws { + guard !isSessionDriver, let schema, !schema.isEmpty, driver.supportsSchemas, driver.currentSchema != schema + else { return } + do { + try await driver.switchSchema(to: schema) + } catch { + throw MobileTransferRefusal(reason: String( + format: String(localized: "Could not switch to schema %1$@: %2$@"), schema, error.localizedDescription + )) + } + } +} + +nonisolated protocol TransferDriverProvider: Sendable { + func openTransferDriver() async throws -> TransferDriverLease +} + +nonisolated struct DedicatedTransferDriverProvider: TransferDriverProvider { + let connectionManager: ConnectionManager + let connectionId: UUID + let database: String? + + // The manager registered the driver, so its own close takes it back off the list a disconnect walks. + func openTransferDriver() async throws -> TransferDriverLease { + let driver = try await connectionManager.openDedicatedDriver(for: connectionId, database: database) + let isSession = connectionManager.session(for: connectionId)?.driver === driver + let manager = connectionManager + let connectionId = connectionId + return TransferDriverLease(driver: driver, isSessionDriver: isSession, close: { + guard !isSession else { return } + await manager.closeDedicatedDriver(driver, for: connectionId) + }) + } +} + +nonisolated enum ExportObjectListing { + static func supportsExport(_ driver: any DatabaseDriver) -> Bool { + driver is any MobileTransferEngine + } + + // Whether a table the list shows is one object of the export, so its own Export entry has something to list. + static func supportsTableExport(_ driver: any DatabaseDriver) -> Bool { + (driver as? any MobileTransferEngine)?.exportsListedTables ?? false + } + + static func catalog( + on driver: any DatabaseDriver, + databaseType: DatabaseType, + database: String?, + schema: String? + ) async throws -> any MobileExportCatalog { + guard let engine = driver as? any MobileTransferEngine else { + throw MobileTransferRefusal(reason: String( + format: String(localized: "%@ cannot export to a file."), databaseType.mobileDisplayName + )) + } + return try await engine.makeExportCatalog(context: MobileTransferContext( + databaseType: databaseType, database: database, schema: schema + )) + } + + // Every object of the database, whatever the format writes, so another format chosen later shows its own kinds. + // Each object starts with the plugin's own defaults for its option columns. + static func load( + connectionManager: ConnectionManager, + connectionId: UUID, + databaseType: DatabaseType, + database: String?, + schema: String?, + plugin: any ExportFormatPlugin + ) async throws -> [PluginExportTable] { + try await load( + drivers: DedicatedTransferDriverProvider( + connectionManager: connectionManager, connectionId: connectionId, database: database + ), + databaseType: databaseType, database: database, schema: schema, plugin: plugin + ) + } + + static func load( + drivers: any TransferDriverProvider, + databaseType: DatabaseType, + database: String?, + schema: String?, + plugin: any ExportFormatPlugin + ) async throws -> [PluginExportTable] { + let lease = try await drivers.openTransferDriver() + do { + try await lease.settle(schema: schema) + let exportCatalog = try await catalog( + on: lease.driver, databaseType: databaseType, database: database, schema: schema + ) + let objects = try await exportCatalog.exportObjects(database: database ?? "", schema: schema) + await lease.close() + return objects.map { $0.withOptionValues(ExportObjectOptions.defaults(for: $0.kind, plugin: plugin)) } + } catch { + await lease.close() + throw error + } + } +} + +nonisolated extension PluginExportTable { + func withOptionValues(_ values: [Bool]) -> PluginExportTable { + PluginExportTable( + name: name, + databaseName: databaseName, + tableType: tableType, + optionValues: values, + schema: schema, + kind: kind, + identity: identity, + parentTable: parentTable, + rowScope: rowScope + ) + } + + func withRowScope(_ scope: PluginExportRowScope) -> PluginExportTable { + PluginExportTable( + name: name, + databaseName: databaseName, + tableType: tableType, + optionValues: optionValues, + schema: schema, + kind: kind, + identity: identity, + parentTable: parentTable, + rowScope: scope + ) + } +} + +// A format's default option values for one kind, with each column the kind does not take cleared. +nonisolated enum ExportObjectOptions { + static func defaults(for kind: PluginExportObjectKind, plugin: any ExportFormatPlugin) -> [Bool] { + let format = type(of: plugin) + let columns = format.perTableOptionColumns + return ExportOptionMask.masked( + plugin.defaultTableOptionValues(), columns: columns, kind: kind, supports: format.supportsOption(columnId:for:) + ) + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Export/TransferFiles.swift b/apple/TableProMobile/TableProMobile/Transfer/Export/TransferFiles.swift new file mode 100644 index 0000000000..0d15b534ac --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Export/TransferFiles.swift @@ -0,0 +1,129 @@ +import Foundation +import os +import TableProLogRedaction + +// Each export and import works in a folder of its own under tmp, so a share, a cancel or a kill removes one folder. +nonisolated enum TransferFiles { + private static let logger = Logger(subsystem: "com.TablePro", category: "TransferFiles") + + // APFS takes 255 bytes of UTF-8 per name; the rest leaves room for ".partN" and an extension. + static let maximumNameBytes = 200 + + static var exportsDirectory: URL { + FileManager.default.temporaryDirectory.appendingPathComponent("Exports", isDirectory: true) + } + + static var importsDirectory: URL { + FileManager.default.temporaryDirectory.appendingPathComponent("Imports", isDirectory: true) + } + + static func makeExportFolder(in root: URL = exportsDirectory) throws -> URL { + try makeFolder(in: root) + } + + static func makeImportFolder(in root: URL = importsDirectory) throws -> URL { + try makeFolder(in: root) + } + + private static func makeFolder(in root: URL) throws -> URL { + let folder = root.appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + return folder + } + + static func remove(_ folder: URL) { + do { + try FileManager.default.removeItem(at: folder) + } catch CocoaError.fileNoSuchFile { + return + } catch { + logger.warning("Could not remove a transfer folder: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + } + } + + // What a run killed with the app left behind; nothing is running at launch. + static func sweep(roots: [URL] = [exportsDirectory, importsDirectory]) { + for root in roots { + remove(root) + } + } + + // The name the recipient sees, so it keeps the table's own letters and drops only what a file name cannot hold. + static func sanitizedFileName(_ name: String, fallback: String = "Export") -> String { + let cleaned = truncated(cleanedName(name), toBytes: maximumNameBytes).trimmingCharacters(in: .whitespaces) + return cleaned.isEmpty ? fallback : cleaned + } + + // An import reads its format and its gzip from the extension, so a long name loses letters from its stem instead. + static func sanitizedFileNameKeepingExtension(_ name: String, fallback: String) -> String { + let cleaned = cleanedName(name) + let suffix = extensionSuffix(of: cleaned) + let stem = String(decoding: cleaned.utf8.dropLast(suffix.utf8.count), as: UTF8.self) + let cut = truncated(stem, toBytes: maximumNameBytes - suffix.utf8.count).trimmingCharacters(in: .whitespaces) + return (cut.isEmpty ? fallback : cut) + suffix + } + + private static let maximumExtensionBytes = 16 + + private static func cleanedName(_ name: String) -> String { + let forbidden = CharacterSet(charactersIn: "/\\:") + .union(.controlCharacters) + .union(.newlines) + .union(.illegalCharacters) + var cleaned = String(String.UnicodeScalarView(name.unicodeScalars.map { forbidden.contains($0) ? "_" : $0 })) + cleaned = cleaned.trimmingCharacters(in: .whitespaces) + // A leading dot hides the file, and "." or ".." name a folder. + while cleaned.hasPrefix(".") { + cleaned.removeFirst() + } + return cleaned + } + + // The last extension, and the one inside a ".gz". + private static func extensionSuffix(of name: String) -> String { + var suffix = "" + var rest = name as NSString + while true { + let ext = rest.pathExtension + guard !ext.isEmpty, ext.utf8.count <= maximumExtensionBytes else { break } + suffix = "." + ext + suffix + rest = rest.deletingPathExtension as NSString + guard ext.lowercased() == "gz", suffix.utf8.count == ext.utf8.count + 1 else { break } + } + return suffix + } + + static func fileURL(in folder: URL, name: String, fileExtension: String) -> URL { + let base = sanitizedFileName(name) + let file = fileExtension.isEmpty ? base : "\(base).\(fileExtension)" + return folder.appendingPathComponent(file, isDirectory: false) + } + + // Every regular file the run wrote, the main file first: SQL export writes split parts beside it. + static func writtenFiles(in folder: URL, primary: URL) -> [URL] { + let contents = (try? FileManager.default.contentsOfDirectory( + at: folder, includingPropertiesForKeys: [.isRegularFileKey], options: [.skipsHiddenFiles] + )) ?? [] + let files = contents.filter { + (try? $0.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile) == true + } + let primaryName = primary.lastPathComponent + let others = files.filter { $0.lastPathComponent != primaryName } + .sorted { $0.lastPathComponent.localizedStandardCompare($1.lastPathComponent) == .orderedAscending } + let hasPrimary = files.contains { $0.lastPathComponent == primaryName } + return (hasPrimary ? [primary] : []) + others + } + + private static func truncated(_ text: String, toBytes limit: Int) -> String { + guard text.utf8.count > limit else { return text } + var result = "" + var used = 0 + for character in text { + let size = String(character).utf8.count + guard used + size <= limit else { break } + result.append(character) + used += size + } + return result + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Import/ImportFileStaging.swift b/apple/TableProMobile/TableProMobile/Transfer/Import/ImportFileStaging.swift new file mode 100644 index 0000000000..321c2c9e3c --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Import/ImportFileStaging.swift @@ -0,0 +1,88 @@ +import Foundation +import os +import TableProLogRedaction + +nonisolated struct StagedImportFile: Sendable, Equatable { + let url: URL + let folder: URL + let originalName: String + + var fileSize: Int64 { + let size = try? FileManager.default.attributesOfItem(atPath: url.path(percentEncoded: false))[.size] as? NSNumber + return size?.int64Value ?? 0 + } + + func discard() { + TransferFiles.remove(folder) + } +} + +nonisolated enum ImportFileStagingError: Error, LocalizedError, Equatable { + case unreadable(String) + + var errorDescription: String? { + switch self { + case .unreadable(let message): + return String(format: String(localized: "Failed to read file: %@"), message) + } + } +} + +// A picked or opened file can sit in iCloud or another app's provider, and another process may change it under a +// memory map. The import parses a private copy taken under a coordinated read, so neither the security scope nor the +// provider's file has to last across a long or background run (critic M10). +nonisolated enum ImportFileStaging { + private static let logger = Logger(subsystem: "com.TablePro", category: "ImportFileStaging") + + @concurrent + static func stage(_ source: URL, into root: URL = TransferFiles.importsDirectory) async throws -> StagedImportFile { + let folder = try TransferFiles.makeImportFolder(in: root) + let name = TransferFiles.sanitizedFileNameKeepingExtension(source.lastPathComponent, fallback: "Import") + let destination = folder.appendingPathComponent(name, isDirectory: false) + let scoped = source.startAccessingSecurityScopedResource() + defer { + if scoped { source.stopAccessingSecurityScopedResource() } + } + do { + try coordinatedCopy(from: source, to: destination) + } catch { + TransferFiles.remove(folder) + throw error + } + if isInInbox(source) { + do { + try FileManager.default.removeItem(at: source) + } catch { + logger.warning("Could not remove an opened file from the Inbox: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + } + } + return StagedImportFile(url: destination, folder: folder, originalName: source.lastPathComponent) + } + + // A coordinated read downloads an iCloud placeholder first and waits for a writer to finish. + private static func coordinatedCopy(from source: URL, to destination: URL) throws { + var coordinationError: NSError? + var copyError: Error? + NSFileCoordinator(filePresenter: nil).coordinate( + readingItemAt: source, options: [], error: &coordinationError + ) { readable in + do { + try FileManager.default.copyItem(at: readable, to: destination) + } catch { + copyError = error + } + } + if let error = coordinationError ?? copyError { + throw ImportFileStagingError.unreadable(error.localizedDescription) + } + } + + // A file another app hands over with "Open in" lands in Documents/Inbox, which the app owns and must empty. + static func isInInbox(_ url: URL) -> Bool { + guard let documents = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first else { + return false + } + let inbox = documents.appendingPathComponent("Inbox", isDirectory: true).standardizedFileURL.path + return url.standardizedFileURL.deletingLastPathComponent().path == inbox + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Import/ImportMappingStore.swift b/apple/TableProMobile/TableProMobile/Transfer/Import/ImportMappingStore.swift new file mode 100644 index 0000000000..37e751d004 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Import/ImportMappingStore.swift @@ -0,0 +1,55 @@ +import Foundation +import os +import TableProLogRedaction + +// The mapping choices a user made for one table, kept as departures from the name match (macOS +// `ImportColumnMappingStore`), so a column the table gains later is still found by name. +nonisolated struct ImportMappingStore: @unchecked Sendable { + nonisolated struct Key: Hashable, Sendable { + let connectionId: UUID + let database: String? + let schema: String? + let table: String + + var defaultsKey: String { + let parts = [database ?? "", schema ?? "", table].map { $0.replacingOccurrences(of: "\u{1F}", with: "", options: .literal) } + return "com.TablePro.importMapping.\(connectionId.uuidString).\(parts.joined(separator: "\u{1F}"))" + } + } + + private static let logger = Logger(subsystem: "com.TablePro", category: "ImportMappingStore") + + private let defaults: UserDefaults + + init(defaults: UserDefaults = .standard) { + self.defaults = defaults + } + + func overrides(for key: Key) -> [String: ImportMappingOverride] { + guard let data = defaults.data(forKey: key.defaultsKey) else { return [:] } + do { + return try JSONDecoder().decode([String: ImportMappingOverride].self, from: data) + } catch { + Self.logger.warning("A saved import mapping could not be read: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + return [:] + } + } + + // Entries for fields this file does not hold are kept, so several layouts into one table all come back. + func remember(_ overrides: [String: ImportMappingOverride], fields: [String], for key: Key) { + let merged = ImportColumnMatcher.merging(overrides, forFields: fields, into: self.overrides(for: key)) + guard !merged.isEmpty else { + defaults.removeObject(forKey: key.defaultsKey) + return + } + do { + defaults.set(try JSONEncoder().encode(merged), forKey: key.defaultsKey) + } catch { + Self.logger.warning("An import mapping could not be saved: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + } + } + + func forget(_ key: Key) { + defaults.removeObject(forKey: key.defaultsKey) + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Import/ImportRun.swift b/apple/TableProMobile/TableProMobile/Transfer/Import/ImportRun.swift new file mode 100644 index 0000000000..440c283fec --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Import/ImportRun.swift @@ -0,0 +1,633 @@ +import Foundation +import os +import TableProDatabase +import TableProModels +import TableProPluginKit +import TableProSQLGrammar + +nonisolated struct RowImportTarget: Sendable { + let table: String + let schema: String? + // Field to column, for every field the user kept. + let columnMapping: [String: String] + // Every field the file holds, in file order. + let sourceFields: [String] + let rowWriting: ImportRowWriting + // Set when the rows go into a table the import makes first. + let newTable: PluginCreateTableDefinition? + + init( + table: String, + schema: String?, + columnMapping: [String: String], + sourceFields: [String], + rowWriting: ImportRowWriting = .insert, + newTable: PluginCreateTableDefinition? = nil + ) { + self.table = table + self.schema = schema + self.columnMapping = columnMapping + self.sourceFields = sourceFields + self.rowWriting = rowWriting + self.newTable = newTable + } +} + +nonisolated struct ImportRequest: Sendable { + let formatId: String + let file: StagedImportFile + let connectionId: UUID + let connectionName: String + let databaseType: DatabaseType + let database: String? + let safeModeLevel: SafeModeLevel + let target: RowImportTarget? + let sqlEncoding: String.Encoding + + init( + formatId: String, + file: StagedImportFile, + connectionId: UUID, + connectionName: String, + databaseType: DatabaseType, + database: String?, + safeModeLevel: SafeModeLevel, + target: RowImportTarget?, + sqlEncoding: String.Encoding = .utf8 + ) { + self.formatId = formatId + self.file = file + self.connectionId = connectionId + self.connectionName = connectionName + self.databaseType = databaseType + self.database = database + self.safeModeLevel = safeModeLevel + self.target = target + self.sqlEncoding = sqlEncoding + } + + // The level the user confirmed under, read live from the connection when they did. + func confirmed(under level: SafeModeLevel) -> ImportRequest { + ImportRequest( + formatId: formatId, file: file, connectionId: connectionId, connectionName: connectionName, + databaseType: databaseType, database: database, safeModeLevel: level, target: target, sqlEncoding: sqlEncoding + ) + } +} + +nonisolated struct ImportStopQuestion: Sendable, Equatable { + // "Stop and Commit" beside "Stop and Rollback" only while the import holds a transaction of its own. + let offersCommit: Bool + let message: String +} + +nonisolated struct DataImportOutcome: Sendable { + nonisolated enum Ending: Sendable, Equatable { + case completed + case stopped + case expired + case failed(String) + } + + let ending: Ending + let transaction: ImportTransactionEnding + let tally: ImportRowTally? + let executedStatements: Int + let skippedStatements: Int + let errors: [PluginImportResult.ImportStatementError] + let createdTable: Bool + let catalogChanges: SQLCatalogObjectKinds + // What a SQL file's statements left on the session, for the window to follow as it follows a query run. + var sessionEffect = SessionLocalStatements.Effect() + var endsTransaction = false + + var summaryLines: [String] { + if let tally { + return tally.summaryLines(failed: skippedStatements) + } + var lines = [String(format: String(localized: "Statements run: %@"), executedStatements.formatted())] + if skippedStatements > 0 { + lines.append(String(format: String(localized: "Statements skipped: %@"), skippedStatements.formatted())) + } + return lines + } + + // Whether the host should check the session is still there: an error mid-run, a statement skipped on one, or the + // server ending the transaction. An expired run may have closed the session itself, and the next use dials again. + var mayHaveLostSession: Bool { + switch ending { + case .failed: + return true + case .completed: + return !errors.isEmpty || transaction == .endedByServer + case .stopped: + return transaction == .endedByServer + case .expired: + return false + } + } + + // What happened to the rows, said only where it is not simply "the import finished and committed". + var transactionMessage: String? { + switch (transaction, ending) { + case (.leftInOpenTransaction, _): + return ImportRun.joinedTransactionMessage + case (.committed, .completed), (.none, .completed): + return nil + case (.rolledBack, _): + return String(localized: "The import's transaction was rolled back, so none of its rows were kept.") + case (.committed, _): + return String(localized: "The rows written before the import ended were committed.") + case (.partlyCommitted, _): + return String(localized: "The file ended the import's transaction early, so the rows written before that stay.") + case (.none, _): + return String(localized: "The rows written before the import ended stay in the table.") + case (.endedByServer, _): + return String(localized: "The database had already ended the import's transaction, so none of its rows were kept.") + } + } +} + +// Tables one import sheet made, so its retry clears and reuses its own table rather than failing on the name or writing +// the kept rows a second time. Another sheet never counts as a retry (macOS `RowImportSheet`). +nonisolated final class ImportTablePlanner: @unchecked Sendable { + private let lock = NSLock() + private var planner = NewTableImportPlanner() + + func plan(for table: NewTableImportPlanner.Table, createTableSQL: String) -> NewTableImportPlan { + lock.withLock { planner.plan(for: table, createTableSQL: createTableSQL) } + } + + func recordCreated(_ table: NewTableImportPlanner.Table, createTableSQL: String) { + lock.withLock { planner.recordCreated(table, createTableSQL: createTableSQL) } + } +} + +// The iOS counterpart of macOS `ImportService` for one file: runs the format plugin on the window's session inside the +// coordinator's tracking, with one safe-mode decision for the whole run (made by the caller's `ConfirmedWriteGate`). +nonisolated final class ImportRun: @unchecked Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "ImportRun") + + static var joinedTransactionMessage: String { + String(localized: "The rows written stay in the transaction that is open on this connection. Commit or roll it back.") + } + + private let request: ImportRequest + private let plugin: any ImportFormatPlugin + private let host: any TransferSessionHost + private let background: TransferBackgroundRun? + private let planner: ImportTablePlanner + private let rollbackGrace: Duration + + private let lock = NSLock() + private var stopChoice: ImportStopChoice? + private var expired = false + private var finished = false + private var sink: MobileImportDataSink? + private var sqlSource: MobileSQLFileImportSource? + private var pluginProgress: PluginImportProgress? + + init( + request: ImportRequest, + plugin: any ImportFormatPlugin, + host: any TransferSessionHost, + background: TransferBackgroundRun? = nil, + planner: ImportTablePlanner = ImportTablePlanner(), + rollbackGrace: Duration = .milliseconds(1_500) + ) { + self.request = request + self.plugin = plugin + self.host = host + self.background = background + self.planner = planner + self.rollbackGrace = rollbackGrace + } + + // Read-Only refuses every import; an engine that takes no rows, or a format it does not read, is refused by name. + static func refusal( + connectionName: String, + databaseType: DatabaseType, + safeModeLevel: SafeModeLevel, + formatId: String + ) -> String? { + guard !safeModeLevel.blocksWrites else { + return String(format: String(localized: "%@ is read-only, so nothing can be written to it."), connectionName) + } + guard databaseType.speaksSQL else { + return String(format: String(localized: "Import is not supported for %@ connections."), databaseType.mobileDisplayName) + } + guard let plugin = MobileImportFormats.plugin(formatId: formatId) else { + return String(format: String(localized: "%@ files cannot be imported into this connection."), formatId.uppercased()) + } + let offered = type(of: plugin).requiresTargetTable + ? MobileImportFormats.rowFormats(for: databaseType).contains { type(of: $0).formatId == formatId } + : MobileImportFormats.offersSQLFile(for: databaseType) + guard offered else { + return String( + format: String(localized: "%@ files cannot be imported into this connection."), + type(of: plugin).formatDisplayName + ) + } + return nil + } + + // MARK: - Stop + + var stopQuestion: ImportStopQuestion { + let sink = lock.withLock { self.sink } + if sink?.holdsOwnTransaction == true { + return ImportStopQuestion( + offersCommit: true, + message: String(localized: "Roll back to remove the rows written so far, or commit to keep them.") + ) + } + if sink?.joinsUserTransaction == true { + return ImportStopQuestion(offersCommit: false, message: Self.joinedTransactionMessage) + } + return ImportStopQuestion( + offersCommit: false, + message: String(localized: "The rows written so far stay in the table.") + ) + } + + // Latched: the first choice wins, and a stop before the run starts keeps it from writing at all. It also stops a + // `.gz` still expanding. + func stop(_ choice: ImportStopChoice) { + let targets = lock.withLock { () -> StopTargets in + if stopChoice == nil { stopChoice = choice } + return StopTargets(sink: sink, progress: pluginProgress, source: sqlSource) + } + targets.sink?.stop(choice) + targets.progress?.cancel() + targets.source?.stop() + } + + nonisolated private struct StopTargets { + let sink: MobileImportDataSink? + let progress: PluginImportProgress? + let source: MobileSQLFileImportSource? + } + + // Background time ran out. An owned rollback that has not reached the server before the grace ends the connection, + // so the server ends the transaction rather than holding its locks while the phone sleeps (critic M8). + func expire() { + lock.withLock { expired = true } + stop(.rollBack) + let grace = rollbackGrace + let host = host + Task { [weak self] in + try? await Task.sleep(for: grace) + guard let self else { return } + let stalled = self.lock.withLock { !self.finished && self.sink?.holdsOwnTransaction == true } + guard stalled else { return } + Self.logger.warning("The import's rollback did not finish before background time ran out; closing the session") + await host.abandonTransferSession() + } + } + + private var isStopped: Bool { + lock.withLock { stopChoice != nil } + } + + // MARK: - Run + + // Throws only before any row is written: `MobileTransferRefusal` for a refused import or a table that could not be + // made, `CancellationError` for one stopped before it began, the session's `AppError` when there is none. + // Everything after the first row comes back as an outcome. + @concurrent + func run(onProgress: @escaping @Sendable (TransferProgress) -> Void) async throws -> DataImportOutcome { + if let reason = Self.refusal( + connectionName: request.connectionName, + databaseType: request.databaseType, + safeModeLevel: request.safeModeLevel, + formatId: request.formatId + ) { + throw MobileTransferRefusal(reason: reason) + } + guard !isStopped else { throw CancellationError() } + + await background?.begin { [weak self] in self?.expire() } + let report: @Sendable (TransferProgress) -> Void = { [background] progress in + onProgress(progress) + guard let background else { return } + Task { @MainActor in background.report(progress) } + } + do { + // A refusal is carried out of the tracked work as a value: thrown through it, the coordinator would treat it + // as a session lost mid-write and redial. A failed run is a value too, and `failed` hands it to that check. + let performed = try await admitted { [self] (admission: ImportAdmission) async throws -> Performed in + if let reason = self.admissionRefusal(liveLevel: admission.safeModeLevel) { + return .refused(MobileTransferRefusal(reason: reason)) + } + do { + return .outcome(try await self.perform(on: admission.driver, report: report), on: admission.driver) + } catch let refusal as MobileTransferRefusal { + return .refused(refusal) + } catch is CancellationError { + return .cancelled + } + } + lock.withLock { finished = true } + let outcome: DataImportOutcome + let driver: any DatabaseDriver + switch performed { + case .outcome(let value, let ranOn): + outcome = value + driver = ranOn + case .refused(let refusal): + throw refusal + case .cancelled: + throw CancellationError() + } + // A SQL file may have run `USE` or moved the search path, however it ended. + let followsSession = request.target == nil + if followsSession { + await host.noteTransferSessionEffect(outcome.sessionEffect, on: driver) + } + if !outcome.catalogChanges.isEmpty { + await host.noteTransferChangedCatalog(outcome.catalogChanges) + } + if followsSession { + await host.followTransferPlacement( + movesDatabase: outcome.sessionEffect.usesDatabase, + movesSchema: outcome.sessionEffect.leavesCatalog || outcome.endsTransaction + ) + } + await background?.end(success: outcome.ending == .completed) + return outcome + } catch { + lock.withLock { finished = true } + await background?.end(success: false) + throw error + } + } + + // The session is the import's for the whole run or not at all: another import, other work or a session that is + // moving refuses it at once, and nothing waits. + private func admitted(_ work: @escaping @Sendable (ImportAdmission) async throws -> Performed) async throws -> Performed { + do { + return try await host.trackImport(failed: { (performed: Performed) in performed.mayHaveLostSession }, work) + } catch let held as SessionHeldByImport { + throw MobileTransferRefusal(reason: held.localizedDescription) + } catch let busy as SessionBusyForImport { + throw MobileTransferRefusal(reason: busy.localizedDescription) + } + } + + // Safe Mode read again as the run is admitted: a level another window made stricter since the confirmation wins. + private func admissionRefusal(liveLevel: SafeModeLevel) -> String? { + if let reason = Self.refusal( + connectionName: request.connectionName, + databaseType: request.databaseType, + safeModeLevel: liveLevel, + formatId: request.formatId + ) { + return reason + } + guard liveLevel.requiresConfirmation, !request.safeModeLevel.requiresConfirmation else { return nil } + return String(localized: "Safe Mode on this connection changed. Start the import again.") + } + + nonisolated private enum Performed: Sendable { + case outcome(DataImportOutcome, on: any DatabaseDriver) + case refused(MobileTransferRefusal) + case cancelled + + var mayHaveLostSession: Bool { + guard case .outcome(let outcome, _) = self else { return false } + return outcome.mayHaveLostSession + } + } + + @concurrent + private func perform( + on driver: any DatabaseDriver, + report: @escaping @Sendable (TransferProgress) -> Void + ) async throws -> DataImportOutcome { + report(TransferProgress(stage: .preparing, unit: request.target == nil ? .statements : .rows)) + let dialect = try await writeDialect(on: driver) + try await refuseIfSessionMoved(on: driver, dialect: dialect) + var catalogChanges: SQLCatalogObjectKinds = request.target == nil ? .everything : [] + let preparation = try await prepareNewTable(on: driver, dialect: dialect) + if preparation.created { catalogChanges.insert(.tables) } + + let host = host + let catalogHold = request.target == nil ? ImportCatalogHold( + hold: { await host.holdTransferCatalogConnections() }, + release: { await host.releaseTransferCatalogConnections() } + ) : nil + let sink = MobileImportDataSink( + driver: driver, + databaseType: request.databaseType, + rowTarget: request.target.map { target in + MobileImportDataSink.RowTarget( + table: target.table, + schema: target.schema, + columnMapping: target.columnMapping, + sourceFields: Set(target.sourceFields), + rowWriting: target.rowWriting, + binaryText: ImportBinaryText(formatId: request.formatId), + database: request.database + ) + }, + dialect: dialect, + clearsTargetFirst: preparation.clearsFirst, + catalogHold: catalogHold + ) + let shared = SharedProgress(Progress(totalUnitCount: 0)) + let progress = PluginImportProgress(progress: shared.progress) + let stopped = lock.withLock { () -> ImportStopChoice? in + self.sink = sink + pluginProgress = progress + return stopChoice + } + if let stopped { + sink.stop(stopped) + progress.cancel() + } + + let bytes = ImportBytesRead() + let source = makeSource(on: driver, bytes: bytes) + if let sqlFile = source as? MobileSQLFileImportSource { + let stoppedBefore = lock.withLock { () -> Bool in + sqlSource = sqlFile + return stopChoice != nil + } + if stoppedBefore { sqlFile.stop() } + } + let fileSize = request.file.fileSize + let unit: TransferProgress.Unit = request.target == nil ? .statements : .rows + let read: @Sendable () -> TransferProgress = { + var running = TransferProgress(stage: .running, unit: unit) + running.completedUnits = progress.processedStatements + running.totalUnits = progress.estimatedTotalStatements > 0 ? progress.estimatedTotalStatements : nil + running.status = shared.status + let file = bytes.value + running.bytesRead = file.bytesRead + running.totalBytes = file.fileSize ?? (fileSize > 0 ? fileSize : nil) + return running + } + let poller = TransferProgressPoller() + poller.start(read: read, report: report) + + var result: PluginImportResult? + var failure: (any Error)? + do { + result = try await plugin.performImport(source: source, sink: sink, progress: progress) + if lock.withLock({ stopChoice }) == nil { + try await sink.clearTargetIfDue() + } + } catch { + failure = error + } + source.cleanup() + + var finishing = read() + finishing.stage = .finishing + poller.stop(final: finishing, report: report) + + let choice = lock.withLock { stopChoice } + await sink.finish(commit: choice == .commit || (failure == nil && choice == nil)) + await catalogHold?.releaseIfHeld() + let transaction = await sink.transactionEnding() + + return DataImportOutcome( + ending: ending(failure: failure, choice: choice), + transaction: transaction, + tally: request.target == nil ? nil : sink.rowTally, + executedStatements: result?.executedStatements ?? progress.processedStatements, + skippedStatements: result?.skippedStatements ?? 0, + errors: result?.errors ?? [], + createdTable: preparation.created, + catalogChanges: catalogChanges, + sessionEffect: sink.sessionEffect, + endsTransaction: sink.endedTransaction + ) + } + + private func ending(failure: (any Error)?, choice: ImportStopChoice?) -> DataImportOutcome.Ending { + if lock.withLock({ expired }) { return .expired } + if choice != nil { return .stopped } + guard let failure else { return .completed } + if failure is PluginImportCancellationError || failure is CancellationError { return .stopped } + return .failed(failure.localizedDescription) + } + + private func writeDialect(on driver: any DatabaseDriver) async throws -> (any MobileWriteDialect)? { + guard let target = request.target else { return nil } + guard let engine = driver as? any MobileTransferEngine else { + throw MobileTransferRefusal(reason: String( + format: String(localized: "%@ cannot take rows from a file."), request.databaseType.mobileDisplayName + )) + } + return try await engine.makeWriteDialect(context: MobileTransferContext( + databaseType: request.databaseType, database: request.database, schema: target.schema + )) + } + + // The sheet named its database, and another window can have moved the session since. Rows that name the database + // land there anyway; anything that does not, a new table or a SQL file's unqualified statements among them, is + // refused rather than written elsewhere. + private func refuseIfSessionMoved(on driver: any DatabaseDriver, dialect: (any MobileWriteDialect)?) async throws { + guard let wanted = request.database, !wanted.isEmpty else { return } + if let target = request.target, dialect?.namesDatabase == true, target.newTable == nil { return } + let current: String? + if let reporting = driver as? any SessionDatabaseReporting { + current = try? await reporting.sessionDatabase() + } else { + current = await host.transferDatabase + } + guard let current, !current.isEmpty, current != wanted else { return } + throw MobileTransferRefusal(reason: request.target == nil ? Self.sqlFileSessionMovedMessage : Self.sessionMovedMessage) + } + + static var sessionMovedMessage: String { + String(localized: "The connection moved to another database or schema. Choose the table again.") + } + + static var sqlFileSessionMovedMessage: String { + String(localized: "The connection moved to another database. Open the SQL file again from that database.") + } + + nonisolated private struct NewTablePreparation { + var created = false + var clearsFirst = false + } + + // The table is made before the rows and committed on its own, as macOS `RowImportTableMaker` does, so a stopped + // import never rolls back the table the user asked for. + private func prepareNewTable( + on driver: any DatabaseDriver, + dialect: (any MobileWriteDialect)? + ) async throws -> NewTablePreparation { + guard let target = request.target, let definition = target.newTable, let dialect else { + return NewTablePreparation() + } + let statements = try dialect.createTableStatements(definition: definition, schema: target.schema) + let createSQL = statements.joined(separator: "\n") + let key = NewTableImportPlanner.Table( + connectionId: request.connectionId, database: request.database, schema: target.schema, name: definition.tableName + ) + switch planner.plan(for: key, createTableSQL: createSQL) { + case .reuseAfterClearing: + return NewTablePreparation(created: false, clearsFirst: true) + case .nameTakenWithDifferentColumns: + throw MobileTransferRefusal(reason: String( + format: String(localized: "An earlier import made a table named %@ with other columns. Choose another name."), + definition.tableName + )) + case .create: + break + } + switch await driver.sessionTransactionState() { + case .explicitTransaction, .implicitTransaction, .abortedTransaction: + throw MobileTransferRefusal(reason: String( + localized: "Commit or roll back the transaction open on this connection before importing into a new table." + )) + case .idle, .unknown: + break + } + guard !isStopped else { throw CancellationError() } + do { + for statement in statements { + _ = try await driver.execute(query: statement) + } + } catch { + throw MobileTransferRefusal(reason: error.localizedDescription) + } + planner.recordCreated(key, createTableSQL: createSQL) + return NewTablePreparation(created: true, clearsFirst: false) + } + + private func makeSource(on driver: any DatabaseDriver, bytes: ImportBytesRead) -> any PluginImportSource { + guard request.target == nil else { + return PlainFileImportSource(url: request.file.url) + } + let databaseType = request.databaseType + let source = MobileSQLFileImportSource( + url: request.file.url, + encoding: request.sqlEncoding, + family: SQLImportFile.dumpFamily(for: databaseType), + grammar: { SQLImportFile.grammar(for: databaseType, session: driver.settledLexicalFacts) } + ) + source.reportsReading { read, size in bytes.update(read: read, size: size) } + return source + } +} + +nonisolated private final class ImportBytesRead: @unchecked Sendable { + private let lock = NSLock() + private var bytesRead: Int64? + private var fileSize: Int64? + + func update(read: Int64, size: Int64) { + lock.withLock { + bytesRead = read + fileSize = size > 0 ? size : nil + } + } + + var value: (bytesRead: Int64?, fileSize: Int64?) { + lock.withLock { (bytesRead, fileSize) } + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Import/MobileImportDataSink.swift b/apple/TableProMobile/TableProMobile/Transfer/Import/MobileImportDataSink.swift new file mode 100644 index 0000000000..420ac2b09c --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Import/MobileImportDataSink.swift @@ -0,0 +1,530 @@ +import Foundation +import os +import TableProDatabase +import TableProLogRedaction +import TableProModels +import TableProPluginKit +import TableProQuery +import TableProSQLFile +import TableProSQLGrammar + +nonisolated enum ImportStopChoice: Sendable, Equatable { + case rollBack + case commit +} + +// How the import's transaction ended, read from what the sink sent and what the session said after. +nonisolated enum ImportTransactionEnding: Sendable, Equatable { + case committed + case rolledBack + // A COMMIT or a DDL statement in the file ended the transaction early, so a rollback after it kept those rows. + case partlyCommitted + // The import joined the user's open transaction, which holds the rows until the user commits or rolls back. + case leftInOpenTransaction + // The rows were written outside any transaction, each kept as it landed. + case none + // The server had already ended the transaction, so no ROLLBACK was sent. + case endedByServer +} + +// The iOS `PluginImportDataSink`. The shared `ImportRowWriter` builds every row statement; the sink owns the +// connection, the transaction, foreign key checks, COPY and the stop. A transaction is opened only when the session is +// idle and is then the import's own; one already open is joined and never committed or rolled back here (critic M8). +nonisolated final class MobileImportDataSink: PluginImportDataSink, @unchecked Sendable { + nonisolated struct RowTarget: Sendable { + let table: String + let schema: String? + let columnMapping: [String: String] + let sourceFields: Set + let rowWriting: ImportRowWriting + let binaryText: ImportBinaryText? + // The database the import was started in, which a read names wherever the session has moved since. + var database: String? + } + + private static let logger = Logger(subsystem: "com.TablePro", category: "MobileImportDataSink") + private static let ddlKeywords: Set = ["CREATE", "ALTER", "DROP", "TRUNCATE", "RENAME", "GRANT", "REVOKE"] + + let databaseTypeId: String + let targetTable: String? + + private let driver: any DatabaseDriver + private let databaseType: DatabaseType + private let rowTarget: RowTarget? + private let dialect: (any MobileWriteDialect)? + private let rowWriter: ImportRowWriter? + private let externallyCancelled: @Sendable () -> Bool + // A table an earlier import made, reused by a retry: emptied inside the import's transaction, so a retry that fails + // or stops gives back the rows the first run kept (macOS `RowImportTableMaker`). + private let clearsTargetFirst: Bool + private let catalogGrammar: SQLLexicalGrammar + private let catalogHold: ImportCatalogHold? + + private let lock = NSLock() + private var stopChoice: ImportStopChoice? + private var ownsTransaction = false + private var joinedTransaction = false + private var ending: ImportTransactionEnding? + // Read statement by statement: a COMMIT, ROLLBACK or DDL statement in the file that ended the import's own + // transaction, and rows that stay whatever the import's end does, committed by the file or written after that end. + private var fileEndedTransaction = false + private var fileRolledBack = false + private var keptWrites = false + private var disabledForeignKeys = false + private var restoresForeignKeysAtFinish = false + private var writerTarget: ImportRowWriter.Target? + private var immutableColumns: Set? + private var clearedTarget = false + private var recordedEffect = SessionLocalStatements.Effect() + private var transactionEnded = false + + init( + driver: any DatabaseDriver, + databaseType: DatabaseType, + rowTarget: RowTarget? = nil, + dialect: (any MobileWriteDialect)? = nil, + clearsTargetFirst: Bool = false, + catalogHold: ImportCatalogHold? = nil, + isCancelled: @escaping @Sendable () -> Bool = { false } + ) { + self.driver = driver + self.catalogHold = catalogHold + self.clearsTargetFirst = clearsTargetFirst + self.catalogGrammar = SQLLexicalReadings.resolve( + databaseTypeId: databaseType.rawValue, declared: nil, session: nil + ).execution + self.databaseType = databaseType + self.databaseTypeId = databaseType.rawValue + self.rowTarget = rowTarget + self.targetTable = rowTarget?.table + self.dialect = dialect + self.externallyCancelled = isCancelled + self.rowWriter = rowTarget.map { + ImportRowWriter( + databaseTypeId: databaseType.rawValue, + columnMapping: $0.columnMapping, + sourceFields: $0.sourceFields, + rowWriting: $0.rowWriting + ) + } + } + + // MARK: - Stop + + // Latched. A rollback stop refuses every write and the commit; a commit stop refuses writes and skips the rollback the + // plugin sends on its way out, so the run's `finish` commits what was written (macOS `ImportRun` admission). + func stop(_ choice: ImportStopChoice) { + lock.withLock { + guard stopChoice == nil else { return } + stopChoice = choice + } + } + + var requestedStop: ImportStopChoice? { + lock.withLock { stopChoice } + } + + private var isStopped: Bool { + requestedStop != nil || externallyCancelled() || Task.isCancelled + } + + private func admitWrite() throws { + if isStopped { throw PluginImportCancellationError() } + } + + var holdsOwnTransaction: Bool { + lock.withLock { ownsTransaction } + } + + var joinsUserTransaction: Bool { + lock.withLock { joinedTransaction } + } + + var rowTally: ImportRowTally? { + rowWriter?.tally + } + + // What the file's statements left on the session, read from each one before it is sent, so a statement that + // failed counts too, as the query editor counts every statement a run planned. + var sessionEffect: SessionLocalStatements.Effect { + lock.withLock { recordedEffect } + } + + // A transaction ended, the file's own or the import's, which takes back a `SET` made inside it. + var endedTransaction: Bool { + lock.withLock { transactionEnded } + } + + // What the import left, once it is over. Asks the session when the sink opened nothing, so rows written into a + // transaction the user holds are never reported as kept. + func transactionEnding() async -> ImportTransactionEnding { + let recorded = lock.withLock { () -> ImportTransactionEnding? in + if keptWrites, ending == .rolledBack { return .partlyCommitted } + if joinedTransaction { + // The file's own COMMIT, ROLLBACK or implicitly committing DDL ended the user's transaction too. + guard fileEndedTransaction else { return .leftInOpenTransaction } + guard fileRolledBack else { return .committed } + return keptWrites ? .partlyCommitted : .rolledBack + } + return ending + } + if let recorded { return recorded } + switch await driver.sessionTransactionState() { + case .explicitTransaction, .implicitTransaction, .abortedTransaction: + return .leftInOpenTransaction + case .idle, .unknown: + return .none + } + } + + // Ends a transaction the sink still owns after the plugin returned: committed when the import finished or the user + // chose to keep the rows, rolled back otherwise. + func finish(commit: Bool) async { + if holdsOwnTransaction { + do { + if commit { + try await commitOwned() + } else { + try await rollbackOwned() + } + } catch { + Self.logger.warning( + "Ending the import's transaction failed: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + } + } + guard lock.withLock({ restoresForeignKeysAtFinish }) else { return } + do { + try await restoreForeignKeyChecks() + } catch { + Self.logger.warning( + "Turning foreign key checks back on failed: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)" + ) + } + } + + // MARK: - Transactions + + // A reused table is emptied right after, in the transaction the sink opened or joined. + func beginTransaction() async throws { + try await openOrJoinTransaction() + try await clearTargetIfDue() + } + + private func openOrJoinTransaction() async throws { + try admitWrite() + guard driver.supportsTransactions else { return } + let state = await driver.sessionTransactionState() + guard WriteTransactionPolicy.opensTransaction(supportsTransactions: true, state: state, statementCount: 2) else { + lock.withLock { + joinedTransaction = state != .idle + ending = nil + } + return + } + try await driver.beginTransaction(mode: .readWrite) + lock.withLock { + ownsTransaction = true + fileEndedTransaction = false + keptWrites = false + ending = nil + } + } + + func commitTransaction() async throws { + switch requestedStop { + case .rollBack?: + throw PluginImportCancellationError() + case .commit?, nil: + try await commitOwned() + } + } + + func rollbackTransaction() async throws { + guard requestedStop != .commit else { return } + try await rollbackOwned() + } + + // A COMMIT in an aborted PostgreSQL transaction ends it as a rollback with no error, so it is not reported as kept. + private func commitOwned() async throws { + guard holdsOwnTransaction else { return } + switch await driver.sessionTransactionState() { + case .abortedTransaction: + try await rollbackOwned() + return + // The file's own COMMIT or ROLLBACK, or a DDL statement, already ended it; a COMMIT now would fail on SQLite. + case .idle: + lock.withLock { + ownsTransaction = false + transactionEnded = true + ending = keptWrites ? .committed : endedBeforeTheImportLocked + } + return + case .explicitTransaction, .implicitTransaction, .unknown: + break + } + try await driver.commitTransaction() + lock.withLock { + ownsTransaction = false + transactionEnded = true + ending = .committed + } + } + + private func rollbackOwned() async throws { + guard holdsOwnTransaction else { return } + switch await driver.sessionTransactionState() { + case .idle: + lock.withLock { + ownsTransaction = false + transactionEnded = true + ending = keptWrites ? .rolledBack : endedBeforeTheImportLocked + } + Self.logger.info("The import's transaction had already ended; no rollback sent") + return + case .explicitTransaction, .implicitTransaction, .abortedTransaction, .unknown: + break + } + try await driver.rollbackTransaction() + lock.withLock { + ownsTransaction = false + transactionEnded = true + ending = .rolledBack + } + } + + // MARK: - Foreign keys + + func disableForeignKeyChecks() async throws { + let statements = driver.foreignKeyCheckStatements(enabled: false) + for statement in statements { + _ = try await driver.execute(query: statement) + } + lock.withLock { disabledForeignKeys = !statements.isEmpty } + } + + // SQLite ignores the re-enable inside a transaction and a rollback takes PostgreSQL's back, so while the sink still + // owns one, as after Stop and Commit, `finish` turns the checks back on once it has ended it. + func enableForeignKeyChecks() async throws { + let restoresNow = lock.withLock { () -> Bool in + guard disabledForeignKeys else { return false } + guard !ownsTransaction else { + restoresForeignKeysAtFinish = true + return false + } + return true + } + guard restoresNow else { return } + try await restoreForeignKeyChecks() + } + + private func restoreForeignKeyChecks() async throws { + for statement in driver.foreignKeyCheckStatements(enabled: true) { + _ = try await driver.execute(query: statement) + } + lock.withLock { + disabledForeignKeys = false + restoresForeignKeysAtFinish = false + } + } + + // MARK: - Statements + + func execute(statement: String) async throws { + try await execute(statement: statement, line: 1) + } + + func execute(statement: String, line: Int) async throws { + try admitWrite() + noteSessionEffect(of: statement) + if let catalogHold, SessionLocalStatements.changesDatabases(statement, databaseType: databaseType) { + await catalogHold.holdOnce() + } + let grammar = SQLImportFile.grammar(for: databaseType, session: driver.settledLexicalFacts) + // MySQL commits the open transaction before a DDL statement runs, so one that then fails still ended it. + do { + if grammar.contains(.batchSeparatorLines), !driver.sendsBatchesWhole { + for piece in SQLStatementScanner.executableStatements(in: statement, grammar: grammar) { + try admitWrite() + _ = try await driver.execute(query: piece.sql) + } + } else { + _ = try await driver.execute(query: statement) + } + } catch { + await noteEffects(of: statement) + throw error + } + await noteEffects(of: statement) + } + + private func noteSessionEffect(of statement: String) { + let effect = SessionLocalStatements.effect(of: [statement], databaseType: databaseType) + let ends = SQLCatalogEffectClassifier.effect(of: statement, grammar: catalogGrammar).endsTransaction + lock.withLock { + recordedEffect.temporaryTables.formUnion(effect.temporaryTables) + recordedEffect.leavesCatalog = recordedEffect.leavesCatalog || effect.leavesCatalog + recordedEffect.usesDatabase = recordedEffect.usesDatabase || effect.usesDatabase + if ends { transactionEnded = true } + } + } + + // Found idle at the import's end with nothing kept: the file rolled the transaction back, or the server ended it. + // Read inside `lock`, which is not recursive. + private var endedBeforeTheImportLocked: ImportTransactionEnding { + fileEndedTransaction ? .rolledBack : .endedByServer + } + + // `SET sql_mode` changes how the next statement reads. A COMMIT, a ROLLBACK or a DDL statement can end the import's + // transaction early, and a write after that runs on its own and stays, so each is read as it lands. + private func noteEffects(of statement: String) async { + let keyword = ExportSnapshotStatements.leadingKeyword(of: statement) + if keyword == "SET" { + _ = await driver.sessionLexicalFacts() + } + guard holdsOwnTransaction || joinsUserTransaction else { return } + let endedBefore = lock.withLock { fileEndedTransaction } + let mayEnd = Self.transactionEndKeywords.contains(keyword) || Self.ddlKeywords.contains(keyword) + let writes = endedBefore && SQLWriteClassifier.isWriteQuery(statement, databaseType: databaseType) + guard mayEnd || writes, await driver.sessionTransactionState() == .idle else { return } + lock.withLock { + if endedBefore { + keptWrites = keptWrites || writes + } else { + fileEndedTransaction = true + fileRolledBack = keyword == "ROLLBACK" + if keyword != "ROLLBACK" { keptWrites = true } + } + } + } + + private static let transactionEndKeywords: Set = ["COMMIT", "END", "ROLLBACK"] + + func copyFromStandardInput(statement: String, line: Int, rows: PluginCopyData) async throws { + try admitWrite() + guard let copier = driver as? any CopyFromStandardInputDriving else { + throw SQLFileParseError.copyFromStandardInput(line: line) + } + _ = try await copier.copyFromStandardInput(statement: statement, rows: rows) + } + + // MARK: - Rows + + func insertRow(_ values: [String: PluginCellValue]) async throws { + try await clearTargetIfDue() + let (writer, target) = try await writing() + try await writer.insertRow(values, into: target, host: host) + } + + func insertRows(_ rows: [[String: PluginCellValue]]) async throws { + try await clearTargetIfDue() + let (writer, target) = try await writing() + try await writer.insertRows(rows, into: target, host: host) + } + + func deleteAllRowsFromTargetTable() async throws { + let (writer, target) = try await writing() + let statement = try writer.clearingStatement(using: target.statements) + _ = try await driver.execute(query: statement) + lock.withLock { clearedTarget = true } + } + + // Before the first row when no transaction was begun, and by the run when a successful import wrote none. + func clearTargetIfDue() async throws { + guard lock.withLock({ clearsTargetFirst && !clearedTarget }) else { return } + try await deleteAllRowsFromTargetTable() + } + + func mappedColumnsAndValues(_ values: [String: PluginCellValue]) -> ([String], [PluginCellValue]) { + rowWriter?.mappedColumnsAndValues(values) ?? ([], []) + } + + private func writing() async throws -> (ImportRowWriter, ImportRowWriter.Target) { + try admitWrite() + guard let rowWriter, let rowTarget else { + throw PluginImportError.importFailed(String(localized: "No target table configured for row import")) + } + if let known = lock.withLock({ writerTarget }) { return (rowWriter, known) } + guard let dialect else { + throw MobileTransferRefusal(reason: String( + format: String(localized: "%@ cannot take rows from a file."), databaseType.mobileDisplayName + )) + } + let columns = try await driver.fetchColumns(table: rowTarget.table, schema: rowTarget.schema, database: rowTarget.database) + let statements = try await dialect.importStatements(table: rowTarget.table, schema: rowTarget.schema, columns: columns) + let target = ImportRowWriter.Target( + tableName: rowTarget.table, + statements: statements, + values: ImportTargetValues(columns: columns, databaseType: databaseType, binaryText: rowTarget.binaryText) + ) + lock.withLock { writerTarget = target } + return (rowWriter, target) + } + + private var host: ImportRowWriter.Host { + ImportRowWriter.Host( + execute: { statement, purpose in try await self.send(statement, purpose) }, + columnsUpdateCannotSet: { try await self.columnsUpdateCannotSet() }, + isCancelled: { [weak self] in self?.isStopped ?? true } + ) + } + + private func send(_ statement: SQLImportStatement, _ purpose: ImportRowWriter.Purpose) async throws -> [[String?]] { + let result = try await driver.executeParameterized( + query: statement.sql, parameters: statement.parameters, typeNames: statement.parameterTypeNames + ) + switch purpose { + case .matchProbe: + return result.rows + case .write: + return [] + } + } + + private func columnsUpdateCannotSet() async throws -> Set { + if let known = lock.withLock({ immutableColumns }) { return known } + guard let dialect, let rowTarget else { return [] } + let fetched = try await dialect.columnsUpdateCannotSet(table: rowTarget.table, schema: rowTarget.schema) + lock.withLock { immutableColumns = fetched } + return fetched + } +} + +nonisolated extension ImportTargetValues { + init(columns: [ColumnInfo], databaseType: DatabaseType, binaryText: ImportBinaryText?) { + self.init( + columns: columns.map { (name: $0.name, typeName: $0.typeName, isGenerated: $0.isServerOwned) }, + databaseTypeId: databaseType.rawValue, + writesNumericBooleans: Self.writesNumericBooleans(databaseType), + binaryText: binaryText + ) + } + + // Each driver plugin's `booleanLiteralStyle`, which iOS has no plugin bundle to ask: PostgreSQL's, which Redshift + // shares, and DuckDB's write `true` and `false`. + static func writesNumericBooleans(_ databaseType: DatabaseType) -> Bool { + ![DatabaseType.postgresql, .redshift, .duckdb].contains(databaseType) + } +} + +nonisolated extension ImportMatchKey { + static func options(columns: [ColumnInfo], indexes: [IndexInfo]) -> [ImportMatchKey] { + options( + columns: columns.map { ImportMatchKey.Column(name: $0.name, isPrimaryKey: $0.isPrimaryKey) }, + indexes: indexes.map(ImportMatchKey.Index.init(mobile:)) + ) + } +} + +nonisolated extension ImportMatchKey.Index { + // The iOS catalog says nothing of validity or expressions, so an index reads as a plain valid one. + init(mobile index: IndexInfo) { + self.init( + name: index.name, + columns: index.columns, + isUnique: index.isUnique, + isPrimary: index.isPrimary, + isPartial: !(index.whereClause ?? "").trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + ) + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Import/MobileImportFormats.swift b/apple/TableProMobile/TableProMobile/Transfer/Import/MobileImportFormats.swift new file mode 100644 index 0000000000..388ce9eee0 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Import/MobileImportFormats.swift @@ -0,0 +1,63 @@ +import Foundation +import TableProModels +import TableProPluginKit + +nonisolated enum MobileImportFormats { + static let csv = CSVImportPlugin() + static let json = JSONImportPlugin() + static let xlsx = XLSXImportPlugin() + static let sql = SQLImportPlugin() + + static var all: [any ImportFormatPlugin] { + [csv, json, xlsx, sql] + } + + static var acceptedFileExtensions: [String] { + all.flatMap { type(of: $0).acceptedFileExtensions.map { $0.lowercased() } } + } + + static func plugin(formatId: String) -> (any ImportFormatPlugin)? { + all.first { type(of: $0).formatId == formatId } + } + + static func frozen(_ shared: any ImportFormatPlugin) -> any ImportFormatPlugin { + let fresh = type(of: shared).init() + RunPluginSettings.copy(from: shared, to: fresh) + return fresh + } + + // By the lowercased extension, never by UTType, as macOS `ImportFileFormatResolver` reads a file. A compressed row + // file is refused by name: only a SQL dump is read through gzip. + static func plugin(forFileExtension fileExtension: String) -> (any ImportFormatPlugin)? { + let lowered = fileExtension.lowercased() + return all.first { type(of: $0).acceptedFileExtensions.contains { $0.lowercased() == lowered } } + } + + static func plugin(for url: URL) -> (any ImportFormatPlugin)? { + let name = url.lastPathComponent.lowercased() + if name.hasSuffix(".gz") { + let inner = url.deletingPathExtension().pathExtension.lowercased() + guard inner.isEmpty || inner == "sql" else { return nil } + return sql + } + return plugin(forFileExtension: url.pathExtension) + } + + // Rows go into a table through SQL; MongoDB and Redis take none (macOS D9), and neither runs a SQL file. + static func rowFormats(for databaseType: DatabaseType) -> [any ImportFormatPlugin] { + guard databaseType.speaksSQL else { return [] } + return all.filter { type(of: $0).requiresTargetTable && accepts($0, databaseTypeId: databaseType.rawValue) } + } + + static func offersSQLFile(for databaseType: DatabaseType) -> Bool { + databaseType.speaksSQL && accepts(sql, databaseTypeId: databaseType.rawValue) + } + + static func accepts(_ plugin: any ImportFormatPlugin, databaseTypeId: String) -> Bool { + let pluginType = type(of: plugin) + guard pluginType.supportedDatabaseTypeIds.isEmpty else { + return pluginType.supportedDatabaseTypeIds.contains(databaseTypeId) + } + return !pluginType.excludedDatabaseTypeIds.contains(databaseTypeId) + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Import/MobileImportSources.swift b/apple/TableProMobile/TableProMobile/Transfer/Import/MobileImportSources.swift new file mode 100644 index 0000000000..e1da110680 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Import/MobileImportSources.swift @@ -0,0 +1,249 @@ +import Foundation +import TableProGzip +import TableProModels +import TableProPluginKit +import TableProSQLFile +import TableProSQLGrammar +import TableProTabularIO + +// The macOS `SqlFileImportSource` over the shared `TableProSQLFile` parser: a `.gz` is expanded beside the staged copy +// first, the session's grammar is read again before each statement, and a dump not in UTF-8 has its literals rewritten. +nonisolated final class MobileSQLFileImportSource: PluginImportSource, @unchecked Sendable { + typealias Expand = @Sendable (_ source: URL, _ destination: URL) async throws -> Void + + private let url: URL + private let encoding: String.Encoding + private let family: SQLDumpFamily + private let grammar: @Sendable () -> SQLLexicalGrammar + private let expand: Expand + private let parser = SQLFileParser() + private let lock = NSLock() + private var expandedURL: URL? + private var expansion: Task? + private var stopped = false + private var onRead: (@Sendable (Int64, Int64) -> Void)? + + // `expand` writes the whole of a `.gz` to the destination and removes it on any failure, a cancel included. + init( + url: URL, + encoding: String.Encoding, + family: SQLDumpFamily, + grammar: @escaping @Sendable () -> SQLLexicalGrammar, + expand: @escaping Expand = { try await GzipFile.decompress(source: $0, destination: $1) } + ) { + self.url = url + self.encoding = encoding + self.family = family + self.grammar = grammar + self.expand = expand + } + + // Latched. The expansion runs in a task of its own, which only this cancel reaches: the import's Stop does not + // cancel the task the plugin reads in. + func stop() { + let running = lock.withLock { () -> Task? in + stopped = true + return expansion + } + running?.cancel() + } + + // Bytes read from the file the parser opens, which for a `.gz` is the expanded copy. + func reportsReading(to handler: @escaping @Sendable (_ bytesRead: Int64, _ fileSize: Int64) -> Void) { + lock.withLock { onRead = handler } + } + + func fileURL() -> URL { + url + } + + func fileSizeBytes() -> Int64 { + let target = lock.withLock { expandedURL } ?? url + let size = try? FileManager.default.attributesOfItem(atPath: target.path(percentEncoded: false))[.size] as? NSNumber + return size?.int64Value ?? 0 + } + + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + let readable = try await resolvedURL() + let parsed = parser.parseFile(url: readable, encoding: encoding, grammar: grammar, onRead: lock.withLock { onRead }) + guard SQLDumpStatementRewriter.rewrites(fileDecodedFrom: encoding) else { return parsed } + return rewriter.rewriting(parsed) + } + + func elements() async throws -> AsyncThrowingStream { + let readable = try await resolvedURL() + var parsed = parser.parseElements(url: readable, encoding: encoding, grammar: grammar, onRead: lock.withLock { onRead }) + if SQLDumpStatementRewriter.rewrites(fileDecodedFrom: encoding) { + parsed = rewriter.rewriting(parsed) + } + let elements = SQLFileElements(parsed) + return AsyncThrowingStream(unfolding: { try await elements.next() }) + } + + func cleanup() { + let expanded = lock.withLock { () -> URL? in + defer { expandedURL = nil } + return expandedURL + } + if let expanded { try? FileManager.default.removeItem(at: expanded) } + } + + deinit { + if let expandedURL { try? FileManager.default.removeItem(at: expandedURL) } + } + + private var rewriter: SQLDumpStatementRewriter { + SQLDumpStatementRewriter(encoding: encoding, family: family, grammar: grammar) + } + + private func resolvedURL() async throws -> URL { + if let expanded = lock.withLock({ expandedURL }) { return expanded } + guard SQLImportFile.isCompressed(url) else { return url } + let expanded = SQLImportFile.expandedURL(for: url) + let source = url + let expand = expand + let task = Task { try await expand(source, expanded) } + let stopped = lock.withLock { () -> Bool in + expansion = task + return self.stopped + } + if stopped { task.cancel() } + defer { lock.withLock { expansion = nil } } + try await withTaskCancellationHandler { + try await task.value + } onCancel: { + task.cancel() + } + lock.withLock { expandedURL = expanded } + return expanded + } +} + +// The parser's elements as the plugin reads them, one pull at a time. +nonisolated private final class SQLFileElements: @unchecked Sendable { + private var iterator: AsyncThrowingStream.AsyncIterator + + init(_ elements: AsyncThrowingStream) { + iterator = elements.makeAsyncIterator() + } + + func next() async throws -> PluginImportElement? { + switch try await iterator.next() { + case let .statement(statement, line)?: + return .statement(statement, lineNumber: line) + case let .copyFromStandardInput(statement, line, readRows)?: + return .copyFromStandardInput(statement: statement, lineNumber: line, rows: PluginCopyData(read: readRows)) + case nil: + return nil + } + } +} + +nonisolated struct SQLImportFilePreview: Sendable, Equatable { + // What to read the file as: the selection, or what the file says it is when the selection does not decode it. + let encoding: String.Encoding + // Set when the preview did not decode as selected and the file named or looked like another encoding. + let detectedEncoding: String.Encoding? + let decodes: Bool + let isCompressed: Bool +} + +nonisolated enum SQLImportFile { + private static let previewBytes = 5 * 1_024 * 1_024 + + static var encodingChoices: [TabularTextEncoding] { + TabularTextEncoding.allCases + } + + static func isCompressed(_ url: URL) -> Bool { + url.pathExtension.lowercased() == "gz" + } + + static func expandedURL(for url: URL) -> URL { + let inner = url.deletingPathExtension().pathExtension.lowercased() + return url.deletingLastPathComponent() + .appendingPathComponent(UUID().uuidString + "." + (inner.isEmpty ? "sql" : inner), isDirectory: false) + } + + // macOS `TransactionEngineFamily.dumpFamily`, keyed by type id. + static func dumpFamily(for databaseType: DatabaseType) -> SQLDumpFamily { + switch databaseType.rawValue { + case "MySQL", "MariaDB", "TiDB", "OceanBase": + return .mysql + case "PostgreSQL", "Redshift", "CockroachDB", "Greenplum", "AlloyDB", "Citus", "KingbaseES": + return .postgres + default: + return .other + } + } + + static func grammar(for databaseType: DatabaseType, session: SQLSessionLexicalFacts?) -> SQLLexicalGrammar { + SQLLexicalReadings.resolve(databaseTypeId: databaseType.rawValue, declared: nil, session: session).execution + } + + // The import dialog's read of a picked dump, as macOS `ImportDialog.loadFile` makes it. + @concurrent + static func preview( + _ file: StagedImportFile, + databaseType: DatabaseType, + selected: String.Encoding = .utf8, + session: SQLSessionLexicalFacts? = nil + ) async throws -> SQLImportFilePreview { + let compressed = isCompressed(file.url) + let head = try await previewData(of: file.url, compressed: compressed) + if SQLDumpEncodingChoice.previewDecodes(head, as: selected, isWholeFile: head.count < previewBytes) { + return SQLImportFilePreview(encoding: selected, detectedEncoding: nil, decodes: true, isCompressed: compressed) + } + let detected = SQLDumpEncodingChoice.encoding( + replacing: selected, + forPreview: head, + isWholeFile: head.count < previewBytes, + family: dumpFamily(for: databaseType), + grammar: grammar(for: databaseType, session: session) + ) + return SQLImportFilePreview( + encoding: detected ?? selected, detectedEncoding: detected, decodes: detected != nil, isCompressed: compressed + ) + } + + @concurrent + static func countStatements( + _ file: StagedImportFile, + encoding: String.Encoding, + databaseType: DatabaseType, + session: SQLSessionLexicalFacts? = nil + ) async throws -> Int { + var readable = file.url + var expanded: URL? + if isCompressed(file.url) { + let target = expandedURL(for: file.url) + try await GzipFile.decompress(source: file.url, destination: target) + readable = target + expanded = target + } + defer { if let expanded { try? FileManager.default.removeItem(at: expanded) } } + return try await SQLFileParser().countStatements( + url: readable, encoding: encoding, grammar: grammar(for: databaseType, session: session) + ) + } + + // The first 5 MiB a parse would read, through the inflater for a `.gz`. + private static func previewData(of url: URL, compressed: Bool) async throws -> Data { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + guard compressed else { return try handle.read(upToCount: previewBytes) ?? Data() } + // No expansion ratio, as for a picked file: a dump of zero-filled BLOBs packs past 500:1. + let inflater = try GzipInflater(budget: GzipFile.budget(availableSpace: nil)) + var output = Data() + while output.count < previewBytes, let block = try handle.read(upToCount: 64 * 1_024), !block.isEmpty { + try Task.checkCancellation() + try block.withUnsafeBytes { raw in + try inflater.inflate(raw) { piece in + guard output.count < previewBytes else { return } + output.append(contentsOf: piece.prefix(previewBytes - output.count)) + } + } + } + return output + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/Import/TransferSessionHost.swift b/apple/TableProMobile/TableProMobile/Transfer/Import/TransferSessionHost.swift new file mode 100644 index 0000000000..266486cf09 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/Import/TransferSessionHost.swift @@ -0,0 +1,110 @@ +import Foundation +import TableProDatabase +import TableProModels +import TableProSQLGrammar + +// What an import is admitted with, read in the same turn that marks the session as the import's. +nonisolated struct ImportAdmission: Sendable { + let driver: any DatabaseDriver + let safeModeLevel: SafeModeLevel +} + +// The window's session an import writes on, as macOS imports on the session driver: the whole run holds the session, +// so every other statement on it, from any window, is refused until it ends, and a session lost under it is restored. +@MainActor +protocol TransferSessionHost: AnyObject, Sendable { + // The connection's Safe Mode as it is now, which another window can change while a sheet is open. + var transferSafeModeLevel: SafeModeLevel { get } + // The database the window's session is on, for an engine whose driver cannot say. + var transferDatabase: String? { get } + // Refused at once (`SessionBusyForImport`, `SessionHeldByImport`) unless nothing else runs on the session and + // nothing moves it. A run that ends on an error comes back as a value, so `failed` says which values may have + // lost the session. + func trackImport( + failed: @Sendable (T) -> Bool, + _ work: @Sendable (ImportAdmission) async throws -> T + ) async throws -> T + // Work an export runs on the session's own driver, as a SQLite file holding the user's transaction requires. + func trackTransfer(_ work: @Sendable () async throws -> T) async throws -> T + func noteTransferChangedCatalog(_ kinds: SQLCatalogObjectKinds) + // What a SQL file left on the session the import ran on, recorded as a query run's is, before the catalog reloads. + func noteTransferSessionEffect(_ effect: SessionLocalStatements.Effect, on driver: any DatabaseDriver) + // The window's database and schema follow where a SQL file left the session, as they follow a query run. + func followTransferPlacement(movesDatabase: Bool, movesSchema: Bool) async + // Background time ran out while the import's own rollback was still on the wire: closing the connection makes the + // server end the transaction, and the next use of the session dials again. + func abandonTransferSession() async + // SQL Server cannot drop or restore a database TablePro's catalog connection sits in (`holdCatalogConnections`). + func holdTransferCatalogConnections() async + func releaseTransferCatalogConnections() +} + +// Taken at a SQL file's first statement that drops, alters or restores a database, and given back when the run ends. +nonisolated final class ImportCatalogHold: @unchecked Sendable { + private let hold: @Sendable () async -> Void + private let release: @Sendable () async -> Void + private let lock = NSLock() + private var held = false + + init(hold: @escaping @Sendable () async -> Void, release: @escaping @Sendable () async -> Void) { + self.hold = hold + self.release = release + } + + func holdOnce() async { + let first = lock.withLock { () -> Bool in + defer { held = true } + return !held + } + if first { await hold() } + } + + func releaseIfHeld() async { + let wasHeld = lock.withLock { () -> Bool in + defer { held = false } + return held + } + if wasHeld { await release() } + } +} + +extension ConnectionCoordinator: TransferSessionHost { + var transferSafeModeLevel: SafeModeLevel { + connection.safeModeLevel + } + + var transferDatabase: String? { + activeDatabase.isEmpty ? nil : activeDatabase + } + + // An export only reads, so it leaves nothing a fresh connection would lack. + func trackTransfer(_ work: @Sendable () async throws -> T) async throws -> T { + try await track(reading: true) { try await work() } + } + + func noteTransferChangedCatalog(_ kinds: SQLCatalogObjectKinds) { + noteCatalogChanged(kinds) + } + + func noteTransferSessionEffect(_ effect: SessionLocalStatements.Effect, on driver: any DatabaseDriver) { + noteSessionEffect(effect, on: driver) + } + + func followTransferPlacement(movesDatabase: Bool, movesSchema: Bool) async { + await followSessionPlacement(after: SessionMove(database: movesDatabase, schema: movesSchema)) + } + + func holdTransferCatalogConnections() async { + await connectionManager.holdCatalogConnections(for: connection.id) + } + + func releaseTransferCatalogConnections() { + connectionManager.releaseCatalogConnections(for: connection.id) + } + + func abandonTransferSession() async { + guard let driver = session?.driver else { return } + try? await driver.cancelCurrentQuery() + try? await driver.disconnect() + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/UI/DataImportConnectionChooser.swift b/apple/TableProMobile/TableProMobile/Transfer/UI/DataImportConnectionChooser.swift new file mode 100644 index 0000000000..6b39f49636 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/UI/DataImportConnectionChooser.swift @@ -0,0 +1,73 @@ +import SwiftUI +import TableProModels + +// A file opened from another app while no connection is open: the person picks where it goes, and that connection +// opens with the import sheet over it. +struct DataImportConnectionChooser: View { + let fileURL: URL + let connections: [DatabaseConnection] + let choose: (UUID) -> Void + + @Environment(\.dismiss) private var dismiss + + init(fileURL: URL, connections: [DatabaseConnection], choose: @escaping (UUID) -> Void) { + self.fileURL = fileURL + self.connections = connections + self.choose = choose + } + + var body: some View { + NavigationStack { + Group { + if connections.isEmpty { + ContentUnavailableView { + Label("No Connections", systemImage: "cylinder.split.1x2") + } description: { + Text("Add a connection, then open the file again.") + } + } else { + List { + Section { + ForEach(connections) { connection in + Button { + choose(connection.id) + } label: { + HStack(spacing: 12) { + DatabaseIconView(type: connection.type, size: 22) + .frame(width: 28) + VStack(alignment: .leading, spacing: 2) { + Text(verbatim: connection.name.isEmpty ? connection.host : connection.name) + .foregroundStyle(.primary) + Text(verbatim: connection.type.mobileDisplayName) + .font(.caption) + .foregroundStyle(.secondary) + } + } + } + } + } header: { + Text(verbatim: fileURL.lastPathComponent) + .textCase(nil) + } + } + } + } + .navigationTitle(Text("Import Into")) + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .cancellationAction) { + Button("Cancel") { + discardInboxCopy() + dismiss() + } + } + } + } + } + + // Another app's "Open in" leaves its copy in the app's own Inbox, which nothing else would empty. + private func discardInboxCopy() { + guard ImportFileStaging.isInInbox(fileURL) else { return } + try? FileManager.default.removeItem(at: fileURL) + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/UI/ExportFormatOptionsSection.swift b/apple/TableProMobile/TableProMobile/Transfer/UI/ExportFormatOptionsSection.swift new file mode 100644 index 0000000000..1e8f777d49 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/UI/ExportFormatOptionsSection.swift @@ -0,0 +1,208 @@ +import SwiftUI +import TableProPluginKit + +// The plugin's own `settings` are the model: these rows write into it and the plugin saves it, so iOS keeps no +// second copy of any option. +struct ExportFormatOptionsSection: View { + let plugin: any ExportFormatPlugin + + var body: some View { + if let csv = plugin as? CSVExportPlugin { + CSVExportOptionsForm(plugin: csv) + } else if let json = plugin as? JSONExportPlugin { + JSONExportOptionsForm(plugin: json) + } else if let sql = plugin as? SQLExportPlugin { + SQLExportOptionsForm(plugin: sql) + } else if let xlsx = plugin as? XLSXExportPlugin { + XLSXExportOptionsForm(plugin: xlsx) + } else if let markdown = plugin as? MarkdownExportPlugin { + MarkdownExportOptionsForm(plugin: markdown) + } else if let mql = plugin as? MQLExportPlugin { + MQLExportOptionsForm(plugin: mql) + } + } +} + +private struct CSVExportOptionsForm: View { + @ObservedObject var plugin: CSVExportPlugin + + var body: some View { + Section { + Picker("Delimiter", selection: $plugin.settings.delimiter) { + Text("Comma (,)").tag(CSVDelimiter.comma) + Text("Semicolon (;)").tag(CSVDelimiter.semicolon) + Text("Tab").tag(CSVDelimiter.tab) + Text("Pipe (|)").tag(CSVDelimiter.pipe) + } + .pickerStyle(.menu) + Picker("Quote", selection: $plugin.settings.quoteHandling) { + ForEach(CSVQuoteHandling.allCases) { handling in + Text(verbatim: handling.displayName).tag(handling) + } + } + .pickerStyle(.menu) + Picker("Line break", selection: $plugin.settings.lineBreak) { + ForEach(CSVLineBreak.allCases) { lineBreak in + Text(verbatim: lineBreak.rawValue).tag(lineBreak) + } + } + .pickerStyle(.menu) + Picker("Decimal", selection: $plugin.settings.decimalFormat) { + ForEach(CSVDecimalFormat.allCases) { format in + Text(verbatim: format.rawValue).tag(format) + } + } + .pickerStyle(.menu) + Toggle("Put field names in the first row", isOn: $plugin.settings.includeFieldNames) + Toggle("Convert NULL to EMPTY", isOn: $plugin.settings.convertNullToEmpty) + Toggle("Convert line break to space", isOn: $plugin.settings.convertLineBreakToSpace) + } header: { + Text("CSV") + } + Section { + Picker("Encoding", selection: $plugin.settings.encoding) { + ForEach(PluginTextEncoding.allCases) { encoding in + Text(verbatim: encoding.displayName).tag(encoding) + } + } + .pickerStyle(.menu) + Toggle("Include byte order mark", isOn: $plugin.settings.writesByteOrderMark) + .disabled(!plugin.settings.encoding.supportsByteOrderMark) + } footer: { + Text("Excel on Windows needs this to read UTF-8. Other readers show it as part of the first value.") + } + Section { + Toggle("Sanitize formula-like values", isOn: $plugin.settings.sanitizeFormulas) + } footer: { + Text(""" + For a file opened in a spreadsheet. Values that would run as formulas, such as those starting \ + with =, +, - or @, get a leading single quote, which stays in the value for anything that \ + imports the file. + """) + } + } +} + +private struct JSONExportOptionsForm: View { + @ObservedObject var plugin: JSONExportPlugin + + var body: some View { + Section { + Picker("Layout", selection: $plugin.settings.layout) { + ForEach(JSONExportLayout.allCases) { layout in + Text(verbatim: layout.label).tag(layout) + } + } + .pickerStyle(.menu) + Toggle("Pretty print (formatted output)", isOn: $plugin.settings.prettyPrint) + .disabled(plugin.settings.layout == .newlineDelimited) + Toggle("Include NULL values", isOn: $plugin.settings.includeNullValues) + Toggle("Preserve all values as strings", isOn: $plugin.settings.preserveAllAsStrings) + } header: { + Text("JSON") + } footer: { + Text("Keep leading zeros in ZIP codes, phone numbers, and IDs by outputting all values as strings") + } + } +} + +private struct SQLExportOptionsForm: View { + @ObservedObject var plugin: SQLExportPlugin + + var body: some View { + Section { + Picker("Rows per INSERT", selection: $plugin.settings.batchSize) { + ForEach(SQLExportOptions.batchSizeChoices, id: \.self) { size in + if size == 1 { + Text("1 (no batching)").tag(size) + } else { + Text(verbatim: "\(size)").tag(size) + } + } + } + .pickerStyle(.menu) + Picker("Max INSERT size", selection: $plugin.settings.maxStatementBytes) { + ForEach(SQLExportOptions.statementSizeChoices, id: \.self) { size in + Text(verbatim: SQLExportOptions.statementSizeLabel(size)).tag(size) + } + } + .pickerStyle(.menu) + Picker("On existing rows", selection: $plugin.settings.insertMode) { + ForEach(SQLExportInsertMode.allCases) { mode in + Text(verbatim: mode.label).tag(mode) + } + } + .pickerStyle(.menu) + Picker("Split every", selection: $plugin.settings.splitSizeMegabytes) { + ForEach(SQLExportOptions.splitSizeChoices, id: \.self) { size in + if size == 0 { + Text("One file").tag(size) + } else { + Text(verbatim: "\(size) MB").tag(size) + } + } + } + .pickerStyle(.menu) + } header: { + Text("SQL") + } + Section { + Toggle("Read every table at one snapshot", isOn: $plugin.settings.consistentSnapshot) + Toggle("Exclude the AUTO_INCREMENT counter", isOn: $plugin.settings.excludeAutoIncrementValue) + Toggle("Exclude DEFINER clauses", isOn: $plugin.settings.excludeDefiner) + Toggle("Compress the file using Gzip", isOn: $plugin.settings.compressWithGzip) + } footer: { + Text("AUTO_INCREMENT and DEFINER apply to MySQL and MariaDB.") + } + } +} + +private struct XLSXExportOptionsForm: View { + @ObservedObject var plugin: XLSXExportPlugin + + var body: some View { + Section { + Toggle("Include column headers", isOn: $plugin.settings.includeHeaderRow) + Toggle("Convert NULL to empty", isOn: $plugin.settings.convertNullToEmpty) + } header: { + Text("Excel") + } + } +} + +private struct MarkdownExportOptionsForm: View { + @ObservedObject var plugin: MarkdownExportPlugin + + var body: some View { + Section { + Toggle("Align columns", isOn: $plugin.settings.alignsColumns) + Toggle("Write each table's name as a heading", isOn: $plugin.settings.includesTableNames) + LabeledContent("NULL shows as") { + LiteralTextField(String("NULL"), text: $plugin.settings.nullPlaceholder) + } + } header: { + Text("Markdown") + } footer: { + Text("An empty Markdown cell and a cell holding an empty string render the same, so a null needs its own text") + } + } +} + +private struct MQLExportOptionsForm: View { + @ObservedObject var plugin: MQLExportPlugin + + var body: some View { + Section { + Picker("Rows per batch", selection: $plugin.settings.batchSize) { + ForEach(MQLExportOptions.batchSizeChoices, id: \.self) { size in + Text(verbatim: "\(size)").tag(size) + } + } + .pickerStyle(.menu) + } header: { + Text("MQL") + } footer: { + Text("Maximum documents per batch. Large documents use smaller batches to fit the server limit.") + } + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/UI/ImportFieldMapping.swift b/apple/TableProMobile/TableProMobile/Transfer/UI/ImportFieldMapping.swift new file mode 100644 index 0000000000..95f7c61d90 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/UI/ImportFieldMapping.swift @@ -0,0 +1,106 @@ +import Foundation + +// The mapping of a file's fields onto an existing table's columns. The rules are the shared `ImportColumnMatcher`; +// this only keeps the sheet's current choice per field and what was saved for the table. +nonisolated struct ImportFieldMapping: Equatable, Sendable { + nonisolated struct Row: Identifiable, Equatable, Sendable { + let field: String + var choice: ImportFieldChoice + + var id: String { field } + } + + private(set) var columns: [String] = [] + private(set) var rows: [Row] = [] + private(set) var saved: [String: ImportMappingOverride] = [:] + private(set) var showsSavedMapping = false + + var fields: [String] { rows.map(\.field) } + + var choices: [ImportFieldChoice] { rows.map(\.choice) } + + var columnMapping: [String: String] { + var mapping: [String: String] = [:] + for row in rows { + guard let column = row.choice.mappedColumn else { continue } + mapping[row.field] = column + } + return mapping + } + + var hasMappedField: Bool { + rows.contains { $0.choice.mappedColumn != nil } + } + + var mapsOneColumnTwice: Bool { + let targets = rows.compactMap { $0.choice.mappedColumn?.lowercased() } + return Set(targets).count != targets.count + } + + var canUseSavedMapping: Bool { + guard !saved.isEmpty else { return false } + let restored = ImportColumnMatcher.applying(saved, fields: fields, columns: columns) + guard restored != choices else { return false } + return !ImportColumnMatcher.restoredFields(saved, fields: fields, columns: columns, choices: restored).isEmpty + } + + // What to remember for the table: only the departures from the name match. + var overrides: [String: ImportMappingOverride] { + ImportColumnMatcher.overrides(fields: fields, columns: columns, choices: choices) + } + + // A reload of the same table keeps the choices made in the sheet, then the saved ones, then the name match. + mutating func load( + fields: [String], + columns: [String], + saved: [String: ImportMappingOverride], + keepingChoices: Bool + ) { + var tiers = [saved.mapValues(\.choice)] + if keepingChoices { + let session = Dictionary(rows.map { ($0.field, $0.choice) }, uniquingKeysWith: { first, _ in first }) + tiers.insert(session, at: 0) + } + self.columns = columns + self.saved = saved + apply(ImportColumnMatcher.resolve(fields: fields, columns: columns, preferring: tiers), to: fields) + showsSavedMapping = (keepingChoices ? showsSavedMapping : true) && appliesSavedChoice + } + + mutating func clear() { + columns = [] + rows = [] + saved = [:] + showsSavedMapping = false + } + + mutating func matchByName() { + apply(ImportColumnMatcher.byName(fields: fields, columns: columns), to: fields) + showsSavedMapping = false + } + + mutating func matchByPosition() { + apply(ImportColumnMatcher.byPosition(fields: fields, columns: columns), to: fields) + showsSavedMapping = false + } + + mutating func useSavedMapping() { + apply(ImportColumnMatcher.applying(saved, fields: fields, columns: columns), to: fields) + showsSavedMapping = appliesSavedChoice + } + + // Nil is "Do not import". + mutating func map(_ field: String, to column: String?) { + guard let index = rows.firstIndex(where: { $0.field == field }) else { return } + rows[index].choice = column.map { ImportFieldChoice(include: true, column: $0) } ?? .skipped + showsSavedMapping = false + } + + private var appliesSavedChoice: Bool { + !ImportColumnMatcher.restoredFields(saved, fields: fields, columns: columns, choices: choices).isEmpty + } + + private mutating func apply(_ choices: [ImportFieldChoice], to fields: [String]) { + rows = zip(fields, choices).map { Row(field: $0, choice: $1) } + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/UI/ImportFormatOptionsSection.swift b/apple/TableProMobile/TableProMobile/Transfer/UI/ImportFormatOptionsSection.swift new file mode 100644 index 0000000000..7b2f2bfdc0 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/UI/ImportFormatOptionsSection.swift @@ -0,0 +1,236 @@ +import SwiftUI +import TableProPluginKit +import TableProTabularIO + +// Rows over the plugin's own `settings`, as on the Mac; the plugin saves what they write. "Delete existing rows +// first" belongs to the Insert write mode, so the destination section draws it with `DeleteExistingRowsToggle`. +struct ImportFormatOptionsSection: View { + let plugin: any ImportFormatPlugin + + var body: some View { + if let csv = plugin as? CSVImportPlugin { + CSVImportOptionsForm(plugin: csv) + } else if let json = plugin as? JSONImportPlugin { + JSONImportOptionsForm(plugin: json) + } else if let xlsx = plugin as? XLSXImportPlugin { + XLSXImportOptionsForm(plugin: xlsx) + } else if let sql = plugin as? SQLImportPlugin { + SQLImportOptionsForm(plugin: sql) + } + } +} + +struct DeleteExistingRowsToggle: View { + let plugin: any ImportFormatPlugin + + var body: some View { + if let csv = plugin as? CSVImportPlugin { + CSVDeleteExistingToggle(plugin: csv) + } else if let json = plugin as? JSONImportPlugin { + JSONDeleteExistingToggle(plugin: json) + } else if let xlsx = plugin as? XLSXImportPlugin { + XLSXDeleteExistingToggle(plugin: xlsx) + } + } +} + +// The plugin runs a clear before the rows whatever the write mode, so a mode that matches rows turns it off first. +enum ImportPluginSettings { + static func deletesExistingRows(_ plugin: any ImportFormatPlugin) -> Bool { + plugin.rowImportConfiguration?.deleteExistingRows ?? false + } + + static func setDeletesExistingRows(_ deletes: Bool, on plugin: any ImportFormatPlugin) { + if let csv = plugin as? CSVImportPlugin, csv.settings.deleteExistingRows != deletes { + csv.settings.deleteExistingRows = deletes + } else if let json = plugin as? JSONImportPlugin, json.settings.deleteExistingRows != deletes { + json.settings.deleteExistingRows = deletes + } else if let xlsx = plugin as? XLSXImportPlugin, xlsx.settings.deleteExistingRows != deletes { + xlsx.settings.deleteExistingRows = deletes + } + } +} + +private struct ErrorHandlingPicker: View { + let selection: Binding + + var body: some View { + Picker("On error", selection: selection) { + Text("Stop and Rollback").tag(ImportErrorHandling.stopAndRollback) + Text("Stop and Commit").tag(ImportErrorHandling.stopAndCommit) + Text("Skip and Continue").tag(ImportErrorHandling.skipAndContinue) + } + .pickerStyle(.menu) + } +} + +private struct TransactionFooter: View { + let errorHandling: ImportErrorHandling + + var body: some View { + if errorHandling == .skipAndContinue { + Text("Not available in skip-and-continue mode") + } else { + Text("Insert all rows in a single transaction. If any row fails, all changes are rolled back.") + } + } +} + +private struct CSVImportOptionsForm: View { + @ObservedObject var plugin: CSVImportPlugin + + var body: some View { + Section { + Picker("Delimiter", selection: $plugin.settings.delimiter) { + Text("Auto-detect").tag(CSVImportOptions.Delimiter.auto) + Text("Comma (,)").tag(CSVImportOptions.Delimiter.comma) + Text("Semicolon (;)").tag(CSVImportOptions.Delimiter.semicolon) + Text("Tab").tag(CSVImportOptions.Delimiter.tab) + Text("Pipe (|)").tag(CSVImportOptions.Delimiter.pipe) + } + .pickerStyle(.menu) + Picker("Quote character", selection: $plugin.settings.quoteCharacter) { + Text("Double quote (\")").tag(CSVImportOptions.QuoteCharacter.doubleQuote) + Text("Single quote (')").tag(CSVImportOptions.QuoteCharacter.singleQuote) + } + .pickerStyle(.menu) + Picker("Encoding", selection: $plugin.settings.encoding) { + Text("Auto-detect").tag(CSVImportOptions.TextEncoding.auto) + ForEach(CSVImportOptions.TextEncoding.allCases.filter { $0 != .auto }) { option in + Text(verbatim: option.tabularEncoding?.displayName ?? option.rawValue).tag(option) + } + } + .pickerStyle(.menu) + Toggle("First row is a header", isOn: $plugin.settings.hasHeaderRow) + Toggle("Trim leading and trailing spaces", isOn: $plugin.settings.trimWhitespace) + Toggle("Treat empty values as NULL", isOn: $plugin.settings.emptyAsNull) + LabeledContent("NULL text") { + LiteralTextField(String("\\N"), text: $plugin.settings.nullString) + } + } header: { + Text("CSV") + } footer: { + Text("An extra value that should be imported as NULL, for example \\N.") + } + Section { + ErrorHandlingPicker(selection: $plugin.settings.errorHandling) + Toggle("Wrap in transaction (BEGIN/COMMIT)", isOn: $plugin.settings.wrapInTransaction) + .disabled(plugin.settings.errorHandling == .skipAndContinue) + } footer: { + TransactionFooter(errorHandling: plugin.settings.errorHandling) + } + } +} + +private struct JSONImportOptionsForm: View { + @ObservedObject var plugin: JSONImportPlugin + + var body: some View { + Section { + ErrorHandlingPicker(selection: $plugin.settings.errorHandling) + Toggle("Wrap in transaction (BEGIN/COMMIT)", isOn: $plugin.settings.wrapInTransaction) + .disabled(plugin.settings.errorHandling == .skipAndContinue) + } footer: { + TransactionFooter(errorHandling: plugin.settings.errorHandling) + } + } +} + +private struct XLSXImportOptionsForm: View { + @ObservedObject var plugin: XLSXImportPlugin + + var body: some View { + Section { + Toggle("First row holds column names", isOn: $plugin.settings.hasHeaderRow) + Toggle("Trim whitespace", isOn: $plugin.settings.trimWhitespace) + Toggle("Treat empty cells as NULL", isOn: $plugin.settings.emptyAsNull) + } header: { + Text("Excel") + } footer: { + Text("Off names the columns column1, column2 and so on") + } + Section { + ErrorHandlingPicker(selection: $plugin.settings.errorHandling) + Toggle("Wrap in transaction", isOn: $plugin.settings.wrapInTransaction) + .disabled(plugin.settings.errorHandling == .skipAndContinue) + } footer: { + TransactionFooter(errorHandling: plugin.settings.errorHandling) + } + } +} + +private struct SQLImportOptionsForm: View { + @ObservedObject var plugin: SQLImportPlugin + + var body: some View { + Section { + ErrorHandlingPicker(selection: $plugin.settings.errorHandling) + Toggle("Wrap in transaction (BEGIN/COMMIT)", isOn: $plugin.settings.wrapInTransaction) + .disabled(plugin.settings.errorHandling == .skipAndContinue) + Toggle("Disable foreign key checks", isOn: $plugin.settings.disableForeignKeyChecks) + } footer: { + if plugin.settings.errorHandling == .skipAndContinue { + Text("Not available in skip-and-continue mode") + } else { + Text(""" + Runs every statement in one transaction, so a failure rolls back what the import changed. A \ + database that commits CREATE, ALTER and DROP as they run, such as MySQL, keeps those statements \ + and everything before them. + """) + } + } + } +} + +private struct CSVDeleteExistingToggle: View { + @ObservedObject var plugin: CSVImportPlugin + + var body: some View { + Toggle("Delete existing rows first", isOn: $plugin.settings.deleteExistingRows) + } +} + +private struct JSONDeleteExistingToggle: View { + @ObservedObject var plugin: JSONImportPlugin + + var body: some View { + Toggle("Delete existing rows first", isOn: $plugin.settings.deleteExistingRows) + } +} + +private struct XLSXDeleteExistingToggle: View { + @ObservedObject var plugin: XLSXImportPlugin + + var body: some View { + Toggle("Delete existing rows first", isOn: $plugin.settings.deleteExistingRows) + } +} + +// The fields come from a read of the file that depends on parsing options (delimiter, header, encoding), so a change +// to them reads the file again. +struct ImportDetectionSignatureObserver: View { + let plugin: any ImportFormatPlugin + let onChange: () -> Void + + var body: some View { + if let csv = plugin as? CSVImportPlugin { + SignatureWatch(plugin: csv, onChange: onChange) + } else if let xlsx = plugin as? XLSXImportPlugin { + SignatureWatch(plugin: xlsx, onChange: onChange) + } else if let json = plugin as? JSONImportPlugin { + SignatureWatch(plugin: json, onChange: onChange) + } + } +} + +private struct SignatureWatch: View { + @ObservedObject var plugin: Plugin + let onChange: () -> Void + + var body: some View { + Color.clear + .frame(width: 0, height: 0) + .accessibilityHidden(true) + .onChange(of: plugin.fieldDetectionSignature) { onChange() } + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/UI/ImportPickerTypes.swift b/apple/TableProMobile/TableProMobile/Transfer/UI/ImportPickerTypes.swift new file mode 100644 index 0000000000..a11492ae47 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/UI/ImportPickerTypes.swift @@ -0,0 +1,19 @@ +import Foundation +import TableProPluginKit +import UniformTypeIdentifiers + +// What the file picker dims and lights. Not the gate: the picked file's extension decides the format again. +nonisolated enum ImportPickerTypes { + static func contentTypes(for plugins: [any ImportFormatPlugin]) -> [UTType] { + var seen = Set() + var types: [UTType] = [] + for plugin in plugins { + for fileExtension in type(of: plugin).acceptedFileExtensions { + guard let type = UTType(filenameExtension: fileExtension.lowercased()), + seen.insert(type.identifier).inserted else { continue } + types.append(type) + } + } + return types + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/UI/MobileExportModel.swift b/apple/TableProMobile/TableProMobile/Transfer/UI/MobileExportModel.swift new file mode 100644 index 0000000000..cd0c0b7a30 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/UI/MobileExportModel.swift @@ -0,0 +1,427 @@ +import Foundation +import Observation +import TableProDatabase +import TableProModels +import TableProPluginKit + +// What an export is asked to cover. Each entry point offers the scopes that make sense where it was opened. +nonisolated struct ExportScopeOption: Identifiable, Sendable { + nonisolated enum Kind: String, Sendable, Hashable { + case currentView + case wholeTable + case selectedRows + case result + case database + } + + nonisolated enum Source: Sendable { + case query(ExportSelect) + // The user's statement run again, under the engine's read-only mode only. + case rerun(String) + case rows(ExportRowSet) + case table(String) + case database + } + + let kind: Kind + let source: Source + let detail: String? + + var id: Kind { kind } + + init(kind: Kind, source: Source, detail: String? = nil) { + self.kind = kind + self.source = source + self.detail = detail + } + + var title: String { + switch kind { + case .currentView: String(localized: "Filtered and Sorted") + case .wholeTable: String(localized: "Whole Table") + case .selectedRows: String(localized: "Selected Rows") + case .result: String(localized: "Query Result") + case .database: String(localized: "Database Objects") + } + } + + var listsObjects: Bool { + switch source { + case .table, .database: true + case .query, .rerun, .rows: false + } + } +} + +nonisolated struct MobileExportInput: Sendable { + let title: String + let fileName: String + let connectionId: UUID + let databaseType: DatabaseType + let database: String? + let schema: String? + let safeModeLevel: SafeModeLevel + let scopes: [ExportScopeOption] +} + +nonisolated struct ExportFormatChoice: Identifiable, Sendable, Equatable { + let id: String + let name: String +} + +nonisolated struct ExportObjectGroup: Identifiable, Sendable { + let kind: PluginExportObjectKind + let rows: [ExportObjectRow] + + var id: PluginExportObjectKind { kind } +} + +nonisolated struct ExportObjectRow: Identifiable, Sendable { + let id: String + var object: PluginExportTable + var isSelected: Bool + + // The type too: a MySQL function and procedure can share a name and a signature, and both are routines. + init(object: PluginExportTable, isSelected: Bool) { + self.id = [ + object.kind.rawValue, object.tableType, object.databaseName, object.schema ?? "", object.parentTable ?? "", + object.identity ?? "", object.name + ].joined(separator: "\u{1F}") + self.object = object + self.isSelected = isSelected + } +} + +// A table's or view's Export names a relation: a routine or trigger of the same name is not it. +nonisolated extension PluginExportObjectKind { + var isRelation: Bool { + switch self { + case .table, .view, .materializedView, .foreignTable: return true + case .sequence, .userType, .routine, .trigger, .event, .grant: return false + } + } +} + +// The seams a test replaces: where the formats, the object list and the run come from. +struct MobileExportEnvironment { + var formats: (DatabaseType) -> [any ExportFormatPlugin] + var loadObjects: @MainActor (MobileExportInput, any ExportFormatPlugin) async throws -> [PluginExportTable] + var makeRunner: @MainActor (ExportRequest, any ExportFormatPlugin) -> any ExportRunning + var sessionDriver: @MainActor () -> (any DatabaseDriver)? + // The connection's Safe Mode as it is when Export is tapped; nil keeps the level the sheet opened with. + var safeModeLevel: @MainActor () -> SafeModeLevel? = { nil } + var sessionHoldsTransaction: @MainActor () -> Bool = { false } + + // The coordinator is the session host: an export that lands on the window's own session runs tracked. + static func live(connectionManager: ConnectionManager, coordinator: ConnectionCoordinator) -> MobileExportEnvironment { + MobileExportEnvironment( + formats: { MobileExportFormats.available(for: $0) }, + loadObjects: { input, plugin in + try await ExportObjectListing.load( + connectionManager: connectionManager, connectionId: input.connectionId, + databaseType: input.databaseType, database: input.database, schema: input.schema, plugin: plugin + ) + }, + makeRunner: { request, plugin in + let background = TransferBackgroundRun( + title: String(format: String(localized: "Exporting %@"), request.fileName), + connectionManager: connectionManager, + connectionId: request.connectionId + ) + return ExportRun( + request: request, plugin: plugin, connectionManager: connectionManager, background: background, + sessionHost: coordinator + ) + }, + sessionDriver: { [weak coordinator] in coordinator?.session?.driver }, + safeModeLevel: { [weak coordinator] in coordinator?.connection.safeModeLevel }, + sessionHoldsTransaction: { [weak coordinator] in coordinator?.transactionHold != nil } + ) + } +} + +@MainActor +@Observable +final class MobileExportModel { + enum Phase: Equatable { + case configuring + case running(TransferProgress) + case sharing + case finished + case failed(String) + case cancelled + } + + struct Finished: Equatable { + let files: [URL] + let rowsWritten: Int + let warnings: [String] + let notes: [String] + } + + let input: MobileExportInput + let formats: [any ExportFormatPlugin] + private(set) var phase: Phase = .configuring + private(set) var finished: Finished? + private(set) var objects: Loadable<[ExportObjectRow]>? + var fileName: String + // Set through `selectFormat` and `selectScope`: an @Observable property's didSet also runs from init. + private(set) var formatId: String + private(set) var scopeKind: ExportScopeOption.Kind + + @ObservationIgnored private let environment: MobileExportEnvironment + @ObservationIgnored private var listed: [PluginExportTable] = [] + @ObservationIgnored private var runner: (any ExportRunning)? + @ObservationIgnored private var outcome: ExportOutcome? + @ObservationIgnored private var objectsRead: Task? + + init(input: MobileExportInput, environment: MobileExportEnvironment) { + self.input = input + self.environment = environment + let formats = environment.formats(input.databaseType) + self.formats = formats + self.formatId = formats.first.map { type(of: $0).formatId } ?? "" + self.scopeKind = input.scopes.first?.kind ?? .database + self.fileName = input.fileName + } + + func selectFormat(_ id: String) { + guard id != formatId, formats.contains(where: { type(of: $0).formatId == id }) else { return } + formatId = id + applyFormatDefaults() + } + + func selectScope(_ kind: ExportScopeOption.Kind) async { + guard kind != scopeKind, input.scopes.contains(where: { $0.kind == kind }) else { return } + scopeKind = kind + await loadObjectsIfNeeded() + } + + var plugin: (any ExportFormatPlugin)? { + formats.first { type(of: $0).formatId == formatId } + } + + var formatChoices: [ExportFormatChoice] { + formats.map { ExportFormatChoice(id: type(of: $0).formatId, name: type(of: $0).formatDisplayName) } + } + + var scope: ExportScopeOption? { + input.scopes.first { $0.kind == scopeKind } + } + + var isRunning: Bool { + if case .running = phase { return true } + return false + } + + var optionColumns: [PluginExportOptionColumn] { + plugin.map { type(of: $0).perTableOptionColumns } ?? [] + } + + // The objects this format can write, grouped in dump order. + var objectGroups: [ExportObjectGroup] { + guard let rows = objects?.value, let plugin else { return [] } + let kinds = Set(type(of: plugin).supportedObjectKinds) + var shown = rows.filter { kinds.contains($0.object.kind) } + if case .table(let name)? = scope?.source { + shown = shown.filter { $0.object.name == name && $0.object.kind.isRelation } + } + let grouped = Dictionary(grouping: shown, by: \.object.kind) + return grouped.keys.sorted { $0.dumpOrder < $1.dumpOrder }.map { kind in + ExportObjectGroup(kind: kind, rows: grouped[kind] ?? []) + } + } + + var exportableObjects: [PluginExportTable] { + guard let plugin else { return [] } + return objectGroups.flatMap { $0.rows } + .filter { $0.isSelected && plugin.isExportable(optionValues: $0.object.optionValues, kind: $0.object.kind) } + .map(\.object) + } + + var canExport: Bool { + guard case .configuring = phase, plugin != nil, let scope, + !fileName.trimmingCharacters(in: .whitespaces).isEmpty else { return false } + return scope.listsObjects ? !exportableObjects.isEmpty : true + } + + // MARK: - Objects + + func loadObjectsIfNeeded() async { + guard let scope, scope.listsObjects, let plugin else { return } + if case .loaded = objects { return } + objectsRead?.cancel() + objects = .loading + let read = Task { [weak self, input, environment] in + do { + let listed = try await environment.loadObjects(input, plugin) + guard !Task.isCancelled else { return } + self?.adopt(listed) + } catch { + guard !Task.isCancelled else { return } + self?.objects = .failed(error) + } + } + objectsRead = read + await read.value + } + + func retryObjects() async { + objects = nil + await loadObjectsIfNeeded() + } + + // The format can change while the list loads, so the options come from the one chosen now. + private func adopt(_ listed: [PluginExportTable]) { + self.listed = listed + let plugin = plugin + objects = .loaded(listed.map { object in + let options = plugin.map { ExportObjectOptions.defaults(for: object.kind, plugin: $0) } ?? object.optionValues + return ExportObjectRow(object: object.withOptionValues(options), isSelected: selectsAtFirst(object)) + }) + } + + // A table's export lists the whole schema so its definition comes out as the Mac writes it, with only that + // table ticked. + private func selectsAtFirst(_ object: PluginExportTable) -> Bool { + guard case .table(let name)? = scope?.source else { return true } + return object.name == name && object.kind.isRelation + } + + // The list holds every kind the database has, so a format that writes more kinds shows them with the selection + // they had; option values belong to a format and start over at its defaults, as on the Mac. + private func applyFormatDefaults() { + guard let plugin, case .loaded(let rows) = objects else { return } + objects = .loaded(rows.map { row in + var row = row + row.object = row.object.withOptionValues(ExportObjectOptions.defaults(for: row.object.kind, plugin: plugin)) + return row + }) + } + + func setSelected(_ isSelected: Bool, for id: String) { + mutateRow(id) { $0.isSelected = isSelected } + } + + func setSelected(_ isSelected: Bool, kind: PluginExportObjectKind) { + guard case .loaded(let rows) = objects else { return } + objects = .loaded(rows.map { row in + guard row.object.kind == kind else { return row } + var row = row + row.isSelected = isSelected + return row + }) + } + + func setOption(_ value: Bool, at column: Int, for id: String) { + mutateRow(id) { row in + var values = row.object.optionValues + guard values.indices.contains(column) else { return } + values[column] = value + row.object = row.object.withOptionValues(values) + } + } + + private func mutateRow(_ id: String, _ change: (inout ExportObjectRow) -> Void) { + guard case .loaded(var rows) = objects, let index = rows.firstIndex(where: { $0.id == id }) else { return } + change(&rows[index]) + objects = .loaded(rows) + } + + // MARK: - Run + + // The run gets a copy of the format's settings as they are now, so another window editing the shared form cannot + // change the name or the content of a file already started. + func start() { + guard canExport, let shared = plugin, let scope else { return } + let plugin = MobileExportFormats.frozen(shared) + let source: ExportSource + switch scope.source { + case .query(let sql): source = .query(sql) + case .rerun(let sql): source = .rerun(sql) + case .rows(let rows): source = .rows(rows) + case .table, .database: source = .objects(exportableObjects) + } + let request = ExportRequest( + formatId: formatId, + source: source, + fileName: fileName.trimmingCharacters(in: .whitespaces), + connectionId: input.connectionId, + databaseType: input.databaseType, + database: input.database, + schema: input.schema, + safeModeLevel: environment.safeModeLevel() ?? input.safeModeLevel, + sessionDriver: environment.sessionDriver(), + sessionHoldsTransaction: environment.sessionHoldsTransaction() + ) + let runner = environment.makeRunner(request, plugin) + self.runner = runner + phase = .running(TransferProgress(stage: .preparing)) + Task { [weak self] in + do { + let outcome = try await runner.run { [weak self] progress in + Task { @MainActor in self?.progressArrived(progress, from: runner) } + } + self?.complete(outcome, from: runner) + } catch { + self?.fail(error, from: runner) + } + } + } + + func cancel() { + runner?.cancel() + } + + private func progressArrived(_ progress: TransferProgress, from source: any ExportRunning) { + guard source === runner, case .running = phase else { return } + phase = .running(progress) + } + + private func complete(_ outcome: ExportOutcome, from source: any ExportRunning) { + guard source === runner else { + outcome.discard() + return + } + runner = nil + self.outcome = outcome + finished = Finished( + files: outcome.files, rowsWritten: outcome.rowsWritten, warnings: outcome.warnings, notes: outcome.notes + ) + phase = .sharing + } + + private func fail(_ error: any Error, from source: any ExportRunning) { + guard source === runner else { return } + runner = nil + if error is CancellationError { + phase = .cancelled + } else { + phase = .failed(error.localizedDescription) + } + } + + func shareEnded() { + guard case .sharing = phase else { return } + phase = .finished + } + + func reconfigure() { + guard !isRunning, phase != .sharing else { return } + discardFile() + phase = .configuring + } + + // The sheet is closing: a running export stops and whatever it wrote goes with it. + func close() { + runner?.cancel() + runner = nil + discardFile() + } + + private func discardFile() { + outcome?.discard() + outcome = nil + finished = nil + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/UI/MobileExportSheet.swift b/apple/TableProMobile/TableProMobile/Transfer/UI/MobileExportSheet.swift new file mode 100644 index 0000000000..7ef0e6fcc2 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/UI/MobileExportSheet.swift @@ -0,0 +1,349 @@ +import SwiftUI +import TableProPluginKit + +struct MobileExportItem: Identifiable { + let id = UUID() + let input: MobileExportInput +} + +// Format, scope, options, generate, then the share sheet: the order Numbers follows for Export. +struct MobileExportSheet: View { + @State private var model: MobileExportModel + @Environment(\.dismiss) private var dismiss + + init(input: MobileExportInput, environment: MobileExportEnvironment) { + _model = State(initialValue: MobileExportModel(input: input, environment: environment)) + } + + var body: some View { + NavigationStack { + content + .navigationTitle(Text(verbatim: model.input.title)) + .navigationBarTitleDisplayMode(.inline) + .toolbar { toolbar } + } + .interactiveDismissDisabled(model.isRunning) + .task { await model.loadObjectsIfNeeded() } + .onDisappear { model.close() } + .sheet(isPresented: sharing, onDismiss: { model.shareEnded() }) { + ActivityViewController(items: model.finished?.files ?? []) + } + } + + private var sharing: Binding { + Binding( + get: { model.phase == .sharing }, + set: { if !$0 { model.shareEnded() } } + ) + } + + @ViewBuilder + private var content: some View { + switch model.phase { + case .configuring: + configuration + case .running(let progress): + Form { + TransferProgressSection(progress: progress) + } + case .sharing: + Form { + Section { + Label("Ready to Share", systemImage: "checkmark.circle") + } + } + case .finished: + finishedView + case .failed(let message): + Form { + Section { + Label { + Text(verbatim: message) + } icon: { + Image(systemName: "exclamationmark.triangle.fill") + .foregroundStyle(.red) + } + } header: { + Text("Export Failed") + } + Section { + Button("Try Again") { model.reconfigure() } + } + } + case .cancelled: + Form { + Section { + Text("The export was cancelled. No file was kept.") + } + Section { + Button("Change Options") { model.reconfigure() } + } + } + } + } + + @ToolbarContentBuilder + private var toolbar: some ToolbarContent { + switch model.phase { + case .configuring: + ToolbarItem(placement: .cancellationAction) { + Button("Cancel") { dismiss() } + } + ToolbarItem(placement: .confirmationAction) { + Button("Export") { model.start() } + .disabled(!model.canExport) + } + case .running: + ToolbarItem(placement: .cancellationAction) { + Button("Cancel", role: .cancel) { model.cancel() } + } + case .sharing, .finished, .failed, .cancelled: + ToolbarItem(placement: .confirmationAction) { + Button("Done") { dismiss() } + } + } + } + + // MARK: - Configuring + + private var configuration: some View { + @Bindable var model = model + return Form { + Section { + Picker("Format", selection: Binding(get: { model.formatId }, set: { model.selectFormat($0) })) { + ForEach(model.formatChoices) { choice in + Text(verbatim: choice.name).tag(choice.id) + } + } + .pickerStyle(.menu) + LabeledContent("File Name") { + LiteralTextField("File Name", text: $model.fileName) + } + } footer: { + if let plugin = model.plugin { + Text(verbatim: ".\(plugin.currentFileExtension)") + } + } + scopeSection + if model.scope?.listsObjects == true { + ExportObjectsSections(model: model) + } + if let plugin = model.plugin { + ExportFormatOptionsSection(plugin: plugin) + } + } + } + + @ViewBuilder + private var scopeSection: some View { + let scopes = model.input.scopes + if scopes.count > 1 { + Section { + Picker("Export", selection: Binding( + get: { model.scopeKind }, + set: { kind in Task { await model.selectScope(kind) } } + )) { + ForEach(scopes) { scope in + Text(verbatim: scope.title).tag(scope.kind) + } + } + .pickerStyle(.inline) + .labelsHidden() + } header: { + Text("Export") + } footer: { + if let detail = model.scope?.detail { + Text(verbatim: detail) + } + } + } else if let detail = model.scope?.detail { + Section { + Text(verbatim: detail) + .foregroundStyle(.secondary) + } + } + } + + // MARK: - Finished + + private var finishedView: some View { + Form { + if let finished = model.finished { + Section { + Label { + Text(verbatim: String( + format: String(localized: "%@ rows written."), finished.rowsWritten.formatted() + )) + } icon: { + Image(systemName: "checkmark.circle.fill") + .foregroundStyle(.green) + } + ForEach(finished.files, id: \.self) { file in + Text(verbatim: file.lastPathComponent) + .font(.footnote) + .foregroundStyle(.secondary) + } + } + if !finished.warnings.isEmpty { + Section { + ForEach(finished.warnings, id: \.self) { warning in + Label { + Text(verbatim: warning) + } icon: { + Image(systemName: "exclamationmark.triangle") + .foregroundStyle(.orange) + } + } + } header: { + Text("Warnings") + } + } + if !finished.notes.isEmpty { + Section { + ForEach(finished.notes, id: \.self) { note in + Text(verbatim: note) + } + } + } + Section { + ShareLink(items: finished.files) { + Label("Share Again", systemImage: "square.and.arrow.up") + } + } + } + } + } +} + +// One section per object kind, with Select All for the kind and the format's per-object options in a menu. +private struct ExportObjectsSections: View { + let model: MobileExportModel + + var body: some View { + switch model.objects { + case .some(.loading), .none: + Section { + HStack(spacing: 12) { + ProgressView() + Text("Reading objects…") + .foregroundStyle(.secondary) + } + } + case .some(.failed(let error)): + Section { + Text(verbatim: error.localizedDescription) + .foregroundStyle(.secondary) + Button("Try Again") { Task { await model.retryObjects() } } + } header: { + Text("Objects") + } + case .some(.loaded): + let groups = model.objectGroups + if groups.isEmpty { + Section { + Text("Nothing here can be exported in this format.") + .foregroundStyle(.secondary) + } + } + ForEach(groups) { group in + Section { + ForEach(group.rows) { row in + ExportObjectRowView(model: model, row: row) + } + } header: { + groupHeader(group.kind, rows: group.rows, offersSelectAll: groups.count > 1 || group.rows.count > 1) + } + } + } + } + + private func groupHeader(_ kind: PluginExportObjectKind, rows: [ExportObjectRow], offersSelectAll: Bool) -> some View { + let allSelected = rows.allSatisfy(\.isSelected) + return HStack { + Text(verbatim: ExportObjectKindLabel.title(for: kind)) + Spacer() + if offersSelectAll { + Button(allSelected ? String(localized: "Deselect All") : String(localized: "Select All")) { + model.setSelected(!allSelected, kind: kind) + } + .font(.footnote) + .textCase(nil) + } + } + } +} + +private struct ExportObjectRowView: View { + let model: MobileExportModel + let row: ExportObjectRow + + private var columns: [PluginExportOptionColumn] { model.optionColumns } + + private var supportedColumns: [Int] { + guard let plugin = model.plugin else { return [] } + return columns.indices.filter { index in + row.object.optionValues.indices.contains(index) + && type(of: plugin).supportsOption(columnId: columns[index].id, for: row.object.kind) + } + } + + var body: some View { + HStack(spacing: 12) { + Button { + model.setSelected(!row.isSelected, for: row.id) + } label: { + HStack(spacing: 12) { + Image(systemName: row.isSelected ? "checkmark.circle.fill" : "circle") + .imageScale(.large) + .foregroundStyle(row.isSelected ? AnyShapeStyle(.tint) : AnyShapeStyle(.secondary)) + Image(systemName: ExportObjectKindLabel.systemImage(for: row.object.kind)) + .foregroundStyle(.secondary) + .frame(width: 20) + VStack(alignment: .leading, spacing: 2) { + Text(verbatim: row.object.name) + .foregroundStyle(.primary) + if let parent = row.object.parentTable, !parent.isEmpty { + Text(verbatim: parent) + .font(.caption) + .foregroundStyle(.secondary) + } + } + } + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + .accessibilityAddTraits(row.isSelected ? .isSelected : []) + Spacer(minLength: 0) + if row.isSelected, !supportedColumns.isEmpty { + optionsMenu + } + } + } + + private var optionsMenu: some View { + Menu { + ForEach(supportedColumns, id: \.self) { index in + Toggle(isOn: Binding( + get: { row.object.optionValues.indices.contains(index) && row.object.optionValues[index] }, + set: { model.setOption($0, at: index, for: row.id) } + )) { + Text(verbatim: columns[index].label) + } + } + } label: { + Text(verbatim: optionSummary) + .font(.footnote) + .lineLimit(1) + } + .accessibilityLabel(Text("Options")) + .accessibilityValue(Text(verbatim: optionSummary)) + } + + private var optionSummary: String { + let enabled = supportedColumns + .filter { row.object.optionValues[$0] } + .map { columns[$0].label } + guard !enabled.isEmpty else { return String(localized: "None") } + return ListFormatter.localizedString(byJoining: enabled) + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/UI/MobileImportModel.swift b/apple/TableProMobile/TableProMobile/Transfer/UI/MobileImportModel.swift new file mode 100644 index 0000000000..b66cb30f80 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/UI/MobileImportModel.swift @@ -0,0 +1,290 @@ +import Foundation +import Observation +import TableProDatabase +import TableProModels +import TableProPluginKit + +// Where an import lands, read once when the sheet opens: a switch in another window cannot move it mid-sheet. +nonisolated struct ImportTargetContext: Sendable { + let connectionId: UUID + let connectionName: String + let databaseType: DatabaseType + let database: String? + let schema: String? + // What a catalog read opens on: the active database where the engine switches databases, else "". + let catalogDatabase: String + let safeModeLevel: SafeModeLevel + let serverVersion: String? +} + +nonisolated struct ImportFinishedSummary: Equatable, Sendable { + let succeeded: Bool + let headline: String + let lines: [String] + let transactionMessage: String? + + init(_ outcome: DataImportOutcome) { + switch outcome.ending { + case .completed: + succeeded = true + headline = String(localized: "Import Complete") + case .stopped: + succeeded = false + headline = String(localized: "Import Stopped") + case .expired: + succeeded = false + headline = String(localized: "Import stopped when TablePro moved to the background.") + case .failed(let message): + succeeded = false + headline = message + } + lines = outcome.summaryLines + transactionMessage = outcome.transactionMessage + } +} + +// A file already copied into tmp/Imports by an earlier sheet, handed on when the person picks another connection. +nonisolated enum ImportFileHandoff { + static func staged(_ url: URL) -> StagedImportFile? { + let root = TransferFiles.importsDirectory.standardizedFileURL.path + let folder = url.deletingLastPathComponent().standardizedFileURL + guard folder.deletingLastPathComponent().path == root else { return nil } + return StagedImportFile(url: url, folder: folder, originalName: url.lastPathComponent) + } + + static func stage(_ url: URL) async throws -> StagedImportFile { + if let staged = staged(url) { return staged } + return try await ImportFileStaging.stage(url) + } +} + +// A confirmed import and the plugin holding the settings it was confirmed with. +nonisolated struct ConfirmedImport: Sendable { + let request: ImportRequest + let plugin: any ImportFormatPlugin +} + +// The staging, the confirmation, the run and the stop question every import sheet goes through. The row and SQL file +// sheets own one and add what they ask before it. +@MainActor +@Observable +final class ImportRunController { + enum Phase: Equatable { + case staging + case configuring + case running(TransferProgress) + case finished(ImportFinishedSummary) + case failed(String) + } + + let context: ImportTargetContext + let plugin: any ImportFormatPlugin + let sourceURL: URL + private(set) var phase: Phase = .staging + private(set) var file: StagedImportFile? + private(set) var stopQuestion: ImportStopQuestion? + private(set) var confirmationMessage: String? + private(set) var blockedMessage: String? + + @ObservationIgnored private let host: any TransferSessionHost + @ObservationIgnored private let connectionManager: ConnectionManager + @ObservationIgnored private var gate = ConfirmedWriteGate() + @ObservationIgnored private var confirmedRequest: ConfirmedImport? + @ObservationIgnored private var run: ImportRun? + @ObservationIgnored private let tablePlanner = ImportTablePlanner() + @ObservationIgnored private var runGeneration = 0 + @ObservationIgnored private var keepsFile = false + @ObservationIgnored var willBegin: (@MainActor (ImportRequest) -> Void)? + @ObservationIgnored var onFinished: (@MainActor () -> Void)? + + init( + sourceURL: URL, + plugin: any ImportFormatPlugin, + context: ImportTargetContext, + host: any TransferSessionHost, + connectionManager: ConnectionManager + ) { + self.sourceURL = sourceURL + self.plugin = plugin + self.context = context + self.host = host + self.connectionManager = connectionManager + } + + // Safe Mode as the connection has it now: another window can make it Read-Only while this sheet is open. + var refusal: String? { + ImportRun.refusal( + connectionName: context.connectionName, + databaseType: context.databaseType, + safeModeLevel: host.transferSafeModeLevel, + formatId: type(of: plugin).formatId + ) + } + + var isRunning: Bool { + if case .running = phase { return true } + return false + } + + var isConfiguring: Bool { + if case .configuring = phase { return true } + return false + } + + var fileName: String { + file?.originalName ?? sourceURL.lastPathComponent + } + + // True once the copy exists, whether this call made it or an earlier one did. + func stageIfNeeded() async -> Bool { + if file != nil { return true } + guard case .staging = phase else { return false } + do { + file = try await ImportFileHandoff.stage(sourceURL) + phase = .configuring + return true + } catch { + phase = .failed(error.localizedDescription) + return false + } + } + + // `extraConfirmation` is asked on top of Safe Mode's question: emptying the table first cannot be taken back. + // `frozen` is the plugin copy the caller decided that question from; the run uses it, not the shared form, so + // another window's later edit cannot change what was confirmed. + func requestImport( + _ request: ImportRequest, + plugin frozen: (any ImportFormatPlugin)? = nil, + extraConfirmation: String? = nil + ) { + guard refusal == nil, isConfiguring else { return } + let level = host.transferSafeModeLevel + let confirmed = ConfirmedImport( + request: request.confirmed(under: level), plugin: frozen ?? MobileImportFormats.frozen(plugin) + ) + switch gate.submit(confirmed, under: level) { + case .run(let approved): + if let extraConfirmation { + confirmedRequest = approved + confirmationMessage = extraConfirmation + } else { + begin(approved) + } + case .awaitConfirmation: + confirmationMessage = extraConfirmation + ?? String(localized: "This import will modify data. Are you sure you want to continue?") + case .blocked: + blockedMessage = refusal + ?? String(localized: "This connection is in read-only mode. Write queries are not allowed.") + } + } + + func confirmImport() { + confirmationMessage = nil + if let confirmed = confirmedRequest { + confirmedRequest = nil + guard refusal == nil else { + blockedMessage = refusal + return + } + begin(confirmed) + } else if let confirmed = gate.confirm(under: host.transferSafeModeLevel) { + begin(confirmed) + } else if let refusal { + blockedMessage = refusal + } + } + + func cancelConfirmation() { + confirmationMessage = nil + confirmedRequest = nil + gate.cancel() + } + + func dismissBlocked() { + blockedMessage = nil + } + + private func begin(_ confirmed: ConfirmedImport) { + willBegin?(confirmed.request) + let background = TransferBackgroundRun( + title: String(format: String(localized: "Importing %@"), fileName), + connectionManager: connectionManager, + connectionId: context.connectionId + ) + let run = ImportRun( + request: confirmed.request, plugin: confirmed.plugin, host: host, background: background, planner: tablePlanner + ) + self.run = run + runGeneration += 1 + let generation = runGeneration + phase = .running(TransferProgress(stage: .preparing)) + Task { [weak self] in + do { + let outcome = try await run.run { [weak self] progress in + Task { @MainActor in self?.progressArrived(progress, generation: generation) } + } + self?.complete(outcome, generation: generation) + } catch { + self?.fail(error, generation: generation) + } + } + } + + private func progressArrived(_ progress: TransferProgress, generation: Int) { + guard generation == runGeneration, isRunning else { return } + phase = .running(progress) + } + + private func complete(_ outcome: DataImportOutcome, generation: Int) { + guard generation == runGeneration else { return } + run = nil + stopQuestion = nil + phase = .finished(ImportFinishedSummary(outcome)) + onFinished?() + } + + private func fail(_ error: any Error, generation: Int) { + guard generation == runGeneration else { return } + run = nil + stopQuestion = nil + if let refusal = error as? MobileTransferRefusal { + phase = .failed(refusal.reason) + } else if let appError = error as? AppError { + phase = .failed([appError.message, appError.recovery].compactMap { $0 }.joined(separator: " ")) + } else if error is CancellationError { + phase = .failed(String(localized: "The import was cancelled. No rows were changed.")) + } else { + phase = .failed(error.localizedDescription) + } + onFinished?() + } + + // MARK: - Stop + + func requestStop() { + guard let run, isRunning else { return } + stopQuestion = run.stopQuestion + } + + func answerStop(_ choice: ImportStopChoice?) { + stopQuestion = nil + guard let choice, let run else { return } + run.stop(choice) + } + + // MARK: - Close + + // The copy goes with the sheet unless another connection's sheet takes it over. + func handOffFile() -> URL? { + guard !isRunning else { return nil } + keepsFile = file != nil + return file?.url ?? sourceURL + } + + func close() { + guard !isRunning, !keepsFile else { return } + file?.discard() + file = nil + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/UI/MobileImportSheet.swift b/apple/TableProMobile/TableProMobile/Transfer/UI/MobileImportSheet.swift new file mode 100644 index 0000000000..6d3cd216c7 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/UI/MobileImportSheet.swift @@ -0,0 +1,721 @@ +import SwiftUI +import TableProDatabase +import TableProModels +import TableProPluginKit +import TableProTabularIO + +// Another connection for a file opened from another app. Picking one closes this sheet and opens that connection +// with the file, the way a link to it opens it. +struct ImportConnectionChoice { + let connections: [DatabaseConnection] + let move: (_ connectionId: UUID, _ fileURL: URL) -> Void +} + +// The file decides the sheet: rows for CSV, JSON and Excel, statements for SQL. +struct MobileImportSheet: View { + @Environment(AppState.self) private var appState + @Environment(ConnectionCoordinator.self) private var coordinator + + private let fileURL: URL + private let presetTable: String? + private let connectionChoice: ImportConnectionChoice? + private let onFinished: @MainActor () -> Void + + init( + fileURL: URL, + presetTable: String? = nil, + connectionChoice: ImportConnectionChoice? = nil, + onFinished: @escaping @MainActor () -> Void = {} + ) { + self.fileURL = fileURL + self.presetTable = presetTable + self.connectionChoice = connectionChoice + self.onFinished = onFinished + } + + var body: some View { + if let plugin = MobileImportFormats.plugin(for: fileURL) { + let controller = ImportRunController( + sourceURL: fileURL, + plugin: plugin, + context: context, + host: coordinator, + connectionManager: appState.connectionManager + ) + if type(of: plugin).formatId == type(of: MobileImportFormats.sql).formatId { + MobileSQLFileSheet(runner: controller, connectionChoice: connectionChoice) { + Task { await coordinator.refreshTables() } + onFinished() + } + } else { + MobileRowImportSheet( + runner: controller, + tables: coordinator.tables, + presetTable: presetTable, + readTable: tableReader, + connectionChoice: connectionChoice, + onFinished: onFinished + ) + } + } else { + UnsupportedImportFileView(fileName: fileURL.lastPathComponent) + } + } + + private var context: ImportTargetContext { + ImportTargetContext( + connectionId: coordinator.connection.id, + connectionName: coordinator.displayName, + databaseType: coordinator.connection.type, + database: coordinator.tableDatabase, + schema: coordinator.tableQualifier, + catalogDatabase: coordinator.supportsDatabaseSwitching ? coordinator.activeDatabase : "", + safeModeLevel: coordinator.connection.safeModeLevel, + serverVersion: coordinator.session?.driver.serverVersion + ) + } + + // The catalog connection first, so the read never waits behind the session; the session when there is none. + private var tableReader: MobileRowImportModel.TableReader { + let manager = appState.connectionManager + let coordinator = coordinator + let context = context + return { table in + let schema = context.schema + let database = context.database + if let read = try await manager.readCatalog(for: context.connectionId, database: context.catalogDatabase, { driver in + let columns = try await driver.fetchColumns(table: table, schema: schema, database: database) + let indexes = try await driver.fetchIndexes(table: table, schema: schema, database: database) + return (columns: columns, indexes: indexes) + }) { + return read + } + guard let driver = coordinator.session?.driver else { + throw MobileTransferRefusal(reason: String(localized: "No active database session.")) + } + return try await coordinator.track(reading: true) { + let columns = try await driver.fetchColumns(table: table, schema: schema, database: database) + let indexes = try await driver.fetchIndexes(table: table, schema: schema, database: database) + return (columns: columns, indexes: indexes) + } + } + } +} + +private struct UnsupportedImportFileView: View { + let fileName: String + @Environment(\.dismiss) private var dismiss + + var body: some View { + NavigationStack { + ContentUnavailableView { + Label("Cannot Import This File", systemImage: "doc.questionmark") + } description: { + Text(verbatim: String( + format: String(localized: "%@ is not a CSV, JSON, Excel or SQL file."), fileName + )) + } + .toolbar { + ToolbarItem(placement: .confirmationAction) { + Button("Done") { dismiss() } + } + } + } + } +} + +// The Stop question, the Safe Mode confirmation and the Read-Only refusal, the same for both sheets. +private struct ImportRunAlerts: ViewModifier { + let runner: ImportRunController + let confirmationTitle: String + + func body(content: Content) -> some View { + content + .alert( + String(localized: "Stop Import?"), + isPresented: Binding(get: { runner.stopQuestion != nil }, set: { if !$0 { runner.answerStop(nil) } }), + presenting: runner.stopQuestion + ) { question in + if question.offersCommit { + Button(String(localized: "Stop and Rollback"), role: .destructive) { runner.answerStop(.rollBack) } + Button(String(localized: "Stop and Commit")) { runner.answerStop(.commit) } + } else { + Button(String(localized: "Stop Import"), role: .destructive) { runner.answerStop(.rollBack) } + } + Button(String(localized: "Continue"), role: .cancel) { runner.answerStop(nil) } + } message: { question in + Text(verbatim: question.message) + } + .confirmationDialog( + confirmationTitle, + isPresented: Binding( + get: { runner.confirmationMessage != nil }, + set: { if !$0 { runner.cancelConfirmation() } } + ), + titleVisibility: .visible + ) { + Button(String(localized: "Import"), role: .destructive) { runner.confirmImport() } + Button(String(localized: "Cancel"), role: .cancel) { runner.cancelConfirmation() } + } message: { + Text(verbatim: runner.confirmationMessage ?? "") + } + .alert( + String(localized: "Import Blocked"), + isPresented: Binding(get: { runner.blockedMessage != nil }, set: { if !$0 { runner.dismissBlocked() } }) + ) { + Button(String(localized: "OK"), role: .cancel) { runner.dismissBlocked() } + } message: { + Text(verbatim: runner.blockedMessage ?? "") + } + } +} + +// What every import sheet shows outside configuring: the copy, the run, the summary, the failure. +private struct ImportRunStateView: View { + let runner: ImportRunController + + var body: some View { + switch runner.phase { + case .staging: + Form { + Section { + HStack(spacing: 12) { + ProgressView() + Text("Reading the file…") + .foregroundStyle(.secondary) + } + } + } + case .configuring: + EmptyView() + case .running(let progress): + Form { + TransferProgressSection(progress: progress) + } + case .finished(let summary): + Form { + Section { + Label { + Text(verbatim: summary.headline) + } icon: { + Image(systemName: summary.succeeded ? "checkmark.circle.fill" : "stop.circle.fill") + .foregroundStyle(summary.succeeded ? Color.green : Color.secondary) + } + ForEach(summary.lines, id: \.self) { line in + Text(verbatim: line) + } + } + if let message = summary.transactionMessage { + Section { + Text(verbatim: message) + .foregroundStyle(.secondary) + } + } + } + case .failed(let message): + Form { + Section { + Label { + Text(verbatim: message) + } icon: { + Image(systemName: "exclamationmark.triangle.fill") + .foregroundStyle(.red) + } + } header: { + Text("Import Failed") + } + } + } + } +} + +@ToolbarContentBuilder +private func importToolbar( + runner: ImportRunController, + actionTitle: String, + canRun: Bool, + run: @escaping () -> Void, + dismiss: DismissAction +) -> some ToolbarContent { + switch runner.phase { + case .staging, .configuring: + ToolbarItem(placement: .cancellationAction) { + Button(String(localized: "Cancel")) { dismiss() } + } + ToolbarItem(placement: .confirmationAction) { + Button(actionTitle, action: run) + .disabled(!canRun) + } + case .running: + ToolbarItem(placement: .cancellationAction) { + Button(String(localized: "Stop"), role: .destructive) { runner.requestStop() } + } + case .finished, .failed: + ToolbarItem(placement: .confirmationAction) { + Button(String(localized: "Done")) { dismiss() } + } + } +} + +private struct ConnectionChoiceSection: View { + let choice: ImportConnectionChoice + let currentId: UUID + let handOff: () -> URL? + @Environment(\.dismiss) private var dismiss + + var body: some View { + Section { + Picker("Connection", selection: Binding( + get: { currentId }, + set: { picked in + guard picked != currentId, let url = handOff() else { return } + choice.move(picked, url) + dismiss() + } + )) { + ForEach(choice.connections) { connection in + Text(verbatim: connection.name.isEmpty ? connection.host : connection.name).tag(connection.id) + } + } + .pickerStyle(.menu) + } footer: { + Text("Another connection closes this one and opens that one with the file.") + } + } +} + +// MARK: - Rows + +struct MobileRowImportSheet: View { + @State private var model: MobileRowImportModel + @Environment(\.dismiss) private var dismiss + private let connectionChoice: ImportConnectionChoice? + + init( + runner: ImportRunController, + tables: [TableInfo], + presetTable: String?, + readTable: @escaping MobileRowImportModel.TableReader, + connectionChoice: ImportConnectionChoice?, + onFinished: @escaping @MainActor () -> Void + ) { + runner.onFinished = onFinished + _model = State(initialValue: MobileRowImportModel( + runner: runner, tables: tables, presetTable: presetTable, readTable: readTable + )) + self.connectionChoice = connectionChoice + } + + private var runner: ImportRunController { model.runner } + + var body: some View { + NavigationStack { + Group { + if runner.isConfiguring { + configuration + } else { + ImportRunStateView(runner: runner) + } + } + .navigationTitle(Text("Import Rows")) + .navigationBarTitleDisplayMode(.inline) + .toolbar { + importToolbar( + runner: runner, actionTitle: String(localized: "Import"), canRun: model.canImport, + run: { model.requestImport() }, dismiss: dismiss + ) + } + } + .interactiveDismissDisabled(runner.isRunning) + .modifier(ImportRunAlerts(runner: runner, confirmationTitle: String(localized: "Import Rows?"))) + .task { await model.start() } + .onDisappear { runner.close() } + } + + private var configuration: some View { + @Bindable var model = model + return Form { + if let refusal = runner.refusal { + Section { + Label(refusal, systemImage: "lock.fill") + .foregroundStyle(.secondary) + } + } + Section { + LabeledContent("File") { + Text(verbatim: runner.fileName) + .lineLimit(1) + .truncationMode(.middle) + } + LabeledContent("Format") { + Text(verbatim: type(of: model.plugin).formatDisplayName) + } + } + if let connectionChoice { + ConnectionChoiceSection( + choice: connectionChoice, currentId: model.context.connectionId, handOff: { runner.handOffFile() } + ) + } + destinationSection + switch model.destination { + case .existingTable: + mappingSection + case .newTable: + newTableSection + } + ImportFormatOptionsSection(plugin: model.plugin) + } + .background { + ImportDetectionSignatureObserver(plugin: model.plugin) { + Task { await model.read() } + } + } + } + + private var destinationSection: some View { + @Bindable var model = model + return Section { + Picker("Destination", selection: Binding( + get: { model.destination }, + set: { destination in Task { await model.selectDestination(destination) } } + )) { + Text("Existing table").tag(MobileRowImportModel.Destination.existingTable) + Text("New table").tag(MobileRowImportModel.Destination.newTable) + } + .pickerStyle(.segmented) + .listRowSeparator(.hidden) + switch model.destination { + case .existingTable: + Picker("Import into", selection: Binding( + get: { model.targetTable }, + set: { table in Task { await model.selectTable(table) } } + )) { + Text("Select a table…").tag(String?.none) + ForEach(model.tables) { table in + Text(verbatim: table.name).tag(String?.some(table.name)) + } + } + .pickerStyle(.menu) + Picker("Import as", selection: Binding( + get: { model.writeMode }, + set: { model.selectWriteMode($0) } + )) { + ForEach(ImportWriteMode.allCases) { mode in + Text(verbatim: mode.label).tag(mode) + } + } + .pickerStyle(.menu) + if model.writeMode.matchesExistingRows { + Picker("Match on", selection: $model.matchKey) { + ForEach(model.matchKeys) { key in + Text(verbatim: key.label).tag(ImportMatchKey?.some(key)) + } + } + .pickerStyle(.menu) + .disabled(model.matchKeys.isEmpty) + } else { + DeleteExistingRowsToggle(plugin: model.plugin) + } + case .newTable: + LabeledContent("New table") { + LiteralTextField(String("table_name"), text: $model.newTableName) + } + } + } footer: { + if let message = model.validationMessage { + Text(verbatim: message) + .foregroundStyle(.red) + } else if model.destination == .existingTable { + Text(""" + Update changes the rows whose key is in the file. Insert or Update also adds the rest, and \ + Insert, Skip Existing adds only rows whose key is not in the table. Columns not mapped keep \ + their values. + """) + } + } + } + + @ViewBuilder + private var readState: some View { + if model.isReading { + HStack(spacing: 12) { + ProgressView() + Text("Reading the file…") + .foregroundStyle(.secondary) + } + } else if let error = model.readError { + VStack(alignment: .leading, spacing: 6) { + Label(String(localized: "Cannot read this file"), systemImage: "exclamationmark.triangle") + Text(verbatim: error) + .font(.footnote) + .foregroundStyle(.secondary) + Button("Try Again") { Task { await model.read() } } + } + } + } + + private var mappingSection: some View { + let mapping = model.mapping + return Section { + if model.isReading || model.readError != nil { + readState + } else if model.targetTable == nil { + Text("Choose a destination table to map fields.") + .foregroundStyle(.secondary) + } else if mapping.rows.isEmpty { + Text("No fields found in the file.") + .foregroundStyle(.secondary) + } else { + ForEach(mapping.rows) { row in + Picker(selection: Binding( + get: { row.choice.mappedColumn }, + set: { model.mapping.map(row.field, to: $0) } + )) { + Text("Do not import").tag(String?.none) + ForEach(mapping.columns, id: \.self) { column in + Text(verbatim: column).tag(String?.some(column)) + } + } label: { + Text(verbatim: row.field) + } + .pickerStyle(.menu) + } + } + } header: { + HStack { + Text("Columns") + Spacer() + Menu { + Button(String(localized: "Match by Name")) { model.mapping.matchByName() } + Button(String(localized: "Match by Position")) { model.mapping.matchByPosition() } + .disabled(!model.fieldsFollowFileOrder) + Divider() + Button(String(localized: "Use Saved Mapping")) { model.mapping.useSavedMapping() } + .disabled(!mapping.canUseSavedMapping) + } label: { + Text("Match Columns") + .font(.footnote) + .textCase(nil) + } + .disabled(mapping.rows.isEmpty) + } + } footer: { + if mapping.showsSavedMapping, let table = model.targetTable { + Text(verbatim: String(format: String(localized: "Restored the mapping saved for %@."), table)) + } + } + } + + private var newTableSection: some View { + let columns = model.newTable.columns + return Section { + if model.isReading || model.readError != nil { + readState + } else if columns.isEmpty { + Text("No columns found in the file.") + .foregroundStyle(.secondary) + } else { + ForEach(columns) { column in + let settings = column.settings + NavigationLink { + NewTableColumnEditor( + field: column.field.name, + settings: newColumnBinding(column.id), + typeSuggestions: typeSuggestions + ) + } label: { + HStack(spacing: 12) { + Image(systemName: settings.include ? "checkmark.circle.fill" : "circle") + .foregroundStyle(settings.include ? AnyShapeStyle(.tint) : AnyShapeStyle(.secondary)) + VStack(alignment: .leading, spacing: 2) { + Text(verbatim: settings.name.isEmpty ? column.field.name : settings.name) + Text(verbatim: settings.isPrimaryKey ? "\(settings.type), PRIMARY KEY" : settings.type) + .font(.caption) + .foregroundStyle(.secondary) + } + } + } + } + } + } header: { + HStack { + Text("Columns") + Spacer() + if !columns.isEmpty { + let includesAll = model.newTable.includesEveryColumn + Button(includesAll ? String(localized: "Deselect All") : String(localized: "Select All")) { + model.newTable.setAllIncluded(!includesAll) + } + .font(.footnote) + .textCase(nil) + } + } + } + } + + private func newColumnBinding(_ id: String) -> Binding { + Binding( + get: { + model.newTable.columns.first { $0.id == id }?.settings ?? .proposed(name: id, type: "") + }, + set: { settings in + guard let index = model.newTable.columns.firstIndex(where: { $0.id == id }) else { return } + model.newTable.columns[index].settings = settings + } + ) + } + + // The type the mapper gives each kind of value, as quick picks beside the free-text type. + private var typeSuggestions: [String] { + let context = model.context + var seen = Set() + return [PluginImportFieldType.integer, .real, .boolean, .text, .json].compactMap { kind in + let type = ImportTypeMapper.sqlType( + for: kind, databaseTypeId: context.databaseType.rawValue, serverVersion: context.serverVersion + ) + return seen.insert(type.lowercased()).inserted ? type : nil + } + } +} + +private struct NewTableColumnEditor: View { + let field: String + @Binding var settings: NewTableColumnSettings + let typeSuggestions: [String] + + var body: some View { + Form { + Section { + Toggle("Import this field", isOn: $settings.include) + LabeledContent("Name") { + LiteralTextField("Name", text: $settings.name) + } + LabeledContent("Type") { + HStack { + LiteralTextField("Type", text: $settings.type) + Menu { + ForEach(typeSuggestions, id: \.self) { type in + Button(type) { settings.type = type } + } + } label: { + Image(systemName: "chevron.up.chevron.down") + } + .accessibilityLabel(Text("Suggested Types")) + } + } + } header: { + Text(verbatim: field) + } + Section { + Toggle("Allows NULL", isOn: $settings.isNullable) + Toggle("Primary Key", isOn: $settings.isPrimaryKey) + LabeledContent("Default") { + LiteralTextField("Default", text: $settings.defaultValue) + } + } footer: { + Text("The default is SQL, written after DEFAULT as it is: quote a text value.") + } + } + .navigationTitle(Text(verbatim: settings.name.isEmpty ? field : settings.name)) + .navigationBarTitleDisplayMode(.inline) + } +} + +// MARK: - SQL file + +struct MobileSQLFileSheet: View { + @State private var model: MobileSQLFileModel + @Environment(\.dismiss) private var dismiss + private let connectionChoice: ImportConnectionChoice? + + init(runner: ImportRunController, connectionChoice: ImportConnectionChoice?, onFinished: @escaping @MainActor () -> Void) { + runner.onFinished = onFinished + _model = State(initialValue: MobileSQLFileModel(runner: runner)) + self.connectionChoice = connectionChoice + } + + private var runner: ImportRunController { model.runner } + + var body: some View { + NavigationStack { + Group { + if runner.isConfiguring { + configuration + } else { + ImportRunStateView(runner: runner) + } + } + .navigationTitle(Text("Run SQL File")) + .navigationBarTitleDisplayMode(.inline) + .toolbar { + importToolbar( + runner: runner, actionTitle: String(localized: "Run"), canRun: model.canRun, + run: { model.requestRun() }, dismiss: dismiss + ) + } + } + .interactiveDismissDisabled(runner.isRunning) + .modifier(ImportRunAlerts(runner: runner, confirmationTitle: String(localized: "Run SQL File?"))) + .task { await model.start() } + .onDisappear { runner.close() } + } + + private var configuration: some View { + Form { + if let refusal = runner.refusal { + Section { + Label(refusal, systemImage: "lock.fill") + .foregroundStyle(.secondary) + } + } + Section { + LabeledContent("File") { + Text(verbatim: runner.fileName) + .lineLimit(1) + .truncationMode(.middle) + } + if let size = runner.file?.fileSize, size > 0 { + LabeledContent("Size") { + Text(verbatim: ByteCountFormatter.string(fromByteCount: size, countStyle: .file)) + } + } + LabeledContent("Statements") { + if model.isReading { + ProgressView() + } else if let count = model.statementCount { + Text(verbatim: count.formatted()) + .monospacedDigit() + } + } + } footer: { + if let error = model.readError { + Text(verbatim: error) + .foregroundStyle(.red) + } + } + if let connectionChoice { + ConnectionChoiceSection( + choice: connectionChoice, currentId: runner.context.connectionId, handOff: { runner.handOffFile() } + ) + } + Section { + Picker("Encoding", selection: Binding( + get: { model.chosenEncoding }, + set: { encoding in Task { await model.choose(encoding) } } + )) { + Text(verbatim: String(format: String(localized: "Automatic (%@)"), model.encodingName)) + .tag(TabularTextEncoding?.none) + ForEach(SQLImportFile.encodingChoices, id: \.self) { encoding in + Text(verbatim: encoding.displayName).tag(TabularTextEncoding?.some(encoding)) + } + } + .pickerStyle(.menu) + } footer: { + if let note = model.overrideNote ?? model.undecodableNote { + Text(verbatim: note) + } + } + ImportFormatOptionsSection(plugin: runner.plugin) + } + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/UI/MobileRowImportModel.swift b/apple/TableProMobile/TableProMobile/Transfer/UI/MobileRowImportModel.swift new file mode 100644 index 0000000000..c65209d987 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/UI/MobileRowImportModel.swift @@ -0,0 +1,232 @@ +import Foundation +import Observation +import TableProModels +import TableProPluginKit + +// Rows from a CSV, JSON or Excel file into an existing table or one made from the file, as the Mac's row import does. +@MainActor +@Observable +final class MobileRowImportModel { + enum Destination: Hashable { + case existingTable + case newTable + } + + typealias TableReader = @MainActor (_ table: String) async throws -> (columns: [ColumnInfo], indexes: [IndexInfo]) + + let runner: ImportRunController + let tables: [TableInfo] + private(set) var destination: Destination + private(set) var targetTable: String? + var newTableName: String + private(set) var writeMode: ImportWriteMode = .insert + var matchKey: ImportMatchKey? + private(set) var matchKeys: [ImportMatchKey] = [] + var mapping = ImportFieldMapping() + var newTable = NewTableDraft() + private(set) var isReading = false + private(set) var readError: String? + + @ObservationIgnored private let readTable: TableReader + @ObservationIgnored private let mappingStore = ImportMappingStore() + @ObservationIgnored private var fields: [PluginImportField] = [] + @ObservationIgnored private var readGeneration = 0 + @ObservationIgnored private var mappedTable: String? + + init(runner: ImportRunController, tables: [TableInfo], presetTable: String?, readTable: @escaping TableReader) { + self.runner = runner + self.tables = tables.filter { $0.type.allowsRowEditing } + self.readTable = readTable + let preset = presetTable.flatMap { name in tables.contains { $0.name == name } ? name : nil } + self.targetTable = preset + self.destination = .existingTable + let base = runner.sourceURL.deletingPathExtension().lastPathComponent + self.newTableName = TransferFiles.sanitizedFileName(base, fallback: "imported_rows") + runner.willBegin = { [weak self] _ in self?.rememberMapping() } + } + + var plugin: any ImportFormatPlugin { runner.plugin } + var context: ImportTargetContext { runner.context } + + var fieldsFollowFileOrder: Bool { type(of: plugin).sourceFieldsFollowFileOrder } + + // MARK: - Choices + + func selectDestination(_ destination: Destination) async { + guard destination != self.destination else { return } + self.destination = destination + if destination == .newTable { + selectWriteMode(.insert) + } + await read() + } + + func selectTable(_ table: String?) async { + guard table != targetTable else { return } + targetTable = table + mapping.clear() + matchKeys = [] + matchKey = nil + mappedTable = nil + await read() + } + + // The plugin clears the table before any mode when the option is on, so a mode that matches rows turns it off. + func selectWriteMode(_ mode: ImportWriteMode) { + writeMode = mode + if mode != .insert { + ImportPluginSettings.setDeletesExistingRows(false, on: plugin) + } + } + + // MARK: - Reading + + func start() async { + guard await runner.stageIfNeeded() else { return } + await read() + } + + // A parsing option changed what the fields are; a read in flight for the old ones is dropped. + func read() async { + guard let file = runner.file else { return } + readGeneration += 1 + let generation = readGeneration + isReading = true + readError = nil + defer { + if generation == readGeneration { isReading = false } + } + do { + switch destination { + case .existingTable: + guard let table = targetTable else { + fields = [] + return + } + let detected = try await ImportFieldDetection.detectFields(plugin: plugin, at: file.url, targetTable: table) + guard generation == readGeneration else { return } + let catalog = try await readTable(table) + guard generation == readGeneration else { return } + fields = detected + applyTable(table, columns: catalog.columns, indexes: catalog.indexes) + case .newTable: + let detected = try await ImportFieldDetection.detectFields(plugin: plugin, at: file.url, targetTable: nil) + guard generation == readGeneration else { return } + fields = detected + let databaseTypeId = context.databaseType.rawValue + let serverVersion = context.serverVersion + newTable.load(fields: detected) { inferred in + ImportTypeMapper.sqlType(for: inferred, databaseTypeId: databaseTypeId, serverVersion: serverVersion) + } + } + } catch { + guard generation == readGeneration, !(error is CancellationError) else { return } + readError = error.localizedDescription + } + } + + private func applyTable(_ table: String, columns: [ColumnInfo], indexes: [IndexInfo]) { + let names = columns.map(\.name) + let saved = mappingStore.overrides(for: mappingKey(table)) + mapping.load(fields: fields.map(\.name), columns: names, saved: saved, keepingChoices: mappedTable == table) + mappedTable = table + matchKeys = ImportMatchKey.options(columns: columns, indexes: indexes) + if let matchKey, matchKeys.contains(matchKey) { return } + matchKey = matchKeys.first + } + + private func mappingKey(_ table: String) -> ImportMappingStore.Key { + ImportMappingStore.Key( + connectionId: context.connectionId, database: context.database, schema: context.schema, table: table + ) + } + + private func rememberMapping() { + guard destination == .existingTable, let table = targetTable, mappedTable == table else { return } + mappingStore.remember(mapping.overrides, fields: mapping.fields, for: mappingKey(table)) + } + + // MARK: - Validation + + var trimmedNewTableName: String { + newTableName.trimmingCharacters(in: .whitespaces) + } + + var validationMessage: String? { + switch destination { + case .existingTable: + if mapping.mapsOneColumnTwice { + return String(localized: "Each column can be mapped from only one field.") + } + guard targetTable != nil, !isReading, readError == nil else { return nil } + return ImportRowWritingProblem.of( + mode: writeMode, key: matchKey, mappedColumns: Array(mapping.columnMapping.values) + )?.message + case .newTable: + let name = trimmedNewTableName + if !name.isEmpty, tables.contains(where: { $0.name.caseInsensitiveCompare(name) == .orderedSame }) { + return String(format: String(localized: "A table named “%@” already exists."), name) + } + return newTable.problem?.message + } + } + + var canImport: Bool { + guard runner.isConfiguring, runner.refusal == nil, !isReading, readError == nil, validationMessage == nil else { + return false + } + switch destination { + case .existingTable: + return targetTable != nil && mappedTable == targetTable && mapping.hasMappedField + case .newTable: + return !trimmedNewTableName.isEmpty && newTable.hasNamedColumn + } + } + + // MARK: - Run + + func requestImport() { + guard canImport, let file = runner.file, let target = makeTarget() else { return } + let request = ImportRequest( + formatId: type(of: plugin).formatId, + file: file, + connectionId: context.connectionId, + connectionName: context.connectionName, + databaseType: context.databaseType, + database: context.database, + safeModeLevel: context.safeModeLevel, + target: target + ) + // The settings are taken now, so the delete question and the run read the same ones whatever another window + // changes on the shared form while the confirmation is open. + let frozen = MobileImportFormats.frozen(plugin) + let clears = destination == .existingTable && writeMode == .insert + && ImportPluginSettings.deletesExistingRows(frozen) + let extra = clears ? String( + format: String(localized: "Every row in “%@” is deleted before the file’s rows are inserted."), + targetTable ?? "" + ) : nil + runner.requestImport(request, plugin: frozen, extraConfirmation: extra) + } + + private func makeTarget() -> RowImportTarget? { + switch destination { + case .existingTable: + guard let table = targetTable else { return nil } + let rowWriting = writeMode == .insert + ? ImportRowWriting.insert + : ImportRowWriting(mode: writeMode, keyColumns: matchKey?.columns ?? []) + return RowImportTarget( + table: table, schema: context.schema, columnMapping: mapping.columnMapping, + sourceFields: mapping.fields, rowWriting: rowWriting + ) + case .newTable: + let name = trimmedNewTableName + guard let definition = newTable.definition(tableName: name) else { return nil } + return RowImportTarget( + table: name, schema: context.schema, columnMapping: newTable.columnMapping, + sourceFields: newTable.fields, rowWriting: .insert, newTable: definition + ) + } + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/UI/MobileSQLFileModel.swift b/apple/TableProMobile/TableProMobile/Transfer/UI/MobileSQLFileModel.swift new file mode 100644 index 0000000000..55bf52ee1d --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/UI/MobileSQLFileModel.swift @@ -0,0 +1,114 @@ +import Foundation +import Observation +import TableProPluginKit +import TableProTabularIO + +// Run SQL File: the file's encoding (read from the file, or the person's choice), how many statements it holds, and +// the SQL import plugin's options, then the same run every import goes through. +@MainActor +@Observable +final class MobileSQLFileModel { + let runner: ImportRunController + // Nil reads the file as UTF-8 and falls back to what the file names or looks like. + private(set) var chosenEncoding: TabularTextEncoding? + private(set) var preview: SQLImportFilePreview? + private(set) var statementCount: Int? + private(set) var isReading = false + private(set) var readError: String? + + @ObservationIgnored private var readGeneration = 0 + + init(runner: ImportRunController) { + self.runner = runner + } + + var encoding: String.Encoding { + preview?.encoding ?? chosenEncoding?.foundationEncoding ?? .utf8 + } + + var encodingName: String { + Self.name(of: encoding) + } + + // The person chose an encoding the file does not read in, and the read goes on in the one the file names. + var overrideNote: String? { + guard let chosenEncoding, let preview, preview.encoding != chosenEncoding.foundationEncoding else { return nil } + return String( + format: String(localized: "The file does not read as %1$@, so it is read as %2$@."), + chosenEncoding.displayName, Self.name(of: preview.encoding) + ) + } + + var undecodableNote: String? { + guard let preview, !preview.decodes else { return nil } + return String( + format: String(localized: "Some of the file does not read as %@. Choose the encoding it was saved in."), + Self.name(of: preview.encoding) + ) + } + + var canRun: Bool { + runner.isConfiguring && runner.refusal == nil && !isReading && readError == nil && statementCount != nil + } + + func start() async { + guard await runner.stageIfNeeded() else { return } + await read() + } + + func choose(_ encoding: TabularTextEncoding?) async { + guard encoding != chosenEncoding else { return } + chosenEncoding = encoding + await read() + } + + private func read() async { + guard let file = runner.file else { return } + readGeneration += 1 + let generation = readGeneration + isReading = true + readError = nil + statementCount = nil + defer { + if generation == readGeneration { isReading = false } + } + let databaseType = runner.context.databaseType + do { + let preview = try await SQLImportFile.preview( + file, databaseType: databaseType, selected: chosenEncoding?.foundationEncoding ?? .utf8 + ) + guard generation == readGeneration else { return } + self.preview = preview + let count = try await SQLImportFile.countStatements(file, encoding: preview.encoding, databaseType: databaseType) + guard generation == readGeneration else { return } + statementCount = count + } catch { + guard generation == readGeneration, !(error is CancellationError) else { return } + readError = error.localizedDescription + } + } + + func requestRun() { + guard canRun, let file = runner.file else { return } + let context = runner.context + let request = ImportRequest( + formatId: type(of: runner.plugin).formatId, + file: file, + connectionId: context.connectionId, + connectionName: context.connectionName, + databaseType: context.databaseType, + database: context.database, + safeModeLevel: context.safeModeLevel, + target: nil, + sqlEncoding: encoding + ) + runner.requestImport(request) + } + + static func name(of encoding: String.Encoding) -> String { + if let known = TabularTextEncoding.allCases.first(where: { $0.foundationEncoding == encoding }) { + return known.displayName + } + return String.localizedName(of: encoding) + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/UI/QueryResultsExport.swift b/apple/TableProMobile/TableProMobile/Transfer/UI/QueryResultsExport.swift new file mode 100644 index 0000000000..6e5f094b7b --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/UI/QueryResultsExport.swift @@ -0,0 +1,61 @@ +import Foundation +import TableProDatabase +import TableProModels + +// What Export Results and the Select bar's Export hand the export sheet for a query's rows. +enum QueryResultsExport { + // A read runs again for every row where the engine can hold it to read-only and a fresh connection reads what the + // session read; anything else exports the rows on screen, which may be a cut result. + static func resultScope( + _ viewModel: QueryEditorViewModel, + databaseType: DatabaseType, + coordinator: ConnectionCoordinator + ) throws(ClipboardExporter.IncompleteValues) -> ExportScopeOption { + switch try viewModel.resultsExportSource( + databaseType: databaseType, + runsReadOnly: coordinator.session?.driver.enforcesReadOnlyMode ?? false, + reproducible: coordinator.reproducesOnFreshConnection(viewModel.resultPlacement) + ) { + case .rerun(let statement): + return ExportScopeOption( + kind: .result, + source: .rerun(statement), + detail: String(localized: "Runs the query again and exports every row it returns.") + ) + case .buffered(let rows): + return ExportScopeOption( + kind: .result, + source: .rows(rows), + detail: String(format: String(localized: "Exports the %d rows shown."), viewModel.bufferedRowCount) + ) + } + } + + static func selectionScope( + columns: [ColumnInfo], + rows: [Row], + indices: [Int] + ) throws(ClipboardExporter.IncompleteValues) -> ExportScopeOption { + let picked = indices.filter(rows.indices.contains).map { rows[$0].cells } + let rowSet = try ExportRowSet(columns: columns, cells: picked) + return ExportScopeOption(kind: .selectedRows, source: .rows(rowSet), detail: selectedRowsDetail(picked.count)) + } + + static func selectedRowsDetail(_ count: Int) -> String { + String(AttributedString(localized: "^[\(count) row](inflect: true) selected.").characters) + } + + // Off the main actor, so a large pick formats without holding up the screen. + @concurrent + nonisolated static func clipboardText( + _ rows: [[Cell]], + columns: [ColumnInfo], + format: ExportFormat, + databaseType: DatabaseType, + driver: (any DatabaseDriver)? + ) async throws -> String { + try ClipboardExporter.exportRows( + columns: columns, rows: rows, format: format, databaseType: databaseType, driver: driver + ) + } +} diff --git a/apple/TableProMobile/TableProMobile/Transfer/UI/TransferProgressSection.swift b/apple/TableProMobile/TableProMobile/Transfer/UI/TransferProgressSection.swift new file mode 100644 index 0000000000..4ae3afa5d9 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Transfer/UI/TransferProgressSection.swift @@ -0,0 +1,90 @@ +import SwiftUI + +// Determinate whenever a total is known; indeterminate only while the run is still counting or preparing. +struct TransferProgressSection: View { + let progress: TransferProgress + + var body: some View { + Section { + VStack(alignment: .leading, spacing: 8) { + if let fraction = shownFraction { + ProgressView(value: fraction) { + Text(verbatim: TransferProgressText.headline(progress)) + } currentValueLabel: { + Text(verbatim: TransferProgressText.counts(progress)) + .monospacedDigit() + } + } else { + HStack(spacing: 12) { + ProgressView() + VStack(alignment: .leading, spacing: 2) { + Text(verbatim: TransferProgressText.headline(progress)) + Text(verbatim: TransferProgressText.counts(progress)) + .font(.footnote) + .foregroundStyle(.secondary) + .monospacedDigit() + } + } + } + if let status = progress.status, !status.isEmpty { + Text(verbatim: status) + .font(.footnote) + .foregroundStyle(.secondary) + } + } + .padding(.vertical, 4) + .accessibilityElement(children: .combine) + } + } + + private var shownFraction: Double? { + switch progress.stage { + case .preparing, .counting: + return nil + case .running, .finishing: + return progress.fractionCompleted + } + } +} + +nonisolated enum TransferProgressText { + static func headline(_ progress: TransferProgress) -> String { + switch progress.stage { + case .preparing: + return String(localized: "Preparing…") + case .counting: + return String(localized: "Counting rows…") + case .running, .finishing: + guard let name = progress.objectName, !name.isEmpty else { + return progress.stage == .finishing ? String(localized: "Finishing…") : String(localized: "Working…") + } + guard progress.objectCount > 1 else { return name } + return String( + format: String(localized: "%1$@ (%2$lld of %3$lld)"), + name, Int64(progress.objectIndex), Int64(progress.objectCount) + ) + } + } + + static func counts(_ progress: TransferProgress) -> String { + let completed = progress.completedUnits.formatted() + guard let total = progress.totalUnits else { + if progress.completedUnits == 0, let read = progress.bytesRead, let size = progress.totalBytes, size > 0 { + return String( + format: String(localized: "%1$@ of %2$@"), + ByteCountFormatter.string(fromByteCount: read, countStyle: .file), + ByteCountFormatter.string(fromByteCount: size, countStyle: .file) + ) + } + switch progress.unit { + case .rows: return String(format: String(localized: "%@ rows"), completed) + case .statements: return String(format: String(localized: "%@ statements"), completed) + } + } + let shownTotal = max(total, progress.completedUnits).formatted() + switch progress.unit { + case .rows: return String(format: String(localized: "%1$@ of %2$@ rows"), completed, shownTotal) + case .statements: return String(format: String(localized: "%1$@ of %2$@ statements"), completed, shownTotal) + } + } +} diff --git a/apple/TableProMobile/TableProMobile/ViewModels/DataBrowserViewModel.swift b/apple/TableProMobile/TableProMobile/ViewModels/DataBrowserViewModel.swift index cc770621ae..545a8c4c82 100644 --- a/apple/TableProMobile/TableProMobile/ViewModels/DataBrowserViewModel.swift +++ b/apple/TableProMobile/TableProMobile/ViewModels/DataBrowserViewModel.swift @@ -36,10 +36,13 @@ final class DataBrowserViewModel { var filterLogicMode: FilterLogicMode = .and private(set) var activeSearchText = "" private(set) var loadError: AppError? + // The last load was refused because an import holds the session; the browser loads again once it ends. + private(set) var waitsForImport = false var operationError: AppError? private(set) var isLoading = true private(set) var isPageLoading = false var memoryWarning: String? + var selection = RowSelection() @ObservationIgnored private var session: ConnectionSession? @ObservationIgnored private var table: TableInfo? @@ -116,7 +119,7 @@ final class DataBrowserViewModel { // MARK: - Attach - typealias SessionReadTracker = @MainActor (_ read: () async -> Void) async -> Void + typealias SessionReadTracker = @MainActor (_ read: () async -> Void) async throws -> Void // `trackRead` marks the session busy for the whole load, so Commit and Roll Back cannot end a page mid-stream. func attach( @@ -140,6 +143,7 @@ final class DataBrowserViewModel { self.trackRead = trackRead if movesSource { resetTotal() + selection.clearRows() } } @@ -168,6 +172,7 @@ final class DataBrowserViewModel { func load(isInitial: Bool = false) async { loadGeneration += 1 let generation = loadGeneration + selection.clearRows() guard let session, let table else { loadError = AppError( @@ -187,24 +192,32 @@ final class DataBrowserViewModel { isPageLoading = true } loadError = nil + waitsForImport = false - await tracked { - if let reader = TableBrowseMode.keyContentsReader(of: session.driver) { - await loadKeyContents(reader: reader, key: table.name, generation: generation) - } else { - await loadRows(session: session, table: table, isInitial: isInitial, generation: generation) + do { + try await tracked { + if let reader = TableBrowseMode.keyContentsReader(of: session.driver) { + await loadKeyContents(reader: reader, key: table.name, generation: generation) + } else { + await loadRows(session: session, table: table, isInitial: isInitial, generation: generation) + } } + } catch { + guard isCurrent(generation) else { return } + loadError = (error as? SessionHeldByImport)?.appError + ?? ErrorClassifier.classify(error, context: ErrorContext(operation: "loadRows", databaseType: databaseType)) + waitsForImport = error is SessionHeldByImport } guard isCurrent(generation) else { return } finishLoad() } - private func tracked(_ read: () async -> Void) async { + private func tracked(_ read: () async -> Void) async throws { guard let trackRead else { await read() return } - await trackRead(read) + try await trackRead(read) } private func isCurrent(_ generation: Int) -> Bool { @@ -569,7 +582,27 @@ final class DataBrowserViewModel { } func deleteRow(_ locator: RowLocator) async -> Bool { - guard let session, let table else { return false } + await deleteRows([locator]) == 1 + } + + // Stops at the first row that fails and reloads once, so a page of selected rows is not read back row by row. + func deleteRows(_ locators: [RowLocator]) async -> Int { + var deletedCount = 0 + var reloads = false + for locator in locators { + let outcome = await deleteOne(locator) + reloads = reloads || outcome.reloads + guard outcome.deleted else { break } + deletedCount += 1 + } + if reloads { + await load() + } + return deletedCount + } + + private func deleteOne(_ locator: RowLocator) async -> (deleted: Bool, reloads: Bool) { + guard let session, let table else { return (false, false) } let schema = schema let database = database if let keys = locator.keyValues, let shared = await SharedRowKey.error( @@ -577,7 +610,7 @@ final class DataBrowserViewModel { columns: columnDetails, type: databaseType, driver: session.driver ) { operationError = shared - return false + return (false, false) } let writer = session.driver as? any DriverRowWriting if let writer, let identical = await SharedRowKey.identicalRowsError( @@ -585,7 +618,7 @@ final class DataBrowserViewModel { columns: locator.columnNames(of: columns), type: databaseType, isUpdate: false ) { operationError = identical - return false + return (false, false) } do { guard let batch = try RowWriteBatch.delete( @@ -597,7 +630,7 @@ final class DataBrowserViewModel { columns: locator.columnNames(of: columns), columnDetails: columnDetails, locator: locator - ) else { return false } + ) else { return (false, false) } let deleted = try await batch.run() // SQL Server under SET NOCOUNT ON reports 0 for every delete, so a zero is checked against the table. // A ClickHouse mutation reports no count, so the row is looked for by every value it held. @@ -612,28 +645,96 @@ final class DataBrowserViewModel { table: table.name, schema: schema, columns: locator.columnNames(of: columns), row: locator.row ) == 0 } - await load() guard isGone else { operationError = Self.rowNotDeleted - return false + return (false, true) } - return true + return (true, true) } catch RowChangeRefusal.rowNotFound { - await load() - return true + return (true, true) } catch RowChangeRefusal.notApplied { - await load() operationError = Self.rowNotDeleted - return false + return (false, true) } catch let refusal as RowChangeRefusal { operationError = SharedRowKey.error(for: refusal) - return false + return (false, false) } catch { operationError = ErrorClassifier.classify( error, context: ErrorContext(operation: "deleteRow", databaseType: databaseType, host: host) ) - return false + return (false, false) + } + } + + func rowLocators(at indices: [Int]) -> [RowLocator]? { + let located = indices.compactMap(rowLocator(forRowAt:)) + return located.count == indices.count ? located : nil + } + + // MARK: - Export Sources + + var narrowsRows: Bool { hasActiveSearch || hasActiveFilters || sortState.isSorting } + + // Every row the browser's search, filter and sort pick, without the page's LIMIT. A driver that builds its own + // reads has no SQL to re-run, so it exports the whole table instead. The literals are written for the export's own + // connection when it runs. + func exportSelect() -> ExportSelect? { + guard let table, let current = session?.driver, !(current is any DriverQueryBuilding) else { return nil } + let name = table.name + let schema = schema + let database = database + let type = databaseType + let filters = filters + let logicMode = filterLogicMode + let columns = columnDetails + let searchText = hasActiveSearch ? activeSearchText : nil + let searchColumns = searchableColumns() + let sortState = sortState + return ExportSelect(relation: ExportRelation(table: name, schema: schema)) { driver in + SQLBuilder.buildExportSelect( + table: name, schema: schema, database: database, type: type, + filters: filters, logicMode: logicMode, columns: columns, + session: driver.settledLexicalFacts, + searchText: searchText, searchColumns: searchColumns, + sortState: sortState + ) + } + } + + // Rows with a key are read again in full, bytes and long values included; the rest export what the page holds. So + // do all of them while the session holds a transaction: the export's own connection cannot see its changes. + func selectedRowsExportSource( + _ indices: [Int], + readsAgain: Bool = true + ) throws(ClipboardExporter.IncompleteValues) -> ExportScopeOption.Source { + if readsAgain, let table, let driver = session?.driver, databaseType.speaksSQL, !(driver is any DriverQueryBuilding), + let locators = rowLocators(at: indices) { + let keys = locators.compactMap(\.keyValues).filter { !$0.isEmpty } + if !keys.isEmpty, keys.count == locators.count { + return .query(Self.rowsSelect( + table: table.name, schema: schema, database: database, type: databaseType, + keys: keys, columns: columnDetails + )) + } + } + return .rows(try ExportRowSet(columns: columns, cells: indices.map(cells(at:)))) + } + + // Each key's literal is written with the escaping of the connection that runs the read, so a key like `a\b` names + // the same row there whatever the window's session reads backslashes as. + nonisolated static func rowsSelect( + table: String, + schema: String?, + database: String?, + type: DatabaseType, + keys: [[(column: String, value: String)]], + columns: [ColumnInfo] + ) -> ExportSelect { + ExportSelect(relation: ExportRelation(table: table, schema: schema)) { driver in + SQLBuilder.buildRowsSelect( + table: table, schema: schema, database: database, type: type, keys: keys, columns: columns, driver: driver + ) } } @@ -779,6 +880,7 @@ final class DataBrowserViewModel { case .warning: Self.logger.warning("Memory pressure warning: shrinking window to 100 rows") self.buffer.shrink(to: 100) + self.selection.clearRows() self.pageWasTrimmed = true if !self.buffer.isEmpty { self.memoryWarning = String(localized: "Results trimmed due to memory pressure.") @@ -786,6 +888,7 @@ final class DataBrowserViewModel { case .critical: Self.logger.error("Memory pressure critical: shrinking window to 50 rows and cancelling") self.buffer.shrink(to: 50) + self.selection.clearRows() self.pageWasTrimmed = true self.fetchTask?.cancel() self.memoryWarning = String(localized: "Results trimmed due to memory pressure.") diff --git a/apple/TableProMobile/TableProMobile/ViewModels/ForeignKeyPreviewModel.swift b/apple/TableProMobile/TableProMobile/ViewModels/ForeignKeyPreviewModel.swift index 461c4b6539..6ec5185197 100644 --- a/apple/TableProMobile/TableProMobile/ViewModels/ForeignKeyPreviewModel.swift +++ b/apple/TableProMobile/TableProMobile/ViewModels/ForeignKeyPreviewModel.swift @@ -28,12 +28,21 @@ final class ForeignKeyPreviewModel { @ObservationIgnored private let session: ConnectionSession? @ObservationIgnored private let databaseType: DatabaseType + @ObservationIgnored private let trackRead: DataBrowserViewModel.SessionReadTracker? @ObservationIgnored private var isRunning = false - init(reference: ForeignKeyReference, session: ConnectionSession?, databaseType: DatabaseType) { + // `trackRead` marks the session busy for the reads, so an import cannot start between them, and refuses them while + // one holds the session. + init( + reference: ForeignKeyReference, + session: ConnectionSession?, + databaseType: DatabaseType, + trackRead: DataBrowserViewModel.SessionReadTracker? = nil + ) { self.reference = reference self.session = session self.databaseType = databaseType + self.trackRead = trackRead } var isLoaded: Bool { @@ -78,6 +87,24 @@ final class ForeignKeyPreviewModel { defer { isRunning = false } let driver = session.driver + guard let trackRead else { + await read(on: driver) + return + } + do { + try await trackRead { await self.read(on: driver) } + } catch { + phase = .failed((error as? SessionHeldByImport)?.appError ?? AppError( + category: .query, + title: String(localized: "Could Not Load the Referenced Row"), + message: error.localizedDescription, + recovery: nil, + underlying: error + )) + } + } + + private func read(on driver: any DatabaseDriver) async { // The key columns' types pick each literal, as they do for the filter Open Table runs. var keyColumns: [ColumnInfo] = [] do { diff --git a/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift b/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift index 73e0a7427f..2064596640 100644 --- a/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift +++ b/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift @@ -55,8 +55,11 @@ final class QueryEditorViewModel { private(set) var rowsAffectedTotal = 0 private(set) var interruption: ScriptInterruption? private(set) var cutLongValues = false + var selection = RowSelection() // Each request as it went to the server, split again after a statement changed how the session reads literals. @ObservationIgnored private(set) var sentStatements: [String] = [] + // The session and the database and schema the result was read on. + @ObservationIgnored private(set) var resultPlacement: ConnectionCoordinator.TablePlacement? @ObservationIgnored private(set) var ranTransactionRollBack = false @ObservationIgnored private var unitKind: QueryScript.UnitKind = .statement @@ -141,6 +144,38 @@ final class QueryEditorViewModel { ) } + nonisolated enum ResultsExportSource: Sendable { + case rerun(String) + case buffered(ExportRowSet) + } + + var bufferedRowCount: Int { buffer.count } + + // A file wants the whole result, which only running the statement again gives, and only a read may run twice: + // `INSERT ... RETURNING`, `CALL` or a batch would write again (critic M9). It runs on a connection of its own, so + // `reproducible` says that connection reads what the session read (`SET @v` then `SELECT @v` does not). Anything + // else exports the rows shown. + func resultsExportSource( + databaseType: DatabaseType, + runsReadOnly: Bool, + reproducible: Bool + ) throws(ClipboardExporter.IncompleteValues) -> ResultsExportSource { + if reproducible, + let statement = Self.rerunnableStatement(sentStatements, databaseType: databaseType, runsReadOnly: runsReadOnly) { + return .rerun(statement) + } + return .buffered(try ExportRowSet(columns: buffer.columns, cells: buffer.window.rows.map(\.cells))) + } + + // `runsReadOnly`: the driver runs the statement in the engine's read-only mode. The classifier alone is not enough, + // since `SELECT post_payment(42)` reads as a read and its function writes, so without that mode nothing runs twice. + nonisolated static func rerunnableStatement(_ sent: [String], databaseType: DatabaseType, runsReadOnly: Bool) -> String? { + guard runsReadOnly, sent.count == 1, let statement = sent.first, + !statement.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty, + !SQLWriteClassifier.isWriteQuery(statement, databaseType: databaseType) else { return nil } + return statement + } + // The export text and the pasteboard's copy of it are held beside the values already on screen. nonisolated static func resultsFitExport(rows: [Row], availableBytes: Int) -> Bool { let textBytes = rows.reduce(0) { total, row in @@ -271,7 +306,8 @@ final class QueryEditorViewModel { driver: DatabaseDriver, script: QueryScript, maxRows: Int = QueryEditorViewModel.maxBufferedRows, - enforcesReadOnly: Bool = false + enforcesReadOnly: Bool = false, + placement: ConnectionCoordinator.TablePlacement? = nil ) async { fetchTask?.cancel() let options = StreamOptions( @@ -284,7 +320,9 @@ final class QueryEditorViewModel { runID &+= 1 let run = runID phase = .running + selection.end() sentStatements = [] + resultPlacement = placement buffer.reset() executionTime = 0 rowsAffectedTotal = 0 @@ -314,6 +352,7 @@ final class QueryEditorViewModel { func reset() { runID &+= 1 fetchTask?.cancel() + selection.end() buffer.reset() executionTime = 0 rowsAffectedTotal = 0 diff --git a/apple/TableProMobile/TableProMobile/ViewModels/RowDetailViewModel.swift b/apple/TableProMobile/TableProMobile/ViewModels/RowDetailViewModel.swift index bdf4e73f61..66906acff4 100644 --- a/apple/TableProMobile/TableProMobile/ViewModels/RowDetailViewModel.swift +++ b/apple/TableProMobile/TableProMobile/ViewModels/RowDetailViewModel.swift @@ -554,7 +554,8 @@ final class RowDetailViewModel { return try ClipboardExporter.exportRow( columns: columns, cells: completeCells(at: currentIndex), format: format, tableName: table?.name, - databaseType: databaseType, driver: openedSession?.driver + databaseType: databaseType, driver: openedSession?.driver, + columnDetails: columnDetails ) } catch { operationError = AppError( diff --git a/apple/TableProMobile/TableProMobile/Views/Components/FKPreviewView.swift b/apple/TableProMobile/TableProMobile/Views/Components/FKPreviewView.swift index f9d1ce1a8d..e8618d2eb3 100644 --- a/apple/TableProMobile/TableProMobile/Views/Components/FKPreviewView.swift +++ b/apple/TableProMobile/TableProMobile/Views/Components/FKPreviewView.swift @@ -15,6 +15,7 @@ struct FKPreviewView: View { let databaseType: DatabaseType let currentSchema: String? let allowsOpen: Bool + let trackRead: DataBrowserViewModel.SessionReadTracker? let refreshTransaction: () async -> Void let onOpen: ((ForeignKeyReference) -> Void)? @@ -24,6 +25,7 @@ struct FKPreviewView: View { databaseType: DatabaseType, currentSchema: String? = nil, allowsOpen: Bool = true, + trackRead: DataBrowserViewModel.SessionReadTracker? = nil, refreshTransaction: @escaping () async -> Void, onOpen: ((ForeignKeyReference) -> Void)? = nil ) { @@ -32,6 +34,7 @@ struct FKPreviewView: View { self.databaseType = databaseType self.currentSchema = currentSchema self.allowsOpen = allowsOpen + self.trackRead = trackRead self.refreshTransaction = refreshTransaction self.onOpen = onOpen } @@ -53,6 +56,7 @@ struct FKPreviewView: View { databaseType: databaseType, currentSchema: currentSchema, allowsOpen: allowsOpen, + trackRead: trackRead, refreshTransaction: refreshTransaction, onOpen: openAndClose, close: { dismiss() } diff --git a/apple/TableProMobile/TableProMobile/Views/Components/ForeignKeyPreviewPage.swift b/apple/TableProMobile/TableProMobile/Views/Components/ForeignKeyPreviewPage.swift index 229a5f4876..4f40e1feb2 100644 --- a/apple/TableProMobile/TableProMobile/Views/Components/ForeignKeyPreviewPage.swift +++ b/apple/TableProMobile/TableProMobile/Views/Components/ForeignKeyPreviewPage.swift @@ -17,6 +17,7 @@ struct ForeignKeyPreviewPage: View { databaseType: DatabaseType, currentSchema: String?, allowsOpen: Bool, + trackRead: DataBrowserViewModel.SessionReadTracker? = nil, refreshTransaction: @escaping () async -> Void, onOpen: ((ForeignKeyReference) -> Void)?, close: @escaping () -> Void @@ -29,7 +30,8 @@ struct ForeignKeyPreviewPage: View { _model = State(initialValue: ForeignKeyPreviewModel( reference: reference, session: session, - databaseType: databaseType + databaseType: databaseType, + trackRead: trackRead )) } diff --git a/apple/TableProMobile/TableProMobile/Views/Components/RowGrid.swift b/apple/TableProMobile/TableProMobile/Views/Components/RowGrid.swift index ce04f75535..f3f8c4661d 100644 --- a/apple/TableProMobile/TableProMobile/Views/Components/RowGrid.swift +++ b/apple/TableProMobile/TableProMobile/Views/Components/RowGrid.swift @@ -92,6 +92,8 @@ struct RowGrid: View { let columns: [ColumnInfo] let window: RowWindow let keyColumn: Int? + // Nil outside Select mode; then no selection column is drawn. + var selectedRows: Set? @ViewBuilder let row: (_ index: Int, _ line: RowGridLine) -> Line @Environment(\.dynamicTypeSize) private var dynamicTypeSize @@ -100,6 +102,7 @@ struct RowGrid: View { @ScaledMetric(relativeTo: .subheadline) private var maximumWidth: CGFloat = 320 @ScaledMetric(relativeTo: .subheadline) private var horizontalPadding: CGFloat = 12 @ScaledMetric(relativeTo: .subheadline) private var verticalPadding: CGFloat = 10 + @ScaledMetric(relativeTo: .subheadline) private var selectionColumnWidth: CGFloat = 44 @State private var computedLayout: (key: LayoutKey, layout: RowGridLayout)? @State private var viewport = RowGridViewport(width: 0, leadingInset: 0) @@ -123,15 +126,17 @@ struct RowGrid: View { horizontalPadding: horizontalPadding, verticalPadding: verticalPadding, hairline: HairlineThickness.points(forDisplayScale: displayScale), - pinnedInset: viewport.leadingInset + pinnedInset: viewport.leadingInset, + selectionWidth: selectedRows == nil ? 0 : selectionColumnWidth ) } var body: some View { let key = layoutKey let layout = currentLayout(for: key).limitingKeyColumn(toViewportWidth: viewport.width) + let selectionWidth = selectedRows == nil ? 0 : selectionColumnWidth ScrollView([.horizontal, .vertical]) { - grid(layout, width: max(layout.totalWidth, viewport.width)) + grid(layout, width: max(layout.totalWidth + selectionWidth, viewport.width)) } .scrollBounceBehavior(.basedOnSize, axes: .horizontal) .defaultScrollAnchor(.topLeading, for: .alignment) @@ -173,7 +178,8 @@ struct RowGrid: View { // Indices of the same array the closure reads, so a result that shrank cannot put one out of range. ForEach(rows.indices, id: \.self) { index in row(index, RowGridLine( - columns: columns, values: rows[index].displayValues(), layout: layout, width: width, style: style + columns: columns, values: rows[index].displayValues(), layout: layout, width: width, style: style, + isSelected: selectedRows.map { $0.contains(index) } )) } } header: { @@ -186,15 +192,20 @@ struct RowGrid: View { private func header(_ layout: RowGridLayout, width: CGFloat, style: RowGridLine.Style) -> some View { HStack(spacing: 0) { ForEach(layout.order.indices, id: \.self) { position in - Text(verbatim: columns[layout.order[position]].name) - .font(.subheadline) - .fontWeight(.semibold) - .foregroundStyle(.secondary) - .lineLimit(1) - .padding(.horizontal, style.horizontalPadding) - .padding(.vertical, style.verticalPadding) - .frame(width: layout.widths[position], alignment: .leading) - .modifier(RowGridCell(isPinned: position == 0, style: style)) + HStack(spacing: 0) { + if position == 0, style.selectionWidth > 0 { + Color.clear.frame(width: style.selectionWidth) + } + Text(verbatim: columns[layout.order[position]].name) + .font(.subheadline) + .fontWeight(.semibold) + .foregroundStyle(.secondary) + .lineLimit(1) + .padding(.horizontal, style.horizontalPadding) + .padding(.vertical, style.verticalPadding) + .frame(width: layout.widths[position], alignment: .leading) + } + .modifier(RowGridCell(isPinned: position == 0, style: style, background: Color(.systemBackground))) } } .frame(width: width, alignment: .leading) @@ -229,6 +240,7 @@ struct RowGridLine: View { let verticalPadding: CGFloat let hairline: CGFloat let pinnedInset: CGFloat + let selectionWidth: CGFloat } let columns: [ColumnInfo] @@ -236,16 +248,29 @@ struct RowGridLine: View { let layout: RowGridLayout let width: CGFloat let style: Style + // Nil outside Select mode. + let isSelected: Bool? + + // Opaque, so the pinned cells still cover the columns scrolled under them. + private var background: Color { + isSelected == true ? Color(.systemGray5) : Color(.systemBackground) + } var body: some View { HStack(spacing: 0) { ForEach(layout.order.indices, id: \.self) { position in - cell(value(at: layout.order[position])) - .frame(width: layout.widths[position], alignment: .leading) - .modifier(RowGridCell(isPinned: position == 0, style: style)) + HStack(spacing: 0) { + if position == 0, let isSelected { + selectionMark(isSelected) + } + cell(value(at: layout.order[position])) + .frame(width: layout.widths[position], alignment: .leading) + } + .modifier(RowGridCell(isPinned: position == 0, style: style, background: background)) } } .frame(width: width, alignment: .leading) + .background(isSelected == true ? Color(.systemGray5) : Color.clear) .overlay(alignment: .bottom) { Rectangle().fill(Color(.separator)).frame(height: style.hairline) } @@ -258,6 +283,13 @@ struct RowGridLine: View { values.indices.contains(column) ? values[column] : nil } + private func selectionMark(_ isSelected: Bool) -> some View { + Image(systemName: isSelected ? "checkmark.circle.fill" : "circle") + .imageScale(.large) + .foregroundStyle(isSelected ? AnyShapeStyle(.tint) : AnyShapeStyle(.secondary)) + .frame(width: style.selectionWidth) + } + private var accessibilityText: String { layout.order .filter { columns.indices.contains($0) } @@ -284,6 +316,7 @@ struct RowGridLine: View { private struct RowGridCell: ViewModifier { let isPinned: Bool let style: RowGridLine.Style + let background: Color func body(content: Content) -> some View { if isPinned { @@ -291,7 +324,7 @@ private struct RowGridCell: ViewModifier { // Offset in the render pass, so a sideways scroll moves no state and rebuilds no row. content // Reaches over the leading inset, so cells scrolled past the key never show beside it. - .background(Color(.systemBackground).padding(.leading, -inset)) + .background(background.padding(.leading, -inset)) .overlay(alignment: .trailing) { Rectangle().fill(Color(.separator)).frame(width: style.hairline) } diff --git a/apple/TableProMobile/TableProMobile/Views/Components/RowSelection.swift b/apple/TableProMobile/TableProMobile/Views/Components/RowSelection.swift new file mode 100644 index 0000000000..0e951a674c --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Views/Components/RowSelection.swift @@ -0,0 +1,128 @@ +import SwiftUI + +// Rows are picked by their place on the loaded page, so anything that replaces the page clears the pick. +nonisolated struct RowSelection: Equatable, Sendable { + private(set) var isActive = false + private(set) var rows: Set = [] + + var count: Int { rows.count } + + var editMode: EditMode { isActive ? .active : .inactive } + + mutating func begin() { + isActive = true + rows = [] + } + + mutating func end() { + isActive = false + rows = [] + } + + mutating func toggle(_ index: Int) { + guard isActive else { return } + if rows.contains(index) { + rows.remove(index) + } else { + rows.insert(index) + } + } + + mutating func replace(with indices: Set) { + guard isActive else { return } + rows = indices + } + + mutating func selectAll(count: Int) { + guard isActive else { return } + rows = Set(0.. Bool { + rows.contains(index) + } + + // Nil outside Select mode, so no row draws a circle or reads as selected there. + func pick(of index: Int) -> Bool? { + isActive ? rows.contains(index) : nil + } + + // In Select mode a tap picks the row, which stays at full contrast; only outside it does a row open its detail. + var opensRowDetail: Bool { + !isActive + } + + func selectsAll(of count: Int) -> Bool { + count > 0 && (0.. [Int] { + rows.filter { $0 >= 0 && $0 < count }.sorted() + } +} + +// The cards layout is a `List`: edit mode draws the circles, and its selection binding picks the rows. +enum RowSelectionBinding { + static func rows(_ selection: Binding) -> Binding>? { + guard selection.wrappedValue.isActive else { return nil } + return Binding( + get: { selection.wrappedValue.rows }, + set: { selection.wrappedValue.replace(with: $0) } + ) + } + + static func editMode(_ selection: Binding) -> Binding { + Binding( + get: { selection.wrappedValue.editMode }, + set: { mode in + if mode == .inactive { + selection.wrappedValue.end() + } else if !selection.wrappedValue.isActive { + selection.wrappedValue.begin() + } + } + ) + } +} + +// Select All, the count and Done, for a results pane that has no navigation bar of its own. +struct RowSelectionHeader: View { + @Binding var selection: RowSelection + let rowCount: Int + + var body: some View { + let selectsAll = selection.selectsAll(of: rowCount) + HStack(spacing: 12) { + Button(selectsAll ? String(localized: "Deselect All") : String(localized: "Select All")) { + if selectsAll { + selection.clearRows() + } else { + selection.selectAll(count: rowCount) + } + } + .buttonStyle(.bordered) + .controlSize(.small) + Spacer() + Text("^[\(selection.count) Row](inflect: true) Selected") + .font(.caption) + .foregroundStyle(.secondary) + .monospacedDigit() + Spacer() + Button { + selection.end() + } label: { + Label("Done", systemImage: "checkmark") + .font(.subheadline.weight(.medium)) + } + .buttonStyle(.bordered) + .controlSize(.small) + } + .padding(.horizontal, 12) + .padding(.vertical, 6) + } +} diff --git a/apple/TableProMobile/TableProMobile/Views/Components/TableTitle.swift b/apple/TableProMobile/TableProMobile/Views/Components/TableTitle.swift index 0c15b95f2e..9125203fb6 100644 --- a/apple/TableProMobile/TableProMobile/Views/Components/TableTitle.swift +++ b/apple/TableProMobile/TableProMobile/Views/Components/TableTitle.swift @@ -1,26 +1,28 @@ import SwiftUI extension View { - func tableTitle(_ table: String, schema: String?) -> some View { - modifier(TableTitle(table: table, schema: schema)) + // `replacement` stands in for the table name, as Select mode's count does, without changing the view's structure. + func tableTitle(_ table: String, schema: String?, replacement: Text? = nil) -> some View { + modifier(TableTitle(table: table, schema: schema, replacement: replacement)) } } private struct TableTitle: ViewModifier { let table: String let schema: String? + let replacement: Text? func body(content: Content) -> some View { if let schema { if #available(iOS 26.0, *) { content - .navigationTitle(Text(verbatim: table)) + .navigationTitle(replacement ?? Text(verbatim: table)) .navigationSubtitle(Text(verbatim: schema)) } else { - content.navigationTitle(Text(verbatim: "\(schema).\(table)")) + content.navigationTitle(replacement ?? Text(verbatim: "\(schema).\(table)")) } } else { - content.navigationTitle(Text(verbatim: table)) + content.navigationTitle(replacement ?? Text(verbatim: table)) } } } diff --git a/apple/TableProMobile/TableProMobile/Views/ConnectedView.swift b/apple/TableProMobile/TableProMobile/Views/ConnectedView.swift index 2fa227aa8a..5a30a36035 100644 --- a/apple/TableProMobile/TableProMobile/Views/ConnectedView.swift +++ b/apple/TableProMobile/TableProMobile/Views/ConnectedView.swift @@ -20,6 +20,8 @@ struct ConnectedView: View { @State private var showDeletedAlert = false @State private var databaseSwitchAwaitingRollBack: String? @State private var retries = 0 + @State private var dataImport: IdentifiableURL? + @State private var dataImportMove: PendingDataImport? private var displayTitle: String { connection.name.isEmpty ? connection.host : connection.name @@ -320,6 +322,36 @@ struct ConnectedView: View { activity.isEligibleForHandoff = true activity.userInfo = ["connectionId": connection.id.uuidString] } + // A file opened from another app waits for the tabs, so it lands on a connected session. + .onChange(of: presenter.pendingDataImport, initial: true) { _, _ in + takePendingDataImport() + } + .sheet(item: $dataImport, onDismiss: dataImportDidDismiss) { item in + MobileImportSheet(fileURL: item.url, connectionChoice: importConnectionChoice) + .environment(appState) + .environment(coordinator) + } + } + + private func takePendingDataImport() { + guard dataImport == nil, let url = presenter.takeDataImport(for: connection.id) else { return } + dataImport = IdentifiableURL(url: url) + } + + private var importConnectionChoice: ImportConnectionChoice { + ImportConnectionChoice(connections: appState.connections.filter { !MacOnlyConnection.isMacOnly($0) }) { id, url in + dataImportMove = PendingDataImport(connectionId: id, fileURL: url) + } + } + + // Moved only once this sheet is gone, so the other connection's cover never meets a sheet still on screen. + private func dataImportDidDismiss() { + guard let move = dataImportMove else { + takePendingDataImport() + return + } + dataImportMove = nil + presenter.moveDataImport(move.fileURL, to: move.connectionId) } // A switch blocks edits on the content only, so the tabs, Connections and the status stay usable. diff --git a/apple/TableProMobile/TableProMobile/Views/ConnectionListView.swift b/apple/TableProMobile/TableProMobile/Views/ConnectionListView.swift index abcb3bbb64..6da0e8a81c 100644 --- a/apple/TableProMobile/TableProMobile/Views/ConnectionListView.swift +++ b/apple/TableProMobile/TableProMobile/Views/ConnectionListView.swift @@ -580,7 +580,7 @@ struct ConnectionListView: View { moreMenu if hasLibraryItems { Button { - withAnimation { + withMotion { editMode = isEditing ? .inactive : .active } } label: { @@ -730,6 +730,14 @@ struct ConnectionListView: View { case .export: MobileConnectionExportSheet(connections: appState.connections.filter(\.participatesInSync)) .environment(appState) + case .importData(let url): + DataImportConnectionChooser( + fileURL: url, + connections: appState.connections.filter { !MacOnlyConnection.isMacOnly($0) } + ) { connectionId in + presenter.requestDataImport(url, in: connectionId) + presenter.sheet = nil + } } } @@ -887,6 +895,14 @@ struct ConnectionListView: View { private func sheetDidDismiss() { presenter.sheetDidDismiss(appState: appState) deliverPendingIntent() + openConnectionForDataImport() + } + + // The chooser has gone, so the cover the chosen connection needs can come up. + private func openConnectionForDataImport() { + guard let pending = presenter.pendingDataImport, !isConnectionCoverOnScreen, + selectedConnectionUUID != pending.connectionId else { return } + open(pending.connectionId) } private func deliverPendingIntent() { @@ -904,6 +920,13 @@ struct ConnectionListView: View { case .importConnections(let url): selectedConnectionIdString = nil presenter.presentImportFile(url, coverIsOnScreen: isConnectionCoverOnScreen) + case .importData(let url): + // The connection on screen, or the one about to be restored, keeps its session and takes the file. + if let connectionId = selectedConnectionUUID, presentableRecord(for: connectionId) != nil { + presenter.requestDataImport(url, in: connectionId) + } else { + presenter.present(.importData(url)) + } } } @@ -916,6 +939,7 @@ struct ConnectionListView: View { private func connectionCoverDidDismiss() { isConnectionCoverOnScreen = false + presenter.discardDataImport(unlessFor: selectedConnectionUUID) presenter.dismissConnectionEditor() coordinatorStore.discardRemovedRecords() presenter.presentHeldImport() diff --git a/apple/TableProMobile/TableProMobile/Views/DataBrowserView.swift b/apple/TableProMobile/TableProMobile/Views/DataBrowserView.swift index a73e67880b..49ee5b4794 100644 --- a/apple/TableProMobile/TableProMobile/Views/DataBrowserView.swift +++ b/apple/TableProMobile/TableProMobile/Views/DataBrowserView.swift @@ -37,6 +37,10 @@ struct DataBrowserView: View { @State private var isCompletingCells = false @State private var hapticSuccess = false @State private var hapticError = false + @State private var exportItem: MobileExportItem? + @State private var showRowFilePicker = false + @State private var importFile: IdentifiableURL? + @State private var pendingSelectionDelete: PendingRowsDelete? @AppStorage(AppPreferences.rowLayoutKey) private var preferredLayout: RowLayout? private var activeSchema: String? { @@ -78,6 +82,22 @@ struct DataBrowserView: View { private var columns: [ColumnInfo] { viewModel.columns } private var rows: [Row] { viewModel.window.rows } + private var isSelecting: Bool { viewModel.selection.isActive } + + private var canExport: Bool { + session.map { ExportObjectListing.supportsExport($0.driver) } ?? false + } + + private var canExportTable: Bool { + session.map { ExportObjectListing.supportsTableExport($0.driver) } ?? false + } + + // Read-Only hides it, as it hides Insert Row; Confirm Writes asks when the import starts. + private var canImportRows: Bool { + allowsRowEditing && browsesSQLRows && !connection.safeModeLevel.blocksWrites && canExport + && !MobileImportFormats.rowFormats(for: connection.type).isEmpty + } + private var pageReads: DataBrowserViewModel.SessionReadTracker { let viewModel = viewModel return coordinator.trackPageRead { viewModel.loadError != nil } @@ -122,8 +142,11 @@ struct DataBrowserView: View { ] } .toolbar { topToolbar } + .toolbar { selectionBar } + .toolbar(isSelecting ? .hidden : .automatic, for: .tabBar) + .navigationBarBackButtonHidden(isSelecting) .bottomSafeAreaBar { - if showsPaginationBar { + if showsPaginationBar && !isSelecting { paginationBar } } @@ -147,6 +170,7 @@ struct DataBrowserView: View { // The confirmation names a row of the table this browser showed before. showDeleteConfirmation = false deleteTarget = nil + pendingSelectionDelete = nil guard source.driver != nil else { return } viewModel.attach( session: session, table: table, databaseType: connection.type, @@ -157,8 +181,51 @@ struct DataBrowserView: View { .onChange(of: appState.sessionTransactions.rollBackCount(for: connection.id)) { Task { await viewModel.load() } } + // A load an import refused runs once the import ends. + .onChange(of: coordinator.isImportRunning) { _, running in + guard !running, viewModel.waitsForImport else { return } + Task { await viewModel.load() } + } .onDisappear { viewModel.cancel() } .sheet(isPresented: $showInsertSheet) { insertSheet } + .sheet(item: $exportItem) { item in + MobileExportSheet( + input: item.input, + environment: .live(connectionManager: appState.connectionManager, coordinator: coordinator) + ) + } + .fileImporter( + isPresented: $showRowFilePicker, + allowedContentTypes: ImportPickerTypes.contentTypes(for: MobileImportFormats.rowFormats(for: connection.type)), + allowsMultipleSelection: false + ) { result in + guard case .success(let urls) = result, let url = urls.first else { return } + importFile = IdentifiableURL(url: url) + } + .sheet(item: $importFile) { item in + MobileImportSheet(fileURL: item.url, presetTable: table.name) { + Task { await viewModel.load() } + } + .environment(appState) + .environment(coordinator) + } + .confirmationDialog( + Text("^[Delete \(pendingSelectionDelete?.locators.count ?? 0) Row](inflect: true)?"), + isPresented: Binding( + get: { pendingSelectionDelete != nil }, + set: { if !$0 { pendingSelectionDelete = nil } } + ), + titleVisibility: .visible, + presenting: pendingSelectionDelete + ) { pending in + Button(role: .destructive) { + Task { await performSelectionDelete(pending) } + } label: { + Text("^[Delete \(pending.locators.count) Row](inflect: true)") + } + } message: { _ in + Text("This action cannot be undone.") + } .sheet(isPresented: $showFilterSheet) { FilterSheetView( filters: $viewModel.filters, @@ -255,12 +322,10 @@ struct DataBrowserView: View { @ViewBuilder private var searchableContent: some View { if !browsesSQLRows { - content - .tableTitle(table.name, schema: displaySchema) + titled(content) .navigationBarTitleDisplayMode(.inline) } else { - content - .tableTitle(table.name, schema: displaySchema) + titled(content) .navigationBarTitleDisplayMode(.inline) .safeAreaInset(edge: .top, spacing: 0) { LiteralSearchField("Search all columns", text: $searchText, isEditing: $searchFocused) { @@ -322,6 +387,14 @@ struct DataBrowserView: View { } } + private func titled(_ view: some View) -> some View { + view.tableTitle( + table.name, + schema: displaySchema, + replacement: isSelecting ? Text("^[\(viewModel.selection.count) Row](inflect: true) Selected") : nil + ) + } + private var rowLayout: RowLayout { RowLayout.resolved(preferred: preferredLayout, idiom: UIDevice.current.userInterfaceIdiom) } @@ -338,12 +411,14 @@ struct DataBrowserView: View { private var rowList: some View { let rows = rows - return List { + @Bindable var viewModel = viewModel + return List(selection: RowSelectionBinding.rows($viewModel.selection)) { ForEach(rows.indices, id: \.self) { index in rowLink(index: index, row: rows[index].displayValues()) } } .listStyle(.plain) + .environment(\.editMode, RowSelectionBinding.editMode($viewModel.selection)) } private var rowGrid: some View { @@ -351,28 +426,38 @@ struct DataBrowserView: View { return RowGrid( columns: columns, window: viewModel.window, - keyColumn: RowKey.columnIndex(columns: columns, primaryKeys: primaryKeys) + keyColumn: RowKey.columnIndex(columns: columns, primaryKeys: primaryKeys), + selectedRows: isSelecting ? viewModel.selection.rows : nil ) { index, line in - NavigationLink { - rowDetail(index: index) - } label: { - line - } - .hoverEffect() - .contextMenu { - rowContextMenu(index: index) - } preview: { - // The row itself can be thousands of points wide; the lift preview shows its card. - RowCard( - columns: columns, - columnDetails: viewModel.columnDetails, - row: rows.indices.contains(index) ? rows[index].displayValues() : [], - previewFieldCount: DuoLayoutResolver.regularPreviewFieldCount - ) - .padding() - .frame(maxWidth: 360, alignment: .leading) + if let isSelected = viewModel.selection.pick(of: index) { + Button { + viewModel.selection.toggle(index) + } label: { + line + } + .accessibilityAddTraits(isSelected ? .isSelected : []) + } else { + NavigationLink { + rowDetail(index: index) + } label: { + line + } + .hoverEffect() + .contextMenu { + rowContextMenu(index: index) + } preview: { + // The row itself can be thousands of points wide; the lift preview shows its card. + RowCard( + columns: columns, + columnDetails: viewModel.columnDetails, + row: rows.indices.contains(index) ? rows[index].displayValues() : [], + previewFieldCount: DuoLayoutResolver.regularPreviewFieldCount + ) + .padding() + .frame(maxWidth: 360, alignment: .leading) + } + .accessibilityActions { deleteAccessibilityAction(index: index) } } - .accessibilityActions { deleteAccessibilityAction(index: index) } } } @@ -386,7 +471,7 @@ struct DataBrowserView: View { .opacity(viewModel.isPageLoading ? 0.5 : 1) .allowsHitTesting(!viewModel.isPageLoading) .overlay { if viewModel.isPageLoading { ProgressView() } } - .animation(.default, value: viewModel.isPageLoading) + .motionAnimation(.default, value: viewModel.isPageLoading) .refreshable { await viewModel.load() } } @@ -424,22 +509,30 @@ struct DataBrowserView: View { ) } + // In Select mode the row is the card alone, which the list's edit mode picks; a disabled link would draw it dimmed. private func rowLink(index: Int, row: [String?]) -> some View { let deleteKeys = deletableRow(at: index) - return NavigationLink { - rowDetail(index: index) - } label: { - RowCard( - columns: columns, - columnDetails: viewModel.columnDetails, - row: row, - previewFieldCount: DuoLayoutResolver.previewFieldCount(for: duoWidthClass) - ) + let card = RowCard( + columns: columns, + columnDetails: viewModel.columnDetails, + row: row, + previewFieldCount: DuoLayoutResolver.previewFieldCount(for: duoWidthClass) + ) + return Group { + if viewModel.selection.opensRowDetail { + NavigationLink { + rowDetail(index: index) + } label: { + card + } + } else { + card + } } .hoverEffect() .contextMenu { rowContextMenu(index: index) } .swipeActions(edge: .trailing, allowsFullSwipe: false) { - if let deleteKeys { + if !isSelecting, let deleteKeys { Button { confirmDelete(deleteKeys) } label: { @@ -515,6 +608,31 @@ struct DataBrowserView: View { @ToolbarContentBuilder private var topToolbar: some ToolbarContent { + if isSelecting { + ToolbarItem(placement: .topBarLeading) { + let selectsAll = viewModel.selection.selectsAll(of: rows.count) + Button(selectsAll ? String(localized: "Deselect All") : String(localized: "Select All")) { + if selectsAll { + viewModel.selection.clearRows() + } else { + viewModel.selection.selectAll(count: rows.count) + } + } + } + ToolbarItem(placement: .topBarTrailing) { + Button { + viewModel.selection.end() + } label: { + Label("Done", systemImage: "checkmark") + } + } + } else { + browsingToolbar + } + } + + @ToolbarContentBuilder + private var browsingToolbar: some ToolbarContent { if browsesSQLRows { sortAndFilterItems } @@ -526,9 +644,29 @@ struct DataBrowserView: View { } Divider() } + Button { + viewModel.selection.begin() + } label: { + Label("Select Rows", systemImage: "checklist") + } + .disabled(rows.isEmpty) RowLayoutPicker() Divider() - Section("Export") { + if canExportTable || canImportRows { + Section { + if canExportTable { + Button { presentTableExport() } label: { + Label("Export…", systemImage: "square.and.arrow.up") + } + } + if canImportRows { + Button { showRowFilePicker = true } label: { + Label("Import Rows…", systemImage: "square.and.arrow.down") + } + } + } + } + Section("Copy Page") { ForEach(ExportFormat.offered(for: connection.type)) { format in Button { exportCompleted(viewModel.window.rows.map(\.cells), asOneRow: false, format: format) { text in @@ -553,6 +691,43 @@ struct DataBrowserView: View { } } + @ToolbarContentBuilder + private var selectionBar: some ToolbarContent { + if isSelecting { + ToolbarItemGroup(placement: .bottomBar) { + let indices = viewModel.selection.indices(within: rows.count) + Button { + presentSelectionExport(indices) + } label: { + Label("Export", systemImage: "square.and.arrow.up") + } + .disabled(indices.isEmpty || !canExport) + Spacer() + Menu { + ForEach(ExportFormat.offered(for: connection.type)) { format in + Button(format.rawValue) { + exportCompleted(indices.map(viewModel.cells(at:)), asOneRow: false, format: format) { text in + ClipboardExporter.copyToClipboard(text) + } + } + } + } label: { + Label("Copy", systemImage: "doc.on.doc") + } + .disabled(indices.isEmpty || isCompletingCells) + if canDeleteRows { + Spacer() + Button(role: .destructive) { + confirmSelectionDelete(indices) + } label: { + Label("Delete", systemImage: "trash") + } + .disabled(indices.isEmpty || viewModel.rowLocators(at: indices) == nil) + } + } + } + } + @ToolbarContentBuilder private var sortAndFilterItems: some ToolbarContent { ToolbarItem(placement: .topBarTrailing) { @@ -664,6 +839,7 @@ struct DataBrowserView: View { let driver = session?.driver let tableName = table.name let databaseType = connection.type + let columnDetails = viewModel.columnDetails isCompletingCells = true Task { defer { isCompletingCells = false } @@ -674,7 +850,7 @@ struct DataBrowserView: View { } deliver(try await Self.exportText( completed, columns: exportColumns, asOneRow: asOneRow, format: format, - tableName: tableName, databaseType: databaseType, driver: driver + tableName: tableName, databaseType: databaseType, driver: driver, columnDetails: columnDetails )) } catch let error where error is ClipboardExporter.IncompleteValues || error is ClipboardExporter.TooLarge || error is CellLookupError { @@ -705,27 +881,117 @@ struct DataBrowserView: View { format: ExportFormat, tableName: String, databaseType: DatabaseType, - driver: (any DatabaseDriver)? + driver: (any DatabaseDriver)?, + columnDetails: [ColumnInfo] ) async throws -> String { if asOneRow, let cells = rows.first { return try ClipboardExporter.exportRow( columns: columns, cells: cells, format: format, tableName: tableName, - databaseType: databaseType, driver: driver + databaseType: databaseType, driver: driver, columnDetails: columnDetails ) } return try ClipboardExporter.exportRows( columns: columns, rows: rows, format: format, tableName: tableName, - databaseType: databaseType, driver: driver + databaseType: databaseType, driver: driver, columnDetails: columnDetails ) } + // MARK: - Export and Selection + + private var exportDatabase: String? { + viewModel.database ?? (coordinator.activeDatabase.isEmpty ? nil : coordinator.activeDatabase) + } + + private var exportSchema: String? { + coordinator.supportsSchemas ? viewModel.schema : nil + } + + private func exportInput(title: String, scopes: [ExportScopeOption]) -> MobileExportInput { + MobileExportInput( + title: title, + fileName: table.name, + connectionId: connection.id, + databaseType: connection.type, + database: exportDatabase, + schema: exportSchema, + safeModeLevel: connection.safeModeLevel, + scopes: scopes + ) + } + + private func presentTableExport() { + var scopes: [ExportScopeOption] = [] + if viewModel.narrowsRows, let query = viewModel.exportSelect() { + scopes.append(ExportScopeOption( + kind: .currentView, + source: .query(query), + detail: String(localized: "Every row that matches the search, filter and sort, not only this page.") + )) + } + scopes.append(ExportScopeOption(kind: .wholeTable, source: .table(table.name))) + exportItem = MobileExportItem(input: exportInput(title: String(localized: "Export Table"), scopes: scopes)) + } + + private func presentSelectionExport(_ indices: [Int]) { + do { + let source = try viewModel.selectedRowsExportSource(indices, readsAgain: coordinator.transactionHold == nil) + let detail = String(AttributedString(localized: "^[\(indices.count) row](inflect: true) selected.").characters) + exportItem = MobileExportItem(input: exportInput( + title: String(localized: "Export Rows"), + scopes: [ExportScopeOption(kind: .selectedRows, source: source, detail: detail)] + )) + } catch { + viewModel.operationError = AppError( + category: .query, + title: String(localized: "Cannot Export"), + message: error.localizedDescription, + recovery: nil, + underlying: error + ) + hapticError.toggle() + } + } + + private func confirmSelectionDelete(_ indices: [Int]) { + guard let locators = viewModel.rowLocators(at: indices), !locators.isEmpty else { return } + pendingSelectionDelete = PendingRowsDelete(locators: locators, source: viewModel.rowSource) + } + + private func performSelectionDelete(_ pending: PendingRowsDelete) async { + guard !viewModel.refusesDelete(raisedOn: pending.source) else { + hapticError.toggle() + return + } + let total = pending.locators.count + let deleted: Int + do { + deleted = try await coordinator.track(failed: { $0 < total }) { await viewModel.deleteRows(pending.locators) } + } catch { + viewModel.operationError = ConnectionCoordinator.refusal(for: error) + hapticError.toggle() + return + } + if deleted == total { + hapticSuccess.toggle() + } else { + hapticError.toggle() + } + } + private func performDelete(_ pending: PendingRowDelete) async { guard !viewModel.refusesDelete(raisedOn: pending.source) else { hapticError.toggle() return } let locator = pending.locator - let success = await coordinator.track(failed: { !$0 }) { await viewModel.deleteRow(locator) } + let success: Bool + do { + success = try await coordinator.track(failed: { !$0 }) { await viewModel.deleteRow(locator) } + } catch { + viewModel.operationError = ConnectionCoordinator.refusal(for: error) + hapticError.toggle() + return + } if success { hapticSuccess.toggle() } else { @@ -739,6 +1005,11 @@ private struct PendingRowDelete { let source: DataBrowserViewModel.RowSource } +private struct PendingRowsDelete { + let locators: [RowLocator] + let source: DataBrowserViewModel.RowSource +} + private struct BrowseSource: Equatable { let driver: ObjectIdentifier? let schema: String? diff --git a/apple/TableProMobile/TableProMobile/Views/ForeignKeyNavigation.swift b/apple/TableProMobile/TableProMobile/Views/ForeignKeyNavigation.swift index 26860ecb69..9161315dfc 100644 --- a/apple/TableProMobile/TableProMobile/Views/ForeignKeyNavigation.swift +++ b/apple/TableProMobile/TableProMobile/Views/ForeignKeyNavigation.swift @@ -45,6 +45,7 @@ private struct ForeignKeyPreviewModifier: ViewModifier { databaseType: databaseType, currentSchema: currentSchema, allowsOpen: allowsOpen && coordinator != nil, + trackRead: sessionCoordinator.trackPageRead(failed: { false }), refreshTransaction: { [sessionCoordinator] in await sessionCoordinator.refreshTransaction() }, onOpen: { requestedOpen = $0 } ) diff --git a/apple/TableProMobile/TableProMobile/Views/InsertRowView.swift b/apple/TableProMobile/TableProMobile/Views/InsertRowView.swift index bd92195011..824e772bf0 100644 --- a/apple/TableProMobile/TableProMobile/Views/InsertRowView.swift +++ b/apple/TableProMobile/TableProMobile/Views/InsertRowView.swift @@ -321,6 +321,10 @@ struct InsertRowView: View { hapticSuccess.toggle() onInserted?() dismiss() + } catch let held as SessionHeldByImport { + operationError = held.appError + showOperationError = true + hapticError.toggle() } catch let unavailable as AppError { operationError = unavailable showOperationError = true diff --git a/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift b/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift index c18385660b..e74bc45df9 100644 --- a/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift +++ b/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift @@ -52,6 +52,7 @@ struct QueryEditorView: View { @State private var containerSize: CGSize = .zero @State private var savedQueriesRoute: SavedQueriesSheetRoute? @State private var savedQueryFolderExpansion: Set = [] + @State private var exportItem: MobileExportItem? @AppStorage(AppPreferences.rowLayoutKey) private var preferredLayout: RowLayout? private var session: ConnectionSession? { coordinator.session } @@ -60,6 +61,11 @@ struct QueryEditorView: View { private var activeSchema: String? { coordinator.supportsSchemas ? coordinator.activeSchema : nil } private var safeModeLevel: SafeModeLevel { coordinator.connection.safeModeLevel } private var connectionId: UUID { coordinator.connection.id } + private var isSelecting: Bool { viewModel.selection.isActive } + + private var canExport: Bool { + session.map { ExportObjectListing.supportsExport($0.driver) } ?? false + } private var completionScope: QueryCompletionScope { QueryCompletionScope( @@ -249,6 +255,14 @@ struct QueryEditorView: View { .sheet(isPresented: $showShareSheet) { ActivityViewController(items: [shareText]) } + .sheet(item: $exportItem) { item in + MobileExportSheet( + input: item.input, + environment: .live(connectionManager: appState.connectionManager, coordinator: coordinator) + ) + } + .toolbar { selectionBar } + .toolbar(isSelecting ? .hidden : .automatic, for: .tabBar) .sheet(item: $savedQueriesRoute) { route in SavedQueriesSheet( route: route, @@ -297,7 +311,19 @@ struct QueryEditorView: View { } } + @ViewBuilder private var actionBar: some View { + if isSelecting { + RowSelectionHeader( + selection: Binding(get: { viewModel.selection }, set: { viewModel.selection = $0 }), + rowCount: viewModel.window.count + ) + } else { + runActionBar + } + } + + private var runActionBar: some View { HStack(spacing: 12) { if isExecuting { Button { @@ -410,7 +436,7 @@ struct QueryEditorView: View { } private var canRun: Bool { - !isExecuting && !coordinator.isDatabaseMoving && !query.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + !isExecuting && coordinator.runRefusal == nil && !query.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } private var runAtCursorTitle: String { @@ -594,20 +620,30 @@ struct QueryEditorView: View { return RowGrid( columns: columns, window: window, - keyColumn: RowKey.columnIndex(columns: columns, primaryKeys: []) + keyColumn: RowKey.columnIndex(columns: columns, primaryKeys: []), + selectedRows: isSelecting ? viewModel.selection.rows : nil ) { index, line in - NavigationLink { - resultRowDetail(columns: columns, initialIndex: index) - } label: { - line - } - .hoverEffect() - .contextMenu { - resultRowContextMenu(columns: columns, rowIndex: index) - } preview: { - resultRowCard(columns: columns, row: rows.indices.contains(index) ? rows[index].displayValues() : []) - .padding() - .frame(maxWidth: 360, alignment: .leading) + if let isSelected = viewModel.selection.pick(of: index) { + Button { + viewModel.selection.toggle(index) + } label: { + line + } + .accessibilityAddTraits(isSelected ? .isSelected : []) + } else { + NavigationLink { + resultRowDetail(columns: columns, initialIndex: index) + } label: { + line + } + .hoverEffect() + .contextMenu { + resultRowContextMenu(columns: columns, rowIndex: index) + } preview: { + resultRowCard(columns: columns, row: rows.indices.contains(index) ? rows[index].displayValues() : []) + .padding() + .frame(maxWidth: 360, alignment: .leading) + } } } } @@ -615,12 +651,21 @@ struct QueryEditorView: View { private var resultList: some View { let columns = viewModel.columns let rows = viewModel.window.rows - return List { + @Bindable var viewModel = viewModel + return List(selection: RowSelectionBinding.rows($viewModel.selection)) { ForEach(rows.indices, id: \.self) { rowIndex in - NavigationLink { - resultRowDetail(columns: columns, initialIndex: rowIndex) - } label: { - resultRowCard(columns: columns, row: rows[rowIndex].displayValues()) + let card = resultRowCard(columns: columns, row: rows[rowIndex].displayValues()) + // In Select mode the row is the card alone, which edit mode picks; a disabled link would draw it dimmed. + Group { + if viewModel.selection.opensRowDetail { + NavigationLink { + resultRowDetail(columns: columns, initialIndex: rowIndex) + } label: { + card + } + } else { + card + } } .contextMenu { resultRowContextMenu(columns: columns, rowIndex: rowIndex) @@ -628,6 +673,31 @@ struct QueryEditorView: View { } } .listStyle(.plain) + .environment(\.editMode, RowSelectionBinding.editMode($viewModel.selection)) + } + + @ToolbarContentBuilder + private var selectionBar: some ToolbarContent { + if isSelecting { + ToolbarItemGroup(placement: .bottomBar) { + let indices = viewModel.selection.indices(within: viewModel.window.count) + Button { + presentSelectionExport(indices) + } label: { + Label("Export", systemImage: "square.and.arrow.up") + } + .disabled(indices.isEmpty || !canExport) + Spacer() + Menu { + ForEach(ExportFormat.offered(for: databaseType)) { format in + Button(format.rawValue) { copySelection(indices, as: format) } + } + } label: { + Label("Copy", systemImage: "doc.on.doc") + } + .disabled(indices.isEmpty || isExportingResults) + } + } } // No table, so nothing to edit; the engine's type and session decide how Copy Row quotes and escapes. @@ -742,7 +812,21 @@ struct QueryEditorView: View { } if !viewModel.window.isEmpty { + Button { + viewModel.selection.begin() + } label: { + Label("Select Rows", systemImage: "checklist") + } + .disabled(isExecuting) RowLayoutPicker() + if canExport { + Button { + presentResultsExport() + } label: { + Label("Export Results…", systemImage: "square.and.arrow.up.on.square") + } + .disabled(isExecuting) + } Section("Share Results") { ForEach(ExportFormat.offered(for: databaseType)) { format in Button { @@ -815,6 +899,61 @@ struct QueryEditorView: View { } } + // MARK: - Export and Selection + + private func exportInput(title: String, scope: ExportScopeOption) -> MobileExportInput { + MobileExportInput( + title: title, + fileName: String(localized: "Query Results"), + connectionId: connectionId, + databaseType: databaseType, + database: coordinator.activeDatabase.isEmpty ? nil : coordinator.activeDatabase, + schema: activeSchema, + safeModeLevel: safeModeLevel, + scopes: [scope] + ) + } + + private func presentResultsExport() { + do { + let scope = try QueryResultsExport.resultScope(viewModel, databaseType: databaseType, coordinator: coordinator) + exportItem = MobileExportItem(input: exportInput(title: String(localized: "Export Results"), scope: scope)) + } catch { + resultsExportError = error + } + } + + private func presentSelectionExport(_ indices: [Int]) { + do { + let scope = try QueryResultsExport.selectionScope( + columns: viewModel.columns, rows: viewModel.window.rows, indices: indices + ) + exportItem = MobileExportItem(input: exportInput(title: String(localized: "Export Rows"), scope: scope)) + } catch { + resultsExportError = error + } + } + + private func copySelection(_ indices: [Int], as format: ExportFormat) { + guard !isExportingResults else { return } + let rows = viewModel.window.rows + let picked = indices.filter(rows.indices.contains).map { rows[$0].cells } + let columns = viewModel.columns + let databaseType = databaseType + let driver = coordinator.session?.driver + isExportingResults = true + Task { + defer { isExportingResults = false } + do { + ClipboardExporter.copyToClipboard(try await QueryResultsExport.clipboardText( + picked, columns: columns, format: format, databaseType: databaseType, driver: driver + )) + } catch { + resultsExportError = error + } + } + } + // MARK: - Execution private func runAtCursor(dismissingKeyboard: Bool) { @@ -896,9 +1035,9 @@ struct QueryEditorView: View { // Claimed before the first await, so a second Run while the session reports how it reads literals starts nothing. private func executeQueryDirect(_ trimmed: String, enforcesReadOnly: Bool) async { guard let session, !isExecuting else { return } - // A run with a `USE` in any window holds the session until the picker follows it. - guard !coordinator.isDatabaseMoving else { - appError = ConnectionCoordinator.databaseMovingRefusal + // A run with a `USE` in any window holds the session until the picker follows it, and an import until it ends. + if let refusal = coordinator.runRefusal { + appError = refusal return } @@ -958,11 +1097,26 @@ struct QueryEditorView: View { let progressUpdater = startActivityProgressUpdater(token: token) - await coordinator.track( - failed: { _ in QueryExecutionOutcome(phase: viewModel.phase) == .failed }, - rolledBack: { _ in viewModel.ranTransactionRollBack } - ) { - await viewModel.run(driver: session.driver, script: script, enforcesReadOnly: enforcesReadOnly) + let placement = coordinator.tablePlacement + do { + try await coordinator.track( + marksSession: false, + failed: { _ in QueryExecutionOutcome(phase: viewModel.phase) == .failed }, + rolledBack: { _ in viewModel.ranTransactionRollBack } + ) { + await viewModel.run(driver: session.driver, script: script, enforcesReadOnly: enforcesReadOnly, placement: placement) + } + } catch { + // An import took the session after the check above; nothing ran. + progressUpdater.cancel() + if movesDatabase { coordinator.endDatabaseMove() } + if holdsCatalog { appState.connectionManager.releaseCatalogConnections(for: connectionId) } + appError = ConnectionCoordinator.refusal(for: error) + isExecuting = false + completion.reopenColumnReads() + executionStartTime = nil + await appState.queryActivities.end(token: token, outcome: QueryExecutionOutcome.failed.activityOutcome) + return } progressUpdater.cancel() @@ -1013,40 +1167,6 @@ struct QueryEditorView: View { await appState.queryActivities.end(token: token, outcome: outcome.activityOutcome) } - // Every statement sent counts as run, a failed one's too: a missed reload leaves the lists naming what is gone. - // They are read as sent, since a run splits the rest of its text again after a statement that changes how the - // session reads literals. - private func postCatalogChanges(on session: ConnectionSession) -> ConnectionCoordinator.SessionMove { - let statements = viewModel.sentStatements - // A key or a collection a write made or removed is on the lists too; only a proven read leaves them as they are. - guard databaseType.speaksSQL else { - if statements.contains(where: { SQLWriteClassifier.isWriteQuery($0, databaseType: databaseType) }) { - coordinator.noteCatalogChanged([.tables, .databases]) - } - return ConnectionCoordinator.SessionMove() - } - let sessionEffect = SessionLocalStatements.effect(of: statements, databaseType: databaseType) - appState.catalogChanges.noteSessionEffect( - sessionEffect, - for: connectionId, - driver: ObjectIdentifier(session.driver) - ) - let grammar = SQLLexicalReadings.resolve( - databaseTypeId: databaseType.rawValue, declared: nil, session: nil - ).execution - let effect = SQLCatalogEffectClassifier.effect(ofStatements: statements, grammar: grammar) - var kinds = effect.kinds - // A commit shows the transaction's DDL to the catalog connection, a rollback takes it back from the session. - if effect.endsTransaction { kinds.formUnion([.tables, .schemas, .databases]) } - coordinator.noteCatalogChanged(kinds) - // A transaction's end takes back a `SET LOCAL` or a `SET` inside it, a function may set the search path, and a - // typed ROLLBACK of a held transaction also reaches the Roll Back reload. - return ConnectionCoordinator.SessionMove( - database: sessionEffect.usesDatabase, - schema: sessionEffect.leavesCatalog || effect.endsTransaction - ) - } - private func plan(_ text: String, on session: ConnectionSession) async -> QueryScript { QueryScript.plan( text, @@ -1093,3 +1213,44 @@ struct QueryEditorView: View { } } } + +// Outside the struct's body, which is long enough already. +fileprivate extension QueryEditorView { + // Every statement sent counts as run, a failed one's too: a missed reload leaves the lists naming what is gone. + // They are read as sent, since a run splits the rest of its text again after a statement that changes how the + // session reads literals. + func postCatalogChanges(on session: ConnectionSession) -> ConnectionCoordinator.SessionMove { + let statements = viewModel.sentStatements + // A fresh connection would not repeat what these left on the session, so Export Results stops running again. + if SessionLocalStatements.changesSession(statements, databaseType: databaseType) { + coordinator.noteSessionChanged(on: session.driver) + } + // A key or a collection a write made or removed is on the lists too; only a proven read leaves them as they are. + guard databaseType.speaksSQL else { + if statements.contains(where: { SQLWriteClassifier.isWriteQuery($0, databaseType: databaseType) }) { + coordinator.noteCatalogChanged([.tables, .databases]) + } + return ConnectionCoordinator.SessionMove() + } + let sessionEffect = SessionLocalStatements.effect(of: statements, databaseType: databaseType) + appState.catalogChanges.noteSessionEffect( + sessionEffect, + for: connectionId, + driver: ObjectIdentifier(session.driver) + ) + let grammar = SQLLexicalReadings.resolve( + databaseTypeId: databaseType.rawValue, declared: nil, session: nil + ).execution + let effect = SQLCatalogEffectClassifier.effect(ofStatements: statements, grammar: grammar) + var kinds = effect.kinds + // A commit shows the transaction's DDL to the catalog connection, a rollback takes it back from the session. + if effect.endsTransaction { kinds.formUnion([.tables, .schemas, .databases]) } + coordinator.noteCatalogChanged(kinds) + // A transaction's end takes back a `SET LOCAL` or a `SET` inside it, a function may set the search path, and a + // typed ROLLBACK of a held transaction also reaches the Roll Back reload. + return ConnectionCoordinator.SessionMove( + database: sessionEffect.usesDatabase, + schema: sessionEffect.leavesCatalog || effect.endsTransaction + ) + } +} diff --git a/apple/TableProMobile/TableProMobile/Views/RowDetailView.swift b/apple/TableProMobile/TableProMobile/Views/RowDetailView.swift index 335954bee4..88dc314da9 100644 --- a/apple/TableProMobile/TableProMobile/Views/RowDetailView.swift +++ b/apple/TableProMobile/TableProMobile/Views/RowDetailView.swift @@ -425,8 +425,14 @@ struct RowDetailView: View { } private func handleSave() async { - let success = await connectionCoordinator.track(failed: { _ in viewModel.operationError != nil }) { - await viewModel.saveChanges() + let success: Bool + do { + success = try await connectionCoordinator.track(failed: { _ in viewModel.operationError != nil }) { + await viewModel.saveChanges() + } + } catch { + refused(error) + return } if viewModel.pendingWriteConfirmation { showSaveConfirmation = true @@ -440,8 +446,14 @@ struct RowDetailView: View { } private func executePendingSave() async { - let success = await connectionCoordinator.track(failed: { _ in viewModel.operationError != nil }) { - await viewModel.executePendingSave() + let success: Bool + do { + success = try await connectionCoordinator.track(failed: { _ in viewModel.operationError != nil }) { + await viewModel.executePendingSave() + } + } catch { + refused(error) + return } if success { hapticSuccess.toggle() @@ -450,6 +462,12 @@ struct RowDetailView: View { } } + // An import holds the session, so nothing was sent. + private func refused(_ error: any Error) { + viewModel.operationError = ConnectionCoordinator.refusal(for: error) + hapticError.toggle() + } + // Names the cell rather than carrying its text, so the route stays cheap to compare. private struct FullValueRoute: Hashable { let row: Int diff --git a/apple/TableProMobile/TableProMobile/Views/TableListView.swift b/apple/TableProMobile/TableProMobile/Views/TableListView.swift index bd66c57ec8..7c79676738 100644 --- a/apple/TableProMobile/TableProMobile/Views/TableListView.swift +++ b/apple/TableProMobile/TableProMobile/Views/TableListView.swift @@ -2,9 +2,12 @@ import SwiftUI import TableProConnectionLibrary import TableProDatabase import TableProModels +import TableProPluginKit import TableProSQLGrammar +import UniformTypeIdentifiers struct TableListView: View { + @Environment(AppState.self) private var appState @Environment(ConnectionCoordinator.self) private var coordinator private var connection: DatabaseConnection { coordinator.connection } @@ -36,6 +39,41 @@ struct TableListView: View { @State private var tableToDrop: PendingTableAction? @State private var errorMessage = "" @State private var showError = false + @State private var exportItem: MobileExportItem? + @State private var showFilePicker = false + @State private var filePickerTypes: [UTType] = [] + @State private var importFile: IdentifiableURL? + + private var canExport: Bool { + session.map { ExportObjectListing.supportsExport($0.driver) } ?? false + } + + private var canExportTable: Bool { + session.map { ExportObjectListing.supportsTableExport($0.driver) } ?? false + } + + // Read-Only hides what writes; Confirm Writes asks when the import starts. + private var writesAllowed: Bool { + !connection.safeModeLevel.blocksWrites && canExport + } + + private var importFormats: [any ImportFormatPlugin] { + guard writesAllowed else { return [] } + let rows = MobileImportFormats.rowFormats(for: connection.type) + return MobileImportFormats.offersSQLFile(for: connection.type) ? rows + [MobileImportFormats.sql] : rows + } + + private var canRunSQLFile: Bool { + writesAllowed && MobileImportFormats.offersSQLFile(for: connection.type) + } + + private var exportDatabase: String? { + coordinator.activeDatabase.isEmpty ? nil : coordinator.activeDatabase + } + + private var exportSchema: String? { + coordinator.supportsSchemas ? coordinator.activeSchema : nil + } private var showTruncateConfirmation: Binding { Binding( @@ -90,6 +128,14 @@ struct TableListView: View { Label("Copy Name", systemImage: "doc.on.doc") } + if canExportTable { + Button { + presentExport(of: table) + } label: { + Label("Export…", systemImage: "square.and.arrow.up") + } + } + let writesAllowed = !connection.safeModeLevel.blocksWrites && engineSpeaksSQLDDL if writesAllowed && (table.type.allowsTruncate || table.type.allowsDrop) { Divider() @@ -125,6 +171,24 @@ struct TableListView: View { } } .listStyle(.insetGrouped) + .toolbar { transferMenu } + .sheet(item: $exportItem) { item in + MobileExportSheet( + input: item.input, + environment: .live(connectionManager: appState.connectionManager, coordinator: coordinator) + ) + } + .fileImporter(isPresented: $showFilePicker, allowedContentTypes: filePickerTypes, allowsMultipleSelection: false) { result in + guard case .success(let urls) = result, let url = urls.first else { return } + importFile = IdentifiableURL(url: url) + } + .sheet(item: $importFile) { item in + MobileImportSheet(fileURL: item.url) { + Task { await coordinator.refreshTables() } + } + .environment(appState) + .environment(coordinator) + } .searchable(text: $searchText, prompt: "Search tables") .searchFocused($searchFocused) .textInputAutocapitalization(.never) @@ -225,6 +289,77 @@ struct TableListView: View { } } + @ToolbarContentBuilder + private var transferMenu: some ToolbarContent { + if canExport || !importFormats.isEmpty { + ToolbarItem(placement: .topBarTrailing) { + Menu { + if canExport { + Button { + presentDatabaseExport() + } label: { + Label("Export Database…", systemImage: "square.and.arrow.up") + } + } + if !importFormats.isEmpty { + Button { + pickFile(for: importFormats) + } label: { + Label("Import…", systemImage: "square.and.arrow.down") + } + } + if canRunSQLFile { + Button { + pickFile(for: [MobileImportFormats.sql]) + } label: { + Label("Run SQL File…", systemImage: "doc.text") + } + } + } label: { + Label("More", systemImage: ToolbarSymbols.more()) + } + } + } + } + + private func pickFile(for formats: [any ImportFormatPlugin]) { + filePickerTypes = ImportPickerTypes.contentTypes(for: formats) + showFilePicker = true + } + + private func exportInput(title: String, fileName: String, scope: ExportScopeOption) -> MobileExportInput { + MobileExportInput( + title: title, + fileName: fileName, + connectionId: connection.id, + databaseType: connection.type, + database: exportDatabase, + schema: exportSchema, + safeModeLevel: connection.safeModeLevel, + scopes: [scope] + ) + } + + private func presentExport(of table: TableInfo) { + exportItem = MobileExportItem(input: exportInput( + title: String(localized: "Export Table"), + fileName: table.name, + scope: ExportScopeOption(kind: .wholeTable, source: .table(table.name)) + )) + } + + // Named as the Mac names it: the container the objects sit in, else the database, else the connection. + private func presentDatabaseExport() { + let fileName = [exportSchema, exportDatabase, coordinator.displayName] + .compactMap { $0 } + .first { !$0.isEmpty } ?? String(localized: "Export") + exportItem = MobileExportItem(input: exportInput( + title: String(localized: "Export Database"), + fileName: fileName, + scope: ExportScopeOption(kind: .database, source: .database) + )) + } + // Another window can switch the shared session between the tap and the confirmation; the table named in the dialog // may then sit in another database or schema, so nothing runs. private func driver(for action: PendingTableAction) -> (any DatabaseDriver)? { diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorBackgroundTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorBackgroundTests.swift index 7e6d2ecb4f..a97b79cfbb 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorBackgroundTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorBackgroundTests.swift @@ -79,7 +79,7 @@ struct ConnectionCoordinatorBackgroundTests { func backgroundCloseDuringWorkIsAmbiguous() async throws { let coordinator = try await connectedCoordinator(on: fileDriver(.explicitTransaction)) let gate = TestGate() - let work = Task { await coordinator.track { await gate.enter() } } + let work = Task { try? await coordinator.track { await gate.enter() } } await gate.waitUntilEntered() await releaseForTheBackground() @@ -99,7 +99,7 @@ struct ConnectionCoordinatorBackgroundTests { await releaseForTheBackground() let gate = TestGate() - let work = Task { await coordinator.track { await gate.enter() } } + let work = Task { try? await coordinator.track { await gate.enter() } } await gate.waitUntilEntered() factory.queue(fileDriver(.idle)) await coordinator.reconnectIfNeeded() diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorCatalogTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorCatalogTests.swift index 1ba67a44a7..acbd2f76d4 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorCatalogTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorCatalogTests.swift @@ -188,7 +188,7 @@ struct ConnectionCoordinatorCatalogTests { let coordinator = await connected(session) session.scriptedTables = [TableInfo(name: "orders"), TableInfo(name: "invoices")] let gate = TestGate() - let page = Task { await appState.sessionTransactions.track(connection.id, reading: true) { await gate.enter() } } + let page = Task { try? await appState.sessionTransactions.track(connection.id, reading: true) { await gate.enter() } } await gate.waitUntilEntered() coordinator.noteCatalogChanged(.tables) diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorReconnectTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorReconnectTests.swift index cd77db9040..6da325f1b6 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorReconnectTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorReconnectTests.swift @@ -73,10 +73,10 @@ struct ConnectionCoordinatorReconnectTests { return (left, right) } - private func runQuery(_ text: String, in coordinator: ConnectionCoordinator, on driver: MockDatabaseDriver) async { + private func runQuery(_ text: String, in coordinator: ConnectionCoordinator, on driver: MockDatabaseDriver) async throws { let viewModel = QueryEditorViewModel() let script = QueryScript(units: [QueryScript.Unit(sql: text)], unitKind: .statement) - await coordinator.track(rolledBack: { _ in viewModel.ranTransactionRollBack }) { + try await coordinator.track(rolledBack: { _ in viewModel.ranTransactionRollBack }) { await viewModel.run(driver: driver, script: script) } } @@ -713,7 +713,7 @@ struct ConnectionCoordinatorReconnectTests { #expect(coordinator.transactionHold == .open) first.scriptedTransactionState = .idle - await runQuery("COMMIT", in: coordinator, on: first) + try await runQuery("COMMIT", in: coordinator, on: first) #expect(coordinator.transactionHold == nil) #expect(monitor.rollBackCount(for: connection.id) == 0) @@ -721,7 +721,7 @@ struct ConnectionCoordinatorReconnectTests { await coordinator.refreshTransaction() #expect(coordinator.transactionHold == .open) first.scriptedTransactionState = .idle - await runQuery("ROLLBACK", in: coordinator, on: first) + try await runQuery("ROLLBACK", in: coordinator, on: first) #expect(coordinator.transactionHold == nil) #expect(monitor.rollBackCount(for: connection.id) == 1) @@ -736,7 +736,7 @@ struct ConnectionCoordinatorReconnectTests { let viewModel = QueryEditorViewModel() let script = QueryScript.plan("SELECT 1; ROLLBACK TRANSACTION;", databaseType: .mssql, sendsBatchesWhole: true) - await coordinator.track(rolledBack: { _ in viewModel.ranTransactionRollBack }) { + try await coordinator.track(rolledBack: { _ in viewModel.ranTransactionRollBack }) { await viewModel.run(driver: first, script: script) } @@ -750,7 +750,7 @@ struct ConnectionCoordinatorReconnectTests { try await dial(first) let coordinator = await attachedCoordinator() - await runQuery("ROLLBACK AND CHAIN", in: coordinator, on: first) + try await runQuery("ROLLBACK AND CHAIN", in: coordinator, on: first) #expect(coordinator.transactionHold == .open) #expect(monitor.rollBackCount(for: connection.id) == 1) @@ -814,7 +814,7 @@ struct ConnectionCoordinatorReconnectTests { try await dial(first) let coordinator = await attachedCoordinator() let gate = TestGate() - let work = Task { await coordinator.track { await gate.enter() } } + let work = Task { try? await coordinator.track { await gate.enter() } } await gate.waitUntilEntered() first.pingError = MockDatabaseDriver.MockError.scripted @@ -957,7 +957,7 @@ struct ConnectionCoordinatorReconnectTests { first.pingError = MockDatabaseDriver.MockError.scripted let second = driver(tables: ["orders"]) factory.queue(second) - let ranIntoAnError = await coordinator.track(failed: { $0 }) { true } + let ranIntoAnError = try await coordinator.track(failed: { $0 }) { true } #expect(ranIntoAnError) #expect(coordinator.session?.driver === second) diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStatusTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStatusTests.swift index 17b19e114f..c1042b322b 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStatusTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStatusTests.swift @@ -65,6 +65,76 @@ struct ConnectionCoordinatorStatusTests { return false } + // MARK: - Import + + @Test("While an import in one window holds the session, another window's query, write and switch are refused at once") + func importHoldsTheSessionForEveryWindow() async throws { + let session = driver() + let importing = try await attachedCoordinator(on: session) + let other = ConnectionCoordinator(connection: connection, appState: appState) + await other.connect() + let log = QueryLog() + let gate = TestGate() + let importRun = Task { + try await importing.trackImport(failed: { (_: Bool) in false }) { admission in + #expect(admission.driver === session) + await gate.enter() + return true + } + } + await gate.waitUntilEntered() + + #expect(other.isImportRunning) + #expect(other.runRefusal?.message == SessionHeldByImport().appError.message) + await #expect(throws: SessionHeldByImport.self) { + try await other.track { log.events.append("query") } + } + await #expect(throws: SessionHeldByImport.self) { + try await other.track(reading: true) { log.events.append("page") } + } + await other.switchDatabase(to: "app_archive") + #expect(other.failureAlertMessage == SessionHeldByImport().localizedDescription) + #expect(log.events.isEmpty) + + await gate.open() + _ = try await importRun.value + try await other.track { log.events.append("query") } + #expect(log.events == ["query"]) + #expect(other.isImportRunning == false) + } + + @Test("An import asked for while other work runs on the session is refused, not queued") + func importRefusedWhileWorkRuns() async throws { + let importing = try await attachedCoordinator(on: driver()) + let gate = TestGate() + let work = Task { try? await importing.track { await gate.enter() } } + await gate.waitUntilEntered() + + await #expect(throws: SessionBusyForImport.self) { + try await importing.trackImport(failed: { (_: Bool) in false }) { _ in true } + } + + await gate.open() + await work.value + #expect(try await importing.trackImport(failed: { (_: Bool) in false }) { _ in true }) + } + + @Test("A result reruns on a fresh connection only from the same unchanged session, still where it was read") + func freshConnectionReproducesOnlyAnUnchangedSession() async throws { + let coordinator = try await attachedCoordinator(on: driver()) + let placement = coordinator.tablePlacement + + try await coordinator.track(reading: true) {} + #expect(coordinator.reproducesOnFreshConnection(placement)) + #expect(coordinator.reproducesOnFreshConnection(nil) == false) + + try await coordinator.track(marksSession: false) {} + #expect(coordinator.reproducesOnFreshConnection(placement)) + + try await coordinator.track {} + #expect(coordinator.reproducesOnFreshConnection(placement) == false) + } + // MARK: - Mapping @Test("Each phase maps to the shared display status, and a redial reads as reconnecting") @@ -226,7 +296,7 @@ struct ConnectionCoordinatorStatusTests { let second = driver(tables: ["orders"]) factory.queue(second) - _ = await coordinator.track(failed: { $0 }) { true } + _ = try await coordinator.track(failed: { $0 }) { true } #expect(coordinator.session?.driver === second) #expect(isConnected(coordinator)) @@ -373,7 +443,7 @@ struct ConnectionCoordinatorStatusTests { await coordinator.reconnectIfNeeded() #expect(coordinator.session?.driver === second) - let saved = await coordinator.track(failed: { _ in viewModel.operationError != nil }) { + let saved = try await coordinator.track(failed: { _ in viewModel.operationError != nil }) { await viewModel.saveChanges() } @@ -419,7 +489,7 @@ struct ConnectionCoordinatorStatusTests { let second = driver(tables: ["users"]) factory.queue(second) - let saved = await coordinator.track(failed: { _ in viewModel.operationError != nil }) { + let saved = try await coordinator.track(failed: { _ in viewModel.operationError != nil }) { await viewModel.saveChanges() } @@ -569,3 +639,8 @@ nonisolated enum ScriptedFailure: LocalizedError { } } } + +@MainActor +private final class QueryLog { + var events: [String] = [] +} diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStoreTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStoreTests.swift index ed43fbfd8f..57ff5120bd 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStoreTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStoreTests.swift @@ -1,8 +1,10 @@ import Foundation import SwiftUI +import TableProDatabase @testable import TableProMobile @testable import TableProModels import TableProSSHTransport +import TableProTestSupport import Testing @Suite("Connection redial detection") @@ -239,6 +241,32 @@ struct ConnectionCoordinatorStoreTests { #expect(store.coordinator(for: second, appState: appState) === secondCoordinator) } + @Test("A connection edited or deleted while an import holds its session drops the session once the import ends") + func dropWaitsForTheImport() async throws { + let factory = StubDriverFactory() + let state = fixture.makeState(syncEnabled: false, driverFactory: factory) + let store = ConnectionCoordinatorStore(connectionManager: state.connectionManager) + let original = connection("A") + let driver = MockDatabaseDriver() + driver.scriptedTransactionState = .idle + factory.queue(driver) + let coordinator = store.coordinator(for: original, appState: state) + await coordinator.connect() + let gate = TestGate() + let importRun = Task { + try await state.sessionTransactions.track(original.id, importing: true) { await gate.enter() } + } + await gate.waitUntilEntered() + + store.reconcile(from: [original], to: []) + try await Task.sleep(for: .milliseconds(50)) + #expect(state.connectionManager.session(for: original.id) != nil) + + await gate.open() + try await importRun.value + try await waitUntil { state.connectionManager.session(for: original.id) == nil } + } + @Test("A deleted connection keeps its screen up until the cover goes, and never reconnects") func deletionKeepsTheLastRecord() { let original = connection("A") diff --git a/apple/TableProMobile/TableProMobileTests/DataBrowserViewModelTests.swift b/apple/TableProMobile/TableProMobileTests/DataBrowserViewModelTests.swift index 0cd439b7fc..3e9659302c 100644 --- a/apple/TableProMobile/TableProMobileTests/DataBrowserViewModelTests.swift +++ b/apple/TableProMobile/TableProMobileTests/DataBrowserViewModelTests.swift @@ -87,6 +87,23 @@ struct DataBrowserViewModelTests { #expect(vm.window.shownValues.count == 1) } + @Test("A page load an import refused says so, reads nothing, and is marked to load again once the import ends") + func importRefusedLoadWaitsForTheImport() async { + let driver = MockDatabaseDriver() + let vm = DataBrowserViewModel() + vm.attach( + session: makeSession(driver: driver), table: TableInfo(name: "users"), databaseType: .mysql, + host: "localhost", trackRead: { _ in throw SessionHeldByImport() } + ) + + await vm.load(isInitial: true) + + #expect(vm.loadError?.message == SessionHeldByImport().appError.message) + #expect(vm.waitsForImport) + #expect(driver.executedQueries.isEmpty) + #expect(vm.isLoading == false) + } + @Test("A page cut short is not taken as the end of the table") func cutPageKeepsTheTotalOpen() async { let driver = MockDatabaseDriver() @@ -204,6 +221,29 @@ struct DataBrowserViewModelTests { #expect(vm.rowLocator(forRowAt: 1) == nil) } + @Test("Selected rows with a key are read again, except while the session holds a transaction, when the page's values go out") + func selectedRowsReadAgainOnlyOutsideATransaction() async throws { + let driver = MockDatabaseDriver() + driver.scriptedColumns = makeColumns() + driver.scriptedExecuteResults = [ + .success(QueryResult(columns: makeColumns(), rows: [["42", "Alice"]], rowsAffected: 0, executionTime: 0)), + .success(QueryResult(columns: [], rows: [["1"]], rowsAffected: 0, executionTime: 0)) + ] + let vm = DataBrowserViewModel() + vm.attach(session: makeSession(driver: driver), table: TableInfo(name: "users"), databaseType: .mysql, host: "localhost") + await vm.load(isInitial: true) + + guard case .query = try vm.selectedRowsExportSource([0]) else { + Issue.record("expected a read by key") + return + } + guard case .rows(let held) = try vm.selectedRowsExportSource([0], readsAgain: false) else { + Issue.record("expected the page's rows") + return + } + #expect(held.rows == [[.text("42"), .text("Alice")]]) + } + @Test("A row whose key holds binary data gives no key values to delete it by") func binaryKeyGivesNoDeleteValues() async { let columns = [ diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/FakePostgreSQLServer.swift b/apple/TableProMobile/TableProMobileTests/Drivers/FakePostgreSQLServer.swift index f17ab72448..2d14fae9bf 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/FakePostgreSQLServer.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/FakePostgreSQLServer.swift @@ -9,11 +9,20 @@ import os final class FakePostgreSQLServer: @unchecked Sendable { struct SetupFailure: Error {} + struct Column: Sendable { + let name: String + let oid: UInt32 + } + enum Reply: Sendable { // One text column `n`, its values `row 0`, `row 1` and on. case rows(Int) + // Columns of the given type oids, their values in text form; nil is NULL. + case typed([Column], rows: [[String?]]) // An ErrorResponse after `rowsFirst` rows. case failure(sqlState: String, message: String, rowsFirst: Int = 0) + // A `COPY ... FROM STDIN` that takes the client's data and completes with its line count. + case copyIn } struct Session: Sendable { @@ -21,6 +30,9 @@ final class FakePostgreSQLServer: @unchecked Sendable { // Every statement in the order it arrived, as a simple Query or a Parse. var queries: [String] = [] var extendedQueries: [String] = [] + // The CopyData a COPY received, and the reason of a CopyFail. + var copied: [UInt8] = [] + var copyFailure: String? var terminated = false } @@ -148,7 +160,23 @@ final class FakePostgreSQLServer: @unchecked Sendable { case UInt8(ascii: "Q"): let query = String(decoding: body.prefix { $0 != 0 }, as: UTF8.self) session.queries.append(query) + if case .copyIn? = server.replies[query] { + // CopyInResponse: text format, one column; ReadyForQuery waits for the copy's end. + return FakePostgreSQLServer.message("G", [0] + FakePostgreSQLServer.bytes16(1) + FakePostgreSQLServer.bytes16(0)) + } return run(query, server: server, describes: true) + FakePostgreSQLServer.message("Z", [status]) + case UInt8(ascii: "d"): + session.copied += body + return [] + case UInt8(ascii: "c"): + let lines = session.copied.filter { $0 == UInt8(ascii: "\n") }.count + return FakePostgreSQLServer.message("C", Array("COPY \(lines)".utf8) + [0]) + + FakePostgreSQLServer.message("Z", [status]) + case UInt8(ascii: "f"): + session.copyFailure = String(decoding: body.prefix { $0 != 0 }, as: UTF8.self) + if status == UInt8(ascii: "T") { status = UInt8(ascii: "E") } + return FakePostgreSQLServer.errorResponse("57014", "COPY from stdin failed: \(session.copyFailure ?? "")") + + FakePostgreSQLServer.message("Z", [status]) case UInt8(ascii: "P"): let fields = body.split(separator: 0, maxSplits: 2, omittingEmptySubsequences: false) parsed = fields.count > 1 ? String(decoding: fields[1], as: UTF8.self) : "" @@ -158,7 +186,9 @@ final class FakePostgreSQLServer: @unchecked Sendable { case UInt8(ascii: "B"): return skippingToSync ? [] : FakePostgreSQLServer.message("2", []) case UInt8(ascii: "D"): - return skippingToSync ? [] : FakePostgreSQLServer.rowDescription + guard !skippingToSync else { return [] } + guard case .typed(let columns, _)? = server.replies[parsed] else { return FakePostgreSQLServer.rowDescription } + return FakePostgreSQLServer.rowDescription(columns) case UInt8(ascii: "E"): guard !skippingToSync else { return [] } if case .failure? = server.replies[parsed] { skippingToSync = true } @@ -179,6 +209,11 @@ final class FakePostgreSQLServer: @unchecked Sendable { case .rows(let count)?: return (describes ? FakePostgreSQLServer.rowDescription : []) + FakePostgreSQLServer.dataRows(count) + FakePostgreSQLServer.message("C", Array("SELECT \(count)".utf8) + [0]) + case .typed(let columns, let rows)?: + return (describes ? FakePostgreSQLServer.rowDescription(columns) : []) + FakePostgreSQLServer.dataRows(rows) + + FakePostgreSQLServer.message("C", Array("SELECT \(rows.count)".utf8) + [0]) + case .copyIn?: + return FakePostgreSQLServer.message("C", Array("SELECT 0".utf8) + [0]) case .failure(let sqlState, let message, let rowsFirst)?: if status == UInt8(ascii: "T") { status = UInt8(ascii: "E") } let header = rowsFirst > 0 && describes ? FakePostgreSQLServer.rowDescription : [] @@ -223,6 +258,24 @@ final class FakePostgreSQLServer: @unchecked Sendable { ) } + private static func rowDescription(_ columns: [Column]) -> [UInt8] { + let fields = columns.flatMap { column in + Array(column.name.utf8) + [0] + bytes(UInt32(0)) + bytes16(0) + bytes(column.oid) + bytes16(0xFFFF) + + bytes(UInt32.max) + bytes16(0) + } + return message("T", bytes16(UInt16(columns.count)) + fields) + } + + private static func dataRows(_ rows: [[String?]]) -> [UInt8] { + rows.flatMap { row in + message("D", bytes16(UInt16(row.count)) + row.flatMap { value -> [UInt8] in + guard let value else { return bytes(UInt32.max) } + let text = Array(value.utf8) + return bytes(UInt32(text.count)) + text + }) + } + } + private static func dataRows(_ count: Int) -> [UInt8] { var rows: [UInt8] = [] for index in 0.. [String?] { - [name, column, table, referencedColumn, onDelete, onUpdate, schema, referencedSchema] - } - - @Test("The catalog read filters on the database it is asked for") - func readsTheRequestedDatabase() { - let query = MySQLDriver.foreignKeyCatalogQuery(table: "orders", database: "crm") - - #expect(query.contains("kcu.TABLE_SCHEMA = 'crm'")) - #expect(query.contains("kcu.TABLE_NAME = 'orders'")) - #expect(query.contains("kcu.REFERENCED_TABLE_SCHEMA")) - #expect(!query.contains("DATABASE()")) - } - - @Test("With no database the catalog read follows the session's current one") - func readsTheSessionDatabase() { - let query = MySQLDriver.foreignKeyCatalogQuery(table: "orders", database: nil) - - #expect(query.contains("kcu.TABLE_SCHEMA = DATABASE()")) - } - - @Test("Quotes and backslashes in the database and table names are escaped") - func escapesNames() { - let query = MySQLDriver.foreignKeyCatalogQuery(table: #"o'b\x"#, database: #"c'r\m"#) - - #expect(query.contains(#"kcu.TABLE_SCHEMA = 'c''r\\m'"#)) - #expect(query.contains(#"kcu.TABLE_NAME = 'o''b\\x'"#)) + onDelete: String = "RESTRICT", + onUpdate: String = "RESTRICT" + ) -> PluginForeignKeyInfo { + PluginForeignKeyInfo( + name: name, + column: column, + referencedTable: table, + referencedColumn: referencedColumn, + referencedSchema: referencedSchema, + onDelete: onDelete, + onUpdate: onUpdate + ) } @Test("Under NO_BACKSLASH_ESCAPES the driver writes a backslash as it is, and doubles it otherwise") @@ -59,54 +40,34 @@ struct MySQLForeignKeyCatalogTests { #expect(MySQLDriver.stringEscapes(for: .mysql, session: escaping, isDatabend: false) == .mysql) #expect(MySQLDriver.stringEscapes(for: .mysql, session: nil, isDatabend: false) == .mysql) #expect(MySQLDriver.stringEscapes(for: .databend, session: nil, isDatabend: true) == .backslash) - #expect( - MySQLDriver.foreignKeyCatalogQuery(table: #"o'b\x"#, database: nil, escapes: .standard) - .contains(#"kcu.TABLE_NAME = 'o''b\x'"#) - ) } @Test("A key into its own database has no schema, one into another database keeps it") func referencedSchemaOnlyWhenItDiffers() { - let keys = MySQLDriver.foreignKeys(fromCatalogRows: [ - row("fk_customer", "customer_id", "customers", "id"), - row("fk_region", "region_id", "regions", "id", referencedSchema: "geo"), - ]) - - #expect(keys.map(\.name) == ["fk_customer", "fk_region"]) - #expect(keys.map(\.referencedTable) == ["customers", "regions"]) - #expect(keys.map(\.referencedSchema) == [nil, "geo"]) - } - - @Test("A composite key keeps one name across its column pairs, in key order") - func compositeKeyKeepsItsName() { - let keys = MySQLDriver.foreignKeys(fromCatalogRows: [ - row("fk_user", "tid", "users", "tenant_id"), - row("fk_user", "uid", "users", "id"), - ]) - - #expect(keys.map(\.name) == ["fk_user", "fk_user"]) - #expect(keys.map(\.column) == ["tid", "uid"]) - #expect(keys.map(\.referencedColumn) == ["tenant_id", "id"]) + let keys = MySQLDriver.foreignKeys([ + key("fk_customer", "customer_id", "customers", "id"), + key("fk_region", "region_id", "regions", "id", referencedSchema: "geo"), + ], database: "shop") + let schemas = keys.map { $0.referencedSchema } + let tables = keys.map { $0.referencedTable } + + #expect(tables == ["customers", "regions"]) + #expect(schemas == [nil, "geo"]) } - @Test("Actions come from the catalog, and a missing one reads as NO ACTION") - func actions() { - let keys = MySQLDriver.foreignKeys(fromCatalogRows: [ - row("fk_a", "a", "p", "id", onDelete: "CASCADE", onUpdate: "SET NULL"), - row("fk_b", "b", "p", "id", onDelete: nil, onUpdate: nil), - ]) - - #expect(keys.map(\.onDelete) == ["CASCADE", "NO ACTION"]) - #expect(keys.map(\.onUpdate) == ["SET NULL", "NO ACTION"]) - } - - @Test("A row missing a name or a column is skipped") - func incompleteRowsAreSkipped() { - var nameless = row("fk_a", "a", "p", "id") - nameless[0] = nil - let short = Array(row("fk_b", "b", "p", "id").prefix(6)) - - #expect(MySQLDriver.foreignKeys(fromCatalogRows: [nameless, short]).isEmpty) + @Test("A composite key keeps one name across its column pairs, in key order, with its actions") + func compositeKeyKeepsItsNameAndActions() { + let keys = MySQLDriver.foreignKeys([ + key("fk_user", "tid", "users", "tenant_id", onDelete: "CASCADE", onUpdate: "SET NULL"), + key("fk_user", "uid", "users", "id", onDelete: "CASCADE", onUpdate: "SET NULL"), + ], database: "shop") + let names = keys.map { $0.name } + let columns = keys.map { $0.column } + let actions = keys.map { "\($0.onDelete) \($0.onUpdate)" } + + #expect(names == ["fk_user", "fk_user"]) + #expect(columns == ["tid", "uid"]) + #expect(actions == ["CASCADE SET NULL", "CASCADE SET NULL"]) } @Test("Indexes of a table in another database are read from that database") diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MySQLTransferTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MySQLTransferTests.swift new file mode 100644 index 0000000000..b7ad40d73c --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MySQLTransferTests.swift @@ -0,0 +1,151 @@ +import Foundation +@testable import TableProMobile +import TableProModels +import TableProMySQLCore +import TableProPluginKit +import TableProSQLGrammar +import Testing + +struct MySQLTransferTests { + private static let mysqlFeatures: SQLLexicalFeatures = [ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .hashLineComments, + .executableComments, .dashCommentsNeedWhitespace, + ] + + @Test("Text is inlined as hex behind the connect-time character set, bytes as a hex literal, NULL as NULL") + func inlinesAsHex() throws { + let sql = try MySQLInlineLiterals.bind( + "INSERT INTO t (a, b, c, d) VALUES (?, ?, ?, ?)", + parameters: [.text(#"it's a\b"#), .bytes(Data([0x00, 0xAB])), .null, .text("00123")], + characterSet: "utf8mb4", + features: Self.mysqlFeatures + ) + + #expect(sql == "INSERT INTO t (a, b, c, d) VALUES (_utf8mb4 X'6974277320615C62', X'00AB', NULL, _utf8mb4 X'3030313233')") + } + + // Under `SET NAMES gbk` the bytes e4 b9 97 5c read as two GBK characters, so a backslash escape let the next quote + // end the literal. In hex the value holds no quote, whatever the session reads. + @Test("A GBK escape payload stays inside its literal") + func gbkPayloadCannotBreakOut() throws { + let payload = #"乗\'), ((SELECT secret FROM secrets LIMIT 1)) #"# + let sql = try MySQLInlineLiterals.bind( + "INSERT INTO t (a) VALUES (?)", parameters: [.text(payload)], characterSet: "utf8mb4", features: Self.mysqlFeatures + ) + + let prefix = "INSERT INTO t (a) VALUES (_utf8mb4 X'" + #expect(sql.hasPrefix(prefix)) + #expect(sql.hasSuffix("')")) + let hex = sql.dropFirst(prefix.count).dropLast(2) + #expect(hex.allSatisfy { $0.isHexDigit }) + let bytes = stride(from: 0, to: hex.count, by: 2).compactMap { offset -> UInt8? in + let start = hex.index(hex.startIndex, offsetBy: offset) + return UInt8(hex[start.. String { + switch cell { + case .null: "null" + case .text(let value): "text \(value)" + case .truncatedText(let prefix, _, _): "truncated \(prefix)" + case .binary(let count, _): "binary \(count)" + } + } + + @Test("A statement's text result reads booleans as the Mac does and keeps a bytea's hex form") + func executeDecodesBooleans() async throws { + let server = try FakePostgreSQLServer(replies: [Self.report: .typed(Self.typedColumns, rows: Self.typedRows)]) + let run = try await Self.run(on: server) { driver in try await driver.execute(query: Self.report) } + + #expect(run.read.rows == [["true", "\\x01ff", nil], ["false", "\\x", "x"]]) + #expect(run.read.columns.map(\.typeName) == ["boolean", "bytea", "text"]) + } + + @Test("A catalog read keeps every row, past the 100,000 the text result keeps, under the pinned search path") + func catalogQueryHasNoCap() async throws { + let statement = "SELECT n FROM pg_catalog.pg_class" + let sent = PostgreSQLCatalogSearchPath.pinned + statement + let server = try FakePostgreSQLServer(replies: [sent: .rows(100_001)]) + let run = try await Self.run(on: server) { driver in try await driver.catalogQuery(statement) } + + #expect(run.read.rows.count == 100_001) + #expect(!run.read.isTruncated) + #expect(run.served.queries.contains(sent)) + } + + @Test("A schema-relative read puts that schema on the path") + func relativeCatalogRead() async throws { + let statement = "SELECT pg_catalog.format_type(1, 2)" + let sent = PostgreSQLCatalogSearchPath.relative(to: "sales") + statement + let server = try FakePostgreSQLServer(replies: [sent: .rows(1)]) + let run = try await Self.run(on: server) { driver in + try await driver.catalogCells(statement, namesRelativeTo: "sales") + } + + #expect(run.read.rows == [[.text("row 0")]]) + #expect(run.served.queries.contains(sent)) + } + + @Test("A catalog read with parameters sets the path on its own inside a block it opens, then binds") + func parameterizedCatalogRead() async throws { + let statement = "SELECT n FROM pg_catalog.pg_class WHERE relname = $1" + let server = try FakePostgreSQLServer(replies: [statement: .rows(1)]) + let run = try await Self.run(on: server) { driver in try await driver.catalogQuery(statement, parameters: ["t"]) } + + #expect(run.read.rows.count == 1) + #expect(Array(run.served.queries.suffix(5)) == [ + "BEGIN", PostgreSQLCatalogSearchPath.pinned, statement, "ROLLBACK", "SELECT 2", + ]) + #expect(run.served.extendedQueries == [statement]) + } + + @Test("A Redshift catalog read runs as it is, as on the Mac") + func redshiftCatalogReadIsNotPinned() async throws { + let statement = "SELECT n FROM pg_catalog.pg_class" + let server = try FakePostgreSQLServer(replies: [statement: .rows(2)]) + let run = try await Self.run(on: server, databaseType: .redshift) { driver in + try await driver.catalogQuery(statement) + } + + #expect(run.read.rows.count == 2) + #expect(!run.served.queries.contains { $0.hasPrefix("SET LOCAL search_path") }) + } + + @Test("Bound values go through the extended protocol, one statement") + func executeParameterizedBinds() async throws { + let insert = "INSERT INTO \"public\".\"t\" (\"a\", \"b\") VALUES ($1, $2)" + let server = try FakePostgreSQLServer() + let run = try await Self.run(on: server) { driver in + try await driver.executeParameterized(query: insert, parameters: [.text("x"), .bytes(Data([1]))], typeNames: nil) + } + + #expect(run.read.rowsAffected == 0) + #expect(run.served.extendedQueries == [insert]) + } + + @Test("COPY FROM STDIN sends every piece as CopyData, then ends the copy") + func copySendsEveryPiece() async throws { + let copy = "COPY public.t (a, b) FROM stdin" + let server = try FakePostgreSQLServer(replies: [copy: .copyIn]) + let pieces = CopyPieces(["1\talpha\n", "2\tbe", "ta\n"]) + let run = try await Self.run(on: server) { driver in + try await driver.copyFromStandardInput(statement: copy, rows: PluginCopyData(read: { try pieces.next() })) + } + + #expect(run.read.rowsAffected == 2) + #expect(String(decoding: run.served.copied, as: UTF8.self) == "1\talpha\n2\tbeta\n") + #expect(run.served.copyFailure == nil) + } + + @Test("A source that fails ends the copy with CopyFail and its own error") + func copySourceFailureFailsTheCopy() async throws { + let copy = "COPY public.t (a) FROM stdin" + let server = try FakePostgreSQLServer(replies: [copy: .copyIn]) + let run = try await Self.run(on: server) { driver in + await #expect(throws: CopySourceFailure.self) { + _ = try await driver.copyFromStandardInput( + statement: copy, rows: PluginCopyData(read: { throw CopySourceFailure() }) + ) + } + } + + #expect(run.served.copyFailure == LibPQCopyIn.sourceFailedReason) + } + + @Test("Redshift refuses COPY FROM STDIN before anything is sent, as on the Mac") + func redshiftRefusesCopy() async throws { + let driver = PostgreSQLDriver( + host: "127.0.0.1", port: 1, user: "app", password: "", database: "app", databaseType: .redshift + ) + await #expect(throws: PluginDriverUnsupportedOperation.copyFromStandardInput) { + _ = try await driver.copyFromStandardInput(statement: "COPY t FROM stdin", rows: PluginCopyData(read: { nil })) + } + } + + @Test("Foreign key checks switch through session_replication_role on PostgreSQL and not on Redshift") + func foreignKeyCheckStatements() { + let postgres = PostgreSQLDriver(host: "h", port: 5_432, user: "u", password: "", database: "d") + let redshift = PostgreSQLDriver(host: "h", port: 5_439, user: "u", password: "", database: "d", databaseType: .redshift) + + #expect(postgres.foreignKeyCheckStatements(enabled: false) == PostgreSQLForeignKeyChecks.disableStatements) + #expect(postgres.foreignKeyCheckStatements(enabled: true) == PostgreSQLForeignKeyChecks.enableStatements) + #expect(redshift.foreignKeyCheckStatements(enabled: false).isEmpty) + } + + // MARK: - Helpers + + private struct CopySourceFailure: Error {} + + private final class CopyPieces: Sendable { + private let pieces: OSAllocatedUnfairLock<[Data]> + + init(_ pieces: [String]) { + self.pieces = OSAllocatedUnfairLock(initialState: pieces.map { Data($0.utf8) }) + } + + func next() throws -> Data? { + pieces.withLock { $0.isEmpty ? nil : $0.removeFirst() } + } + } + + private struct Collected { + var columns: [String] = [] + var typeNames: [String] = [] + var rows: [[PluginCellValue]] = [] + var batchSizes: [Int] = [] + } + + private struct Run { + let read: Value + let served: FakePostgreSQLServer.Session + } + + private static func collect(_ stream: AsyncThrowingStream) async throws -> Collected { + var collected = Collected() + for try await element in stream { + switch element { + case .header(let header): + collected.columns = header.columns + collected.typeNames = header.columnTypeNames + case .rows(let rows): + collected.rows += rows + collected.batchSizes.append(rows.count) + } + } + return collected + } + + // Connects, runs `body`, sends `SELECT 2` so the server shows what came after it, and disconnects. + private static func run( + on server: FakePostgreSQLServer, + databaseType: DatabaseType = .postgresql, + _ body: (PostgreSQLDriver) async throws -> Value + ) async throws -> Run { + defer { server.close() } + async let session = server.serveOneSession() + let driver = PostgreSQLDriver( + host: "127.0.0.1", port: server.port, user: "app", password: "app-password", database: "app", + databaseType: databaseType + ) + try await driver.connect() + let read = try await body(driver) + _ = try await driver.execute(query: "SELECT 2") + try await driver.disconnect() + return Run(read: read, served: await session) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/RedisExportRowsTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/RedisExportRowsTests.swift new file mode 100644 index 0000000000..e9257b842d --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Drivers/RedisExportRowsTests.swift @@ -0,0 +1,80 @@ +import Foundation +@testable import TableProMobile +import TableProPluginKit +import Testing + +struct RedisExportRowsTests { + // Answers by the command's first two words; anything unscripted is an error reply. + private struct Server: Sendable { + let replies: [String: RedisReplyValue] + let bytes: [String: Data] + + var send: RedisExportRows.Send { + let replies = replies + return { arguments in replies[arguments.prefix(2).joined(separator: " ")] ?? .error("NOPERM") } + } + + var readBytes: RedisExportRows.ReadBytes { + let bytes = bytes + return { arguments in bytes[arguments.prefix(2).joined(separator: " ")] } + } + } + + @Test("The export statement names the database it reads, and anything else runs as a command") + func readsTheRequest() { + #expect(RedisExportRows.request(for: "KEYBROWSE DB 3") == .keyBrowse(database: 3)) + #expect(RedisExportRows.request(for: "keybrowse") == .keyBrowse(database: nil)) + #expect(RedisExportRows.request(for: "GET a") == .command) + } + + @Test("A string key reads whole: UTF-8 as text, anything else as its bytes, with its type, TTL and length") + func stringKeys() async throws { + let server = Server( + replies: [ + "TYPE text": .status("string"), "TTL text": .integer(-1), "STRLEN text": .integer(5), + "TYPE raw": .status("string"), "TTL raw": .integer(30), "STRLEN raw": .integer(2), + ], + bytes: ["GET text": Data("héllo".utf8), "GET raw": Data([0xFF, 0x00])] + ) + + let rows = try await RedisExportRows.rows(keys: ["text", "raw"], send: server.send, readBytes: server.readBytes) + + #expect(rows == [ + [.text("text"), .text("STRING"), .text("-1"), .text("5"), .text("héllo")], + [.text("raw"), .text("STRING"), .text("30"), .text("2"), .bytes(Data([0xFF, 0x00]))], + ]) + } + + @Test("A collection's value is the Mac's JSON preview of it") + func collectionPreviews() async throws { + let server = Server( + replies: [ + "TYPE h": .status("hash"), "TTL h": .integer(-1), "HLEN h": .integer(2), + "HSCAN h": .array([.string("0"), .array([.string("b"), .string("2"), .string("a"), .string("1")])]), + "TYPE l": .status("list"), "TTL l": .integer(-1), "LLEN l": .integer(2), + "LRANGE l": .array([.string("x"), .string("y")]), + ], + bytes: [:] + ) + + let rows = try await RedisExportRows.rows(keys: ["h", "l"], send: server.send, readBytes: server.readBytes) + + #expect(rows[0][4] == .text(#"{"a":"1","b":"2"}"#)) + #expect(rows[1][4] == .text(#"["x","y"]"#)) + } + + @Test("A key the server will not describe keeps its name and leaves the other cells empty") + func refusedKey() async throws { + let server = Server(replies: [:], bytes: [:]) + + let rows = try await RedisExportRows.rows(keys: ["secret"], send: server.send, readBytes: server.readBytes) + + #expect(rows == [[.text("secret"), .null, .null, .null, .null]]) + } + + @Test("The columns are the Mac's KEYBROWSE columns, in its order") + func columns() { + #expect(RedisExportRows.columns == ["Key", "Type", "TTL", "Length", "Value"]) + #expect(RedisExportRows.columnTypeNames == ["String", "RedisType", "RedisInt", "Int64", "RedisRaw"]) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/ClipboardExporterFidelityTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/ClipboardExporterFidelityTests.swift index 0f68fa77de..3fcd09c052 100644 --- a/apple/TableProMobile/TableProMobileTests/Helpers/ClipboardExporterFidelityTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Helpers/ClipboardExporterFidelityTests.swift @@ -94,7 +94,9 @@ struct ClipboardExporterFidelityTests { } } - @Test("each column's declared type picks the JSON type") + // The macOS JSON export's rule (D6): a boolean column folds only true and false, `bit`, an unsigned ClickHouse type + // and a column with no type stay strings. The PostgreSQL driver now sends true and false (D7). + @Test("each column's declared type picks the JSON type the macOS export writes") func declaredTypePicksTheJSONType() throws { func value(_ text: String, typeName: String) throws -> String { let output = ClipboardExporter.exportRow( @@ -105,24 +107,24 @@ struct ClipboardExporterFidelityTests { let line = try #require(output.split(separator: "\n").first { $0.contains("\"v\"") }) return String(line.drop { $0 != ":" }.dropFirst(2)) } - #expect(try value("t", typeName: "bool") == "true") - #expect(try value("f", typeName: "bool") == "false") + #expect(try value("true", typeName: "bool") == "true") + #expect(try value("false", typeName: "bool") == "false") + #expect(try value("t", typeName: "bool") == "\"t\"") #expect(try value("TRUE", typeName: "BOOLEAN") == "true") #expect(try value("false", typeName: "Nullable(Bool)") == "false") #expect(try value("yes", typeName: "bool") == "\"yes\"") #expect(try value("42", typeName: "int4") == "42") #expect(try value("12.50", typeName: "numeric(10,2)") == "12.50") #expect(try value("-1.5e3", typeName: "Float64") == "-1.5e3") - #expect(try value("18446744073709551615", typeName: "UInt64") == "18446744073709551615") - #expect(try value("1", typeName: "bit") == "1") + #expect(try value("18446744073709551615", typeName: "UInt64") == "\"18446744073709551615\"") + #expect(try value("1", typeName: "bit") == "\"1\"") #expect(try value("NaN", typeName: "float8") == "\"NaN\"") #expect(try value("$5.00", typeName: "money") == "\"$5.00\"") #expect(try value("t", typeName: "text") == "\"t\"") - #expect(try value("1", typeName: "BOOLEAN") == "true") - #expect(try value("0", typeName: "BOOLEAN") == "false") - // SQLite reports no type for an expression such as count(*), so its value decides. - #expect(try value("42", typeName: "") == "42") - #expect(try value("true", typeName: "") == "true") + #expect(try value("1", typeName: "BOOLEAN") == "\"1\"") + #expect(try value("0", typeName: "BOOLEAN") == "\"0\"") + #expect(try value("42", typeName: "") == "\"42\"") + #expect(try value("true", typeName: "") == "\"true\"") #expect(try value("01234", typeName: "") == "\"01234\"") } @@ -263,6 +265,50 @@ struct ClipboardExporterFidelityTests { #expect(MySQLDriver.keptTextEscapes(for: .mysql, banner: nil) == .mysql) #expect(MySQLDriver.keptTextEscapes(for: .mysql, banner: "8.0.90-v1.2.725-nightly") == .backslash) } + // D2: MySQL stores a quoted '5' in a BIT column as the byte 53. + @Test("A MySQL BIT value is copied bare, and every other value stays quoted") + func mysqlBitIsBare() { + let bits = [ + ColumnInfo(name: "id", typeName: "INT", ordinalPosition: 0), + ColumnInfo(name: "flags", typeName: "BIT(8)", ordinalPosition: 1), + ColumnInfo(name: "on", typeName: "BIT", ordinalPosition: 2) + ] + let text = ClipboardExporter.exportRow( + columns: bits, row: ["1", "5", "1"], format: .sqlInsert, tableName: "t", databaseType: .mysql, driver: nil + ) + #expect(text == "INSERT INTO `t` (`id`, `flags`, `on`) VALUES ('1', 5, 1);") + let postgres = ClipboardExporter.exportRow( + columns: bits, row: ["1", "101", "1"], format: .sqlInsert, tableName: "t", databaseType: .postgresql, driver: nil + ) + #expect(postgres.contains("'101'")) + } + + // D3: the server refuses a value for a generated or identity column. + @Test("A column the server fills in is left out of the INSERT, and a row of only those writes its defaults") + func serverOwnedColumnsAreLeftOut() { + let shown = [ + ColumnInfo(name: "id", typeName: "int4", ordinalPosition: 0), + ColumnInfo(name: "qty", typeName: "int4", ordinalPosition: 1), + ColumnInfo(name: "total", typeName: "numeric", ordinalPosition: 2) + ] + let catalog = [ + ColumnInfo(name: "id", typeName: "int4", isPrimaryKey: true, ordinalPosition: 0, rejectsWrittenValues: true), + ColumnInfo(name: "qty", typeName: "int4", ordinalPosition: 1), + ColumnInfo(name: "total", typeName: "numeric", ordinalPosition: 2, isGenerated: true) + ] + let text = ClipboardExporter.exportRow( + columns: shown, row: ["1", "3", "9.00"], format: .sqlInsert, tableName: "orders", + databaseType: .postgresql, driver: nil, columnDetails: catalog + ) + #expect(text == "INSERT INTO \"orders\" (\"qty\") VALUES ('3');") + + let onlyOwned = ClipboardExporter.exportRow( + columns: [shown[0], shown[2]], row: ["1", "9.00"], format: .sqlInsert, tableName: "orders", + databaseType: .postgresql, driver: nil, columnDetails: catalog + ) + #expect(onlyOwned == "INSERT INTO \"orders\" DEFAULT VALUES;") + } + @Test("SQL Server copy as INSERT keeps Unicode and backslash line endings without a driver") func sqlServerLiteral() { #expect(insert(["1", "日本\\\n", nil], .mssql) diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/RowInsertPlannerTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/RowInsertPlannerTests.swift index 855e0d68f9..144819bb5f 100644 --- a/apple/TableProMobile/TableProMobileTests/Helpers/RowInsertPlannerTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Helpers/RowInsertPlannerTests.swift @@ -249,13 +249,13 @@ struct RowInsertPlannerTests { #expect(statements == [#"INSERT INTO "people" DEFAULT VALUES"#]) } - @Test("an all-default row produces nothing on Oracle, which has no such statement") + @Test("an all-default row on Oracle names its first writable column, as the Mac writes it") func allDefaultsOnOracle() throws { let statements = try RowInsertPlanner.statements( table: "people", schema: nil, type: .oracle, driver: ansiDriver(), columns: defaultsColumns, rows: [PayloadRow(values: [:])], allowAllDefaults: true ) - #expect(statements.isEmpty) + #expect(statements == [#"INSERT INTO "people" ("id") VALUES (DEFAULT)"#]) } @Test("a caller that distinguishes omission writes an explicitly empty primary key") diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/SQLBuilderDefaultValuesTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/SQLBuilderDefaultValuesTests.swift index a71ba93091..b1b885b580 100644 --- a/apple/TableProMobile/TableProMobileTests/Helpers/SQLBuilderDefaultValuesTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Helpers/SQLBuilderDefaultValuesTests.swift @@ -44,10 +44,14 @@ struct SQLBuilderDefaultValuesTests { #expect(SQLBuilder.supportsAllDefaultsInsert(type)) } - @Test("Oracle has no all-defaults statement") - func oracleHasNoForm() { - #expect(SQLBuilder.buildAllDefaultsInsert(qualifiedTable: "\"t\"", for: .oracle) == nil) - #expect(!SQLBuilder.supportsAllDefaultsInsert(.oracle)) + // D12: the macOS spelling through the shared `AllDefaultsInsert`; it needs a column to name, so none without one. + @Test("Oracle names its first writable column with VALUES (DEFAULT)") + func oracleNamesAColumn() { + #expect(SQLBuilder.buildAllDefaultsInsert(qualifiedTable: "\"T\"", for: .oracle, firstWritableColumn: "ID") + == "INSERT INTO \"T\" (\"ID\") VALUES (DEFAULT)") + #expect(SQLBuilder.buildAllDefaultsInsert(qualifiedTable: "\"T\"", for: .oracle) == nil) + #expect(SQLBuilder.supportsAllDefaultsInsert(.oracle)) + #expect(SQLBuilder.buildAllDefaultsInsert(qualifiedTable: "t", for: .clickhouse, firstWritableColumn: "id") == nil) } @Test("an empty column list routes through the all-defaults form") @@ -59,5 +63,8 @@ struct SQLBuilderDefaultValuesTests { #expect(SQLBuilder.buildInsert( table: "people", schema: nil, type: .oracle, driver: driver, columns: [], values: [] ) == nil) + #expect(SQLBuilder.buildInsert( + table: "PEOPLE", schema: nil, type: .oracle, driver: driver, columns: [], values: [], firstWritableColumn: "ID" + ) == "INSERT INTO \"PEOPLE\" (\"ID\") VALUES (DEFAULT)") } } diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/SQLBuilderExportSelectTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/SQLBuilderExportSelectTests.swift new file mode 100644 index 0000000000..d0836657c6 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Helpers/SQLBuilderExportSelectTests.swift @@ -0,0 +1,55 @@ +import Foundation +import TableProModels +import TableProQuery +import Testing +@testable import TableProMobile + +struct SQLBuilderExportSelectTests { + private let idFilter = [TableFilter(columnName: "id", filterOperator: .equal, value: "5")] + private let sorted = SortState(columns: [SortColumn(name: "name", ascending: false)]) + + @Test("A table export reads every row, with no page limit on any engine") + func exportSelectHasNoPagination() { + for type in [DatabaseType.postgresql, .mysql, .mssql, .oracle, .sqlite, .clickhouse] { + let sql = SQLBuilder.buildExportSelect(table: "orders", schema: nil, type: type) + #expect(!sql.contains("LIMIT"), "\(type.rawValue)") + #expect(!sql.contains("FETCH NEXT"), "\(type.rawValue)") + #expect(!sql.contains("OFFSET"), "\(type.rawValue)") + } + #expect(SQLBuilder.buildExportSelect(table: "orders", schema: "public", type: .postgresql) + == "SELECT * FROM \"public\".\"orders\"") + } + + @Test("The browser's filter and sort carry over, as the export of a filtered view") + func exportSelectKeepsFilterAndSort() { + #expect(SQLBuilder.buildExportSelect( + table: "orders", schema: nil, type: .mssql, filters: idFilter, sortState: sorted + ) == "SELECT * FROM [orders] WHERE ([id] = 5) ORDER BY [name] DESC") + } + + @Test("A search narrows the export the way it narrows the page") + func exportSelectKeepsSearch() { + let columns = [ColumnInfo(name: "name", typeName: "text", ordinalPosition: 0)] + let sql = SQLBuilder.buildExportSelect( + table: "people", schema: nil, type: .postgresql, searchText: "ada", searchColumns: columns + ) + #expect(sql.hasPrefix("SELECT * FROM \"people\" WHERE (CAST(\"name\" AS TEXT) ILIKE '%ada%'")) + } + + @Test("Selected rows are read again by each one's whole key") + func rowsSelectMatchesEveryKey() { + let columns = [ + ColumnInfo(name: "tenant", typeName: "integer", isPrimaryKey: true, ordinalPosition: 0), + ColumnInfo(name: "id", typeName: "integer", isPrimaryKey: true, ordinalPosition: 1) + ] + let sql = SQLBuilder.buildRowsSelect( + table: "orders", schema: "public", type: .postgresql, + keys: [[(column: "tenant", value: "1"), (column: "id", value: "7")], [(column: "tenant", value: "2"), (column: "id", value: "7")]], + columns: columns, driver: MockDatabaseDriver() + ) + #expect(sql == "SELECT * FROM \"public\".\"orders\" WHERE (\"tenant\" = 1 AND \"id\" = 7) OR (\"tenant\" = 2 AND \"id\" = 7)") + #expect(SQLBuilder.buildRowsSelect( + table: "orders", schema: nil, type: .postgresql, keys: [], columns: columns, driver: MockDatabaseDriver() + ) == "SELECT * FROM \"orders\" WHERE 1 = 0") + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/SessionLocalStatementsTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/SessionLocalStatementsTests.swift index 6936bedadb..5daccdc59a 100644 --- a/apple/TableProMobile/TableProMobileTests/Helpers/SessionLocalStatementsTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Helpers/SessionLocalStatementsTests.swift @@ -4,6 +4,22 @@ import TableProModels import Testing struct SessionLocalStatementsTests { + @Test("A write, a SET, a USE, a temporary table or a transaction statement changes the session; a read does not") + func changesSession() { + for statement in [ + "SET @v = 7", "USE shop", "CREATE TEMPORARY TABLE scratch (id int)", "BEGIN", "COMMIT", + "INSERT INTO t VALUES (1)", "CALL refresh()" + ] { + #expect(SessionLocalStatements.changesSession([statement], databaseType: .mysql), "\(statement)") + } + #expect(SessionLocalStatements.changesSession(["SELECT @v", "SELECT * FROM t"], databaseType: .mysql) == false) + #expect(SessionLocalStatements.changesSession(["SELECT @v := 7"], databaseType: .mysql)) + #expect(SessionLocalStatements.changesSession( + ["SELECT set_config('search_path', 'app', false)"], databaseType: .postgresql + )) + #expect(SessionLocalStatements.changesSession(["SELECT ':=' AS text"], databaseType: .mysql) == false) + } + @Test("A temporary table a statement creates is noted, by its unquoted lowercased name") func createdTables() { let mysql = SessionLocalStatements.effect( diff --git a/apple/TableProMobile/TableProMobileTests/Onboarding/SceneIntentTests.swift b/apple/TableProMobile/TableProMobileTests/Onboarding/SceneIntentTests.swift index 6be1f71dc7..be38c570f3 100644 --- a/apple/TableProMobile/TableProMobileTests/Onboarding/SceneIntentTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Onboarding/SceneIntentTests.swift @@ -58,4 +58,25 @@ struct SceneIntentTests { ) #expect(SceneIntent.parse(activityType: "com.example.other", userInfo: ["connectionId": id]) == nil) } + + @Test( + "Every file an import plugin reads goes to the data import", + arguments: ["csv", "tsv", "json", "jsonl", "ndjson", "xlsx", "sql", "gz"] + ) + func dataFiles(fileExtension: String) { + let file = URL(fileURLWithPath: "/tmp/orders.\(fileExtension)") + #expect(SceneIntent.parse(url: file) == .importData(file)) + } + + @Test("A data file's extension is matched in any case") + func dataFileExtensionCase() { + let file = URL(fileURLWithPath: "/tmp/Orders.CSV") + #expect(SceneIntent.parse(url: file) == .importData(file)) + } + + @Test("A file no import plugin reads is ignored", arguments: ["numbers", "txt", "parquet", ""]) + func unknownDataFiles(fileExtension: String) { + let name = fileExtension.isEmpty ? "orders" : "orders.\(fileExtension)" + #expect(SceneIntent.parse(url: URL(fileURLWithPath: "/tmp/\(name)")) == nil) + } } diff --git a/apple/TableProMobile/TableProMobileTests/Onboarding/ScenePresenterTests.swift b/apple/TableProMobile/TableProMobileTests/Onboarding/ScenePresenterTests.swift index d78b220b26..b40c677179 100644 --- a/apple/TableProMobile/TableProMobileTests/Onboarding/ScenePresenterTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Onboarding/ScenePresenterTests.swift @@ -248,6 +248,70 @@ struct ScenePresenterTests { #expect(presenter.takeDeliverableIntent(isLocked: false, isLibraryWritable: true) != nil) } + @Test("An opened data file waits for a sheet like a link, and needs no writable library") + func dataImportWaitsForSheetOnly() { + let presenter = ScenePresenter() + let file = URL(fileURLWithPath: "/tmp/orders.csv") + presenter.present(.settings) + presenter.receive(.importData(file)) + + #expect(presenter.takeDeliverableIntent(isLocked: false, isLibraryWritable: false) == nil) + + presenter.sheet = nil + #expect(presenter.takeDeliverableIntent(isLocked: false, isLibraryWritable: false) == .importData(file)) + #expect(presenter.holdsConnectionRestore == false) + } + + @Test("An opened data file waits while an editor holds unsaved changes") + func dataImportWaitsForEditor() { + let presenter = ScenePresenter() + let editor = UUID() + presenter.setEditorHold(editor, isHolding: true) + presenter.receive(.importData(URL(fileURLWithPath: "/tmp/orders.csv"))) + + #expect(presenter.takeDeliverableIntent(isLocked: false, isLibraryWritable: true) == nil) + + presenter.setEditorHold(editor, isHolding: false) + #expect(presenter.takeDeliverableIntent(isLocked: false, isLibraryWritable: true) != nil) + } + + @Test("A file handed to a connection is taken once, and only by that connection") + func dataImportTakenByItsConnection() { + let presenter = ScenePresenter() + let file = URL(fileURLWithPath: "/tmp/orders.csv") + presenter.requestDataImport(file, in: connectionId) + + #expect(presenter.takeDataImport(for: UUID()) == nil) + #expect(presenter.takeDataImport(for: connectionId) == file) + #expect(presenter.takeDataImport(for: connectionId) == nil) + } + + @Test("A file waiting for a connection that is no longer on screen is dropped, and one for the next is kept") + func dataImportDiscardedWithItsConnection() { + let presenter = ScenePresenter() + let file = URL(fileURLWithPath: "/tmp/orders.csv") + let next = UUID() + + presenter.requestDataImport(file, in: next) + presenter.discardDataImport(unlessFor: next) + #expect(presenter.pendingDataImport == PendingDataImport(connectionId: next, fileURL: file)) + + presenter.discardDataImport(unlessFor: nil) + #expect(presenter.pendingDataImport == nil) + } + + @Test("Moving a file to another connection opens that connection the way a link does") + func dataImportMovesToAnotherConnection() { + let presenter = ScenePresenter() + let file = URL(fileURLWithPath: "/tmp/orders.csv") + let other = UUID() + + presenter.moveDataImport(file, to: other) + #expect(presenter.restoreHolds == [.pendingIntent]) + #expect(presenter.takeDeliverableIntent(isLocked: false, isLibraryWritable: true) == .openConnection(other, table: nil)) + #expect(presenter.takeDataImport(for: other) == file) + } + @Test("An editor that goes away lets go of its hold", .timeLimit(.minutes(1))) func releasedEditorLetsGo() async { let presenter = ScenePresenter() diff --git a/apple/TableProMobile/TableProMobileTests/Platform/SessionTransactionMonitorTests.swift b/apple/TableProMobile/TableProMobileTests/Platform/SessionTransactionMonitorTests.swift index be0bde871b..893aa30a32 100644 --- a/apple/TableProMobile/TableProMobileTests/Platform/SessionTransactionMonitorTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Platform/SessionTransactionMonitorTests.swift @@ -37,7 +37,7 @@ struct SessionTransactionMonitorTests { let log = EventLog() let gate = TestGate() let write = Task { - await monitor.track(id) { + try? await monitor.track(id) { await gate.enter() log.events.append("write") } @@ -57,12 +57,86 @@ struct SessionTransactionMonitorTests { #expect(log.events == ["waiting", "write", "run"]) } + @Test("An import holding the session refuses every other statement at once, reads and a second import included") + func importRefusesEveryOtherStatement() async throws { + attach() + let gate = TestGate() + let importRun = Task { + try await monitor.track(id, importing: true) { await gate.enter() } + } + await gate.waitUntilEntered() + #expect(monitor.isHeldByImport(id)) + #expect(monitor.isBusy(id)) + + await #expect(throws: SessionHeldByImport.self) { try await monitor.track(id) {} } + await #expect(throws: SessionHeldByImport.self) { try await monitor.track(id, reading: true) {} } + await #expect(throws: SessionHeldByImport.self) { try await monitor.track(id, importing: true) {} } + + await gate.open() + try await importRun.value + #expect(monitor.isHeldByImport(id) == false) + try await monitor.track(id, reading: true) {} + } + + @Test("An import is refused at once while other work runs on the session or the session is moving") + func importRefusedOnABusyOrMovingSession() async throws { + attach() + let gate = TestGate() + let write = Task { try? await monitor.track(id) { await gate.enter() } } + await gate.waitUntilEntered() + + await #expect(throws: SessionBusyForImport.self) { try await monitor.track(id, importing: true) {} } + #expect(monitor.isHeldByImport(id) == false) + + await gate.open() + await write.value + await #expect(throws: SessionBusyForImport.self) { + try await monitor.track(id, importing: true, sessionMoving: true) {} + } + try await monitor.track(id, importing: true) {} + } + + // Round 4 deadlocked here: a delete's reload is tracked inside the delete's own tracked work. + @Test("Work nested inside tracked work runs, and an import asked meanwhile is refused instead of waiting") + func nestedWorkRunsAndAnImportIsRefused() async throws { + attach() + let log = EventLog() + try await monitor.track(id) { + log.events.append("delete") + do { + try await monitor.track(id, importing: true) { log.events.append("import") } + } catch { + log.events.append("import refused") + } + try? await monitor.track(id, reading: true) { log.events.append("reload") } + } + + #expect(log.events == ["delete", "import refused", "reload"]) + } + + @Test("Work held until an import ends runs once it ends, or at once when none holds the session") + func afterImportRunsOnceTheImportEnds() async throws { + attach() + let log = EventLog() + monitor.afterImport(id) { log.events.append("now") } + let gate = TestGate() + let importRun = Task { try await monitor.track(id, importing: true) { await gate.enter() } } + await gate.waitUntilEntered() + monitor.afterImport(id) { log.events.append("after") } + #expect(log.events == ["now"]) + + await gate.open() + try await importRun.value + + #expect(log.events == ["now", "after"]) + } + @Test("Stop ends the wait for a write that is still running") func stopEndsTheWait() async throws { attach() let log = EventLog() let gate = TestGate() - let write = Task { await monitor.track(id) { await gate.enter() } } + let write = Task { try? await monitor.track(id) { await gate.enter() } } await gate.waitUntilEntered() let run = Task { log.events.append("waiting") @@ -82,29 +156,29 @@ struct SessionTransactionMonitorTests { } @Test("A run that leaves a transaction open shows it, explicit or implicit") - func runLeavesTheTransactionOpen() async { + func runLeavesTheTransactionOpen() async throws { let driver = attach(.explicitTransaction) - await monitor.track(id) {} + try await monitor.track(id) {} #expect(monitor.hold(for: id) == .open) driver.scriptedTransactionState = .implicitTransaction - await monitor.track(id) {} + try await monitor.track(id) {} #expect(monitor.hold(for: id) == .open) } @Test("A typed COMMIT clears it on the next settle") - func typedCommitClears() async { + func typedCommitClears() async throws { let driver = attach(.explicitTransaction) - await monitor.track(id) {} + try await monitor.track(id) {} driver.scriptedTransactionState = .idle - await monitor.track(id) {} + try await monitor.track(id) {} #expect(monitor.hold(for: id) == nil) } @Test("Work that throws still refreshes") - func throwingWorkRefreshes() async { + func throwingWorkRefreshes() async throws { attach(.abortedTransaction) await #expect(throws: MockDatabaseDriver.MockError.self) { @@ -172,7 +246,7 @@ struct SessionTransactionMonitorTests { } @Test("A window on a replaced session sees no hold and cannot act on the newer session's transaction") - func staleWindowCannotActOnTheNewerSession() async { + func staleWindowCannotActOnTheNewerSession() async throws { let old = attach(.idle) let newer = attach(.explicitTransaction) await monitor.refresh(id) @@ -194,7 +268,7 @@ struct SessionTransactionMonitorTests { await monitor.refresh(id) let token = try #require(monitor.token(for: id, on: driver)) - await monitor.track(id) {} + try await monitor.track(id) {} await #expect(throws: TransactionActionError.transactionChanged) { try await monitor.rollBack(id, as: token) @@ -214,7 +288,7 @@ struct SessionTransactionMonitorTests { let commit = Task { try await monitor.commit(id, as: token) } await gate.waitUntilEntered() driver.beforeTransactionState = nil - await monitor.track(id) {} + try await monitor.track(id) {} await gate.open() await #expect(throws: TransactionActionError.transactionChanged) { @@ -292,7 +366,7 @@ struct SessionTransactionMonitorTests { await monitor.refresh(id) let gate = TestGate() - let work = Task { await monitor.track(id) { await gate.enter() } } + let work = Task { try? await monitor.track(id) { await gate.enter() } } await gate.waitUntilEntered() live.drivers[id] = nil await gate.open() @@ -319,7 +393,7 @@ struct SessionTransactionMonitorTests { attach() let gate = TestGate() - let work = Task { await monitor.track(id) { await gate.enter() } } + let work = Task { try? await monitor.track(id) { await gate.enter() } } await gate.waitUntilEntered() #expect(monitor.isBusy(id)) @@ -331,11 +405,11 @@ struct SessionTransactionMonitorTests { @Test("A page read keeps the connection busy, never counts as statements, and stales a commit dialog") func pageReadIsBusyButNotStatements() async throws { let driver = attach(.explicitTransaction) - await monitor.track(id) {} + try await monitor.track(id) {} let token = try #require(monitor.token(for: id, on: driver)) let gate = TestGate() - let read = Task { await monitor.track(id, reading: true) { await gate.enter() } } + let read = Task { try? await monitor.track(id, reading: true) { await gate.enter() } } await gate.waitUntilEntered() let busyDuring = monitor.isBusy(id) let statementsDuring = monitor.isRunningStatements(id) @@ -356,7 +430,7 @@ struct SessionTransactionMonitorTests { attach() let gate = TestGate() - let work = Task { await monitor.track(id) { await gate.enter() } } + let work = Task { try? await monitor.track(id) { await gate.enter() } } await gate.waitUntilEntered() let during = monitor.isRunningStatements(id) await gate.open() diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Background/TransferBackgroundRunTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Background/TransferBackgroundRunTests.swift new file mode 100644 index 0000000000..fb86ac052a --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Background/TransferBackgroundRunTests.swift @@ -0,0 +1,174 @@ +import Foundation +@testable import TableProMobile +import TableProTestSupport +import Testing +import UIKit + +@MainActor +private final class FakeContinuedTask: ContinuedProcessingTask { + let progress = Progress(totalUnitCount: 0) + private(set) var titles: [(String, String)] = [] + private(set) var completions: [Bool] = [] + private var expiration: (@MainActor () -> Void)? + + func updateTitle(_ title: String, subtitle: String) { + titles.append((title, subtitle)) + } + + func setTaskCompleted(success: Bool) { + completions.append(success) + } + + func onExpiration(_ handler: @escaping @MainActor () -> Void) { + expiration = handler + } + + func expire() { + expiration?() + } +} + +@MainActor +private final class FakeScheduler: ContinuedProcessingScheduling { + struct Refused: Error {} + + var refuses = false + private(set) var identifiers: [String] = [] + private var launch: (@MainActor (any ContinuedProcessingTask) -> Void)? + + func submit( + identifier: String, + title: String, + subtitle: String, + launch: @escaping @MainActor (any ContinuedProcessingTask) -> Void + ) throws { + identifiers.append(identifier) + if refuses { throw Refused() } + self.launch = launch + } + + func start(_ task: FakeContinuedTask) { + launch?(task) + } +} + +@MainActor +private final class Counter { + var value = 0 +} + +@MainActor +struct TransferBackgroundRunTests { + @Test("A refused continued task leaves the run on the assertion, which ends with it") + func submissionFailureFallsBackToTheAssertion() { + let asserter = RecordingAsserter() + let scheduler = FakeScheduler() + scheduler.refuses = true + let hold = FakeHold() + let run = TransferBackgroundRun(title: "Export", asserter: asserter, scheduler: scheduler, hold: { hold }) + let expired = Counter() + + run.begin { expired.value += 1 } + + #expect(asserter.names == ["Export"]) + #expect(!run.isRunningContinuedTask) + #expect(run.submittedIdentifier == nil) + #expect(scheduler.identifiers.first?.hasPrefix("com.TablePro.TableProMobile.transfer.") == true) + run.end(success: true) + #expect(asserter.ended == [asserter.identifier]) + #expect(hold.releases == 1) + #expect(expired.value == 0) + } + + @Test("Each run asks for its own identifier, never one registered before") + func identifiersAreUnique() { + let scheduler = FakeScheduler() + for _ in 0..<2 { + let run = TransferBackgroundRun(title: "Import", asserter: RecordingAsserter(), scheduler: scheduler, hold: nil) + run.begin {} + run.end(success: true) + } + #expect(Set(scheduler.identifiers).count == 2) + } + + @Test("When the assertion runs out with no continued task, the run is told once and the assertion ends") + func assertionExpiryStopsTheRun() async throws { + let asserter = RecordingAsserter() + let run = TransferBackgroundRun( + title: "Export", asserter: asserter, scheduler: nil, hold: nil, expiryGrace: .milliseconds(1) + ) + let expired = Counter() + run.begin { expired.value += 1 } + + asserter.fireExpiration() + asserter.fireExpiration() + + #expect(expired.value == 1) + try await waitUntil { asserter.ended == [asserter.identifier] } + run.end(success: false) + #expect(asserter.ended == [asserter.identifier]) + } + + @Test("A continued task carries the progress and title, completes once, and outlives the assertion") + func continuedTaskMirrorsProgress() { + let asserter = RecordingAsserter() + let scheduler = FakeScheduler() + let run = TransferBackgroundRun(title: "Export", asserter: asserter, scheduler: scheduler, hold: nil) + let expired = Counter() + run.begin { expired.value += 1 } + let task = FakeContinuedTask() + scheduler.start(task) + + var progress = TransferProgress(stage: .running) + progress.completedUnits = 250 + progress.totalUnits = 1_000 + run.report(progress) + + #expect(run.isRunningContinuedTask) + #expect(task.progress.fractionCompleted == 0.25) + #expect(task.titles.last?.1 == TransferBackgroundRun.subtitle(for: progress)) + + asserter.fireExpiration() + #expect(expired.value == 0) + #expect(asserter.ended == [asserter.identifier]) + + run.end(success: true) + run.end(success: false) + #expect(task.completions == [true]) + } + + @Test("A progress with no total still moves the continued task forward") + func indeterminateProgressStillAdvances() { + let scheduler = FakeScheduler() + let run = TransferBackgroundRun(title: "Export", asserter: RecordingAsserter(), scheduler: scheduler, hold: nil) + run.begin {} + let task = FakeContinuedTask() + scheduler.start(task) + + var progress = TransferProgress(stage: .running) + progress.completedUnits = 10 + run.report(progress) + let first = task.progress.completedUnitCount + progress.completedUnits = 20 + run.report(progress) + + #expect(task.progress.completedUnitCount > first) + run.end(success: true) + } + + @Test("The person cancelling the continued task stops the run") + func continuedTaskExpiryStopsTheRun() { + let scheduler = FakeScheduler() + let run = TransferBackgroundRun(title: "Import", asserter: RecordingAsserter(), scheduler: scheduler, hold: nil) + let expired = Counter() + run.begin { expired.value += 1 } + let task = FakeContinuedTask() + scheduler.start(task) + + task.expire() + run.end(success: false) + + #expect(expired.value == 1) + #expect(task.completions == [false]) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Engines/DuckDBTransferEngineTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Engines/DuckDBTransferEngineTests.swift new file mode 100644 index 0000000000..2ebc5d5f04 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Engines/DuckDBTransferEngineTests.swift @@ -0,0 +1,144 @@ +import Foundation +import TableProDatabase +@testable import TableProMobile +import TableProModels +import TableProPluginKit +import Testing + +struct DuckDBTransferEngineTests { + private func session() async throws -> DuckDBDriver { + let driver = DuckDBDriver(source: .inMemory) + try await driver.connect() + return driver + } + + private static let context = MobileTransferContext(databaseType: .duckdb, database: nil, schema: nil) + + @Test("Bound text, bytes and NULL land as written, and export rows read the blob back as its bytes") + func bindsAndExportsBytes() async throws { + let driver = try await session() + _ = try await driver.execute(query: "CREATE TABLE t (id INTEGER, body VARCHAR, data BLOB)") + _ = try await driver.executeParameterized( + query: "INSERT INTO t VALUES ($1, $2, $3)", + parameters: [.text("1"), .text("it's \\ here"), .bytes(Data([0x00, 0xFF, 0x7F]))], + typeNames: nil + ) + _ = try await driver.executeParameterized( + query: "INSERT INTO t VALUES ($1, $2, $3)", parameters: [.text("2"), .null, .bytes(Data())], typeNames: nil + ) + _ = try await driver.execute(query: "INSERT INTO t SELECT i, 'x', NULL FROM range(3, 5003) AS r(i)") + + var header: PluginStreamHeader? + var rows: [[PluginCellValue]] = [] + for try await element in driver.exportRows(query: "SELECT * FROM t ORDER BY id", readOnly: false) { + switch element { + case .header(let value): header = value + case .rows(let batch): rows += batch + } + } + try await driver.disconnect() + + #expect(header?.columns == ["id", "body", "data"]) + #expect(header?.columnTypeNames == ["INTEGER", "VARCHAR", "BLOB"]) + #expect(rows.count == 5_002) + #expect(rows[0] == [.text("1"), .text("it's \\ here"), .bytes(Data([0x00, 0xFF, 0x7F]))]) + #expect(rows[1] == [.text("2"), .null, .bytes(Data())]) + } + + @Test("A catalog read binds its values and keeps bytes") + func catalogQueryBinds() async throws { + let driver = try await session() + + let result = try await driver.catalogQuery("SELECT $1 AS a, $2::BLOB AS b", parameters: [.text("x"), .bytes(Data([1, 2]))]) + try await driver.disconnect() + + #expect(result.rows == [[.text("x"), .bytes(Data([1, 2]))]]) + } + + @Test("The catalog lists tables, views and both macro kinds, and drops a table macro as one") + func catalogListsMacros() async throws { + let driver = try await session() + _ = try await driver.execute(query: "CREATE TABLE facts (id INTEGER PRIMARY KEY, label VARCHAR)") + _ = try await driver.execute(query: "CREATE VIEW labels AS SELECT label FROM facts") + _ = try await driver.execute(query: "CREATE MACRO plus_one(x) AS x + 1") + _ = try await driver.execute(query: "CREATE MACRO rows_of(n) AS TABLE SELECT * FROM range(n)") + let catalog = try await driver.makeExportCatalog(context: Self.context) + + let objects = try await catalog.exportObjects(database: "", schema: nil) + let scalar = try #require(objects.first { $0.name == "plus_one" }) + let table = try #require(objects.first { $0.name == "rows_of" }) + let scalarDDL = try await catalog.fetchObjectDDL(scalar) + try await driver.disconnect() + let relations = objects.filter { $0.kind != .routine }.map { "\($0.kind.rawValue) \($0.name)" } + + #expect(relations.sorted() == ["table facts", "view labels"]) + #expect(scalar.identity == "(x)") + #expect(table.identity == "(n) AS TABLE") + #expect(scalar.tableType == PluginRoutineKind.function.rawValue) + #expect(scalarDDL.hasPrefix("CREATE OR REPLACE MACRO")) + #expect(catalog.dropStatement(for: scalar) == "DROP MACRO IF EXISTS \"main\".\"plus_one\"") + #expect(catalog.dropStatement(for: table) == "DROP MACRO TABLE IF EXISTS \"main\".\"rows_of\"") + } + + @Test("The CREATE TABLE writer's statements run, and import statements bind with $n in the schema") + func writeDialect() async throws { + let driver = try await session() + let dialect = try await driver.makeWriteDialect(context: Self.context) + let definition = PluginCreateTableDefinition( + tableName: "made", + columns: [ + PluginColumnDefinition(name: "id", dataType: "INTEGER", isNullable: false, isPrimaryKey: true), + PluginColumnDefinition(name: "note", dataType: "VARCHAR"), + ], + primaryKeyColumns: ["id"] + ) + for statement in try dialect.createTableStatements(definition: definition, schema: nil) { + _ = try await driver.execute(query: statement) + } + let statements = try await dialect.importStatements(table: "made", schema: "main", columns: []) + let insert = try #require(statements.insertStatement(columns: ["id", "note"], values: [.text("7"), .null])) + _ = try await driver.executeParameterized(query: insert.sql, parameters: insert.parameters, typeNames: nil) + let stored = try await driver.execute(query: "SELECT id, note FROM main.made").rows + try await driver.disconnect() + + #expect(insert.sql == "INSERT INTO \"main\".\"made\" (\"id\", \"note\") VALUES ($1, $2)") + #expect(stored == [["7", nil]]) + } + + @Test("A transfer connection reads its own snapshot while the session writes, and the session's close ends it") + func transferConnectionLivesInsideTheSession() async throws { + let driver = try await session() + _ = try await driver.execute(query: "CREATE TABLE t (id INTEGER)") + _ = try await driver.execute(query: "INSERT INTO t VALUES (1)") + + let transfer = try await driver.openTransferDriver() + try await transfer.beginTransaction() + let before = try await transfer.execute(query: "SELECT count(*) FROM t").rows.first?.first ?? nil + _ = try await driver.execute(query: "INSERT INTO t VALUES (2)") + let during = try await transfer.execute(query: "SELECT count(*) FROM t").rows.first?.first ?? nil + try await driver.disconnect() + + #expect(transfer !== driver) + #expect(!transfer.holdsSuspensionBlockingResource) + #expect(before == "1") + #expect(during == "1") + await #expect(throws: DuckDBDriverError.self) { _ = try await transfer.ping() } + } + + @Test("A grid statement on the session does not end a read the session streams for an export") + func sessionExportSurvivesGridStatements() async throws { + let driver = try await session() + _ = try await driver.execute(query: "CREATE TABLE t AS SELECT i FROM range(0, 10000) AS r(i)") + + var iterator = driver.exportRows(query: "SELECT i FROM t ORDER BY i", readOnly: false).makeAsyncIterator() + _ = try await iterator.next() + _ = try await driver.execute(query: "SELECT 1") + var count = 0 + while let element = try await iterator.next() { + if case .rows(let batch) = element { count += batch.count } + } + try await driver.disconnect() + + #expect(count == 10_000) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Engines/MSSQLTransferEngineTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Engines/MSSQLTransferEngineTests.swift new file mode 100644 index 0000000000..8d97202122 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Engines/MSSQLTransferEngineTests.swift @@ -0,0 +1,131 @@ +import Foundation +@testable import TableProMobile +import TableProModels +import TableProMSSQLCore +import TableProPluginKit +import Testing + +struct MSSQLTransferEngineTests { + @Test("Bound values go as sp_executesql parameters typed by their values, as the Mac binds them") + func boundStatementUsesSpExecuteSql() { + let sql = MSSQLDriver.boundStatement( + "INSERT INTO [dbo].[t] ([a], [b], [c]) VALUES (?, ?, ?)", + parameters: [.text("it's"), .bytes(Data([0xAB, 0x01])), .null] + ) + + #expect(sql == "EXEC sp_executesql N'INSERT INTO [dbo].[t] ([a], [b], [c]) VALUES " + + "(@__tablepro_p1, @__tablepro_p2, @__tablepro_p3)', " + + "N'@__tablepro_p1 NVARCHAR(MAX), @__tablepro_p2 VARBINARY(MAX), @__tablepro_p3 NVARCHAR(MAX)', " + + "@__tablepro_p1 = N'it''s', @__tablepro_p2 = 0xAB01, @__tablepro_p3 = NULL") + } + + @Test("A statement with no values, or no placeholders, goes as it is") + func unboundStatementIsUnchanged() { + #expect(MSSQLDriver.boundStatement("DELETE FROM [dbo].[t]", parameters: []) == "DELETE FROM [dbo].[t]") + #expect(MSSQLDriver.boundStatement("SELECT 1", parameters: [.text("x")]) == "SELECT 1") + } + + private static let salesReplies: [String: [[PluginCellValue]]] = [ + MSSQLSchemaQueries.tables(schema: "sales"): [["orders", "BASE TABLE"], ["recent", "VIEW"]], + MSSQLObjectQueries.routineList(schema: "sales"): [ + ["archive", "sales", "P ", "0", "0", "@before date", nil], + ["total", "sales", "FN", "0", "0", "", "int"], + ["concat_all", "sales", "AF", "0", "0", "@value nvarchar(4000)", "nvarchar"], + ], + ] + + private static func catalog(_ reads: RecordedCatalogReads) -> MSSQLMobileExportCatalog { + MSSQLMobileExportCatalog( + catalog: MSSQLExportCatalog(run: { try reads.result($0) }, currentSchema: "dbo"), + script: ExportCatalogScript(databaseTypeId: DatabaseType.mssql.rawValue, session: nil) + ) + } + + @Test("Tables, views and routines are listed under their schema, each routine with its kind") + func listsObjects() async throws { + let objects = try await Self.catalog(RecordedCatalogReads(Self.salesReplies)) + .exportObjects(database: "shop", schema: "sales") + + #expect(objects.map(\.name) == ["orders", "recent", "archive", "total", "concat_all"]) + #expect(objects.map(\.kind) == [.table, .view, .routine, .routine, .routine]) + #expect(objects.map(\.tableType) == ["table", "view", "procedure", "function", "function"]) + #expect(objects.allSatisfy { $0.databaseName == "sales" && $0.schema == "sales" }) + } + + @Test("An empty schema is the session's own, never a schema named \"\"") + func emptySchemaIsCurrent() async throws { + let reads = RecordedCatalogReads([MSSQLSchemaQueries.tables(schema: "dbo"): [["t", "BASE TABLE"]]]) + let objects = try await Self.catalog(reads).exportObjects(database: "shop", schema: "") + + #expect(objects.map(\.name) == ["t"]) + #expect(objects.first?.databaseName == "dbo") + #expect(!reads.statements.contains(MSSQLSchemaQueries.tables(schema: ""))) + } + + @Test("A routine drops with its own verb, a CLR aggregate as one, never DROP ROUTINE") + func routineDrops() async throws { + let catalog = Self.catalog(RecordedCatalogReads(Self.salesReplies)) + let objects = try await catalog.exportObjects(database: "shop", schema: "sales") + + #expect(catalog.dropStatement(for: objects[2]) == "DROP PROCEDURE IF EXISTS [sales].[archive]") + #expect(catalog.dropStatement(for: objects[3]) == "DROP FUNCTION IF EXISTS [sales].[total]") + #expect(catalog.dropStatement(for: objects[4]) == "DROP AGGREGATE IF EXISTS [sales].[concat_all]") + #expect(catalog.dropStatement(for: objects[0]) == nil) + #expect(catalog.quoteIdentifier("a]b") == "[a]]b]") + #expect(catalog.tableDDLIncludesForeignKeys) + } + + // MARK: - Write dialect + + private static let dialect = MSSQLMobileWriteDialect(databaseTypeId: DatabaseType.mssql.rawValue, currentSchema: "dbo") + + @Test("Rows bind as ? into the bracketed, schema-qualified table, within SQL Server's ceilings") + func importStatements() async throws { + let columns = [ColumnInfo(name: "a", typeName: "int", ordinalPosition: 0)] + let statements = try await Self.dialect.importStatements(table: "t", schema: "sales", columns: columns) + + #expect(statements.insertStatement(columns: ["a"], values: [.text("1")])?.sql + == "INSERT INTO [sales].[t] ([a]) VALUES (?)") + #expect(statements.maxBindParameters == 2_100) + #expect(statements.maxRowsPerInsert == 1_000) + } + + @Test("With the import's database the rows go to a three-part name, whatever database the session is on") + func importStatementsNameTheDatabase() async throws { + let dialect = MSSQLMobileWriteDialect( + databaseTypeId: DatabaseType.mssql.rawValue, currentSchema: "dbo", database: "sales_db" + ) + let columns = [ColumnInfo(name: "a", typeName: "int", ordinalPosition: 0)] + + let named = try await dialect.importStatements(table: "t", schema: "sales", columns: columns) + let defaulted = try await dialect.importStatements(table: "t", schema: nil, columns: columns) + + #expect(named.insertStatement(columns: ["a"], values: [.text("1")])?.sql + == "INSERT INTO [sales_db].[sales].[t] ([a]) VALUES (?)") + #expect(defaulted.insertStatement(columns: ["a"], values: [.text("1")])?.sql + == "INSERT INTO [sales_db].[dbo].[t] ([a]) VALUES (?)") + #expect(dialect.namesDatabase) + #expect(Self.dialect.namesDatabase == false) + } + + @Test("A new table goes into the session's schema when the import names an empty one") + func createTableInCurrentSchema() throws { + let definition = PluginCreateTableDefinition( + tableName: "t", columns: [PluginColumnDefinition(name: "a", dataType: "int", isPrimaryKey: true)] + ) + let statements = try Self.dialect.createTableStatements(definition: definition, schema: "") + + #expect(statements.first?.hasPrefix("CREATE TABLE [dbo].[t] (") == true) + } + + @Test("A column type the writer cannot spell refuses the table with its reason") + func createTableRefusalNamesTheColumn() throws { + let column = PluginColumnDefinition(name: "a", dataType: "int; DROP TABLE x") + let definition = PluginCreateTableDefinition(tableName: "t", columns: [column]) + let reason = try #require(MSSQLTableDefinitionSQL.refusal(for: column)) + + #expect(throws: MobileTransferRefusal(reason: reason)) { + _ = try Self.dialect.createTableStatements(definition: definition, schema: "dbo") + } + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Engines/OracleTransferEngineTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Engines/OracleTransferEngineTests.swift new file mode 100644 index 0000000000..b4e541d346 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Engines/OracleTransferEngineTests.swift @@ -0,0 +1,128 @@ +import Foundation +import os +@testable import TableProMobile +import TableProModels +import TableProOracleCore +import TableProPluginKit +import Testing + +struct OracleTransferEngineTests { + @Test("Untyped values stay literals where a literal carries them; bytes and long text bind as :n") + func boundStatementSubstitutes() throws { + let long = String(repeating: "a", count: OracleBindPlaceholders.maxLiteralBytes + 1) + let statement = try OracleDriver.boundStatement( + "INSERT INTO \"APP\".\"T\" (\"A\", \"B\", \"C\", \"D\") VALUES (?, ?, ?, ?)", + parameters: [.text("00123"), .bytes(Data([0x01])), .null, .text(long)], + typeNames: nil, + writesBooleans: false + ) + + #expect(statement.sql == "INSERT INTO \"APP\".\"T\" (\"A\", \"B\", \"C\", \"D\") VALUES ('00123', :1, NULL, :2)") + #expect(statement.binds == [.bytes(Data([0x01])), .text(long)]) + } + + @Test("A ? inside a literal, a quoted name or a comment is text, not a placeholder") + func placeholdersInTextStay() throws { + let statement = try OracleDriver.boundStatement( + "SELECT '?' AS \"Paid?\" FROM DUAL WHERE A = ? -- ?", + parameters: [.text("x")], + typeNames: nil, + writesBooleans: false + ) + + #expect(statement.sql == "SELECT '?' AS \"Paid?\" FROM DUAL WHERE A = 'x' -- ?") + #expect(statement.binds.isEmpty) + } + + private static let appReplies: [String: [[PluginCellValue]]] = [ + OracleSchemaQueries.tables(schema: "APP"): [["ORDERS", "BASE TABLE", "N"], ["RECENT", "VIEW", "N"]], + OracleObjectQueries.routineList(schema: "APP"): [["ARCHIVE", "APP", "PROCEDURE", "VALID"], ["TOTAL", "APP", "FUNCTION", "VALID"]], + OracleObjectQueries.triggerList(schema: "APP", table: nil): [[ + "AUDIT_ORDERS", "ORDERS", "APP", "BEFORE EACH ROW", "INSERT", "ENABLED", nil, + "AUDIT_ORDERS BEFORE INSERT ON ORDERS FOR EACH ROW", "PL/SQL ", "APP", "BEGIN NULL; END;", + ]], + ] + + private static func catalog(_ reads: RecordedCatalogReads) -> OracleMobileExportCatalog { + OracleMobileExportCatalog( + catalog: OracleExportCatalog( + run: { statement in + OracleRawResult( + columns: [], + rows: try reads.rows(statement).map { $0.map(Self.rawCell) }, + affectedRows: 0, + isTruncated: false + ) + }, + currentSchema: "APP", + fallbackSchema: "APP", + release: OracleServerRelease(major: 23), + principalState: OSAllocatedUnfairLock(initialState: OraclePrincipalState()), + coreError: { $0 as? OracleCoreError } + ), + script: ExportCatalogScript(databaseTypeId: DatabaseType.oracle.rawValue, session: nil) + ) + } + + private static func rawCell(_ value: PluginCellValue) -> OracleRawCell { + switch value { + case .null: .null + case .text(let text): .string(text) + case .bytes(let data): .bytes(data) + } + } + + @Test("Tables, views, routines and triggers are listed under their owner (D14)") + func listsObjectsWithTriggers() async throws { + let objects = try await Self.catalog(RecordedCatalogReads(Self.appReplies)) + .exportObjects(database: "APP", schema: "APP") + + #expect(objects.map(\.name) == ["ORDERS", "RECENT", "ARCHIVE", "TOTAL", "AUDIT_ORDERS"]) + #expect(objects.map(\.kind) == [.table, .view, .routine, .routine, .trigger]) + #expect(objects.map(\.tableType) == ["table", "view", "procedure", "function", "trigger"]) + #expect(objects[4].parentTable == "ORDERS") + #expect(objects.allSatisfy { $0.databaseName == "APP" && $0.schema == "APP" }) + } + + @Test("A trigger's definition is the one the listing read, and every object drops by its own verb") + func triggerDefinitionAndDrops() async throws { + let catalog = Self.catalog(RecordedCatalogReads(Self.appReplies)) + let objects = try await catalog.exportObjects(database: "APP", schema: "APP") + + let definition = try await catalog.fetchObjectDDL(objects[4]) + #expect(definition.contains("BEGIN NULL; END;")) + #expect(catalog.dropStatement(for: objects[0]) == "DROP TABLE IF EXISTS \"ORDERS\"") + #expect(catalog.dropStatement(for: objects[1]) == "DROP VIEW IF EXISTS \"RECENT\"") + #expect(catalog.dropStatement(for: objects[4]) == "DROP TRIGGER IF EXISTS \"AUDIT_ORDERS\"") + #expect(catalog.dropStatement(for: objects[2])?.contains("PROCEDURE") == true) + #expect(catalog.dropStatement(for: objects[3])?.contains("FUNCTION") == true) + #expect(!catalog.tableDDLIncludesForeignKeys) + } + + // MARK: - Write dialect + + private static let dialect = OracleMobileWriteDialect( + databaseTypeId: DatabaseType.oracle.rawValue, createTableSchema: "APP", supportsIdentity: true + ) + + @Test("Rows insert one per statement with each value's column type, for the binder to write it") + func importStatementsCarryTypes() async throws { + let columns = [ColumnInfo(name: "A", typeName: "NUMBER(10,2)", ordinalPosition: 0)] + let statements = try await Self.dialect.importStatements(table: "T", schema: "APP", columns: columns) + let insert = try #require(statements.insertStatement(columns: ["A"], values: [.text("1.5")])) + + #expect(insert.sql == "INSERT INTO \"APP\".\"T\" (\"A\") VALUES (?)") + #expect(insert.parameterTypeNames == ["NUMBER(10,2)"]) + #expect(statements.maxRowsPerInsert == 1) + } + + @Test("A new table goes into the session's schema when the import names none") + func createTableInSessionSchema() throws { + let definition = PluginCreateTableDefinition( + tableName: "T", columns: [PluginColumnDefinition(name: "A", dataType: "NUMBER", isPrimaryKey: true)] + ) + let statements = try Self.dialect.createTableStatements(definition: definition, schema: nil) + + #expect(statements.first?.hasPrefix("CREATE TABLE \"APP\".\"T\" (") == true) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Engines/PostgreSQLExportCatalogTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Engines/PostgreSQLExportCatalogTests.swift new file mode 100644 index 0000000000..30e3e13065 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Engines/PostgreSQLExportCatalogTests.swift @@ -0,0 +1,178 @@ +import Foundation +@testable import TableProMobile +import TableProModels +import TableProPluginKit +import TableProPostgreSQLCore +import Testing + +struct PostgreSQLExportCatalogTests { + private static let capabilities = PostgreSQLCapabilities(serverVersion: 170_000) + + private static func listing(_ schema: String) -> String { + PostgreSQLTableListing.query( + schema: schema, + includeMaterializedViews: true, + includeForeignTables: true, + includeComments: false, + includePartitionAwareness: true + ) + } + + private static let salesReplies: [String: [[PluginCellValue]]] = [ + listing("sales"): [ + ["orders", "BASE TABLE", nil, nil], + ["events", "PARTITIONED TABLE", nil, "2"], + ["recent", "VIEW", nil, nil], + ["totals", "MATERIALIZED VIEW", nil, nil], + ["remote", "FOREIGN TABLE", nil, nil], + ], + PostgreSQLObjectQueries.routineList(schema: "sales", capabilities: capabilities): [ + ["16401", "f", "sales", "(integer)", "integer", "sql", "f"], + ["16402", "f", "sales", "(text)", nil, "plpgsql", "p"], + ], + PostgreSQLObjectQueries.triggerList(schema: "sales", table: nil): [ + ["audit", "orders", "sales", "AFTER", "INSERT", "ROW", "O", "CREATE TRIGGER audit AFTER INSERT ON sales.orders"], + ], + PostgreSQLObjectQueries.routineDefinition(identity: "16402"): [["CREATE PROCEDURE sales.f(text) AS $$ $$"]], + ] + + private static func catalog(_ reads: RecordedCatalogReads) -> PostgreSQLMobileExportCatalog { + PostgreSQLMobileExportCatalog( + catalog: PostgreSQLCatalogExport( + read: { statement, _ in try reads.rows(statement) }, + currentSchema: "public", + capabilities: capabilities, + catalogCapabilities: capabilities, + catalogPresence: nil + ), + sessionScope: nil, + script: ExportCatalogScript(databaseTypeId: DatabaseType.postgresql.rawValue, session: nil), + standardConformingStrings: true + ) + } + + @Test("Every kind the Mac exports is listed under its schema, a partitioned table once as a table") + func listsEveryKind() async throws { + let objects = try await Self.catalog(RecordedCatalogReads(Self.salesReplies)) + .exportObjects(database: "app", schema: "sales") + + #expect(objects.map(\.name) == ["orders", "events", "recent", "totals", "remote", "f", "f", "audit"]) + #expect(objects.map(\.kind) == [ + .table, .table, .view, .materializedView, .foreignTable, .routine, .routine, .trigger, + ]) + #expect(objects.map(\.tableType) == [ + "table", "table", "view", "materializedView", "foreignTable", "function", "procedure", "trigger", + ]) + #expect(objects.allSatisfy { $0.databaseName == "sales" && $0.schema == "sales" }) + #expect(objects[5].identity == "(integer)") + #expect(objects[6].identity == "(text)") + #expect(objects[7].parentTable == "orders") + } + + @Test("No schema lists the session's own") + func listsTheCurrentSchema() async throws { + let reads = RecordedCatalogReads([Self.listing("public"): [["t", "BASE TABLE", nil, nil]]]) + let objects = try await Self.catalog(reads).exportObjects(database: "app", schema: nil) + + #expect(objects.map(\.name) == ["t"]) + #expect(objects.first?.databaseName == "public") + } + + @Test("A server that refuses the catalog listing is listed from information_schema") + func listingFallsBack() async throws { + let fallback = PostgreSQLTableListing.informationSchemaQuery(in: .schema("sales")) + let reads = RecordedCatalogReads([fallback: [["orders", "BASE TABLE", nil, nil]]], failing: [Self.listing("sales")]) + let objects = try await Self.catalog(reads).exportObjects(database: "app", schema: "sales") + + #expect(objects.map(\.name) == ["orders"]) + #expect(reads.statements.contains(fallback)) + } + + @Test("A routine's definition is read for the routine of its own kind and signature") + func routineDDLMatchesKind() async throws { + let catalog = Self.catalog(RecordedCatalogReads(Self.salesReplies)) + let objects = try await catalog.exportObjects(database: "app", schema: "sales") + let procedure = try #require(objects.first(where: { $0.tableType == "procedure" })) + + #expect(try await catalog.fetchObjectDDL(procedure) == "CREATE PROCEDURE sales.f(text) AS $$ $$") + } + + @Test("Drops name the routine's kind and signature, and the trigger's table") + func dropStatements() async throws { + let catalog = Self.catalog(RecordedCatalogReads(Self.salesReplies)) + let objects = try await catalog.exportObjects(database: "app", schema: "sales") + + #expect(catalog.dropStatement(for: objects[5]) == "DROP FUNCTION IF EXISTS \"sales\".\"f\"(integer)") + #expect(catalog.dropStatement(for: objects[6]) == "DROP PROCEDURE IF EXISTS \"sales\".\"f\"(text)") + #expect(catalog.dropStatement(for: objects[7]) == "DROP TRIGGER IF EXISTS \"audit\" ON \"sales\".\"orders\"") + #expect(catalog.dropStatement(for: objects[0]) == nil) + } + + @Test("Literals follow the session's standard_conforming_strings, as the Mac's driver writes them") + func literalEscaping() { + let reads = RecordedCatalogReads() + let standard = Self.catalog(reads) + let backslash = PostgreSQLMobileExportCatalog( + catalog: standard.catalog, sessionScope: nil, script: standard.script, standardConformingStrings: false + ) + + #expect(standard.escapeStringLiteral("it's \\ here") == "it''s \\ here") + #expect(backslash.escapeStringLiteral("it's \\ here") == "it''s \\\\ here") + #expect(standard.quoteIdentifier("a\"b") == "\"a\"\"b\"") + #expect(standard.supportsCascadeDrop) + #expect(!standard.tableDDLIncludesForeignKeys) + } + + // MARK: - Write dialect + + private static let columns = [ + ColumnInfo(name: "a", typeName: "integer", ordinalPosition: 0), + ColumnInfo(name: "b", typeName: "text", ordinalPosition: 1), + ] + + private static func dialect(buildsTables: Bool = true) -> PostgreSQLMobileWriteDialect { + PostgreSQLMobileWriteDialect( + databaseTypeId: DatabaseType.postgresql.rawValue, + currentSchema: "public", + capabilities: capabilities, + buildsTables: buildsTables + ) + } + + @Test("Rows bind as $n into the schema-qualified table") + func importStatementsBindDollar() async throws { + let statements = try await Self.dialect().importStatements(table: "t", schema: "sales", columns: Self.columns) + let insert = try #require(statements.insertStatement(columns: ["a", "b"], values: [.text("1"), .null])) + + #expect(insert.sql == "INSERT INTO \"sales\".\"t\" (\"a\", \"b\") VALUES ($1, $2)") + #expect(insert.parameters == [.text("1"), .null]) + } + + @Test("A new table is created in the import's schema, or the session's when it names none") + func createTableSchema() throws { + let definition = PluginCreateTableDefinition( + tableName: "t", + columns: [PluginColumnDefinition(name: "a", dataType: "integer", isNullable: false, isPrimaryKey: true)] + ) + + let named = try Self.dialect().createTableStatements(definition: definition, schema: "sales") + let unnamed = try Self.dialect().createTableStatements(definition: definition, schema: nil) + + #expect(named.first?.hasPrefix("CREATE TABLE \"sales\".\"t\" (") == true) + #expect(unnamed.first?.hasPrefix("CREATE TABLE \"public\".\"t\" (") == true) + } + + @Test("Redshift refuses a new table as the Mac does, and still takes rows into an existing one") + func redshiftRefusesNewTables() async throws { + let definition = PluginCreateTableDefinition( + tableName: "t", columns: [PluginColumnDefinition(name: "a", dataType: "integer")] + ) + let redshift = Self.dialect(buildsTables: false) + + #expect(throws: ImportTableNames.noStatement) { + _ = try redshift.createTableStatements(definition: definition, schema: "public") + } + let statements = try await redshift.importStatements(table: "t", schema: nil, columns: Self.columns) + #expect(statements.insertStatement(columns: ["a"], values: [.text("1")])?.sql == "INSERT INTO \"t\" (\"a\") VALUES ($1)") + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Engines/RecordedCatalogReads.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Engines/RecordedCatalogReads.swift new file mode 100644 index 0000000000..d343e4834c --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Engines/RecordedCatalogReads.swift @@ -0,0 +1,35 @@ +import Foundation +import os +import TableProPluginKit + +// Answers each catalog statement with the rows recorded for its exact text, nothing for any other, and remembers what +// was read. A statement listed in `failing` throws. +final class RecordedCatalogReads: Sendable { + struct Refused: Error {} + + private let replies: [String: [[PluginCellValue]]] + private let failing: Set + private let readStatements = OSAllocatedUnfairLock<[String]>(initialState: []) + + init(_ replies: [String: [[PluginCellValue]]] = [:], failing: Set = []) { + self.replies = replies + self.failing = failing + } + + var statements: [String] { + readStatements.withLock { $0 } + } + + func rows(_ statement: String) throws -> [[PluginCellValue]] { + readStatements.withLock { $0.append(statement) } + if failing.contains(statement) { throw Refused() } + return replies[statement] ?? [] + } + + func result(_ statement: String) throws -> PluginQueryResult { + let rows = try rows(statement) + return PluginQueryResult( + columns: [], columnTypeNames: [], rows: rows, rowsAffected: 0, executionTime: 0 + ) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Engines/SQLiteTransferEngineTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Engines/SQLiteTransferEngineTests.swift new file mode 100644 index 0000000000..3b46b67d84 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Engines/SQLiteTransferEngineTests.swift @@ -0,0 +1,233 @@ +import Foundation +import TableProDatabase +@testable import TableProMobile +import TableProModels +import TableProPluginKit +import TableProTestSupport +import Testing + +struct SQLiteTransferEngineTests { + private let directory: URL + + init() throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("sqlite-transfer-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + } + + private func driver(_ name: String) async throws -> SQLiteDriver { + let driver = SQLiteDriver(source: .file(directory.appendingPathComponent(name)), openMode: .createNew) + try await driver.connect() + return driver + } + + @Test("Export rows keep a blob's bytes, a NUL inside text and a NULL, with every row and the declared types") + func exportRowsKeepValues() async throws { + let driver = try await driver("rows.db") + _ = try await driver.execute(query: "CREATE TABLE t (id INTEGER, body TEXT, data BLOB)") + _ = try await driver.executeParameterized( + query: "INSERT INTO t VALUES (?, ?, ?)", + parameters: [.text("1"), .text("a\0b"), .bytes(Data([0x00, 0xFF, 0x10]))], + typeNames: nil + ) + _ = try await driver.executeParameterized( + query: "INSERT INTO t VALUES (?, ?, ?)", parameters: [.text("2"), .null, .bytes(Data())], typeNames: nil + ) + _ = try await driver.execute(query: """ + WITH RECURSIVE n(x) AS (SELECT 3 UNION ALL SELECT x + 1 FROM n WHERE x < 1203) + INSERT INTO t SELECT x, 'x', NULL FROM n + """) + + let (header, rows) = try await collect(driver.exportRows(query: "SELECT * FROM t ORDER BY id", readOnly: false)) + try await driver.disconnect() + + #expect(header?.columns == ["id", "body", "data"]) + #expect(header?.columnTypeNames == ["INTEGER", "TEXT", "BLOB"]) + #expect(rows.count == 1_203) + #expect(rows[0] == [.text("1"), .text("a\0b"), .bytes(Data([0x00, 0xFF, 0x10]))]) + #expect(rows[1] == [.text("2"), .null, .bytes(Data())]) + } + + @Test("A NUL inside text reaches the SQL dump as char(0), so the value restores whole") + func dumpLiteralKeepsNul() async throws { + let driver = try await driver("nul.db") + let body = SQLiteTransferCatalog(driver: driver).escapeStringLiteral("a\0b'c") + + let restored = try await driver.execute(query: "SELECT hex('\(body)')").rows.first?.first + try await driver.disconnect() + + #expect(body == "a'||char(0)||'b''c") + #expect(restored == "6100622763") + } + + // SQLite refuses to drop a table a statement still reads (SQLITE_LOCKED), so the drop lands once the read is gone. + @Test("A reader that stops early finalizes its statement when it lets go") + func earlyStopFinalizes() async throws { + let driver = try await driver("stop.db") + _ = try await driver.execute(query: "CREATE TABLE t (id INTEGER)") + _ = try await driver.execute(query: """ + WITH RECURSIVE n(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM n WHERE x < 2000) INSERT INTO t SELECT x FROM n + """) + try await readTwoElements(of: driver.exportRows(query: "SELECT id FROM t", readOnly: false)) + + try await waitUntil { (try? await driver.execute(query: "DROP TABLE t")) != nil } + try await driver.disconnect() + } + + private func readTwoElements(of stream: AsyncThrowingStream) async throws { + var iterator = stream.makeAsyncIterator() + _ = try await iterator.next() + _ = try await iterator.next() + } + + @Test("A bound value count that does not match the statement is refused") + func refusesMismatchedValues() async throws { + let driver = try await driver("binds.db") + _ = try await driver.execute(query: "CREATE TABLE t (a TEXT, b TEXT)") + + await #expect(throws: SQLiteError.self) { + _ = try await driver.executeParameterized( + query: "INSERT INTO t VALUES (?, ?)", parameters: [.text("only one")], typeNames: nil + ) + } + try await driver.disconnect() + } + + @Test("The catalog lists tables, views and triggers, and reads each one's definition") + func catalogListsEveryKind() async throws { + let driver = try await driver("catalog.db") + _ = try await driver.execute(query: "CREATE TABLE parent (id INTEGER PRIMARY KEY, name TEXT)") + _ = try await driver.execute(query: "CREATE VIEW names AS SELECT name FROM parent") + _ = try await driver.execute(query: """ + CREATE TRIGGER stamp AFTER INSERT ON parent BEGIN UPDATE parent SET name = name WHERE id = NEW.id; END + """) + let catalog = try await driver.makeExportCatalog( + context: MobileTransferContext(databaseType: .sqlite, database: nil, schema: nil) + ) + + let objects = try await catalog.exportObjects(database: "", schema: nil) + let trigger = try #require(objects.first { $0.kind == .trigger }) + let triggerDDL = try await catalog.fetchObjectDDL(trigger) + let viewDDL = try await catalog.fetchObjectDDL(try #require(objects.first { $0.kind == .view })) + try await driver.disconnect() + let names = objects.map { $0.name } + let kinds = objects.map { $0.kind } + + #expect(names == ["names", "parent", "stamp"]) + #expect(kinds == [.view, .table, .trigger]) + #expect(trigger.parentTable == "parent") + #expect(triggerDDL.hasPrefix("CREATE TRIGGER stamp")) + #expect(viewDDL.hasPrefix("CREATE VIEW names")) + #expect(catalog.dropStatement(for: trigger) == "DROP TRIGGER IF EXISTS `stamp`") + } + + @Test("The CREATE TABLE writer's statement runs, and the import statements quote and bind for SQLite") + func writeDialect() async throws { + let driver = try await driver("write.db") + let dialect = try await driver.makeWriteDialect( + context: MobileTransferContext(databaseType: .sqlite, database: nil, schema: nil) + ) + let definition = PluginCreateTableDefinition( + tableName: "new table", + columns: [ + PluginColumnDefinition(name: "id", dataType: "INTEGER", isNullable: false, isPrimaryKey: true), + PluginColumnDefinition(name: "note", dataType: "TEXT"), + ], + primaryKeyColumns: ["id"] + ) + for statement in try dialect.createTableStatements(definition: definition, schema: nil) { + _ = try await driver.execute(query: statement) + } + let statements = try await dialect.importStatements( + table: "new table", schema: nil, columns: [ColumnInfo(name: "id", typeName: "INTEGER")] + ) + let insert = try #require(statements.insertStatement(columns: ["id", "note"], values: [.text("7"), .text("x")])) + _ = try await driver.executeParameterized(query: insert.sql, parameters: insert.parameters, typeNames: nil) + let stored = try await driver.execute(query: "SELECT id, note FROM \"new table\"").rows + try await driver.disconnect() + + #expect(insert.sql == "INSERT INTO `new table` (`id`, `note`) VALUES (?, ?)") + #expect(stored == [["7", "x"]]) + } + + @Test("Foreign key checks turn off and back on with PRAGMA foreign_keys") + func foreignKeyStatements() async throws { + let driver = SQLiteDriver(source: .inMemory) + #expect(driver.foreignKeyCheckStatements(enabled: false) == ["PRAGMA foreign_keys = OFF"]) + #expect(driver.foreignKeyCheckStatements(enabled: true) == ["PRAGMA foreign_keys = ON"]) + } + + @Test("A WAL file gets a second, read-only reader that sees committed rows and refuses writes") + func walGetsASecondReader() async throws { + let driver = try await driver("wal.db") + _ = try await driver.execute(query: "PRAGMA journal_mode = WAL") + _ = try await driver.execute(query: "CREATE TABLE t (id INTEGER)") + _ = try await driver.execute(query: "INSERT INTO t VALUES (1)") + + let reader = try await driver.openTransferDriver() + let count = try await reader.execute(query: "SELECT count(*) FROM t").rows.first?.first ?? nil + await #expect(throws: SQLiteError.self) { _ = try await reader.execute(query: "INSERT INTO t VALUES (2)") } + try await reader.disconnect() + try await driver.disconnect() + + #expect(reader !== driver) + #expect(count == "1") + } + + @Test("A rollback-journal file is read from a snapshot, removed when the transfer ends") + func rollbackJournalReadsASnapshot() async throws { + let driver = try await driver("journal.db") + _ = try await driver.execute(query: "CREATE TABLE t (id INTEGER)") + _ = try await driver.execute(query: "INSERT INTO t VALUES (1)") + + let snapshot = try await driver.openTransferDriver() + _ = try await driver.execute(query: "INSERT INTO t VALUES (2)") + let count = try await snapshot.execute(query: "SELECT count(*) FROM t").rows.first?.first ?? nil + let leftBefore = try FileManager.default.contentsOfDirectory(atPath: SQLiteDriver.snapshotDirectory.path) + try await snapshot.disconnect() + let leftAfter = (try? FileManager.default.contentsOfDirectory(atPath: SQLiteDriver.snapshotDirectory.path)) ?? [] + try await driver.disconnect() + + #expect(snapshot !== driver) + #expect(count == "1") + #expect(leftAfter.count == leftBefore.count - 1) + } + + @Test("With the user's transaction open the transfer reads the session inside it") + func openTransactionReadsTheSession() async throws { + let driver = try await driver("busy.db") + _ = try await driver.execute(query: "CREATE TABLE t (id INTEGER)") + try await driver.beginTransaction() + + let transfer = try await driver.openTransferDriver() + try await driver.rollbackTransaction() + try await driver.disconnect() + + #expect(transfer === driver) + } + + @Test("An in-memory database is read on its session") + func inMemoryReadsTheSession() async throws { + let driver = SQLiteDriver(source: .inMemory) + try await driver.connect() + + let transfer = try await driver.openTransferDriver() + try await driver.disconnect() + + #expect(transfer === driver) + } + + private func collect( + _ stream: AsyncThrowingStream + ) async throws -> (header: PluginStreamHeader?, rows: [[PluginCellValue]]) { + var header: PluginStreamHeader? + var rows: [[PluginCellValue]] = [] + for try await element in stream { + switch element { + case .header(let value): header = value + case .rows(let batch): rows += batch + } + } + return (header, rows) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Export/ExportRunTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Export/ExportRunTests.swift new file mode 100644 index 0000000000..8a5e6ba045 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Export/ExportRunTests.swift @@ -0,0 +1,442 @@ +import Foundation +import TableProDatabase +@testable import TableProMobile +import TableProModels +import TableProPluginKit +import TableProTestSupport +import Testing + +private struct SchemaRefused: LocalizedError { + var errorDescription: String? { "schema \"sales\" does not exist" } +} + +struct ExportRunTests { + private func request(_ source: ExportSource, name: String = "orders", schema: String? = "public") -> ExportRequest { + ExportRequest( + formatId: "csv", source: source, fileName: name, connectionId: UUID(), databaseType: .postgresql, + database: "shop", schema: schema, safeModeLevel: .off + ) + } + + private let rows = ExportRowSet( + columns: ["id", "name"], columnTypeNames: ["int4", "text"], rows: [[.text("1"), .text("Ada")], [.text("2"), .null]] + ) + + @Test("Held rows export to one file named after the export, in a folder of its own") + func rowsExportWritesTheFile() async throws { + let root = try TransferTestFolders.make("export-rows") + let run = ExportRun( + request: request(.rows(rows), name: "a/b"), plugin: MobileExportFormats.csv, + drivers: UnreachableDriverProvider(), exportsRoot: root + ) + + let outcome = try await run.run { _ in } + + #expect(outcome.files.map(\.lastPathComponent) == ["a_b.csv"]) + #expect(outcome.folder.deletingLastPathComponent().resolvingSymlinksInPath().path + == root.resolvingSymlinksInPath().path) + #expect(outcome.rowsWritten == 2) + let text = try String(contentsOf: try #require(outcome.files.first), encoding: .utf8) + #expect(text.contains("Ada")) + outcome.discard() + #expect(!FileManager.default.fileExists(atPath: outcome.folder.path)) + TransferFiles.remove(root) + } + + @Test("A stop before the run starts throws and writes nothing") + func stopBeforeStartLeavesNothing() async throws { + let root = try TransferTestFolders.make("export-early-stop") + let run = ExportRun( + request: request(.rows(rows)), plugin: MobileExportFormats.csv, + drivers: UnreachableDriverProvider(), exportsRoot: root + ) + run.cancel() + + await #expect(throws: CancellationError.self) { + _ = try await run.run { _ in } + } + #expect(TransferTestFolders.contents(of: root).isEmpty) + TransferFiles.remove(root) + } + + @Test("A stop while the plugin writes removes its partial file and the folder") + func stopMidRunLeavesNoFile() async throws { + let root = try TransferTestFolders.make("export-stop") + let plugin = GatedExportPlugin() + let run = ExportRun( + request: request(.rows(rows)), plugin: plugin, drivers: UnreachableDriverProvider(), exportsRoot: root + ) + + let running = Task { try await run.run { _ in } } + await plugin.gate.waitUntilEntered() + #expect(TransferTestFolders.contents(of: root).count == 1) + run.cancel() + await plugin.gate.open() + + await #expect(throws: CancellationError.self) { + _ = try await running.value + } + #expect(TransferTestFolders.contents(of: root).isEmpty) + TransferFiles.remove(root) + } + + @Test("A query export reads on the leased connection, asks for data only, and closes the lease") + func queryExportIsDataOnly() async throws { + let root = try TransferTestFolders.make("export-query") + let driver = RecordingTransferDriver() + driver.scriptedRows["SELECT * FROM orders WHERE total > 10"] = ( + PluginStreamHeader(columns: ["id"], columnTypeNames: ["int4"]), [[.text("7")]] + ) + let run = ExportRun( + request: request(.query("SELECT * FROM orders WHERE total > 10")), plugin: MobileExportFormats.sql, + drivers: FixedDriverProvider(driver: driver), exportsRoot: root + ) + + let outcome = try await run.run { _ in } + + let text = try String(contentsOf: try #require(outcome.files.first), encoding: .utf8) + #expect(!text.contains("CREATE TABLE")) + #expect(!text.contains("DROP TABLE")) + #expect(text.contains("INSERT INTO")) + #expect(driver.isDisconnected) + outcome.discard() + TransferFiles.remove(root) + } + + @Test("Rows read from a table go into that table, not the file's name, and leave its generated column out") + func tableQueryKeepsItsRelation() async throws { + let root = try TransferTestFolders.make("export-relation") + let driver = RecordingTransferDriver() + let catalog = StubExportCatalog() + catalog.columns = ["orders": [ + PluginColumnInfo(name: "id", dataType: "int4", isPrimaryKey: true), + PluginColumnInfo(name: "total", dataType: "numeric", isGenerated: true) + ]] + driver.exportCatalog = catalog + driver.scriptedRows["SELECT id, total FROM orders WHERE id > 1"] = ( + PluginStreamHeader(columns: ["id", "total"], columnTypeNames: ["int4", "numeric"]), [[.text("2"), .text("9")]] + ) + let select = ExportSelect(relation: ExportRelation(table: "orders", schema: "public")) { _ in + "SELECT id, total FROM orders WHERE id > 1" + } + let run = ExportRun( + request: request(.query(select), name: "my rows"), plugin: MobileExportFormats.sql, + drivers: FixedDriverProvider(driver: driver), exportsRoot: root + ) + + let outcome = try await run.run { _ in } + + let text = try String(contentsOf: try #require(outcome.files.first), encoding: .utf8) + #expect(text.contains("\"orders\"")) + #expect(text.contains("(\"id\")")) + #expect(!text.contains("\"total\"")) + #expect(!text.contains("my rows")) + outcome.discard() + TransferFiles.remove(root) + } + + @Test("The session's own connection is never closed by the export") + func sessionLeaseStaysOpen() async throws { + let root = try TransferTestFolders.make("export-session") + let driver = RecordingTransferDriver() + let run = ExportRun( + request: request(.query("SELECT 1")), plugin: MobileExportFormats.csv, + drivers: FixedDriverProvider(driver: driver, isSessionDriver: true), exportsRoot: root + ) + + let outcome = try await run.run { _ in } + + #expect(!driver.isDisconnected) + outcome.discard() + TransferFiles.remove(root) + } + + @Test("A transaction open on the window's session is named in the warnings only when another connection read the rows", + arguments: [false, true]) + func openTransactionWarnsWhenReadElsewhere(onSession: Bool) async throws { + let root = try TransferTestFolders.make("export-transaction") + let driver = RecordingTransferDriver() + let held = ExportRequest( + formatId: "csv", source: .query("SELECT 1"), fileName: "orders", connectionId: UUID(), + databaseType: .postgresql, database: "shop", schema: "public", safeModeLevel: .off, + sessionHoldsTransaction: true + ) + let run = ExportRun( + request: held, plugin: MobileExportFormats.csv, + drivers: FixedDriverProvider(driver: driver, isSessionDriver: onSession), exportsRoot: root + ) + + let outcome = try await run.run { _ in } + + #expect(outcome.warnings.contains(ExportRun.missedTransactionWarning) == !onSession) + outcome.discard() + TransferFiles.remove(root) + } + + @Test("The formats offered follow each plugin's engine list, in the macOS order") + func formatsPerEngine() { + let ids = { (type: DatabaseType) in MobileExportFormats.available(for: type).map { Swift.type(of: $0).formatId } } + #expect(ids(.postgresql) == ["csv", "json", "sql", "xlsx", "md"]) + #expect(ids(.mongodb) == ["csv", "json", "xlsx", "md", "mql"]) + #expect(ids(.redis) == ["csv", "json", "xlsx", "md"]) + } + + @Test("Export Results runs a read again, and only where the engine holds it to read-only") + func resultsRerunOnlyReads() { + func rerun(_ sent: [String], _ type: DatabaseType, readOnly: Bool = true) -> String? { + QueryEditorViewModel.rerunnableStatement(sent, databaseType: type, runsReadOnly: readOnly) + } + #expect(rerun(["SELECT * FROM t"], .postgresql) == "SELECT * FROM t") + #expect(rerun(["SELECT public.post_payment(42)"], .postgresql, readOnly: false) == nil) + #expect(rerun(["DELETE FROM t RETURNING *"], .postgresql) == nil) + #expect(rerun(["CALL refresh()"], .mysql) == nil) + #expect(rerun(["CREATE TEMP TABLE x AS SELECT 1", "SELECT * FROM x"], .postgresql) == nil) + #expect(rerun([], .postgresql) == nil) + } + + @Test("A result runs again in the engine's read-only mode even with Safe Mode off") + func rerunReadsReadOnly() async throws { + let root = try TransferTestFolders.make("export-rerun") + let driver = RecordingTransferDriver() + driver.enforcesReadOnlyMode = true + let run = ExportRun( + request: request(.rerun("SELECT public.post_payment(42)")), plugin: MobileExportFormats.csv, + drivers: FixedDriverProvider(driver: driver), exportsRoot: root + ) + + let outcome = try await run.run { _ in } + + #expect(driver.executed == ["SELECT public.post_payment(42)"]) + #expect(driver.exportReadOnlyFlags == [true]) + outcome.discard() + TransferFiles.remove(root) + } + + @Test("A result is not run again on a connection that cannot hold it to read-only") + func rerunRefusedWithoutReadOnlyMode() async throws { + let root = try TransferTestFolders.make("export-rerun-refused") + let driver = RecordingTransferDriver() + let run = ExportRun( + request: request(.rerun("SELECT public.post_payment(42)")), plugin: MobileExportFormats.csv, + drivers: FixedDriverProvider(driver: driver), exportsRoot: root + ) + + await #expect(throws: MobileTransferRefusal.self) { + _ = try await run.run { _ in } + } + #expect(driver.executed.isEmpty) + #expect(driver.isDisconnected) + #expect(TransferTestFolders.contents(of: root).isEmpty) + TransferFiles.remove(root) + } + + @Test("A connection of the export's own is set to the window's schema before it reads") + func dedicatedConnectionReadsTheChosenSchema() async throws { + let root = try TransferTestFolders.make("export-schema") + let driver = RecordingTransferDriver() + driver.supportsSchemas = true + let run = ExportRun( + request: request(.query("SELECT * FROM orders"), schema: "sales"), plugin: MobileExportFormats.csv, + drivers: FixedDriverProvider(driver: driver), exportsRoot: root + ) + + let outcome = try await run.run { _ in } + + #expect(driver.executed == ["switchSchema(sales)", "SELECT * FROM orders"]) + outcome.discard() + TransferFiles.remove(root) + } + + @Test("A connection that opened on the window's schema is left on the server's own search path") + func connectionAlreadyOnTheSchemaIsLeftAlone() async throws { + let root = try TransferTestFolders.make("export-schema-same") + let driver = RecordingTransferDriver() + driver.supportsSchemas = true + driver.currentSchema = "sales" + let run = ExportRun( + request: request(.query("SELECT * FROM orders"), schema: "sales"), plugin: MobileExportFormats.csv, + drivers: FixedDriverProvider(driver: driver), exportsRoot: root + ) + + let outcome = try await run.run { _ in } + + #expect(driver.executed == ["SELECT * FROM orders"]) + outcome.discard() + TransferFiles.remove(root) + } + + @Test("A schema the connection cannot switch to fails the export before it reads, and leaves no file") + func schemaSwitchFailureReadsNothing() async throws { + let root = try TransferTestFolders.make("export-schema-refused") + let driver = RecordingTransferDriver() + driver.supportsSchemas = true + driver.schemaSwitchFailure = SchemaRefused() + let run = ExportRun( + request: request(.query("SELECT * FROM orders"), schema: "sales"), plugin: MobileExportFormats.csv, + drivers: FixedDriverProvider(driver: driver), exportsRoot: root + ) + + do { + _ = try await run.run { _ in } + Issue.record("The export read without its schema") + } catch let refusal as MobileTransferRefusal { + #expect(refusal.reason.contains("sales")) + } + #expect(driver.executed == ["switchSchema(sales)"]) + #expect(driver.isDisconnected) + #expect(TransferTestFolders.contents(of: root).isEmpty) + TransferFiles.remove(root) + } + + @Test("The window's own session keeps the schema it reads in") + func sessionLeaseKeepsItsSchema() async throws { + let root = try TransferTestFolders.make("export-session-schema") + let driver = RecordingTransferDriver() + driver.supportsSchemas = true + let run = ExportRun( + request: request(.query("SELECT 1"), schema: "sales"), plugin: MobileExportFormats.csv, + drivers: FixedDriverProvider(driver: driver, isSessionDriver: true), exportsRoot: root + ) + + let outcome = try await run.run { _ in } + + #expect(driver.executed == ["SELECT 1"]) + outcome.discard() + TransferFiles.remove(root) + } + + @Test("The object list keeps every kind the database has, whatever format it was read for") + func listingKeepsEveryKind() async throws { + let driver = RecordingTransferDriver() + let catalog = StubExportCatalog() + catalog.objects = [ + PluginExportTable(name: "orders", databaseName: "shop", tableType: "table", schema: "public", kind: .table), + PluginExportTable(name: "total", databaseName: "shop", tableType: "function", schema: "public", kind: .routine), + PluginExportTable(name: "audit", databaseName: "shop", tableType: "trigger", schema: "public", kind: .trigger), + PluginExportTable(name: "ids", databaseName: "shop", tableType: "sequence", schema: "public", kind: .sequence) + ] + driver.exportCatalog = catalog + + let objects = try await ExportObjectListing.load( + drivers: FixedDriverProvider(driver: driver), databaseType: .postgresql, database: "shop", schema: "public", + plugin: MobileExportFormats.csv + ) + + #expect(objects.map(\.name) == ["orders", "total", "audit", "ids"]) + #expect(driver.isDisconnected) + } + + @Test("Selected rows are matched with the escaping of the connection that reads them, not the window's") + func rowsSelectFollowsTheReadingConnection() async throws { + let select = DataBrowserViewModel.rowsSelect( + table: "t", schema: nil, database: "shop", type: .mysql, keys: [[(column: "k", value: #"a\b"#)]], + columns: [ColumnInfo(name: "k", typeName: "varchar(10)", ordinalPosition: 0)] + ) + let escaping = RecordingTransferDriver() + escaping.literalEscaping = { $0.replacingOccurrences(of: "\\", with: "\\\\") } + let literal = RecordingTransferDriver() + + #expect(select.statement(on: escaping).hasSuffix(#"= 'a\\b')"#)) + #expect(select.statement(on: literal).hasSuffix(#"= 'a\b')"#)) + + let root = try TransferTestFolders.make("export-rows-escaping") + let run = ExportRun( + request: request(.query(select)), plugin: MobileExportFormats.csv, + drivers: FixedDriverProvider(driver: escaping), exportsRoot: root + ) + let outcome = try await run.run { _ in } + + #expect(escaping.executed == [select.statement(on: escaping)]) + outcome.discard() + TransferFiles.remove(root) + } + + @Test("A run's plugin holds the settings the shared one had when it was taken, whatever is changed after") + func frozenPluginKeepsItsSettings() throws { + let shared = SettingsProbePlugin() + shared.settings.compresses = true + + let frozen = try #require(MobileExportFormats.frozen(shared) as? SettingsProbePlugin) + shared.settings.compresses = false + + #expect(frozen !== shared) + #expect(frozen.settings.compresses) + #expect(frozen.currentFileExtension == "txt.gz") + } + + @Test("Each shipped format gives a run its own instance with the same settings") + func shippedFormatsFreeze() { + for shared in MobileExportFormats.all { + let frozen = MobileExportFormats.frozen(shared) + #expect(frozen !== shared) + #expect(type(of: frozen) == type(of: shared)) + #expect(frozen.currentFileExtension == shared.currentFileExtension) + } + for shared in MobileImportFormats.all { + let frozen = MobileImportFormats.frozen(shared) + #expect(frozen !== shared) + #expect(type(of: frozen) == type(of: shared)) + } + } + + @Test("Redis keys have no Export of their own: the export's objects are its databases") + func redisKeysExportByDatabase() { + let redis = RedisDriver(host: "localhost", port: 6_379, password: nil) + #expect(ExportObjectListing.supportsExport(redis)) + #expect(!ExportObjectListing.supportsTableExport(redis)) + #expect(ExportObjectListing.supportsTableExport(RecordingTransferDriver())) + } + + @Test("A stop ends the run at once, stops the driver's statement and closes the connection") + func stopInterruptsTheRead() async throws { + let root = try TransferTestFolders.make("export-stop-read") + let driver = RecordingTransferDriver() + driver.stall = .untilInterrupted + let run = ExportRun( + request: request(.query("SELECT 1")), plugin: ExecutingExportPlugin(), + drivers: FixedDriverProvider(driver: driver), exportsRoot: root + ) + let running = Task { try await run.run { _ in } } + await driver.waitUntilStalled() + + run.cancel() + + let ended = try #require(await BoundedCall.result(within: .seconds(5)) { await running.result }) + #expect(throws: CancellationError.self) { try ended.get() } + try await waitUntil { driver.isDisconnected } + #expect(driver.interrupts == 1) + #expect(TransferTestFolders.contents(of: root).isEmpty) + TransferFiles.remove(root) + } + + @MainActor + @Test("Background expiry ends the run within the grace, releases the hold, and closes a connection that never answers") + func expiryReleasesAnUnresponsiveConnection() async throws { + let root = try TransferTestFolders.make("export-expiry") + let driver = RecordingTransferDriver() + driver.stall = .unresponsive + let asserter = RecordingAsserter() + let hold = FakeHold() + let background = TransferBackgroundRun( + title: "Export", asserter: asserter, scheduler: nil, hold: { hold }, expiryGrace: .milliseconds(1) + ) + let run = ExportRun( + request: request(.query("SELECT 1")), plugin: ExecutingExportPlugin(), + drivers: FixedDriverProvider(driver: driver), background: background, exportsRoot: root, + stopGrace: .milliseconds(50) + ) + let running = Task { try await run.run { _ in } } + await driver.waitUntilStalled() + + asserter.fireExpiration() + + let ended = try #require(await BoundedCall.result(within: .seconds(5)) { await running.result }) + #expect(throws: CancellationError.self) { try ended.get() } + #expect(hold.releases == 1) + try await waitUntil { driver.isDisconnected } + #expect(driver.interrupts == 1) + #expect(TransferTestFolders.contents(of: root).isEmpty) + driver.endStall() + TransferFiles.remove(root) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Export/MobileExportDataSourceTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Export/MobileExportDataSourceTests.swift new file mode 100644 index 0000000000..dad54aaaac --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Export/MobileExportDataSourceTests.swift @@ -0,0 +1,176 @@ +import Foundation +import TableProDatabase +@testable import TableProMobile +import TableProModels +import TableProPluginKit +import Testing + +struct MobileExportDataSourceTests { + private func table(_ name: String, schema: String? = "public", scope: PluginExportRowScope = .unrestricted) -> PluginExportTable { + PluginExportTable( + name: name, databaseName: "", tableType: "table", schema: schema, kind: .table, rowScope: scope + ) + } + + private func collect(_ stream: AsyncThrowingStream) async throws + -> (header: PluginStreamHeader?, rows: [PluginRow]) { + var header: PluginStreamHeader? + var rows: [PluginRow] = [] + for try await element in stream { + switch element { + case .header(let value): header = value + case .rows(let batch): rows += batch + } + } + return (header, rows) + } + + @Test("Rows reach the format as the driver read them: bytes, NULL and the column types kept") + func bytesNullAndTypesPassThrough() async throws { + let driver = RecordingTransferDriver() + let blob = Data([0x00, 0xFF, 0x10]) + driver.scriptedRows["SELECT * FROM \"public\".\"files\""] = ( + PluginStreamHeader(columns: ["id", "body", "note"], columnTypeNames: ["int4", "bytea", "text"]), + [[.text("1"), .bytes(blob), .null]] + ) + let source = MobileExportDataSource( + catalog: StubExportCatalog(), driver: driver, databaseType: .postgresql, readOnly: false + ) + + let read = try await collect(source.streamRows(for: table("files"))) + + #expect(read.header?.columnTypeNames == ["int4", "bytea", "text"]) + #expect(read.rows == [[.text("1"), .bytes(blob), .null]]) + } + + @Test("Read-Only reads every row through the driver's read-only mode") + func readOnlyReachesTheDriver() async throws { + let driver = RecordingTransferDriver() + let source = MobileExportDataSource( + catalog: StubExportCatalog(), driver: driver, databaseType: .postgresql, readOnly: true + ) + _ = try await collect(source.streamRows(for: table("t"))) + #expect(driver.exportReadOnlyFlags == [true]) + } + + @Test("A row scope narrows the columns and rows the way the macOS adapter does") + func rowScopeNarrowsTheSelect() throws { + let source = MobileExportDataSource( + catalog: StubExportCatalog(), driver: RecordingTransferDriver(), databaseType: .postgresql, readOnly: false + ) + let scoped = table("t", scope: PluginExportRowScope(filter: "id > 3;", rowLimit: 10, columns: ["id", "name"])) + + #expect(try source.rowsQuery(for: scoped) + == "SELECT \"id\", \"name\" FROM \"public\".\"t\" WHERE id > 3 LIMIT 10 OFFSET 0") + #expect(source.rowCountQuery(for: scoped) == nil) + #expect(source.rowCountQuery(for: table("t")) == "SELECT COUNT(*) FROM \"public\".\"t\"") + } + + @Test("The engine's own rows statement wins when the scope is the whole object") + func catalogRowsQueryWins() throws { + let catalog = StubExportCatalog() + catalog.customRowsQuery = "SELECT * FROM ONLY \"public\".\"t\"" + let source = MobileExportDataSource( + catalog: catalog, driver: RecordingTransferDriver(), databaseType: .postgresql, readOnly: false + ) + #expect(try source.rowsQuery(for: table("t")) == "SELECT * FROM ONLY \"public\".\"t\"") + #expect(source.rowCountQuery(for: table("t")) == nil) + } + + @Test("An engine with no schema layer qualifies by the database the catalog named") + func databaseQualifiesWithoutSchema() { + let source = MobileExportDataSource( + catalog: StubExportCatalog(databaseTypeId: "MySQL"), driver: RecordingTransferDriver(), + databaseType: .mysql, readOnly: false + ) + let object = PluginExportTable(name: "t", databaseName: "shop", tableType: "table", schema: nil, kind: .table) + #expect(source.qualifiedName(of: object) == "\"shop\".\"t\"") + } + + // On the session holding the user's transaction, the dump's COMMIT would end the user's work. + @Test("Snapshot statements are dropped on the user's session and at Read-Only, and sent otherwise") + func snapshotStatementsSkipWhereTheyCannotRun() async throws { + let onSession = RecordingTransferDriver() + let sessionSource = MobileExportDataSource( + catalog: StubExportCatalog(), driver: onSession, databaseType: .sqlite, readOnly: false, + skipsSnapshotStatements: true + ) + _ = try await sessionSource.execute(query: "BEGIN") + _ = try await sessionSource.execute(query: " COMMIT") + #expect(onSession.executed.isEmpty) + + let readOnly = RecordingTransferDriver() + let readOnlySource = MobileExportDataSource( + catalog: StubExportCatalog(), driver: readOnly, databaseType: .postgresql, readOnly: true + ) + _ = try await readOnlySource.execute(query: "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + #expect(readOnly.executed.isEmpty) + + let dedicated = RecordingTransferDriver() + let dedicatedSource = MobileExportDataSource( + catalog: StubExportCatalog(), driver: dedicated, databaseType: .postgresql, readOnly: false + ) + _ = try await dedicatedSource.execute(query: "BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + #expect(dedicated.executed == ["BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY"]) + } + + @Test("A routine whose kind is unknown gets no drop, since PostgreSQL refuses the wrong one") + func routineWithoutKindWritesNoDrop() { + let catalog = StubExportCatalog() + catalog.dropStatements = ["f": "DROP FUNCTION IF EXISTS f"] + let source = MobileExportDataSource( + catalog: catalog, driver: RecordingTransferDriver(), databaseType: .postgresql, readOnly: false + ) + let known = PluginExportTable(name: "f", databaseName: "", tableType: "function", schema: nil, kind: .routine) + let unknown = PluginExportTable(name: "f", databaseName: "", tableType: "routine", schema: nil, kind: .routine) + #expect(source.dropStatement(for: known) == "DROP FUNCTION IF EXISTS f") + #expect(source.dropStatement(for: unknown) == nil) + } + + @Test("Selected rows refuse a cut or binary cell and keep every other value") + func rowSetRefusesIncompleteCells() throws { + let columns = [ + ColumnInfo(name: "id", typeName: "int", ordinalPosition: 0), + ColumnInfo(name: "photo", typeName: "blob", ordinalPosition: 1) + ] + #expect(throws: ClipboardExporter.IncompleteValues(columns: ["photo"])) { + try ExportRowSet(columns: columns, cells: [[.text("1"), .binary(byteCount: 4, ref: nil)]]) + } + let rows = try ExportRowSet(columns: columns, cells: [[.text("1"), .null]]) + #expect(rows.rows == [[.text("1"), .null]]) + #expect(rows.columnTypeNames == ["int", "blob"]) + } + + @Test("Held rows take the SQL format's snapshot statements as no-ops and refuse any other statement") + func rowsSourcePassesSnapshotStatements() async throws { + let source = MobileRowsExportDataSource( + rows: ExportRowSet(columns: ["n"], columnTypeNames: ["int"], rows: [[.text("1")]]), + databaseType: .postgresql, sessionDriver: nil + ) + for dialect in [SqlDialect.mysql, .postgres, .sqlite] { + let snapshot = SQLExportSnapshot(dialect: dialect) + try await snapshot.begin(on: source) + await snapshot.end(on: source) + } + _ = try await source.execute(query: "COMMIT") + _ = try await source.execute(query: "ROLLBACK") + await #expect(throws: PluginExportError.self) { + _ = try await source.execute(query: "SELECT 1") + } + } + + @Test("Held rows stream a slice at a time after the header") + func rowsSourceStreamsInSlices() async throws { + let values = (0..<1_200).map { [PluginCellValue.text("\($0)")] } + let source = MobileRowsExportDataSource( + rows: ExportRowSet(columns: ["n"], columnTypeNames: ["int"], rows: values), + databaseType: .postgresql, sessionDriver: nil + ) + var batches: [Int] = [] + for try await element in source.streamRows(table: "q", databaseName: "") { + if case .rows(let rows) = element { batches.append(rows.count) } + } + #expect(batches == [500, 500, 200]) + #expect(try await source.fetchApproximateRowCount(table: "q", databaseName: "") == 1_200) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Export/TransferFakes.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Export/TransferFakes.swift new file mode 100644 index 0000000000..15eb5f2826 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Export/TransferFakes.swift @@ -0,0 +1,332 @@ +import Foundation +import TableProDatabase +@testable import TableProMobile +import TableProModels +import TableProPluginKit +import TableProSQLGrammar +import TableProSQLQuoting +import UIKit + +// A driver that answers export reads from scripted rows and records every statement it is sent. +final class RecordingTransferDriver: DatabaseDriver, ExportRowStreaming, CatalogQuerying, @unchecked Sendable { + // How a catalog read behaves: answered at once, held until `cancelCurrentQuery`, or held with the stop itself + // never answering either, as a driver blocked on a dead socket. + enum ReadStall { + case none + case untilInterrupted + case unresponsive + } + + struct Interrupted: Error {} + + private let lock = NSLock() + private var sent: [String] = [] + private var readOnlyFlags: [Bool] = [] + private var stalledReads: [CheckedContinuation] = [] + private var stalledStops: [CheckedContinuation] = [] + private var stallObservers: [CheckedContinuation] = [] + private var interruptCount = 0 + var scriptedRows: [String: (header: PluginStreamHeader, rows: [PluginRow])] = [:] + var transactionState: DriverTransactionState = .idle + var supportsSchemas = false + var currentSchema: String? + var schemaSwitchFailure: (any Error)? + var enforcesReadOnlyMode = false + var stall = ReadStall.none + var exportCatalog: (any MobileExportCatalog)? + var literalEscaping: (@Sendable (String) -> String)? + private(set) var disconnected = false + + var executed: [String] { lock.withLock { sent } } + var exportReadOnlyFlags: [Bool] { lock.withLock { readOnlyFlags } } + var interrupts: Int { lock.withLock { interruptCount } } + var isDisconnected: Bool { lock.withLock { disconnected } } + + private func record(_ query: String) { + lock.withLock { sent.append(query) } + } + + func exportRows(query: String, readOnly: Bool) -> AsyncThrowingStream { + lock.withLock { + sent.append(query) + readOnlyFlags.append(readOnly) + } + let script = scriptedRows[query] + return AsyncThrowingStream { continuation in + if let script { + continuation.yield(.header(script.header)) + if !script.rows.isEmpty { continuation.yield(.rows(script.rows)) } + } + continuation.finish() + } + } + + func catalogQuery(_ sql: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + record(sql) + if stall != .none { + try await withCheckedThrowingContinuation { continuation in + let observers = lock.withLock { () -> [CheckedContinuation] in + stalledReads.append(continuation) + defer { stallObservers = [] } + return stallObservers + } + observers.forEach { $0.resume() } + } + } + return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func waitUntilStalled() async { + await withCheckedContinuation { observer in + let stalled = lock.withLock { () -> Bool in + guard stalledReads.isEmpty else { return true } + stallObservers.append(observer) + return false + } + if stalled { observer.resume() } + } + } + + // Lets every held read and stop go, so a test leaves no continuation behind. + func endStall() { + let (reads, stops) = lock.withLock { () -> ([CheckedContinuation], [CheckedContinuation]) in + defer { + stalledReads = [] + stalledStops = [] + } + return (stalledReads, stalledStops) + } + reads.forEach { $0.resume(throwing: Interrupted()) } + stops.forEach { $0.resume() } + } + + func escapeStringLiteral(_ value: String) -> String { + literalEscaping?(value) ?? SQLQuoting.stringLiteralBody(value) + } + + func connect() async throws {} + func disconnect() async throws { lock.withLock { disconnected = true } } + func ping() async throws -> Bool { true } + func execute(query: String) async throws -> QueryResult { + record(query) + return QueryResult(columns: [], rows: [], rowsAffected: 0, executionTime: 0) + } + func cancelCurrentQuery() async throws { + let mode = lock.withLock { () -> ReadStall in + interruptCount += 1 + return stall + } + switch mode { + case .none: + return + case .untilInterrupted: + let reads = lock.withLock { () -> [CheckedContinuation] in + defer { stalledReads = [] } + return stalledReads + } + reads.forEach { $0.resume(throwing: Interrupted()) } + case .unresponsive: + await withCheckedContinuation { stop in + lock.withLock { stalledStops.append(stop) } + } + } + } + func fetchTables(schema: String?) async throws -> [TableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [ColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [IndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [ForeignKeyInfo] { [] } + func fetchDatabases() async throws -> [String] { [] } + func switchDatabase(to name: String) async throws {} + func switchSchema(to name: String) async throws { + record("switchSchema(\(name))") + if let schemaSwitchFailure { throw schemaSwitchFailure } + } + func fetchSchemas() async throws -> [String] { [] } + var supportsTransactions: Bool { true } + func beginTransaction() async throws { record("BEGIN") } + func commitTransaction() async throws { record("COMMIT") } + func rollbackTransaction() async throws { record("ROLLBACK") } + func sessionTransactionState() async -> DriverTransactionState { transactionState } + var serverVersion: String? { nil } +} + +extension RecordingTransferDriver: MobileTransferEngine { + func makeExportCatalog(context: MobileTransferContext) async throws -> any MobileExportCatalog { + guard let exportCatalog else { throw MobileTransferRefusal(reason: "No catalog scripted") } + return exportCatalog + } +} + +// Every catalog answer at its default but the ones a test sets. +final class StubExportCatalog: MobileExportCatalog, @unchecked Sendable { + let databaseTypeId: String + var objects: [PluginExportTable] = [] + var customRowsQuery: String? + var dropStatements: [String: String] = [:] + var columns: [String: [PluginColumnInfo]] = [:] + + init(databaseTypeId: String = "PostgreSQL") { + self.databaseTypeId = databaseTypeId + } + + func exportObjects(database: String, schema: String?) async throws -> [PluginExportTable] { objects } + func rowsQuery(for object: PluginExportTable) -> String? { customRowsQuery } + func quoteIdentifier(_ identifier: String) -> String { SQLQuoting.identifier(identifier) } + func escapeStringLiteral(_ value: String) -> String { SQLEscaping.ansiStringLiteral(value) } + func fetchTableDDL(table: String, databaseName: String) async throws -> String { "CREATE TABLE \(table) ()" } + func dropStatement(for object: PluginExportTable) -> String? { dropStatements[object.name] } + func fetchColumns(table: String, databaseName: String) async throws -> [PluginColumnInfo] { columns[table] ?? [] } + func fetchAllColumns(databaseName: String) async throws -> [String: [PluginColumnInfo]] { columns } +} + +// A provider that hands out one driver and says whether it is the window's session. +struct FixedDriverProvider: TransferDriverProvider { + let driver: any DatabaseDriver + var isSessionDriver = false + + func openTransferDriver() async throws -> TransferDriverLease { + let driver = driver + let isSession = isSessionDriver + return TransferDriverLease(driver: driver, isSessionDriver: isSession) { + guard !isSession else { return } + try? await driver.disconnect() + } + } +} + +struct UnreachableDriverProvider: TransferDriverProvider { + struct Unreachable: Error {} + + func openTransferDriver() async throws -> TransferDriverLease { + throw Unreachable() + } +} + +// Writes a partial file, then waits for the test before it looks at the stop, as a plugin between two batches does. +final class GatedExportPlugin: ExportFormatPlugin, @unchecked Sendable { + static let pluginName = "Gated" + static let pluginVersion = "1.0" + static let pluginDescription = "Test export" + static let formatId = "gated" + static let formatDisplayName = "Gated" + static let defaultFileExtension = "txt" + static let iconName = "doc" + + let gate = TestGate() + + required init() {} + + func export( + tables: [PluginExportTable], + dataSource: any PluginExportDataSource, + destination: URL, + progress: PluginExportProgress + ) async throws -> ExportFormatResult { + try Data("partial".utf8).write(to: destination) + await gate.enter() + try progress.checkCancellation() + return ExportFormatResult() + } +} + +// Reads through the data source's `execute`, which the driver may hold, as a format's snapshot BEGIN does. +final class ExecutingExportPlugin: ExportFormatPlugin, @unchecked Sendable { + static let pluginName = "Executing" + static let pluginVersion = "1.0" + static let pluginDescription = "Test export" + static let formatId = "executing" + static let formatDisplayName = "Executing" + static let defaultFileExtension = "txt" + static let iconName = "doc" + + required init() {} + + func export( + tables: [PluginExportTable], + dataSource: any PluginExportDataSource, + destination: URL, + progress: PluginExportProgress + ) async throws -> ExportFormatResult { + try Data("partial".utf8).write(to: destination) + _ = try await dataSource.execute(query: "SELECT pg_sleep(3600)") + try progress.checkCancellation() + return ExportFormatResult() + } +} + +// A format with one setting kept under a test-only storage key, so changing it touches no shipped format's defaults. +final class SettingsProbePlugin: ExportFormatPlugin, SettablePlugin, @unchecked Sendable { + struct Settings: Codable, Equatable { + var compresses = false + } + + static let pluginName = "Settings Probe" + static let pluginVersion = "1.0" + static let pluginDescription = "Test export" + static let formatId = "settings-probe" + static let formatDisplayName = "Settings Probe" + static let defaultFileExtension = "txt" + static let iconName = "doc" + static let settingsStorageId = "tests.settings-probe" + + var settings = Settings() { + didSet { saveSettings() } + } + + required init() {} + + var currentFileExtension: String { settings.compresses ? "txt.gz" : "txt" } + + func export( + tables: [PluginExportTable], + dataSource: any PluginExportDataSource, + destination: URL, + progress: PluginExportProgress + ) async throws -> ExportFormatResult { + ExportFormatResult() + } +} + +@MainActor +final class RecordingAsserter: BackgroundTaskAsserting { + let identifier = UIBackgroundTaskIdentifier(rawValue: 11) + private(set) var names: [String] = [] + private(set) var ended: [UIBackgroundTaskIdentifier] = [] + private var expirationHandler: (() -> Void)? + + func beginBackgroundTask(name: String, expirationHandler: @escaping () -> Void) -> UIBackgroundTaskIdentifier { + names.append(name) + self.expirationHandler = expirationHandler + return identifier + } + + func endBackgroundTask(_ identifier: UIBackgroundTaskIdentifier) { + ended.append(identifier) + } + + func fireExpiration() { + expirationHandler?() + } +} + +@MainActor +final class FakeHold: TransferHolding { + private(set) var releases = 0 + + func release() { + releases += 1 + } +} + +enum TransferTestFolders { + static func make(_ label: String) throws -> URL { + let folder = FileManager.default.temporaryDirectory + .appendingPathComponent("transfer-\(label)-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + return folder + } + + static func contents(of folder: URL) -> [URL] { + (try? FileManager.default.contentsOfDirectory(at: folder, includingPropertiesForKeys: nil)) ?? [] + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Export/TransferFilesTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Export/TransferFilesTests.swift new file mode 100644 index 0000000000..5c2b030595 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Export/TransferFilesTests.swift @@ -0,0 +1,78 @@ +import Foundation +import Testing +@testable import TableProMobile + +struct TransferFilesTests { + @Test("A name that holds a path separator or a colon keeps its other letters") + func separatorsBecomeUnderscores() { + #expect(TransferFiles.sanitizedFileName("a/b") == "a_b") + #expect(TransferFiles.sanitizedFileName("a\\b:c") == "a_b_c") + #expect(TransferFiles.sanitizedFileName("line\nbreak") == "line_break") + } + + @Test("A name of dots, spaces or nothing falls back, so it never names a folder or a hidden file") + func dotNamesFallBack() { + #expect(TransferFiles.sanitizedFileName("..") == "Export") + #expect(TransferFiles.sanitizedFileName(".") == "Export") + #expect(TransferFiles.sanitizedFileName(" ") == "Export") + #expect(TransferFiles.sanitizedFileName("") == "Export") + #expect(TransferFiles.sanitizedFileName(".hidden") == "hidden") + } + + @Test("A long name is cut on a character boundary under the byte limit") + func longNamesAreCut() { + let name = String(repeating: "日本", count: 200) + let cleaned = TransferFiles.sanitizedFileName(name) + #expect(cleaned.utf8.count <= TransferFiles.maximumNameBytes) + #expect(cleaned.allSatisfy { $0 == "日" || $0 == "本" }) + #expect(TransferFiles.sanitizedFileName("Cafe\u{301} orders") == "Cafe\u{301} orders") + } + + @Test("A long imported name is cut in its stem, so its extension and a gzip's inner one survive") + func longImportNamesKeepTheirExtensions() { + let stem = String(repeating: "a", count: 201) + let gzipped = TransferFiles.sanitizedFileNameKeepingExtension("\(stem).sql.gz", fallback: "Import") + #expect(gzipped == String(repeating: "a", count: TransferFiles.maximumNameBytes - 7) + ".sql.gz") + #expect(SQLImportFile.isCompressed(URL(fileURLWithPath: "/tmp/\(gzipped)"))) + let plain = TransferFiles.sanitizedFileNameKeepingExtension("\(stem).CSV", fallback: "Import") + #expect(plain == String(repeating: "a", count: TransferFiles.maximumNameBytes - 4) + ".CSV") + #expect(TransferFiles.sanitizedFileNameKeepingExtension("orders.2024.csv", fallback: "Import") == "orders.2024.csv") + #expect(TransferFiles.sanitizedFileNameKeepingExtension("../x.sql", fallback: "Import") == "_x.sql") + #expect(TransferFiles.sanitizedFileNameKeepingExtension(".gz", fallback: "Import") == "gz") + #expect(TransferFiles.sanitizedFileNameKeepingExtension("", fallback: "Import") == "Import") + } + + @Test("The export file sits in its folder under the clean name with the format's extension") + func fileURLUsesTheCleanName() { + let folder = URL(fileURLWithPath: "/tmp/x", isDirectory: true) + #expect(TransferFiles.fileURL(in: folder, name: "a/b", fileExtension: "csv").lastPathComponent == "a_b.csv") + #expect(TransferFiles.fileURL(in: folder, name: "..", fileExtension: "sql").lastPathComponent == "Export.sql") + } + + @Test("The sweep removes what an earlier run left in its roots") + func sweepRemovesLeftovers() throws { + let root = try TransferTestFolders.make("sweep") + let exports = root.appendingPathComponent("Exports", isDirectory: true) + let folder = try TransferFiles.makeExportFolder(in: exports) + try Data("x".utf8).write(to: folder.appendingPathComponent("left.csv")) + + TransferFiles.sweep(roots: [exports]) + + #expect(!FileManager.default.fileExists(atPath: exports.path)) + TransferFiles.remove(root) + } + + @Test("A SQL export's split parts are shared after the main file, in part order") + func writtenFilesListsPartsAfterThePrimary() throws { + let folder = try TransferTestFolders.make("parts") + let primary = folder.appendingPathComponent("dump.sql") + for name in ["dump.part10.sql", "dump.sql", "dump.part2.sql"] { + try Data("x".utf8).write(to: folder.appendingPathComponent(name)) + } + + let names = TransferFiles.writtenFiles(in: folder, primary: primary).map(\.lastPathComponent) + + #expect(names == ["dump.sql", "dump.part2.sql", "dump.part10.sql"]) + TransferFiles.remove(folder) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Import/ImportRunTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Import/ImportRunTests.swift new file mode 100644 index 0000000000..d0a0f16677 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Import/ImportRunTests.swift @@ -0,0 +1,504 @@ +import Foundation +import TableProDatabase +import TableProModels +import TableProPluginKit +import TableProSQLGrammar +import TableProTestSupport +import Testing + +@testable import TableProMobile + +@MainActor +private final class FakeSessionHost: TransferSessionHost { + let driver: any DatabaseDriver + private(set) var trackedRuns = 0 + private(set) var sessionChecks = 0 + private(set) var changedKinds: [SQLCatalogObjectKinds] = [] + private(set) var sessionEffects: [SessionLocalStatements.Effect] = [] + private(set) var placementFollows: [[Bool]] = [] + + init(driver: any DatabaseDriver) { + self.driver = driver + } + + var transferDatabase: String? + var transferSafeModeLevel: SafeModeLevel = .off + // The level the import is admitted under, which a test can make stricter than the one it was confirmed under. + var admittedLevel: SafeModeLevel = .off + var refusesImport: (any Error)? + + // As `ConnectionCoordinator.trackImport` does: refused before anything runs, else a value the test calls failed + // sends the session to its check. + func trackImport( + failed: @Sendable (T) -> Bool, + _ work: @Sendable (ImportAdmission) async throws -> T + ) async throws -> T { + if let refusesImport { throw refusesImport } + trackedRuns += 1 + let value = try await work(ImportAdmission(driver: driver, safeModeLevel: admittedLevel)) + if failed(value) { sessionChecks += 1 } + return value + } + + func trackTransfer(_ work: @Sendable () async throws -> T) async throws -> T { + try await work() + } + + func noteTransferChangedCatalog(_ kinds: SQLCatalogObjectKinds) { + changedKinds.append(kinds) + } + + func noteTransferSessionEffect(_ effect: SessionLocalStatements.Effect, on driver: any DatabaseDriver) { + sessionEffects.append(effect) + } + + func followTransferPlacement(movesDatabase: Bool, movesSchema: Bool) async { + placementFollows.append([movesDatabase, movesSchema]) + } + + func abandonTransferSession() async {} + + private(set) var catalogHolds = 0 + private(set) var catalogReleases = 0 + + func holdTransferCatalogConnections() async { + catalogHolds += 1 + } + + func releaseTransferCatalogConnections() { + catalogReleases += 1 + } +} + +struct ImportRunTests { + private let directory: URL + + init() throws { + directory = try TransferTestFolders.make("import-run") + } + + private func database() async throws -> SQLiteDriver { + let driver = SQLiteDriver(source: .file(directory.appendingPathComponent("run.db")), openMode: .createNew) + try await driver.connect() + return driver + } + + private func staged(_ name: String, _ text: String) throws -> StagedImportFile { + let folder = try TransferFiles.makeImportFolder(in: directory) + let url = folder.appendingPathComponent(name) + try Data(text.utf8).write(to: url) + return StagedImportFile(url: url, folder: folder, originalName: name) + } + + private func request( + _ file: StagedImportFile, formatId: String, target: RowImportTarget? = nil, connectionId: UUID = UUID(), + database: String? = nil + ) -> ImportRequest { + ImportRequest( + formatId: formatId, file: file, connectionId: connectionId, connectionName: "Local", databaseType: .sqlite, + database: database, safeModeLevel: .off, target: target + ) + } + + @Test("A SQL file runs on the session inside the host's tracking and tells it the catalog moved") + func sqlFileRunsTracked() async throws { + let driver = try await database() + let host = await FakeSessionHost(driver: driver) + let file = try staged("seed.sql", "CREATE TABLE t (a INTEGER);\nINSERT INTO t VALUES (1);\nINSERT INTO t VALUES (2);\n") + let run = ImportRun(request: request(file, formatId: "sql"), plugin: MobileImportFormats.sql, host: host) + + let outcome = try await run.run { _ in } + + #expect(outcome.ending == .completed) + #expect(outcome.executedStatements == 3) + #expect(try await driver.execute(query: "SELECT count(*) FROM t").rows.first?.first == "2") + #expect(await host.trackedRuns == 1) + #expect(await host.changedKinds == [.everything]) + } + + @Test("A failing statement rolls back the import's own transaction and reports it as failed") + func failedSQLFileRollsBack() async throws { + let driver = try await database() + _ = try await driver.execute(query: "CREATE TABLE t (a INTEGER)") + let host = await FakeSessionHost(driver: driver) + let file = try staged("bad.sql", "INSERT INTO t VALUES (1);\nINSERT INTO missing VALUES (2);\n") + let run = ImportRun(request: request(file, formatId: "sql"), plugin: MobileImportFormats.sql, host: host) + + let outcome = try await run.run { _ in } + + guard case .failed = outcome.ending else { + Issue.record("expected a failed import, got \(outcome.ending)") + return + } + #expect(outcome.transaction == .rolledBack) + #expect(try await driver.execute(query: "SELECT count(*) FROM t").rows.first?.first == "0") + } + + @Test("A failed import still returns its outcome, and the host checks the session it ran on") + func failedImportReachesTheSessionCheck() async throws { + let driver = try await database() + _ = try await driver.execute(query: "CREATE TABLE t (a INTEGER)") + let host = await FakeSessionHost(driver: driver) + let file = try staged("bad.sql", "INSERT INTO t VALUES (1);\nINSERT INTO missing VALUES (2);\n") + let run = ImportRun(request: request(file, formatId: "sql"), plugin: MobileImportFormats.sql, host: host) + + let outcome = try await run.run { _ in } + + guard case .failed = outcome.ending else { + Issue.record("expected a failed import, got \(outcome.ending)") + return + } + #expect(await host.sessionChecks == 1) + } + + @Test("A completed import leaves the session alone") + func completedImportSkipsTheSessionCheck() async throws { + let driver = try await database() + let host = await FakeSessionHost(driver: driver) + let file = try staged("seed.sql", "CREATE TABLE t (a INTEGER);\nINSERT INTO t VALUES (1);\n") + let run = ImportRun(request: request(file, formatId: "sql"), plugin: MobileImportFormats.sql, host: host) + + let outcome = try await run.run { _ in } + + #expect(outcome.ending == .completed) + #expect(await host.sessionChecks == 0) + } + + @Test("Which endings send the session to its check", arguments: [ + (DataImportOutcome.Ending.failed("gone"), ImportTransactionEnding.rolledBack, true), + (.completed, .committed, false), + (.completed, .endedByServer, true), + (.stopped, .rolledBack, false), + (.stopped, .endedByServer, true), + (.expired, .endedByServer, false) + ]) + func endingsThatCheckTheSession( + _ ending: DataImportOutcome.Ending, _ transaction: ImportTransactionEnding, _ checks: Bool + ) { + let outcome = DataImportOutcome( + ending: ending, transaction: transaction, tally: nil, executedStatements: 0, skippedStatements: 0, + errors: [], createdTable: false, catalogChanges: [] + ) + #expect(outcome.mayHaveLostSession == checks) + } + + @Test("Read-Only refuses the run before it reaches the session") + func readOnlyRefusesBeforeWriting() async throws { + let driver = try await database() + let host = await FakeSessionHost(driver: driver) + let file = try staged("seed.sql", "CREATE TABLE t (a INTEGER);\n") + let refused = ImportRequest( + formatId: "sql", file: file, connectionId: UUID(), connectionName: "Local", databaseType: .sqlite, + database: nil, safeModeLevel: .readOnly, target: nil + ) + let run = ImportRun(request: refused, plugin: MobileImportFormats.sql, host: host) + + await #expect(throws: MobileTransferRefusal.self) { + _ = try await run.run { _ in } + } + #expect(await host.trackedRuns == 0) + } + + @Test("A stop before the run starts writes nothing") + func stopBeforeStartWritesNothing() async throws { + let driver = try await database() + let host = await FakeSessionHost(driver: driver) + let file = try staged("seed.sql", "CREATE TABLE t (a INTEGER);\n") + let run = ImportRun(request: request(file, formatId: "sql"), plugin: MobileImportFormats.sql, host: host) + run.stop(.rollBack) + + await #expect(throws: CancellationError.self) { + _ = try await run.run { _ in } + } + #expect(try await driver.fetchTables(schema: nil).isEmpty) + } + + // Goes through the SQLite engine's own write dialect, so it also covers the engine lane's conformance. + @Test("CSV rows land in a new table the run makes first, and a retry clears and reuses it") + func csvIntoNewTable() async throws { + let driver = try await database() + let host = await FakeSessionHost(driver: driver) + let planner = ImportTablePlanner() + let definition = PluginCreateTableDefinition( + tableName: "people", + columns: [ + PluginColumnDefinition(name: "id", dataType: "INTEGER", isNullable: false, isPrimaryKey: true), + PluginColumnDefinition(name: "name", dataType: "TEXT") + ], + primaryKeyColumns: ["id"] + ) + let target = RowImportTarget( + table: "people", schema: nil, columnMapping: ["id": "id", "name": "name"], sourceFields: ["id", "name"], + newTable: definition + ) + + let connectionId = UUID() + for _ in 0..<2 { + let file = try staged("people.csv", "id,name\n1,Ada\n2,Bo\n") + let run = ImportRun( + request: request(file, formatId: "csv", target: target, connectionId: connectionId), + plugin: MobileImportFormats.csv, + host: host, planner: planner + ) + let outcome = try await run.run { _ in } + #expect(outcome.ending == .completed) + #expect(outcome.tally?.inserted == 2) + } + + #expect(try await driver.execute(query: "SELECT count(*) FROM people").rows.first?.first == "2") + #expect(await host.changedKinds == [.tables]) + } + + @Test("A retry into the table an earlier import made empties it inside its own transaction, so a failing retry keeps the rows") + func failedRetryKeepsTheFirstRunsRows() async throws { + let driver = try await database() + let host = await FakeSessionHost(driver: driver) + let planner = ImportTablePlanner() + let definition = PluginCreateTableDefinition( + tableName: "people", + columns: [ + PluginColumnDefinition(name: "id", dataType: "INTEGER", isNullable: false, isPrimaryKey: true), + PluginColumnDefinition(name: "name", dataType: "TEXT") + ], + primaryKeyColumns: ["id"] + ) + let target = RowImportTarget( + table: "people", schema: nil, columnMapping: ["id": "id", "name": "name"], sourceFields: ["id", "name"], + newTable: definition + ) + let connectionId = UUID() + func importing(_ text: String) async throws -> DataImportOutcome { + let run = ImportRun( + request: request(try staged("people.csv", text), formatId: "csv", target: target, connectionId: connectionId), + plugin: MobileImportFormats.csv, + host: host, planner: planner + ) + return try await run.run { _ in } + } + + #expect(try await importing("id,name\n1,Ada\n2,Bo\n").ending == .completed) + let retry = try await importing("id,name\n3,Cy\n3,Dup\n") + + guard case .failed = retry.ending else { + Issue.record("expected the retry to fail, got \(retry.ending)") + return + } + #expect(retry.transaction == .rolledBack) + let names = try await driver.execute(query: "SELECT name FROM people ORDER BY id").rows.compactMap { $0.first ?? nil } + #expect(names == ["Ada", "Bo"]) + } + + @Test("Another sheet's import into the same new table is no retry: it is refused on the name and leaves the rows") + func anotherSheetNeverClearsTheTable() async throws { + let driver = try await database() + let host = await FakeSessionHost(driver: driver) + let definition = PluginCreateTableDefinition( + tableName: "people", + columns: [ + PluginColumnDefinition(name: "id", dataType: "INTEGER", isNullable: false, isPrimaryKey: true), + PluginColumnDefinition(name: "name", dataType: "TEXT") + ], + primaryKeyColumns: ["id"] + ) + let target = RowImportTarget( + table: "people", schema: nil, columnMapping: ["id": "id", "name": "name"], sourceFields: ["id", "name"], + newTable: definition + ) + let connectionId = UUID() + func importing(_ text: String) async throws -> DataImportOutcome { + let run = ImportRun( + request: request(try staged("people.csv", text), formatId: "csv", target: target, connectionId: connectionId), + plugin: MobileImportFormats.csv, + host: host, planner: ImportTablePlanner() + ) + return try await run.run { _ in } + } + + #expect(try await importing("id,name\n1,Ada\n2,Bo\n").ending == .completed) + await #expect(throws: MobileTransferRefusal.self) { + _ = try await importing("id,name\n3,Cy\n") + } + let names = try await driver.execute(query: "SELECT name FROM people ORDER BY id").rows.compactMap { $0.first ?? nil } + #expect(names == ["Ada", "Bo"]) + } + + @Test("A SQL file that drops, alters or restores a database holds the catalog connections until the run ends", + arguments: [true, false]) + func databaseChangesHoldTheCatalog(changesDatabases: Bool) async throws { + let driver = try await database() + let host = await FakeSessionHost(driver: driver) + let text = changesDatabases ? "CREATE TABLE t (a INTEGER);\nDROP DATABASE other;\n" : "CREATE TABLE t (a INTEGER);\n" + let run = ImportRun(request: request(try staged("run.sql", text), formatId: "sql"), plugin: MobileImportFormats.sql, host: host) + + _ = try await run.run { _ in } + + #expect(await host.catalogHolds == (changesDatabases ? 1 : 0)) + #expect(await host.catalogReleases == (changesDatabases ? 1 : 0)) + } + + @Test("A SQL file opened on one database is refused after the session moved to another, before any statement runs") + func sqlFileRefusedAfterTheSessionMoved() async throws { + let driver = try await database() + let host = await FakeSessionHost(driver: driver) + await MainActor.run { host.transferDatabase = "other" } + let file = try staged("run.sql", "CREATE TABLE t (a INTEGER);\n") + let run = ImportRun( + request: request(file, formatId: "sql", database: "shop"), plugin: MobileImportFormats.sql, host: host + ) + + await #expect(throws: MobileTransferRefusal.self) { + _ = try await run.run { _ in } + } + #expect(try await driver.fetchTables(schema: nil).isEmpty) + } + + @Test("An import the session cannot take now is refused with why, before it writes anything", arguments: [ + SessionAdmissionCase.otherWork, .anotherImport + ]) + func refusedAdmissionWritesNothing(_ refusal: SessionAdmissionCase) async throws { + let driver = try await database() + _ = try await driver.execute(query: "CREATE TABLE t (a INTEGER)") + let host = await FakeSessionHost(driver: driver) + await MainActor.run { host.refusesImport = refusal.error } + let file = try staged("seed.sql", "INSERT INTO t VALUES (1);\n") + let run = ImportRun(request: request(file, formatId: "sql"), plugin: MobileImportFormats.sql, host: host) + + await #expect(throws: MobileTransferRefusal(reason: refusal.error.localizedDescription)) { + _ = try await run.run { _ in } + } + #expect(try await driver.execute(query: "SELECT count(*) FROM t").rows.first?.first == "0") + } + + @Test("Safe Mode read again at admission refuses an import another window made stricter since it was confirmed", + arguments: [SafeModeLevel.readOnly, .confirmWrites]) + func stricterSafeModeAtAdmissionRefuses(_ live: SafeModeLevel) async throws { + let driver = try await database() + _ = try await driver.execute(query: "CREATE TABLE t (a INTEGER)") + let host = await FakeSessionHost(driver: driver) + await MainActor.run { host.admittedLevel = live } + let file = try staged("seed.sql", "INSERT INTO t VALUES (1);\n") + let run = ImportRun(request: request(file, formatId: "sql"), plugin: MobileImportFormats.sql, host: host) + + await #expect(throws: MobileTransferRefusal.self) { + _ = try await run.run { _ in } + } + #expect(try await driver.execute(query: "SELECT count(*) FROM t").rows.first?.first == "0") + } + + @Test("A SQL file's temporary table is recorded on the session and the window follows where the file left it") + func sqlFileSessionEffectsReachTheWindow() async throws { + let driver = try await database() + let host = await FakeSessionHost(driver: driver) + let file = try staged("temp.sql", "CREATE TEMP TABLE scratch (a INTEGER);\nINSERT INTO scratch VALUES (1);\n") + let run = ImportRun(request: request(file, formatId: "sql"), plugin: MobileImportFormats.sql, host: host) + + let outcome = try await run.run { _ in } + + #expect(outcome.ending == .completed) + #expect(await host.sessionEffects.map(\.temporaryTables) == [["scratch"]]) + #expect(await host.placementFollows.count == 1) + } + + @Test("A SQL file that fails on its USE still has the window follow the session's database") + func failedSQLFileStillFollowsTheSession() async throws { + let driver = try await database() + let host = await FakeSessionHost(driver: driver) + let file = try staged("use.sql", "USE other_db;\n") + let run = ImportRun(request: request(file, formatId: "sql"), plugin: MobileImportFormats.sql, host: host) + + let outcome = try await run.run { _ in } + + guard case .failed = outcome.ending else { + Issue.record("expected a failed import, got \(outcome.ending)") + return + } + #expect(await host.sessionEffects.first?.usesDatabase == true) + #expect(await host.placementFollows.first == [true, true]) + } + + @Test("A row import leaves the session's placement to the window") + func rowImportDoesNotFollowTheSession() async throws { + let driver = try await database() + _ = try await driver.execute(query: "CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT)") + let host = await FakeSessionHost(driver: driver) + let target = RowImportTarget( + table: "people", schema: nil, columnMapping: ["id": "id", "name": "name"], sourceFields: ["id", "name"] + ) + let file = try staged("people.csv", "id,name\n1,Ada\n") + let run = ImportRun(request: request(file, formatId: "csv", target: target), plugin: MobileImportFormats.csv, host: host) + + _ = try await run.run { _ in } + + #expect(await host.sessionEffects.isEmpty) + #expect(await host.placementFollows.isEmpty) + } + + @Test("A row import whose session moved to another database since the sheet opened is refused before it writes") + func movedSessionRefusesTheImport() async throws { + let driver = try await database() + _ = try await driver.execute(query: "CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT)") + let host = await FakeSessionHost(driver: driver) + await MainActor.run { host.transferDatabase = "archive" } + let target = RowImportTarget( + table: "people", schema: nil, columnMapping: ["id": "id", "name": "name"], sourceFields: ["id", "name"] + ) + let file = try staged("people.csv", "id,name\n1,Ada\n") + let moved = ImportRequest( + formatId: "csv", file: file, connectionId: UUID(), connectionName: "Local", databaseType: .sqlite, + database: "main", safeModeLevel: .off, target: target + ) + let run = ImportRun(request: moved, plugin: MobileImportFormats.csv, host: host) + + await #expect(throws: MobileTransferRefusal(reason: ImportRun.sessionMovedMessage)) { + _ = try await run.run { _ in } + } + #expect(try await driver.execute(query: "SELECT count(*) FROM people").rows.first?.first == "0") + } + + @Test("A row import on the session's own database goes ahead") + func sameDatabaseImports() async throws { + let driver = try await database() + _ = try await driver.execute(query: "CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT)") + let host = await FakeSessionHost(driver: driver) + await MainActor.run { host.transferDatabase = "main" } + let target = RowImportTarget( + table: "people", schema: nil, columnMapping: ["id": "id", "name": "name"], sourceFields: ["id", "name"] + ) + let file = try staged("people.csv", "id,name\n1,Ada\n") + let request = ImportRequest( + formatId: "csv", file: file, connectionId: UUID(), connectionName: "Local", databaseType: .sqlite, + database: "main", safeModeLevel: .off, target: target + ) + + let outcome = try await ImportRun(request: request, plugin: MobileImportFormats.csv, host: host).run { _ in } + + #expect(outcome.ending == .completed) + } + + @Test("A SQL file that rolls back the import's transaction and then inserts keeps that row and says so") + func sqlFileRollbackThenInsertIsReported() async throws { + let driver = try await database() + _ = try await driver.execute(query: "CREATE TABLE t (a INTEGER)") + let host = await FakeSessionHost(driver: driver) + let file = try staged("rollback.sql", "INSERT INTO t VALUES (1);\nROLLBACK;\nINSERT INTO t VALUES (2);\n") + let run = ImportRun(request: request(file, formatId: "sql"), plugin: MobileImportFormats.sql, host: host) + + let outcome = try await run.run { _ in } + + #expect(outcome.ending == .completed) + #expect(outcome.transaction == .committed) + #expect(try await driver.execute(query: "SELECT group_concat(a) FROM t").rows.first?.first == "2") + } +} + +nonisolated enum SessionAdmissionCase: Sendable, CaseIterable { + case otherWork + case anotherImport + + var error: any Error { + switch self { + case .otherWork: SessionBusyForImport() + case .anotherImport: SessionHeldByImport() + } + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Import/ImportSourcesTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Import/ImportSourcesTests.swift new file mode 100644 index 0000000000..1b0dcf9a14 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Import/ImportSourcesTests.swift @@ -0,0 +1,204 @@ +import Foundation +import TableProGzip +@testable import TableProMobile +import TableProModels +import TableProPluginKit +import TableProSQLFile +import TableProTestSupport +import Testing + +struct ImportSourcesTests { + private static let script = "CREATE TABLE t (a int);\nINSERT INTO t VALUES (1);\nINSERT INTO t VALUES ('x;y');\n" + + private func gzipped(in folder: URL) throws -> URL { + let url = folder.appendingPathComponent("dump.sql.gz") + try GzipDeflater.compress(Data(Self.script.utf8)).write(to: url) + return url + } + + @Test("A gzipped dump is expanded beside its copy, split into its statements, and the expansion removed") + func gzipDumpSplitsIntoStatements() async throws { + let folder = try TransferTestFolders.make("sql-gz") + let source = MobileSQLFileImportSource( + url: try gzipped(in: folder), + encoding: .utf8, + family: .other, + grammar: { SQLImportFile.grammar(for: .sqlite, session: nil) } + ) + + var statements: [String] = [] + for try await element in try await source.elements() { + statements.append(element.statementText.trimmingCharacters(in: .whitespacesAndNewlines)) + } + #expect(statements.count == 3) + #expect(statements.last?.contains("'x;y'") == true) + #expect(TransferTestFolders.contents(of: folder).count == 2) + + source.cleanup() + #expect(TransferTestFolders.contents(of: folder).map(\.lastPathComponent) == ["dump.sql.gz"]) + TransferFiles.remove(folder) + } + + @Test("A stop during the expansion of a gzipped dump ends it at once and leaves no partial file") + func stopEndsTheExpansion() async throws { + let folder = try TransferTestFolders.make("sql-gz-stop") + let started = TestGate() + await started.open() + let source = MobileSQLFileImportSource( + url: try gzipped(in: folder), + encoding: .utf8, + family: .other, + grammar: { SQLImportFile.grammar(for: .sqlite, session: nil) }, + expand: { _, destination in + // As `GzipFile.decompress` does: a partial file while it runs, removed when it is cancelled. + try Data("CREATE TABLE".utf8).write(to: destination) + await started.enter() + do { + try await Task.sleep(for: .seconds(60)) + } catch { + try? FileManager.default.removeItem(at: destination) + throw error + } + } + ) + + let reading = Task { _ = try await source.elements() } + await started.waitUntilEntered() + #expect(TransferTestFolders.contents(of: folder).count == 2) + source.stop() + + let ended = try #require(await BoundedCall.result(within: .seconds(5)) { await reading.result }) + #expect(throws: CancellationError.self) { try ended.get() } + #expect(TransferTestFolders.contents(of: folder).map(\.lastPathComponent) == ["dump.sql.gz"]) + TransferFiles.remove(folder) + } + + @Test("A stop before the dump is read keeps the real expansion from starting and leaves only the dump") + func stopBeforeReadingExpandsNothing() async throws { + let folder = try TransferTestFolders.make("sql-gz-early-stop") + let source = MobileSQLFileImportSource( + url: try gzipped(in: folder), + encoding: .utf8, + family: .other, + grammar: { SQLImportFile.grammar(for: .sqlite, session: nil) } + ) + source.stop() + + await #expect(throws: CancellationError.self) { + _ = try await source.statements() + } + #expect(TransferTestFolders.contents(of: folder).map(\.lastPathComponent) == ["dump.sql.gz"]) + TransferFiles.remove(folder) + } + + @Test("The preview and the statement count read through gzip") + func previewAndCountReadGzip() async throws { + let folder = try TransferTestFolders.make("sql-preview") + let file = StagedImportFile(url: try gzipped(in: folder), folder: folder, originalName: "dump.sql.gz") + + let preview = try await SQLImportFile.preview(file, databaseType: .postgresql) + let count = try await SQLImportFile.countStatements(file, encoding: .utf8, databaseType: .postgresql) + + #expect(preview.isCompressed) + #expect(preview.decodes) + #expect(preview.encoding == .utf8) + #expect(count == 3) + #expect(TransferTestFolders.contents(of: folder).count == 1) + TransferFiles.remove(folder) + } + + @Test("A dump that says it is Latin-1 and does not decode as UTF-8 is read as what it says") + func declaredEncodingIsDetected() async throws { + let folder = try TransferTestFolders.make("sql-latin1") + let url = folder.appendingPathComponent("dump.sql") + var bytes = Data("SET NAMES latin1;\nINSERT INTO t VALUES ('caf".utf8) + bytes.append(0xE9) + bytes.append(contentsOf: Data("');\n".utf8)) + try bytes.write(to: url) + + let preview = try await SQLImportFile.preview( + StagedImportFile(url: url, folder: folder, originalName: "dump.sql"), databaseType: .mysql + ) + + #expect(preview.detectedEncoding == .windowsCP1252) + #expect(preview.encoding == .windowsCP1252) + TransferFiles.remove(folder) + } + + @Test("A picked file is copied whole into a folder of its own and the source is left where it was") + func stagingCopiesTheFile() async throws { + let picked = try TransferTestFolders.make("picked") + let root = try TransferTestFolders.make("imports") + let source = picked.appendingPathComponent("people.csv") + try Data("id,name\n1,Ada\n".utf8).write(to: source) + + let staged = try await ImportFileStaging.stage(source, into: root) + + #expect(try Data(contentsOf: staged.url) == Data(contentsOf: source)) + #expect(staged.url.lastPathComponent == "people.csv") + #expect(staged.folder.deletingLastPathComponent().resolvingSymlinksInPath().path == root.resolvingSymlinksInPath().path) + #expect(FileManager.default.fileExists(atPath: source.path)) + staged.discard() + #expect(!FileManager.default.fileExists(atPath: staged.folder.path)) + TransferFiles.remove(picked) + TransferFiles.remove(root) + } + + @Test("A file that is not there fails the staging and leaves no folder") + func stagingAMissingFileLeavesNothing() async throws { + let root = try TransferTestFolders.make("imports-missing") + let missing = root.appendingPathComponent("gone.csv") + await #expect(throws: ImportFileStagingError.self) { + _ = try await ImportFileStaging.stage(missing, into: root) + } + #expect(TransferTestFolders.contents(of: root).isEmpty) + TransferFiles.remove(root) + } + + @Test("The format comes from the extension, and a compressed row file is refused") + func formatByExtension() { + let id = { (name: String) in + MobileImportFormats.plugin(for: URL(fileURLWithPath: "/tmp/\(name)")).map { type(of: $0).formatId } + } + #expect(id("a.CSV") == "csv") + #expect(id("a.tsv") == "csv") + #expect(id("a.ndjson") == "json") + #expect(id("a.xlsx") == "xlsx") + #expect(id("a.sql") == "sql") + #expect(id("a.sql.gz") == "sql") + #expect(id("a.csv.gz") == nil) + #expect(id("a.txt") == nil) + } + + @Test("Row import and SQL files are offered by engine, and refused on Read-Only") + func offersAndRefusals() { + #expect(MobileImportFormats.rowFormats(for: .mongodb).isEmpty) + #expect(MobileImportFormats.rowFormats(for: .redis).isEmpty) + #expect(MobileImportFormats.rowFormats(for: .postgresql).map { type(of: $0).formatId } == ["csv", "json", "xlsx"]) + #expect(!MobileImportFormats.offersSQLFile(for: .mongodb)) + #expect(MobileImportFormats.offersSQLFile(for: .sqlite)) + + #expect(ImportRun.refusal(connectionName: "Prod", databaseType: .postgresql, safeModeLevel: .readOnly, formatId: "csv") + != nil) + #expect(ImportRun.refusal(connectionName: "Prod", databaseType: .mongodb, safeModeLevel: .off, formatId: "json") + != nil) + #expect(ImportRun.refusal(connectionName: "Dev", databaseType: .postgresql, safeModeLevel: .confirmWrites, formatId: "csv") + == nil) + } + + @Test("A saved mapping keeps entries for fields another file holds") + func mappingStoreMerges() throws { + let suite = "transfer-mapping-\(UUID().uuidString)" + let defaults = try #require(UserDefaults(suiteName: suite)) + defer { defaults.removePersistentDomain(forName: suite) } + let store = ImportMappingStore(defaults: defaults) + let key = ImportMappingStore.Key(connectionId: UUID(), database: "shop", schema: "public", table: "people") + + store.remember(["Full Name": .column("name")], fields: ["Full Name", "id"], for: key) + store.remember(["Email": .skip], fields: ["Email", "id"], for: key) + + #expect(store.overrides(for: key) == ["Full Name": .column("name"), "Email": .skip]) + let other = ImportMappingStore.Key(connectionId: key.connectionId, database: "shop", schema: "public", table: "orders") + #expect(store.overrides(for: other).isEmpty) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/Import/MobileImportDataSinkTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/Import/MobileImportDataSinkTests.swift new file mode 100644 index 0000000000..3b228e8a5d --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/Import/MobileImportDataSinkTests.swift @@ -0,0 +1,366 @@ +import Foundation +import TableProDatabase +import TableProModels +import TableProPluginKit +import Testing + +@testable import TableProMobile + +// The SQLite statements the shared writer builds, without the engine lane's own dialect, so these cases test the sink. +private struct SQLiteTestDialect: MobileWriteDialect { + let databaseTypeId = "SQLite" + + func importStatements(table: String, schema: String?, columns: [ColumnInfo]) async throws -> SQLImportStatements { + SQLImportStatements( + databaseTypeId: "SQLite", + qualifiedTableName: SQLBuilder.qualifiedIdentifier(table: table, schema: schema, for: .sqlite), + parameterStyle: .questionMark, + quoteIdentifier: { SQLBuilder.quoteIdentifier($0, for: .sqlite) }, + columnTypeNames: Dictionary(columns.map { ($0.name, $0.typeName) }, uniquingKeysWith: { first, _ in first }) + ) + } + + func createTableStatements(definition: PluginCreateTableDefinition, schema: String?) throws -> [String] { + throw MobileTransferRefusal(reason: "not in these tests") + } +} + +private struct DuplicateTable: Error {} + +struct MobileImportDataSinkTests { + private let directory: URL + + init() throws { + directory = try TransferTestFolders.make("sink") + } + + private func people(_ rows: [(Int, String)] = []) async throws -> SQLiteDriver { + let driver = SQLiteDriver(source: .file(directory.appendingPathComponent("people.db")), openMode: .createNew) + try await driver.connect() + _ = try await driver.execute(query: "CREATE TABLE people (id INTEGER PRIMARY KEY, name TEXT)") + for (id, name) in rows { + _ = try await driver.execute(query: "INSERT INTO people (id, name) VALUES (\(id), '\(name)')") + } + return driver + } + + private func sink( + on driver: SQLiteDriver, + mode: ImportWriteMode = .insert, + clearsTargetFirst: Bool = false + ) -> MobileImportDataSink { + MobileImportDataSink( + driver: driver, + databaseType: .sqlite, + rowTarget: MobileImportDataSink.RowTarget( + table: "people", + schema: nil, + columnMapping: ["id": "id", "name": "name"], + sourceFields: ["id", "name"], + rowWriting: ImportRowWriting(mode: mode, keyColumns: mode == .insert ? [] : ["id"]), + binaryText: nil + ), + dialect: SQLiteTestDialect(), + clearsTargetFirst: clearsTargetFirst + ) + } + + private func names(_ driver: SQLiteDriver) async throws -> [String] { + try await driver.execute(query: "SELECT id || ':' || name FROM people ORDER BY id").rows.compactMap { $0.first ?? nil } + } + + private func row(_ id: String, _ name: String) -> [String: PluginCellValue] { + ["id": .text(id), "name": .text(name)] + } + + @Test("Insert writes every row in a transaction of the import's own and commits it") + func insertCommitsItsOwnTransaction() async throws { + let driver = try await people() + let sink = sink(on: driver) + + try await sink.beginTransaction() + #expect(sink.holdsOwnTransaction) + try await sink.insertRows([row("1", "Ada"), row("2", "Bo")]) + try await sink.commitTransaction() + + #expect(try await names(driver) == ["1:Ada", "2:Bo"]) + #expect(await sink.transactionEnding() == .committed) + #expect(sink.rowTally?.inserted == 2) + } + + @Test("Update changes the matched row and leaves an unmatched one out") + func updateChangesMatchedRowsOnly() async throws { + let driver = try await people([(1, "Ada"), (2, "Bo")]) + let sink = sink(on: driver, mode: .update) + + try await sink.insertRows([row("1", "Ann"), row("3", "Cy")]) + + #expect(try await names(driver) == ["1:Ann", "2:Bo"]) + #expect(sink.rowTally?.updated == 1) + #expect(sink.rowTally?.unmatched == 1) + } + + @Test("Insert or Update updates the matched row and inserts the new one") + func upsertUpdatesAndInserts() async throws { + let driver = try await people([(1, "Ada")]) + let sink = sink(on: driver, mode: .upsert) + + try await sink.insertRows([row("1", "Ann"), row("3", "Cy")]) + + #expect(try await names(driver) == ["1:Ann", "3:Cy"]) + #expect(sink.rowTally?.updated == 1) + #expect(sink.rowTally?.inserted == 1) + } + + @Test("Insert, Skip Existing keeps the stored row and inserts the new one") + func skipExistingKeepsStoredRows() async throws { + let driver = try await people([(1, "Ada")]) + let sink = sink(on: driver, mode: .insertSkippingExisting) + + try await sink.insertRows([row("1", "Ann"), row("3", "Cy")]) + + #expect(try await names(driver) == ["1:Ada", "3:Cy"]) + #expect(sink.rowTally?.existing == 1) + #expect(sink.rowTally?.inserted == 1) + } + + @Test("Clearing the table first is refused in a mode that matches rows") + func clearingRefusedWhenMatching() async throws { + let driver = try await people([(1, "Ada")]) + let sink = sink(on: driver, mode: .update) + await #expect(throws: PluginImportError.self) { + try await sink.deleteAllRowsFromTargetTable() + } + #expect(try await names(driver) == ["1:Ada"]) + } + + @Test("Stop and Rollback refuses the next write and rolls back the rows the import wrote") + func stopRollsBackOwnedRows() async throws { + let driver = try await people([(1, "Ada")]) + let sink = sink(on: driver) + try await sink.beginTransaction() + try await sink.insertRows([row("2", "Bo")]) + + sink.stop(.rollBack) + await #expect(throws: PluginImportCancellationError.self) { + try await sink.insertRows([row("3", "Cy")]) + } + await #expect(throws: PluginImportCancellationError.self) { + try await sink.commitTransaction() + } + try await sink.rollbackTransaction() + + #expect(try await names(driver) == ["1:Ada"]) + #expect(await sink.transactionEnding() == .rolledBack) + } + + @Test("Stop and Commit skips the plugin's rollback and the run's finish keeps the rows") + func stopAndCommitKeepsRows() async throws { + let driver = try await people() + let sink = sink(on: driver) + try await sink.beginTransaction() + try await sink.insertRows([row("1", "Ada")]) + + sink.stop(.commit) + try await sink.rollbackTransaction() + #expect(sink.holdsOwnTransaction) + await sink.finish(commit: true) + + #expect(try await names(driver) == ["1:Ada"]) + #expect(await sink.transactionEnding() == .committed) + } + + @Test("After Stop and Commit the foreign key checks come back once the run's finish has committed") + func stopAndCommitRestoresForeignKeysAfterTheCommit() async throws { + let driver = try await people() + let sink = sink(on: driver) + try await sink.disableForeignKeyChecks() + try await sink.beginTransaction() + try await sink.insertRows([row("1", "Ada")]) + + sink.stop(.commit) + try await sink.rollbackTransaction() + try await sink.enableForeignKeyChecks() + await sink.finish(commit: true) + + let enforced = try await driver.execute(query: "PRAGMA foreign_keys").rows.first?.first ?? nil + #expect(enforced == "1") + #expect(try await names(driver) == ["1:Ada"]) + } + + @Test("A DDL statement that fails after MySQL committed the rows before it reports them as kept") + func failedDDLAfterAnImplicitCommitKeepsTheRows() async throws { + let driver = MockDatabaseDriver() + driver.scriptedTransactionState = .idle + driver.scriptedExecuteResults = [ + .success(QueryResult(columns: [], rows: [], rowsAffected: 1, executionTime: 0)), + .failure(DuplicateTable()) + ] + let sink = MobileImportDataSink(driver: driver, databaseType: .mysql) + try await sink.beginTransaction() + try await sink.execute(statement: "INSERT INTO people (id) VALUES (1)") + + await #expect(throws: DuplicateTable.self) { + try await sink.execute(statement: "CREATE TABLE people (id INT)") + } + try await sink.rollbackTransaction() + + #expect(await sink.transactionEnding() == .partlyCommitted) + } + + @Test("A file's own COMMIT or ROLLBACK ends the user's transaction the import joined, and the outcome says how", + arguments: [("COMMIT", ImportTransactionEnding.committed), ("ROLLBACK", .rolledBack)]) + func fileEndsTheJoinedTransaction(_ statement: String, _ expected: ImportTransactionEnding) async throws { + let driver = try await people() + _ = try await driver.execute(query: "BEGIN") + let sink = sink(on: driver) + try await sink.beginTransaction() + #expect(sink.joinsUserTransaction) + try await sink.execute(statement: "INSERT INTO people (id, name) VALUES (1, 'Ada')") + try await sink.execute(statement: statement) + + #expect(await sink.transactionEnding() == expected) + } + + // Critic M8: the import never commits or rolls back the user's transaction, and says the rows wait in it. + @Test("An import into the user's open transaction joins it and leaves the rows there") + func joinsTheUsersTransaction() async throws { + let driver = try await people() + _ = try await driver.execute(query: "BEGIN") + let sink = sink(on: driver) + + try await sink.beginTransaction() + #expect(sink.joinsUserTransaction) + #expect(!sink.holdsOwnTransaction) + try await sink.insertRows([row("1", "Ada")]) + sink.stop(.rollBack) + try await sink.rollbackTransaction() + await sink.finish(commit: false) + + #expect(await driver.sessionTransactionState() != .idle) + #expect(try await names(driver) == ["1:Ada"]) + #expect(await sink.transactionEnding() == .leftInOpenTransaction) + _ = try await driver.execute(query: "ROLLBACK") + #expect(try await names(driver).isEmpty) + } + + @Test("A reused table is emptied inside the transaction the import owns, so its rollback brings the rows back") + func clearRollsBackWithTheImport() async throws { + let driver = try await people([(1, "Ada")]) + let sink = sink(on: driver, clearsTargetFirst: true) + + try await sink.beginTransaction() + #expect(sink.holdsOwnTransaction) + #expect(try await names(driver).isEmpty) + try await sink.insertRows([row("2", "Bo")]) + try await sink.rollbackTransaction() + + #expect(try await names(driver) == ["1:Ada"]) + } + + @Test("A reused table is emptied inside the user's open transaction the import joins") + func clearJoinsTheUsersTransaction() async throws { + let driver = try await people([(1, "Ada")]) + _ = try await driver.execute(query: "BEGIN") + let sink = sink(on: driver, clearsTargetFirst: true) + + try await sink.beginTransaction() + #expect(sink.joinsUserTransaction) + try await sink.insertRows([row("2", "Bo")]) + #expect(try await names(driver) == ["2:Bo"]) + + _ = try await driver.execute(query: "ROLLBACK") + #expect(try await names(driver) == ["1:Ada"]) + } + + @Test("Without a transaction the reused table is emptied once, before the first row") + func clearRunsBeforeTheFirstRow() async throws { + let driver = try await people([(1, "Ada")]) + let sink = sink(on: driver, clearsTargetFirst: true) + + try await sink.insertRow(row("2", "Bo")) + try await sink.insertRow(row("3", "Cy")) + + #expect(try await names(driver) == ["2:Bo", "3:Cy"]) + } + + @Test("A write after the file's own ROLLBACK stays, and the import says rows were kept, on success and on failure", + arguments: [true, false]) + func writeAfterTheFilesRollbackIsKept(commits: Bool) async throws { + let driver = try await people() + let sink = sink(on: driver) + try await sink.beginTransaction() + try await sink.execute(statement: "INSERT INTO people (id, name) VALUES (1, 'Ada')") + try await sink.execute(statement: "ROLLBACK") + try await sink.execute(statement: "INSERT INTO people (id, name) VALUES (2, 'Bo')") + + await sink.finish(commit: commits) + + #expect(try await names(driver) == ["2:Bo"]) + #expect(await sink.transactionEnding() == (commits ? .committed : .partlyCommitted)) + } + + @Test("The file's own ROLLBACK with nothing after it reports the rows as rolled back, not ended by the server") + func filesRollbackKeepsNothing() async throws { + let driver = try await people() + let sink = sink(on: driver) + try await sink.beginTransaction() + try await sink.execute(statement: "INSERT INTO people (id, name) VALUES (1, 'Ada')") + try await sink.execute(statement: "ROLLBACK") + try await sink.execute(statement: "SELECT count(*) FROM people") + + await sink.finish(commit: true) + + #expect(try await names(driver).isEmpty) + #expect(await sink.transactionEnding() == .rolledBack) + } + + @Test("Rows written with no transaction report as kept, not rolled back") + func noTransactionReportsKeptRows() async throws { + let driver = try await people() + let sink = sink(on: driver) + try await sink.insertRow(row("1", "Ada")) + #expect(await sink.transactionEnding() == .none) + } + + @Test("The expiry and stop messages say where the rows are") + func outcomeMessagesNameTheRows() { + func outcome(_ ending: DataImportOutcome.Ending, _ transaction: ImportTransactionEnding) -> DataImportOutcome { + DataImportOutcome( + ending: ending, transaction: transaction, tally: nil, executedStatements: 0, skippedStatements: 0, + errors: [], createdTable: false, catalogChanges: [] + ) + } + #expect(outcome(.expired, .leftInOpenTransaction).transactionMessage == ImportRun.joinedTransactionMessage) + #expect(outcome(.expired, .rolledBack).transactionMessage?.contains("rolled back") == true) + #expect(outcome(.completed, .committed).transactionMessage == nil) + #expect(outcome(.stopped, .none).transactionMessage != nil) + } + + @Test("Boolean words become 1 and 0 only for a flag column on an engine whose dialect writes numbers") + func numericBooleansFollowTheEngine() throws { + func written(_ typeName: String, on databaseType: DatabaseType) throws -> PluginCellValue? { + let values = ImportTargetValues( + columns: [ColumnInfo(name: "flag", typeName: typeName)], databaseType: databaseType, binaryText: nil + ) + return try values.values([.text("true")], columns: ["flag"]).first + } + #expect(try written("tinyint(1)", on: .mysql) == .text("1")) + #expect(try written("bit", on: .mssql) == .text("1")) + #expect(try written("boolean", on: .sqlite) == .text("1")) + #expect(try written("boolean", on: .postgresql) == .text("true")) + #expect(try written("BOOLEAN", on: .duckdb) == .text("true")) + #expect(!ImportTargetValues.writesNumericBooleans(.redshift)) + } + + @Test("A CSV value in a binary column is read as bytes, and text that only starts like hex is refused") + func binaryColumnReadsPrefixedHex() throws { + let values = ImportTargetValues( + columns: [ColumnInfo(name: "data", typeName: "BLOB")], databaseType: .sqlite, binaryText: .prefixedHex + ) + #expect(try values.values([.text("0x00ff")], columns: ["data"]) == [.bytes(Data([0x00, 0xFF]))]) + #expect(throws: PluginImportError.self) { + _ = try values.values([.text("0x0")], columns: ["data"]) + } + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/UI/ImportFieldMappingTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/UI/ImportFieldMappingTests.swift new file mode 100644 index 0000000000..7bdfca63af --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/UI/ImportFieldMappingTests.swift @@ -0,0 +1,78 @@ +import Foundation +@testable import TableProMobile +import Testing + +struct ImportFieldMappingTests { + private let fields = ["id", "Name", "email"] + private let columns = ["id", "name", "created_at"] + + private func mappedColumns(_ mapping: ImportFieldMapping) -> [String?] { + mapping.rows.map { $0.choice.mappedColumn } + } + + @Test("By default each field goes to the column of its name, in any case, and the rest are not imported") + func defaultsByName() { + var mapping = ImportFieldMapping() + mapping.load(fields: fields, columns: columns, saved: [:], keepingChoices: false) + + #expect(mappedColumns(mapping) == ["id", "name", nil]) + #expect(mapping.columnMapping == ["id": "id", "Name": "name"]) + #expect(mapping.showsSavedMapping == false) + #expect(mapping.overrides.isEmpty) + } + + @Test("Match by Position pairs fields and columns in order, and Match by Name puts the names back") + func byPositionThenName() { + var mapping = ImportFieldMapping() + mapping.load(fields: ["a", "b", "c", "d"], columns: columns, saved: [:], keepingChoices: false) + + mapping.matchByPosition() + #expect(mappedColumns(mapping) == ["id", "name", "created_at", nil]) + + mapping.matchByName() + #expect(mappedColumns(mapping) == [nil, nil, nil, nil]) + } + + @Test("A mapping saved for the table comes back, and only departures from the name match are saved") + func savedMappingRestores() { + var mapping = ImportFieldMapping() + mapping.load( + fields: fields, columns: columns, saved: ["email": .column("created_at"), "Name": .skip], + keepingChoices: false + ) + + #expect(mappedColumns(mapping) == ["id", nil, "created_at"]) + #expect(mapping.showsSavedMapping) + #expect(mapping.overrides == ["email": .column("created_at"), "Name": .skip]) + + mapping.matchByName() + #expect(mapping.canUseSavedMapping) + mapping.useSavedMapping() + #expect(mappedColumns(mapping) == ["id", nil, "created_at"]) + } + + @Test("Reading the same table again keeps what was chosen in the sheet") + func reloadKeepsChoices() { + var mapping = ImportFieldMapping() + mapping.load(fields: fields, columns: columns, saved: [:], keepingChoices: false) + mapping.map("Name", to: nil) + mapping.map("email", to: "created_at") + + mapping.load(fields: fields, columns: columns, saved: [:], keepingChoices: true) + #expect(mappedColumns(mapping) == ["id", nil, "created_at"]) + + mapping.load(fields: fields, columns: columns, saved: [:], keepingChoices: false) + #expect(mappedColumns(mapping) == ["id", "name", nil]) + } + + @Test("Two fields sent to one column are caught") + func oneColumnTwice() { + var mapping = ImportFieldMapping() + mapping.load(fields: fields, columns: columns, saved: [:], keepingChoices: false) + #expect(mapping.mapsOneColumnTwice == false) + + mapping.map("email", to: "id") + #expect(mapping.mapsOneColumnTwice) + #expect(mapping.hasMappedField) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Transfer/UI/MobileExportModelTests.swift b/apple/TableProMobile/TableProMobileTests/Transfer/UI/MobileExportModelTests.swift new file mode 100644 index 0000000000..4a32b10880 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Transfer/UI/MobileExportModelTests.swift @@ -0,0 +1,374 @@ +import Foundation +@testable import TableProMobile +import TableProModels +import TableProPluginKit +import Testing + +// Stands in for `ExportRun`: the test decides when the run reports and how it ends. +nonisolated final class ScriptedExportRunner: ExportRunning, @unchecked Sendable { + private let lock = NSLock() + private var continuation: CheckedContinuation? + private var pending: Result? + private var onProgress: (@Sendable (TransferProgress) -> Void)? + private var waiters: [CheckedContinuation] = [] + private(set) var cancelCount = 0 + + func run(onProgress: @escaping @Sendable (TransferProgress) -> Void) async throws -> ExportOutcome { + try await withCheckedThrowingContinuation { continuation in + let (ready, waiting) = lock.withLock { () -> (Result?, [CheckedContinuation]) in + self.onProgress = onProgress + defer { waiters = [] } + if let pending { + self.pending = nil + return (pending, waiters) + } + self.continuation = continuation + return (nil, waiters) + } + waiting.forEach { $0.resume() } + if let ready { continuation.resume(with: ready) } + } + } + + func cancel() { + lock.withLock { cancelCount += 1 } + finish(.failure(CancellationError())) + } + + func waitUntilRunning() async { + await withCheckedContinuation { waiter in + let running = lock.withLock { () -> Bool in + guard onProgress == nil else { return true } + waiters.append(waiter) + return false + } + if running { waiter.resume() } + } + } + + func report(_ progress: TransferProgress) { + let handler = lock.withLock { onProgress } + handler?(progress) + } + + func finish(_ result: Result) { + let waiting = lock.withLock { () -> CheckedContinuation? in + guard let continuation else { + pending = result + return nil + } + self.continuation = nil + return continuation + } + waiting?.resume(with: result) + } +} + +nonisolated struct ScriptedExportFailure: LocalizedError { + var errorDescription: String? { "The server went away." } +} + +@MainActor +struct MobileExportModelTests { + private func input(scopes: [ExportScopeOption], type: DatabaseType = .postgresql) -> MobileExportInput { + MobileExportInput( + title: "Export", fileName: "orders", connectionId: UUID(), databaseType: type, database: "shop", + schema: "public", safeModeLevel: .off, scopes: scopes + ) + } + + private func object(_ name: String, kind: PluginExportObjectKind) -> PluginExportTable { + PluginExportTable( + name: name, databaseName: "shop", tableType: kind.rawValue, + optionValues: MobileExportFormats.sql.defaultTableOptionValues(), schema: "public", kind: kind + ) + } + + private func environment( + runner: ScriptedExportRunner = ScriptedExportRunner(), + objects: [PluginExportTable] = [], + requests: RequestLog = RequestLog() + ) -> MobileExportEnvironment { + MobileExportEnvironment( + formats: { MobileExportFormats.available(for: $0) }, + loadObjects: { _, _ in objects }, + makeRunner: { request, _ in + requests.requests.append(request) + return runner + }, + sessionDriver: { nil } + ) + } + + private func outcomeFolder() throws -> (folder: URL, file: URL) { + let folder = FileManager.default.temporaryDirectory + .appendingPathComponent("ExportModelTests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + let file = folder.appendingPathComponent("orders.csv") + try Data("id\n1\n".utf8).write(to: file) + return (folder, file) + } + + @Test("The formats offered follow the engine, in the Mac's order", arguments: [ + (DatabaseType.postgresql, ["csv", "json", "sql", "xlsx", "md"]), + (DatabaseType.mongodb, ["csv", "json", "xlsx", "md", "mql"]), + (DatabaseType.redis, ["csv", "json", "xlsx", "md"]) + ]) + func formatsPerEngine(type: DatabaseType, expected: [String]) { + let model = MobileExportModel( + input: input(scopes: [ExportScopeOption(kind: .result, source: .query("SELECT 1"))], type: type), + environment: environment() + ) + #expect(model.formatChoices.map { $0.id } == expected) + #expect(model.formatId == expected.first) + } + + @Test("An export runs, shares, keeps the file for Share Again, and removes it when the sheet closes") + func runShareAndClose() async throws { + let runner = ScriptedExportRunner() + let requests = RequestLog() + let model = MobileExportModel( + input: input(scopes: [ExportScopeOption(kind: .result, source: .query("SELECT * FROM orders"))]), + environment: environment(runner: runner, requests: requests) + ) + #expect(model.canExport) + + model.start() + #expect(model.phase == .running(TransferProgress(stage: .preparing))) + #expect(model.canExport == false) + await runner.waitUntilRunning() + + var progress = TransferProgress(stage: .running) + progress.completedUnits = 12_400 + progress.totalUnits = 48_000 + runner.report(progress) + await ObservedCondition.wait { model.phase == .running(progress) } + + let (folder, file) = try outcomeFolder() + runner.finish(.success(ExportOutcome(folder: folder, files: [file], rowsWritten: 48_000, warnings: [], notes: []))) + await ObservedCondition.wait { model.phase == .sharing } + #expect(model.finished?.files == [file]) + #expect(model.finished?.rowsWritten == 48_000) + if case .query(let select)? = requests.requests.first?.source { + #expect(select.statement(on: RecordingTransferDriver()) == "SELECT * FROM orders") + } else { + Issue.record("The result was not exported as a query") + } + + model.shareEnded() + #expect(model.phase == .finished) + #expect(FileManager.default.fileExists(atPath: file.path)) + + model.close() + #expect(FileManager.default.fileExists(atPath: folder.path) == false) + #expect(model.finished == nil) + } + + @Test("Cancel stops the run and ends in the cancelled state") + func cancel() async { + let runner = ScriptedExportRunner() + let model = MobileExportModel( + input: input(scopes: [ExportScopeOption(kind: .result, source: .query("SELECT 1"))]), + environment: environment(runner: runner) + ) + model.start() + await runner.waitUntilRunning() + + model.cancel() + await ObservedCondition.wait { model.phase == .cancelled } + #expect(runner.cancelCount == 1) + + model.reconfigure() + #expect(model.phase == .configuring) + } + + @Test("A failed run says why, and Try Again goes back to the options") + func failure() async { + let runner = ScriptedExportRunner() + let model = MobileExportModel( + input: input(scopes: [ExportScopeOption(kind: .result, source: .query("SELECT 1"))]), + environment: environment(runner: runner) + ) + model.start() + runner.finish(.failure(ScriptedExportFailure())) + + await ObservedCondition.wait { model.phase == .failed("The server went away.") } + model.reconfigure() + #expect(model.phase == .configuring) + } + + @Test("A database export lists every object by kind in dump order, all picked, and sends only what writes something") + func databaseObjects() async { + let requests = RequestLog() + let runner = ScriptedExportRunner() + let model = MobileExportModel( + input: input(scopes: [ExportScopeOption(kind: .database, source: .database)]), + environment: environment( + runner: runner, + objects: [ + object("orders_view", kind: .view), object("orders", kind: .table), + object("order_status", kind: .userType), object("customers", kind: .table) + ], + requests: requests + ) + ) + model.selectFormat(type(of: MobileExportFormats.sql).formatId) + await model.loadObjectsIfNeeded() + + #expect(model.objectGroups.map { $0.kind } == [.userType, .table, .view]) + #expect(model.exportableObjects.count == 4) + + let typeRow = model.objectGroups[0].rows[0].id + for column in model.optionColumns.indices { + model.setOption(false, at: column, for: typeRow) + } + model.setSelected(false, kind: .view) + #expect(model.exportableObjects.map { $0.name }.sorted() == ["customers", "orders"]) + + model.start() + if case .objects(let objects)? = requests.requests.first?.source { + #expect(objects.map { $0.name }.sorted() == ["customers", "orders"]) + } else { + Issue.record("The database was not exported as objects") + } + model.cancel() + await ObservedCondition.wait { model.phase == .cancelled } + } + + @Test("A table export shows and picks only that table") + func tableObjects() async { + let model = MobileExportModel( + input: input(scopes: [ExportScopeOption(kind: .wholeTable, source: .table("orders"))]), + environment: environment(objects: [object("customers", kind: .table), object("orders", kind: .table)]) + ) + await model.loadObjectsIfNeeded() + + #expect(model.objectGroups.flatMap { $0.rows }.map { $0.object.name } == ["orders"]) + #expect(model.exportableObjects.map { $0.name } == ["orders"]) + } + + @Test("A view's export shows and picks that view, and leaves out a routine of the same name") + func viewObjects() async { + let model = MobileExportModel( + input: input(scopes: [ExportScopeOption(kind: .wholeTable, source: .table("totals"))]), + environment: environment(objects: [ + object("orders", kind: .table), object("totals", kind: .view), object("totals", kind: .routine) + ]) + ) + model.selectFormat(type(of: MobileExportFormats.sql).formatId) + await model.loadObjectsIfNeeded() + + #expect(model.objectGroups.flatMap { $0.rows }.map { $0.object.kind } == [.view]) + #expect(model.exportableObjects.map { $0.name } == ["totals"]) + #expect(model.canExport) + } + + @Test("Another format gives every object that format's default options") + func formatChangeResetsOptions() async { + let model = MobileExportModel( + input: input(scopes: [ExportScopeOption(kind: .database, source: .database)]), + environment: environment(objects: [object("orders", kind: .table)]) + ) + model.selectFormat(type(of: MobileExportFormats.sql).formatId) + await model.loadObjectsIfNeeded() + let row = model.objectGroups[0].rows[0].id + model.setOption(false, at: 0, for: row) + #expect(model.objectGroups[0].rows[0].object.optionValues == [false, true, true]) + + model.selectFormat(type(of: MobileExportFormats.csv).formatId) + #expect(model.objectGroups[0].rows[0].object.optionValues == MobileExportFormats.csv.defaultTableOptionValues()) + } + + @Test("Another format shows the kinds it writes with the selection they had, each with only the options it takes") + func formatChangeKeepsEveryKind() async throws { + let model = MobileExportModel( + input: input(scopes: [ExportScopeOption(kind: .database, source: .database)]), + environment: environment(objects: [ + object("orders", kind: .table), object("total", kind: .routine), object("audit", kind: .trigger) + ]) + ) + await model.loadObjectsIfNeeded() + #expect(model.objectGroups.map { $0.kind } == [.table]) + + model.selectFormat(type(of: MobileExportFormats.sql).formatId) + #expect(Set(model.objectGroups.map { $0.kind }) == [.table, .routine, .trigger]) + let rows = model.objectGroups.flatMap { $0.rows } + let routine = try #require(rows.first { $0.object.kind == .routine }) + #expect(routine.isSelected) + #expect(routine.object.optionValues == [true, true, false]) + #expect(rows.first { $0.object.kind == .table }?.object.optionValues == [true, true, true]) + + model.setSelected(false, for: routine.id) + model.selectFormat(type(of: MobileExportFormats.csv).formatId) + model.selectFormat(type(of: MobileExportFormats.sql).formatId) + #expect(model.objectGroups.flatMap { $0.rows }.first { $0.id == routine.id }?.isSelected == false) + } + + @Test("A function and a procedure with one name are two rows, and picking one leaves the other") + func sameNamedRoutinesAreTwoRows() async throws { + let function = PluginExportTable( + name: "total", databaseName: "shop", tableType: "function", schema: nil, kind: .routine + ) + let procedure = PluginExportTable( + name: "total", databaseName: "shop", tableType: "procedure", schema: nil, kind: .routine + ) + let model = MobileExportModel( + input: input(scopes: [ExportScopeOption(kind: .database, source: .database)]), + environment: environment(objects: [function, procedure]) + ) + model.selectFormat(type(of: MobileExportFormats.sql).formatId) + await model.loadObjectsIfNeeded() + + let rows = model.objectGroups.flatMap { $0.rows } + #expect(Set(rows.map(\.id)).count == 2) + let procedureRow = try #require(rows.first { $0.object.tableType == "procedure" }) + model.setSelected(false, for: procedureRow.id) + + #expect(model.exportableObjects.map(\.tableType) == ["function"]) + } + + @Test("Export hands the run its own copy of the format, and the live Safe Mode level") + func exportRunsWithAFrozenPlugin() async { + let plugins = PluginLog() + let requests = RequestLog() + var live = environment(requests: requests) + live.makeRunner = { request, plugin in + requests.requests.append(request) + plugins.plugins.append(plugin) + return ScriptedExportRunner() + } + live.safeModeLevel = { .readOnly } + let model = MobileExportModel( + input: input(scopes: [ExportScopeOption(kind: .result, source: .query("SELECT 1"))]), + environment: live + ) + + model.start() + + let used = plugins.plugins.first + #expect(used != nil) + #expect(used.map { $0 !== MobileExportFormats.csv } == true) + #expect(requests.requests.first?.safeModeLevel == .readOnly) + model.close() + } + + @Test("Without a file name nothing exports") + func emptyFileName() { + let model = MobileExportModel( + input: input(scopes: [ExportScopeOption(kind: .result, source: .query("SELECT 1"))]), + environment: environment() + ) + model.fileName = " " + #expect(model.canExport == false) + } +} + +@MainActor +final class RequestLog { + var requests: [ExportRequest] = [] +} + +@MainActor +final class PluginLog { + var plugins: [any ExportFormatPlugin] = [] +} diff --git a/apple/TableProMobile/TableProMobileTests/Views/RowSelectionTests.swift b/apple/TableProMobile/TableProMobileTests/Views/RowSelectionTests.swift new file mode 100644 index 0000000000..03662cc1e9 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Views/RowSelectionTests.swift @@ -0,0 +1,144 @@ +import Foundation +import SwiftUI +import TableProDatabase +@testable import TableProMobile +import TableProModels +import TableProQuery +import Testing + +struct RowSelectionTests { + @Test("Rows are picked only in Select mode, and leaving it drops them") + func selectModeOwnsThePick() { + var selection = RowSelection() + selection.toggle(1) + #expect(selection.rows.isEmpty) + #expect(selection.editMode == .inactive) + + selection.begin() + selection.toggle(1) + selection.toggle(3) + selection.toggle(1) + #expect(selection.rows == [3]) + #expect(selection.editMode == .active) + + selection.end() + #expect(selection.isActive == false) + #expect(selection.rows.isEmpty) + } + + @Test("Outside Select mode no row reads as picked and a row opens its detail; in it a tap picks the row") + func pickStateFollowsSelectMode() { + var selection = RowSelection() + #expect(selection.pick(of: 0) == nil) + #expect(selection.opensRowDetail) + + selection.begin() + selection.toggle(0) + #expect(selection.pick(of: 0) == true) + #expect(selection.pick(of: 1) == false) + #expect(selection.opensRowDetail == false) + + selection.end() + #expect(selection.pick(of: 0) == nil) + #expect(selection.opensRowDetail) + } + + @Test("Select All picks every loaded row, and says so only while all are picked") + func selectAll() { + var selection = RowSelection() + selection.begin() + selection.selectAll(count: 4) + #expect(selection.count == 4) + #expect(selection.selectsAll(of: 4)) + + selection.toggle(2) + #expect(selection.selectsAll(of: 4) == false) + #expect(selection.selectsAll(of: 0) == false) + } + + @Test("The picked rows come back in page order, without rows the page no longer holds") + func indicesWithinThePage() { + var selection = RowSelection() + selection.begin() + selection.replace(with: [7, 2, 0, 12]) + #expect(selection.indices(within: 8) == [0, 2, 7]) + } +} + +nonisolated enum PageChange: CaseIterable, Sendable { + case nextPage + case reload + case filter + case sort + case search + case memoryTrim +} + +@MainActor +struct RowSelectionClearingTests { + private let columns = [ + ColumnInfo(name: "id", typeName: "INT", isPrimaryKey: true, isNullable: false, ordinalPosition: 0), + ColumnInfo(name: "name", typeName: "VARCHAR(64)", ordinalPosition: 1) + ] + + private func page(_ viewModel: DataBrowserViewModel, from start: Int = 0) -> Result { + let rows = (start.. (DataBrowserViewModel, MockDatabaseDriver) { + let driver = MockDatabaseDriver() + driver.scriptedColumns = columns + let viewModel = DataBrowserViewModel() + driver.scriptedExecuteResults = [ + page(viewModel), + .success(QueryResult(columns: [], rows: [["5000"]], rowsAffected: 0, executionTime: 0)) + ] + viewModel.attach( + session: ConnectionSession(connectionId: UUID(), driver: driver, activeDatabase: "test", tables: []), + table: TableInfo(name: "users"), databaseType: .mysql, host: "localhost" + ) + await viewModel.load(isInitial: true) + viewModel.selection.begin() + viewModel.selection.selectAll(count: viewModel.window.count) + return (viewModel, driver) + } + + @Test("Anything that replaces the page clears the picked rows and stays in Select mode", arguments: PageChange.allCases) + func pageChangeClearsThePick(_ change: PageChange) async { + let (viewModel, driver) = await browserWithEveryRowPicked() + #expect(viewModel.selection.count == viewModel.pagination.pageSize) + + driver.scriptedExecuteResults = [page(viewModel)] + switch change { + case .nextPage: + await viewModel.goToNextPage() + case .reload: + await viewModel.load() + case .filter: + viewModel.filters = [TableFilter(columnName: "name", value: "name 1")] + await viewModel.applyFilters() + case .sort: + viewModel.sortState.columns = [SortColumn(name: "name", ascending: false)] + await viewModel.applySort() + case .search: + await viewModel.applySearch("name") + case .memoryTrim: + await viewModel.handlePressure(.warning) + } + + #expect(viewModel.selection.rows.isEmpty) + #expect(viewModel.selection.isActive) + } + + @Test("A new run or a cleared query leaves Select mode in the query results") + func queryResultsLeaveSelectMode() { + let viewModel = QueryEditorViewModel() + viewModel.selection.begin() + viewModel.selection.selectAll(count: 3) + + viewModel.reset() + #expect(viewModel.selection.isActive == false) + #expect(viewModel.selection.rows.isEmpty) + } +} diff --git a/apple/TableProMobile/project.yml b/apple/TableProMobile/project.yml index da9cb32d25..bfb88ea509 100644 --- a/apple/TableProMobile/project.yml +++ b/apple/TableProMobile/project.yml @@ -82,6 +82,8 @@ targets: - ../Plugins/Shared/PluginQueryTimeout.swift # The toolbar glyph rule both apps follow on and before the 26 releases. - ../TablePro/Core/Services/Infrastructure/Toolbar/ToolbarSymbols.swift + # Animations that follow Reduce Motion, the same in both apps. + - ../TablePro/Theme/MotionAccessibility.swift # The PostHog client and what it may collect, the same in both apps. - ../TablePro/Core/Services/Telemetry/PostHogTelemetryClient.swift # The ClickHouse plugin driver runs whole behind PluginBackedDriver, so every file of the plugin is compiled in. @@ -102,7 +104,7 @@ targets: - MongoDBPlugin.swift - CLibMongoc/** # The column default reading both apps show and the version floor that says which catalog - # quotes its defaults. The DDL writers stay on macOS. + # quotes its defaults. - ../Plugins/MySQLDriverPlugin/MySQLCatalogDefault.swift - ../Plugins/MySQLDriverPlugin/MySQLGeneratedColumnClassification.swift - ../Plugins/MySQLDriverPlugin/MySQLServerVersion.swift @@ -119,8 +121,107 @@ targets: - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLObjectQueries.swift # libpq COPY handling the iOS PostgreSQL driver shares with the macOS plugin. - ../Plugins/PostgreSQLDriverPlugin/LibPQCopyState.swift + - ../Plugins/PostgreSQLDriverPlugin/LibPQCopyIn.swift # The read-write BEGIN both platforms open a write transaction with. - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLTransactionStatement.swift + # Data export and import: the format plugins both apps run, without the macOS option panes. + - path: ../Plugins/CSVExportPlugin + excludes: + - Info.plist + - "*OptionsView.swift" + - path: ../Plugins/JSONExportPlugin + excludes: + - Info.plist + - "*OptionsView.swift" + - path: ../Plugins/SQLExportPlugin + excludes: + - Info.plist + - "*OptionsView.swift" + - path: ../Plugins/XLSXExportPlugin + excludes: + - Info.plist + - "*OptionsView.swift" + - path: ../Plugins/MarkdownExportPlugin + excludes: + - Info.plist + - "*OptionsView.swift" + - path: ../Plugins/MQLExportPlugin + excludes: + - Info.plist + - "*OptionsView.swift" + - path: ../Plugins/CSVImportPlugin + excludes: + - Info.plist + - "*OptionsView.swift" + - path: ../Plugins/JSONImportPlugin + excludes: + - Info.plist + - "*OptionsView.swift" + - path: ../Plugins/XLSXImportPlugin + excludes: + - Info.plist + - "*OptionsView.swift" + - path: ../Plugins/SQLImportPlugin + excludes: + - Info.plist + - "*OptionsView.swift" + # The import write path and script text both apps share. + - ../TablePro/Core/Transfer/Shared + # Each engine's export catalog, CREATE TABLE writer and binding helpers, shared with the macOS plugin. + - ../Plugins/MySQLDriverPlugin/MySQLExportCatalog.swift + - ../Plugins/MySQLDriverPlugin/MySQLSessionScopePlan.swift + - ../Plugins/MySQLDriverPlugin/MySQLForeignKeyCatalog.swift + - ../Plugins/MySQLDriverPlugin/MySQLPrivilegeCatalog.swift + - ../Plugins/MySQLDriverPlugin/MySQLCreateTableSQL.swift + - ../Plugins/MySQLDriverPlugin/MySQLColumnDefinitionSQL.swift + - ../Plugins/SQLiteDriverPlugin/SQLiteCatalogReads.swift + - ../Plugins/SQLiteDriverPlugin/SQLiteCreateTableDDL.swift + - ../Plugins/SQLiteDriverPlugin/SQLiteTemporarySchema.swift + - ../Plugins/SQLiteDriverPlugin/SQLiteCheckConstraintParser.swift + - ../Plugins/SQLiteDriverPlugin/SQLiteDefaultValue.swift + - ../Plugins/DuckDBDriverPlugin/DuckDBExportCatalog.swift + - ../Plugins/DuckDBDriverPlugin/DuckDBCreateTableSQL.swift + - ../Plugins/DuckDBDriverPlugin/DuckDBSchemaQueries.swift + - ../Plugins/DuckDBDriverPlugin/DuckDBIndexClauses.swift + - ../Plugins/DuckDBDriverPlugin/DuckDBLexicalFeatures.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLCatalogExport.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLCreateTableSQL.swift + - ../Plugins/PostgreSQLDriverPlugin/RedshiftCatalogExport.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLSchemaQueries.swift + - ../Plugins/PostgreSQLDriverPlugin/ColumnQueryShape.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLSequenceQueries.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLSequenceReference.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLCommentStatements.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLRelationSQL.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLViewDefinition.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLForeignKeyQueries.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLTypeDefinition.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLColumnTypeSpelling.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLSessionScopeSQL.swift + - ../Plugins/PostgreSQLDriverPlugin/LibPQCellDecoding.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLPrincipalQueries.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLVersionedStatements.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLColumnClauses.swift + - ../Plugins/PostgreSQLDriverPlugin/PostgreSQLIndexClauses.swift + - ../Plugins/PostgreSQLDriverPlugin/LibPQParameterBuffers.swift + - ../Plugins/PostgreSQLDriverPlugin/RedshiftSchemaQueries.swift + - ../Plugins/PostgreSQLDriverPlugin/RedshiftExternalSchemaQueries.swift + - ../Plugins/MSSQLDriverPlugin/MSSQLExportCatalog.swift + - ../Plugins/MSSQLDriverPlugin/MSSQLObjectQueries.swift + - ../Plugins/MSSQLDriverPlugin/MSSQLTableDefinitionSQL.swift + - ../Plugins/OracleDriverPlugin/OracleExportCatalog.swift + - ../Plugins/OracleDriverPlugin/OracleStatementError.swift + - ../Plugins/OracleDriverPlugin/OraclePluginValues.swift + - ../Plugins/OracleDriverPlugin/OracleCreateTableSQL.swift + - ../Plugins/OracleDriverPlugin/OracleObjectQueries.swift + - ../Plugins/OracleDriverPlugin/OracleIndexStatements.swift + - ../Plugins/OracleDriverPlugin/OracleTableDDL.swift + - ../Plugins/OracleDriverPlugin/OracleTypeCatalog.swift + - ../Plugins/OracleDriverPlugin/OracleBindPlaceholders.swift + - ../Plugins/OracleDriverPlugin/OracleRowWriter.swift + - ../Plugins/OracleDriverPlugin/OracleNationalText.swift + - ../Plugins/OracleDriverPlugin/OracleColumnStatements.swift # The libssh2 half of the SSH transport, shared with the macOS app by file reference. # The C-free half is the TableProSSHTransport package target; these touch libssh2 # and cannot be, because a manifest cannot express the slice-dependent header path the @@ -155,10 +256,13 @@ targets: - TableProConnectionLibrary - TableProCoreTypes - TableProDatabase + - TableProDuckDBCore - TableProGeometry - TableProGoogleCloud + - TableProGzip - TableProHTTP - TableProImport + - TableProJavaScriptText - TableProLogRedaction - TableProModels - TableProMSSQLCore @@ -168,6 +272,7 @@ targets: - TableProPostgreSQLCore - TableProQuery - TableProRedisCore + - TableProSQLFile - TableProSQLGrammar - TableProSQLQuoting - TableProSQLVocabulary diff --git a/apple/TableProTests/Core/Export/DataSourceExportTests.swift b/apple/TableProTests/Core/Export/DataSourceExportTests.swift index 5f8f572dbf..9020c0a8c1 100644 --- a/apple/TableProTests/Core/Export/DataSourceExportTests.swift +++ b/apple/TableProTests/Core/Export/DataSourceExportTests.swift @@ -131,17 +131,17 @@ struct DataSourceExportTests { @Test("A supported list wins over an excluded list") func supportedListWins() { - #expect(ExportFormatCatalog.accepts( + #expect(ExportFormatOffer.accepts( databaseTypeId: "MongoDB", supportedDatabaseTypeIds: ["MongoDB"], excludedDatabaseTypeIds: ["MongoDB"] )) - #expect(!ExportFormatCatalog.accepts( + #expect(!ExportFormatOffer.accepts( databaseTypeId: "DataFile", supportedDatabaseTypeIds: ["MongoDB"], excludedDatabaseTypeIds: [] )) - #expect(ExportFormatCatalog.accepts( + #expect(ExportFormatOffer.accepts( databaseTypeId: "DataFile", supportedDatabaseTypeIds: [], excludedDatabaseTypeIds: ["MongoDB", "Redis"] @@ -170,6 +170,18 @@ struct DataSourceExportTests { #expect(dataSource.escapeStringLiteral("O'Brien\\n") == "O''Brien\\n") } + @Test("Held rows pass a SQL export's snapshot BEGIN and COMMIT as no-ops and refuse any other statement") + func heldRowsPassSnapshotStatements() async throws { + let dataSource = QueryResultExportDataSource(detachedRows: tableRows(), databaseTypeId: "PostgreSQL") + for statement in ["BEGIN", "START TRANSACTION READ ONLY", "COMMIT", "/* snapshot */ ROLLBACK"] { + let result = try await dataSource.execute(query: statement) + #expect(result.rows.isEmpty) + } + await #expect(throws: (any Error).self) { + _ = try await dataSource.execute(query: "SELECT 1") + } + } + @Test("A detached data source streams every row, or only the rows it was given in that order") func detachedDataSourceStreamsRowSubset() async throws { let all = QueryResultExportDataSource(detachedRows: tableRows(), databaseTypeId: "DataFile") diff --git a/apple/TableProTests/Core/Export/ExportFormatCatalogTests.swift b/apple/TableProTests/Core/Export/ExportFormatCatalogTests.swift index 594518434d..bdd2c42c3c 100644 --- a/apple/TableProTests/Core/Export/ExportFormatCatalogTests.swift +++ b/apple/TableProTests/Core/Export/ExportFormatCatalogTests.swift @@ -87,7 +87,7 @@ struct ExportFormatCatalogTests { /// unknown format at the end of the list and showed no description at all. @Test("Parquet is ordered and described like the formats that shipped before it") func parquetIsCurated() { - let sorted = ExportFormatCatalog.sorted([StubParquet(), StubCSV(), StubSQL()]) + let sorted = ExportFormatOffer.sorted([StubParquet(), StubCSV(), StubSQL()]) #expect(ids(sorted) == ["csv", "sql", "parquet"]) let description = ExportFormatCatalog.description(for: StubParquet()) @@ -99,7 +99,7 @@ struct ExportFormatCatalogTests { /// land somewhere stable rather than tying with every other unknown one. @Test("An unknown format sorts after the curated ones, by display name") func unknownFormatsSortLast() { - let sorted = ExportFormatCatalog.sorted([StubZebra(), StubAardvark(), StubSQL()]) + let sorted = ExportFormatOffer.sorted([StubZebra(), StubAardvark(), StubSQL()]) #expect(ids(sorted) == ["sql", "aardvark", "zebra"]) } @@ -117,9 +117,9 @@ struct ExportFormatCatalogTests { @Test("Sorting does not depend on the order it is handed") func sortIsOrderIndependent() { let forwards = ids( - ExportFormatCatalog.sorted([StubCSV(), StubSQL(), StubParquet(), StubZebra()])) + ExportFormatOffer.sorted([StubCSV(), StubSQL(), StubParquet(), StubZebra()])) let backwards = ids( - ExportFormatCatalog.sorted([StubZebra(), StubParquet(), StubSQL(), StubCSV()])) + ExportFormatOffer.sorted([StubZebra(), StubParquet(), StubSQL(), StubCSV()])) #expect(forwards == backwards) #expect(forwards == ["csv", "sql", "parquet", "zebra"]) } diff --git a/apple/TableProTests/Core/Export/ExportRoutineKindTests.swift b/apple/TableProTests/Core/Export/ExportRoutineKindTests.swift new file mode 100644 index 0000000000..d2738a5202 --- /dev/null +++ b/apple/TableProTests/Core/Export/ExportRoutineKindTests.swift @@ -0,0 +1,162 @@ +// +// ExportRoutineKindTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct DropRoutineCall: Equatable { + let name: String + let signature: String? + let isFunction: Bool +} + +private final class SameNamedRoutinesDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var calls: [DropRoutineCall] = [] + let spellsDrop: Bool + + init(spellsDrop: Bool = true) { + self.spellsDrop = spellsDrop + } + + var dropCalls: [DropRoutineCall] { lock.withLock { calls } } + + func fetchRoutines(schema: String?) async throws -> [PluginRoutineInfo] { + [ + PluginRoutineInfo(name: "f", kind: .function, schema: schema, argumentSignature: "()"), + PluginRoutineInfo(name: "f", kind: .procedure, schema: schema, argumentSignature: "()"), + ] + } + + func fetchRoutineDDL(_ routine: PluginRoutineInfo) async throws -> String { + "CREATE \(routine.kind.rawValue.uppercased()) f()" + } + + func generateDropRoutineSQL(name: String, signature: String?, schema: String?, isFunction: Bool) -> String? { + lock.withLock { calls.append(DropRoutineCall(name: name, signature: signature, isFunction: isFunction)) } + guard spellsDrop else { return nil } + return "DROP \(isFunction ? "FUNCTION" : "PROCEDURE") IF EXISTS \(name)" + } +} + +private final class TableRecordingFormat: ExportFormatPlugin, @unchecked Sendable { + static let pluginName = "Table Recorder" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Records the objects it is handed" + static let formatId = "table-recorder" + static let formatDisplayName = "Table Recorder" + static let defaultFileExtension = "txt" + static let iconName = "doc" + + private(set) var receivedTables: [PluginExportTable] = [] + + required init() {} + + func export( + tables: [PluginExportTable], + dataSource: any PluginExportDataSource, + destination: URL, + progress: PluginExportProgress + ) async throws -> ExportFormatResult { + receivedTables = tables + try Data().write(to: destination) + return ExportFormatResult() + } +} + +struct ExportRoutineKindTests { + private func pluginAdapter(_ stub: SameNamedRoutinesDriver) -> PluginDriverAdapter { + PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .mysql), pluginDriver: stub) + } + + private func source(_ stub: SameNamedRoutinesDriver) -> ExportDataSourceAdapter { + ExportDataSourceAdapter(driver: pluginAdapter(stub), databaseType: .mysql) + } + + private func routine(tableType: String) -> PluginExportTable { + PluginExportTable( + name: "f", databaseName: "app", tableType: tableType, schema: "app", kind: .routine, identity: "()" + ) + } + + @Test("The loader keeps each routine's own kind") + func loaderKeepsTheKind() async { + let items = await ExportObjectLoader.loadObjects( + request: .init(containerName: "app", schema: "app", kinds: [.routine]), + tables: [], + driver: pluginAdapter(SameNamedRoutinesDriver()) + ) + + #expect(items.map(\.kind) == [.routine, .routine]) + #expect(items.map(\.routineKind) == [.function, .procedure]) + #expect(items.map(\.identity) == ["()", "()"]) + } + + @Test("A procedure is dropped as a procedure and a function as a function") + func dropAsksForTheRoutinesKind() { + let stub = SameNamedRoutinesDriver() + let adapter = source(stub) + + #expect(adapter.dropStatement(for: routine(tableType: "procedure")) == "DROP PROCEDURE IF EXISTS f") + #expect(adapter.dropStatement(for: routine(tableType: "function")) == "DROP FUNCTION IF EXISTS f") + #expect(stub.dropCalls == [ + DropRoutineCall(name: "f", signature: "()", isFunction: false), + DropRoutineCall(name: "f", signature: "()", isFunction: true), + ]) + } + + @Test("A routine whose kind was not carried gets no drop, and the driver is not asked to guess") + func unknownKindWritesNoDrop() { + let stub = SameNamedRoutinesDriver() + + #expect(source(stub).dropStatement(for: routine(tableType: PluginExportObjectKind.routine.rawValue)) == nil) + #expect(stub.dropCalls.isEmpty) + } + + @Test("A driver that cannot spell the drop leaves it out rather than writing DROP ROUTINE") + func driverWithoutDropWritesNone() { + let adapter = source(SameNamedRoutinesDriver(spellsDrop: false)) + let statements = ["procedure", "function", "routine"].map { adapter.dropStatement(for: routine(tableType: $0)) } + + #expect(statements == [nil, nil, nil]) + } + + @Test("A function and a procedure that share a name and signature each export their own definition") + func definitionMatchesTheKind() async throws { + let adapter = source(SameNamedRoutinesDriver()) + + #expect(try await adapter.fetchObjectDDL(routine(tableType: "procedure")) == "CREATE PROCEDURE f()") + #expect(try await adapter.fetchObjectDDL(routine(tableType: "function")) == "CREATE FUNCTION f()") + } + + @Test("The export service hands the format a routine's own kind") + @MainActor + func serviceCarriesTheKind() async throws { + let format = TableRecordingFormat() + let driver = MockDatabaseDriver(connection: TestFixtures.makeConnection(type: .postgresql)) + let service = ExportService(driver: driver, databaseType: .postgresql) { requested in + requested == TableRecordingFormat.formatId ? format : nil + } + var config = ExportConfiguration() + config.formatId = TableRecordingFormat.formatId + let url = FileManager.default.temporaryDirectory.appendingPathComponent("\(UUID().uuidString).txt") + defer { try? FileManager.default.removeItem(at: url) } + + try await service.export( + objects: [ + ExportObjectItem(name: "p_proc", kind: .routine, identity: "(integer)", routineKind: .procedure), + ExportObjectItem(name: "f_func", kind: .routine, identity: "(integer)", routineKind: .function), + ExportObjectItem(name: "orders", kind: .table), + ], + config: config, + to: url + ) + + #expect(format.receivedTables.map(\.tableType) == ["procedure", "function", "table"]) + #expect(format.receivedTables.map(\.kind) == [.routine, .routine, .table]) + } +} diff --git a/apple/TableProTests/Core/Export/ExportTriggerListingTests.swift b/apple/TableProTests/Core/Export/ExportTriggerListingTests.swift new file mode 100644 index 0000000000..06ff83d799 --- /dev/null +++ b/apple/TableProTests/Core/Export/ExportTriggerListingTests.swift @@ -0,0 +1,68 @@ +// +// ExportTriggerListingTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +// Shaped like Oracle: a schema-wide list it answers, with the Compare flag left false on purpose. +private final class SchemaTriggerListDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var schemas: [String?] = [] + + var triggerListReads: [String?] { lock.withLock { schemas } } + + func fetchAllTriggers(schema: String?) async throws -> [PluginTriggerInfo] { + lock.withLock { schemas.append(schema) } + return [ + PluginTriggerInfo(name: "ORDERS_BI", table: "ORDERS", timing: "BEFORE", event: "INSERT", statement: "BEGIN NULL; END;") + ] + } +} + +private final class NoTriggerListDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var queries: [String] = [] + + var executedQueries: [String] { lock.withLock { queries } } + + func execute(query: String) async throws -> PluginQueryResult { + lock.withLock { queries.append(query) } + return .empty + } +} + +struct ExportTriggerListingTests { + private func load(_ driver: any PluginDatabaseDriver) async -> [ExportObjectItem] { + await ExportObjectLoader.loadObjects( + request: .init(containerName: "SALES", schema: "SALES", kinds: [.table, .trigger]), + tables: [TableInfo(name: "ORDERS", type: .table, rowCount: nil)], + driver: PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .oracle), pluginDriver: driver) + ) + } + + @Test("Triggers reach the export from a driver whose Compare flag stays false") + func triggersListedWithoutTheCompareFlag() async { + let driver = SchemaTriggerListDriver() + let items = await load(driver) + + #expect(!driver.providesBulkTriggerFetch) + #expect(driver.triggerListReads == ["SALES"]) + let trigger = items.first { $0.kind == .trigger } + #expect(trigger?.name == "ORDERS_BI") + #expect(trigger?.parentTable == "ORDERS") + #expect(items.contains { $0.name == "ORDERS" && $0.kind == .table }) + } + + @Test("A driver with no schema-wide trigger list adds no triggers and runs no query") + func driverWithoutTriggerListReadsNothing() async { + let driver = NoTriggerListDriver() + let items = await load(driver) + + #expect(items.map(\.kind) == [.table]) + #expect(driver.executedQueries.isEmpty) + } +} diff --git a/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterRelationDropTests.swift b/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterRelationDropTests.swift new file mode 100644 index 0000000000..d7309a208f --- /dev/null +++ b/apple/TableProTests/Core/Plugins/ExportDataSourceAdapterRelationDropTests.swift @@ -0,0 +1,65 @@ +// +// ExportDataSourceAdapterRelationDropTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct RelationDropCall: Equatable { + let name: String + let kind: PluginExportObjectKind + let schema: String? +} + +private final class RelationDropDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var calls: [RelationDropCall] = [] + + var dropCalls: [RelationDropCall] { lock.withLock { calls } } + + func generateDropRelationSQL(name: String, kind: PluginExportObjectKind, schema: String?) -> String? { + lock.withLock { calls.append(RelationDropCall(name: name, kind: kind, schema: schema)) } + return "GUARDED DROP \(kind.rawValue) \(name)" + } +} + +private final class GenericDropDriver: PluginDriverStub, @unchecked Sendable {} + +struct ExportDataSourceAdapterRelationDropTests { + private func source(_ pluginDriver: any PluginDatabaseDriver) -> ExportDataSourceAdapter { + let driver = PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .oracle), pluginDriver: pluginDriver) + return ExportDataSourceAdapter(driver: driver, databaseType: .oracle) + } + + private func object(_ name: String, kind: PluginExportObjectKind) -> PluginExportTable { + PluginExportTable(name: name, databaseName: "APP", tableType: kind.rawValue, schema: "APP", kind: kind) + } + + @Test("A table, view or materialized view drops the way the driver spells it", arguments: [ + PluginExportObjectKind.table, .view, .materializedView, + ]) + func driverSpellsTheRelationDrop(kind: PluginExportObjectKind) { + let driver = RelationDropDriver() + + #expect(source(driver).dropStatement(for: object("ORDERS", kind: kind)) == "GUARDED DROP \(kind.rawValue) ORDERS") + #expect(driver.dropCalls == [RelationDropCall(name: "ORDERS", kind: kind, schema: "APP")]) + } + + @Test("A driver with no relation drop leaves the export its own", arguments: [ + PluginExportObjectKind.table, .view, .materializedView, + ]) + func driverWithoutRelationDropFallsBack(kind: PluginExportObjectKind) { + #expect(source(GenericDropDriver()).dropStatement(for: object("ORDERS", kind: kind)) == nil) + } + + @Test("Another kind never asks for a relation drop") + func otherKindsAreNotAsked() { + let driver = RelationDropDriver() + + #expect(source(driver).dropStatement(for: object("SEQ", kind: .sequence)) == nil) + #expect(driver.dropCalls.isEmpty) + } +} diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterCopyTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterCopyTests.swift index 2df5b46f5f..839ad0e75b 100644 --- a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterCopyTests.swift +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterCopyTests.swift @@ -7,6 +7,7 @@ import Foundation import os @testable import TablePro import TableProPluginKit +import TableProSQLFile import TableProTestSupport import Testing diff --git a/apple/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift b/apple/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift index dd89bb8dab..8ac76cf51f 100644 --- a/apple/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift +++ b/apple/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift @@ -6,6 +6,7 @@ import Foundation @testable import TablePro import TableProPluginKit +import TableProSQLFile import TableProSQLGrammar import TableProTabularIO import Testing diff --git a/apple/TableProTests/Core/Services/Export/RowImportWritabilityTests.swift b/apple/TableProTests/Core/Services/Export/RowImportWritabilityTests.swift new file mode 100644 index 0000000000..e4c3fb1f48 --- /dev/null +++ b/apple/TableProTests/Core/Services/Export/RowImportWritabilityTests.swift @@ -0,0 +1,108 @@ +// +// RowImportWritabilityTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class RowFormatStub: ImportFormatPlugin, @unchecked Sendable { + static let pluginName = "Row Format" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Writes rows into a table" + static let formatId = "row-format" + static let formatDisplayName = "Rows" + static let acceptedFileExtensions = ["rows"] + static let iconName = "doc" + static let requiresTargetTable = true + + required init() {} + + func performImport( + source: any PluginImportSource, + sink: any PluginImportDataSink, + progress: PluginImportProgress + ) async throws -> PluginImportResult { + PluginImportResult(executedStatements: 0, executionTime: 0) + } +} + +private final class StatementFormatStub: ImportFormatPlugin, @unchecked Sendable { + static let pluginName = "Statement Format" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Runs statements" + static let formatId = "statement-format" + static let formatDisplayName = "Statements" + static let acceptedFileExtensions = ["statements"] + static let iconName = "doc" + + required init() {} + + func performImport( + source: any PluginImportSource, + sink: any PluginImportDataSink, + progress: PluginImportProgress + ) async throws -> PluginImportResult { + PluginImportResult(executedStatements: 0, executionTime: 0) + } +} + +@MainActor +struct RowImportWritabilityTests { + private static let offersCSVAndSQL = ImportFormatLookup( + supportsImport: { _ in true }, + offeredFormats: { _ in + [ImportFormatOption(id: "csv", name: "CSV"), ImportFormatOption(id: "sql", name: "SQL")] + }, + requiresTargetTable: { $0 == "csv" } + ) + + private func check(_ formatId: String, into databaseType: DatabaseType) -> RowImportRefusal? { + RowImportEligibility.refusal( + formatId: formatId, + databaseType: databaseType, + connectionName: "Catalog", + safeModeLevel: .silent, + isPresentingSheet: false, + lookup: Self.offersCSVAndSQL + ) + } + + @Test("A row file into MongoDB is refused before it is read, with a reason the user can act on") + func mongoDBRefusesRowFiles() { + let refusal = check("csv", into: .mongodb) + + #expect(refusal == .rowsNotWritable(.mongodb)) + #expect(refusal?.localizedDescription == String( + format: String(localized: "%@ cannot take rows from a file."), "MongoDB" + )) + } + + @Test("An engine the sink can write rows into is not refused") + func sqlEngineTakesRowFiles() { + #expect(check("csv", into: .postgresql) == nil) + } + + @Test("A statement format is judged by its offer, not by whether rows can be written") + func statementFormatSkipsTheRowCheck() { + #expect(check("sql", into: .mongodb) == .formatNotOffered(formatId: "sql")) + } + + @Test("The refusal asks the same question the sink asks before its first batch") + func refusalFollowsTheSink() { + #expect(!ImportDataSinkAdapter.canWriteRows(into: .mongodb)) + #expect(ImportDataSinkAdapter.canWriteRows(into: .postgresql)) + } + + @Test("A row format is not offered to an engine the sink cannot write rows into") + func rowFormatIsNotOfferedWhereRowsCannotBeWritten() { + let mongoWritesRows = ImportDataSinkAdapter.canWriteRows(into: .mongodb) + let postgresWritesRows = ImportDataSinkAdapter.canWriteRows(into: .postgresql) + + #expect(!PluginManager.offers(RowFormatStub.self, to: .mongodb, writesRows: mongoWritesRows)) + #expect(PluginManager.offers(RowFormatStub.self, to: .postgresql, writesRows: postgresWritesRows)) + #expect(PluginManager.offers(StatementFormatStub.self, to: .mongodb, writesRows: mongoWritesRows)) + } +} diff --git a/apple/TableProTests/Core/Transfer/ColumnTypeTraitsTests.swift b/apple/TableProTests/Core/Transfer/ColumnTypeTraitsTests.swift new file mode 100644 index 0000000000..a49083f90e --- /dev/null +++ b/apple/TableProTests/Core/Transfer/ColumnTypeTraitsTests.swift @@ -0,0 +1,57 @@ +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +struct ColumnTypeTraitsTests { + // One or more ids of every `SQLTypeFamily`, and engines that family leaves generic. + private static let databaseTypeIds = [ + "MySQL", "MariaDB", "TiDB", "OceanBase", "StarRocks", + "PostgreSQL", "Redshift", "CockroachDB", "PGlite", "KingbaseES", "Greenplum", + "SQLite", "libSQL", "Cloudflare D1", "SQL Server", "Oracle", "Dameng", "ClickHouse", "DuckDB", + "MongoDB", "Snowflake", "BigQuery" + ] + + private static let typeNames = [ + "", " bit", "bit ", "BLOB", "blob", "BLOB SUB_TYPE 1", "BLOBTEXT", "BLOBINT", "TIMEBLOB", "tinyblob", "MEDIUMBLOB", + "longblob", "bytea", "bytea[]", "BYTES", "bytes", "Bytes", "VARBINARY(16)", "varbinary(max)", "binary(16)", + "BINARY_DOUBLE", "RAW(16)", "LONG RAW", "long raw", "IMAGE", "Nullable(String)", "LowCardinality(Nullable(String))", + "Nullable(Bool)", "boolean", "BOOL", "_bool", "boolean[]", "ARRAY", "STRUCT", "Array(Bool)", + "MAP(String, Bool)", "tinyint(1)", "TINYINT(1) UNSIGNED", "tinyint( 1 )", "tinyint(4)", "TINYINT", "bit", "BIT", + "bit(1)", "bit(8)", "BIT(64)", "bit varying(5)", "bit(3) varying", "varbit", "bit(1)[]", "BITSTRING", + "INT(10) UNSIGNED ZEROFILL", "ENUM[](mood)", "varchar(255)", "text", "json", "uuid", "VARIANT", "geometry" + ] + + @Test("The import reads bytes and flags from a type name as the classifier does", arguments: databaseTypeIds) + func matchesTheClassifier(_ databaseTypeId: String) { + let classifier = ColumnTypeClassifier(family: SQLTypeFamily.of(DatabaseType(rawValue: databaseTypeId))) + for name in Self.typeNames { + let type = classifier.classify(rawTypeName: name) + #expect(ColumnTypeTraits.holdsBytes(name) == type.isBlobType, "\(name) on \(databaseTypeId)") + #expect( + ColumnTypeTraits.isBooleanFlag(name, databaseTypeId: databaseTypeId) == type.isBooleanType, + "\(name) on \(databaseTypeId)" + ) + } + } + + @Test("A boolean word becomes a digit only for a flag column on a dialect that writes numbers") + func numericBooleansNeedBoth() { + typealias Traits = ImportTargetValues.ColumnTraits + #expect(Traits(typeName: "tinyint(1)", databaseTypeId: "MySQL", writesNumericBooleans: true).takesNumericBooleans) + #expect(!Traits(typeName: "tinyint(1)", databaseTypeId: "MySQL", writesNumericBooleans: false).takesNumericBooleans) + #expect(!Traits(typeName: "bit(8)", databaseTypeId: "MySQL", writesNumericBooleans: true).takesNumericBooleans) + #expect(!Traits(typeName: "bit", databaseTypeId: "PostgreSQL", writesNumericBooleans: true).takesNumericBooleans) + #expect(Traits(typeName: "LONG RAW", databaseTypeId: "Oracle", writesNumericBooleans: true).holdsBytes) + } + + @Test("Text in a binary column is bytes only as 0x and an even run of hex digits") + func prefixedHexReadsStrictly() { + typealias PrefixedHex = ImportTargetValues.PrefixedHex + #expect(PrefixedHex(reading: "0x00ff") == .bytes(Data([0x00, 0xFF]))) + #expect(PrefixedHex(reading: "0X") == .bytes(Data())) + #expect(PrefixedHex(reading: "0x0") == .malformed) + #expect(PrefixedHex(reading: "0xzz") == .malformed) + #expect(PrefixedHex(reading: "plain") == .notHex) + } +} diff --git a/apple/TableProTests/Core/Transfer/ImportRowWriterTests.swift b/apple/TableProTests/Core/Transfer/ImportRowWriterTests.swift new file mode 100644 index 0000000000..13537fc4ba --- /dev/null +++ b/apple/TableProTests/Core/Transfer/ImportRowWriterTests.swift @@ -0,0 +1,150 @@ +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +// Answers a probe from how many rows hold each key, read from the probe's own parameters in order. +private final class RecordingExecutor: @unchecked Sendable { + private(set) var statements: [SQLImportStatement] = [] + private(set) var purposes: [ImportRowWriter.Purpose] = [] + private let rowsPerKey: [String: Int] + + init(rowsPerKey: [String: Int] = [:]) { + self.rowsPerKey = rowsPerKey + } + + var host: ImportRowWriter.Host { + ImportRowWriter.Host( + execute: { statement, purpose in + self.statements.append(statement) + self.purposes.append(purpose) + guard purpose == .matchProbe else { return [] } + return statement.parameters.enumerated().map { position, key -> [String?] in + [String(position), String(self.rowsPerKey[key.asText ?? ""] ?? 0)] + } + }, + columnsUpdateCannotSet: { [] }, + isCancelled: { false } + ) + } + + var sql: [String] { statements.map(\.sql) } +} + +struct ImportRowWriterTests { + private var target: ImportRowWriter.Target { + ImportRowWriter.Target( + tableName: "people", + statements: SQLImportStatements( + databaseTypeId: "PostgreSQL", + qualifiedTableName: "\"people\"", + parameterStyle: .dollar, + quoteIdentifier: { "\"\($0)\"" } + ), + values: ImportTargetValues( + columns: [], + binaryText: nil, + prefixedHex: { _ in .notHex }, + traits: { _ in ImportTargetValues.ColumnTraits(holdsBytes: false, takesNumericBooleans: false) } + ) + ) + } + + private let rows: [[String: PluginCellValue]] = [ + ["id": "1", "name": "Ada"], + ["id": "2", "name": "Grace"], + ] + + private func makeWriter(_ mode: ImportWriteMode) -> ImportRowWriter { + ImportRowWriter( + databaseTypeId: "PostgreSQL", + columnMapping: ["id": "id", "name": "name"], + sourceFields: ["id", "name"], + rowWriting: ImportRowWriting(mode: mode, keyColumns: mode == .insert ? [] : ["id"]) + ) + } + + private let probe = """ + SELECT 0, COUNT(*) FROM "people" WHERE "id" = $1 UNION ALL SELECT 1, COUNT(*) FROM "people" WHERE "id" = $2 + """ + + @Test("Insert writes the rows in one multi-row statement") + func insertWritesOneBatch() async throws { + let executor = RecordingExecutor() + let writer = makeWriter(.insert) + try await writer.insertRows(rows, into: target, host: executor.host) + #expect(executor.sql == ["INSERT INTO \"people\" (\"id\", \"name\") VALUES ($1, $2), ($3, $4)"]) + #expect(executor.purposes == [.write(rows: 2)]) + #expect(executor.statements.first?.parameters == ["1", "Ada", "2", "Grace"]) + #expect(writer.tally.inserted == 2) + } + + @Test("Update writes the matched row and leaves out the one that matches nothing") + func updateSkipsUnmatched() async throws { + let executor = RecordingExecutor(rowsPerKey: ["1": 1]) + let writer = makeWriter(.update) + try await writer.insertRows(rows, into: target, host: executor.host) + #expect(executor.sql == [probe, "UPDATE \"people\" SET \"name\" = $1 WHERE \"id\" = $2"]) + #expect(executor.purposes == [.matchProbe, .write(rows: 1)]) + #expect(writer.tally.updated == 1) + #expect(writer.tally.unmatched == 1) + #expect(writer.tally.inserted == 0) + } + + @Test("Insert or Update updates the matched row and inserts the other, in file order") + func upsertUpdatesThenInserts() async throws { + let executor = RecordingExecutor(rowsPerKey: ["1": 1]) + let writer = makeWriter(.upsert) + try await writer.insertRows(rows, into: target, host: executor.host) + #expect(executor.sql == [ + probe, + "UPDATE \"people\" SET \"name\" = $1 WHERE \"id\" = $2", + "INSERT INTO \"people\" (\"id\", \"name\") VALUES ($1, $2)", + ]) + #expect(writer.tally.updated == 1) + #expect(writer.tally.inserted == 1) + } + + @Test("Insert, Skip Existing inserts only the row the table does not hold") + func skipExistingInsertsOnlyNewRows() async throws { + let executor = RecordingExecutor(rowsPerKey: ["1": 1]) + let writer = makeWriter(.insertSkippingExisting) + try await writer.insertRows(rows, into: target, host: executor.host) + #expect(executor.sql == [probe, "INSERT INTO \"people\" (\"id\", \"name\") VALUES ($1, $2)"]) + #expect(executor.statements.last?.parameters == ["2", "Grace"]) + #expect(writer.tally.existing == 1) + #expect(writer.tally.inserted == 1) + } + + @Test("A key that matches two rows fails the batch before anything is written") + func keyMatchingTwoRowsFails() async throws { + let executor = RecordingExecutor(rowsPerKey: ["1": 2]) + let writer = makeWriter(.update) + do { + try await writer.insertRows(rows, into: target, host: executor.host) + Issue.record("The import went on past an ambiguous key") + } catch PluginImportError.importFailed(let message) { + #expect(message == "The key id = 1 matched 2 rows, so the row was not written.") + } + #expect(executor.purposes == [.matchProbe]) + #expect(writer.tally == ImportRowTally(mode: .update)) + } + + @Test("A key repeated in one batch is probed again after its first row is written") + func repeatedKeySplitsTheRun() async throws { + let executor = RecordingExecutor() + let writer = makeWriter(.upsert) + let repeated: [[String: PluginCellValue]] = [["id": "1", "name": "Ada"], ["id": "1", "name": "Ada L."]] + try await writer.insertRows(repeated, into: target, host: executor.host) + #expect(executor.purposes == [.matchProbe, .write(rows: 1), .matchProbe, .write(rows: 1)]) + #expect(writer.tally.inserted == 2) + } + + @Test("Clearing the table first is refused when rows are matched") + func clearingRefusedWhenMatching() throws { + #expect(try makeWriter(.insert).clearingStatement(using: target.statements) == "DELETE FROM \"people\"") + #expect(throws: PluginImportError.self) { + try makeWriter(.upsert).clearingStatement(using: target.statements) + } + } +} diff --git a/apple/TableProTests/Core/Transfer/SQLImportStatementsTests.swift b/apple/TableProTests/Core/Transfer/SQLImportStatementsTests.swift new file mode 100644 index 0000000000..ef336d4131 --- /dev/null +++ b/apple/TableProTests/Core/Transfer/SQLImportStatementsTests.swift @@ -0,0 +1,122 @@ +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +struct SQLImportStatementsTests { + private func statements( + _ databaseTypeId: String = "MySQL", + style: ParameterStyle = .questionMark, + typeNames: [String: String] = [:], + bitColumns: Set = [], + rowDelete: ((_ table: String, _ predicate: String?) -> String?)? = nil + ) -> SQLImportStatements { + SQLImportStatements( + databaseTypeId: databaseTypeId, + qualifiedTableName: "\"t\"", + parameterStyle: style, + quoteIdentifier: { "\"\($0)\"" }, + columnTypeNames: typeNames, + numberBitColumns: bitColumns, + rowDeleteStatement: rowDelete + ) + } + + @Test("Question-mark placeholders repeat, dollar placeholders count across every row") + func placeholderStyles() throws { + let rows: [[PluginCellValue]] = [["1", "a"], ["2", "b"]] + let question = try #require(statements().insertStatement(columns: ["id", "name"], rows: rows)) + #expect(question.sql == "INSERT INTO \"t\" (\"id\", \"name\") VALUES (?, ?), (?, ?)") + let dollar = try #require( + statements("PostgreSQL", style: .dollar).insertStatement(columns: ["id", "name"], rows: rows) + ) + #expect(dollar.sql == "INSERT INTO \"t\" (\"id\", \"name\") VALUES ($1, $2), ($3, $4)") + #expect(dollar.parameters == ["1", "a", "2", "b"]) + #expect(dollar.parameterTypeNames == nil) + } + + @Test("A keyed update numbers its SET and WHERE placeholders as one list") + func keyedUpdateNumbersAcrossClauses() throws { + let update = try #require(statements("PostgreSQL", style: .dollar).keyedUpdateStatement( + columns: ["name"], values: ["Ada"], keyColumns: ["tenant", "id"], keyValues: ["t1", "7"] + )) + #expect(update.sql == "UPDATE \"t\" SET \"name\" = $1 WHERE \"tenant\" = $2 AND \"id\" = $3") + #expect(update.parameters == ["Ada", "t1", "7"]) + } + + @Test("The match probe tags each key with its position and binds a NULL key part as NULL") + func matchProbe() throws { + let probe = try #require(statements().keyMatchCountStatement(keyColumns: ["id"], keys: [["1"], [.null]])) + #expect(probe.sql == """ + SELECT 0, COUNT(*) FROM "t" WHERE "id" = ? UNION ALL SELECT 1, COUNT(*) FROM "t" WHERE "id" = ? + """) + #expect(probe.parameters == ["1", .null]) + } + + @Test("A MySQL BIT column takes decimal text through CAST(? AS UNSIGNED), with no column type on the operand") + func mysqlBitColumn() throws { + let insert = try #require( + statements(typeNames: ["flags": "bit(8)", "name": "varchar(10)"], bitColumns: ["flags"]) + .insertStatement(columns: ["flags", "name"], values: ["b'101'", "x"]) + ) + #expect(insert.sql == "INSERT INTO \"t\" (\"flags\", \"name\") VALUES (CAST(? AS UNSIGNED), ?)") + #expect(insert.parameters == ["5", "x"]) + #expect(insert.parameterTypeNames == [nil, "varchar(10)"]) + } + + @Test("Blank text for a BIT column goes as empty text bound to the column") + func mysqlBitBlank() throws { + let insert = try #require( + statements(typeNames: ["flags": "bit(8)"], bitColumns: ["flags"]) + .insertStatement(columns: ["flags"], values: [" "]) + ) + #expect(insert.sql == "INSERT INTO \"t\" (\"flags\") VALUES (?)") + #expect(insert.parameters == [""]) + #expect(insert.parameterTypeNames == ["bit(8)"]) + } + + @Test("Bit literals and boolean words read as decimal text") + func decimalBitText() { + #expect(SQLImportStatements.decimalBitText("0b11") == "3") + #expect(SQLImportStatements.decimalBitText("TRUE") == "1") + #expect(SQLImportStatements.decimalBitText("f") == "0") + #expect(SQLImportStatements.decimalBitText("12") == "12") + } + + @Test("Oracle writes one row per INSERT, so its batches hold one row") + func oracleOneRowCeiling() { + let oracle = statements("Oracle") + #expect(oracle.maxRowsPerInsert == 1) + #expect(SQLWriteBatchBudget(columnCount: 2, statements: oracle).maxRows == 1) + } + + @Test("Cloudflare D1 binds 100 parameters, so a batch holds 100 over the row width") + func cloudflareD1BindCeiling() { + let d1 = statements("Cloudflare D1") + #expect(d1.maxBindParameters == 100) + #expect(SQLWriteBatchBudget(columnCount: 3, statements: d1).maxRows == 33) + } + + @Test("A host's own ceilings and literal rendering replace the type id's") + func hostOverrides() { + let inlined = SQLImportStatements( + databaseTypeId: "MySQL", + qualifiedTableName: "`t`", + parameterStyle: .questionMark, + quoteIdentifier: { "`\($0)`" }, + maxBindParameters: 10, + maxRowsPerInsert: 4, + rendersValuesAsLiterals: true + ) + let budget = SQLWriteBatchBudget(columnCount: 2, statements: inlined) + #expect(budget.maxRows == 4) + #expect(budget.rendersValuesAsLiterals) + } + + @Test("Clearing the table uses the host's row-delete form when it has one") + func deleteAllRows() { + #expect(statements().deleteAllRowsStatement() == "DELETE FROM \"t\"") + let truncating = statements(rowDelete: { table, _ in "TRUNCATE TABLE \(table)" }) + #expect(truncating.deleteAllRowsStatement() == "TRUNCATE TABLE \"t\"") + } +} diff --git a/apple/TableProTests/Core/Transfer/TransferEngineIdTests.swift b/apple/TableProTests/Core/Transfer/TransferEngineIdTests.swift new file mode 100644 index 0000000000..2fa7fd35fb --- /dev/null +++ b/apple/TableProTests/Core/Transfer/TransferEngineIdTests.swift @@ -0,0 +1,116 @@ +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +// The shared write path is keyed by the type id exactly as both apps spell it; a lower-cased or respelled id falls to +// the generic answers without a word. +struct TransferEngineIdTests { + struct Engine: Sendable, CustomTestStringConvertible { + let databaseType: DatabaseType + let id: String + let maxBindParameters: Int + let maxRowsPerInsert: Int + let allDefaults: String? + let integerType: String + let booleanType: String + let rendersValuesAsLiterals: Bool + + var testDescription: String { id } + } + + static let iosEngines: [Engine] = [ + Engine( + databaseType: .mysql, id: "MySQL", maxBindParameters: 65_535, maxRowsPerInsert: .max, + allDefaults: "INSERT INTO t () VALUES ()", integerType: "BIGINT", booleanType: "TINYINT(1)", + rendersValuesAsLiterals: false + ), + Engine( + databaseType: .mariadb, id: "MariaDB", maxBindParameters: 65_535, maxRowsPerInsert: .max, + allDefaults: "INSERT INTO t () VALUES ()", integerType: "BIGINT", booleanType: "TINYINT(1)", + rendersValuesAsLiterals: false + ), + Engine( + databaseType: .tidb, id: "TiDB", maxBindParameters: 65_535, maxRowsPerInsert: .max, + allDefaults: "INSERT INTO t () VALUES ()", integerType: "BIGINT", booleanType: "TINYINT(1)", + rendersValuesAsLiterals: false + ), + Engine( + databaseType: .oceanbase, id: "OceanBase", maxBindParameters: 65_535, maxRowsPerInsert: .max, + allDefaults: "INSERT INTO t () VALUES ()", integerType: "BIGINT", booleanType: "TINYINT(1)", + rendersValuesAsLiterals: false + ), + Engine( + databaseType: .postgresql, id: "PostgreSQL", maxBindParameters: 65_535, maxRowsPerInsert: .max, + allDefaults: "INSERT INTO t DEFAULT VALUES", integerType: "BIGINT", booleanType: "BOOLEAN", + rendersValuesAsLiterals: false + ), + Engine( + databaseType: .redshift, id: "Redshift", maxBindParameters: 65_535, maxRowsPerInsert: .max, + allDefaults: "INSERT INTO t DEFAULT VALUES", integerType: "BIGINT", booleanType: "BOOLEAN", + rendersValuesAsLiterals: false + ), + Engine( + databaseType: .sqlite, id: "SQLite", maxBindParameters: 32_766, maxRowsPerInsert: .max, + allDefaults: "INSERT INTO t DEFAULT VALUES", integerType: "INTEGER", booleanType: "BOOLEAN", + rendersValuesAsLiterals: false + ), + Engine( + databaseType: .duckdb, id: "DuckDB", maxBindParameters: 65_535, maxRowsPerInsert: .max, + allDefaults: "INSERT INTO t DEFAULT VALUES", integerType: "INTEGER", booleanType: "BOOLEAN", + rendersValuesAsLiterals: false + ), + Engine( + databaseType: .mssql, id: "SQL Server", maxBindParameters: 2_100, maxRowsPerInsert: 1_000, + allDefaults: "INSERT INTO t DEFAULT VALUES", integerType: "BIGINT", booleanType: "BIT", + rendersValuesAsLiterals: false + ), + Engine( + databaseType: .oracle, id: "Oracle", maxBindParameters: 65_535, maxRowsPerInsert: 1, + allDefaults: "INSERT INTO t (c) VALUES (DEFAULT)", integerType: "NUMBER(19)", + booleanType: "VARCHAR2(5 CHAR)", rendersValuesAsLiterals: false + ), + Engine( + databaseType: .clickhouse, id: "ClickHouse", maxBindParameters: 65_535, maxRowsPerInsert: .max, + allDefaults: nil, integerType: "INTEGER", booleanType: "BOOLEAN", rendersValuesAsLiterals: true + ), + Engine( + databaseType: .cloudflareD1, id: "Cloudflare D1", maxBindParameters: 100, maxRowsPerInsert: .max, + allDefaults: "INSERT INTO t DEFAULT VALUES", integerType: "INTEGER", booleanType: "BOOLEAN", + rendersValuesAsLiterals: false + ), + ] + + @Test( + "Each engine the iOS app imports into gets its own answers from its type id", + arguments: TransferEngineIdTests.iosEngines + ) + func engineAnswers(engine: Engine) { + #expect(engine.databaseType.rawValue == engine.id) + + let statements = SQLImportStatements( + databaseTypeId: engine.id, qualifiedTableName: "t", parameterStyle: .questionMark, quoteIdentifier: { $0 } + ) + #expect(statements.maxBindParameters == engine.maxBindParameters) + #expect(statements.maxRowsPerInsert == engine.maxRowsPerInsert) + #expect(statements.rendersValuesAsLiterals == engine.rendersValuesAsLiterals) + #expect(statements.allDefaultsInsertStatement(firstColumn: "c")?.sql == engine.allDefaults) + #expect(ImportTypeMapper.sqlType(for: .integer, databaseTypeId: engine.id) == engine.integerType) + #expect(ImportTypeMapper.sqlType(for: .boolean, databaseTypeId: engine.id) == engine.booleanType) + } + + @Test("A lower-cased type id gets the generic answers") + func lowerCasedIdIsGeneric() { + let statements = SQLImportStatements( + databaseTypeId: "sql server", qualifiedTableName: "t", parameterStyle: .questionMark, quoteIdentifier: { $0 } + ) + #expect(statements.maxBindParameters == 65_535) + #expect(ImportTypeMapper.sqlType(for: .boolean, databaseTypeId: "sql server") == "BOOLEAN") + } + + @Test("Only PostgreSQL itself reads its JSON column type from the server version") + func postgresJSONColumnTypeById() { + #expect(ImportTypeMapper.postgresJSONColumnType(databaseTypeId: "PostgreSQL", serverVersion: "9.3.25") == .json) + #expect(ImportTypeMapper.postgresJSONColumnType(databaseTypeId: "Redshift", serverVersion: "8.0.2") == .jsonb) + } +} diff --git a/apple/TableProTests/Core/Utilities/FileTextLoaderTests.swift b/apple/TableProTests/Core/Utilities/FileTextLoaderTests.swift index b099822899..5b7f59f8a5 100644 --- a/apple/TableProTests/Core/Utilities/FileTextLoaderTests.swift +++ b/apple/TableProTests/Core/Utilities/FileTextLoaderTests.swift @@ -5,6 +5,7 @@ import Foundation @testable import TablePro +import TableProTabularIO @testable import TableProUtilities import Testing diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserPLSQLTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserPLSQLTests.swift index 1210ccc017..2db1a55aca 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserPLSQLTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserPLSQLTests.swift @@ -10,6 +10,7 @@ import Foundation @testable import TablePro import TableProPluginKit +import TableProSQLFile import TableProSQLGrammar import Testing diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLScriptTextTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLScriptTextTests.swift index f48a21d31b..10eac609dc 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLScriptTextTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLScriptTextTests.swift @@ -10,6 +10,7 @@ import Foundation @testable import TablePro import TableProPluginKit +import TableProSQLFile import TableProSQLGrammar import Testing diff --git a/apple/TableProTests/Models/FileTabBaselineTests.swift b/apple/TableProTests/Models/FileTabBaselineTests.swift index b5b50a7bbe..b057c422d1 100644 --- a/apple/TableProTests/Models/FileTabBaselineTests.swift +++ b/apple/TableProTests/Models/FileTabBaselineTests.swift @@ -8,6 +8,7 @@ import Foundation @testable import TablePro +import TableProTabularIO @testable import TableProUtilities import Testing diff --git a/apple/TableProTests/Models/Query/TabQueryContentEqualityTests.swift b/apple/TableProTests/Models/Query/TabQueryContentEqualityTests.swift index b9263d55fa..9c0857c3b2 100644 --- a/apple/TableProTests/Models/Query/TabQueryContentEqualityTests.swift +++ b/apple/TableProTests/Models/Query/TabQueryContentEqualityTests.swift @@ -1,5 +1,6 @@ import Foundation @testable import TablePro +import TableProTabularIO @testable import TableProUtilities import Testing diff --git a/apple/TableProTests/Views/Export/ExportTreeBuilderTests.swift b/apple/TableProTests/Views/Export/ExportTreeBuilderTests.swift index 96d014b5b5..51ffccfd29 100644 --- a/apple/TableProTests/Views/Export/ExportTreeBuilderTests.swift +++ b/apple/TableProTests/Views/Export/ExportTreeBuilderTests.swift @@ -351,6 +351,25 @@ struct ExportTreeBuilderTests { #expect(loaded.map(\.name) == ["purge"]) } + @Test("A routine's own kind survives the rebuild the tree makes of every row") + func routineKindSurvivesTheTreeBuild() async throws { + let reader = Self.threeSchemaReader() + reader.sideObjectsBySchema = ["archive": [ + ExportObjectItem(name: "purge", databaseName: "archive", kind: .routine, routineKind: .procedure) + ]] + let builder = Self.builder( + typeId: Self.schemaTypeId, + preselection: .containers([.database("app")]), + reader: reader, + supportedObjectKinds: Self.sqlKinds + ) + + let items = try await builder.build(priorRows: [:]) + let purge = items.first { $0.name == "archive" }?.objects.first { $0.name == "purge" } + + #expect(purge?.routineKind == .procedure) + } + @Test("Exporting a whole database reads every kind of every schema in it") func wholeDatabaseExportReadsEveryKind() async throws { let reader = Self.threeSchemaReader() diff --git a/apple/changelog.d/4302.added.1.md b/apple/changelog.d/4302.added.1.md new file mode 100644 index 0000000000..8774435700 --- /dev/null +++ b/apple/changelog.d/4302.added.1.md @@ -0,0 +1 @@ +Import CSV, JSON and XLSX files into a table, and run SQL files, on iPhone and iPad. diff --git a/apple/changelog.d/4302.added.2.md b/apple/changelog.d/4302.added.2.md new file mode 100644 index 0000000000..33611a6de5 --- /dev/null +++ b/apple/changelog.d/4302.added.2.md @@ -0,0 +1 @@ +Select several rows on iPhone and iPad to export, copy or delete them. diff --git a/apple/changelog.d/4302.added.md b/apple/changelog.d/4302.added.md new file mode 100644 index 0000000000..b28b023b33 --- /dev/null +++ b/apple/changelog.d/4302.added.md @@ -0,0 +1 @@ +Export tables, query results, selected rows or a whole database to a file on iPhone and iPad. diff --git a/apple/changelog.d/4302.fixed.1.md b/apple/changelog.d/4302.fixed.1.md new file mode 100644 index 0000000000..71c28e5e35 --- /dev/null +++ b/apple/changelog.d/4302.fixed.1.md @@ -0,0 +1 @@ +XLSX export dropped leading and trailing spaces in text cells. diff --git a/apple/changelog.d/4302.fixed.10.md b/apple/changelog.d/4302.fixed.10.md new file mode 100644 index 0000000000..28db38bce4 --- /dev/null +++ b/apple/changelog.d/4302.fixed.10.md @@ -0,0 +1 @@ +A small SQL file whose last characters were Latin-1 was read as UTF-8. diff --git a/apple/changelog.d/4302.fixed.11.md b/apple/changelog.d/4302.fixed.11.md new file mode 100644 index 0000000000..9d9e9f9f15 --- /dev/null +++ b/apple/changelog.d/4302.fixed.11.md @@ -0,0 +1 @@ +An XLSX file with one huge shared string could use up memory while it was previewed. diff --git a/apple/changelog.d/4302.fixed.12.md b/apple/changelog.d/4302.fixed.12.md new file mode 100644 index 0000000000..6ae7fcc421 --- /dev/null +++ b/apple/changelog.d/4302.fixed.12.md @@ -0,0 +1 @@ +Copy as SQL INSERT on iPhone and iPad quoted MySQL BIT values and listed generated columns. diff --git a/apple/changelog.d/4302.fixed.13.md b/apple/changelog.d/4302.fixed.13.md new file mode 100644 index 0000000000..35afe462a3 --- /dev/null +++ b/apple/changelog.d/4302.fixed.13.md @@ -0,0 +1 @@ +PostgreSQL booleans showed as t and f on iPhone and iPad. diff --git a/apple/changelog.d/4302.fixed.14.md b/apple/changelog.d/4302.fixed.14.md new file mode 100644 index 0000000000..ebab50b3d2 --- /dev/null +++ b/apple/changelog.d/4302.fixed.14.md @@ -0,0 +1 @@ +Inserting a row of only defaults on Oracle did nothing on iPhone and iPad. diff --git a/apple/changelog.d/4302.fixed.2.md b/apple/changelog.d/4302.fixed.2.md new file mode 100644 index 0000000000..d60418c2b3 --- /dev/null +++ b/apple/changelog.d/4302.fixed.2.md @@ -0,0 +1 @@ +Stopping an XLSX or Markdown export after its last row still saved the file. diff --git a/apple/changelog.d/4302.fixed.3.md b/apple/changelog.d/4302.fixed.3.md new file mode 100644 index 0000000000..f4f831b5ef --- /dev/null +++ b/apple/changelog.d/4302.fixed.3.md @@ -0,0 +1 @@ +SQL exports with Drop wrote DROP ROUTINE, which no database accepts, and dropped PostgreSQL procedures as functions. diff --git a/apple/changelog.d/4302.fixed.4.md b/apple/changelog.d/4302.fixed.4.md new file mode 100644 index 0000000000..9cdebf3179 --- /dev/null +++ b/apple/changelog.d/4302.fixed.4.md @@ -0,0 +1 @@ +Oracle triggers were left out of SQL exports, and Oracle drops failed to restore before 23ai. diff --git a/apple/changelog.d/4302.fixed.5.md b/apple/changelog.d/4302.fixed.5.md new file mode 100644 index 0000000000..11e7d44725 --- /dev/null +++ b/apple/changelog.d/4302.fixed.5.md @@ -0,0 +1 @@ +SQL exports wrote wrong trigger or routine drops for Cassandra, Teradata, SQL Server aggregates and DuckDB macros. diff --git a/apple/changelog.d/4302.fixed.6.md b/apple/changelog.d/4302.fixed.6.md new file mode 100644 index 0000000000..6f06ee58ed --- /dev/null +++ b/apple/changelog.d/4302.fixed.6.md @@ -0,0 +1 @@ +CSV, JSON and XLSX import was offered for MongoDB and then failed. diff --git a/apple/changelog.d/4302.fixed.7.md b/apple/changelog.d/4302.fixed.7.md new file mode 100644 index 0000000000..5dac5d24f7 --- /dev/null +++ b/apple/changelog.d/4302.fixed.7.md @@ -0,0 +1 @@ +SQL exports from two windows at once could leave out sequences and mix up their warnings. diff --git a/apple/changelog.d/4302.fixed.8.md b/apple/changelog.d/4302.fixed.8.md new file mode 100644 index 0000000000..b3fe04e86d --- /dev/null +++ b/apple/changelog.d/4302.fixed.8.md @@ -0,0 +1 @@ +A SQL export of query results failed with Read every table at one snapshot turned on. diff --git a/apple/changelog.d/4302.fixed.9.md b/apple/changelog.d/4302.fixed.9.md new file mode 100644 index 0000000000..0c0869c148 --- /dev/null +++ b/apple/changelog.d/4302.fixed.9.md @@ -0,0 +1 @@ +SQLite SQL exports dropped NUL characters inside text. diff --git a/apple/changelog.d/4302.fixed.md b/apple/changelog.d/4302.fixed.md new file mode 100644 index 0000000000..252e7f0962 --- /dev/null +++ b/apple/changelog.d/4302.fixed.md @@ -0,0 +1 @@ +Markdown export cut emoji and dropped accents in aligned columns. diff --git a/apple/project.yml b/apple/project.yml index 482c603ffd..379fad71f9 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -421,7 +421,7 @@ targets: - package: Yams - package: PostHog - package: TableProCore - products: [TableProAutocomplete, TableProConnectionLibrary, TableProCoreTypes, TableProEtcdCore, TableProFirebaseCore, TableProGeometry, TableProGoogleCloud, TableProGzip, TableProHTTP, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProMemcachedCore, TableProNumberFormatting, TableProProcess, TableProQdrantCore, TableProRedisCore, TableProSQLGrammar, TableProSQLQuoting, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTLSClientIdentity, TableProTabular, TableProTabularIO, TableProTheme, TableProWeaviateCore] + products: [TableProAutocomplete, TableProConnectionLibrary, TableProCoreTypes, TableProEtcdCore, TableProFirebaseCore, TableProGeometry, TableProGoogleCloud, TableProGzip, TableProHTTP, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProMemcachedCore, TableProNumberFormatting, TableProProcess, TableProQdrantCore, TableProRedisCore, TableProSQLFile, TableProSQLGrammar, TableProSQLQuoting, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTLSClientIdentity, TableProTabular, TableProTabularIO, TableProTheme, TableProWeaviateCore] - package: TableProMacCore products: [TableProCloudAPI, TableProMCP, TableProMCPClients, TableProSSHConfig, TableProUtilities] # Writes nothing for a version CHANGELOG.md has no section for, so a 0.0.0-dev build ships no notes. @@ -745,7 +745,7 @@ targets: principalClass: SQLExportPlugin dependencies: - package: TableProCore - product: TableProProcess + product: TableProGzip settings: base: PRODUCT_BUNDLE_IDENTIFIER: com.TablePro.SQLExportPlugin @@ -846,7 +846,7 @@ targets: - TablePro/Core/RemoteSQLite/RemoteSQLiteWire.swift dependencies: - package: TableProCore - products: [TableProProcess, TableProSQLGrammar, TableProTestSupport] + products: [TableProGzip, TableProSQLGrammar, TableProTestSupport] ClickHouseDriverTests: templates: [PluginTests, ClickHouseLinkage] @@ -898,7 +898,7 @@ targets: dependencies: - target: TableProPluginKit - package: TableProCore - products: [TableProJavaScriptText, TableProNumberFormatting, TableProProcess, TableProTabularIO] + products: [TableProJavaScriptText, TableProGzip, TableProNumberFormatting, TableProTabularIO] settings: base: GENERATE_INFOPLIST_FILE: YES diff --git a/apple/scripts/probes/check-mssql-merge-terminator.sh b/apple/scripts/probes/check-mssql-merge-terminator.sh index 9927118f98..f28c74c864 100755 --- a/apple/scripts/probes/check-mssql-merge-terminator.sh +++ b/apple/scripts/probes/check-mssql-merge-terminator.sh @@ -9,8 +9,8 @@ # MERGE, and where one does not is the server's decision: a MERGE inside parentheses, a MERGE JOIN hint, and a name # such as @merge or x$merge need none, while a MERGE after a name and a space, or on a table named range, does. # -# So this builds a harness from the real Plugins/MSSQLDriverPlugin sources, the TableProCore and TableProMacCore -# packages, the app's TablePro/Core/Utilities/SQL/SQLFileParser.swift and the shipped Libs/libsybdb.a. For each +# So this builds a harness from the real Plugins/MSSQLDriverPlugin sources, the TableProCore package with its SQL +# file parser in TableProSQLFile, and the shipped Libs/libsybdb.a. For each # editor text it takes what the scanner sends, as one batch cut from the first statement to the last the way the # app's batch planner cuts it, statement by statement, and what the import parser reads from the text saved as a # file, and runs each through MSSQLPluginDriver. Each must run and leave the rows expected. For a MERGE it also sends @@ -78,10 +78,9 @@ fi mkdir -p "$WORK/Sources/Check" ln -s "$ROOT/Plugins/MSSQLDriverPlugin/CFreeTDS" "$WORK/CFreeTDS" -for source in "$ROOT"/Plugins/MSSQLDriverPlugin/*.swift "$ROOT"/TablePro/Core/Utilities/SQL/SQLFileParser.swift; do +for source in "$ROOT"/Plugins/MSSQLDriverPlugin/*.swift; do ln -s "$source" "$WORK/Sources/Check/$(basename "$source")" done -echo 'enum DecompressionError: Error { case fileReadFailed(String) }' > "$WORK/Sources/Check/DecompressionError.swift" cat > "$WORK/Package.swift" << MANIFEST // swift-tools-version: 6.0 @@ -92,7 +91,6 @@ let package = Package( platforms: [.macOS(.v14)], dependencies: [ .package(path: "$ROOT/Packages/TableProCore"), - .package(path: "$ROOT/Packages/TableProMacCore"), ], targets: [ .systemLibrary(name: "CFreeTDS", path: "CFreeTDS"), @@ -105,7 +103,7 @@ let package = Package( .product(name: "TableProMSSQLCore", package: "TableProCore"), .product(name: "TableProLogRedaction", package: "TableProCore"), .product(name: "TableProSQLGrammar", package: "TableProCore"), - .product(name: "TableProUtilities", package: "TableProMacCore"), + .product(name: "TableProSQLFile", package: "TableProCore"), ], path: "Sources/Check", swiftSettings: [.swiftLanguageMode(.v6)], @@ -122,6 +120,7 @@ cat > "$WORK/Sources/Check/Check.swift" << 'SWIFT' import Foundation import TableProCoreTypes import TableProPluginKit +import TableProSQLFile import TableProSQLGrammar @main diff --git a/apple/scripts/probes/check-mssql-sql-import.sh b/apple/scripts/probes/check-mssql-sql-import.sh index 3ea3628180..3e66f86350 100755 --- a/apple/scripts/probes/check-mssql-sql-import.sh +++ b/apple/scripts/probes/check-mssql-sql-import.sh @@ -11,9 +11,8 @@ # runs none of it. None of that shows in a unit test, because each case depends on what the server does with the text # it is sent. # -# So this builds a harness from the real TablePro/Core/Utilities/SQL/SQLFileParser.swift, the TableProUtilities module it -# reads lines and chunks with, the real Plugins/MSSQLDriverPlugin sources, the TableProCore package and the shipped -# Libs/libsybdb.a. Each script is read by SQLFileParser with the SQL Server grammar and every batch it hands out goes +# So this builds a harness from the real SQL file parser in TableProSQLFile, the real Plugins/MSSQLDriverPlugin +# sources, the TableProCore package and the shipped Libs/libsybdb.a. Each script is read by SQLFileParser with the SQL Server grammar and every batch it hands out goes # through MSSQLPluginDriver.executeBatch, as ImportDataSinkAdapter sends it: the first batch that raises an error stops # the import, with the server's line placed in the file. Then the database is read back. # @@ -78,7 +77,7 @@ fi mkdir -p "$WORK/Sources/Check" ln -s "$ROOT/Plugins/MSSQLDriverPlugin/CFreeTDS" "$WORK/CFreeTDS" -for source in "$ROOT"/Plugins/MSSQLDriverPlugin/*.swift "$ROOT/TablePro/Core/Utilities/SQL/SQLFileParser.swift"; do +for source in "$ROOT"/Plugins/MSSQLDriverPlugin/*.swift; do ln -s "$source" "$WORK/Sources/Check/$(basename "$source")" done @@ -91,7 +90,6 @@ let package = Package( platforms: [.macOS(.v14)], dependencies: [ .package(path: "$ROOT/Packages/TableProCore"), - .package(path: "$ROOT/Packages/TableProMacCore"), ], targets: [ .systemLibrary(name: "CFreeTDS", path: "CFreeTDS"), @@ -104,7 +102,7 @@ let package = Package( .product(name: "TableProMSSQLCore", package: "TableProCore"), .product(name: "TableProLogRedaction", package: "TableProCore"), .product(name: "TableProSQLGrammar", package: "TableProCore"), - .product(name: "TableProUtilities", package: "TableProMacCore"), + .product(name: "TableProSQLFile", package: "TableProCore"), ], path: "Sources/Check", swiftSettings: [.swiftLanguageMode(.v6)], @@ -121,14 +119,9 @@ cat > "$WORK/Sources/Check/Check.swift" << 'SWIFT' import Foundation import TableProCoreTypes import TableProPluginKit +import TableProSQLFile import TableProSQLGrammar -/// The one type SQLFileParser needs from the app beyond the files linked in beside it. -enum DecompressionError: Error { - case decompressFailed - case fileReadFailed(String) -} - @main enum Check { nonisolated(unsafe) static var failures = 0