diff --git a/apple/Configs/PluginKit.xcconfig b/apple/Configs/PluginKit.xcconfig index 5a97fb9713..fef1b5c3ce 100644 --- a/apple/Configs/PluginKit.xcconfig +++ b/apple/Configs/PluginKit.xcconfig @@ -1,3 +1,3 @@ // When to raise each, and how the build stamps them: .claude/rules/plugin-system.md, ABI. -TABLEPRO_PLUGINKIT_VERSION = 42 +TABLEPRO_PLUGINKIT_VERSION = 43 TABLEPRO_PLUGINKIT_MINIMUM_VERSION = 19 diff --git a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLSocketKeepalive.swift b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLSocketKeepalive.swift new file mode 100644 index 0000000000..1ff480a9ea --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLSocketKeepalive.swift @@ -0,0 +1,24 @@ +import Foundation + +// Left to the kernel, Darwin sends the first keepalive after 7,200 s of idle, while an AWS Network Load Balancer drops +// an idle flow after 350 s. A session with no query timeout has no read timeout either, so a reply lost with the flow, +// or on a path that died, blocked the statement for hours, and Stop's KILL QUERY could not reach the blocked read. +// These keep the flow open and fail a dead peer about 90 s after the last traffic, the values the PostgreSQL plugin +// gives libpq. A slow statement on a live server still completes: its kernel answers the probes. +public enum MySQLSocketKeepalive { + public static let idleSeconds: Int32 = 60 + public static let intervalSeconds: Int32 = 10 + public static let probeCount: Int32 = 3 + + public static func enable(on socket: Int32) -> Bool { + set(socket, SOL_SOCKET, SO_KEEPALIVE, 1) + && set(socket, IPPROTO_TCP, TCP_KEEPALIVE, idleSeconds) + && set(socket, IPPROTO_TCP, TCP_KEEPINTVL, intervalSeconds) + && set(socket, IPPROTO_TCP, TCP_KEEPCNT, probeCount) + } + + private static func set(_ socket: Int32, _ level: Int32, _ name: Int32, _ value: Int32) -> Bool { + var value = value + return setsockopt(socket, level, name, &value, socklen_t(MemoryLayout.size)) == 0 + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift index e5bf1f8b15..3b0a77dded 100644 --- a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift @@ -61,7 +61,13 @@ public enum PostgreSQLCatalogTypeNames { case 1_083: return "time" case 1_114: return "timestamp" case 1_184: return "timestamptz" + case 1_186: return "interval" case 1_266: return "timetz" + // The names the catalog lookup gives them, so a fresh session's stream header reads a bit string as one. + case 1_560: return "bit" + case 1_561: return "bit[]" + case 1_562: return "varbit" + case 1_563: return "varbit[]" case 1_700: return "numeric" case 2_950: return "uuid" case 3_802: return "jsonb" diff --git a/apple/Packages/TableProCore/Sources/TableProSQLiteCore/SQLiteAuthorizer.swift b/apple/Packages/TableProCore/Sources/TableProSQLiteCore/SQLiteAuthorizer.swift index 84b75ee13c..9d5bdb744b 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLiteCore/SQLiteAuthorizer.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLiteCore/SQLiteAuthorizer.swift @@ -120,6 +120,8 @@ public final class SQLiteAuthorizerPolicy: @unchecked Sendable { "table_info", "table_xinfo", "table_list", "index_info", "index_xinfo", "index_list", "foreign_key_list", "foreign_key_check", "integrity_check", "quick_check", "function_list", "pragma_list", "module_list", "collation_list", "database_list", "compile_options", + // FTS5 runs it on every read of its table. + "data_version", ] static func decision( diff --git a/apple/Packages/TableProCore/Sources/TableProSyncTransport/CloudKitSyncEngine.swift b/apple/Packages/TableProCore/Sources/TableProSyncTransport/CloudKitSyncEngine.swift index d9992388dd..78795f78de 100644 --- a/apple/Packages/TableProCore/Sources/TableProSyncTransport/CloudKitSyncEngine.swift +++ b/apple/Packages/TableProCore/Sources/TableProSyncTransport/CloudKitSyncEngine.swift @@ -6,11 +6,19 @@ public struct PullResult: Sendable { public let changedRecords: [CKRecord] public let deletedRecordIDs: [CKRecord.ID] public let newToken: CKServerChangeToken? - - public init(changedRecords: [CKRecord], deletedRecordIDs: [CKRecord.ID], newToken: CKServerChangeToken?) { + // Records the zone holds that the fetch could not read, so their absence from `changedRecords` proves nothing. + public let unreadRecordIDs: [CKRecord.ID] + + public init( + changedRecords: [CKRecord], + deletedRecordIDs: [CKRecord.ID], + newToken: CKServerChangeToken?, + unreadRecordIDs: [CKRecord.ID] = [] + ) { self.changedRecords = changedRecords self.deletedRecordIDs = deletedRecordIDs self.newToken = newToken + self.unreadRecordIDs = unreadRecordIDs } } @@ -228,6 +236,7 @@ public actor CloudKitSyncEngine { public func pull(since token: CKServerChangeToken?) async throws -> PullResult { var changedRecords: [CKRecord] = [] var deletedRecordIDs: [CKRecord.ID] = [] + var unreadRecordIDs: [CKRecord.ID] = [] var cursor = token while true { @@ -237,13 +246,15 @@ public actor CloudKitSyncEngine { changedRecords.append(contentsOf: page.result.changedRecords) deletedRecordIDs.append(contentsOf: page.result.deletedRecordIDs) + unreadRecordIDs.append(contentsOf: page.result.unreadRecordIDs) cursor = page.result.newToken ?? cursor guard page.moreComing, page.result.newToken != nil else { return PullResult( changedRecords: changedRecords, deletedRecordIDs: deletedRecordIDs, - newToken: cursor + newToken: cursor, + unreadRecordIDs: unreadRecordIDs ) } } @@ -266,14 +277,19 @@ public actor CloudKitSyncEngine { var changedRecords: [CKRecord] = [] var deletedRecordIDs: [CKRecord.ID] = [] + var unreadRecordIDs: [CKRecord.ID] = [] var newToken: CKServerChangeToken? var moreComing = false var zoneError: Error? return try await withCheckedThrowingContinuation { continuation in - operation.recordWasChangedBlock = { _, result in - if case .success(let record) = result { + operation.recordWasChangedBlock = { recordID, result in + switch result { + case .success(let record): changedRecords.append(record) + case .failure(let error): + Self.logger.warning("Record fetch error: \(error.localizedDescription)") + unreadRecordIDs.append(recordID) } } @@ -310,7 +326,8 @@ public actor CloudKitSyncEngine { result: PullResult( changedRecords: changedRecords, deletedRecordIDs: deletedRecordIDs, - newToken: newToken + newToken: newToken, + unreadRecordIDs: unreadRecordIDs ), moreComing: moreComing )) diff --git a/apple/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLSocketKeepaliveTests.swift b/apple/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLSocketKeepaliveTests.swift new file mode 100644 index 0000000000..9b6db413d2 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProMySQLCoreTests/MySQLSocketKeepaliveTests.swift @@ -0,0 +1,41 @@ +import Darwin +import Foundation +import TableProMySQLCore +import Testing + +struct MySQLSocketKeepaliveTests { + @Test("A TCP socket probes a silent peer after 60 s idle, every 10 s, and gives up after 3") + func tcpSocketTakesTheValues() throws { + let socket = Darwin.socket(AF_INET, SOCK_STREAM, 0) + try #require(socket >= 0) + defer { close(socket) } + + #expect(MySQLSocketKeepalive.enable(on: socket)) + #expect(Self.intOption(socket, SOL_SOCKET, SO_KEEPALIVE) != 0) + #expect(Self.intOption(socket, IPPROTO_TCP, TCP_KEEPALIVE) == 60) + #expect(Self.intOption(socket, IPPROTO_TCP, TCP_KEEPINTVL) == 10) + #expect(Self.intOption(socket, IPPROTO_TCP, TCP_KEEPCNT) == 3) + } + + @Test("A dead peer fails about 90 s after the last traffic") + func deadPeerBound() { + let seconds = MySQLSocketKeepalive.idleSeconds + MySQLSocketKeepalive.intervalSeconds * MySQLSocketKeepalive.probeCount + #expect(seconds == 90) + } + + @Test("A socket that is not TCP reports that the keepalive could not be set") + func localSocketRefuses() throws { + let socket = Darwin.socket(AF_UNIX, SOCK_STREAM, 0) + try #require(socket >= 0) + defer { close(socket) } + + #expect(!MySQLSocketKeepalive.enable(on: socket)) + } + + private static func intOption(_ socket: Int32, _ level: Int32, _ name: Int32) -> Int32 { + var value: Int32 = 0 + var length = socklen_t(MemoryLayout.size) + getsockopt(socket, level, name, &value, &length) + return value + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginColumnOriginsTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginColumnOriginsTests.swift new file mode 100644 index 0000000000..5d4198352d --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginColumnOriginsTests.swift @@ -0,0 +1,41 @@ +import Foundation +import TableProPluginKit +import Testing + +struct PluginColumnOriginsTests { + private let origins: [PluginColumnOrigin?] = [PluginColumnOrigin(schema: "public", table: "users", column: "email"), nil] + + @Test("A capped stream keeps the origins its header reported") + func boundedStreamKeepsHeaderOrigins() async throws { + var header = PluginStreamHeader(columns: ["e", "lower"], columnTypeNames: ["text", "text"]) + header.columnOrigins = origins + let stream = AsyncThrowingStream { continuation in + continuation.yield(.header(header)) + continuation.yield(.rows([[.text("a"), .text("b")], [.text("c"), .text("d")]])) + continuation.finish() + } + + let result = try await PluginBoundedStream.collect(stream, rowCap: 1, startedAt: Date()) + + #expect(result.columnOrigins == origins) + #expect(result.isTruncated) + } + + @Test("Origins survive encoding, and a result without the key decodes with none") + func codableRoundTrip() throws { + var result = PluginQueryResult( + columns: ["e", "lower"], + columnTypeNames: ["text", "text"], + rows: [], + rowsAffected: 0, + timing: PluginQueryTiming(total: 0) + ) + result.columnOrigins = origins + + let decoded = try JSONDecoder().decode(PluginQueryResult.self, from: JSONEncoder().encode(result)) + #expect(decoded.columnOrigins == origins) + + let unreported = try JSONDecoder().decode(PluginQueryResult.self, from: JSONEncoder().encode(PluginQueryResult.empty)) + #expect(unreported.columnOrigins == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPostgreSQLCoreTests/PostgreSQLCatalogTypeNamesTests.swift b/apple/Packages/TableProCore/Tests/TableProPostgreSQLCoreTests/PostgreSQLCatalogTypeNamesTests.swift index 739bbba76c..c9cf429a06 100644 --- a/apple/Packages/TableProCore/Tests/TableProPostgreSQLCoreTests/PostgreSQLCatalogTypeNamesTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProPostgreSQLCoreTests/PostgreSQLCatalogTypeNamesTests.swift @@ -109,7 +109,12 @@ struct PostgreSQLCatalogTypeNamesTests { 1_009: "text[]", 3_807: "jsonb[]", 2_950: "uuid", - 701: "double precision" + 701: "double precision", + 1_560: "bit", + 1_561: "bit[]", + 1_562: "varbit", + 1_563: "varbit[]", + 1_186: "interval" ] for (oid, name) in expected { #expect(PostgreSQLCatalogTypeNames.builtinTypeName(for: oid) == name, "oid \(oid)") diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditorHost.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditorHost.swift index c748ce3c59..bd58e9ef40 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditorHost.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditorHost.swift @@ -40,6 +40,12 @@ public final class SourceEditorHost { SourceEditor.dismantleNSViewController(controller, coordinator: coordinator) } + // A large document's edit reaches the binding on a delay; nil when nothing is waiting. + public func takePendingTextWriteback() -> String? { + guard let textView = controller.textView else { return nil } + return coordinator.textSync.takePendingWriteback(from: textView) + } + // A SwiftUI state write leads to an update pass that consumes the editor's change latch; without one the latch // would swallow the next text the owner pushes. private func stateDidChange(from old: SourceEditorState, to new: SourceEditorState) { diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/TextBindingSync.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/TextBindingSync.swift index 60b3403155..9bf4a5641b 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/TextBindingSync.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/TextBindingSync.swift @@ -67,13 +67,27 @@ final class TextBindingSync { // A run or save reads the binding, which trails the editor by the debounce on a large document. func flushPendingWriteback(from textView: TextView) { - guard let pending = writebackTask else { return } - pending.cancel() - writebackTask = nil + guard writebackTask != nil else { return } + let newText = endPendingWriteback(from: textView) guard case .binding(let binding) = text else { return } + binding.wrappedValue = newText + } + + // For an owner about to point the binding at another document, or to drop the editor: the debounced edit goes + // to the caller, and the edit latch is released so the next document's text is applied. + func takePendingWriteback(from textView: TextView) -> String? { + guard writebackTask != nil else { return nil } + let newText = endPendingWriteback(from: textView) + phase.consumePendingEditorChange() + return newText + } + + private func endPendingWriteback(from textView: TextView) -> String { + writebackTask?.cancel() + writebackTask = nil let newText = textView.string lastSyncedText = newText - binding.wrappedValue = newText + return newText } /// Pushes an external binding change down into the text view. The editor's diff --git a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/SourceEditorBindingSyncTests.swift b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/SourceEditorBindingSyncTests.swift index 99bea4bb35..d78673c597 100644 --- a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/SourceEditorBindingSyncTests.swift +++ b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/SourceEditorBindingSyncTests.swift @@ -147,6 +147,43 @@ final class SourceEditorBindingSyncTests { #expect(bound == "external") } + @MainActor + @Test func test_takeHandsTheDebouncedEditOverAndLetsTheNextDocumentIn() { + var bound = "" + var setterCallCount = 0 + let coordinator = makeCoordinator(get: { bound }, set: { bound = $0; setterCallCount += 1 }) + let largeText = String(repeating: "a", count: 500_001) + controller.textView.setText(largeText) + coordinator.textViewDidChangeText( + Notification(name: TextView.textDidChangeNotification, object: controller.textView) + ) + #expect(coordinator.phase.isEditorChangePending) + + let taken = coordinator.textSync.takePendingWriteback(from: controller.textView) + + #expect(taken == largeText) + #expect(setterCallCount == 0) + #expect(coordinator.textSync.writebackTask == nil) + #expect(!coordinator.phase.isEditorChangePending) + + coordinator.textSync.applyRepresentableText("SELECT 2", controller: controller) + #expect(controller.textView.string == "SELECT 2") + } + + @MainActor + @Test func test_takeWithoutAPendingWritebackKeepsTheEditLatch() { + var bound = "" + let coordinator = makeCoordinator(get: { bound }, set: { bound = $0 }) + controller.textView.setText("small edit") + coordinator.textViewDidChangeText( + Notification(name: TextView.textDidChangeNotification, object: controller.textView) + ) + #expect(bound == "small edit") + + #expect(coordinator.textSync.takePendingWriteback(from: controller.textView) == nil) + #expect(coordinator.phase.isEditorChangePending) + } + @MainActor @Test func test_syncSkipsStaleBindingSnapshotWhileEditIsInFlight() { var bound = "" diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Auth/MCPPrincipal.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Auth/MCPPrincipal.swift index 19c316e8e9..fe6855968c 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Auth/MCPPrincipal.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Auth/MCPPrincipal.swift @@ -5,12 +5,20 @@ public struct MCPPrincipalMetadata: Sendable, Equatable { public let issuedAt: Date public let expiresAt: Date? public let isBridgeCredential: Bool + public let isClaudeAgentCredential: Bool - public init(label: String?, issuedAt: Date, expiresAt: Date?, isBridgeCredential: Bool = false) { + public init( + label: String?, + issuedAt: Date, + expiresAt: Date?, + isBridgeCredential: Bool = false, + isClaudeAgentCredential: Bool = false + ) { self.label = label self.issuedAt = issuedAt self.expiresAt = expiresAt self.isBridgeCredential = isBridgeCredential + self.isClaudeAgentCredential = isClaudeAgentCredential } } diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Identity/MCPGrantSubject.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Identity/MCPGrantSubject.swift index 92ae89907a..c7c949748d 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Identity/MCPGrantSubject.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Identity/MCPGrantSubject.swift @@ -12,11 +12,24 @@ import Foundation /// ask to be the bridge and inherit every grant the bridge had earned. public enum MCPGrantSubject: Sendable, Equatable { case bridge + // The Claude Agent chat mints a token for every turn and deletes it after, the way the bridge rotates. + case claudeAgent case token(UUID) + public init(tokenId: UUID, isBridgeCredential: Bool, isClaudeAgentCredential: Bool) { + if isBridgeCredential { + self = .bridge + } else if isClaudeAgentCredential { + self = .claudeAgent + } else { + self = .token(tokenId) + } + } + public var storageKey: String { switch self { case .bridge: return "bridge" + case .claudeAgent: return "claude-agent" case .token(let id): return id.uuidString } } @@ -24,6 +37,10 @@ public enum MCPGrantSubject: Sendable, Equatable { public extension MCPPrincipal { var grantSubject: MCPGrantSubject { - metadata.isBridgeCredential ? .bridge : .token(tokenId) + MCPGrantSubject( + tokenId: tokenId, + isBridgeCredential: metadata.isBridgeCredential, + isClaudeAgentCredential: metadata.isClaudeAgentCredential + ) } } diff --git a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection+Transport.swift b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection+Transport.swift index 27d7b4811b..dc1c4b5612 100644 --- a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection+Transport.swift +++ b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection+Transport.swift @@ -5,9 +5,13 @@ import CMariaDB import Foundation +import os +import TableProMySQLCore import TableProPluginKit enum MySQLTransport: Sendable, Equatable { + private static let logger = Logger(subsystem: "com.TablePro", category: "MySQLTransport") + case tcp(host: String, port: UInt32) case unixSocket(path: String) @@ -44,7 +48,12 @@ enum MySQLTransport: Sendable, Equatable { ) -> UnsafeMutablePointer? { switch self { case .tcp(let host, let port): - return mysql_real_connect(handle, host, user, password, database, port, nil, 0) + let connected = mysql_real_connect(handle, host, user, password, database, port, nil, 0) + // Every session and kill connection dials through here, and a session's socket outlives the connect. + if connected != nil, !MySQLSocketKeepalive.enable(on: mysql_get_socket(handle)) { + Self.logger.warning("TCP keepalive could not be set: errno \(errno, privacy: .public)") + } + return connected case .unixSocket(let path): return mysql_real_connect(handle, "localhost", user, password, database, 0, path, 0) } diff --git a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift index 98ca6125e2..edea147522 100644 --- a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift +++ b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift @@ -81,6 +81,8 @@ struct MariaDBPluginQueryResult { /// Send to first row, measured from just before the statement goes out. Separates the server's /// own work from the time spent pulling the rest of the result across the wire. var firstRowTime: TimeInterval? + + var columnOrigins: [PluginColumnOrigin?]? } // MARK: - Connection Class @@ -1018,6 +1020,13 @@ final class MariaDBPluginConnection: @unchecked Sendable { flavor: sessionFlavor, binaryStringsAreText: binaryStringsAreText(in: query, flavor: sessionFlavor) ) + let origins = MySQLColumnOrigins.origins( + of: mysql_fetch_fields(resultPtr), + count: Int(mysql_num_fields(resultPtr)), + encoding: connectionEncoding, + flavor: sessionFlavor, + statement: query + ) var rows: [[PluginCellValue]] = [] rows.reserveCapacity(min(1_000, PluginRowLimits.emergencyMax)) @@ -1100,7 +1109,8 @@ final class MariaDBPluginConnection: @unchecked Sendable { columns: columns.names, columnTypes: columns.typeCodes, columnTypeNames: columns.typeNames, rows: rows, affectedRows: UInt64(rows.count), insertId: 0, isTruncated: truncated, columnMeta: columns.metadata, - firstRowTime: firstRowTime ?? Date().timeIntervalSince(sentAt) + firstRowTime: firstRowTime ?? Date().timeIntervalSince(sentAt), + columnOrigins: origins ) } @@ -1406,7 +1416,14 @@ final class MariaDBPluginConnection: @unchecked Sendable { rows: fetchResult.rows, affectedRows: UInt64(fetchResult.rows.count), insertId: 0, isTruncated: fetchResult.isTruncated, columnMeta: columns.metadata, - firstRowTime: fetchResult.firstRowTime + firstRowTime: fetchResult.firstRowTime, + columnOrigins: MySQLColumnOrigins.origins( + of: mysql_fetch_fields(metadata), + count: Int(mysql_num_fields(metadata)), + encoding: connectionEncoding, + flavor: sessionFlavor, + statement: query + ) ) } diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLColumnOrigins.swift b/apple/Plugins/MySQLDriverPlugin/MySQLColumnOrigins.swift new file mode 100644 index 0000000000..124fda688d --- /dev/null +++ b/apple/Plugins/MySQLDriverPlugin/MySQLColumnOrigins.swift @@ -0,0 +1,90 @@ +// +// MySQLColumnOrigins.swift +// MySQLDriverPlugin +// + +import CMariaDB +import Foundation +import TableProPluginKit + +// The server names the table and column a result column reads, but a column read through a derived table, a CTE, +// JSON_TABLE or a procedure comes back under the alias it was given there, which the statement chooses. Measured on +// MySQL 8.4.13 and MariaDB 13.0.2. So origins are kept only for a statement that has one SELECT and none of those, +// and the server reports none for an expression or a set operation. +nonisolated internal enum MySQLColumnOrigins { + // Only the two servers measured; another flavor's field names are not vouched for. + static func origins( + of fields: UnsafeMutablePointer?, + count: Int, + encoding: MySQLConnectionEncoding, + flavor: MySQLServerFlavor, + statement: String + ) -> [PluginColumnOrigin?]? { + guard flavor == .mysql || flavor == .mariadb, let fields else { return nil } + // Every column untrusted, not "no origins": nil would let a browse prove by name, and a filter value can hold the word. + guard readsTableColumnsDirectly(statement) else { return Array(repeating: nil, count: count) } + return (0.. PluginColumnOrigin? { + guard let schema = text(field.db, length: field.db_length, encoding: encoding), schema != "*any*", + !systemSchemas.contains(schema.lowercased()), + let table = text(field.org_table, length: field.org_table_length, encoding: encoding), + let column = text(field.org_name, length: field.org_name_length, encoding: encoding) + else { return nil } + return PluginColumnOrigin(schema: schema, table: table, column: column) + } + + // Only a user table is vouched for. These repeat other tables' values: `data_locks.LOCK_DATA` and + // `INNODB_LOCKS.lock_data` an indexed key, `events_statements_history.MESSAGE_TEXT` an error's text, + // `user_variables_by_thread` what was read into a variable, and the histograms and the optimizer trace samples. + private static let systemSchemas = Set(MySQLSystemDatabases.mysql.map { $0.lowercased() }) + + private static func text( + _ pointer: UnsafeMutablePointer?, + length: UInt32, + encoding: MySQLConnectionEncoding + ) -> String? { + guard let pointer, length > 0 else { return nil } + let bytes = UnsafeRawBufferPointer(start: pointer, count: strnlen(pointer, Int(length))) + let decoded = mysqlSessionText(bytes, encoding: encoding) + return decoded.isEmpty ? nil : decoded + } + + // A set operation needs a second SELECT, TABLE or VALUES, and a derived table one of those or WITH, so a second + // SELECT or any of these words ends the trust. A word inside a literal or a comment counts too: it masks that one + // result whole. + private static let executableCommentOpeners = ["/*!", "/*M!"] + + private static let renamingWords: Set = [ + "WITH", "TABLE", "VALUES", "JSON_TABLE", "CALL", "EXECUTE", "UNION", "INTERSECT", "EXCEPT", + ] + + static func readsTableColumnsDirectly(_ statement: String) -> Bool { + // The server runs an executable comment's body, and in `/*!50000SELECT` the version digits glue to the keyword. + guard !executableCommentOpeners.contains(where: { statement.range(of: $0, options: .caseInsensitive) != nil }) + else { return false } + var selects = 0 + var word = "" + for scalar in statement.unicodeScalars { + if scalar == "_" || scalar == "$" || CharacterSet.alphanumerics.contains(scalar) { + word.unicodeScalars.append(scalar) + continue + } + guard admits(&word, selects: &selects) else { return false } + } + return admits(&word, selects: &selects) + } + + private static func admits(_ word: inout String, selects: inout Int) -> Bool { + defer { word = "" } + guard !word.isEmpty else { return true } + let keyword = word.uppercased() + if keyword == "SELECT" { + selects += 1 + return selects == 1 + } + return !renamingWords.contains(keyword) + } +} diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift index 85effd3772..e63ca8032d 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift @@ -471,18 +471,7 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { rowCap: rowCap, sessionState: readOnlySessionState ) - return PluginQueryResult( - columns: result.columns, - columnTypeNames: result.columnTypeNames, - rows: result.rows, - rowsAffected: Int(result.affectedRows), - timing: PluginQueryTiming( - total: Date().timeIntervalSince(startTime), - firstRow: result.firstRowTime - ), - isTruncated: result.isTruncated, - columnMeta: result.columnMeta - ) + return Self.pluginResult(result, startedAt: startTime) } catch let error as MariaDBPluginError where !isRetry && parameters == nil && mysqlConnectionLossMayReplay(code: error.code, outlastedSocketTimeout: error.outlastedSocketTimeout) @@ -524,7 +513,11 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { await refreshTransactionFlag(after: error, on: conn) throw error } - return PluginQueryResult( + return Self.pluginResult(result, startedAt: startTime) + } + + private static func pluginResult(_ result: MariaDBPluginQueryResult, startedAt startTime: Date) -> PluginQueryResult { + var answer = PluginQueryResult( columns: result.columns, columnTypeNames: result.columnTypeNames, rows: result.rows, @@ -536,6 +529,8 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { isTruncated: result.isTruncated, columnMeta: result.columnMeta ) + answer.columnOrigins = result.columnOrigins + return answer } func cancelQuery() throws { @@ -579,19 +574,7 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { do { let result = try await conn.executeQuery(query, rowCap: rowCap) - - return PluginQueryResult( - columns: result.columns, - columnTypeNames: result.columnTypeNames, - rows: result.rows, - rowsAffected: Int(result.affectedRows), - timing: PluginQueryTiming( - total: Date().timeIntervalSince(startTime), - firstRow: result.firstRowTime - ), - isTruncated: result.isTruncated, - columnMeta: result.columnMeta - ) + return Self.pluginResult(result, startedAt: startTime) } catch let error as MariaDBPluginError where !isRetry && mysqlConnectionLossMayReplay(code: error.code, outlastedSocketTimeout: error.outlastedSocketTimeout) && mayReplay(query) { diff --git a/apple/Plugins/MySQLDriverTests/MariaDBTLSModeTests.swift b/apple/Plugins/MySQLDriverTests/MariaDBTLSModeTests.swift index 6a6c2858f2..ae2dca5232 100644 --- a/apple/Plugins/MySQLDriverTests/MariaDBTLSModeTests.swift +++ b/apple/Plugins/MySQLDriverTests/MariaDBTLSModeTests.swift @@ -224,7 +224,7 @@ struct MariaDBTLSModeTests { // One session: a handshake without CLIENT_SSL, then a switch to mysql_clear_password (or to caching_sha2_password and // its full authentication), then OK. -private final class PlaintextMySQLServer: @unchecked Sendable { +final class PlaintextMySQLServer: @unchecked Sendable { struct SetupFailure: Error {} struct Session: Sendable { diff --git a/apple/Plugins/MySQLDriverTests/MySQLColumnOriginsTests.swift b/apple/Plugins/MySQLDriverTests/MySQLColumnOriginsTests.swift new file mode 100644 index 0000000000..92918c171e --- /dev/null +++ b/apple/Plugins/MySQLDriverTests/MySQLColumnOriginsTests.swift @@ -0,0 +1,147 @@ +import CMariaDB +import Foundation +import TableProPluginKit +import Testing + +struct MySQLColumnOriginsTests { + @Test("A plain projection keeps the origins the server reports, an alias included") + func plainProjectionIsTrusted() { + #expect(MySQLColumnOrigins.readsTableColumnsDirectly("SELECT email AS e FROM users")) + #expect(MySQLColumnOrigins.readsTableColumnsDirectly("select u.email id, lower(name) from shop.users u where id = 1")) + #expect(MySQLColumnOrigins.readsTableColumnsDirectly("SELECT table_name FROM information_schema.tables")) + #expect(MySQLColumnOrigins.readsTableColumnsDirectly("DELETE FROM users RETURNING email AS e")) + } + + // Measured: the server reports `s.e`, `c.e` or `users.e` for these, the alias the statement chose. + @Test("A derived table, a CTE, JSON_TABLE or a procedure drops every origin") + func renamingShapesAreNotTrusted() { + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("SELECT e FROM (SELECT email AS e FROM users) s")) + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("SELECT e FROM(SELECT email AS e FROM users)s")) + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("WITH c AS (SELECT email AS e FROM users) SELECT e FROM c")) + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("SELECT * FROM (TABLE users) x")) + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("SELECT e FROM (VALUES ROW('a')) x(e)")) + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly( + "SELECT j.id FROM users, json_table(users.doc, '$' COLUMNS (id VARCHAR(50) PATH '$.email')) j" + )) + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("CALL leak()")) + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("EXECUTE prepared")) + } + + @Test("A set operation drops every origin, even with its keyword glued to a number") + func setOperationsAreNotTrusted() { + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("SELECT id FROM t UNION SELECT email FROM users")) + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("SELECT id FROM t WHERE id = 1.0union SELECT email FROM users")) + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("SELECT id FROM t WHERE id = 1e0union TABLE users")) + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("SELECT id FROM t /*!UNION*/ SELECT 1")) + } + + // Measured on MariaDB 13.0.2: the server ran the inner SELECT and reported `p.s.e` for the e-mail column. + @Test("An executable comment drops every origin, even with its version glued to a keyword") + func executableCommentsAreNotTrusted() { + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("SELECT e FROM (/*!100000SELECT*/ email AS e FROM users) s")) + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("SELECT e FROM (/*!50000SELECT*/ email AS e FROM users) s")) + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("SELECT e FROM (/*M!100000SELECT*/ email AS e FROM users) s")) + #expect(!MySQLColumnOrigins.readsTableColumnsDirectly("SELECT email /*!AS e*/ FROM users")) + #expect(MySQLColumnOrigins.readsTableColumnsDirectly("SELECT email /* AS e */ FROM users")) + } + + @Test("An identifier that only contains a keyword keeps the origins") + func identifiersAreNotKeywords() { + #expect(MySQLColumnOrigins.readsTableColumnsDirectly("SELECT withdrawn, table_name, selected FROM tables_with_values")) + } + + @Test("A field names its schema, table and column only when the server gave all three") + func fieldOrigin() { + let traced = Self.field(db: "shop", table: "users", column: "email") + defer { Self.release(traced) } + #expect( + MySQLColumnOrigins.origin(of: traced, encoding: .utf8) + == PluginColumnOrigin(schema: "shop", table: "users", column: "email") + ) + + let expression = Self.field(db: "", table: "", column: "") + defer { Self.release(expression) } + #expect(MySQLColumnOrigins.origin(of: expression, encoding: .utf8) == nil) + + let jsonTable = Self.field(db: "*any*", table: "json_table", column: "x") + defer { Self.release(jsonTable) } + #expect(MySQLColumnOrigins.origin(of: jsonTable, encoding: .utf8) == nil) + + let recursive = Self.field(db: "", table: "", column: "e") + defer { Self.release(recursive) } + #expect(MySQLColumnOrigins.origin(of: recursive, encoding: .utf8) == nil) + } + + // Measured on MySQL 8.4.13 and MariaDB 13.0.2: each carried an e-mail address read from a user table. + @Test("A system schema's column is never vouched for, only a user table's", arguments: [ + ("performance_schema", "events_statements_history", "MESSAGE_TEXT"), + ("performance_schema", "data_locks", "LOCK_DATA"), + ("performance_schema", "user_variables_by_thread", "VARIABLE_VALUE"), + ("information_schema", "INNODB_LOCKS", "lock_data"), + ("INFORMATION_SCHEMA", "COLUMN_STATISTICS", "HISTOGRAM"), + ("information_schema", "OPTIMIZER_TRACE", "TRACE"), + ("mysql", "column_stats", "max_value"), + ("sys", "x$innodb_lock_waits", "locked_index"), + ]) + func systemSchemasHaveNoOrigin(schema: String, table: String, column: String) { + let field = Self.field(db: schema, table: table, column: column) + defer { Self.release(field) } + #expect(MySQLColumnOrigins.origin(of: field, encoding: .utf8) == nil) + } + + @Test("Only MySQL and MariaDB, whose field names were measured, report origins") + func onlyMeasuredFlavorsReportOrigins() { + var traced = Self.field(db: "shop", table: "users", column: "email") + defer { Self.release(traced) } + let email: [PluginColumnOrigin?] = [PluginColumnOrigin(schema: "shop", table: "users", column: "email")] + + let mysql = Self.origins(of: &traced, flavor: .mysql) + let mariadb = Self.origins(of: &traced, flavor: .mariadb) + let starrocks = Self.origins(of: &traced, flavor: .starrocks) + let databend = Self.origins(of: &traced, flavor: .databend) + + #expect(mysql == email) + #expect(mariadb == email) + #expect(starrocks == nil) + #expect(databend == nil) + } + + // A browse writes filter values as literals, so `'union'` in one used to read as "no origins", which a browse + // takes as proof by name. + @Test("A statement the scan distrusts leaves every column untrusted, never without origins") + func distrustedStatementKeepsColumnsUntrusted() { + var traced = Self.field(db: "shop", table: "users", column: "email") + defer { Self.release(traced) } + + let filtered = Self.origins(of: &traced, flavor: .mysql, statement: "SELECT email FROM users WHERE name = 'union'") + + #expect(filtered == [nil]) + } + + private static func origins( + of field: inout MYSQL_FIELD, + flavor: MySQLServerFlavor, + statement: String = "SELECT email AS e FROM users" + ) -> [PluginColumnOrigin?]? { + withUnsafeMutablePointer(to: &field) { pointer in + MySQLColumnOrigins.origins(of: pointer, count: 1, encoding: .utf8, flavor: flavor, statement: statement) + } + } + + private static func field(db: String, table: String, column: String) -> MYSQL_FIELD { + var field = MYSQL_FIELD() + field.db = strdup(db) + field.db_length = UInt32(db.utf8.count) + field.org_table = strdup(table) + field.org_table_length = UInt32(table.utf8.count) + field.org_name = strdup(column) + field.org_name_length = UInt32(column.utf8.count) + return field + } + + private static func release(_ field: MYSQL_FIELD) { + free(field.db) + free(field.org_table) + free(field.org_name) + } +} diff --git a/apple/Plugins/MySQLDriverTests/MySQLSessionSocketKeepaliveTests.swift b/apple/Plugins/MySQLDriverTests/MySQLSessionSocketKeepaliveTests.swift new file mode 100644 index 0000000000..a6f2aed4aa --- /dev/null +++ b/apple/Plugins/MySQLDriverTests/MySQLSessionSocketKeepaliveTests.swift @@ -0,0 +1,60 @@ +// +// MySQLSessionSocketKeepaliveTests.swift +// MySQLDriverTests +// + +import Darwin +import Foundation +import TableProPluginKit +import Testing + +// The test process holds the connection's end of the socket, so its options are read off it after the sign-in. +@Suite(.timeLimit(.minutes(1))) +struct MySQLSessionSocketKeepaliveTests { + @Test("A session dialed over TCP probes a silent peer, so a lost reply fails in about 90 s rather than hours") + func sessionSocketTakesTheKeepalive() async throws { + let server = try PlaintextMySQLServer(asksForCleartext: false) + defer { server.close() } + async let session = server.serveOneSession() + let connection = MariaDBPluginConnection( + transport: .tcp(host: "127.0.0.1", port: UInt32(server.port)), + user: "app", + password: "app-password", + database: "", + sslConfig: SSLConfiguration(mode: .disabled), + enableCleartextPlugin: false + ) + + try await connection.connect() + let socket = try #require(Self.clientSocket(connectedTo: server.port)) + let keepalive = Self.intOption(socket, SOL_SOCKET, SO_KEEPALIVE) + let idle = Self.intOption(socket, IPPROTO_TCP, TCP_KEEPALIVE) + let interval = Self.intOption(socket, IPPROTO_TCP, TCP_KEEPINTVL) + let probes = Self.intOption(socket, IPPROTO_TCP, TCP_KEEPCNT) + connection.disconnect() + _ = await session + + #expect(keepalive != 0) + #expect(idle == 60) + #expect(interval == 10) + #expect(probes == 3) + } + + private static func clientSocket(connectedTo port: Int) -> Int32? { + (0...size) + let named = withUnsafeMutablePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { getpeername(descriptor, $0, &length) } + } + return named == 0 && address.sin_family == sa_family_t(AF_INET) && Int(address.sin_port.bigEndian) == port + } + } + + private static func intOption(_ socket: Int32, _ level: Int32, _ name: Int32) -> Int32 { + var value: Int32 = 0 + var length = socklen_t(MemoryLayout.size) + getsockopt(socket, level, name, &value, &length) + return value + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift index 4fa8c7c42f..9b6c67a479 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift @@ -288,6 +288,45 @@ final class LibPQDriverCore: @unchecked Sendable { ) } + // The statement an external caller approved, sent as `executeReadOnly`, `executeParameterized` or + // `executeSingleStatement` sends it, with the relation column each result column reads. + func executeExternalStatement( + query: String, + parameters: [PluginCellValue]?, + readOnly: Bool + ) async throws -> PluginQueryResult { + let pqConn = try connection() + let startTime = Date() + let result: LibPQPluginQueryResult + if readOnly { + result = try await pqConn.executeReadOnlyStatement(query, parameters: parameters) + } else if let parameters { + result = try await pqConn.executeParameterizedQuery(query, parameters: parameters) + } else { + result = try await pqConn.executeSingleStatement(query) + } + var answer = PluginQueryResult( + columns: result.columns, + columnTypeNames: result.columnTypeNames, + rows: result.rows, + rowsAffected: result.affectedRows, + executionTime: Date().timeIntervalSince(startTime), + isTruncated: result.isTruncated + ) + answer.columnOrigins = await columnOrigins(for: result.columnSources) + return answer + } + + // Unpinned (Redshift, CockroachDB) the lookup would run unscoped inside the user's transaction, so it reports none. + private func columnOrigins(for sources: [PostgreSQLColumnOrigins.Source?]) async -> [PluginColumnOrigin?]? { + guard pinsCatalogSearchPath else { return nil } + guard let query = PostgreSQLColumnOrigins.lookupQuery(for: sources) else { + return Array(repeating: nil, count: sources.count) + } + guard let lookup = try? await executeCatalogRead(query) else { return Array(repeating: nil, count: sources.count) } + return PostgreSQLColumnOrigins.origins(for: sources, rows: lookup.rows) + } + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { let pqConn = try connection() let startTime = Date() @@ -490,6 +529,12 @@ extension LibPQBackedDriver { parameters: [PluginCellValue]?, context: PluginStatementContext ) async throws -> PluginQueryResult { + if context.origin == .external { + let answer = try await core.executeExternalStatement( + query: query, parameters: parameters, readOnly: runsReadOnly(context) + ) + return Self.capped(answer, rowCap: rowCap) + } guard runsReadOnly(context) else { return try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) } @@ -518,7 +563,7 @@ extension LibPQBackedDriver { private static func capped(_ raw: PluginQueryResult, rowCap: Int?) -> PluginQueryResult { guard let cap = rowCap, cap > 0, raw.rows.count > cap else { return raw } - return PluginQueryResult( + var kept = PluginQueryResult( columns: raw.columns, columnTypeNames: raw.columnTypeNames, rows: Array(raw.rows.prefix(cap)), @@ -526,6 +571,8 @@ extension LibPQBackedDriver { executionTime: raw.executionTime, isTruncated: true ) + kept.columnOrigins = raw.columnOrigins + return kept } func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection+ResultReading.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection+ResultReading.swift index cd2708a115..d2e7ea8c04 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection+ResultReading.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection+ResultReading.swift @@ -67,7 +67,9 @@ internal extension LibPQPluginConnection { generation: generation ) - return PostGISTypes.rendering(parsed, types: typeNames.postgisTypes) + var rendered = PostGISTypes.rendering(parsed, types: typeNames.postgisTypes) + rendered.columnSources = PostgreSQLColumnOrigins.sources(of: result) + return rendered } static func decodeCell( diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginQueryResult.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginQueryResult.swift index a4109d2502..27ee6c342f 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginQueryResult.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginQueryResult.swift @@ -18,4 +18,6 @@ struct LibPQPluginQueryResult { /// Send to first row, when the read went through single-row mode and could see one. The /// buffered `PQexec` path has no such boundary and leaves it nil. var firstRowTime: TimeInterval? + + var columnSources: [PostgreSQLColumnOrigins.Source?] = [] } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLColumnOrigins.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLColumnOrigins.swift new file mode 100644 index 0000000000..4f940ee9b5 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLColumnOrigins.swift @@ -0,0 +1,70 @@ +// +// PostgreSQLColumnOrigins.swift +// PostgreSQLDriverPlugin +// + +import CLibPQ +import Foundation +import TableProPluginKit + +// The server resolves each result column to the relation column it reads through aliases, subqueries and CTEs, and +// reports none for an expression, a set operation or a whole-row reference. Measured on PostgreSQL 17. +enum PostgreSQLColumnOrigins { + struct Source: Hashable, Sendable { + let relation: UInt32 + let attribute: Int32 + } + + // A system column has a negative number and a whole-row reference reports 0, so only a positive one is a column. + static func source(relation: UInt32, attribute: Int32) -> Source? { + guard relation != 0, attribute > 0 else { return nil } + return Source(relation: relation, attribute: attribute) + } + + static func sources(of result: OpaquePointer) -> [Source?] { + (0.. String? { + let relations = Set(sources.compactMap { $0?.relation }).sorted() + guard !relations.isEmpty else { return nil } + let list = relations.map(String.init).joined(separator: ",") + return """ + SELECT a.attrelid, a.attnum, n.nspname, c.relname, a.attname \ + FROM pg_catalog.pg_attribute a \ + JOIN pg_catalog.pg_class c ON c.oid = a.attrelid \ + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace \ + WHERE a.attrelid = ANY ('{\(list)}'::pg_catalog.oid[]) AND a.attnum > 0 AND NOT a.attisdropped + """ + } + + // Only a user table is vouched for. A catalog or system view repeats other tables' values: `pg_stats` samples + // them in `histogram_bounds`, and `pg_stat_activity` and `pg_locks` show what sessions sent and hold. Whatever + // initdb made has an oid below FirstNormalObjectId. KingbaseES builds these sources, so its `sys_catalog` and `sys` + // are named too, in case its bootstrap counts past that oid. + private static let firstNormalObjectId: UInt32 = 16_384 + private static let systemSchemas = Set(PostgreSQLPlugin.systemSchemaNames + ["sys_catalog", "sys"]) + + static func isUserRelation(_ relation: UInt32, schema: String) -> Bool { + relation >= firstNormalObjectId && !systemSchemas.contains(schema) && !schema.hasPrefix("pg_toast") + } + + static func origins(for sources: [Source?], rows: [[PluginCellValue]]) -> [PluginColumnOrigin?] { + var named: [Source: PluginColumnOrigin] = [:] + for row in rows where row.count == 5 { + guard let relation = row[0].asText.flatMap({ UInt32($0) }), + let attribute = row[1].asText.flatMap({ Int32($0) }), + let schema = row[2].asText, + let table = row[3].asText, + let column = row[4].asText, + isUserRelation(relation, schema: schema) + else { continue } + named[Source(relation: relation, attribute: attribute)] = PluginColumnOrigin( + schema: schema, table: table, column: column + ) + } + return sources.map { $0.flatMap { named[$0] } } + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/PostgreSQLColumnOriginsTests.swift b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLColumnOriginsTests.swift new file mode 100644 index 0000000000..0b37fefdce --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLColumnOriginsTests.swift @@ -0,0 +1,82 @@ +import Foundation +import TableProPluginKit +import Testing + +struct PostgreSQLColumnOriginsTests { + // Measured on PostgreSQL 17: `SELECT u FROM users u` reports the table with column 0, `ctid` column -1. + @Test("Only a relation and a positive column number name a column") + func sourceNeedsARelationColumn() { + #expect(PostgreSQLColumnOrigins.source(relation: 16_384, attribute: 2) + == PostgreSQLColumnOrigins.Source(relation: 16_384, attribute: 2)) + #expect(PostgreSQLColumnOrigins.source(relation: 0, attribute: 0) == nil) + #expect(PostgreSQLColumnOrigins.source(relation: 16_384, attribute: 0) == nil) + #expect(PostgreSQLColumnOrigins.source(relation: 16_384, attribute: -1) == nil) + } + + @Test("The lookup reads each relation once, by oid, from the catalogs") + func lookupQueryNamesEachRelationOnce() throws { + let sources = [ + PostgreSQLColumnOrigins.Source(relation: 16_394, attribute: 1), + nil, + PostgreSQLColumnOrigins.Source(relation: 16_384, attribute: 2), + PostgreSQLColumnOrigins.Source(relation: 16_384, attribute: 3), + ] + let query = try #require(PostgreSQLColumnOrigins.lookupQuery(for: sources)) + #expect(query.contains("a.attrelid = ANY ('{16384,16394}'::pg_catalog.oid[])")) + #expect(query.contains("FROM pg_catalog.pg_attribute a")) + #expect(query.contains("a.attnum > 0")) + } + + @Test("A result of expressions only needs no lookup") + func noLookupWithoutSources() { + #expect(PostgreSQLColumnOrigins.lookupQuery(for: [nil, nil]) == nil) + #expect(PostgreSQLColumnOrigins.lookupQuery(for: []) == nil) + } + + @Test("Each column takes the schema, relation and column its source names") + func originsFollowTheirSources() { + let email = PostgreSQLColumnOrigins.Source(relation: 16_384, attribute: 2) + let viewColumn = PostgreSQLColumnOrigins.Source(relation: 16_394, attribute: 2) + let dropped = PostgreSQLColumnOrigins.Source(relation: 16_384, attribute: 9) + let rows: [[PluginCellValue]] = [ + [.text("16384"), .text("1"), .text("public"), .text("users"), .text("id")], + [.text("16384"), .text("2"), .text("public"), .text("users"), .text("email")], + [.text("16394"), .text("2"), .text("public"), .text("v"), .text("e")], + [.text("x"), .text("2"), .text("public"), .text("users"), .text("email")], + ] + + let origins = PostgreSQLColumnOrigins.origins(for: [email, nil, viewColumn, dropped], rows: rows) + + #expect(origins == [ + PluginColumnOrigin(schema: "public", table: "users", column: "email"), + nil, + PluginColumnOrigin(schema: "public", table: "v", column: "e"), + nil, + ]) + } + + // Measured: `histogram_bounds` of `pg_stats` listed the sampled e-mail addresses of the column it describes. + @Test("Only a user table's column is vouched for, never a catalog's or a system view's") + func systemRelationsHaveNoOrigin() { + let rows: [[PluginCellValue]] = [ + [.text("12053"), .text("10"), .text("pg_catalog"), .text("pg_stats"), .text("histogram_bounds")], + [.text("12222"), .text("21"), .text("pg_catalog"), .text("pg_stat_activity"), .text("query")], + [.text("13400"), .text("3"), .text("information_schema"), .text("columns"), .text("column_default")], + [.text("20000"), .text("2"), .text("sys_catalog"), .text("sys_stats"), .text("histogram_bounds")], + [.text("20002"), .text("1"), .text("sys"), .text("dba_users"), .text("username")], + [.text("20001"), .text("1"), .text("pg_toast"), .text("pg_toast_16384"), .text("chunk_data")], + [.text("12500"), .text("1"), .text("public"), .text("made_by_initdb"), .text("x")], + [.text("16500"), .text("1"), .text("public"), .text("pg_stats"), .text("note")], + ] + let sources = rows.map { row in + PostgreSQLColumnOrigins.Source( + relation: UInt32(row[0].asText ?? "") ?? 0, attribute: Int32(row[1].asText ?? "") ?? 0 + ) + } + + #expect(PostgreSQLColumnOrigins.origins(for: sources, rows: rows) == [ + nil, nil, nil, nil, nil, nil, nil, + PluginColumnOrigin(schema: "public", table: "pg_stats", column: "note"), + ]) + } +} diff --git a/apple/Plugins/RedisDriverPlugin/RedisConditionalWrite.swift b/apple/Plugins/RedisDriverPlugin/RedisConditionalWrite.swift index 8d41c646ca..b00d5cc665 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisConditionalWrite.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisConditionalWrite.swift @@ -6,6 +6,7 @@ // import Foundation +import TableProRedisCore // `RENAMENX` answers 0 and `SET ... NX` answers nil when the key they would create is already // there, and neither is an error to the server. A save that sends one has to fail on that answer, @@ -17,6 +18,48 @@ enum RedisConditionalWrite { "\(keyword) \(command)" } + // HSET, RPUSH, SADD and ZADD have no create-only form, so a new collection is built under a + // staging key and moved onto its name with RENAMENX. The staging key starts with the key, so an + // ACL key pattern that admits the key admits it too. + static let stagingMarker = ":tablepro-staging:" + + // A cluster takes RENAMENX only within one slot. A key with a hash tag keeps it as the first tag + // of the staging key; one that hashes whole gets a counter searched to land in its slot, and the + // suffix has no braces, so it never adds a tag. A counter covers every slot well inside the bound. + static func stagingKey(for key: String, token: String, partitioned: Bool) -> String? { + let staging = key + stagingMarker + token + guard partitioned else { return staging } + let slot = RedisKeySlot.slot(for: key) + if RedisKeySlot.slot(for: staging) == slot { return staging } + for attempt in 0 ..< maxSlotSearch { + let candidate = "\(staging).\(attempt)" + if RedisKeySlot.slot(for: candidate) == slot { return candidate } + } + return nil + } + + // The token after the last marker, without the slot counter. + static func stagingToken(in key: String) -> String? { + guard let marker = key.range(of: stagingMarker, options: .backwards) else { return nil } + let token = key[marker.upperBound...].prefix { $0 != "." } + return token.isEmpty ? nil : String(token) + } + + // The staging key a failed write leaves behind outside a block, where the save stops before its DEL. + static func stagingKey(written operation: RedisOperation) -> String? { + switch operation { + case .mustApply(let args): + guard args.count == 3, args[0].text.uppercased() == "RENAMENX" else { return nil } + return stagingToken(in: args[1].text) == nil ? nil : args[1].text + case .expire(let key, _): + return stagingToken(in: key) == nil ? nil : key + default: + return nil + } + } + + private static let maxSlotSearch = 1_000_000 + static func applied(_ reply: RedisReply) -> Bool { switch reply { case .null: return false diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift index c02cd0baa2..c2f93088de 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift @@ -34,6 +34,25 @@ extension RedisPluginDriver { _ operation: RedisOperation, connection conn: any RedisCommandChannel, startTime: Date + ) async throws -> PluginQueryResult { + guard let staging = RedisConditionalWrite.stagingKey(written: operation), isIssuedStagingKey(staging) else { + return try await dispatchOperation(operation, connection: conn, startTime: startTime) + } + do { + return try await dispatchOperation(operation, connection: conn, startTime: startTime) + } catch let queued as RedisQueuedCommand { + throw queued + } catch { + // Sent outside any block: inside one it would be queued with the rest, and EXEC drops it there anyway. + _ = try? await conn.run(["DEL", staging], scope: .outsideBlock) + throw error + } + } + + private func dispatchOperation( + _ operation: RedisOperation, + connection conn: any RedisCommandChannel, + startTime: Date ) async throws -> PluginQueryResult { switch operation { case .get, .set, .del, .keys, .scan, .type, .ttl, .pttl, .expire, .persist, .rename, .exists: diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift index 1e832eec3c..43d70ee2b9 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift @@ -52,6 +52,12 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private var queuedCount = 0 private var conditionalPositions: Set = [] + // Only a staging key this driver generated is ever deleted on a failed write: a leftover one the + // user renames from the grid is their data. + private let stagingLock = NSLock() + private var issuedStagingTokens: Set = [] + private static let maxIssuedStagingTokens = 10_000 + var serverVersion: String? { redisConnection?.serverVersion() } @@ -223,6 +229,20 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { queuedCommandsLock.unlock() } + func issueStagingToken() -> String { + let token = UUID().uuidString.lowercased() + stagingLock.withLock { + if issuedStagingTokens.count >= Self.maxIssuedStagingTokens { issuedStagingTokens.removeAll() } + issuedStagingTokens.insert(token) + } + return token + } + + func isIssuedStagingKey(_ key: String) -> Bool { + guard let token = RedisConditionalWrite.stagingToken(in: key) else { return false } + return stagingLock.withLock { issuedStagingTokens.contains(token) } + } + private func takeQueuedCommands() -> (labels: [String], conditional: Set) { queuedCommandsLock.lock() defer { queuedCommandsLock.unlock() } @@ -635,11 +655,14 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + let partitioned = redisConnection?.partitionsKeyspace == true let generator = RedisStatementGenerator( namespaceName: table, columns: columns, - deleteBatching: redisConnection?.partitionsKeyspace == true ? .perHashSlot : .singleCommand, - keepsTTLOnSet: RedisStatementGenerator.serverKeepsTTLOnSet(version: redisConnection?.serverVersion()) + deleteBatching: partitioned ? .perHashSlot : .singleCommand, + keepsTTLOnSet: RedisStatementGenerator.serverKeepsTTLOnSet(version: redisConnection?.serverVersion()), + partitionsKeyspace: partitioned, + stagingToken: { [weak self] in self?.issueStagingToken() ?? UUID().uuidString.lowercased() } ) let writes = try generator.generateRowWrites( from: changes, insertedRowData: insertedRowData, diff --git a/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift b/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift index 22922a53b4..1bb4883226 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift @@ -26,6 +26,8 @@ struct RedisStatementGenerator { var deleteBatching: RedisDeleteBatching = .singleCommand // `SET ... KEEPTTL` needs Redis 6.0. An older server gets the expiry put back from the grid. var keepsTTLOnSet = true + var partitionsKeyspace = false + var stagingToken: @Sendable () -> String = { UUID().uuidString.lowercased() } static func serverKeepsTTLOnSet(version: String?) -> Bool { guard let major = version?.split(separator: ".").first.flatMap({ Int($0) }) else { return true } @@ -146,13 +148,7 @@ struct RedisStatementGenerator { let typeName = type?.lowercased() ?? "string" guard typeName.isEmpty || Self.insertableTypes.contains(typeName) else { - throw PluginRowWriteRefusal( - rowIndex: change.rowIndex, - reason: String( - format: String(localized: "A %@ key cannot be added from the grid. Add it with a command in the query editor."), - typeName - ) - ) + throw Self.notAddable(typeName, rowIndex: change.rowIndex) } var ttl: Int? @@ -165,19 +161,42 @@ struct RedisStatementGenerator { let encodedValue = value ?? RedisArgumentCodec.quote("") let expiry = ttl.flatMap { $0 > 0 ? $0 : nil } - guard let collectionCommand = collectionInsertCommand(key: k, encodedValue: encodedValue, type: typeName) else { + guard Self.collectionTypes.contains(typeName) else { let expiryOption = expiry.map { " EX \($0)" } ?? "" let set = "SET \(RedisArgumentCodec.quote(k)) \(encodedValue)\(expiryOption) NX" return [RedisConditionalWrite.statement(set)] } + guard let staging = RedisConditionalWrite.stagingKey( + for: k, token: stagingToken(), partitioned: partitionsKeyspace + ), + let collectionCommand = collectionInsertCommand(key: staging, encodedValue: encodedValue, type: typeName) + else { + throw Self.notAddable(typeName, rowIndex: change.rowIndex) + } + // RENAMENX carries the staging key's TTL over. In a block the DEL drops a staging key the rename + // refused; run a statement at a time, the driver drops it before it fails the save. + let quotedStaging = RedisArgumentCodec.quote(staging) var commands = [collectionCommand] if let expiry { - commands.append("EXPIRE \(RedisArgumentCodec.quote(k)) \(expiry)") + commands.append("EXPIRE \(quotedStaging) \(expiry)") } + commands.append(RedisConditionalWrite.statement("RENAMENX \(quotedStaging) \(RedisArgumentCodec.quote(k))")) + commands.append("DEL \(quotedStaging)") return commands } - // Nil for a string, which is written by a `SET` that carries its own expiry and key check. + private static let collectionTypes: Set = ["hash", "list", "set", "zset"] + + private static func notAddable(_ typeName: String, rowIndex: Int) -> PluginRowWriteRefusal { + PluginRowWriteRefusal( + rowIndex: rowIndex, + reason: String( + format: String(localized: "A %@ key cannot be added from the grid. Add it with a command in the query editor."), + typeName + ) + ) + } + private func collectionInsertCommand(key: String, encodedValue: String, type: String) -> String? { let quotedKey = RedisArgumentCodec.quote(key) switch type { diff --git a/apple/Plugins/RedisDriverTests/RedisCommandParserTests.swift b/apple/Plugins/RedisDriverTests/RedisCommandParserTests.swift index 9ec7c88924..c14f1717d3 100644 --- a/apple/Plugins/RedisDriverTests/RedisCommandParserTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisCommandParserTests.swift @@ -1080,7 +1080,7 @@ struct RedisCommandParserAppStatementTests { func insertsStayTyped(type: String) throws { let value = type == "hash" ? #"{"f":"v w"}"# : "a \"quoted\" value" let statements = try insertStatements(key: "user:1 x", type: type, value: value) - #expect(statements.count == (type == "string" ? 1 : 2)) + #expect(statements.count == (type == "string" ? 1 : 4)) for statement in statements { #expect(try !isVerbatim(statement), "\(statement)") } diff --git a/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift b/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift index f62754b3b6..72e4aa3bd3 100644 --- a/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift @@ -99,6 +99,78 @@ struct RedisConditionalWriteTests { #expect(channel.sentCommands == [["RENAMENX", "old", "new"]]) } + // Outside a block the save stops at the failed statement, so the DEL after the rename never runs. + @Test("A staged insert refused because the key exists drops its staging key") + func refusedStagedInsertDropsItsStagingKey() async throws { + let driver = self.driver() + let staging = "queue:tablepro-staging:\(driver.issueStagingToken())" + let channel = StubRedisChannel([.integer(0), .integer(1)]) + await #expect(throws: RedisPluginError.self) { + _ = try await driver.executeOperation( + try RedisCommandParser.parse("MUSTAPPLY RENAMENX \(staging) queue"), connection: channel, startTime: Date() + ) + } + #expect(channel.sentCommands == [["RENAMENX", staging, "queue"], ["DEL", staging]]) + #expect(channel.sentScopes.last == .outsideBlock) + } + + @Test("A staged rename the server answers with an error drops its staging key") + func failedStagedRenameDropsItsStagingKey() async throws { + let driver = self.driver() + let staging = "queue:tablepro-staging:\(driver.issueStagingToken())" + let channel = StubRedisChannel(outcomes: [ + .failure(RedisPluginError(code: 0, message: "NOPERM No permissions to access a key")), + .success(.integer(1)) + ]) + await #expect(throws: RedisPluginError.self) { + _ = try await driver.executeOperation( + try RedisCommandParser.parse("MUSTAPPLY RENAMENX \(staging) queue"), connection: channel, startTime: Date() + ) + } + #expect(channel.sentCommands == [["RENAMENX", staging, "queue"], ["DEL", staging]]) + } + + @Test("A refused expiry on a staging key drops it before the save stops") + func failedStagingExpiryDropsItsStagingKey() async throws { + let driver = self.driver() + let staging = "queue:tablepro-staging:\(driver.issueStagingToken())" + let channel = StubRedisChannel(outcomes: [ + .failure(RedisPluginError(code: 0, message: "NOPERM this user has no permissions to run the 'expire' command")), + .success(.integer(1)) + ]) + await #expect(throws: RedisPluginError.self) { + _ = try await driver.executeOperation( + try RedisCommandParser.parse("EXPIRE \(staging) 60"), connection: channel, startTime: Date() + ) + } + #expect(channel.sentCommands == [["EXPIRE", staging, "60"], ["DEL", staging]]) + } + + // A leftover staging key shows in the grid, and renaming it onto a taken name must not delete it. + @Test("A refused rename of a staging key this driver did not generate keeps it") + func refusedRenameOfALeftoverKeepsIt() async throws { + let channel = StubRedisChannel([.integer(0)]) + await #expect(throws: RedisPluginError.self) { + _ = try await driver().executeOperation( + try RedisCommandParser.parse("MUSTAPPLY RENAMENX queue:tablepro-staging:old queue"), + connection: channel, + startTime: Date() + ) + } + #expect(channel.sentCommands == [["RENAMENX", "queue:tablepro-staging:old", "queue"]]) + } + + @Test("A staged insert onto a free name leaves nothing to drop") + func appliedStagedInsertSendsNoDelete() async throws { + let driver = self.driver() + let staging = "queue:tablepro-staging:\(driver.issueStagingToken())" + let channel = StubRedisChannel([.integer(1)]) + _ = try await driver.executeOperation( + try RedisCommandParser.parse("MUSTAPPLY RENAMENX \(staging) queue"), connection: channel, startTime: Date() + ) + #expect(channel.sentCommands == [["RENAMENX", staging, "queue"]]) + } + @Test("A rename onto a free name succeeds") func appliedRenameSucceeds() async throws { let channel = StubRedisChannel([.integer(1)]) diff --git a/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift b/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift index 74584377dc..8710b7d525 100644 --- a/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift @@ -979,7 +979,8 @@ struct RedisStatementGeneratorBrowseColumnTests { @Test("A new hash stores each JSON field value as the text it was written as") func hashInsertKeepsJSONValues() throws { - let gen = RedisStatementGenerator(namespaceName: "", columns: Self.browseColumns) + var gen = RedisStatementGenerator(namespaceName: "", columns: Self.browseColumns) + gen.stagingToken = { "t" } let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) let value = #"{"s": "t", "n": 2, "on": true, "none": null, "nested": {"x": 1}, "list": [1, 2]}"# let results = try gen.generateRowWrites( @@ -989,7 +990,114 @@ struct RedisStatementGeneratorBrowseColumnTests { insertedRowIndices: [0] ) #expect(results.first?.statement - == #"HSET h list [1,2] n 2 nested "{\"x\":1}" none null on true s t"#) + == #"HSET h:tablepro-staging:t list [1,2] n 2 nested "{\"x\":1}" none null on true s t"#) + } +} + +// HSET, RPUSH, SADD and ZADD write into a key that already exists, so a new collection row is +// built under a staging key and moved onto its name only when the name is free. +struct RedisCollectionInsertTests { + private static let browseColumns = ["Key", "Type", "TTL", "Length", "Value"] + + private func statements(key: String, type: String, ttl: PluginCellValue = nil, value: PluginCellValue) throws -> [String] { + var gen = RedisStatementGenerator(namespaceName: "", columns: Self.browseColumns) + gen.stagingToken = { "t" } + let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + return try gen.generateRowWrites( + from: [change], + insertedRowData: [0: [.text(key), .text(type), ttl, nil, value]], + deletedRowIndices: [], + insertedRowIndices: [0] + ).map { $0.statement } + } + + @Test("Each collection type is built under a staging key and renamed only onto a free name", arguments: [ + ("list", "RPUSH queue:tablepro-staging:t job"), + ("set", "SADD queue:tablepro-staging:t job"), + ("zset", "ZADD queue:tablepro-staging:t 0 job"), + ("hash", "HSET queue:tablepro-staging:t value job") + ]) + func collectionInsertIsStaged(type: String, write: String) throws { + #expect(try statements(key: "queue", type: type, value: "job") == [ + write, + "MUSTAPPLY RENAMENX queue:tablepro-staging:t queue", + "DEL queue:tablepro-staging:t" + ]) + } + + @Test("The expiry is set on the staging key, which the rename carries over") + func stagedInsertCarriesItsTTL() throws { + #expect(try statements(key: "queue", type: "list", ttl: "60", value: "job") == [ + "RPUSH queue:tablepro-staging:t job", + "EXPIRE queue:tablepro-staging:t 60", + "MUSTAPPLY RENAMENX queue:tablepro-staging:t queue", + "DEL queue:tablepro-staging:t" + ]) + } + + @Test("A string insert keeps its single SET NX") + func stringInsertIsNotStaged() throws { + #expect(try statements(key: "queue", type: "string", value: "job") == ["MUSTAPPLY SET queue job NX"]) + } + + @Test("Every statement of a staged insert names the row it writes") + func stagedStatementsNameTheRow() throws { + var gen = RedisStatementGenerator(namespaceName: "", columns: Self.browseColumns) + gen.stagingToken = { "t" } + let change = PluginRowChange(rowIndex: 3, type: .insert, cellChanges: [], originalRow: nil) + let writes = try gen.generateRowWrites( + from: [change], + insertedRowData: [3: ["queue", "list", nil, nil, "job"]], + deletedRowIndices: [], + insertedRowIndices: [3] + ) + #expect(writes.count == 3) + #expect(writes.allSatisfy { $0.rowIndices == [3] }) + } + + // An ACL key pattern such as `~app:*` admits the staging key only when it starts with the key. + @Test("On a cluster a staging key starts with its key and hashes to the key's slot", arguments: [ + "queue", "app:1", "user:{42}:profile", "{tenant}", "a}b", "{}x}", "x{", "}{", "user:{}{7}" + ]) + func clusterStagingKeySharesTheSlot(key: String) throws { + let staging = try #require(RedisConditionalWrite.stagingKey(for: key, token: "t", partitioned: true)) + #expect(staging.hasPrefix(key + RedisConditionalWrite.stagingMarker + "t")) + #expect(RedisKeySlot.slot(for: staging) == RedisKeySlot.slot(for: key)) + #expect(RedisConditionalWrite.stagingToken(in: staging) == "t") + } + + @Test("A tagged key keeps its tag, so its staging key needs no counter") + func taggedKeyNeedsNoCounter() { + #expect(RedisConditionalWrite.stagingKey(for: "user:{42}:profile", token: "t", partitioned: true) + == "user:{42}:profile:tablepro-staging:t") + } + + @Test("Without a cluster the staging key is the key and the marker, with no braces") + func standaloneStagingKeyIsTheKeyWithASuffix() { + #expect(RedisConditionalWrite.stagingKey(for: "app:1", token: "t", partitioned: false) == "app:1:tablepro-staging:t") + } + + @Test("The grid on a cluster stages a key in its own slot") + func clusterGridInsertIsStagedInTheKeysSlot() throws { + var gen = RedisStatementGenerator(namespaceName: "", columns: Self.browseColumns, partitionsKeyspace: true) + gen.stagingToken = { "t" } + let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + let statements = try gen.generateRowWrites( + from: [change], + insertedRowData: [0: ["app:1", "list", nil, nil, "job"]], + deletedRowIndices: [], + insertedRowIndices: [0] + ).map { $0.statement } + let staging = try #require(statements.first?.split(separator: " ").dropFirst().first.map(String.init)) + #expect(staging.hasPrefix("app:1:tablepro-staging:t")) + #expect(RedisKeySlot.slot(for: staging) == RedisKeySlot.slot(for: "app:1")) + #expect(statements.last == "DEL \(staging)") + } + + @Test("The staging token is read past the last marker, without the slot counter") + func stagingTokenIsReadPastTheLastMarker() { + #expect(RedisConditionalWrite.stagingToken(in: "a:tablepro-staging:old:tablepro-staging:t.42") == "t") + #expect(RedisConditionalWrite.stagingToken(in: "queue") == nil) } } diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLiteColumnOrigins.swift b/apple/Plugins/SQLiteDriverPlugin/SQLiteColumnOrigins.swift new file mode 100644 index 0000000000..fbd60a7e6c --- /dev/null +++ b/apple/Plugins/SQLiteDriverPlugin/SQLiteColumnOrigins.swift @@ -0,0 +1,86 @@ +// +// SQLiteColumnOrigins.swift +// SQLiteDriverPlugin +// + +import CSQLite +import Foundation +import TableProPluginKit + +// SQLite traces a result column through aliases, subqueries, CTEs and views to the table column it reads, and reports +// none for an expression. A compound SELECT, a multi-row VALUES among them, reports one arm's column for the values of +// every arm, and a table-valued function such as json_each reports its own columns for whatever it was handed. +// Measured on SQLite 3.53.4. +enum SQLiteColumnOrigins { + // Read after the first step, which may prepare the statement again, and only after one that did not fail: + // `sqlite3_table_column_metadata` replaces the connection's error message. + static func origins(of statement: OpaquePointer?, on db: OpaquePointer, text: String) -> [PluginColumnOrigin?]? { + guard let statement else { return nil } + let count = sqlite3_column_count(statement) + guard count > 0 else { return [] } + // Every column untrusted, not "no origins": nil would let a browse prove by name, and a filter value can hold the word. + guard !hasCompoundSelect(text) else { return Array(repeating: nil, count: Int(count)) } + let origins = (0.. Set? { + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + // The PRAGMA statement, not the pragma_table_list function: a read-only handle's authorizer allows the + // reporting pragma and refuses reading the eponymous table. + guard sqlite3_prepare_v2(db, "PRAGMA table_list", -1, &statement, nil) == SQLITE_OK else { return nil } + var shadows: Set = [] + var step = sqlite3_step(statement) + while step == SQLITE_ROW { + if let schema = sqlite3_column_text(statement, 0), let name = sqlite3_column_text(statement, 1), + let type = sqlite3_column_text(statement, 2), String(cString: type) == "shadow" { + shadows.insert(key(schema: String(cString: schema), table: String(cString: name))) + } + step = sqlite3_step(statement) + } + return step == SQLITE_DONE ? shadows : nil + } + + private static func key(schema: String, table: String) -> String { + "\(schema.lowercased()).\(table.lowercased())" + } + + // Only a user table is vouched for: a `sqlite_` table is SQLite's own, and `sqlite_stat4.sample` holds the index + // keys ANALYZE sampled. A table-valued function is no table in the schema, so the metadata lookup refuses it. + private static func origin(of statement: OpaquePointer, column index: Int32, on db: OpaquePointer) -> PluginColumnOrigin? { + guard let schema = sqlite3_column_database_name(statement, index), + let table = sqlite3_column_table_name(statement, index), + let column = sqlite3_column_origin_name(statement, index), + !String(cString: table).lowercased().hasPrefix("sqlite_"), + sqlite3_table_column_metadata(db, schema, table, column, nil, nil, nil, nil, nil) == SQLITE_OK + else { return nil } + return PluginColumnOrigin( + schema: String(cString: schema), table: String(cString: table), column: String(cString: column) + ) + } + + // `VALUES ((SELECT id FROM t)), ((SELECT email FROM users))` reports `t.id` for both rows. + private static let compoundOperators: Set = ["UNION", "INTERSECT", "EXCEPT", "VALUES"] + + // A word inside a literal or a comment counts too: it masks that one result whole. + static func hasCompoundSelect(_ text: String) -> Bool { + var word = "" + for scalar in text.unicodeScalars { + if scalar == "_" || scalar == "$" || CharacterSet.alphanumerics.contains(scalar) { + word.unicodeScalars.append(scalar) + continue + } + if compoundOperators.contains(word.uppercased()) { return true } + word = "" + } + return compoundOperators.contains(word.uppercased()) + } +} diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLiteExecutionBackend.swift b/apple/Plugins/SQLiteDriverPlugin/SQLiteExecutionBackend.swift index 3e6abdc630..5906915c69 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLiteExecutionBackend.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLiteExecutionBackend.swift @@ -87,6 +87,7 @@ struct SQLiteRawResult: Sendable { let rowsAffected: Int let executionTime: TimeInterval let isTruncated: Bool + var columnOrigins: [PluginColumnOrigin?]? } // MARK: - Busy Wait @@ -363,6 +364,7 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend { let columnCount = firstStep.count let columns = firstStep.names let columnTypeNames = firstStep.typeNames + let columnOrigins = Self.columnOrigins(after: firstStep, of: statement, on: db, text: query) var rows: [[PluginCellValue]] = [] var truncated = false @@ -387,10 +389,21 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend { rows: rows, rowsAffected: rowsAffected, executionTime: Date().timeIntervalSince(startTime), - isTruncated: truncated + isTruncated: truncated, + columnOrigins: columnOrigins ) } + private static func columnOrigins( + after firstStep: SQLiteResultColumns.FirstStep, + of statement: OpaquePointer?, + on db: OpaquePointer, + text: String + ) -> [PluginColumnOrigin?]? { + guard firstStep.result == SQLITE_ROW || firstStep.result == SQLITE_DONE else { return nil } + return SQLiteColumnOrigins.origins(of: statement, on: db, text: text) + } + // SQLite compiles only a text's first statement; a second one in the tail would be dropped without a word. The // tail is read, never prepared: preparing a PRAGMA already applies it. static func prepareOnlyStatement(_ query: String, db: OpaquePointer) throws -> OpaquePointer? { @@ -441,9 +454,11 @@ actor SQLiteLocalBackend: SQLiteExecutionBackend { let firstStep = SQLiteResultColumns.stepFirst(statement) let columnCount = firstStep.count - continuation.yield(.header(PluginStreamHeader( + var header = PluginStreamHeader( columns: firstStep.names, columnTypeNames: firstStep.typeNames, estimatedRowCount: nil - ))) + ) + header.columnOrigins = Self.columnOrigins(after: firstStep, of: statement, on: db, text: query) + continuation.yield(.header(header)) let batchSize = 5_000 var batch: [PluginRow] = [] diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift b/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift index c2ab24fe0a..439acba77e 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift @@ -246,7 +246,7 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } private static func queryResult(_ rawResult: SQLiteRawResult) -> PluginQueryResult { - PluginQueryResult( + var result = PluginQueryResult( columns: rawResult.columns, columnTypeNames: rawResult.columnTypeNames, rows: rawResult.rows, @@ -254,6 +254,8 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { executionTime: rawResult.executionTime, isTruncated: rawResult.isTruncated ) + result.columnOrigins = rawResult.columnOrigins + return result } /// `sqlite3_interrupt` ends a statement that is running. A connection waiting for a lock is @@ -366,7 +368,7 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { private static func capped(_ raw: PluginQueryResult, to rowCap: Int?) -> PluginQueryResult { guard let cap = rowCap, cap > 0, raw.rows.count > cap else { return raw } - return PluginQueryResult( + var kept = PluginQueryResult( columns: raw.columns, columnTypeNames: raw.columnTypeNames, rows: Array(raw.rows.prefix(cap)), @@ -375,6 +377,8 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { isTruncated: true, statusMessage: raw.statusMessage ) + kept.columnOrigins = raw.columnOrigins + return kept } private static let rowReturningKeywords: Set = ["SELECT", "WITH", "VALUES", "TABLE", "PRAGMA", "EXPLAIN"] diff --git a/apple/Plugins/SQLiteDriverTests/SQLiteColumnOriginsTests.swift b/apple/Plugins/SQLiteDriverTests/SQLiteColumnOriginsTests.swift new file mode 100644 index 0000000000..0d5075a053 --- /dev/null +++ b/apple/Plugins/SQLiteDriverTests/SQLiteColumnOriginsTests.swift @@ -0,0 +1,159 @@ +import Foundation +import TableProPluginKit +import Testing + +struct SQLiteColumnOriginsTests { + private let directory: URL + + init() throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("sqlite-column-origins-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + } + + private static var external: PluginStatementContext { + var context = PluginStatementContext() + context.origin = .external + context.hostSplit = true + context.allowsServerCode = false + return context + } + + private static var externalRead: PluginStatementContext { + var context = external + context.readOnly = true + return context + } + + private static let email = PluginColumnOrigin(schema: "main", table: "users", column: "email") + + private func connect() async throws -> SQLitePluginDriver { + let driver = SQLitePluginDriver(config: DriverConnectionConfig( + host: "", port: 0, username: "", password: "", database: directory.appendingPathComponent("app.db").path + )) + try await driver.connect() + _ = try await driver.execute(query: "CREATE TABLE users (id INTEGER PRIMARY KEY, email TEXT)") + _ = try await driver.execute(query: "CREATE TABLE t (id INTEGER)") + _ = try await driver.execute(query: "INSERT INTO users VALUES (1, 'jane@example.com')") + _ = try await driver.execute(query: "INSERT INTO t VALUES (7)") + return driver + } + + // The streamed read is the one MCP takes, with a row cap; the whole read is the one without. + private func origins(of sql: String, on driver: SQLitePluginDriver) async throws -> [[PluginColumnOrigin?]?] { + let streamed = try await driver.executeUserQuery(query: sql, rowCap: 10, parameters: nil, context: Self.externalRead) + let whole = try await driver.executeUserQuery(query: sql, rowCap: nil, parameters: nil, context: Self.externalRead) + return [streamed.columnOrigins, whole.columnOrigins] + } + + @Test("A column renamed by an alias, a subquery or a CTE still names the column it reads") + func renamedColumnsKeepTheirOrigin() async throws { + let driver = try await connect() + defer { driver.disconnect() } + + for sql in [ + "SELECT email AS e FROM users", + "SELECT e FROM (SELECT email AS e FROM users) s", + "WITH c AS (SELECT email AS e FROM users) SELECT e FROM c", + ] { + #expect(try await origins(of: sql, on: driver) == [[Self.email], [Self.email]], "\(sql)") + } + } + + @Test("An expression reports no origin") + func expressionHasNoOrigin() async throws { + let driver = try await connect() + defer { driver.disconnect() } + + #expect(try await origins(of: "SELECT lower(email), id FROM users", on: driver) == [ + [nil, PluginColumnOrigin(schema: "main", table: "users", column: "id")], + [nil, PluginColumnOrigin(schema: "main", table: "users", column: "id")], + ]) + } + + // Measured: SQLite reports `t.id` for this column, which holds the e-mail addresses too. + @Test("A compound SELECT reports no origins at all") + func compoundSelectHasNoOrigins() async throws { + let driver = try await connect() + defer { driver.disconnect() } + + #expect(try await origins(of: "SELECT id FROM t UNION SELECT email FROM users", on: driver) == [[nil], [nil]]) + #expect(try await origins(of: "SELECT e FROM (SELECT id AS e FROM t EXCEPT SELECT 1)", on: driver) == [[nil], [nil]]) + } + + // A browse writes filter values as literals, so `'union'` in one used to read as "no origins", which a browse + // takes as proof by name. + @Test("A word in a filter value leaves every column untrusted, never without origins") + func filterValueWordKeepsColumnsUntrusted() async throws { + let driver = try await connect() + defer { driver.disconnect() } + _ = try await driver.execute(query: "CREATE VIRTUAL TABLE contacts USING fts5(email, note)") + _ = try await driver.execute(query: "INSERT INTO contacts VALUES ('jane@example.com', 'n')") + + #expect(try await origins( + of: "SELECT c0, c1 FROM contacts_content WHERE c0 != 'union'", on: driver + ) == [[nil, nil], [nil, nil]]) + } + + // Measured: both report `main.t.id` for a column holding the e-mail address too. + @Test("A multi-row VALUES reports no origins at all") + func valuesRowsHaveNoOrigins() async throws { + let driver = try await connect() + defer { driver.disconnect() } + + #expect(try await origins(of: "VALUES ((SELECT id FROM t)), ((SELECT email FROM users))", on: driver) == [[nil], [nil]]) + #expect(try await origins( + of: "SELECT column1 FROM (VALUES ((SELECT email FROM users)), ((SELECT id FROM t)))", on: driver + ) == [[nil], [nil]]) + } + + @Test("A statement with no result columns is not scanned") + func noColumnsNoOrigins() async throws { + let driver = try await connect() + defer { driver.disconnect() } + + let insert = try await driver.executeUserQuery( + query: "INSERT INTO t VALUES (8)", rowCap: nil, parameters: nil, context: Self.external + ) + #expect(insert.columnOrigins == []) + } + + @Test("A table-valued function's column is no table column") + func tableValuedFunctionHasNoOrigin() async throws { + let driver = try await connect() + defer { driver.disconnect() } + + let sql = "SELECT j.value AS id FROM users, json_each(json_array(users.email)) j" + #expect(try await origins(of: sql, on: driver) == [[nil], [nil]]) + } + + @Test("SQLite's own tables report no origin") + func systemTablesHaveNoOrigin() async throws { + let driver = try await connect() + defer { driver.disconnect() } + + #expect(try await origins(of: "SELECT name, sql FROM sqlite_schema", on: driver) == [[nil, nil], [nil, nil]]) + } + + // Measured: `contacts_content.c0` passed the metadata lookup and held the indexed address. + @Test("An FTS table's shadow copy reports no origin, while the FTS table's own columns do") + func shadowTablesHaveNoOrigin() async throws { + let driver = try await connect() + defer { driver.disconnect() } + _ = try await driver.execute(query: "CREATE VIRTUAL TABLE contacts USING fts5(email, note)") + _ = try await driver.execute(query: "INSERT INTO contacts VALUES ('jane@example.com', 'n')") + + #expect(try await origins(of: "SELECT c0, c1 FROM contacts_content", on: driver) == [[nil, nil], [nil, nil]]) + let email = PluginColumnOrigin(schema: "main", table: "contacts", column: "email") + #expect(try await origins(of: "SELECT email AS e FROM contacts", on: driver) == [[email], [email]]) + } + + @Test("A compound operator is found as a word, not inside an identifier") + func compoundWords() { + #expect(SQLiteColumnOrigins.hasCompoundSelect("select 1 union select 2")) + #expect(SQLiteColumnOrigins.hasCompoundSelect("SELECT 1 INTERSECT SELECT 1")) + #expect(SQLiteColumnOrigins.hasCompoundSelect("SELECT 1 /* x */EXCEPT SELECT 2")) + #expect(SQLiteColumnOrigins.hasCompoundSelect("VALUES (1), (2)")) + #expect(!SQLiteColumnOrigins.hasCompoundSelect("SELECT reunion, exceptions, value_count FROM union_members")) + } +} diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealCellCoder.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealCellCoder.swift index cf9af0f35d..f40f7591c4 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealCellCoder.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealCellCoder.swift @@ -33,10 +33,12 @@ public enum SurrealCellCoder { // `held` is every kind of value the field was read holding. It decides text typed into a // field with no declared type, where the shape of the text alone would retype a string. + // `recordHeld` is the kind this record's own field held when it was read, and decides first. public static func value( from cell: PluginCellValue, kind: SurrealFieldKind?, - held: Set = [] + held: Set = [], + recordHeld: SurrealFieldKind.Base? = nil ) -> SurrealValue { switch cell { case .null: @@ -44,10 +46,27 @@ public enum SurrealCellCoder { case let .bytes(data): return .bytes(data) case let .text(text): + if let recordHeld, let typed = value(fromText: text, keeping: recordHeld, kind: kind) { return typed } return value(fromText: text, kind: kind, held: held) } } + // A cell shows the string '1001' and the int 1001 alike, and a field with no declared type, or a + // union, can hold both. Nil when the text cannot be a value of the kind the record held. + private static func value( + fromText text: String, + keeping base: SurrealFieldKind.Base, + kind: SurrealFieldKind? + ) -> SurrealValue? { + let trimmed = text.trimmingCharacters(in: .whitespaces) + guard base != .any, !trimmed.isEmpty else { return nil } + if base == .decimal, Double(trimmed) == nil { return nil } + let heldKind = kind.flatMap { $0.base == base ? $0 : nil } + ?? SurrealFieldKind(base: base, isOptional: true, raw: base.rawValue) + let typed = value(fromText: text, kind: heldKind, held: []) + return base == .string || typed != .string(text) ? typed : nil + } + // MARK: - Text coercion private static func value( diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver+Schema.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver+Schema.swift index a1a2f5d233..77d70a9b7c 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver+Schema.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver+Schema.swift @@ -65,6 +65,7 @@ extension SurrealDBPluginDriver { ) var columns = SurrealInfoParser.columns(from: value, isRelation: descriptor?.isRelation ?? false) + rememberDeclaredKinds(Self.kinds(from: columns), for: table) if columns.count <= 1 || !(descriptor?.isSchemafull ?? false) { columns = try await sampledColumns(table: table, scope: scope, declared: columns) } diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift index 83744d2156..1edab23077 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift @@ -15,6 +15,15 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private var database: String? private var kindCache: [String: [String: SurrealFieldKind]] = [:] private var heldKindCache: [String: [String: Set]] = [:] + // What `DEFINE FIELD` says, kept apart from the kinds the first rows read happened to hold. + private var declaredKindCache: [String: [String: SurrealFieldKind]] = [:] + // Table, then record literal, then field: the kind each record's field held when last read. + // Bounded by field entries, and evicted a table at a time, least recently read first. + private var recordKindCache: [String: [String: [String: SurrealFieldKind.Base]]] = [:] + private var recordKindEntries: [String: Int] = [:] + private var recordKindTables: [String] = [] + private static let recordKindEntryLimit = 200_000 + private static let recordKindRowLimit = 20_000 init(config: DriverConnectionConfig) { let settings = SurrealDBConnectionConfig(config: config) @@ -64,14 +73,25 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } + // The record kinds outlive a reconnect: they describe rows of the same database that a grid still shows. func disconnect() { client.stop() lock.withLock { kindCache.removeAll() heldKindCache.removeAll() + declaredKindCache.removeAll() } } + private func clearKindCaches() { + kindCache.removeAll() + heldKindCache.removeAll() + declaredKindCache.removeAll() + recordKindCache.removeAll() + recordKindEntries.removeAll() + recordKindTables.removeAll() + } + func ping() async throws { let scope = currentScope() _ = try await client.query("RETURN 1;", namespace: scope.namespace, database: scope.database) @@ -116,16 +136,14 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { lock.withLock { namespace = database self.database = nil - kindCache.removeAll() - heldKindCache.removeAll() + clearKindCaches() } } func switchSchema(to schema: String) async throws { lock.withLock { database = schema - kindCache.removeAll() - heldKindCache.removeAll() + clearKindCaches() } } @@ -133,10 +151,22 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { lock.withLock { heldKindCache[table] ?? [:] } } + func cachedRecordKinds(for table: String) -> [String: [String: SurrealFieldKind.Base]] { + lock.withLock { recordKindCache[table] ?? [:] } + } + func cachedKinds(for table: String) -> [String: SurrealFieldKind] { lock.withLock { kindCache[table] ?? [:] } } + func cachedDeclaredKinds(for table: String) -> [String: SurrealFieldKind] { + lock.withLock { declaredKindCache[table] ?? [:] } + } + + func rememberDeclaredKinds(_ kinds: [String: SurrealFieldKind], for table: String) { + lock.withLock { declaredKindCache[table] = kinds } + } + func mergeKinds(_ kinds: [String: SurrealFieldKind], for table: String) { guard !kinds.isEmpty else { return } lock.withLock { @@ -144,7 +174,8 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } - func learnKinds(from value: SurrealValue) { + // `remembersRecords` is false for a whole-table read, an export or a stream, which no grid edits. + func learnKinds(from value: SurrealValue, remembersRecords: Bool = true) { let rows: [SurrealValue] switch value { case let .array(items): @@ -155,31 +186,91 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return } + let keepsRecords = remembersRecords && rows.count <= Self.recordKindRowLimit var learned: [String: [String: SurrealFieldKind]] = [:] var held: [String: [String: Set]] = [:] + var byRecord: [String: [String: RecordFieldKinds]] = [:] for row in rows { guard let pairs = row.objectPairs, case let .recordId(record)? = row[SurrealInfoParser.recordIdColumn] else { continue } + var fields = RecordFieldKinds() for pair in pairs where !SurrealInfoParser.isReservedColumn(pair.key) { - guard let kind = SurrealFieldKind.infer(from: pair.value) else { continue } + guard let kind = SurrealFieldKind.infer(from: pair.value) else { + if keepsRecords { fields.empty.insert(pair.key) } + continue + } + if keepsRecords { + if kind.base == .any { fields.empty.insert(pair.key) } else { fields.held[pair.key] = kind.base } + } held[record.table, default: [:]][pair.key, default: []].insert(kind.base) guard learned[record.table]?[pair.key] == nil else { continue } learned[record.table, default: [:]][pair.key] = kind } + if keepsRecords { byRecord[record.table, default: [:]][record.literal] = fields } } for (table, kinds) in learned { mergeKinds(kinds, for: table) } - guard !held.isEmpty else { return } + guard !held.isEmpty || !byRecord.isEmpty else { return } lock.withLock { for (table, fields) in held { for (field, bases) in fields { heldKindCache[table, default: [:]][field, default: []].formUnion(bases) } } + rememberRecordKinds(byRecord) } } + private struct RecordFieldKinds { + var held: [String: SurrealFieldKind.Base] = [:] + var empty: Set = [] + } + + // A projection names some fields, so each read updates the ones it names and keeps the rest. + // Called under `lock`. + private func rememberRecordKinds(_ reads: [String: [String: RecordFieldKinds]]) { + let incoming = reads.values.reduce(0) { total, records in total + records.values.reduce(0) { $0 + $1.held.count } } + guard incoming <= Self.recordKindEntryLimit else { return } + for (table, records) in reads { + mergeRecordKinds(records, into: table) + } + var total = recordKindEntries.values.reduce(0, +) + while total > Self.recordKindEntryLimit, let oldest = recordKindTables.first(where: { reads[$0] == nil }) { + total -= recordKindEntries[oldest] ?? 0 + forgetRecordKinds(of: oldest) + } + guard total > Self.recordKindEntryLimit else { return } + // The tables just read outgrew the limit on their own over many pages: only this read's rows stay. + for (table, records) in reads { + forgetRecordKinds(of: table) + mergeRecordKinds(records, into: table) + } + } + + private func mergeRecordKinds(_ records: [String: RecordFieldKinds], into table: String) { + var cached = recordKindCache[table] ?? [:] + var entries = recordKindEntries[table] ?? 0 + for (record, fields) in records { + var known = cached[record] ?? [:] + entries -= known.count + known.merge(fields.held) { _, read in read } + for field in fields.empty { known.removeValue(forKey: field) } + entries += known.count + cached[record] = known.isEmpty ? nil : known + } + recordKindCache[table] = cached + recordKindEntries[table] = entries + recordKindTables.removeAll { $0 == table } + recordKindTables.append(table) + } + + private func forgetRecordKinds(of table: String) { + recordKindCache.removeValue(forKey: table) + recordKindEntries.removeValue(forKey: table) + recordKindTables.removeAll { $0 == table } + } + // MARK: - Execution func execute(query: String) async throws -> PluginQueryResult { @@ -193,7 +284,7 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { if let failure = SurrealRPCClient.firstFailure(results) { throw failure } - results.forEach { learnKinds(from: $0.value) } + results.forEach { learnKinds(from: $0.value, remembersRecords: length == .display) } return Self.result(from: results, elapsed: Date().timeIntervalSince(started), length: length) } @@ -373,6 +464,8 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { columns: columns, kinds: cachedKinds(for: table), heldKinds: cachedHeldKinds(for: table), + recordKinds: cachedRecordKinds(for: table), + declaredKinds: cachedDeclaredKinds(for: table), changes: changes, insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift index 20a4970ec0..c98626d2bd 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift @@ -21,13 +21,15 @@ public enum SurrealStatementGenerator { columns: [String], kinds: [String: SurrealFieldKind], heldKinds: [String: Set] = [:], + recordKinds: [String: [String: SurrealFieldKind.Base]] = [:], + declaredKinds: [String: SurrealFieldKind] = [:], changes: [PluginRowChange], insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set ) throws(PluginRowWriteRefusal) -> [PluginRowWrite] { var writes: [PluginRowWrite] = [] - let typing = FieldTyping(kinds: kinds, held: heldKinds) + let typing = FieldTyping(kinds: kinds, held: heldKinds, records: recordKinds, declared: declaredKinds) for change in changes where change.type == .update && !insertedRowIndices.contains(change.rowIndex) { let write = try update(table: table, scope: scope, columns: columns, typing: typing, change: change) @@ -54,9 +56,29 @@ public enum SurrealStatementGenerator { private struct FieldTyping { let kinds: [String: SurrealFieldKind] let held: [String: Set] + // Keyed by record literal, the text the grid shows in the id column. + let records: [String: [String: SurrealFieldKind.Base]] + let declared: [String: SurrealFieldKind] + + // A single declared type decides: SurrealDB leaves rows written before `DEFINE FIELD ... TYPE` as + // they were, and rejects another kind written into the field now. + func value(of cell: PluginCellValue, in column: String, record: String? = nil) -> SurrealValue { + let recordHeld = Self.declaresOneKind(declared[column]) ? nil : record.flatMap { records[$0]?[column] } + return SurrealCellCoder.value( + from: cell, + kind: declared[column] ?? kinds[column], + held: held[column] ?? [], + recordHeld: recordHeld + ) + } - func value(of cell: PluginCellValue, in column: String) -> SurrealValue { - SurrealCellCoder.value(from: cell, kind: kinds[column], held: held[column] ?? []) + private static func declaresOneKind(_ kind: SurrealFieldKind?) -> Bool { + guard let kind, kind.base != .any else { return false } + let concrete = kind.raw.split(separator: "|").filter { alternative in + let name = alternative.trimmingCharacters(in: .whitespaces).lowercased() + return name != "none" && name != "null" + } + return concrete.count <= 1 } } @@ -72,7 +94,8 @@ public enum SurrealStatementGenerator { if let reserved = change.cellChanges.first(where: { SurrealInfoParser.isReservedColumn($0.columnName) }) { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: reservedColumnReason(reserved.columnName)) } - guard let record = recordId(table: table, columns: columns, originalRow: change.originalRow) else { return nil } + let recordKey = Self.recordText(columns: columns, originalRow: change.originalRow) + guard let record = recordKey.flatMap({ SurrealQL.parseRecordId($0, fallbackTable: table) }) else { return nil } let editable = change.cellChanges.filter { !Self.isAutoDefault($0.newValue) } guard !editable.isEmpty else { return nil } @@ -84,7 +107,7 @@ public enum SurrealStatementGenerator { if !cell.newValue.isNull { try refuseShortened(cell.oldValue, in: cell.columnName, rowIndex: change.rowIndex) } - let value = typing.value(of: cell.newValue, in: cell.columnName) + let value = typing.value(of: cell.newValue, in: cell.columnName, record: recordKey) parameters.append(SurrealCellCoder.parameter(value)) assignments.append(SurrealQL.quoteIdentifier(cell.columnName) + " = $p\(parameters.count - 1)") } @@ -187,10 +210,16 @@ public enum SurrealStatementGenerator { columns: [String], originalRow: [PluginCellValue]? ) -> SurrealRecordID? { + recordText(columns: columns, originalRow: originalRow).flatMap { + SurrealQL.parseRecordId($0, fallbackTable: table) + } + } + + private static func recordText(columns: [String], originalRow: [PluginCellValue]?) -> String? { guard let originalRow, let index = columns.firstIndex(of: SurrealInfoParser.recordIdColumn), index < originalRow.count, case let .text(text) = originalRow[index] else { return nil } - return SurrealQL.parseRecordId(text, fallbackTable: table) + return text } } diff --git a/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift b/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift index 15deee830f..702eaa1e56 100644 --- a/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift +++ b/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift @@ -27,6 +27,7 @@ public enum PluginBoundedStream { let cap = max(rowCap, 1) var columns: [String] = [] var columnTypeNames: [String] = [] + var columnOrigins: [PluginColumnOrigin?]? var rows: [PluginRow] = [] rows.reserveCapacity(min(cap, 10_000)) var truncated = false @@ -39,6 +40,7 @@ public enum PluginBoundedStream { case .header(let header): columns = header.columns columnTypeNames = header.columnTypeNames + columnOrigins = header.columnOrigins case .rows(let batch): if firstRowTime == nil, !batch.isEmpty { firstRowTime = Date().timeIntervalSince(startedAt) @@ -54,7 +56,7 @@ public enum PluginBoundedStream { if truncated { break } } - return PluginQueryResult( + var result = PluginQueryResult( columns: columns, columnTypeNames: columnTypeNames, rows: rows, @@ -66,5 +68,7 @@ public enum PluginBoundedStream { ), isTruncated: truncated ) + result.columnOrigins = columnOrigins + return result } } diff --git a/apple/Plugins/TableProPluginKit/PluginColumnOrigin.swift b/apple/Plugins/TableProPluginKit/PluginColumnOrigin.swift new file mode 100644 index 0000000000..5374235895 --- /dev/null +++ b/apple/Plugins/TableProPluginKit/PluginColumnOrigin.swift @@ -0,0 +1,14 @@ +import Foundation + +// The table or view column a result column reads, as the engine resolved it, whatever the statement named the result. +public struct PluginColumnOrigin: Codable, Sendable, Hashable { + public let schema: String? + public let table: String + public let column: String + + public init(schema: String?, table: String, column: String) { + self.schema = schema + self.table = table + self.column = column + } +} diff --git a/apple/Plugins/TableProPluginKit/PluginQueryResult.swift b/apple/Plugins/TableProPluginKit/PluginQueryResult.swift index 00abc72b4c..5a06c825f5 100644 --- a/apple/Plugins/TableProPluginKit/PluginQueryResult.swift +++ b/apple/Plugins/TableProPluginKit/PluginQueryResult.swift @@ -21,6 +21,9 @@ public struct PluginQueryResult: Codable, Sendable { /// document store sets it, because a missing field and a field holding null both read `.null`. public var absentCells: [Int: Set]? public var exportDocuments: [PluginExportDocument]? + // Per column, nil where the column is an expression or the driver cannot vouch for what it reads. Nil when the + // driver reports no origins; the host then masks every unknown column on a connection with masking rules. + public var columnOrigins: [PluginColumnOrigin?]? public init( columns: [String], @@ -108,6 +111,7 @@ public struct PluginQueryResult: Codable, Sendable { rowLocators = try container.decodeIfPresent([String?].self, forKey: .rowLocators) absentCells = try container.decodeIfPresent([Int: Set].self, forKey: .absentCells) exportDocuments = try container.decodeIfPresent([PluginExportDocument].self, forKey: .exportDocuments) + columnOrigins = try container.decodeIfPresent([PluginColumnOrigin?].self, forKey: .columnOrigins) } public static let empty = PluginQueryResult( diff --git a/apple/Plugins/TableProPluginKit/PluginStreamTypes.swift b/apple/Plugins/TableProPluginKit/PluginStreamTypes.swift index 7ccf6bcd5b..12cec34a01 100644 --- a/apple/Plugins/TableProPluginKit/PluginStreamTypes.swift +++ b/apple/Plugins/TableProPluginKit/PluginStreamTypes.swift @@ -6,6 +6,8 @@ public struct PluginStreamHeader: Sendable { public let columns: [String] public let columnTypeNames: [String] public let estimatedRowCount: Int? + // See `PluginQueryResult.columnOrigins`. + public var columnOrigins: [PluginColumnOrigin?]? public init(columns: [String], columnTypeNames: [String], estimatedRowCount: Int? = nil) { self.columns = columns diff --git a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentMCPBridge.swift b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentMCPBridge.swift index ee01dce6bc..3450ce5425 100644 --- a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentMCPBridge.swift +++ b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentMCPBridge.swift @@ -47,11 +47,12 @@ struct ClaudeAgentMCPBridge: ClaudeAgentMCPBridging { connectionAccess: connectionAccess, expiresAt: Date.now.addingTimeInterval(Self.tokenLifetime), isBridgeCredential: false, + isClaudeAgentCredential: true, extraScopes: [] ) else { return nil } guard let configPath = Self.writeConfig(endpoint: endpoint, token: generated.plaintext) else { - await tokenStore.delete(tokenId: generated.token.id) + await tokenStore.retire(tokenId: generated.token.id) return nil } @@ -63,7 +64,7 @@ struct ClaudeAgentMCPBridge: ClaudeAgentMCPBridging { guard let grant else { return } try? FileManager.default.removeItem(atPath: grant.configPath) guard let tokenStore = await MCPServerManager.shared.tokenStore else { return } - await tokenStore.delete(tokenId: grant.tokenId) + await tokenStore.retire(tokenId: grant.tokenId) Self.logger.debug("Released Claude Agent MCP grant") } diff --git a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift index 162b793ee8..17f1871fa9 100644 --- a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift +++ b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift @@ -196,18 +196,35 @@ struct SQLStatementGenerator { return Set(bitColumns.keys) } + // Blank text goes as '', which the server stores as 0: cast, strict mode refuses it as a truncated integer, and a + // space bound as it is would be its byte, 32. private func bind(_ value: PluginCellValue, toColumn column: String?, into parameters: inout [Any?]) -> String { guard let column, numberBitColumns.contains(column), case .text(let text) = value else { parameters.append(value.asAny) return placeholder(at: parameters.count - 1) } + guard !text.trimmingCharacters(in: .whitespaces).isEmpty else { + parameters.append("") + return placeholder(at: parameters.count - 1) + } parameters.append(Self.decimalBitText(text)) return "CAST(\(placeholder(at: parameters.count - 1)) AS UNSIGNED)" } - // b'101' and 0b101 are SQL, not values: bound as text they would cast to 0. + // b'101' and 0b101 are SQL, not values, and a boolean word is no number: cast as text, each is 0 or an error. static func decimalBitText(_ text: String) -> String { let trimmed = text.trimmingCharacters(in: .whitespaces) + switch PluginSQLLiteral.booleanSynonym(for: trimmed) { + case .isTrue: return "1" + case .isFalse: return "0" + case nil: break + @unknown default: break + } + switch trimmed.lowercased() { + case "t": return "1" + case "f": return "0" + default: break + } let digits: Substring if trimmed.count > 3, trimmed.hasPrefix("b'") || trimmed.hasPrefix("B'"), trimmed.hasSuffix("'") { digits = trimmed.dropFirst(2).dropLast() diff --git a/apple/TablePro/Core/Compare/CompareSQLLiteral.swift b/apple/TablePro/Core/Compare/CompareSQLLiteral.swift index 4354e2a505..24cc1a74a8 100644 --- a/apple/TablePro/Core/Compare/CompareSQLLiteral.swift +++ b/apple/TablePro/Core/Compare/CompareSQLLiteral.swift @@ -50,6 +50,9 @@ internal enum CompareSQLLiteral { if let columnType, ColumnTypeSQLQuoting.isNumericLiteral(text, for: columnType) { return text } + if isBitNumber(text, columnType: columnType, databaseType: databaseType) { + return text + } // ClickHouse and MySQL read a backslash as one, so a value ending in `\'` left single closed the literal early // (#3777). The script is also copied and saved with no mode declared, so there MySQL gets its own set; a lookup // that only runs takes the session's escapes, so it finds a key holding a backslash. @@ -60,6 +63,14 @@ internal enum CompareSQLLiteral { return prefixed(SQLQuoting.stringLiteral(text, escapes: escapes), databaseType: databaseType) } + // MySQL reads a BIT back as the number it holds and takes a quoted one as its characters' bytes: '5' is 53. + private static func isBitNumber(_ text: String, columnType: ColumnType?, databaseType: DatabaseType) -> Bool { + let family = SQLTypeFamily.of(databaseType) + guard family.readsBitsAsDecimal, case .boolean(let rawType)? = columnType, let rawType, + case .bitString = SQLTypeParser.parse(rawType, family: family).kind else { return false } + return UInt64(text) != nil && text.allSatisfy { $0.isASCII && $0.isNumber } + } + // `N` in front of `NULL` or a bare number is a syntax error, so the opening quote decides. internal static func prefixed(_ literal: String, databaseType: DatabaseType) -> String { guard literal.unicodeScalars.first == "'" else { return literal } diff --git a/apple/TablePro/Core/Compare/DataSyncScriptBuilder.swift b/apple/TablePro/Core/Compare/DataSyncScriptBuilder.swift index 101f4ce620..5988478c3f 100644 --- a/apple/TablePro/Core/Compare/DataSyncScriptBuilder.swift +++ b/apple/TablePro/Core/Compare/DataSyncScriptBuilder.swift @@ -30,6 +30,8 @@ internal struct DataSyncScriptBuilder { private let options: DataCompareOptions private let plan: DataComparePlan private let targetTypes: [String: ColumnType] + // Bit strings the two sides spell differently, so a source value is respelled before it is written. + private let bitRespellings: [String: (pair: CrossEngineValueCoercer.ColumnPair, from: SQLTypeFamily)] private let immutableColumns: Set internal init( @@ -45,11 +47,37 @@ internal struct DataSyncScriptBuilder { self.plan = plan self.immutableColumns = Set(immutableColumns.map { $0.lowercased() }) var types: [String: ColumnType] = [:] + var respellings: [String: (pair: CrossEngineValueCoercer.ColumnPair, from: SQLTypeFamily)] = [:] for column in plan.columns { + if let respelling = Self.bitRespelling(of: column, targetDatabaseType: targetDatabaseType) { + respellings[column.name.lowercased()] = respelling + } guard let type = column.targetColumnType else { continue } types[column.name.lowercased()] = type } self.targetTypes = types + self.bitRespellings = respellings + } + + private static func bitRespelling( + of column: CompareColumn, + targetDatabaseType: DatabaseType + ) -> (pair: CrossEngineValueCoercer.ColumnPair, from: SQLTypeFamily)? { + guard let sourceFamily = column.sourceFamily, let sourceType = column.sourceType else { return nil } + let targetFamily = column.targetFamily ?? SQLTypeFamily.of(targetDatabaseType) + let pair = CrossEngineValueCoercer.ColumnPair( + source: SQLTypeParser.parse(sourceType, family: sourceFamily).kind, + target: SQLTypeParser.parse(column.targetType ?? sourceType, family: targetFamily).kind + ) + guard CrossEngineValueCoercer.respellsBits(pair, from: sourceFamily, to: targetFamily) else { return nil } + return (pair, sourceFamily) + } + + // A value read from the source, which the target may spell differently. + private func sourceLiteral(_ value: PluginCellValue, column: String) -> String { + guard let respelling = bitRespellings[column.lowercased()] else { return literal(value, column: column) } + let respelled = CrossEngineValueCoercer.respelledBits(value, pair: respelling.pair, from: respelling.from) + return literal(respelled, column: column) } internal func build(entries: [RowDiffEntry]) -> [SyncStatement] { @@ -132,7 +160,7 @@ internal struct DataSyncScriptBuilder { private func insertStatement(row: DataRow, entry: RowDiffEntry) -> SyncStatement { let columns = plan.writeColumns let columnList = columns.map { targetDriver.quoteIdentifier($0) }.joined(separator: ", ") - let valueList = columns.map { literal(row.value(for: $0), column: $0) }.joined(separator: ", ") + let valueList = columns.map { sourceLiteral(row.value(for: $0), column: $0) }.joined(separator: ", ") let override = identityInsertStyle == .overridingSystemValue ? " OVERRIDING SYSTEM VALUE" : "" return SyncStatement( sql: "INSERT INTO \(qualifiedTable) (\(columnList))\(override) VALUES (\(valueList))", @@ -151,7 +179,7 @@ internal struct DataSyncScriptBuilder { } + changedKeys guard !assignable.isEmpty, let predicate = keyPredicate(row: target) else { return nil } let assignments = assignable - .map { "\(targetDriver.quoteIdentifier($0)) = \(literal(source.value(for: $0), column: $0))" } + .map { "\(targetDriver.quoteIdentifier($0)) = \(sourceLiteral(source.value(for: $0), column: $0))" } .joined(separator: ", ") let table = qualifiedTable return SyncStatement( diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift index f451693062..aec7e5162d 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift @@ -578,7 +578,8 @@ extension QueryExecutionCoordinator { absentCells: result.absentCells, masks: parent.columnMasks(forTabId: tabId, table: nil, columns: columns, columnTypes: result.columnTypes), exportPolicy: parent.exportMaskingPolicy(forTabId: tabId), - exportTable: nil + exportTable: nil, + columnOrigins: result.columnOrigins ) ) resultSet.statementAnchor = anchor diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift index 1b775282a5..4289d16183 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift @@ -209,7 +209,8 @@ extension QueryExecutionCoordinator { rowLocators: [String?]? = nil, exportDocuments: [PluginExportDocument]? = nil, absentCells: [Int: Set] = [:], - unloadedCells: [Int: [Int: Int]] = [:] + unloadedCells: [Int: [Int: Int]] = [:], + columnOrigins: [PluginColumnOrigin?]? = nil ) { guard let idx = parent.tabManager.tabs.firstIndex(where: { $0.id == tabId }) else { return } @@ -272,7 +273,8 @@ extension QueryExecutionCoordinator { unloadedCells: unloadedCells, masks: masks, exportPolicy: parent.exportMaskingPolicy(forTabId: tabId), - exportTable: tableName + exportTable: tableName, + columnOrigins: columnOrigins ) let previousTableName = parent.tabManager.tabs[idx].tableContext.tableName let definitionChanged = parent.tabSessionRegistry.needsDefinition(existingTabId) diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift index e90dd0ded3..08d9ddbe9a 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift @@ -79,7 +79,8 @@ extension QueryExecutionCoordinator { absentCells: result.absentCells, masks: parent.columnMasks(forTabId: tabId, table: tableName, columns: columns, columnTypes: result.columnTypes), exportPolicy: parent.exportMaskingPolicy(forTabId: tabId), - exportTable: tableName + exportTable: tableName, + columnOrigins: result.columnOrigins ) let resultSet = ResultSet( label: ResultSet.label(tableName: tableName, anchor: anchor, index: index), diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift index ac3f5b8ec1..b3492b2e3b 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift @@ -763,7 +763,8 @@ extension QueryExecutionCoordinator { source: install.source, serverOutput: fetchResult.serverOutput, exportDocuments: fetchResult.exportDocuments, - absentCells: fetchResult.absentCells + absentCells: fetchResult.absentCells, + columnOrigins: fetchResult.columnOrigins ) let parameterValues = nativeParameters.map { $0 as? String } diff --git a/apple/TablePro/Core/CrossEngine/CrossEngineValueCoercer.swift b/apple/TablePro/Core/CrossEngine/CrossEngineValueCoercer.swift index c39008be0e..89c29b16ff 100644 --- a/apple/TablePro/Core/CrossEngine/CrossEngineValueCoercer.swift +++ b/apple/TablePro/Core/CrossEngine/CrossEngineValueCoercer.swift @@ -7,7 +7,7 @@ // Values cross as `PluginCellValue`, which is text, bytes or null, and each // one is bound as a parameter rather than written into SQL. That is already // enough for almost everything: a number, a string and a blob all mean the -// same thing to both sides. Three things do not, and each of them fails in a +// same thing to both sides. A few things do not, and each of them fails in a // way that is hard to read from the far end of a long copy. // // PostgreSQL renders a boolean as `t` and `f`. Bound into a MySQL @@ -38,13 +38,15 @@ internal struct CrossEngineValueCoercer: Sendable { private let pairs: [ColumnPair] private let sourceFamily: SQLTypeFamily + private let targetFamily: SQLTypeFamily? private let positions: [Int] - internal init(pairs: [ColumnPair], from sourceFamily: SQLTypeFamily) { + internal init(pairs: [ColumnPair], from sourceFamily: SQLTypeFamily, to targetFamily: SQLTypeFamily? = nil) { self.pairs = pairs self.sourceFamily = sourceFamily + self.targetFamily = targetFamily self.positions = pairs.enumerated() - .filter { Self.needsCoercion($0.element) } + .filter { Self.needsCoercion($0.element, from: sourceFamily, to: targetFamily) } .map(\.offset) } @@ -65,6 +67,9 @@ internal struct CrossEngineValueCoercer: Sendable { private func coerce(_ value: PluginCellValue, pair: ColumnPair) -> PluginCellValue { if Self.carriesBoolean(pair) { return boolean(value) } + if Self.respellsBits(pair, from: sourceFamily, to: targetFamily) { + return Self.respelledBits(value, pair: pair, from: sourceFamily) + } switch pair.target { case .date, .time, .timestamp: return temporal(value, kind: pair.target) @@ -121,6 +126,34 @@ internal struct CrossEngineValueCoercer: Sendable { } } + // MySQL reads a BIT as the number it holds, PostgreSQL and DuckDB as its digits, and each reads the other's + // spelling as another value: '5' is no bit string, and 101 is not 0b101. + internal static func respellsBits(_ pair: ColumnPair, from source: SQLTypeFamily, to target: SQLTypeFamily?) -> Bool { + guard let target, case .bitString = pair.source, case .bitString = pair.target else { return false } + return source.readsBitsAsDecimal != target.readsBitsAsDecimal + } + + // Only for a pair `respellsBits` accepts. Text that is no bit value of the source's reading is left as it came. + internal static func respelledBits(_ value: PluginCellValue, pair: ColumnPair, from sourceFamily: SQLTypeFamily) -> PluginCellValue { + guard case .text(let text) = value else { return value } + let trimmed = text.trimmingCharacters(in: .whitespaces) + guard sourceFamily.readsBitsAsDecimal else { + guard !trimmed.isEmpty, trimmed.allSatisfy({ $0 == "0" || $0 == "1" }), + let number = UInt64(trimmed, radix: 2) else { return value } + return .text(String(number)) + } + guard let number = UInt64(trimmed) else { return value } + let digits = String(number, radix: 2) + // A fixed-width bit string takes only its own length, so the digits are padded to it. + let width = Self.bitLength(pair.target) ?? Self.bitLength(pair.source) ?? digits.count + return .text(String(repeating: "0", count: max(0, width - digits.count)) + digits) + } + + private static func bitLength(_ kind: CanonicalTypeKind?) -> Int? { + guard case .bitString(let length) = kind else { return nil } + return length + } + private func temporal(_ value: PluginCellValue, kind: CanonicalTypeKind?) -> PluginCellValue { guard case .text(let text) = value else { return value } if sourceFamily == .mysql, Self.isZeroDate(text) { return .null } @@ -167,8 +200,8 @@ internal struct CrossEngineValueCoercer: Sendable { // MARK: - Shapes - private static func needsCoercion(_ pair: ColumnPair) -> Bool { - if carriesBoolean(pair) { return true } + private static func needsCoercion(_ pair: ColumnPair, from source: SQLTypeFamily, to target: SQLTypeFamily?) -> Bool { + if carriesBoolean(pair) || respellsBits(pair, from: source, to: target) { return true } switch pair.target { case .date, .time, .timestamp, .json: return true default: return false diff --git a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift index 1206c4838b..fb726e0bd5 100644 --- a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift +++ b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift @@ -36,6 +36,23 @@ extension DatabaseAccessBridge { maxRows: Int, timeoutSeconds: Int, cancellation: (any StatementCancellationSignal)? + ) async throws -> ScriptOutcome { + let reach = DriverReach() + do { + return try await runApprovedScript( + operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, reach: reach + ) + } catch { + throw await externalError(error, on: operation.scope.connectionId, reach: reach) + } + } + + private func runApprovedScript( + _ operation: ApprovedOperation, + maxRows: Int, + timeoutSeconds: Int, + cancellation: (any StatementCancellationSignal)?, + reach: DriverReach ) async throws -> ScriptOutcome { guard operation.unit == .script else { let outcome = try await runStatement( @@ -63,12 +80,13 @@ extension DatabaseAccessBridge { rowCap: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, - context: operation.context + context: operation.context, + reach: reach ) let policy = await maskingPolicy(for: scope.connectionId) return ScriptOutcome( - resultSets: outcome.resultSets.map { $0.masked(by: policy) }, - primary: outcome.primary.masked(by: policy), + resultSets: outcome.resultSets.map { $0.maskedForExternalCaller(by: policy) }, + primary: outcome.primary.maskedForExternalCaller(by: policy), executionTimeMs: outcome.executionTimeMs ) } @@ -83,7 +101,8 @@ extension DatabaseAccessBridge { rowCap: Int, timeoutSeconds: Int, cancellation: (any StatementCancellationSignal)?, - context: PluginStatementContext + context: PluginStatementContext, + reach: DriverReach ) async throws -> ScriptOutcome { let classification = QueryClassifier.classify(text, databaseType: databaseType) let connectionId = scope.connectionId @@ -114,7 +133,8 @@ extension DatabaseAccessBridge { timeoutSeconds: timeoutSeconds, onLateCompletion: { CatalogChangeService.post(statementsRan) } ) { driver in - try await ScriptBatchRun.run( + reach.mark() + return try await ScriptBatchRun.run( batches, startLines: startLines, rowCap: rowCap, driver: driver, progress: progress, context: context ) } @@ -275,6 +295,7 @@ struct ScriptBatchRun: Sendable { error: nil ) primary.isTruncated = first?.isTruncated ?? false + primary.columnOrigins = first?.columnOrigins let notes = [ first?.statusMessage, BatchRunNotice.text( diff --git a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift index e0c3b5f772..d3577f51c1 100644 --- a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift +++ b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift @@ -342,14 +342,32 @@ internal actor DatabaseAccessBridge { internal let executionTimeMs: Double } - // Runs the one approved statement as it was approved. `maskingTable` names the table the rows come from when - // the caller built the query for one, so a `table.column` rule matches by table. + // Runs the one approved statement as it was approved, its rows masked as `masking` says. internal func runStatement( _ operation: ApprovedOperation, maxRows: Int, timeoutSeconds: Int, cancellation: (any StatementCancellationSignal)?, - maskingTable: String? = nil + masking: ExternalRowMasking = .byOrigin + ) async throws -> StatementOutcome { + let reach = DriverReach() + do { + return try await runApprovedStatement( + operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, masking: masking, + reach: reach + ) + } catch { + throw await externalError(error, on: operation.scope.connectionId, reach: reach) + } + } + + private func runApprovedStatement( + _ operation: ApprovedOperation, + maxRows: Int, + timeoutSeconds: Int, + cancellation: (any StatementCancellationSignal)?, + masking: ExternalRowMasking, + reach: DriverReach ) async throws -> StatementOutcome { guard operation.unit == .statement, operation.statements.count == 1, let statement = operation.statements.first else { throw Self.notApproved } @@ -396,6 +414,7 @@ internal actor DatabaseAccessBridge { timeoutSeconds: timeoutSeconds, onLateCompletion: { CatalogChangeService.post(statementRan) } ) { driver in + reach.mark() let answer = try await driver.executeUserQuery( query: sql, rowCap: fetchCap, @@ -419,16 +438,46 @@ internal actor DatabaseAccessBridge { CatalogChangeService.post(.statementsSucceeded(tableEdits)) } let executionTimeMs = (CFAbsoluteTimeGetCurrent() - startTime) * 1_000 - let masking = await maskingPolicy(for: connectionId) - return StatementOutcome(result: result.masked(by: masking, table: maskingTable), executionTimeMs: executionTimeMs) + let rules = await maskingPolicy(for: connectionId) + let applied = await MainActor.run { masking.resolved(in: scope, databaseType: databaseType) } + return StatementOutcome(result: applied.apply(rules, to: result), executionTimeMs: executionTimeMs) } - // A plan the driver builds for the approved statement. + // A plan the driver builds for the approved statement, masked as `masking` says. internal func runExplainPlan( _ operation: ApprovedOperation, step: ExplainRequest.DriverStep, timeoutSeconds: Int, - cancellation: (any StatementCancellationSignal)? + cancellation: (any StatementCancellationSignal)?, + masking: ExternalRowMasking = .byOrigin + ) async throws -> StatementOutcome { + let reach = DriverReach() + do { + let outcome = try await runDriverPlan( + operation, step: step, timeoutSeconds: timeoutSeconds, cancellation: cancellation, reach: reach + ) + let rules = await maskingPolicy(for: operation.scope.connectionId) + return StatementOutcome(result: masking.apply(rules, to: outcome.result), executionTimeMs: outcome.executionTimeMs) + } catch { + throw await externalError(error, on: operation.scope.connectionId, reach: reach) + } + } + + // A server's message can repeat a value the statement read, as PostgreSQL's `invalid input syntax for type + // integer: "..."` does, so on a connection with masking rules an external caller gets the error's code only. + // What failed before a driver took the statement, a connect, a route or a pin, is the app's own and keeps its words. + internal func externalError(_ error: Error, on connectionId: UUID, reach: DriverReach) async -> Error { + guard reach.isReached, ServerErrorText.mayRepeatAValue(error) else { return error } + guard await !maskingPolicy(for: connectionId).isEmpty else { return error } + return ServerErrorText.withholding(error) + } + + private func runDriverPlan( + _ operation: ApprovedOperation, + step: ExplainRequest.DriverStep, + timeoutSeconds: Int, + cancellation: (any StatementCancellationSignal)?, + reach: DriverReach ) async throws -> StatementOutcome { guard operation.unit == .statement, operation.statements.count == 1, let approved = operation.statements.first else { throw Self.notApproved } @@ -461,7 +510,8 @@ internal actor DatabaseAccessBridge { timeoutSeconds: timeoutSeconds, onLateCompletion: { CatalogChangeService.post(statementRan) } ) { driver in - try await driver.explainPlan(statement: statement, variantId: variantId) + reach.mark() + return try await driver.explainPlan(statement: statement, variantId: variantId) } } catch { if writes { @@ -477,6 +527,15 @@ internal actor DatabaseAccessBridge { // One approved INSERT per row, on one write lease, each with the values it was approved with. internal func runInserts(_ operation: ApprovedOperation) async throws -> Int { + let reach = DriverReach() + do { + return try await runApprovedInserts(operation, reach: reach) + } catch { + throw await externalError(error, on: operation.scope.connectionId, reach: reach) + } + } + + private func runApprovedInserts(_ operation: ApprovedOperation, reach: DriverReach) async throws -> Int { guard operation.kind == .writeQuery, operation.unit == .statement, !operation.statements.isEmpty else { throw Self.notApproved } let scope = operation.scope @@ -492,6 +551,7 @@ internal actor DatabaseAccessBridge { route: route, cancellation: .protectedWrite ) { driver -> Int in + reach.mark() var affected = 0 for statement in statements { let result = try await driver.executeParameterized( @@ -512,12 +572,31 @@ internal actor DatabaseAccessBridge { filters: [TableFilter], logicMode: FilterLogicMode, exact: Bool + ) async throws -> (count: Int?, isApproximate: Bool) { + let reach = DriverReach() + do { + return try await countApprovedRows( + operation, table: table, filters: filters, logicMode: logicMode, exact: exact, reach: reach + ) + } catch { + throw await externalError(error, on: operation.scope.connectionId, reach: reach) + } + } + + private func countApprovedRows( + _ operation: ApprovedOperation, + table: String, + filters: [TableFilter], + logicMode: FilterLogicMode, + exact: Bool, + reach: DriverReach ) async throws -> (count: Int?, isApproximate: Bool) { guard operation.kind == .readQuery, operation.statements.isEmpty else { throw Self.notApproved } let scope = operation.scope try await ensureConnected(scope.connectionId) let manager = await databaseManager return try await manager.withMetadataDriver(scope: scope) { driver -> (count: Int?, isApproximate: Bool) in + reach.mark() if exact { let value = try await driver.fetchExactRowCount(table: table, filters: filters, logicMode: logicMode) return (value, false) @@ -530,7 +609,17 @@ internal actor DatabaseAccessBridge { } } + // A deferred constraint fails at COMMIT, naming the key values it checked. internal func transactionControl(_ operation: ApprovedOperation, action: String) async throws { + let reach = DriverReach() + do { + try await runTransactionControl(operation, action: action, reach: reach) + } catch { + throw await externalError(error, on: operation.scope.connectionId, reach: reach) + } + } + + private func runTransactionControl(_ operation: ApprovedOperation, action: String, reach: DriverReach) async throws { guard operation.kind == .writeQuery, operation.statements.isEmpty else { throw Self.notApproved } let scope = operation.scope try await ensureConnected(scope.connectionId) @@ -546,6 +635,7 @@ internal actor DatabaseAccessBridge { route: .sessionDriver, cancellation: .protectedWrite ) { driver in + reach.mark() switch action { case "begin": try await driver.beginTransaction(mode: .readWrite) case "commit": try await driver.commitTransaction() @@ -603,10 +693,25 @@ internal actor DatabaseAccessBridge { } // A grid save's plan, run only when it is the plan that was approved. + // An UPDATE that breaks a NOT NULL or CHECK constraint fails listing the row's other values. internal func runRowWrites( _ operation: ApprovedOperation, plan: DataWritePlan, before: @Sendable @escaping (DatabaseDriver, _ ownsConnection: Bool) async throws -> Void + ) async throws -> DataWriteRun { + let reach = DriverReach() + do { + return try await runApprovedRowWrites(operation, plan: plan, before: before, reach: reach) + } catch { + throw await externalError(error, on: operation.scope.connectionId, reach: reach) + } + } + + private func runApprovedRowWrites( + _ operation: ApprovedOperation, + plan: DataWritePlan, + before: @Sendable @escaping (DatabaseDriver, _ ownsConnection: Bool) async throws -> Void, + reach: DriverReach ) async throws -> DataWriteRun { guard operation.kind == .writeQuery, operation.scope == plan.scope, @@ -622,6 +727,7 @@ internal actor DatabaseAccessBridge { route: route, cancellation: .protectedWrite ) { driver in + reach.mark() try await before(driver, ownsConnection) return try await DataWriteExecutor.run(plan, on: driver, context: context) } @@ -802,6 +908,128 @@ internal actor DatabaseAccessBridge { } } +// How the rows an external caller reads meet the connection's masking rules. +internal enum ExternalRowMasking: Sendable, Equatable { + // The caller's own statement, which can rename or compute any column: matched by what each column reads. + case byOrigin + // A statement the app wrote that selects this table's own columns by name. + case table(String) + // A plan measured to hold only the statement's own literals, or an answer to a server call the app wrote: + // matched by column name. + case byName + + func apply(_ policy: MaskingPolicy, to result: QueryResult) -> QueryResult { + switch self { + case .byOrigin: return result.maskedForExternalCaller(by: policy) + case .table(let table): return result.maskedForExternalCaller(by: policy, table: table) + case .byName: return result.masked(by: policy) + } + } + + // Measured: PostgreSQL 17 plans, with ANALYZE and VERBOSE too, and SQLite 3.54 plans and bytecode hold only the + // statement's own literals. MariaDB writes the values it read from const tables into its plan, so a format nobody + // measured is masked as rows are, which masks a plan whole. + static func plan(in format: ExplainPlanFormat?) -> ExternalRowMasking { + guard let format, valueFreePlanFormats.contains(format) else { return .byOrigin } + return .byName + } + + private static let valueFreePlanFormats: Set = [.postgresJson, .sqliteQueryPlan] + + // The app's projection of a table proves by name only for a user table. A system one, ClickHouse's + // `system.query_log` or `pg_stat_activity`, repeats other tables' values and what sessions sent, so it is masked + // as a result with no origins is. The system names are the ones the engine's plugin declares. + @MainActor + func resolved(in scope: DatabaseScope, databaseType: DatabaseType) -> ExternalRowMasking { + guard case .table(let table) = self else { return self } + let databases = PluginManager.shared.systemContainerNames(for: databaseType, target: .database) + let schemas = PluginManager.shared.systemContainerNames(for: databaseType, target: .schema) + let containers = [scope.database] + [scope.schema].compactMap { $0 } + table.split(separator: ".").dropLast().map(String.init) + guard !containers.contains(where: { databases.contains($0) || schemas.contains($0) }) else { return .byOrigin } + return self + } +} + +// Set as a driver takes the statement. +internal final class DriverReach: Sendable { + private let reached = OSAllocatedUnfairLock(initialState: false) + + func mark() { + reached.withLock { $0 = true } + } + + var isReached: Bool { + reached.withLock { $0 } + } +} + +// What an external caller learns of a failed statement on a connection with masking rules. +internal enum ServerErrorText { + // The app's own refusals, timeouts and cancels name no value; whatever a driver or a server wrote might. + static func mayRepeatAValue(_ error: Error) -> Bool { + switch error { + case is CancellationError, is DatabaseAccessError, is ExternalStatementGateError, is ShortcutsActionError: + return false + case let failure as DatabaseError: + if case .queryFailed = failure { return true } + return false + case let refusal as DataWriteError: + // Only this case quotes the driver; the rest are the app's own counts and names. + if case .changeRefused = refusal { return true } + return false + default: + return true + } + } + + static func withholding(_ error: Error) -> Error { + guard mayRepeatAValue(error) else { return error } + guard let partial = error as? DataWritePartialCommitError else { + return DatabaseAccessError.dataSourceError(withheld(for: error)) + } + // What was already written is the app's own account, and what the caller must do next depends on it. + return DataWritePartialCommitError( + committed: partial.committed, + totalStatements: partial.totalStatements, + engine: partial.engine, + disposition: partial.disposition, + underlying: withholding(partial.underlying) + ) + } + + static var withheldMessage: String { + String( + localized: "The statement failed. Its message is withheld because this connection has masking rules, and the message can repeat a masked value." + ) + } + + static var withheldStatementText: String { + String(localized: "The statement text is withheld because this connection has masking rules, and it can repeat a masked value.") + } + + static func withheld(for error: Error) -> String { + let labels = (error as? any PluginDriverError).map { Self.codes(of: $0) } ?? [] + guard !labels.isEmpty else { return withheldMessage } + return String( + format: String( + localized: "The statement failed with %@. Its message is withheld because this connection has masking rules, and the message can repeat a masked value." + ), + labels.joined(separator: ", ") + ) + } + + private static func codes(of error: any PluginDriverError) -> [String] { + var codes: [String] = [] + if let code = error.pluginErrorCode, code != 0 { + codes.append("error \(code)") + } + if let state = error.pluginSqlState, !state.isEmpty { + codes.append("SQLSTATE \(state)") + } + return codes + } +} + /// What a transport offers a running statement so its own cancellation reaches the driver. /// /// MCP has `notifications/cancelled` and passes its token; AppleScript has no such notification and diff --git a/apple/TablePro/Core/Database/DatabaseManager+Schema.swift b/apple/TablePro/Core/Database/DatabaseManager+Schema.swift index 1f5e81f63a..fd4b884352 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Schema.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Schema.swift @@ -83,7 +83,13 @@ extension DatabaseManager { Self.logger.error("Rollback failed after schema change error: \(error.localizedDescription)") } } - throw SchemaChangeFailedAfterWriting(message: "Schema change failed: \(error.localizedDescription)") + let applied: [SchemaStatement] = useTransaction + ? [] + : statements.prefix(measured.count).filter { !$0.carriesCredentials } + throw SchemaChangeFailedAfterWriting( + message: "Schema change failed: \(error.localizedDescription)", + appliedStatements: applied.map(\.sql) + ) } try await Self.confirmFinished(script, scope: scope, on: driver) return measured @@ -93,7 +99,7 @@ extension DatabaseManager { } catch let failure as SchemaChangeFailedAfterWriting { Self.reportCatalogChangeAfterFailure(in: scope) reportSavedChange(of: script, in: scope) - throw DatabaseError.queryFailed(failure.message) + throw DatabaseError.queryFailed(failure.reportedMessage) } catch { Self.reportCatalogChangeAfterFailure(in: scope) throw error @@ -335,6 +341,14 @@ struct CreateTableIncompleteError: LocalizedError { /// changed even though the save did not finish. private struct SchemaChangeFailedAfterWriting: Error { let message: String + // Statements an engine that commits each DDL statement kept after a later one failed. + var appliedStatements: [String] = [] + + var reportedMessage: String { + guard !appliedStatements.isEmpty else { return message } + let heading = String(localized: "These statements ran before the failure and were not rolled back:") + return message + "\n\n" + ([heading] + appliedStatements).joined(separator: "\n") + } } private extension SchemaChangeScript { diff --git a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift index aeb3174daa..2e0bfa525e 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift @@ -658,6 +658,12 @@ extension DatabaseManager { guard let previous = session(for: connectionId) else { throw DatabaseError.notConnected } + // The reconnect would end a live transaction; one on a connection that already stopped answering is gone anyway. + if previous.liveness == .live, await browsedDriverHoldsTransaction(previous.driver, for: connectionId) { + throw DatabaseError.connectionFailed( + Self.switchEndsTransactionMessage(database: previous.resolvedBrowseDatabase) + ) + } let previousDatabase = previous.connection.database let previousBrowseDatabase = previous.browseDatabase let previousBrowseSchema = previous.browseSchema @@ -688,6 +694,27 @@ extension DatabaseManager { } } + // The tracker's last answer, or the driver's own: a refusal over a transaction that is there is never wrong. + private func browsedDriverHoldsTransaction(_ driver: DatabaseDriver?, for connectionId: UUID) async -> Bool { + guard let driver, driver.supportsTransactions else { return false } + let tracked = sessionTransactions.holds(for: connectionId).contains { sessionTransactions.driver(for: $0) === driver } + if tracked { return true } + let state = await Self.boundedTransactionState(of: driver, clock: ContinuousClock()) + return state == .inTransaction || state == .abortedTransaction + } + + static func switchEndsTransactionMessage(database: String) -> String { + String( + format: String( + localized: """ + A transaction is open on %@. Switching databases reconnects, and the server would \ + roll it back. Commit or roll back the transaction, then switch again. + """ + ), + database + ) + } + /// In the same turn as the session's browse cursor, so nothing reads one database's catalog under /// another's name. private func showBrowsedCatalog(_ connectionId: UUID) { diff --git a/apple/TablePro/Core/MCP/Auth/MCPBearerTokenAuthenticator.swift b/apple/TablePro/Core/MCP/Auth/MCPBearerTokenAuthenticator.swift index 276094fe9f..c93f2427a7 100644 --- a/apple/TablePro/Core/MCP/Auth/MCPBearerTokenAuthenticator.swift +++ b/apple/TablePro/Core/MCP/Auth/MCPBearerTokenAuthenticator.swift @@ -12,6 +12,7 @@ public struct MCPValidatedToken: Sendable, Equatable { public let issuedAt: Date public let expiresAt: Date? public let isBridgeCredential: Bool + public let isClaudeAgentCredential: Bool public init( tokenId: UUID, @@ -20,13 +21,15 @@ public struct MCPValidatedToken: Sendable, Equatable { connectionAccess: ConnectionAccess, issuedAt: Date, expiresAt: Date?, - isBridgeCredential: Bool = false + isBridgeCredential: Bool = false, + isClaudeAgentCredential: Bool = false ) { self.tokenId = tokenId self.label = label self.scopes = scopes self.connectionAccess = connectionAccess self.isBridgeCredential = isBridgeCredential + self.isClaudeAgentCredential = isClaudeAgentCredential self.issuedAt = issuedAt self.expiresAt = expiresAt } @@ -54,7 +57,8 @@ internal extension MCPTokenStore { connectionAccess: authToken.connectionAccess, issuedAt: authToken.createdAt, expiresAt: authToken.expiresAt, - isBridgeCredential: authToken.isBridgeCredential + isBridgeCredential: authToken.isBridgeCredential, + isClaudeAgentCredential: authToken.isClaudeAgentCredential ) } } @@ -127,7 +131,8 @@ public actor MCPBearerTokenAuthenticator: MCPAuthenticator { label: validated.label, issuedAt: validated.issuedAt, expiresAt: validated.expiresAt, - isBridgeCredential: validated.isBridgeCredential + isBridgeCredential: validated.isBridgeCredential, + isClaudeAgentCredential: validated.isClaudeAgentCredential ) ) MCPAuditLogger.logAuthSuccess( diff --git a/apple/TablePro/Core/MCP/Lifecycle/MCPServerManager.swift b/apple/TablePro/Core/MCP/Lifecycle/MCPServerManager.swift index ad761974f2..cc9c7020ee 100644 --- a/apple/TablePro/Core/MCP/Lifecycle/MCPServerManager.swift +++ b/apple/TablePro/Core/MCP/Lifecycle/MCPServerManager.swift @@ -168,8 +168,14 @@ internal final class MCPServerManager: ObservableObject { return true } + var withdrawn: MCPGrantSubject? = .token(tokenId) + if let ended = await tokenStore?.token(id: tokenId) { + withdrawn = ended.subjectWithdrawnWhenEnded + } let persisted = await revokeToken(id: tokenId) - await composition.authPolicy.clearApprovals(tokenId: tokenId) + if let withdrawn { + await composition.authPolicy.clearApprovals(subject: withdrawn) + } await composition.activityLedger.forget(tokenId: tokenId) await refreshClients() return persisted @@ -318,6 +324,7 @@ internal final class MCPServerManager: ObservableObject { connectionAccess: .all, expiresAt: expiresAt, isBridgeCredential: true, + isClaudeAgentCredential: false, extraScopes: [] ) return BridgeCredential( @@ -418,11 +425,11 @@ internal final class MCPServerManager: ObservableObject { store: MCPTokenStore, generation: Int ) async { - revocationObserverId = await store.addRevocationObserver { [weak self] rawTokenId, wasBridge in + revocationObserverId = await store.addRevocationObserver { [weak self] rawTokenId, withdrawn in guard let tokenId = UUID(uuidString: rawTokenId), let self else { return } await self.handleTokenRevoked( tokenId: tokenId, - wasBridgeCredential: wasBridge, + withdrawing: withdrawn, dispatcher: built.dispatcher, generation: generation ) @@ -431,16 +438,15 @@ internal final class MCPServerManager: ObservableObject { private func handleTokenRevoked( tokenId: UUID, - wasBridgeCredential: Bool, + withdrawing subject: MCPGrantSubject?, dispatcher: MCPProtocolDispatcher, generation: Int ) async { guard isCurrent(generation) else { return } let cancelled = await dispatcher.cancelInflight(matchingTokenId: tokenId) - await composition?.authPolicy.clearApprovals( - tokenId: tokenId, - wasBridgeCredential: wasBridgeCredential - ) + if let subject { + await composition?.authPolicy.clearApprovals(subject: subject) + } guard cancelled > 0 else { return } Self.logger.info( "Token \(tokenId.uuidString, privacy: .public) revoked: cancelled \(cancelled, privacy: .public) request(s)" diff --git a/apple/TablePro/Core/MCP/MCPAuthPolicy.swift b/apple/TablePro/Core/MCP/MCPAuthPolicy.swift index 1a0e67e84e..1fc13c4522 100644 --- a/apple/TablePro/Core/MCP/MCPAuthPolicy.swift +++ b/apple/TablePro/Core/MCP/MCPAuthPolicy.swift @@ -274,7 +274,11 @@ public actor MCPAuthPolicy { /// passes the token's name and the bridge's own rotation is skipped. func clearApprovals(tokenId: UUID, wasBridgeCredential: Bool = false) async { guard !wasBridgeCredential else { return } - await approvalLedger.clear(subject: .token(tokenId)) + await clearApprovals(subject: .token(tokenId)) + } + + func clearApprovals(subject: MCPGrantSubject) async { + await approvalLedger.clear(subject: subject) } func forgetEveryGrant() async { diff --git a/apple/TablePro/Core/MCP/MCPConnectionBridge+Data.swift b/apple/TablePro/Core/MCP/MCPConnectionBridge+Data.swift index 20dd275e4e..6912d91cf7 100644 --- a/apple/TablePro/Core/MCP/MCPConnectionBridge+Data.swift +++ b/apple/TablePro/Core/MCP/MCPConnectionBridge+Data.swift @@ -107,7 +107,7 @@ extension MCPConnectionBridge { maxRows: limit, timeoutSeconds: timeoutSeconds, cancellation: cancellation, - maskingTable: request.table + masking: .table(request.table) ) if case .object(var fields) = payload { fields["table"] = .string(request.table) @@ -285,6 +285,8 @@ struct MCPExplainRequest: Equatable { let subjectSQL: String let driverStep: ExplainRequest.DriverStep? + // The plan's shape: the variant's, or for a statement the caller wrote as EXPLAIN, the engine's first variant's. + var format: ExplainPlanFormat? } extension MCPConnectionBridge { @@ -295,6 +297,7 @@ extension MCPConnectionBridge { cancellation: MCPCancellationToken? ) async throws -> JsonValue { let databaseType = operation.databaseType + let masking = ExternalRowMasking.plan(in: request.format) let outcome: (result: QueryResult, executionTimeMs: Double) let routed: ExplainResultRouter.RoutedPlan? if let step = request.driverStep { @@ -302,7 +305,8 @@ extension MCPConnectionBridge { operation, step: step, timeoutSeconds: timeoutSeconds, - cancellation: cancellation + cancellation: cancellation, + masking: masking ) routed = ExplainResultRouter.routeDriverPlan( rows: outcome.result.rows, @@ -316,7 +320,8 @@ extension MCPConnectionBridge { operation, maxRows: Self.explainRowLimit, timeoutSeconds: timeoutSeconds, - cancellation: cancellation + cancellation: cancellation, + masking: masking ) routed = ExplainResultRouter.route( sql: request.statement, @@ -333,6 +338,9 @@ extension MCPConnectionBridge { "columns": .array(outcome.result.columns.map { .string($0) }), "rows": .array(outcome.result.rows.map { row in .array(row.map(Self.cellValue)) }) ] + if let statusMessage = outcome.result.statusMessage { + payload["status_message"] = .string(statusMessage) + } if let routed { payload["plan_text"] = .string(routed.rawText) if let plan = routed.plan { @@ -380,7 +388,9 @@ extension MCPConnectionBridge { throw DatabaseAccessError.invalidArgument(String(localized: "The query is empty.")) } guard !QueryClassifier.isExplainStatement(trimmed, grammar: databaseType.lexicalGrammar) else { - return MCPExplainRequest(statement: trimmed, gatedSQL: trimmed, subjectSQL: trimmed, driverStep: nil) + return MCPExplainRequest( + statement: trimmed, gatedSQL: trimmed, subjectSQL: trimmed, driverStep: nil, format: variants.first?.format + ) } guard let first = variants.first else { throw DatabaseAccessError.invalidArgument(String(localized: "This database does not explain statements.")) @@ -389,7 +399,9 @@ extension MCPConnectionBridge { switch variant.step { case .sqlPrefix: let prefixed = "\(variant.sqlPrefix) \(trimmed)" - return MCPExplainRequest(statement: prefixed, gatedSQL: prefixed, subjectSQL: trimmed, driverStep: nil) + return MCPExplainRequest( + statement: prefixed, gatedSQL: prefixed, subjectSQL: trimmed, driverStep: nil, format: variant.format + ) case .driver(let runsStatement): return MCPExplainRequest( statement: trimmed, @@ -397,7 +409,8 @@ extension MCPConnectionBridge { subject: trimmed, runsStatement: runsStatement, databaseType: databaseType ), subjectSQL: trimmed, - driverStep: ExplainRequest.DriverStep(variantId: variant.id, runsStatement: runsStatement) + driverStep: ExplainRequest.DriverStep(variantId: variant.id, runsStatement: runsStatement), + format: variant.format ) @unknown default: throw DatabaseAccessError.invalidArgument( diff --git a/apple/TablePro/Core/MCP/MCPConnectionBridge+Server.swift b/apple/TablePro/Core/MCP/MCPConnectionBridge+Server.swift index 4e1f43278c..ed5436db9e 100644 --- a/apple/TablePro/Core/MCP/MCPConnectionBridge+Server.swift +++ b/apple/TablePro/Core/MCP/MCPConnectionBridge+Server.swift @@ -93,6 +93,8 @@ extension MCPConnectionBridge { guard let scope else { throw DatabaseAccessError.notConnected(connectionId) } + // A session's statement, idle or slow, is what a client sent, literals and all. + let withholdsQueries = await !maskingPolicy(for: connectionId).isEmpty return try await DatabaseManager.shared.withMetadataDriver(scope: scope) { driver in guard let provider = ServerDashboardQueryProviderFactory.provider( @@ -111,7 +113,9 @@ extension MCPConnectionBridge { if panels.contains("sessions") { do { let sessions = try await provider.fetchSessions(execute: execute) - result["sessions"] = .array(sessions.map { Self.sessionPayload($0, provider: provider) }) + result["sessions"] = .array(sessions.map { + Self.sessionPayload($0, provider: provider, withholdingQuery: withholdsQueries) + }) } catch { failures["sessions"] = Self.dashboardPanelFailure(panel: "sessions", error: error) } @@ -134,14 +138,7 @@ extension MCPConnectionBridge { if panels.contains("slow_queries") { do { let slow = try await provider.fetchSlowQueries(execute: execute) - result["slow_queries"] = .array(slow.map { entry in - .object([ - "duration": .string(entry.duration), - "query": .string(entry.query), - "user": .string(entry.user), - "database": .string(entry.database) - ]) - }) + result["slow_queries"] = .array(slow.map { Self.slowQueryPayload($0, withholdingQuery: withholdsQueries) }) } catch { failures["slow_queries"] = Self.dashboardPanelFailure(panel: "slow_queries", error: error) } @@ -150,19 +147,32 @@ extension MCPConnectionBridge { } } - static func sessionPayload(_ session: DashboardSession, provider: ServerDashboardQueryProvider) -> JsonValue { + static func sessionPayload( + _ session: DashboardSession, + provider: ServerDashboardQueryProvider, + withholdingQuery: Bool = false + ) -> JsonValue { .object([ "id": .string(session.id), "user": .string(session.user), "database": .string(session.database), "state": .string(session.state), "duration_seconds": .int(session.durationSeconds), - "query": .string(session.query), + "query": .string(withholdingQuery ? ServerErrorText.withheldStatementText : session.query), "can_kill": .bool(provider.canKill(session)), "can_cancel": .bool(provider.canCancel(session)) ]) } + static func slowQueryPayload(_ entry: DashboardSlowQuery, withholdingQuery: Bool = false) -> JsonValue { + .object([ + "duration": .string(entry.duration), + "query": .string(withholdingQuery ? ServerErrorText.withheldStatementText : entry.query), + "user": .string(entry.user), + "database": .string(entry.database) + ]) + } + /// The paired client is told which panel the server refused, never what the server said. The /// server's own text can name a role, a database, a column or a statement, and this reply /// leaves the user's Mac; the full message stays in the log for the user to read. @@ -458,9 +468,11 @@ extension MCPConnectionBridge { default: since = nil } + let masksValues = await !maskingPolicy(for: connectionId).isEmpty let page = await QueryHistoryManager.shared.fetch( QueryHistoryFilter( scope: .connection(connectionId), + sources: Self.searchableSources(by: search, masksValues: masksValues), searchText: search, since: since, allowedConnectionIds: [connectionId] @@ -468,7 +480,8 @@ extension MCPConnectionBridge { limit: limit ) - return .object(["history": .array(page.entries.map(Self.encode(historyEntry:)))]) + let entries = page.entries.filter { Self.isSearchable($0, by: search, masksValues: masksValues) } + return .object(["history": .array(entries.map { Self.encode(historyEntry: $0, masksValues: masksValues) })]) } static func scopePath(_ scope: PluginPrivilegeScope) -> String { @@ -488,10 +501,11 @@ extension MCPConnectionBridge { } } - static func encode(historyEntry entry: QueryHistoryEntry) -> JsonValue { + // `masksValues` is whether the entry's connection has masking rules. + static func encode(historyEntry entry: QueryHistoryEntry, masksValues: Bool = false) -> JsonValue { var fields: [String: JsonValue] = [ "id": .string(entry.id.uuidString), - "query": .string(entry.query), + "query": .string(externalQueryText(of: entry, masksValues: masksValues)), "connection_id": .string(entry.connectionId.uuidString), "database_name": .string(entry.databaseName), "database_type": .string(entry.databaseType.rawValue), @@ -506,10 +520,38 @@ extension MCPConnectionBridge { fields["schema_name"] = .string(schemaName) } if let error = entry.errorMessage { - fields["error_message"] = .string(MCPErrorRedactor.redact(error)) + fields["error_message"] = .string(externalErrorText(error, masksValues: masksValues)) } return .object(fields) } + + // A statement the app wrote from the data itself carries stored values: a document edit's `replaceOne` holds every + // field of the original and the edit, an import the file's values. What a person or a client typed is theirs. + static func externalQueryText(of entry: QueryHistoryEntry, masksValues: Bool) -> String { + withholdsQuery(of: entry, masksValues: masksValues) ? ServerErrorText.withheldStatementText : entry.query + } + + // The search matches the stored statement, so an entry whose statement is withheld would answer for its text. + static func isSearchable(_ entry: QueryHistoryEntry, by search: String?, masksValues: Bool) -> Bool { + guard let search, !search.isEmpty else { return true } + return !withholdsQuery(of: entry, masksValues: masksValues) + } + + // Left out of the storage query itself, so a withheld match never takes a slot in the page and its size answers nothing. + static func searchableSources(by search: String?, masksValues: Bool) -> Set { + let all = Set(QueryHistorySource.allCases) + guard masksValues, let search, !search.isEmpty else { return all } + return all.subtracting([.rowEdit, .dataImport]) + } + + private static func withholdsQuery(of entry: QueryHistoryEntry, masksValues: Bool) -> Bool { + masksValues && (entry.source == .rowEdit || entry.source == .dataImport) + } + + // A failure the editor or a grid save recorded keeps the server's words, which can repeat a value it read. + static func externalErrorText(_ error: String, masksValues: Bool) -> String { + masksValues ? ServerErrorText.withheldMessage : MCPErrorRedactor.redact(error) + } } extension MCPConnectionBridge { diff --git a/apple/TablePro/Core/MCP/MCPConnectionBridge.swift b/apple/TablePro/Core/MCP/MCPConnectionBridge.swift index 676b475379..8a3f3dcbd2 100644 --- a/apple/TablePro/Core/MCP/MCPConnectionBridge.swift +++ b/apple/TablePro/Core/MCP/MCPConnectionBridge.swift @@ -86,6 +86,10 @@ public actor MCPConnectionBridge { return .object(result) } + func maskingPolicy(for connectionId: UUID) async -> MaskingPolicy { + await access.maskingPolicy(for: connectionId) + } + func resolveScope(connectionId: UUID, database: String?, schema: String?) async throws -> DatabaseScope { try await access.resolveScope(connectionId: connectionId, database: database, schema: schema) } @@ -121,14 +125,14 @@ public actor MCPConnectionBridge { maxRows: Int, timeoutSeconds: Int, cancellation: MCPCancellationToken?, - maskingTable: String? = nil + masking: ExternalRowMasking = .byOrigin ) async throws -> JsonValue { let outcome = try await runStatement( operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, - maskingTable: maskingTable + masking: masking ) return Self.encode( result: outcome.result, @@ -157,14 +161,14 @@ public actor MCPConnectionBridge { maxRows: Int, timeoutSeconds: Int, cancellation: MCPCancellationToken?, - maskingTable: String? = nil + masking: ExternalRowMasking = .byOrigin ) async throws -> (result: QueryResult, executionTimeMs: Double) { let outcome = try await access.runStatement( operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, - maskingTable: maskingTable + masking: masking ) return (outcome.result, outcome.executionTimeMs) } @@ -173,13 +177,15 @@ public actor MCPConnectionBridge { _ operation: ApprovedOperation, step: ExplainRequest.DriverStep, timeoutSeconds: Int, - cancellation: MCPCancellationToken? + cancellation: MCPCancellationToken?, + masking: ExternalRowMasking = .byOrigin ) async throws -> (result: QueryResult, executionTimeMs: Double) { let outcome = try await access.runExplainPlan( operation, step: step, timeoutSeconds: timeoutSeconds, - cancellation: cancellation + cancellation: cancellation, + masking: masking ) return (outcome.result, outcome.executionTimeMs) } diff --git a/apple/TablePro/Core/MCP/MCPPairingService.swift b/apple/TablePro/Core/MCP/MCPPairingService.swift index 4333769928..23a1e369fe 100644 --- a/apple/TablePro/Core/MCP/MCPPairingService.swift +++ b/apple/TablePro/Core/MCP/MCPPairingService.swift @@ -179,6 +179,7 @@ final class MCPPairingService { connectionAccess: connectionAccess, expiresAt: approval.expiresAt, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: extraScopes ) diff --git a/apple/TablePro/Core/MCP/MCPTokenStore.swift b/apple/TablePro/Core/MCP/MCPTokenStore.swift index f7dde8f6d0..0e01d81646 100644 --- a/apple/TablePro/Core/MCP/MCPTokenStore.swift +++ b/apple/TablePro/Core/MCP/MCPTokenStore.swift @@ -36,6 +36,8 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { var isActive: Bool // Set only by the bridge's own mint, never from the name, which a pairing request supplies. let isBridgeCredential: Bool + // Set only by the Claude Agent chat's own mint, on the same terms. + let isClaudeAgentCredential: Bool let extraScopes: Set var scopes: Set { @@ -49,6 +51,15 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { var isEffectivelyActive: Bool { isActive && !isExpired } + var grantSubject: MCPGrantSubject { + MCPGrantSubject(tokenId: id, isBridgeCredential: isBridgeCredential, isClaudeAgentCredential: isClaudeAgentCredential) + } + + // The bridge's credential rotates on its own, so ending one withdraws nothing the bridge was granted. + var subjectWithdrawnWhenEnded: MCPGrantSubject? { + isBridgeCredential ? nil : grantSubject + } + init( id: UUID, name: String, @@ -62,6 +73,7 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { expiresAt: Date?, isActive: Bool, isBridgeCredential: Bool = false, + isClaudeAgentCredential: Bool = false, extraScopes: Set = [] ) { self.id = id @@ -76,6 +88,7 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { self.expiresAt = expiresAt self.isActive = isActive self.isBridgeCredential = isBridgeCredential + self.isClaudeAgentCredential = isClaudeAgentCredential self.extraScopes = extraScopes.intersection(MCPScope.optionalGrants) } @@ -92,6 +105,7 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { case expiresAt case isActive case isBridgeCredential + case isClaudeAgentCredential case extraScopes } @@ -109,6 +123,7 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { self.isActive = try container.decode(Bool.self, forKey: .isActive) self.connectionAccess = try container.decodeIfPresent(ConnectionAccess.self, forKey: .connectionAccess) ?? .all self.isBridgeCredential = try container.decodeIfPresent(Bool.self, forKey: .isBridgeCredential) ?? false + self.isClaudeAgentCredential = try container.decodeIfPresent(Bool.self, forKey: .isClaudeAgentCredential) ?? false // Lenient, so a bad value costs this grant only; the filter keeps a tier scope such as admin out. let storedExtraScopes = (try? container.decodeIfPresent([String].self, forKey: .extraScopes)) ?? [] self.extraScopes = Set(storedExtraScopes.compactMap(MCPScope.init(rawValue:))) @@ -128,6 +143,9 @@ struct MCPAuthToken: Codable, Identifiable, Sendable { try container.encodeIfPresent(lastUsedAt, forKey: .lastUsedAt) try container.encodeIfPresent(expiresAt, forKey: .expiresAt) try container.encode(isBridgeCredential, forKey: .isBridgeCredential) + if isClaudeAgentCredential { + try container.encode(isClaudeAgentCredential, forKey: .isClaudeAgentCredential) + } try container.encode(isActive, forKey: .isActive) if !extraScopes.isEmpty { try container.encode(extraScopes.map(\.rawValue).sorted(), forKey: .extraScopes) @@ -182,7 +200,8 @@ actor MCPTokenStore { private var lastSavedAt: ContinuousClock.Instant = .now private static let saveCooldown: Duration = .seconds(60) - private var revocationObservers: [UUID: @Sendable (String, Bool) async -> Void] = [:] + // Called with the token id and the subject whose approvals ended with it, nil when they outlive it. + private var revocationObservers: [UUID: @Sendable (String, MCPGrantSubject?) async -> Void] = [:] init(credentialStore: any MCPTokenCredentialStoring = MCPTokenKeychainStore()) { self.credentialStore = credentialStore @@ -191,7 +210,7 @@ actor MCPTokenStore { } @discardableResult - func addRevocationObserver(_ handler: @escaping @Sendable (String, Bool) async -> Void) -> UUID { + func addRevocationObserver(_ handler: @escaping @Sendable (String, MCPGrantSubject?) async -> Void) -> UUID { let id = UUID() revocationObservers[id] = handler return id @@ -208,6 +227,7 @@ actor MCPTokenStore { connectionAccess: ConnectionAccess, expiresAt: Date?, isBridgeCredential: Bool, + isClaudeAgentCredential: Bool, extraScopes: Set ) throws -> (token: MCPAuthToken, plaintext: String) { let name = (!isBridgeCredential && name == Self.stdioBridgeTokenName) @@ -235,6 +255,7 @@ actor MCPTokenStore { expiresAt: expiresAt, isActive: true, isBridgeCredential: isBridgeCredential, + isClaudeAgentCredential: isClaudeAgentCredential && !isBridgeCredential, extraScopes: isBridgeCredential ? [] : extraScopes ) @@ -270,9 +291,9 @@ actor MCPTokenStore { tokens[index].isActive = false let revokedName = tokens[index].name - let wasBridge = tokens[index].isBridgeCredential + let withdrawn = tokens[index].subjectWithdrawnWhenEnded let persisted = save() - notifyRevocationObservers(tokenId: tokenId, wasBridgeCredential: wasBridge) + notifyRevocationObservers(tokenId: tokenId, withdrawing: withdrawn) Self.logger.info("Revoked MCP token '\(revokedName, privacy: .private(mask: .hash))'") MCPAuditLogger.logTokenRevoked(tokenId: tokenId, tokenName: revokedName) @@ -292,25 +313,34 @@ actor MCPTokenStore { } func delete(tokenId: UUID) { + remove(tokenId: tokenId, withdrawsApprovals: true) + } + + // A Claude Agent turn ends its own token this way: the approvals belong to the agent, not to one turn. + func retire(tokenId: UUID) { + remove(tokenId: tokenId, withdrawsApprovals: false) + } + + private func remove(tokenId: UUID, withdrawsApprovals: Bool) { guard let index = tokens.firstIndex(where: { $0.id == tokenId }) else { Self.logger.warning("Attempted to delete non-existent token \(tokenId.uuidString, privacy: .public)") return } let name = tokens[index].name - let wasBridge = tokens[index].isBridgeCredential + let withdrawn = withdrawsApprovals ? tokens[index].subjectWithdrawnWhenEnded : nil tokens.remove(at: index) save() - notifyRevocationObservers(tokenId: tokenId, wasBridgeCredential: wasBridge) + notifyRevocationObservers(tokenId: tokenId, withdrawing: withdrawn) Self.logger.info("Deleted MCP token '\(name, privacy: .private(mask: .hash))'") } - private func notifyRevocationObservers(tokenId: UUID, wasBridgeCredential: Bool) { + private func notifyRevocationObservers(tokenId: UUID, withdrawing subject: MCPGrantSubject?) { let observers = Array(revocationObservers.values) let key = tokenId.uuidString for observer in observers { - Task { await observer(key, wasBridgeCredential) } + Task { await observer(key, subject) } } } diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/ExplainQueryTool.swift b/apple/TablePro/Core/MCP/Protocol/Tools/ExplainQueryTool.swift index 41a2eec9ec..057a66292c 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/ExplainQueryTool.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/ExplainQueryTool.swift @@ -53,6 +53,7 @@ public struct ExplainQueryTool: MCPToolImplementation { of: .object(["type": .string("array"), "items": MCPToolSchema.cell]) ), "plan_text": MCPToolSchema.string(String(localized: "Flattened plan text")), + "status_message": MCPToolSchema.string(String(localized: "Why values are masked, when they are")), "plan": MCPToolSchema.object( properties: [ "root": MCPToolSchema.anyValue, diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/ExportDataTool.swift b/apple/TablePro/Core/MCP/Protocol/Tools/ExportDataTool.swift index 5a827a6e73..14e1f492ab 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/ExportDataTool.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/ExportDataTool.swift @@ -197,7 +197,8 @@ public struct ExportDataTool: MCPToolImplementation { maxRows: maxRows + 1, timeoutSeconds: timeoutSeconds, context: context, - secrets: meta.redactionSecrets + secrets: meta.redactionSecrets, + masking: statement.masking ) let rowCount = min(result.rows.count, maxRows) @@ -260,6 +261,7 @@ public struct ExportDataTool: MCPToolImplementation { struct ExportStatement { let label: String let sql: String + var masking: ExternalRowMasking = .byOrigin } static func buildStatements( @@ -294,7 +296,8 @@ public struct ExportDataTool: MCPToolImplementation { .joined(separator: ".") return ExportStatement( label: table, - sql: limitedSelect(from: quoted, limit: maxRows + 1, style: dialect.autoLimitStyle) + sql: limitedSelect(from: quoted, limit: maxRows + 1, style: dialect.autoLimitStyle), + masking: .table(table) ) } } diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/MaintenanceTools.swift b/apple/TablePro/Core/MCP/Protocol/Tools/MaintenanceTools.swift index c90e5f7e22..832833c5bb 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/MaintenanceTools.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/MaintenanceTools.swift @@ -215,7 +215,8 @@ public struct RunMaintenanceTool: MCPToolImplementation { maxRows: Self.rowLimit, timeoutSeconds: timeoutSeconds, context: context, - secrets: meta.redactionSecrets + secrets: meta.redactionSecrets, + masking: .byName ) results.append(result) } diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/SearchQueryHistoryTool.swift b/apple/TablePro/Core/MCP/Protocol/Tools/SearchQueryHistoryTool.swift index ff1a975ed2..46bac19014 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/SearchQueryHistoryTool.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/SearchQueryHistoryTool.swift @@ -107,9 +107,17 @@ public struct SearchQueryHistoryTool: MCPToolImplementation { return .structured(.object(["entries": .array([])])) } + var anyMasksValues = false + for allowed in allowlist { + if await !services.connectionBridge.maskingPolicy(for: allowed).isEmpty { + anyMasksValues = true + break + } + } let page = await services.queryHistoryManager.fetch( QueryHistoryFilter( scope: connectionId.map { .connection($0) } ?? .all, + sources: MCPConnectionBridge.searchableSources(by: query, masksValues: anyMasksValues), searchText: query.isEmpty ? nil : query, since: since, until: until, @@ -118,10 +126,20 @@ public struct SearchQueryHistoryTool: MCPToolImplementation { limit: limit ) - let payload = page.entries.map { entry -> JsonValue in + var masksValues: [UUID: Bool] = [:] + for entryConnection in Set(page.entries.map(\.connectionId)) { + masksValues[entryConnection] = await !services.connectionBridge.maskingPolicy(for: entryConnection).isEmpty + } + + let entries = page.entries.filter { + MCPConnectionBridge.isSearchable($0, by: query, masksValues: masksValues[$0.connectionId] == true) + } + let payload = entries.map { entry -> JsonValue in var fields: [String: JsonValue] = [ "id": .string(entry.id.uuidString), - "query": .string(entry.query), + "query": .string( + MCPConnectionBridge.externalQueryText(of: entry, masksValues: masksValues[entry.connectionId] == true) + ), "connection_id": .string(entry.connectionId.uuidString), "database_name": .string(entry.databaseName), "database_type": .string(entry.databaseType.rawValue), @@ -136,7 +154,9 @@ public struct SearchQueryHistoryTool: MCPToolImplementation { fields["schema_name"] = .string(schemaName) } if let error = entry.errorMessage { - fields["error_message"] = .string(MCPErrorRedactor.redact(error)) + fields["error_message"] = .string( + MCPConnectionBridge.externalErrorText(error, masksValues: masksValues[entry.connectionId] == true) + ) } return .object(fields) } diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/ServerTools.swift b/apple/TablePro/Core/MCP/Protocol/Tools/ServerTools.swift index 010e471b81..512044e903 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/ServerTools.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/ServerTools.swift @@ -215,7 +215,8 @@ public struct StopServerSessionTool: MCPToolImplementation { maxRows: 1, timeoutSeconds: settings.validatedQueryTimeoutSeconds, context: context, - secrets: meta.redactionSecrets + secrets: meta.redactionSecrets, + masking: .byName ) return .structured(result) } diff --git a/apple/TablePro/Core/MCP/Protocol/Tools/ToolQueryExecutor.swift b/apple/TablePro/Core/MCP/Protocol/Tools/ToolQueryExecutor.swift index 82dba63ddf..81d490792e 100644 --- a/apple/TablePro/Core/MCP/Protocol/Tools/ToolQueryExecutor.swift +++ b/apple/TablePro/Core/MCP/Protocol/Tools/ToolQueryExecutor.swift @@ -8,7 +8,8 @@ enum ToolQueryExecutor { maxRows: Int, timeoutSeconds: Int, context: MCPRequestContext, - secrets: [String] + secrets: [String], + masking: ExternalRowMasking = .byOrigin ) async throws -> JsonValue { try await context.cancellation.throwIfCancelled() return try await executeAndLog( @@ -19,7 +20,8 @@ enum ToolQueryExecutor { principal: context.principal, cancellation: context.cancellation, secrets: secrets, - source: historySource(for: context) + source: historySource(for: context), + masking: masking ) } @@ -36,7 +38,8 @@ enum ToolQueryExecutor { principal: MCPPrincipal, cancellation: MCPCancellationToken? = nil, secrets: [String] = [], - source: QueryHistorySource = .mcp + source: QueryHistorySource = .mcp, + masking: ExternalRowMasking = .byOrigin ) async throws -> JsonValue { try await logged( services: services, @@ -50,7 +53,8 @@ enum ToolQueryExecutor { services: services, maxRows: maxRows, timeoutSeconds: timeoutSeconds, - cancellation: cancellation + cancellation: cancellation, + masking: masking ) } } @@ -61,7 +65,8 @@ enum ToolQueryExecutor { maxRows: Int, timeoutSeconds: Int, context: MCPRequestContext, - secrets: [String] + secrets: [String], + masking: ExternalRowMasking = .byOrigin ) async throws -> QueryResult { try await context.cancellation.throwIfCancelled() return try await logged( @@ -75,7 +80,8 @@ enum ToolQueryExecutor { operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, - cancellation: context.cancellation + cancellation: context.cancellation, + masking: masking ) return (outcome.result, outcome.result.rows.count) } @@ -163,7 +169,8 @@ enum ToolQueryExecutor { services: MCPToolServices, maxRows: Int, timeoutSeconds: Int, - cancellation: MCPCancellationToken? + cancellation: MCPCancellationToken?, + masking: ExternalRowMasking ) async throws -> (payload: JsonValue, rowsReturned: Int) { switch operation.unit { case .statement: @@ -171,7 +178,8 @@ enum ToolQueryExecutor { operation, maxRows: maxRows, timeoutSeconds: timeoutSeconds, - cancellation: cancellation + cancellation: cancellation, + masking: masking ) return (payload, payload["row_count"]?.intValue ?? 0) case .script: diff --git a/apple/TablePro/Core/ObjectCopy/ObjectCopyPlan.swift b/apple/TablePro/Core/ObjectCopy/ObjectCopyPlan.swift index 0d1be56987..90b790623e 100644 --- a/apple/TablePro/Core/ObjectCopy/ObjectCopyPlan.swift +++ b/apple/TablePro/Core/ObjectCopy/ObjectCopyPlan.swift @@ -39,6 +39,8 @@ internal struct ObjectCopyTableStep: Identifiable, Sendable { /// The columns written, source order, with generated columns already removed. Empty when this /// step copies structure only. internal let columns: [String] + // The target's type of each written column, which decides how the INSERT binds a MySQL BIT. + internal let targetColumnTypeNames: [String: String] internal let primaryKeyColumns: [String] internal let sourceQuery: String internal let targetTable: String @@ -79,6 +81,7 @@ internal struct ObjectCopyTableStep: Identifiable, Sendable { createStatements: [SyncStatement], truncateStatements: [SyncStatement], columns: [String], + targetColumnTypeNames: [String: String] = [:], primaryKeyColumns: [String], sourceQuery: String, targetTable: String, @@ -102,6 +105,7 @@ internal struct ObjectCopyTableStep: Identifiable, Sendable { self.createStatements = createStatements self.truncateStatements = truncateStatements self.columns = columns + self.targetColumnTypeNames = targetColumnTypeNames self.primaryKeyColumns = primaryKeyColumns self.sourceQuery = sourceQuery self.targetTable = targetTable diff --git a/apple/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift b/apple/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift index 704ba5fd7d..80a2cb599f 100644 --- a/apple/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift +++ b/apple/TablePro/Core/ObjectCopy/ObjectCopyPlanner.swift @@ -534,6 +534,7 @@ internal struct ObjectCopyPlanner { createStatements: statements.create, truncateStatements: statements.truncate, columns: draft.targetColumns, + targetColumnTypeNames: draft.targetColumnTypeNames, primaryKeyColumns: draft.snapshot.primaryKeyColumns, sourceQuery: parts?.query ?? "", targetTable: draft.targetTable, diff --git a/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift b/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift index 2ed4008ce1..164f6c3613 100644 --- a/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift +++ b/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift @@ -163,6 +163,7 @@ internal struct ObjectCopyRowCopier: Sendable { columns: step.columns, primaryKeyColumns: step.primaryKeyColumns, databaseType: targetDatabaseType, + columnTypeNames: step.targetColumnTypeNames, parameterStyle: targetDriver.parameterStyle, quoteIdentifier: targetDriver.quoteIdentifier ) diff --git a/apple/TablePro/Core/ObjectCopy/ObjectCopyTableDraft.swift b/apple/TablePro/Core/ObjectCopy/ObjectCopyTableDraft.swift index 3fd3ed1724..a21a3c5eeb 100644 --- a/apple/TablePro/Core/ObjectCopy/ObjectCopyTableDraft.swift +++ b/apple/TablePro/Core/ObjectCopy/ObjectCopyTableDraft.swift @@ -30,6 +30,8 @@ internal struct ObjectCopyTableDraft { /// column, and each side has to be quoted the way its own server spells it. let sourceColumns: [String] let targetColumns: [String] + // Keyed by target column. The INSERT binds a MySQL BIT as a number, not as text whose bytes it would store. + let targetColumnTypeNames: [String: String] let writesStructure: Bool let dropsFirst: Bool let emptiesFirst: Bool @@ -110,6 +112,9 @@ internal struct ObjectCopyTableDraft { ) self.sourceColumns = pairs.map(\.source) self.targetColumns = pairs.map(\.target) + self.targetColumnTypeNames = Self.typeNames( + of: pairs.map(\.target), in: writesStructure ? translation.snapshot : targetSnapshot + ) let copiesData = request.content.includesData && !pairs.isEmpty && (writesStructure || existsInTarget) self.copiesData = copiesData @@ -185,6 +190,19 @@ internal struct ObjectCopyTableDraft { } } + static func typeNames(of targetColumns: [String], in structure: TableStructureSnapshot?) -> [String: String] { + guard let structure else { return [:] } + var declared: [String: String] = [:] + for column in structure.columns where declared[column.name] == nil { + declared[column.name] = column.typeNameForClassification + } + var names: [String: String] = [:] + for column in targetColumns { + names[column] = declared[column] + } + return names + } + /// Nil where the two engines share a type family, and nil where the columns that are written /// hold nothing whose spelling differs between them. A copy that needs no reshaping runs the /// loop it ran before this existed. @@ -212,7 +230,9 @@ internal struct ObjectCopyTableDraft { target: targetKinds[$0.target.lowercased()] ) } - let coercer = CrossEngineValueCoercer(pairs: columnPairs, from: source) + let coercer = CrossEngineValueCoercer( + pairs: columnPairs, from: source, to: SQLTypeFamily.of(request.target.databaseType) + ) return coercer.isNeeded ? coercer : nil } diff --git a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift index e06d24a770..cb350fb44f 100644 --- a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift +++ b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift @@ -30,6 +30,7 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { private let sourceFields: Set private let sourceFieldsPerFoldedName: [String: Int] private let rowGenerator: SQLStatementGenerator? + private let typedRowGenerator = OSAllocatedUnfairLock(uncheckedState: nil) private let rowWriting: ImportRowWriting private let tally: OSAllocatedUnfairLock private let immutableColumns = OSAllocatedUnfairLock?>(initialState: nil) @@ -94,23 +95,57 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { self.sourceFieldsPerFoldedName = Dictionary(sourceFields.map { ($0.lowercased(), 1) }, uniquingKeysWith: +) self.foldedMapping = Dictionary(grouping: columnMapping, by: { $0.key.lowercased() }) .compactMapValues { $0.count == 1 ? $0.first?.value : nil } - // The driver's quoting, not the dialect's: ClickHouse reads a backslash escape inside backquotes. if let targetTable, Self.canWriteRows(into: databaseType) { - self.rowGenerator = try? SQLStatementGenerator( - tableName: targetTable, - schemaName: targetSchema, - columns: [], - primaryKeyColumns: [], - databaseType: databaseType, - quoteIdentifier: driver.quoteIdentifier, - rowUpdateStatement: driver.rowUpdateStatement, - rowDeleteStatement: driver.rowDeleteStatement + self.rowGenerator = Self.makeRowGenerator( + table: targetTable, schema: targetSchema, databaseType: databaseType, driver: driver ) } else { self.rowGenerator = nil } } + // The driver's quoting, not the dialect's: ClickHouse reads a backslash escape inside backquotes. + private static func makeRowGenerator( + table: String, + schema: String?, + databaseType: DatabaseType, + driver: DatabaseDriver, + columnTypeNames: [String: String] = [:] + ) -> SQLStatementGenerator? { + try? SQLStatementGenerator( + tableName: table, + schemaName: schema, + columns: [], + primaryKeyColumns: [], + databaseType: databaseType, + columnTypeNames: columnTypeNames, + quoteIdentifier: driver.quoteIdentifier, + rowUpdateStatement: driver.rowUpdateStatement, + rowDeleteStatement: driver.rowDeleteStatement + ) + } + + // MySQL stores text bound to a BIT column as its bytes, so a 5 lands as 53 unless the INSERT knows the column is BIT. + private func writingGenerator() async throws -> SQLStatementGenerator? { + guard let rowGenerator, let targetTable, SQLTypeFamily.of(databaseType).readsBitsAsDecimal else { + return rowGenerator + } + if let typed = typedRowGenerator.withLockUnchecked({ $0 }) { return typed } + let columns = try await importedData { + try await self.driver.fetchColumns(table: targetTable, schema: self.targetSchema) + } + var typeNames: [String: String] = [:] + for column in columns where typeNames[column.name] == nil { + typeNames[column.name] = column.typeNameForClassification + } + let typed = Self.makeRowGenerator( + table: targetTable, schema: targetSchema, databaseType: databaseType, driver: driver, + columnTypeNames: typeNames + ) ?? rowGenerator + typedRowGenerator.withLockUnchecked { $0 = typed } + return typed + } + func execute(statement: String) async throws { try await execute(statement: statement, line: 1) } @@ -189,7 +224,7 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { guard let targetTable else { throw PluginImportError.importFailed("No target table configured for row import") } - guard let rowGenerator else { + guard let rowGenerator = try await writingGenerator() else { throw PluginImportError.importFailed("Could not resolve SQL dialect for \(targetTable)") } @@ -220,7 +255,7 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { guard targetTable != nil else { throw PluginImportError.importFailed("No target table configured for row import") } - guard let rowGenerator else { + guard let rowGenerator = try await writingGenerator() else { throw PluginImportError.importFailed("Could not resolve SQL dialect for row import") } @@ -269,11 +304,11 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { values rows: [[PluginCellValue]], onWrite: (Int) -> Void = { _ in } ) async throws { - guard let rowGenerator else { + guard rowGenerator != nil else { throw PluginImportError.importFailed("No target table configured for row import") } - guard !columns.isEmpty, !rows.isEmpty else { return } - try await insertGroup(rows, columns: columns, generator: rowGenerator, onWrite: onWrite) + guard !columns.isEmpty, !rows.isEmpty, let generator = try await writingGenerator() else { return } + try await insertGroup(rows, columns: columns, generator: generator, onWrite: onWrite) } private func insertGroup( diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift index 8e0083c5fb..587b6fe793 100644 --- a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift @@ -1340,6 +1340,7 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor ResultColumnMeta(isPrimaryKey: $0.isPrimaryKey, isNullable: $0.isNullable, isAutoIncrement: $0.isIdentity) } result.rowLocators = pluginResult.rowLocators + result.columnOrigins = pluginResult.columnOrigins result.exportDocuments = pluginResult.exportDocuments result.absentCells = pluginResult.absentCells ?? [:] return result diff --git a/apple/TablePro/Core/Scripting/ScriptingSnapshot.swift b/apple/TablePro/Core/Scripting/ScriptingSnapshot.swift index 13b8df1972..48ab2df429 100644 --- a/apple/TablePro/Core/Scripting/ScriptingSnapshot.swift +++ b/apple/TablePro/Core/Scripting/ScriptingSnapshot.swift @@ -117,9 +117,12 @@ internal enum ScriptingSnapshot { } internal static func query(ofTab tabId: UUID, connectionId: UUID) -> String? { - guard isVisibleToScripts(connectionId: connectionId) else { return nil } - return coordinator(ofTab: tabId, connectionId: connectionId)? - .tabManager.tabs.first { $0.id == tabId }?.content.query + guard isVisibleToScripts(connectionId: connectionId), + let coordinator = coordinator(ofTab: tabId, connectionId: connectionId) + else { return nil } + // A large document's last keys reach its tab on a delay; a script reads what the editor shows. + coordinator.flushEditorText(forTab: tabId) + return coordinator.tabManager.tabs.first { $0.id == tabId }?.content.query } // MARK: - Results @@ -150,7 +153,7 @@ internal enum ScriptingSnapshot { ? coordinator.changeManager.deletedRowIDs : tab.pendingChanges.deletedRowIDs - let tableRows = coordinator.tabSessionRegistry.tableRows(for: tabId) + let tableRows = coordinator.externallyMaskedRows(forTab: tabId) let read = DisplayedResultReader.read( tableRows: tableRows, displayIDs: coordinator.displayIDs(forTab: tabId), diff --git a/apple/TablePro/Core/Services/Export/TableTransferService.swift b/apple/TablePro/Core/Services/Export/TableTransferService.swift index 6220b1a1ac..98d3a27644 100644 --- a/apple/TablePro/Core/Services/Export/TableTransferService.swift +++ b/apple/TablePro/Core/Services/Export/TableTransferService.swift @@ -252,8 +252,91 @@ final class TableTransferService: ObservableObject { sourceFields: Set(request.sourceColumns[object.name] ?? []), isCancelled: { [flag = cancellationFlag] in flag.isCancelled } ) - try await transferOne(object: object, from: source, into: sink, request: request) + let bitKinds = try await Self.bitColumnKinds( + for: object, source: source, request: request, destinationDriver: destinationDriver) + // A Stop during those reads must land before "Delete existing rows first" empties the target. + try checkCancellation() + try await transferOne( + object: object, from: source, into: sink, request: request, + bitSpelling: { header in + Self.bitRespelling(for: header, mapping: mapping, request: request, kinds: bitKinds) + } + ) + } + } + + struct BitColumnKinds { + let source: [String: CanonicalTypeKind] + let target: [String: CanonicalTypeKind] + } + + // PostgreSQL and DuckDB read a BIT as its digits and MySQL as its number, so 00000101 bound to a MySQL BIT stores 101. + // Read before the row stream opens: a catalog read queued behind the source's own open stream waits for the whole + // table, and behind DuckDB's bounded one never ends. The catalog, because a fresh PostgreSQL session's header names + // a bit column `unknown` and MySQL's drops the width. + static func bitColumnKinds( + for object: ExportObjectItem, + source: ExportDataSourceAdapter, + request: Request, + destinationDriver: DatabaseDriver + ) async throws -> BitColumnKinds? { + let sourceFamily = SQLTypeFamily.of(request.sourceType) + let targetFamily = SQLTypeFamily.of(request.destinationType) + guard sourceFamily.readsBitsAsDecimal != targetFamily.readsBitsAsDecimal, + Self.spellsBitStrings(sourceFamily), Self.spellsBitStrings(targetFamily) else { return nil } + + let sourceColumns = try await source.fetchColumns(table: object.name, databaseName: object.databaseName) + let sourceKinds = Self.kinds(of: sourceColumns.map { ($0.name, $0.typeNameForClassification, $0.ddlSpelling) }, + family: sourceFamily) + // A catalog that lists the table's columns and no bit string leaves nothing to respell. + guard sourceKinds.isEmpty || sourceKinds.values.contains(where: { Self.isBitString($0) }) else { return nil } + + let targetColumns = try await destinationDriver.fetchColumns(table: object.name, schema: request.destinationSchema) + let targetKinds = Self.kinds(of: targetColumns.map { ($0.name, $0.typeNameForClassification, $0.ddlSpelling) }, + family: targetFamily) + return BitColumnKinds(source: sourceKinds, target: targetKinds) + } + + // A column the catalog did not list keeps the header's type. + static func bitRespelling( + for header: PluginStreamHeader, + mapping: [String: String], + request: Request, + kinds: BitColumnKinds? + ) -> CrossEngineValueCoercer? { + guard let kinds else { return nil } + let sourceFamily = SQLTypeFamily.of(request.sourceType) + let pairs = header.columns.enumerated().map { index, column -> CrossEngineValueCoercer.ColumnPair in + let sourceKind = kinds.source[column] ?? (index < header.columnTypeNames.count + ? SQLTypeParser.parse(header.columnTypeNames[index], family: sourceFamily).kind + : nil) + let targetKind = mapping[column].flatMap { kinds.target[$0] } + guard Self.isBitString(sourceKind), Self.isBitString(targetKind) else { return .init(source: nil, target: nil) } + return .init(source: sourceKind, target: targetKind) } + let coercer = CrossEngineValueCoercer(pairs: pairs, from: sourceFamily, to: SQLTypeFamily.of(request.destinationType)) + return coercer.isNeeded ? coercer : nil + } + + // Only a family whose BIT is a bit string can take part: SQL Server reads BIT as a boolean, SQLite not at all. + private static func spellsBitStrings(_ family: SQLTypeFamily) -> Bool { + isBitString(SQLTypeParser.parse("BIT", family: family).kind) + } + + private static func kinds( + of columns: [(name: String, type: String, catalogSpelling: String?)], + family: SQLTypeFamily + ) -> [String: CanonicalTypeKind] { + var kinds: [String: CanonicalTypeKind] = [:] + for column in columns { + kinds[column.name] = SQLTypeParser.parse(column.type, catalogSpelling: column.catalogSpelling, family: family).kind + } + return kinds + } + + private static func isBitString(_ kind: CanonicalTypeKind?) -> Bool { + guard case .bitString = kind else { return false } + return true } /// Checked for every table before the first is written: the INSERT would name the contested @@ -317,7 +400,8 @@ final class TableTransferService: ObservableObject { object: ExportObjectItem, from source: ExportDataSourceAdapter, into sink: ImportDataSinkAdapter, - request: Request + request: Request, + bitSpelling: (PluginStreamHeader) -> CrossEngineValueCoercer? ) async throws { let exportTable = PluginExportTable( name: object.name, @@ -332,6 +416,7 @@ final class TableTransferService: ObservableObject { ) var columns: [String] = [] + var coercer: CrossEngineValueCoercer? var filler = SQLWriteBatchFiller<[String: PluginCellValue]>(budget: Self.batchBudget) var wroteAnything = false @@ -351,9 +436,10 @@ final class TableTransferService: ObservableObject { switch element { case .header(let header): columns = header.columns + coercer = bitSpelling(header) case .rows(let rows): for row in rows { - let keyed = Self.dictionary(columns: columns, row: row) + let keyed = Self.dictionary(columns: columns, row: coercer?.coerce(row) ?? row) let bytes = Self.batchBudget.byteCount(of: keyed.values) guard let batch = filler.append(keyed, bytes: bytes) else { continue } try await sink.insertRows(batch) diff --git a/apple/TablePro/Core/Services/Infrastructure/QuitApproval.swift b/apple/TablePro/Core/Services/Infrastructure/QuitApproval.swift index 24ef8917e4..9d14e0be65 100644 --- a/apple/TablePro/Core/Services/Infrastructure/QuitApproval.swift +++ b/apple/TablePro/Core/Services/Infrastructure/QuitApproval.swift @@ -23,6 +23,8 @@ internal enum QuitApproval { action: RunningWorkAction, stopsWork: Bool = true ) -> Bool { + // A large document's last keys reach its tab on a delay; without them a dirty file reads as clean. + EditorEventRouter.shared.flushAllPendingText() let work = registry.items(in: .app) let hasUnsavedTabs = MainContentCoordinator.hasAnyUnsavedChanges() let losses = hasUnsavedTabs ? [String(localized: "Unsaved edits in open tabs")] : [] diff --git a/apple/TablePro/Core/Services/Infrastructure/WorkspaceCloseAction.swift b/apple/TablePro/Core/Services/Infrastructure/WorkspaceCloseAction.swift index 9db31a5d0f..b124761aaf 100644 --- a/apple/TablePro/Core/Services/Infrastructure/WorkspaceCloseAction.swift +++ b/apple/TablePro/Core/Services/Infrastructure/WorkspaceCloseAction.swift @@ -96,7 +96,7 @@ internal enum WorkspaceCloseAction { return } // Taken after the transaction prompt: the tabs stay editable while it is answered. - let victims = coordinators.flatMap { tabs(in: workspace.container, of: $0) } + let victims = committedTabs(in: workspace.container, across: coordinators) // Closing a tab stops its query, so a yes needs no cancel of its own. let running = RunningWorkRegistry.shared.items(in: .tabs(Set(victims.map(\.id)))) let holdsUnsavedWork = coordinators.contains { coordinator in @@ -352,6 +352,17 @@ internal enum WorkspaceCloseAction { .map(\.container) } + // An open cell editor or the editor's pending text in a window's selected tab is unsaved work too, so it lands + // before the tabs are read. + internal static func committedTabs(in container: String, across coordinators: [MainContentCoordinator]) -> [QueryTab] { + for coordinator in coordinators { + let selected = coordinator.tabManager.selectedTabId + guard tabs(in: container, of: coordinator).contains(where: { $0.id == selected }) else { continue } + coordinator.commitActiveGridEdit() + } + return coordinators.flatMap { tabs(in: container, of: $0) } + } + private static func tabs(in container: String, of coordinator: MainContentCoordinator?) -> [QueryTab] { guard let coordinator else { return [] } let target = PluginManager.shared.containerSwitchTarget(for: coordinator.connection.type) diff --git a/apple/TablePro/Core/Services/Masking/MaskingPolicy.swift b/apple/TablePro/Core/Services/Masking/MaskingPolicy.swift index 629a6432bc..d14aeb5aa4 100644 --- a/apple/TablePro/Core/Services/Masking/MaskingPolicy.swift +++ b/apple/TablePro/Core/Services/Masking/MaskingPolicy.swift @@ -6,7 +6,7 @@ import Foundation import TableProPluginKit -/// The rules one connection masks its values by, and the only place they are matched against columns. +// The rules one connection masks its values by, and the only place they are matched against columns. struct MaskingPolicy: Equatable, Sendable { let rules: [MaskingRule] @@ -18,8 +18,8 @@ struct MaskingPolicy: Equatable, Sendable { var isEmpty: Bool { rules.isEmpty } - /// The first rule that matches wins. Without a table, as for most query results, a `table.column` - /// pattern matches by its column part alone, so an unknown source masks more rather than less. + // The first rule that matches wins. Without a table, as for most query results, a `table.column` + // pattern matches by its column part alone, so an unknown source masks more rather than less. func style(table: String?, column: String, typeName: String?) -> MaskingStyle? { for rule in rules where Self.matches(rule, table: table, column: column, typeName: typeName) { return rule.style @@ -43,6 +43,55 @@ struct MaskingPolicy: Equatable, Sendable { columnMasks(table: table, columns: columns, typeNames: columnTypes.map(\.rawType)) } + // An external caller can rename a column or compute one from it, so an origin only ever adds masking. A column + // nobody traced is masked whole, whatever rule its name matches, since the caller chose that name. The exception + // is `table`: the app selected that one table's own columns by name and the driver reported nothing against it. + func externalColumnMasks( + table: String?, + columns: [String], + typeNames: [String?], + origins: [PluginColumnOrigin?]? + ) -> ColumnMasks { + guard !isEmpty else { return .none } + let provesByName = Self.provesByName(table: table, origins: origins) + var styles: [Int: MaskingStyle] = [:] + for (index, column) in columns.enumerated() { + let typeName = typeNames.indices.contains(index) ? typeNames[index] : nil + if let origin = Self.origin(at: index, in: origins) { + // The relation an engine names can be a view, a partition or a child of the rule's table, so a + // `table.column` rule also matches by its column part; naming the relation only picks the style. A + // name the statement chose adds its own rule, and the stricter of the two holds. + let relation = origin.schema.map { "\($0).\(origin.table)" } ?? origin.table + let read = style(table: relation, column: origin.column, typeName: typeName) + ?? style(table: nil, column: origin.column, typeName: typeName) + let isAlias = column.caseInsensitiveCompare(origin.column) != .orderedSame + let named = isAlias ? style(table: nil, column: column, typeName: typeName) : nil + styles[index] = MaskingStyle.strictest(read, named) + } else if provesByName { + styles[index] = style(table: table, column: column, typeName: typeName) + } else { + styles[index] = .full + } + } + return ColumnMasks(styles: styles) + } + + // The columns `externalColumnMasks` masks only because nothing said what they read. + func untracedColumns(table: String?, columns: [String], origins: [PluginColumnOrigin?]?) -> [Int] { + guard !isEmpty, !Self.provesByName(table: table, origins: origins) else { return [] } + return columns.indices.filter { Self.origin(at: $0, in: origins) == nil } + } + + // A driver that reports origins and leaves one out withholds it on purpose, as for a statistics view. + private static func provesByName(table: String?, origins: [PluginColumnOrigin?]?) -> Bool { + table != nil && origins == nil + } + + private static func origin(at index: Int, in origins: [PluginColumnOrigin?]?) -> PluginColumnOrigin? { + guard let origins, origins.indices.contains(index) else { return nil } + return origins[index] + } + private static func matches(_ rule: MaskingRule, table: String?, column: String, typeName: String?) -> Bool { let pattern = rule.trimmedPattern switch rule.target { @@ -64,7 +113,16 @@ struct MaskingPolicy: Equatable, Sendable { } } -/// `*` matches any run of characters and `?` exactly one, ignoring case. +extension MaskingStyle { + // Two styles that differ each show something the other hides, except `.null`, which shows nothing at all. + static func strictest(_ first: MaskingStyle?, _ second: MaskingStyle?) -> MaskingStyle? { + guard let first else { return second } + guard let second, second != first else { return first } + return first == .null || second == .null ? .null : .full + } +} + +// `*` matches any run of characters and `?` exactly one, ignoring case. enum MaskingGlob { static func matches(_ pattern: String, _ text: String) -> Bool { let pattern = Array(pattern.lowercased()) @@ -96,7 +154,7 @@ enum MaskingGlob { } } -/// Which columns of one result are masked, and how. Every surface masks rows through this. +// Which columns of one result are masked, and how. Every surface masks rows through this. struct ColumnMasks: Equatable, Sendable { let styles: [Int: MaskingStyle] diff --git a/apple/TablePro/Core/Services/Masking/QueryResult+Masking.swift b/apple/TablePro/Core/Services/Masking/QueryResult+Masking.swift index 7ab1ade5fb..03340665dc 100644 --- a/apple/TablePro/Core/Services/Masking/QueryResult+Masking.swift +++ b/apple/TablePro/Core/Services/Masking/QueryResult+Masking.swift @@ -8,8 +8,41 @@ import TableProPluginKit extension QueryResult { func masked(by policy: MaskingPolicy, table: String? = nil) -> QueryResult { + guard !policy.isEmpty else { return self } let masks = policy.columnMasks(table: table, columns: columns, columnTypes: columnTypes) + return masked(by: masks, policy: policy, table: table) + } + + // For MCP, the assistant and scripts: matched by what each column reads, not by what the statement named it. + func maskedForExternalCaller(by policy: MaskingPolicy, table: String? = nil) -> QueryResult { guard !policy.isEmpty else { return self } + let masks = policy.externalColumnMasks( + table: table, columns: columns, typeNames: columnTypes.map(\.rawType), origins: columnOrigins + ) + var result = masked(by: masks, policy: policy, table: table) + let untraced = policy.untracedColumns(table: table, columns: columns, origins: columnOrigins) + if !rows.isEmpty, let note = Self.untracedNote(untraced.map { columns[$0] }, columnCount: columns.count) { + result.statusMessage = [statusMessage, note].compactMap { $0 }.joined(separator: " ") + } + return result + } + + private static func untracedNote(_ names: [String], columnCount: Int) -> String? { + guard !names.isEmpty else { return nil } + guard names.count < columnCount else { + return String( + localized: "Masked every column: this connection has masking rules, and TablePro could not trace any of them to a table column." + ) + } + return String( + format: String( + localized: "Masked %@: this connection has masking rules, and TablePro could not trace these columns to a table column." + ), + ListFormatter.localizedString(byJoining: names) + ) + } + + private func masked(by masks: ColumnMasks, policy: MaskingPolicy, table: String?) -> QueryResult { var masked = QueryResult( columns: columns, columnTypes: columnTypes, @@ -22,6 +55,7 @@ extension QueryResult { masked.isTruncated = isTruncated masked.statusMessage = statusMessage masked.columnMeta = columnMeta + masked.columnOrigins = columnOrigins masked.rowLocators = rowLocators masked.serverOutput = serverOutput masked.exportDocuments = exportDocuments?.enumerated().map { index, document in diff --git a/apple/TablePro/Core/Services/Query/QueryExecutor.swift b/apple/TablePro/Core/Services/Query/QueryExecutor.swift index 13c42a9140..f67b7650aa 100644 --- a/apple/TablePro/Core/Services/Query/QueryExecutor.swift +++ b/apple/TablePro/Core/Services/Query/QueryExecutor.swift @@ -27,6 +27,8 @@ struct QueryFetchResult { var exportDocuments: [PluginExportDocument]? var absentCells: [Int: Set] = [:] var unloadedCells: [Int: [Int: Int]] = [:] + // See `PluginQueryResult.columnOrigins`. + var columnOrigins: [PluginColumnOrigin?]? var resolvedTiming: PluginQueryTiming { timing ?? PluginQueryTiming(total: executionTime) @@ -171,7 +173,8 @@ final class QueryExecutor { timing: result.timing, rowLocators: result.rowLocators, exportDocuments: result.exportDocuments, - absentCells: result.absentCells + absentCells: result.absentCells, + columnOrigins: result.columnOrigins ) } @@ -219,7 +222,8 @@ final class QueryExecutor { timing: result.timing, rowLocators: result.rowLocators, exportDocuments: result.exportDocuments, - absentCells: result.absentCells + absentCells: result.absentCells, + columnOrigins: result.columnOrigins ) } @@ -246,7 +250,8 @@ final class QueryExecutor { timing: result.timing, rowLocators: result.rowLocators, exportDocuments: result.exportDocuments, - absentCells: result.absentCells + absentCells: result.absentCells, + columnOrigins: result.columnOrigins ) } diff --git a/apple/TablePro/Core/Services/Query/RowOperationsManager.swift b/apple/TablePro/Core/Services/Query/RowOperationsManager.swift index 6fba4619aa..66f5ebf5e4 100644 --- a/apple/TablePro/Core/Services/Query/RowOperationsManager.swift +++ b/apple/TablePro/Core/Services/Query/RowOperationsManager.swift @@ -366,8 +366,10 @@ final class RowOperationsManager { switch parseResult { case .success(let parsedRows): + let unfilled = Self.unfilledCellValues(columns: columns, tableRows: tableRows) + let filled = parsedRows.map { Self.fillingBlankCells($0, with: unfilled) } return insertParsedRows( - keepsMissingFields ? parsedRows.map(Self.nullsAsMissingFields) : parsedRows, + keepsMissingFields ? filled.map(Self.nullsAsMissingFields) : filled, uuidKeyForms: uuidKeyForms, into: &tableRows ) @@ -443,6 +445,33 @@ final class RowOperationsManager { } } + // A blank cell of pasted text was left unfilled, so it takes what a new row's cell does: the server's value where + // the column has a default, else NULL. A NOT NULL text column with no default takes the empty string it can hold. + private static func unfilledCellValues(columns: [String], tableRows: TableRows) -> [PluginCellValue] { + columns.map { name in + guard let index = tableRows.columns.firstIndex(of: name) else { return .null } + if tableRows.generatedColumns.contains(name) || tableRows.serverAssignsValue(forColumn: name) { + return .text("__DEFAULT__") + } + if tableRows.refusesNull(columnAt: index), tableRows.editingColumnType(at: index)?.hasEmptyValue == true { + return .text("") + } + return .null + } + } + + private static func fillingBlankCells(_ row: ParsedRow, with unfilled: [PluginCellValue]) -> ParsedRow { + guard !row.blankColumns.isEmpty else { return row } + var values = row.values + for index in row.blankColumns where index < values.count && index < unfilled.count { + values[index] = unfilled[index] + } + var filled = ParsedRow(values: values, sourceLineNumber: row.sourceLineNumber) + filled.absentColumns = row.absentColumns + filled.expressionColumns = row.expressionColumns + return filled + } + // Text can't say which fields a row lacks, so a pasted NULL leaves the field out, as a new row does. private static func nullsAsMissingFields(_ row: ParsedRow) -> ParsedRow { var missing = row diff --git a/apple/TablePro/Core/Services/Query/RowParser.swift b/apple/TablePro/Core/Services/Query/RowParser.swift index e0a31868b3..cdd13c09a6 100644 --- a/apple/TablePro/Core/Services/Query/RowParser.swift +++ b/apple/TablePro/Core/Services/Query/RowParser.swift @@ -19,6 +19,35 @@ protocol RowDataParser { func parse(_ text: String, schema: TableSchema) -> Result<[ParsedRow], RowParseError> } +// One line's fields fitted to the table: a short line's missing fields are blank cells, extra fields are dropped, +// and the key is left to the server. +private func fittedRow( + _ fields: [String], + lineNumber: Int, + schema: TableSchema, + normalize: (String) -> String? +) -> ParsedRow { + var values: [String?] = [] + var blankColumns: Set = [] + for (column, field) in fields.prefix(schema.columnCount).enumerated() { + if field.trimmingCharacters(in: .whitespaces).isEmpty { blankColumns.insert(column) } + values.append(normalize(field)) + } + while values.count < schema.columnCount { + blankColumns.insert(values.count) + values.append(nil) + } + if let pkIndex = schema.primaryKeyIndex, pkIndex < values.count { + values[pkIndex] = "__DEFAULT__" + blankColumns.remove(pkIndex) + } + return ParsedRow( + values: values.map(PluginCellValue.fromOptional), + sourceLineNumber: lineNumber, + blankColumns: blankColumns + ) +} + /// TSV (Tab-Separated Values) parser /// Matches the format produced by RowOperationsManager.copySelectedRowsToClipboard() struct TSVRowParser: RowDataParser { @@ -38,26 +67,12 @@ struct TSVRowParser: RowDataParser { var parsedRows: [ParsedRow] = [] for (index, line) in lines.enumerated() { - let lineNumber = index + 1 - - let rawValues = line.components(separatedBy: "\t") - var values = rawValues.map { normalizeValue($0) } - - if values.count < schema.columnCount { - while values.count < schema.columnCount { - values.append(nil) - } - } else if values.count > schema.columnCount { - values = Array(values.prefix(schema.columnCount)) - } - - if let pkIndex = schema.primaryKeyIndex, pkIndex < values.count { - values[pkIndex] = "__DEFAULT__" - } - - let typedValues = values.map(PluginCellValue.fromOptional) - let parsedRow = ParsedRow(values: typedValues, sourceLineNumber: lineNumber) - parsedRows.append(parsedRow) + parsedRows.append(fittedRow( + line.components(separatedBy: "\t"), + lineNumber: index + 1, + schema: schema, + normalize: normalizeValue + )) } guard !parsedRows.isEmpty else { @@ -109,23 +124,12 @@ struct CSVRowParser: RowDataParser { var parsedRows: [ParsedRow] = [] for recordIndex in startIndex.. schema.columnCount { - values = Array(values.prefix(schema.columnCount)) - } - - if let pkIndex = schema.primaryKeyIndex, pkIndex < values.count { - values[pkIndex] = "__DEFAULT__" - } - - let typedValues = values.map(PluginCellValue.fromOptional) - parsedRows.append(ParsedRow(values: typedValues, sourceLineNumber: lineNumber)) + parsedRows.append(fittedRow( + records[recordIndex], + lineNumber: recordIndex + 1, + schema: schema, + normalize: normalizeValue + )) } guard !parsedRows.isEmpty else { diff --git a/apple/TablePro/Core/Services/Query/UnloadedValuePlan.swift b/apple/TablePro/Core/Services/Query/UnloadedValuePlan.swift index 43e1a65992..7a92ab8c35 100644 --- a/apple/TablePro/Core/Services/Query/UnloadedValuePlan.swift +++ b/apple/TablePro/Core/Services/Query/UnloadedValuePlan.swift @@ -143,6 +143,8 @@ extension QueryResult { absentCells: absentCells ) folded.unloadedCells = fold.unloadedCells + // The fold drops the trailing length columns, so the origins of the columns it keeps still line up. + folded.columnOrigins = columnOrigins.map { Array($0.prefix(fold.columns.count)) } return folded } } @@ -169,6 +171,8 @@ extension QueryFetchResult { absentCells: absentCells ) folded.unloadedCells = fold.unloadedCells + // The fold drops the trailing length columns, so the origins of the columns it keeps still line up. + folded.columnOrigins = columnOrigins.map { Array($0.prefix(fold.columns.count)) } return folded } } diff --git a/apple/TablePro/Core/Storage/ConnectionStorage.swift b/apple/TablePro/Core/Storage/ConnectionStorage.swift index 382d89fd20..f1f9cfbd49 100644 --- a/apple/TablePro/Core/Storage/ConnectionStorage.swift +++ b/apple/TablePro/Core/Storage/ConnectionStorage.swift @@ -832,6 +832,11 @@ final class ConnectionStorage { @discardableResult private func writeSecret(_ value: String, forKey key: String, connectionId: UUID) -> Bool { let staged = staging?.connectionId == connectionId + // Writing it again would date the item as a new save, and a newer copy on another device would lose to it. + if staged, staging?.loaded[key] == value { + staging?.edits.deletedKeys.removeAll { $0 == key } + return true + } let deviceOnly = staged ? staging?.deviceOnly ?? false : keepsSecretsOnDevice(connectionId) let written = keychain.writeString(value, forKey: key, deviceOnly: deviceOnly) if staged { @@ -915,34 +920,51 @@ final class ConnectionStorage { let connectionId: UUID let deviceOnly: Bool let keptKeys: Set + let loaded: [String: String] var edits = StagedSecretEdits() } private var staging: SecretStaging? private var unreadableSecretsSeen: Set? + private var loadedSecretsSeen: [String: String]? + + struct SecretReads { + var unreadable: Set = [] + var loaded: [String: String] = [:] + } // The keys a read inside `body` could not read. A form runs its prefill here, so the field left empty and the // key it must not clear come from one read. func recordingUnreadableSecrets(_ body: () -> Void) -> Set { - let outer = unreadableSecretsSeen + recordingSecretReads(body).unreadable + } + + // Also the values the reads found, so a save can leave alone what the form loaded and did not change. + func recordingSecretReads(_ body: () -> Void) -> SecretReads { + let outerUnreadable = unreadableSecretsSeen + let outerLoaded = loadedSecretsSeen unreadableSecretsSeen = [] + loadedSecretsSeen = [:] body() - let seen = unreadableSecretsSeen ?? [] - unreadableSecretsSeen = outer.map { $0.union(seen) } - return seen + let reads = SecretReads(unreadable: unreadableSecretsSeen ?? [], loaded: loadedSecretsSeen ?? [:]) + unreadableSecretsSeen = outerUnreadable.map { $0.union(reads.unreadable) } + loadedSecretsSeen = outerLoaded.map { $0.merging(reads.loaded) { _, inner in inner } } + return reads } // Inside `body`, this connection's secret writes take `deviceOnly` and report a failure, and its deletes wait - // for `applyStagedDeletions`, which the caller runs once the record is on disk. A key in `keeping` is never deleted. + // for `applyStagedDeletions`, which the caller runs once the record is on disk. A key in `keeping` is never deleted, + // and a write of the value `unchanged` holds for its key is skipped. func stagingSecretEdits( for connectionId: UUID, deviceOnly: Bool, keeping keptKeys: Set = [], + unchanged loaded: [String: String] = [:], _ body: () -> Void ) -> StagedSecretEdits { let outer = staging - staging = SecretStaging(connectionId: connectionId, deviceOnly: deviceOnly, keptKeys: keptKeys) + staging = SecretStaging(connectionId: connectionId, deviceOnly: deviceOnly, keptKeys: keptKeys, loaded: loaded) body() let edits = staging?.edits ?? StagedSecretEdits() staging = outer @@ -995,7 +1017,9 @@ final class ConnectionStorage { private func readSecret(forKey key: String, connectionId: UUID) -> KeychainStringResult { let result = keychain.readStringResult(forKey: key, deviceOnly: readsOwnItemFirst(connectionId)) switch result { - case .found, .notFound: + case .found(let value): + loadedSecretsSeen?[key] = value + case .notFound: break case .locked, .userCancelled, .authFailed, .error: unreadableSecretsSeen?.insert(key) diff --git a/apple/TablePro/Core/Storage/KeychainHelper.swift b/apple/TablePro/Core/Storage/KeychainHelper.swift index e45b42dfee..eba7a3b291 100644 --- a/apple/TablePro/Core/Storage/KeychainHelper.swift +++ b/apple/TablePro/Core/Storage/KeychainHelper.swift @@ -48,10 +48,17 @@ extension KeychainStoring { } } +struct KeychainCopiedItem: Sendable { + let status: OSStatus + let data: Data? + let modified: Date? +} + protocol KeychainItemStore: Sendable { func add(_ attributes: [String: Any]) -> OSStatus func update(_ query: [String: Any], attributes: [String: Any]) -> OSStatus - func copyData(_ query: [String: Any]) -> (status: OSStatus, data: Data?) + // The first item the query matches, with its data and modification date. + func copyItem(_ query: [String: Any]) -> KeychainCopiedItem func delete(_ query: [String: Any]) -> OSStatus } @@ -64,10 +71,19 @@ struct SystemKeychainItemStore: KeychainItemStore { SecItemUpdate(query as CFDictionary, attributes as CFDictionary) } - func copyData(_ query: [String: Any]) -> (status: OSStatus, data: Data?) { + func copyItem(_ query: [String: Any]) -> KeychainCopiedItem { + var query = query + query[kSecReturnData as String] = true + query[kSecReturnAttributes as String] = true + query[kSecMatchLimit as String] = kSecMatchLimitOne var result: AnyObject? let status = SecItemCopyMatching(query as CFDictionary, &result) - return (status, result as? Data) + let item = result as? [String: Any] + return KeychainCopiedItem( + status: status, + data: item?[kSecValueData as String] as? Data, + modified: item?[kSecAttrModificationDate as String] as? Date + ) } func delete(_ query: [String: Any]) -> OSStatus { @@ -166,23 +182,30 @@ final class KeychainHelper: KeychainStoring { read(forKey: key, deviceOnly: false) } + // With sync on, a newer local item is a save made while sync was off; with sync off, this Mac's own item always wins. func read(forKey key: String, deviceOnly: Bool) -> KeychainResult { - var preferred = itemQuery(forKey: key, synchronizable: !deviceOnly && isPasswordSyncEnabled) - preferred[kSecReturnData as String] = true - preferred[kSecMatchLimit as String] = kSecMatchLimitOne - let first = items.copyData(preferred) + let prefersSynced = !deviceOnly && isPasswordSyncEnabled + let first = items.copyItem(itemQuery(forKey: key, synchronizable: prefersSynced)) guard first.status == errSecItemNotFound else { - return result(for: first, key: key) + guard first.status == errSecSuccess, prefersSynced else { + return result(for: first, key: key) + } + let other = items.copyItem(itemQuery(forKey: key, synchronizable: !prefersSynced)) + return result(for: Self.isNewer(other, than: first) ? other : first, key: key) } var fallback = baseQuery(forKey: key) fallback[kSecAttrSynchronizable as String] = kSecAttrSynchronizableAny - fallback[kSecReturnData as String] = true - fallback[kSecMatchLimit as String] = kSecMatchLimitOne - return result(for: items.copyData(fallback), key: key) + return result(for: items.copyItem(fallback), key: key) + } + + private static func isNewer(_ candidate: KeychainCopiedItem, than current: KeychainCopiedItem) -> Bool { + guard candidate.status == errSecSuccess, let candidateDate = candidate.modified, + let currentDate = current.modified else { return false } + return candidateDate > currentDate } - private func result(for copied: (status: OSStatus, data: Data?), key: String) -> KeychainResult { + private func result(for copied: KeychainCopiedItem, key: String) -> KeychainResult { switch copied.status { case errSecSuccess: if let data = copied.data { diff --git a/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+DeferredDeletions.swift b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+DeferredDeletions.swift index 272a90623b..1f6fa77487 100644 --- a/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+DeferredDeletions.swift +++ b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+DeferredDeletions.swift @@ -6,7 +6,7 @@ import TableProSyncTransport // record name is kept until the category is read. extension SyncCoordinator { // Deleting a profile removes keychain secrets no pull can bring back, so a late deletion never does it. - private static let typesNeverDeletedLate: Set = [.sshProfile, .credentialProfile] + static let typesNeverDeletedLate: Set = [.sshProfile, .credentialProfile] struct DeferredDeletions { var due: [CKRecord.ID] = [] diff --git a/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+FullFetchDeletions.swift b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+FullFetchDeletions.swift new file mode 100644 index 0000000000..31ee8e5537 --- /dev/null +++ b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+FullFetchDeletions.swift @@ -0,0 +1,94 @@ +import CloudKit +import Foundation +import os +import TableProSyncTransport + +// A full fetch lists the records a zone holds and never the ones it lost, so a deletion passed by an expired +// or cleared token is found by what the fetch no longer lists. A zone deleted and made again holds only +// records created after it was made, so a cached copy created no earlier than the oldest fetched record +// belongs to the zone this fetch read, and its absence from a complete fetch proves it was deleted there. +enum SyncFullFetchDeletions { + struct Candidate: Equatable { + let recordID: CKRecord.ID + let cachedCreation: Date? + } + + static func proven( + candidates: [Candidate], + fetchedNames: Set, + fetchedCreations: [Date] + ) -> [CKRecord.ID] { + guard let oldestFetched = fetchedCreations.min() else { return [] } + return candidates.compactMap { candidate in + guard !fetchedNames.contains(candidate.recordID.recordName), + let created = candidate.cachedCreation, + created >= oldestFetched + else { return nil } + return candidate.recordID + } + } +} + +extension SyncCoordinator { + func deletionsProvenByFullFetch( + _ result: PullResult, + settings: SyncSettings, + zoneID: CKRecordZone.ID, + pushed: Set + ) async -> [CKRecord.ID] { + let fetchedCreations = result.changedRecords.compactMap(recordCreationDate) + guard result.unreadRecordIDs.isEmpty, !fetchedCreations.isEmpty else { return [] } + let fetchedNames = Set(result.changedRecords.map(\.recordID.recordName)) + let boundary = syncBoundary(settings: settings) + var candidates: [SyncFullFetchDeletions.Candidate] = [] + for (type, ids) in await localRecordIds() + where settings.syncs(type) && !Self.typesNeverDeletedLate.contains(type) { + let dirty = changeTracker.dirtyRecords(for: type) + let tombstoned = Set(metadataStorage.tombstones(for: type).map(\.id)) + for id in ids where !dirty.contains(id) && !tombstoned.contains(id) { + guard boundary.includes(type, owner: SyncBoundary.owner(ofRecordId: id, type: type)) else { continue } + let recordID = SyncRecordMapper.recordID(type: type, id: id, in: zoneID) + guard !fetchedNames.contains(recordID.recordName), + !pushed.contains(recordID), + let cached = recordCache.record(for: recordID) + else { continue } + candidates.append(.init(recordID: recordID, cachedCreation: recordCreationDate(cached))) + } + } + let proven = SyncFullFetchDeletions.proven( + candidates: candidates, + fetchedNames: fetchedNames, + fetchedCreations: fetchedCreations + ) + if !proven.isEmpty { + Self.logger.info("A full fetch no longer lists \(proven.count, privacy: .public) records this Mac holds; deleting them here") + } + return proven + } + + // Only the settings records a deletion acts on are listed: column layouts and themes. + private func localRecordIds() async -> [(type: SyncRecordType, ids: [String])] { + let tableFolderIds = tableFolderSyncIds() + var listed: [(type: SyncRecordType, ids: [String])] = [ + (.connection, services.connectionStorage.loadConnections().filter(\.participatesInSync).map { $0.id.uuidString }), + (.group, services.groupStorage.loadGroups().map { $0.id.uuidString }), + (.tag, services.tagStorage.loadTags().map { $0.id.uuidString }), + (.tableFavorite, services.favoriteTablesStorage.loadFavorites().map { FavoriteTablesStorage.syncId(for: $0) }), + (.favoriteDatabase, services.favoriteDatabasesStorage.loadFavorites().map { FavoriteDatabasesStorage.syncId(for: $0) }), + (.tableFolder, tableFolderIds.folders), + (.tableFolderItem, tableFolderIds.items), + ( + .settings, + columnLayouts().customizedStorageKeys().map { FileColumnLayoutPersister.syncCategory(for: $0) } + + themeCatalog.syncCategories + ) + ] + if let favorites = await services.sqlFavoriteManager.favoritesForSync() { + listed.append((.favorite, favorites.map { $0.id.uuidString })) + } + if let folders = await services.sqlFavoriteManager.foldersForSync() { + listed.append((.favoriteFolder, folders.map { $0.id.uuidString })) + } + return listed + } +} diff --git a/apple/TablePro/Core/Sync/SyncCoordinator.swift b/apple/TablePro/Core/Sync/SyncCoordinator.swift index 917a6eaf64..3455061e16 100644 --- a/apple/TablePro/Core/Sync/SyncCoordinator.swift +++ b/apple/TablePro/Core/Sync/SyncCoordinator.swift @@ -33,6 +33,7 @@ final class SyncCoordinator: ObservableObject { private let makeNetworkMonitor: () -> SyncNetworkMonitor? /// Why the license keeps sync off, or nil while it allows sync. private let licenseGate: @MainActor () -> DisableReason? + let recordCreationDate: (CKRecord) -> Date? private var triggers: SyncTriggers? private var licenseCancellable: AnyCancellable? private var debounceTask: Task? @@ -68,11 +69,13 @@ final class SyncCoordinator: ObservableObject { makeNetworkMonitor: @escaping () -> SyncNetworkMonitor? = { SyncNetworkMonitor() }, columnLayouts: @escaping @autoclosure () -> FileColumnLayoutPersister = .shared, themeCatalog: ThemeCatalog = .shared, - licenseGate: (@MainActor () -> DisableReason?)? = nil + licenseGate: (@MainActor () -> DisableReason?)? = nil, + recordCreationDate: @escaping (CKRecord) -> Date? = { $0.creationDate } ) { self.services = services self.makeTransport = transport self.makeNetworkMonitor = makeNetworkMonitor + self.recordCreationDate = recordCreationDate self.changeTracker = services.syncTracker self.metadataStorage = services.syncMetadataStorage self.recordCache = recordCache @@ -714,7 +717,9 @@ final class SyncCoordinator: ObservableObject { settings: SyncSettings, echoGuard: SyncEchoGuard? ) async -> Bool { - guard await applyPullResult(result, settings: settings, echoGuard: echoGuard) else { return false } + guard await applyPullResult(result, settings: settings, echoGuard: echoGuard, isFullFetch: isFullFetch) else { + return false + } if isFullFetch { seedLocalThemesIfPending(after: result) } @@ -725,12 +730,18 @@ final class SyncCoordinator: ObservableObject { internal func applyPullResult( _ result: PullResult, settings: SyncSettings? = nil, - echoGuard: SyncEchoGuard? = nil + echoGuard: SyncEchoGuard? = nil, + isFullFetch: Bool = false ) async -> Bool { let settings = settings ?? services.appSettingsStorage.loadSync() let zoneID = await transport.currentZoneID let deferred = deferredDeletions(dueIn: result, settings: settings, zoneID: zoneID) - let deletedRecordIDs = (result.deletedRecordIDs + deferred.due).filter { recordID in + // Read before anything is applied, while the local stores still hold every record the fetch is checked against. + let pushed = echoGuard?.pushedRecordIDs ?? [] + let missing = isFullFetch + ? await deletionsProvenByFullFetch(result, settings: settings, zoneID: zoneID, pushed: pushed) + : [] + let deletedRecordIDs = (result.deletedRecordIDs + deferred.due + missing).filter { recordID in echoGuard?.withholdsDeletion(recordID, tracker: changeTracker) != true } let pendingEdits = await recordNamesWithPendingEdits() @@ -769,7 +780,7 @@ final class SyncCoordinator: ObservableObject { echoGuard?.withholds(record.recordID, tracker: changeTracker) != true && SyncRecordType(rawValue: record.recordType).map { settings.syncs($0) } == true }) - recordCache.remove(result.deletedRecordIDs + deferred.due) + recordCache.remove(result.deletedRecordIDs + deferred.due + missing) settleDeferredDeletions(deferred, after: result, settings: settings) Self.logger.info( diff --git a/apple/TablePro/Core/Sync/SyncEchoGuard.swift b/apple/TablePro/Core/Sync/SyncEchoGuard.swift index 4c9e90640c..0353d7d78b 100644 --- a/apple/TablePro/Core/Sync/SyncEchoGuard.swift +++ b/apple/TablePro/Core/Sync/SyncEchoGuard.swift @@ -8,6 +8,8 @@ struct SyncEchoGuard: Sendable { let snapshot: SyncEditSnapshot let savedRecords: [CKRecord.ID: SyncRecordIdentity] let deletedRecords: [CKRecord.ID: SyncRecordIdentity] + // Every record the push saved, connections included, which a fetch right after it may not list yet. + let pushedRecordIDs: Set private let savedIdentities: Set init( @@ -17,6 +19,7 @@ struct SyncEchoGuard: Sendable { ) { let guarded = savedRecords.filter { !Self.typesMergedOnPull.contains($0.value.type) } self.snapshot = snapshot + self.pushedRecordIDs = Set(savedRecords.keys) self.savedRecords = guarded self.deletedRecords = deletedRecords self.savedIdentities = Set(guarded.values) diff --git a/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift b/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift index 1dd8f10e35..dab2c5e5b4 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift @@ -60,6 +60,11 @@ enum SQLParameterExtractor { values.append(nil) } + // `LIMIT:n` becomes `LIMIT $1`: PostgreSQL reads `$` inside a name, so `LIMIT$1` would be one identifier. + if range.location > 0, SQLNonCodeSpan.isWordUnit(nsSQL.character(at: range.location - 1)) { + resultSQL += " " + } + switch style { case .questionMark: resultSQL += "?" @@ -81,10 +86,35 @@ enum SQLParameterExtractor { // MARK: - Private private static let colon = UInt16(UnicodeScalar(":").value) + private static let dot = UInt16(UnicodeScalar(".").value) + private static let openBrace = UInt16(UnicodeScalar("{").value) private static let closeBracket = UInt16(UnicodeScalar("]").value) + // Reserved on every engine that takes `:name`, Snowflake included, and always followed by an operand, so a `:` + // glued to one starts a bind: `IN:ids`, `BETWEEN:a AND:b`, `THEN:p`. + private static let operandKeywords: Set = [ + "AND", "BETWEEN", "ELSE", "HAVING", "ILIKE", "IN", "LIKE", "NOT", "OR", "SELECT", "THEN", "WHEN", "WHERE", + ] + + // Snowflake leaves both unreserved, so `SELECT limit:currency` there is a path into a column. Each is the + // row-limit clause, and the `:` after it a bind, only right after an operand: `FROM t LIMIT:n`, `10 OFFSET:m`. + private static let rowLimitKeywords: Set = ["LIMIT", "OFFSET"] + + // An operand comes next after one of these, so a `limit` there is a column. `ALL` is one only after `SELECT` + // (`LIMIT ALL OFFSET:m` is the clause) and `TOP n` is checked two tokens back. + private static let operandStarters: Set = operandKeywords.union([ + "AS", "BY", "CASE", "DISTINCT", "FROM", "ON", "QUALIFY", "RETURNING", "SET", + ]) + + private enum PrecedingToken { + case operandStart + case operandEnd + case word(NSRange) + } + // A `:` written against the end of a name, a literal, a subscript, a call or another `:` is the engine's own // syntax: a cast (`a::int`), a path (`src:customer`), a slice (`arr[lo:hi]`), a typed slot (`{id:UInt32}`). + // Against a keyword that takes an operand it is a bind. private static func scan( sql: String, grammar: SQLLexicalGrammar, @@ -94,16 +124,56 @@ enum SQLParameterExtractor { let length = text.length var index = 0 var followsOperand = false + var precedingToken = PrecedingToken.operandStart + var tokenBeforePreceding = PrecedingToken.operandStart + var wordStart: Int? + var tokenBeforeWord = PrecedingToken.operandStart + var tokenTwoBeforeWord = PrecedingToken.operandStart + var wordFollowsPath = false + + func push(_ token: PrecedingToken) { + tokenBeforePreceding = precedingToken + precedingToken = token + } + + func endWord(at end: Int) { + guard let start = wordStart else { return } + wordStart = nil + push(.word(NSRange(location: start, length: end - start))) + } + + func upperWord(_ range: NSRange) -> String { + text.substring(with: range).uppercased() + } + + func keywordBeforeTakesBind(colonAt colonIndex: Int) -> Bool { + guard let start = wordStart, !wordFollowsPath else { return false } + let word = upperWord(NSRange(location: start, length: colonIndex - start)) + if operandKeywords.contains(word) { return true } + guard rowLimitKeywords.contains(word) else { return false } + switch tokenBeforeWord { + case .operandStart: return false + case .operandEnd: return true + case .word(let range): + let previous = upperWord(range) + if operandStarters.contains(previous) { return false } + guard case .word(let earlierRange) = tokenTwoBeforeWord else { return true } + let earlier = upperWord(earlierRange) + return earlier != "TOP" && !(previous == "ALL" && earlier == "SELECT") + } + } while index < length { if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + endWord(at: index) index = max(span.end, index + 1) followsOperand = !span.kind.isComment + if followsOperand { push(.operandEnd) } continue } let unit = text.character(at: index) - if unit == colon, !followsOperand, index + 1 < length, - SqlDollarQuote.isIdentifierStart(text.character(at: index + 1)) { + if unit == colon, index + 1 < length, SqlDollarQuote.isIdentifierStart(text.character(at: index + 1)), + !followsOperand || keywordBeforeTakesBind(colonAt: index) { var nameEnd = index + 2 while nameEnd < length, SqlDollarQuote.isIdentifierPart(text.character(at: nameEnd)) { nameEnd += 1 @@ -112,11 +182,36 @@ enum SQLParameterExtractor { onParameter(name, NSRange(location: index, length: nameEnd - index)) index = nameEnd followsOperand = true + wordStart = nil + push(.operandEnd) continue } - followsOperand = unit == colon || unit == closeBracket || unit == SqlLexer.closeParen - || SQLNonCodeSpan.isWordUnit(unit) + if SQLNonCodeSpan.isWordUnit(unit) { + if wordStart == nil { + wordStart = index + tokenBeforeWord = precedingToken + tokenTwoBeforeWord = tokenBeforePreceding + // A path step (`src:in`, `t.limit`) or a typed slot's name (`{limit:UInt32}`) is never a keyword. + let before = index > 0 ? text.character(at: index - 1) : 0 + wordFollowsPath = before == dot || before == colon || before == openBrace + } + followsOperand = true + } else { + endWord(at: index) + let closes = unit == closeBracket || unit == SqlLexer.closeParen + followsOperand = unit == colon || closes + if closes { + push(.operandEnd) + } else if !isSpace(unit) { + push(.operandStart) + } + } index += 1 } } + + private static func isSpace(_ unit: UInt16) -> Bool { + if unit < 0x80 { return unit == 0x20 || (unit >= 0x09 && unit <= 0x0D) } + return UnicodeScalar(unit).map { CharacterSet.whitespacesAndNewlines.contains($0) } ?? false + } } diff --git a/apple/TablePro/Models/Query/ParsedRow.swift b/apple/TablePro/Models/Query/ParsedRow.swift index 359054fe45..3b836fea46 100644 --- a/apple/TablePro/Models/Query/ParsedRow.swift +++ b/apple/TablePro/Models/Query/ParsedRow.swift @@ -22,6 +22,9 @@ struct ParsedRow { /// The columns holding SQL the grid staged, which only the grid's own copy carries. var expressionColumns: Set = [] + // Pasted text left these cells blank or had no field for them; their values are NULL until a paste fills them. + var blankColumns: Set = [] + /// Check if row has valid data var isValid: Bool { !values.isEmpty diff --git a/apple/TablePro/Models/Query/QueryResult.swift b/apple/TablePro/Models/Query/QueryResult.swift index 44be1ab384..43fe862ed9 100644 --- a/apple/TablePro/Models/Query/QueryResult.swift +++ b/apple/TablePro/Models/Query/QueryResult.swift @@ -32,6 +32,9 @@ struct QueryResult { var columnMeta: [ResultColumnMeta]? + // See `PluginQueryResult.columnOrigins`. + var columnOrigins: [PluginColumnOrigin?]? + /// Per row, what the driver finds that row by again. Only the driver reads it. var rowLocators: [String?]? diff --git a/apple/TablePro/Models/Query/TableRows.swift b/apple/TablePro/Models/Query/TableRows.swift index f1fb269893..c175fc3c73 100644 --- a/apple/TablePro/Models/Query/TableRows.swift +++ b/apple/TablePro/Models/Query/TableRows.swift @@ -39,6 +39,13 @@ struct TableRows: Sendable { private(set) var masks: ColumnMasks = .none private var exportPolicy: MaskingPolicy = .none private var exportTable: String? + private var reportedOrigins: (columns: [String], origins: [PluginColumnOrigin?])? + + // What the driver said each column reads, while the columns are still the ones it said it for. + var columnOrigins: [PluginColumnOrigin?]? { + guard let reportedOrigins, reportedOrigins.columns == columns else { return nil } + return reportedOrigins.origins + } var maskedColumnNames: Set { Set(masks.maskedColumns.compactMap { columns.indices.contains($0) ? columns[$0] : nil }) @@ -184,6 +191,31 @@ struct TableRows: Sendable { return .fullReplace } + // What to lay over the held values for a reader outside the app, `external` being the bridge's masking of these + // columns. A column the grid masked another way takes the stricter style, over what the grid already hid. + func externalOverlay(_ external: ColumnMasks) -> ColumnMasks { + var styles: [Int: MaskingStyle] = [:] + for (column, style) in external.styles where column < columns.count { + let shown = masks.styles[column] + guard shown != style else { continue } + styles[column] = MaskingStyle.strictest(shown, style) + } + return ColumnMasks(styles: styles) + } + + func externallyMasked(by external: ColumnMasks) -> TableRows { + let overlay = externalOverlay(external) + guard !overlay.isEmpty else { return self } + var copy = self + for index in copy.rows.indices { + for column in overlay.styles.keys where column < copy.rows[index].values.count { + copy.rows[index].values[column] = overlay.value(copy.rows[index].values[column], column: column) + } + } + copy.masks = ColumnMasks(styles: masks.styles.merging(overlay.styles) { _, stricter in stricter }) + return copy + } + // A cell an edit or an earlier load already filled keeps what it holds. @discardableResult mutating func fillUnloaded(_ values: [RowID: [Int: PluginCellValue]]) -> Delta { @@ -475,7 +507,8 @@ struct TableRows: Sendable { unloadedCells: [Int: [Int: Int]] = [:], masks: ColumnMasks = .none, exportPolicy: MaskingPolicy = .none, - exportTable: String? = nil + exportTable: String? = nil, + columnOrigins: [PluginColumnOrigin?]? = nil ) -> TableRows { var rows = ContiguousArray() rows.reserveCapacity(queryRows.count) @@ -513,6 +546,7 @@ struct TableRows: Sendable { tableRows.masks = masks tableRows.exportPolicy = exportPolicy tableRows.exportTable = exportTable + tableRows.reportedOrigins = columnOrigins.map { (columns: columns, origins: $0) } return tableRows } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index c89f3aec2a..01982c4328 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -17582,6 +17582,9 @@ } } } + }, + "A transaction is open on %@. Switching databases reconnects, and the server would roll it back. Commit or roll back the transaction, then switch again." : { + }, "A transaction is open on this connection, and its uncommitted rows aren't in these counts." : { @@ -107796,9 +107799,15 @@ }, "Masked" : { + }, + "Masked %@: this connection has masking rules, and TablePro could not trace these columns to a table column." : { + }, "Masked by a masking rule" : { + }, + "Masked every column: this connection has masking rules, and TablePro could not trace any of them to a table column." : { + }, "Masking" : { @@ -194353,6 +194362,12 @@ }, "The statement did not finish within %d seconds and is still running. It may have been applied; check before running it again." : { + }, + "The statement failed with %@. Its message is withheld because this connection has masking rules, and the message can repeat a masked value." : { + + }, + "The statement failed. Its message is withheld because this connection has masking rules, and the message can repeat a masked value." : { + }, "The statement has %1$lld placeholders but %2$lld values were given." : { "extractionState" : "manual", @@ -194399,6 +194414,9 @@ } } } + }, + "The statement text is withheld because this connection has masking rules, and it can repeat a masked value." : { + }, "The statement that was sent, with its prefix" : { "extractionState" : "stale", @@ -196702,6 +196720,12 @@ }, "These rows show masked values, which cannot be written back. Reveal them to edit." : { + }, + "These statements ran before the failure and were not rolled back:" : { + + }, + "These statements ran inside the transaction already open on this connection, so rolling it back undoes them:" : { + }, "These two engines cannot share a script." : { "localizations" : { @@ -207657,6 +207681,9 @@ } } } + }, + "Undoing what had already run also failed, so the table may be left part way. Run these statements to restore it:" : { + }, "Unexpected “%@” in the .proto file." : { @@ -216961,6 +216988,9 @@ } } } + }, + "Why values are masked, when they are" : { + }, "Wi-Fi" : { "extractionState" : "manual", diff --git a/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift b/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift index 13ef2193b5..3be561720c 100644 --- a/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift +++ b/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift @@ -66,6 +66,8 @@ final class ConnectionFormCoordinator: ObservableObject { // Filled by the reads that prefill the form: a field left empty because its secret could not be read is not a cleared one. private var unreadableSecretKeys: Set = [] + // What those reads found. A save leaves a secret the user did not change as it is in the Keychain. + private var loadedSecrets: [String: String] = [:] private var childChangeForwarding: AnyCancellable? @@ -225,7 +227,7 @@ final class ConnectionFormCoordinator: ObservableObject { "[trace] load found existing id=\(existing.id.uuidString, privacy: .public) name='\(existing.name, privacy: .private(mask: .hash))' promptForPassword=\(existing.promptForPassword)" ) originalConnection = existing - unreadableSecretKeys = storage.recordingUnreadableSecrets { + let reads = storage.recordingSecretReads { network.load(from: existing) auth.load(from: existing, storage: storage) ssh.load(from: existing, storage: storage) @@ -239,6 +241,8 @@ final class ConnectionFormCoordinator: ObservableObject { aiRules.load(from: existing) maskingRules.load(from: existing) } + unreadableSecretKeys = reads.unreadable + loadedSecrets = reads.loaded normalizeTransport() } hasLoadedData = true @@ -458,7 +462,8 @@ final class ConnectionFormCoordinator: ObservableObject { let stagedSecrets = storage.stagingSecretEdits( for: finalId, deviceOnly: !connectionToSave.participatesInSync, - keeping: unreadableSecretKeys + keeping: unreadableSecretKeys, + unchanged: loadedSecrets ) { for field in secureFieldValues { if field.value.isEmpty { @@ -523,6 +528,8 @@ final class ConnectionFormCoordinator: ObservableObject { cloudSQLProxy.save(to: finalId, storage: storage) socksProxy.save(to: finalId, storage: storage) } + // The Keychain may hold this save's values now, whether or not the record saves. + loadedSecrets = [:] guard !stagedSecrets.writeFailed else { saveError = String(localized: "Could not save this connection's passwords to the Keychain. Your changes were not saved.") return diff --git a/apple/TablePro/Views/Editor/EditorEventRouter.swift b/apple/TablePro/Views/Editor/EditorEventRouter.swift index 300d8e2b9d..a6a2ed5293 100644 --- a/apple/TablePro/Views/Editor/EditorEventRouter.swift +++ b/apple/TablePro/Views/Editor/EditorEventRouter.swift @@ -158,7 +158,7 @@ internal final class EditorEventRouter { coordinator.runStatementAtCursorAndAdvance() } - // A large document writes its text to the tab on a delay; a run or an AI action reads the tab. + // A large document writes its text to the tab on a delay; a run, save, close or quit reads the tab. internal func flushPendingText(forTab tabId: UUID, connectionId: UUID) { for ref in editors.values { guard let coordinator = ref.coordinator, @@ -167,6 +167,12 @@ internal final class EditorEventRouter { } } + internal func flushAllPendingText() { + for ref in editors.values { + ref.coordinator?.controller?.flushPendingTextWriteback() + } + } + /// Called when the Edit menu's "Clear Selection" item is chosen with the pointer. /// /// Only with the pointer. AppKit never dispatches an unmodified Escape as a key equivalent: diff --git a/apple/TablePro/Views/Editor/SQLEditorController.swift b/apple/TablePro/Views/Editor/SQLEditorController.swift index 1b218c27bc..e62f529488 100644 --- a/apple/TablePro/Views/Editor/SQLEditorController.swift +++ b/apple/TablePro/Views/Editor/SQLEditorController.swift @@ -100,6 +100,9 @@ final class SQLEditorController { func apply(_ newInput: SQLEditorInput) { guard !isTornDown else { return } let old = input + if newInput.tabID != old.tabID { + handOverPendingText(to: old.text) + } input = newInput applyInput(newInput) pushEditor() @@ -132,6 +135,7 @@ final class SQLEditorController { func teardown() { guard !isTornDown else { return } isTornDown = true + handOverPendingText(to: input.text) profileTask?.cancel() profileTask = nil revisionSubscription = nil @@ -140,6 +144,13 @@ final class SQLEditorController { host?.dismantle() } + // A large document's last keys are still on their way to the binding when the editor moves to another tab or + // goes away. Both run inside a SwiftUI update, where writing the tab would publish mid-update, so it lands next turn. + private func handOverPendingText(to binding: Binding) { + guard let text = host?.takePendingTextWriteback() else { return } + Task { @MainActor in binding.wrappedValue = text } + } + private func applyInput(_ input: SQLEditorInput) { coordinator.onCloseTab = input.onCloseTab coordinator.onExecuteQuery = input.onExecuteQuery diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AIQueryActions.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AIQueryActions.swift index 1dc5c1c0d3..750e98c33f 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AIQueryActions.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AIQueryActions.swift @@ -43,7 +43,7 @@ extension MainContentCoordinator { target: AssistantEditorTarget( tabId: tab.id, scope: scope(for: tab), - errorMessage: tab.latestError?.message, + errorMessage: externalErrorText(tab.latestError?.message), errorQuery: tab.latestError?.query ) ) @@ -67,7 +67,7 @@ extension MainContentCoordinator { scope: scope(for: tab), databaseType: connection.type, source: source, - errorMessage: error + errorMessage: externalErrorText(error) ) presentIdleAssistant()?.runQueryAction(request) } @@ -99,10 +99,16 @@ extension MainContentCoordinator { scope: scope(for: tab), databaseType: connection.type, source: QueryEditorAnchor(tabId: tab.id, range: range), - explainPlan: explainPlan(in: tab, covering: range) + explainPlan: externalPlanText(explainPlan(in: tab, covering: range)) ) } + // A plan can hold values the planner read, as MySQL's does from a const table; see `ExternalRowMasking.plan(in:)`. + private func externalPlanText(_ plan: String?) -> String? { + guard plan != nil, !maskingPolicy.isEmpty else { return plan } + return ExternalRowMasking.plan(in: connection.type.explainVariants.first?.format) == .byName ? plan : nil + } + func editorText(for target: AIQueryTarget, in fullQuery: String) -> (sql: String, offset: Int) { switch target { case .selectionOrStatementAtCursor: diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Favorites.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Favorites.swift index 7716ecc16f..bec094a2e0 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Favorites.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Favorites.swift @@ -32,6 +32,7 @@ extension MainContentCoordinator { } func saveCurrentQueryAsFavorite() { + if let tabId = tabManager.selectedTabId { flushEditorText(forTab: tabId) } guard let tab = tabManager.selectedTab, tab.tabType == .query else { return } let query = tab.content.query.trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+GridModels.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+GridModels.swift index 6cce31bb76..c9ee39a429 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+GridModels.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+GridModels.swift @@ -139,6 +139,9 @@ extension MainContentCoordinator { @discardableResult func commitActiveGridEdit() -> Bool { + guard let selectedTabId = tabManager.selectedTabId else { return true } + // A large document's last keys reach the tab on a delay, and a save or close reads the tab. + flushEditorText(forTab: selectedTabId) guard let tab = tabManager.selectedTab else { return true } var committed = existingGridModel(forTab: tab.id)?.commitEditing() ?? true if let schemaModel = activeGridModelProvider(for: tab)?.activeGridModel { diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Masking.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Masking.swift index 014180446e..663fe1f4f4 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Masking.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Masking.swift @@ -21,6 +21,32 @@ extension MainContentCoordinator { tabManager.tabs.first(where: { $0.id == tabId })?.revealsMaskedValues == true ? .none : maskingPolicy } + // The grid matched only the names a statement chose, so a tab's rows leave for the assistant or a script masked + // the way the bridge masks an external caller's result, whatever the person revealed in the grid. + func externalOverlay(forTab tabId: UUID) -> ColumnMasks { + let policy = maskingPolicy + guard !policy.isEmpty else { return .none } + let rows = tabSessionRegistry.tableRows(for: tabId) + let tab = tabManager.tabs.first { $0.id == tabId } + let table = tab?.tabType == .table ? tab?.tableContext.tableName : nil + let external = policy.externalColumnMasks( + table: table, columns: rows.columns, typeNames: rows.columnTypes.map(\.rawType), origins: rows.columnOrigins + ) + return rows.externalOverlay(external) + } + + func externallyMaskedRows(forTab tabId: UUID) -> TableRows { + let rows = tabSessionRegistry.tableRows(for: tabId) + let overlay = externalOverlay(forTab: tabId) + return overlay.isEmpty ? rows : rows.externallyMasked(by: overlay) + } + + // A failed statement's text can repeat a value the server read. + func externalErrorText(_ message: String?) -> String? { + guard let message, !maskingPolicy.isEmpty else { return message } + return ServerErrorText.withheldMessage + } + func observeMaskingRuleChanges() -> AnyCancellable { services.appEvents.connectionUpdated .filter { [connectionId] changed in changed == nil || changed == connectionId } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift index 45e6d2c25f..581bd55786 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift @@ -135,7 +135,8 @@ extension MainContentCoordinator { rowLocators: [String?]? = nil, exportDocuments: [PluginExportDocument]? = nil, absentCells: [Int: Set] = [:], - unloadedCells: [Int: [Int: Int]] = [:] + unloadedCells: [Int: [Int: Int]] = [:], + columnOrigins: [PluginColumnOrigin?]? = nil ) { queryExecutionCoordinator.applyPhase1Result( tabId: tabId, @@ -162,7 +163,8 @@ extension MainContentCoordinator { rowLocators: rowLocators, exportDocuments: exportDocuments, absentCells: absentCells, - unloadedCells: unloadedCells + unloadedCells: unloadedCells, + columnOrigins: columnOrigins ) } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentView+Helpers.swift b/apple/TablePro/Views/Main/Extensions/MainContentView+Helpers.swift index a2b789bbdb..41deb3a3e6 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentView+Helpers.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentView+Helpers.swift @@ -109,6 +109,8 @@ extension MainContentView { guard let tab = currentTab else { return nil } let tableRows = coordinator.tabSessionRegistry.tableRows(for: tab.id) guard !tableRows.columns.isEmpty, !tableRows.rows.isEmpty else { return nil } + // The summary goes to the provider, so it is masked as an external caller's result is. + let overlay = coordinator.externalOverlay(forTab: tab.id) let columns = tableRows.columns let rows = tableRows.rows @@ -125,7 +127,7 @@ extension MainContentView { return DataGridCellContent.unloadedSpokenText(byteCount: byteCount) } let raw: String - switch row.values[i] { + switch overlay.value(row.values[i], column: i) { case .null: raw = "NULL" case .text(let s): diff --git a/apple/TablePro/Views/Main/MainContentCommandActions+BulkClose.swift b/apple/TablePro/Views/Main/MainContentCommandActions+BulkClose.swift index 52c901f5c2..49f8f70d4e 100644 --- a/apple/TablePro/Views/Main/MainContentCommandActions+BulkClose.swift +++ b/apple/TablePro/Views/Main/MainContentCommandActions+BulkClose.swift @@ -79,12 +79,11 @@ extension MainContentCommandActions { // A list edit, not a window close: the window hosts every connection, so closing all tabs leaves this one empty. private func runBatchClose(kind: BatchCloseKind) async { - guard let coordinator else { return } + guard let coordinator, !tabsToClose(kind: kind).isEmpty else { return } + // An open cell editor or the editor's pending text is unsaved work, so it lands before the victims are read. + coordinator.commitActiveGridEdit() let victims = tabsToClose(kind: kind) - guard !victims.isEmpty else { return } let work = runningWork(.tabs(Set(victims.map(\.id)))) - // An open cell editor is unsaved work, so it lands before the question is chosen. - coordinator.commitActiveGridEdit() if !hasUnsavedWork(among: victims) { guard await confirmsStoppingWork(work) else { return } } diff --git a/apple/TablePro/Views/Main/MainContentCommandActions+Closing.swift b/apple/TablePro/Views/Main/MainContentCommandActions+Closing.swift index 58f247dd5e..5c96cd52ee 100644 --- a/apple/TablePro/Views/Main/MainContentCommandActions+Closing.swift +++ b/apple/TablePro/Views/Main/MainContentCommandActions+Closing.swift @@ -31,12 +31,12 @@ extension MainContentCommandActions { // Asks before closing unsaved work (HIG); like NSDocument, the tab stays open when the save leaves work unsaved. func closeTabAwaiting(id: UUID) async { - guard let coordinator, - let tab = coordinator.tabManager.tabs.first(where: { $0.id == id }) else { return } - // An open cell editor is unsaved work too, so it lands before the question is asked. + guard let coordinator else { return } + // An open cell editor or the editor's pending text is unsaved work too, so it lands before the tab is read. if coordinator.tabManager.selectedTabId == id { coordinator.commitActiveGridEdit() } + guard let tab = coordinator.tabManager.tabs.first(where: { $0.id == id }) else { return } let work = runningWork(.tabs([id])) guard !work.isEmpty || coordinator.hasUnsavedWork(in: tab) else { coordinator.closeTabsByUser(ids: [id]) diff --git a/apple/TablePro/Views/Main/MainContentCommandActions.swift b/apple/TablePro/Views/Main/MainContentCommandActions.swift index 5b5536605c..ec02fa09d7 100644 --- a/apple/TablePro/Views/Main/MainContentCommandActions.swift +++ b/apple/TablePro/Views/Main/MainContentCommandActions.swift @@ -928,6 +928,9 @@ final class MainContentCommandActions: ObservableObject { @discardableResult func saveFileAsAwaiting() async -> Bool { + if let tabId = coordinator?.tabManager.selectedTabId { + coordinator?.flushEditorText(forTab: tabId) + } guard let tab = coordinator?.tabManager.selectedTab, tab.tabType == .query else { return false } let content = tab.content.query diff --git a/apple/TablePro/Views/Main/MainContentCoordinator.swift b/apple/TablePro/Views/Main/MainContentCoordinator.swift index 5a73e9d4fe..8ec9847488 100644 --- a/apple/TablePro/Views/Main/MainContentCoordinator.swift +++ b/apple/TablePro/Views/Main/MainContentCoordinator.swift @@ -1359,7 +1359,8 @@ final class MainContentCoordinator: ObservableObject { rowLocators: isTableTab ? fetchResult.rowLocators : nil, exportDocuments: fetchResult.exportDocuments, absentCells: fetchResult.absentCells, - unloadedCells: fetchResult.unloadedCells + unloadedCells: fetchResult.unloadedCells, + columnOrigins: fetchResult.columnOrigins ) scheduleTraceCompletion(traceToken, outcome: .completed) diff --git a/apple/TablePro/Views/Settings/Sections/MCPGrantListView.swift b/apple/TablePro/Views/Settings/Sections/MCPGrantListView.swift index c7dd80d753..55efe881c7 100644 --- a/apple/TablePro/Views/Settings/Sections/MCPGrantListView.swift +++ b/apple/TablePro/Views/Settings/Sections/MCPGrantListView.swift @@ -4,12 +4,14 @@ // import SwiftUI +import TableProMCP /// The approvals the user has already given, so a remembered answer stays visible and revocable. struct MCPGrantListView: View { @ObservedObject private var mcpServerManager = MCPServerManager.shared @State private var grants: [MCPConnectionGrant] = [] @State private var connectionNames: [UUID: String] = [:] + @State private var tokenNames: [UUID: String] = [:] var body: some View { Group { @@ -18,10 +20,15 @@ struct MCPGrantListView: View { .foregroundStyle(.secondary) } else { ForEach(grants) { grant in - LabeledContent(name(for: grant)) { + LabeledContent { Button(String(localized: "Forget")) { forget(grant) } + } label: { + Text(name(for: grant)) + if let client = Self.clientName(forSubject: grant.subject, tokenNames: tokenNames) { + Text(client) + } } } @@ -37,6 +44,18 @@ struct MCPGrantListView: View { connectionNames[grant.connectionId] ?? String(localized: "Deleted connection") } + // Two clients approved for one connection would otherwise show as two identical rows. + nonisolated static func clientName(forSubject subject: String, tokenNames: [UUID: String]) -> String? { + switch subject { + case MCPGrantSubject.bridge.storageKey: + return IntegrationsFormatting.displayTokenName(MCPTokenStore.stdioBridgeTokenName) + case MCPGrantSubject.claudeAgent.storageKey: + return AIProviderType.claudeAgent.displayName + default: + return UUID(uuidString: subject).flatMap { tokenNames[$0] }.map(IntegrationsFormatting.displayTokenName) + } + } + private func forget(_ grant: MCPConnectionGrant) { Task { await MCPServerManager.shared.forgetGrant(subject: grant.subject, connectionId: grant.connectionId) @@ -58,5 +77,7 @@ struct MCPGrantListView: View { ConnectionStorage.shared.loadConnections().map { ($0.id, $0.name) }, uniquingKeysWith: { first, _ in first } ) + let tokens = await mcpServerManager.tokenStore?.list() ?? [] + tokenNames = Dictionary(tokens.map { ($0.id, $0.name) }, uniquingKeysWith: { first, _ in first }) } } diff --git a/apple/TablePro/Views/Settings/Sections/MCPSection.swift b/apple/TablePro/Views/Settings/Sections/MCPSection.swift index f86c8a8841..66a2b46c14 100644 --- a/apple/TablePro/Views/Settings/Sections/MCPSection.swift +++ b/apple/TablePro/Views/Settings/Sections/MCPSection.swift @@ -160,6 +160,7 @@ struct MCPSection: View { connectionAccess: access, expiresAt: expiresAt, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) else { return } revealedToken = result.token diff --git a/apple/TablePro/Views/Structure/StructureRebuildPlanRunner.swift b/apple/TablePro/Views/Structure/StructureRebuildPlanRunner.swift index a3ff46c722..4524bc9ac7 100644 --- a/apple/TablePro/Views/Structure/StructureRebuildPlanRunner.swift +++ b/apple/TablePro/Views/Structure/StructureRebuildPlanRunner.swift @@ -181,6 +181,7 @@ enum StructureRebuildPlanRunner { try await driver.commitTransaction() } } catch { + var leftover: Leftover? if usesTransaction { do { try await driver.rollbackTransaction() @@ -188,16 +189,17 @@ enum StructureRebuildPlanRunner { logger.error("Rollback failed: \(error.publicLogShape, privacy: .public)") } } else if completed > 0 { - /// An engine whose DDL commits statement by statement has nothing to roll back, - /// so it supplies statements that put back what already ran. - for sql in plan.compensation { - _ = try? await driver.execute(query: sql) - } + leftover = await undo( + plan, + completed: completed, + insideOpenTransaction: plan.isTransactional && owner == .session, + driver: driver + ) } for sql in plan.epilogue { _ = try? await driver.execute(query: sql) } - throw runFailure(from: error) + throw runFailure(from: error, leftover: leftover) } for sql in plan.epilogue { @@ -224,12 +226,69 @@ enum StructureRebuildPlanRunner { } } + // What a failed plan left on the server that nothing took back, so the user can see the table moved. + enum Leftover: Equatable { + case ran([String]) + case pendingInOpenTransaction([String]) + case unrestored([String]) + + var note: String { + switch self { + case .ran(let statements): + return ([String(localized: "These statements ran before the failure and were not rolled back:")] + statements) + .joined(separator: "\n") + case .pendingInOpenTransaction(let statements): + let heading = String( + localized: "These statements ran inside the transaction already open on this connection, so rolling it back undoes them:" + ) + return ([heading] + statements).joined(separator: "\n") + case .unrestored(let statements): + let heading = String( + localized: "Undoing what had already run also failed, so the table may be left part way. Run these statements to restore it:" + ) + return ([heading] + statements).joined(separator: "\n") + } + } + } + + // An engine whose DDL commits statement by statement has nothing to roll back, so the plan carries statements that put back what ran. + nonisolated private static func undo( + _ plan: PluginColumnReorderPlan, + completed: Int, + insideOpenTransaction: Bool, + driver: any DatabaseDriver + ) async -> Leftover? { + // The user's transaction holds them, and only the user may end it. + guard !insideOpenTransaction else { + return .pendingInOpenTransaction(Array(plan.statements.prefix(completed))) + } + guard !plan.compensation.isEmpty else { + return .ran(Array(plan.statements.prefix(completed))) + } + var failed: [String] = [] + for sql in plan.compensation { + do { + _ = try await driver.execute(query: sql) + } catch { + logger.error("Compensation failed: \(error.publicLogShape, privacy: .public)") + failed.append(sql) + } + } + return failed.isEmpty ? nil : .unrestored(failed) + } + /// A refusal this runner raised keeps its own wording; anything from the driver is wrapped so /// the message says which step failed. - nonisolated private static func runFailure(from error: any Error) -> any Error { - if let runError = error as? RunError { return runError } - return RunError.executionFailed( - String(format: String(localized: "The change could not be applied: %@"), error.localizedDescription) - ) + nonisolated private static func runFailure(from error: any Error, leftover: Leftover?) -> any Error { + let failure: RunError + if let runError = error as? RunError { + failure = runError + } else { + failure = .executionFailed( + String(format: String(localized: "The change could not be applied: %@"), error.localizedDescription) + ) + } + guard let leftover else { return failure } + return RunError.executionFailed(failure.localizedDescription + "\n\n" + leftover.note) } } diff --git a/apple/TableProTests/Core/AI/ClaudeAgentGrantSubjectTests.swift b/apple/TableProTests/Core/AI/ClaudeAgentGrantSubjectTests.swift new file mode 100644 index 0000000000..a0e784a697 --- /dev/null +++ b/apple/TableProTests/Core/AI/ClaudeAgentGrantSubjectTests.swift @@ -0,0 +1,203 @@ +// +// ClaudeAgentGrantSubjectTests.swift +// TableProTests +// +// The Claude Agent chat mints a token for each turn and deletes it after, so an Ask Once answer +// filed under the token id was gone by the next turn and the alert came back every time. +// + +import Foundation +import TableProMCP +import TableProMCPTestSupport +import TableProTestSupport +import Testing + +@testable import TablePro + +private final class MemoryTokenCredentials: MCPTokenCredentialStoring, @unchecked Sendable { + private let lock = NSLock() + private var stored: Data? = Data("[]".utf8) + + func read() -> MCPTokenBlob { + lock.withLock { stored.map(MCPTokenBlob.found) ?? .missing } + } + + @discardableResult + func write(_ data: Data) -> Bool { + lock.withLock { stored = data } + return true + } + + func delete() { + lock.withLock { stored = nil } + } +} + +private actor EndedTokens { + private var keys: Set = [] + + func record(_ key: String) { + keys.insert(key) + } + + func contains(_ key: String) -> Bool { + keys.contains(key) + } +} + +struct ClaudeAgentGrantSubjectTests { + private let connectionId = UUID() + + private func mint( + in store: MCPTokenStore, + isClaudeAgentCredential: Bool + ) async throws -> (token: MCPAuthToken, plaintext: String) { + try await store.generate( + name: "__claude_agent__", + permissions: .readOnly, + connectionAccess: .limited([connectionId]), + expiresAt: Date.now.addingTimeInterval(900), + isBridgeCredential: false, + isClaudeAgentCredential: isClaudeAgentCredential, + extraScopes: [] + ) + } + + private func principal(presenting plaintext: String, to store: MCPTokenStore) async throws -> MCPPrincipal { + let clock = MCPTestClock() + let authenticator = MCPBearerTokenAuthenticator( + tokenStore: store, + rateLimiter: MCPRateLimiter(clock: clock), + clock: clock + ) + let decision = await authenticator.authenticate(authorizationHeader: "Bearer \(plaintext)", clientAddress: .loopback) + guard case .allow(let principal) = decision else { + Issue.record("Expected allow, got \(decision)") + throw CancellationError() + } + return principal + } + + // Clears what each ended token withdraws, as the server manager's observer does. + private func watch(_ store: MCPTokenStore, clearing ledger: MCPApprovalLedger) async -> EndedTokens { + let ended = EndedTokens() + await store.addRevocationObserver { key, withdrawn in + if let withdrawn { + await ledger.clear(subject: withdrawn) + } + await ended.record(key) + } + return ended + } + + private func approvedTurn( + in store: MCPTokenStore, + ledger: MCPApprovalLedger + ) async throws -> (token: MCPAuthToken, plaintext: String) { + let minted = try await mint(in: store, isClaudeAgentCredential: true) + let turn = try await principal(presenting: minted.plaintext, to: store) + await ledger.record(principal: turn, connectionId: connectionId, approved: true) + return minted + } + + @Test("An approval given on one turn still stands on the next turn's token") + func approvalOutlivesTheTurnToken() async throws { + let store = MCPTokenStore(credentialStore: MemoryTokenCredentials()) + let ledger = MCPApprovalLedger(clock: MCPTestClock(), store: MCPInMemoryApprovalStore()) + let ended = await watch(store, clearing: ledger) + + let first = try await approvedTurn(in: store, ledger: ledger) + await store.retire(tokenId: first.token.id) + let key = first.token.id.uuidString + try await waitUntil { await ended.contains(key) } + + let second = try await mint(in: store, isClaudeAgentCredential: true) + let secondTurn = try await principal(presenting: second.plaintext, to: store) + + #expect(secondTurn.grantSubject == .claudeAgent) + #expect(await ledger.isApproved(principal: secondTurn, connectionId: connectionId)) + #expect(await ledger.isApproved(principal: secondTurn, connectionId: UUID()) == false) + } + + @Test("Revoking the agent's token in Settings withdraws the approval it was carrying") + func revokingTheAgentTokenWithdrawsItsApproval() async throws { + let store = MCPTokenStore(credentialStore: MemoryTokenCredentials()) + let ledger = MCPApprovalLedger(clock: MCPTestClock(), store: MCPInMemoryApprovalStore()) + let ended = await watch(store, clearing: ledger) + + let turn = try await approvedTurn(in: store, ledger: ledger) + await store.revoke(tokenId: turn.token.id) + let key = turn.token.id.uuidString + try await waitUntil { await ended.contains(key) } + + let next = try await mint(in: store, isClaudeAgentCredential: true) + let nextTurn = try await principal(presenting: next.plaintext, to: store) + #expect(await ledger.isApproved(principal: nextTurn, connectionId: connectionId) == false) + } + + @Test("Deleting the agent's token in Settings withdraws the approval it was carrying") + func deletingTheAgentTokenWithdrawsItsApproval() async throws { + let store = MCPTokenStore(credentialStore: MemoryTokenCredentials()) + let ledger = MCPApprovalLedger(clock: MCPTestClock(), store: MCPInMemoryApprovalStore()) + let ended = await watch(store, clearing: ledger) + + let turn = try await approvedTurn(in: store, ledger: ledger) + await store.delete(tokenId: turn.token.id) + let key = turn.token.id.uuidString + try await waitUntil { await ended.contains(key) } + + let next = try await mint(in: store, isClaudeAgentCredential: true) + let nextTurn = try await principal(presenting: next.plaintext, to: store) + #expect(await ledger.isApproved(principal: nextTurn, connectionId: connectionId) == false) + } + + @Test("A grant row names the client it belongs to") + func grantRowsNameTheirClient() { + let tokenId = UUID() + let names = [tokenId: "Raycast"] + + #expect( + MCPGrantListView.clientName(forSubject: MCPGrantSubject.claudeAgent.storageKey, tokenNames: names) + == AIProviderType.claudeAgent.displayName + ) + #expect( + MCPGrantListView.clientName(forSubject: MCPGrantSubject.bridge.storageKey, tokenNames: names) + == IntegrationsFormatting.displayTokenName(MCPTokenStore.stdioBridgeTokenName) + ) + #expect(MCPGrantListView.clientName(forSubject: tokenId.uuidString, tokenNames: names) == "Raycast") + #expect(MCPGrantListView.clientName(forSubject: UUID().uuidString, tokenNames: names) == nil) + } + + @Test("A token that only takes the agent's name inherits none of its approvals") + func nameAloneIsNotTheAgent() async throws { + let store = MCPTokenStore(credentialStore: MemoryTokenCredentials()) + let ledger = MCPApprovalLedger(clock: MCPTestClock(), store: MCPInMemoryApprovalStore()) + + let agent = try await mint(in: store, isClaudeAgentCredential: true) + await ledger.record( + principal: try await principal(presenting: agent.plaintext, to: store), + connectionId: connectionId, + approved: true + ) + + let impostor = try await mint(in: store, isClaudeAgentCredential: false) + let impostorPrincipal = try await principal(presenting: impostor.plaintext, to: store) + + #expect(impostorPrincipal.grantSubject == .token(impostor.token.id)) + #expect(await ledger.isApproved(principal: impostorPrincipal, connectionId: connectionId) == false) + } + + @Test("The agent's mark survives a save and reload, and a token saved without it reads as a client") + func markRoundTrips() async throws { + let store = MCPTokenStore(credentialStore: MemoryTokenCredentials()) + let agent = try await mint(in: store, isClaudeAgentCredential: true).token + let client = try await mint(in: store, isClaudeAgentCredential: false).token + + let data = try MCPTokenStore.encodeStored(tokens: [agent, client], undecoded: []) + let marks = MCPTokenStore.decodeStored(data).tokens.map { $0.isClaudeAgentCredential } + let mentions = String(bytes: data, encoding: .utf8)?.components(separatedBy: "isClaudeAgentCredential").count + + #expect(marks == [true, false]) + #expect(mentions == 2) + } +} diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift index 984194c139..d3d4c2d300 100644 --- a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift @@ -131,6 +131,28 @@ struct SQLStatementGeneratorBitColumnTests { #expect(texts(statement) == ["5", "7"]) } + // Cast as text, a boolean word is a truncated integer: 0 outside strict mode, an error inside it. + @Test("A boolean word, as JSON and PostgreSQL write one, is bound as 1 or 0") + func booleanWordsAreBoundAsBits() throws { + #expect(SQLStatementGenerator.decimalBitText("true") == "1") + #expect(SQLStatementGenerator.decimalBitText("FALSE") == "0") + #expect(SQLStatementGenerator.decimalBitText("t") == "1") + #expect(SQLStatementGenerator.decimalBitText("f") == "0") + #expect(SQLStatementGenerator.decimalBitText("yes") == "1") + + let statement = try makeGenerator().insertStatement(columns: ["id", "flags"], values: ["1", "true"]) + #expect(statement?.sql == "INSERT INTO `t` (`id`, `flags`) VALUES (?, CAST(? AS UNSIGNED))") + #expect(texts(statement) == ["1", "1"]) + } + + // Bound as it is, a space is its byte: 32 in a BIT(8), and too long for a BIT(1) in strict mode. + @Test("An empty or blank BIT value is bound as empty text, which the server stores as 0", arguments: ["", " ", "\t "]) + func blankTextIsBoundEmpty(text: String) throws { + let statement = try makeGenerator().insertStatement(columns: ["id", "flags"], values: ["1", .text(text)]) + #expect(statement?.sql == "INSERT INTO `t` (`id`, `flags`) VALUES (?, ?)") + #expect(texts(statement) == ["1", ""]) + } + @Test("Other columns, unknown types and engines whose BIT is a string of digits keep the plain placeholder") func everythingElseIsUnchanged() throws { let untyped = try makeGenerator(typeNames: [:]).generateUpdateSQL(for: flagsEdit(to: "5")) diff --git a/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift b/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift index e112359dac..d6ba71ac85 100644 --- a/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift +++ b/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift @@ -44,6 +44,30 @@ struct CompareSQLLiteralTests { #expect(literal == #"'x\'"#) } + // Quoted, MySQL stores and compares a BIT value as its characters' bytes, so a synced 5 lands as 53. + @Test + func mysqlBitIsWrittenAsItsNumber() { + let driver = NoBackslashEscapesDriver() + let bit = ColumnType.boolean(rawType: "bit(8)") + for type in [DatabaseType.mysql, .mariadb] { + #expect(CompareSQLLiteral.textLiteral("5", columnType: bit, databaseType: type, driver: driver) == "5") + #expect(CompareSQLLiteral.literal( + for: .text("170"), columnType: bit, databaseType: type, driver: driver, keptInScript: false + ) == "170") + } + #expect(CompareSQLLiteral.textLiteral("abc", columnType: bit, databaseType: .mysql, driver: driver) == "'abc'") + #expect(CompareSQLLiteral.textLiteral("+5", columnType: bit, databaseType: .mysql, driver: driver) == "'+5'") + } + + @Test + func otherBooleansAndBitStringsStayQuoted() { + let driver = NoBackslashEscapesDriver() + let tinyint = ColumnType.boolean(rawType: "tinyint(1)") + #expect(CompareSQLLiteral.textLiteral("1", columnType: tinyint, databaseType: .mysql, driver: driver) == "'1'") + let bit = ColumnType.boolean(rawType: "bit(8)") + #expect(CompareSQLLiteral.textLiteral("5", columnType: bit, databaseType: .postgresql, driver: driver) != "5") + } + @Test func bitStringEnginesKeepTheDefaultSpelling() throws { for type in [DatabaseType.mysql, .mariadb, .tidb, .databend, .oceanbase, .sqlite, .clickhouse, .libsql, .turso, .cloudflareD1] { diff --git a/apple/TableProTests/Core/Compare/DataSyncScriptBuilderTests.swift b/apple/TableProTests/Core/Compare/DataSyncScriptBuilderTests.swift index fd0aa06749..72c7c861e1 100644 --- a/apple/TableProTests/Core/Compare/DataSyncScriptBuilderTests.swift +++ b/apple/TableProTests/Core/Compare/DataSyncScriptBuilderTests.swift @@ -5,6 +5,7 @@ import Foundation import TableProPluginKit +import Testing import XCTest @testable import TablePro @@ -889,3 +890,57 @@ final class DataSyncScriptBuilderColumnTests: XCTestCase { ) } } + +// PostgreSQL reads a bit string as its digits and MySQL a BIT as its number, so a source value is respelled for the +// target: written as it came, 00000101 lands in MySQL as 101. +struct DataSyncBitRespellingTests { + private func columns(from source: SQLTypeFamily, to target: SQLTypeFamily) -> [CompareColumn] { + [ + CompareColumn(name: "id", sourceType: "int", targetType: "int", sourceFamily: source, targetFamily: target), + CompareColumn(name: "flags", sourceType: "bit(8)", targetType: "bit(8)", sourceFamily: source, targetFamily: target) + ] + } + + @Test + func aPostgresBitStringIsWrittenIntoMySQLAsItsNumber() { + let statements = PlanSyncFixture.build( + [ + PlanSyncFixture.insertEntry(["id": .text("1"), "flags": .text("00000101")]), + PlanSyncFixture.updateEntry( + source: ["id": .text("2"), "flags": .text("00000110")], + target: ["id": .text("2"), "flags": .text("5")], + changed: ["flags"], + key: "2" + ) + ], + plan: PlanSyncFixture.makePlan(columns: columns(from: .postgres, to: .mysql)), + databaseType: .mysql + ) + let sql = statements.map { $0.sql } + #expect(sql == [ + "INSERT INTO `users` (`id`, `flags`) VALUES (1, 5)", + "UPDATE `users` SET `flags` = 6 WHERE `id` = 2" + ]) + } + + @Test + func aMySQLBitIsWrittenIntoPostgresAsItsDigits() { + let statements = PlanSyncFixture.build( + [PlanSyncFixture.insertEntry(["id": .text("1"), "flags": .text("5")])], + plan: PlanSyncFixture.makePlan(columns: columns(from: .mysql, to: .postgres)), + databaseType: .postgresql + ) + #expect(statements.first?.sql.contains("'00000101'") == true) + } + + @Test + func aKeyReadFromTheTargetIsNotRespelled() { + let plan = PlanSyncFixture.makePlan(columns: columns(from: .postgres, to: .mysql), keys: ["flags"]) + let statements = PlanSyncFixture.build( + [PlanSyncFixture.deleteEntry(["id": .text("1"), "flags": .text("5")])], + plan: plan, + databaseType: .mysql + ) + #expect(statements.first?.sql == "DELETE FROM `users` WHERE `flags` = 5") + } +} diff --git a/apple/TableProTests/Core/CrossEngine/CrossEngineValueCoercerTests.swift b/apple/TableProTests/Core/CrossEngine/CrossEngineValueCoercerTests.swift index 895a80b358..9607aea9f8 100644 --- a/apple/TableProTests/Core/CrossEngine/CrossEngineValueCoercerTests.swift +++ b/apple/TableProTests/Core/CrossEngine/CrossEngineValueCoercerTests.swift @@ -224,6 +224,53 @@ struct CrossEngineValueCoercerTests { #expect(subject.coerce([.text("{a,b}")]) == [.text("{a,b}")]) } + // MARK: - Bit strings + + private func bitCoercer( + source: CanonicalTypeKind, + target: CanonicalTypeKind, + from sourceFamily: SQLTypeFamily, + to targetFamily: SQLTypeFamily + ) -> CrossEngineValueCoercer { + CrossEngineValueCoercer(pairs: [.init(source: source, target: target)], from: sourceFamily, to: targetFamily) + } + + @Test + func aPostgresBitStringBecomesTheNumberMySQLReads() { + let subject = bitCoercer(source: .bitString(length: 8), target: .bitString(length: 8), from: .postgres, to: .mysql) + #expect(subject.coerce([.text("00000101")]) == [.text("5")]) + #expect(subject.coerce([.text("00000000")]) == [.text("0")]) + #expect(subject.coerce([.null]) == [.null]) + } + + @Test + func aMySQLBitBecomesDigitsPaddedToTheTargetLength() { + let subject = bitCoercer(source: .bitString(length: 8), target: .bitString(length: 8), from: .mysql, to: .postgres) + #expect(subject.coerce([.text("5")]) == [.text("00000101")]) + #expect(subject.coerce([.text("170")]) == [.text("10101010")]) + } + + @Test + func aVaryingTargetTakesTheSourceLength() { + let subject = bitCoercer(source: .bitString(length: 4), target: .bitString(length: nil), from: .mysql, to: .duckdb) + #expect(subject.coerce([.text("5")]) == [.text("0101")]) + } + + @Test + func bitTextThatIsNotABitValueIsLeftAlone() { + let toMySQL = bitCoercer(source: .bitString(length: 8), target: .bitString(length: 8), from: .postgres, to: .mysql) + #expect(toMySQL.coerce([.text("0012")]) == [.text("0012")]) + let toPostgres = bitCoercer(source: .bitString(length: 8), target: .bitString(length: 8), from: .mysql, to: .postgres) + #expect(toPostgres.coerce([.text("b'101'")]) == [.text("b'101'")]) + } + + @Test + func twoEnginesThatReadBitsAlikeKeepTheValue() { + let subject = bitCoercer(source: .bitString(length: 8), target: .bitString(length: 8), from: .postgres, to: .duckdb) + #expect(!subject.isNeeded) + #expect(!coercer([.bitString(length: 8)], from: .mysql).isNeeded) + } + // MARK: - Shape /// The width is checked before the coercion, so a short row can never reach it. Guarded here diff --git a/apple/TableProTests/Core/Database/SchemaDDLTransactionTests.swift b/apple/TableProTests/Core/Database/SchemaDDLTransactionTests.swift index 64c93f89e3..f46a92ed52 100644 --- a/apple/TableProTests/Core/Database/SchemaDDLTransactionTests.swift +++ b/apple/TableProTests/Core/Database/SchemaDDLTransactionTests.swift @@ -145,6 +145,52 @@ struct SchemaDDLTransactionTests { #expect(pooled.executedQueries == ["BEGIN", Self.first, Self.second, "ROLLBACK"]) } + @Test("A Structure save that fails partway where DDL is not transactional names the statements that stayed applied") + func failedSaveNamesTheStatementsThatRan() async throws { + let (connection, scope, pooled) = try await Self.inject(transactions: true, transactionalDDL: false) + defer { Self.tearDown(connection) } + pooled.failingStatement = Self.second + + let message = await Self.failureMessage { try await Self.save(scope) } + + #expect(message.contains("statement refused")) + #expect(message.contains(Self.first)) + #expect(!message.contains(Self.second)) + } + + @Test("A Structure save whose first statement fails names no statement as applied") + func failedFirstStatementNamesNothing() async throws { + let (connection, scope, pooled) = try await Self.inject(transactions: true, transactionalDDL: false) + defer { Self.tearDown(connection) } + pooled.failingStatement = Self.first + + let message = await Self.failureMessage { try await Self.save(scope) } + + #expect(message.contains("statement refused")) + #expect(!message.contains(Self.first)) + } + + @Test("A Structure save rolled back where DDL is transactional names no statement as applied") + func rolledBackSaveNamesNothing() async throws { + let (connection, scope, pooled) = try await Self.inject(transactions: true, transactionalDDL: true) + defer { Self.tearDown(connection) } + pooled.failingStatement = Self.second + + let message = await Self.failureMessage { try await Self.save(scope) } + + #expect(message.contains("statement refused")) + #expect(!message.contains(Self.first)) + } + + private static func failureMessage(_ save: () async throws -> Void) async -> String { + do { + try await save() + return "" + } catch { + return error.localizedDescription + } + } + @Test("Create Table opens no transaction where DDL is not transactional") func createSkipsTheTransactionWithoutTransactionalDDL() async throws { let (connection, scope, pooled) = try await Self.inject(transactions: true, transactionalDDL: false) diff --git a/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift b/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift index a8e7e7852d..1d8ab421c1 100644 --- a/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift +++ b/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift @@ -93,8 +93,8 @@ struct SwitchDatabasePooledConnectionTests { #expect(!MetadataConnectionPool.shared.isReplacingTransport(for: connection.id)) } - @Test("A switch that reconnects reports the transaction the connection it closed was holding") - func reconnectingSwitchReportsTheTransactionItEnded() async throws { + @Test("A switch that would reconnect over an open transaction is refused and the transaction stays") + func reconnectingSwitchKeepsAnOpenTransaction() async throws { let connection = makeSession() defer { cleanUp(connection) } let manager = DatabaseManager.shared @@ -104,6 +104,44 @@ struct SwitchDatabasePooledConnectionTests { try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in } #expect(manager.sessionTransactions.holdsTransaction(connectionId: connection.id, container: "app")) + await #expect(throws: DatabaseError.self) { + try await manager.switchDatabase(to: "reports", for: connection.id, persist: false) + } + + #expect(driver.disconnectCallCount == 0) + #expect(manager.sessionTransactions.holdsTransaction(connectionId: connection.id, container: "app")) + #expect(!manager.sessionLanes.hasTransactionLoss(database: "app", for: connection.id)) + #expect(manager.session(for: connection.id)?.resolvedBrowseDatabase == "app") + #expect(manager.session(for: connection.id)?.status == .connected) + } + + @Test("A switch is refused when the driver reports a transaction the tracker has not recorded") + func reconnectingSwitchAsksTheDriver() async throws { + let connection = makeSession() + defer { cleanUp(connection) } + let manager = DatabaseManager.shared + let driver = try #require(manager.session(for: connection.id)?.driver as? MockDatabaseDriver) + driver.sessionTransactionStateToReturn = .abortedTransaction + + await #expect(throws: DatabaseError.self) { + try await manager.switchDatabase(to: "reports", for: connection.id, persist: false) + } + + #expect(driver.disconnectCallCount == 0) + #expect(manager.session(for: connection.id)?.resolvedBrowseDatabase == "app") + } + + @Test("A switch over a connection that stopped answering goes ahead and reports the transaction it held") + func reconnectingSwitchReportsALostTransaction() async throws { + let connection = makeSession() + defer { cleanUp(connection) } + let manager = DatabaseManager.shared + let driver = try #require(manager.session(for: connection.id)?.driver as? MockDatabaseDriver) + driver.sessionTransactionStateToReturn = .inTransaction + let scope = DatabaseScope(connectionId: connection.id, database: "app", schema: nil) + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in } + manager.updateSession(connection.id) { $0.liveness = .unreachable(nil) } + try? await manager.switchDatabase(to: "unreachable", for: connection.id, persist: false) #expect(driver.disconnectCallCount >= 1) diff --git a/apple/TableProTests/Core/Export/TableTransferServiceTests.swift b/apple/TableProTests/Core/Export/TableTransferServiceTests.swift index cbbe7db329..127e7522a9 100644 --- a/apple/TableProTests/Core/Export/TableTransferServiceTests.swift +++ b/apple/TableProTests/Core/Export/TableTransferServiceTests.swift @@ -14,13 +14,30 @@ private final class TransferStubDriver: PluginDriverStub, @unchecked Sendable { let header: [String] let rows: [PluginRow] let columns: [String] + let types: [String: String] + // What the stream header names a column, where the driver's header and its catalog disagree. + let headerTypes: [String: String] private(set) var executedQueries: [String] = [] private(set) var executedParameters: [[PluginCellValue]] = [] + private let eventLock = NSLock() + private var recordedEvents: [String] = [] + var onFetchColumns: (@Sendable () async -> Void)? - init(header: [String] = [], rows: [PluginRow] = [], columns: [String] = []) { + // `stream` when a row stream opens, `columns` when the catalog is read: a real source serves one at a time. + var events: [String] { eventLock.withLock { recordedEvents } } + + init( + header: [String] = [], + rows: [PluginRow] = [], + columns: [String] = [], + types: [String: String] = [:], + headerTypes: [String: String] = [:] + ) { self.header = header self.rows = rows self.columns = columns + self.types = types + self.headerTypes = headerTypes } var insertStatements: [String] { @@ -37,8 +54,9 @@ private final class TransferStubDriver: PluginDriverStub, @unchecked Sendable { } func streamRows(query: String) -> AsyncThrowingStream { + eventLock.withLock { recordedEvents.append("stream") } let header = PluginStreamHeader( - columns: header, columnTypeNames: header.map { _ in "TEXT" }) + columns: header, columnTypeNames: header.map { headerTypes[$0] ?? types[$0] ?? "TEXT" }) let rows = rows return AsyncThrowingStream { continuation in continuation.yield(.header(header)) @@ -59,7 +77,9 @@ private final class TransferStubDriver: PluginDriverStub, @unchecked Sendable { } func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { - columns.map { PluginColumnInfo(name: $0, dataType: "TEXT") } + eventLock.withLock { recordedEvents.append("columns") } + await onFetchColumns?() + return columns.map { PluginColumnInfo(name: $0, dataType: types[$0] ?? "TEXT") } } } @@ -238,6 +258,128 @@ struct TableTransferServiceTests { #expect(service.state.warnings.contains { $0.contains("Name") }) } + // PostgreSQL reads a bit string as its digits; bound to a MySQL BIT as a number, 00000101 would store 101. A fresh + // session's stream header names the bit column `unknown`, so only the catalog says what it is. + @MainActor @Test("A PostgreSQL bit string transferred to a MySQL BIT arrives as its number") + func postgresBitReachesMySQLAsItsNumber() async throws { + let source = TransferStubDriver( + header: ["id", "flags"], + rows: [[.text("1"), .text("00000101")]], + columns: ["id", "flags"], + types: ["id": "integer", "flags": "bit(8)"], + headerTypes: ["id": "integer", "flags": "unknown"] + ) + let destination = TransferStubDriver(columns: ["id", "flags"], types: ["id": "int", "flags": "bit(8)"]) + let request = TableTransferService.Request( + objects: [ExportObjectItem(name: "devices", kind: .table, isSelected: true)], + sourceType: .postgresql, + destinationType: .mysql, + sourceColumns: ["devices": ["id", "flags"]], + wrapInTransaction: false + ) + + try await TableTransferService().transfer( + request: request, + sourceDriver: adapter(source, type: .postgresql), + destinationDriver: adapter(destination, type: .mysql) + ) + + let insert = try #require(destination.insertStatements.first) + #expect(insert.hasSuffix("VALUES (CAST(? AS UNSIGNED), ?)")) + #expect(destination.executedParameters.last == [.text("5"), .text("1")]) + // Behind its own open stream, a real source answers the catalog only after the whole table, or never. + #expect(source.events == ["columns", "stream"]) + } + + @MainActor @Test("A Stop during the catalog reads leaves the destination's rows alone") + func stopDuringCatalogReadsDeletesNothing() async { + let source = TransferStubDriver( + header: ["id", "flags"], + rows: [[.text("1"), .text("00000101")]], + columns: ["id", "flags"], + types: ["id": "integer", "flags": "bit(8)"] + ) + let destination = TransferStubDriver(columns: ["id", "flags"], types: ["id": "int", "flags": "bit(8)"]) + let service = TableTransferService() + source.onFetchColumns = { await MainActor.run { service.cancel() } } + let request = TableTransferService.Request( + objects: [ExportObjectItem(name: "devices", kind: .table, isSelected: true)], + sourceType: .postgresql, + destinationType: .mysql, + sourceColumns: ["devices": ["id", "flags"]], + deleteExistingRows: true, + wrapInTransaction: false + ) + + _ = try? await service.transfer( + request: request, + sourceDriver: adapter(source, type: .postgresql), + destinationDriver: adapter(destination, type: .mysql) + ) + + #expect(source.events == ["columns"]) + #expect(destination.executedQueries.allSatisfy { query in + !query.uppercased().hasPrefix("DELETE") && !query.uppercased().hasPrefix("TRUNCATE") + }) + #expect(destination.insertStatements.isEmpty) + } + + @MainActor @Test("A transfer whose engines cannot both hold a bit string never reads the source catalog") + func noBitStringsNoSourceCatalogRead() async throws { + let source = TransferStubDriver( + header: ["id", "flags"], + rows: [[.text("1"), .text("5")]], + columns: ["id", "flags"], + types: ["id": "INTEGER", "flags": "BIT"] + ) + let destination = TransferStubDriver(columns: ["id", "flags"], types: ["id": "int", "flags": "bit(8)"]) + let request = TableTransferService.Request( + objects: [ExportObjectItem(name: "devices", kind: .table, isSelected: true)], + sourceType: .sqlite, + destinationType: .mysql, + sourceColumns: ["devices": ["id", "flags"]], + wrapInTransaction: false + ) + + try await TableTransferService().transfer( + request: request, + sourceDriver: adapter(source, type: .sqlite), + destinationDriver: adapter(destination, type: .mysql) + ) + + #expect(source.events == ["stream"]) + #expect(destination.executedParameters.last == [.text("5"), .text("1")]) + } + + // MySQL's header says BIT without the width, so the digits take the catalog's BIT(8). + @MainActor @Test("A MySQL BIT transferred to a PostgreSQL bit varying arrives as its digits, at the source's width") + func mysqlBitReachesPostgresAsItsDigits() async throws { + let source = TransferStubDriver( + header: ["id", "flags"], + rows: [[.text("1"), .text("5")]], + columns: ["id", "flags"], + types: ["id": "int", "flags": "bit(8)"], + headerTypes: ["id": "INT", "flags": "BIT"] + ) + let destination = TransferStubDriver(columns: ["id", "flags"], types: ["id": "integer", "flags": "varbit"]) + let request = TableTransferService.Request( + objects: [ExportObjectItem(name: "devices", kind: .table, isSelected: true)], + sourceType: .mysql, + destinationType: .postgresql, + sourceColumns: ["devices": ["id", "flags"]], + wrapInTransaction: false + ) + + try await TableTransferService().transfer( + request: request, + sourceDriver: adapter(source, type: .mysql), + destinationDriver: adapter(destination, type: .postgresql) + ) + + #expect(destination.executedParameters.last == [.text("00000101"), .text("1")]) + #expect(source.events == ["columns", "stream"]) + } + @MainActor @Test("Twins mapped by the user to their own columns each reach their own column") func mappedTwinsReachTheirOwnColumns() async throws { let source = TransferStubDriver( diff --git a/apple/TableProTests/Core/MCP/MCPTokenStoreRecoveryTests.swift b/apple/TableProTests/Core/MCP/MCPTokenStoreRecoveryTests.swift index a7c3a442b7..fd4f17d505 100644 --- a/apple/TableProTests/Core/MCP/MCPTokenStoreRecoveryTests.swift +++ b/apple/TableProTests/Core/MCP/MCPTokenStoreRecoveryTests.swift @@ -101,6 +101,7 @@ struct MCPTokenStoreRecoveryTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: bridge, + isClaudeAgentCredential: false, extraScopes: [] ).plaintext } @@ -220,6 +221,7 @@ struct MCPTokenStoreRecoveryTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: true, + isClaudeAgentCredential: false, extraScopes: [] ).token diff --git a/apple/TableProTests/Core/MCP/MCPTokenStoreTests.swift b/apple/TableProTests/Core/MCP/MCPTokenStoreTests.swift index afb4cb3e57..7bea3598b0 100644 --- a/apple/TableProTests/Core/MCP/MCPTokenStoreTests.swift +++ b/apple/TableProTests/Core/MCP/MCPTokenStoreTests.swift @@ -145,6 +145,7 @@ struct MCPTokenStoreTests { connectionAccess: .limited(allowed), expiresAt: expiry, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -172,6 +173,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) let second = try await store.generate( @@ -180,6 +182,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -200,6 +203,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -218,6 +222,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -235,6 +240,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: Date.now.addingTimeInterval(-1), isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -251,6 +257,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -270,6 +277,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: Date.now.addingTimeInterval(-1), isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -286,6 +294,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -304,6 +313,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: Date.now.addingTimeInterval(-1), isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) let authenticator = MCPBearerTokenAuthenticator( @@ -333,6 +343,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -349,6 +360,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -366,6 +378,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) let recorder = RevocationRecorder() @@ -390,6 +403,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -414,6 +428,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -445,6 +460,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -464,6 +480,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -482,6 +499,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -491,6 +509,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -510,6 +529,7 @@ struct MCPTokenStoreTests { connectionAccess: .limited([UUID()]), expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -533,6 +553,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: Date.now.addingTimeInterval(3_600), isBridgeCredential: true, + isClaudeAgentCredential: false, extraScopes: [] ) let survivor = try await writer.generate( @@ -541,6 +562,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) @@ -562,6 +584,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [.connectionsDisplay] ) @@ -584,6 +607,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) #expect(credentials.storedText.contains("extraScopes") == false) @@ -626,6 +650,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: true, + isClaudeAgentCredential: false, extraScopes: [.connectionsDisplay] ) #expect(result.token.extraScopes.isEmpty) @@ -640,6 +665,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [] ) let launcher = try await store.generate( @@ -648,6 +674,7 @@ struct MCPTokenStoreTests { connectionAccess: .all, expiresAt: nil, isBridgeCredential: false, + isClaudeAgentCredential: false, extraScopes: [.connectionsDisplay] ) diff --git a/apple/TableProTests/Core/MCP/Transport/MCPTokenRevocationRouteTests.swift b/apple/TableProTests/Core/MCP/Transport/MCPTokenRevocationRouteTests.swift index a3c11c700b..09cc32331f 100644 --- a/apple/TableProTests/Core/MCP/Transport/MCPTokenRevocationRouteTests.swift +++ b/apple/TableProTests/Core/MCP/Transport/MCPTokenRevocationRouteTests.swift @@ -70,6 +70,7 @@ private struct RevocationServer { connectionAccess: .all, expiresAt: nil, isBridgeCredential: bridge, + isClaudeAgentCredential: false, extraScopes: [] ) return (minted.token.id, minted.plaintext) diff --git a/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift b/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift index a9eee6587a..772fdf6ba5 100644 --- a/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift +++ b/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift @@ -59,6 +59,7 @@ private final class CopyDriver: PluginDriverStub, @unchecked Sendable { struct ObjectCopyRowCopierTests { private func step( columns: [String] = ["id", "name"], + columnTypeNames: [String: String] = [:], schema: String? = "public", coercer: CrossEngineValueCoercer? = nil, serverSideInsert: SyncStatement? = nil, @@ -71,6 +72,7 @@ struct ObjectCopyRowCopierTests { createStatements: [], truncateStatements: [], columns: columns, + targetColumnTypeNames: columnTypeNames, primaryKeyColumns: ["id"], sourceQuery: "SELECT `id`, `name` FROM `public`.`orders`", targetTable: "orders", @@ -267,6 +269,140 @@ struct ObjectCopyRowCopierTests { #expect(target.executedParameters.first?.map(\.sortKey) == ["1", "1"]) } + // MARK: - BIT columns + + // Bound as text, MySQL stores the characters' bytes in a BIT column: 5 lands as 53. + @Test + func aMySQLBitColumnIsWrittenAsTheNumber() async throws { + let source = CopyDriver() + source.streamed = [[.text("1"), .text("5")]] + let target = CopyDriver() + + _ = try await ObjectCopyRowCopier( + step: step(columns: ["id", "flags"], columnTypeNames: ["id": "int", "flags": "bit(8)"]), + sourceDatabaseType: .mysql, + targetDatabaseType: .mysql + ).copy(from: source, to: target) { _ in } + + #expect(target.executedQueries.first == "INSERT INTO `public`.`orders` (`id`, `flags`) VALUES (?, CAST(? AS UNSIGNED))") + #expect(target.executedParameters.first?.map(\.sortKey) == ["1", "5"]) + } + + @Test + func aMySQLTableCopyCarriesItsBitTypes() throws { + let draft = try Self.draft( + columns: [ + PluginColumnInfo(name: "id", dataType: "int", isNullable: false, isPrimaryKey: true), + PluginColumnInfo(name: "flags", dataType: "bit(8)") + ], + from: .mysql, + to: .mysql + ) + + #expect(draft.targetColumnTypeNames["flags"] == "bit(8)") + #expect(draft.coercer == nil) + } + + // PostgreSQL reads a bit string as its digits, which MySQL would take as the decimal number 101. + @Test + func aPostgresBitStringCopiedToMySQLArrivesAsItsNumber() async throws { + let draft = try Self.draft( + columns: [ + PluginColumnInfo(name: "id", dataType: "integer", isNullable: false, isPrimaryKey: true), + PluginColumnInfo(name: "flags", dataType: "bit(8)") + ], + from: .postgresql, + to: .mysql + ) + let source = CopyDriver() + source.streamed = [[.text("1"), .text("00000101")]] + let target = CopyDriver() + + _ = try await ObjectCopyRowCopier( + step: step( + columns: draft.targetColumns, + columnTypeNames: draft.targetColumnTypeNames, + coercer: draft.coercer + ), + sourceDatabaseType: .postgresql, + targetDatabaseType: .mysql + ).copy(from: source, to: target) { _ in } + + #expect(target.executedQueries.first?.hasSuffix("VALUES (?, CAST(? AS UNSIGNED))") == true) + #expect(target.executedParameters.first?.map(\.sortKey) == ["1", "5"]) + } + + @Test + func aMySQLBitCopiedToPostgresArrivesAsItsDigits() async throws { + let draft = try Self.draft( + columns: [ + PluginColumnInfo(name: "id", dataType: "int", isNullable: false, isPrimaryKey: true), + PluginColumnInfo(name: "flags", dataType: "bit(8)") + ], + from: .mysql, + to: .postgresql + ) + let source = CopyDriver() + source.streamed = [[.text("1"), .text("5")]] + let target = CopyDriver() + target.declaredParameterStyle = .dollar + + _ = try await ObjectCopyRowCopier( + step: step( + columns: draft.targetColumns, + columnTypeNames: draft.targetColumnTypeNames, + coercer: draft.coercer + ), + sourceDatabaseType: .mysql, + targetDatabaseType: .postgresql + ).copy(from: source, to: target) { _ in } + + #expect(target.executedQueries.first?.contains("CAST") == false) + #expect(target.executedParameters.first?.map(\.sortKey) == ["1", "00000101"]) + } + + private static func draft( + columns: [PluginColumnInfo], + from source: DatabaseType, + to target: DatabaseType + ) throws -> ObjectCopyTableDraft { + let read = TableStructureRead( + table: PluginTableInfo(name: "orders", type: "TABLE", schema: nil, comment: nil), + columns: columns, + indexes: [], + foreignKeys: [], + metadata: nil, + failure: nil + ) + return ObjectCopyTableDraft( + selection: ObjectCopySelection(kind: .table, name: "orders", schema: nil), + read: read, + snapshot: try #require(read.snapshot), + targetSnapshot: nil, + existsInTarget: false, + sourceSchema: nil, + targetSchema: nil, + targetServerVersion: nil, + request: ObjectCopyRequest( + source: endpoint(source), + destination: .existing(endpoint(target)), + objects: [], + content: .structureAndData, + existingPolicy: .skip + ) + ) + } + + private static func endpoint(_ type: DatabaseType) -> DatabaseEndpoint { + DatabaseEndpoint( + scope: DatabaseScope(connectionId: UUID(), database: "shop", schema: nil), + connectionName: type.rawValue, + databaseType: type, + safeModeLevel: .silent, + color: .blue + ) + } + // MARK: - The server-side path /// One statement instead of a stream, and the source driver is never read from at all. diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift new file mode 100644 index 0000000000..6af9834a0a --- /dev/null +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift @@ -0,0 +1,113 @@ +// +// ImportDataSinkAdapterBitColumnTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct WrittenStatement { + let sql: String + let parameters: [PluginCellValue] +} + +private final class BitTableDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var written: [WrittenStatement] = [] + private var fetches = 0 + + var statements: [WrittenStatement] { lock.withLock { written } } + var columnFetches: Int { lock.withLock { fetches } } + + func quoteIdentifier(_ name: String) -> String { "`\(name)`" } + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + lock.withLock { fetches += 1 } + return [ + PluginColumnInfo(name: "id", dataType: "int", isNullable: false, isPrimaryKey: true), + PluginColumnInfo(name: "flags", dataType: "bit(8)"), + PluginColumnInfo(name: "name", dataType: "varchar(20)") + ] + } + + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + lock.withLock { written.append(WrittenStatement(sql: query, parameters: parameters)) } + guard query.hasPrefix("SELECT") else { return .empty } + return PluginQueryResult( + columns: ["position", "count"], columnTypeNames: [], rows: [[.text("0"), .text("1")]], + rowsAffected: 0, executionTime: 0 + ) + } +} + +// MySQL stores text bound to a BIT column as its bytes, so an imported 5 read back as 53. +struct ImportDataSinkAdapterBitColumnTests { + private static let mapping = ["id": "id", "flags": "flags", "name": "name"] + + private func sink( + on driver: BitTableDriver, + type: DatabaseType = .mysql, + rowWriting: ImportRowWriting = .insert + ) -> ImportDataSinkAdapter { + ImportDataSinkAdapter( + driver: PluginDriverAdapter(connection: TestFixtures.makeConnection(type: type), pluginDriver: driver), + databaseType: type, + targetTable: "devices", + columnMapping: Self.mapping, + sourceFields: Set(Self.mapping.keys), + rowWriting: rowWriting + ) + } + + @Test + func anImportedBitValueIsWrittenAsTheNumber() async throws { + let driver = BitTableDriver() + + try await sink(on: driver).insertRows([["id": .text("1"), "flags": .text("5"), "name": .text("a")]]) + + let insert = try #require(driver.statements.first) + let parameters = insert.parameters.map { $0.sortKey } + #expect(insert.sql == "INSERT INTO `devices` (`flags`, `id`, `name`) VALUES (CAST(? AS UNSIGNED), ?, ?)") + #expect(parameters == ["5", "1", "a"]) + } + + @Test + func aSingleRowAndOrderedRowsGetTheSameBinding() async throws { + let driver = BitTableDriver() + let target = sink(on: driver) + + try await target.insertRow(["id": .text("1"), "flags": .text("5")]) + try await target.insertRows(columns: ["id", "flags"], values: [[.text("2"), .text("6")]]) + + let sql = driver.statements.map { $0.sql } + #expect(sql == [ + "INSERT INTO `devices` (`flags`, `id`) VALUES (CAST(? AS UNSIGNED), ?)", + "INSERT INTO `devices` (`id`, `flags`) VALUES (?, CAST(? AS UNSIGNED))" + ]) + #expect(driver.columnFetches == 1) + } + + @Test + func matchingOnABitKeyComparesTheNumber() async throws { + let driver = BitTableDriver() + let target = sink(on: driver, rowWriting: ImportRowWriting(mode: .update, keyColumns: ["flags"])) + + try await target.insertRows([["flags": .text("5"), "name": .text("b")]]) + + let sql = driver.statements.map { $0.sql } + #expect(sql.first == "SELECT 0, COUNT(*) FROM `devices` WHERE `flags` = CAST(? AS UNSIGNED)") + #expect(sql.last == "UPDATE `devices` SET `name` = ? WHERE `flags` = CAST(? AS UNSIGNED)") + } + + @Test + func anEngineThatReadsBitsAsDigitsIsNotAsked() async throws { + let driver = BitTableDriver() + + try await sink(on: driver, type: .postgresql).insertRows([["id": .text("1"), "flags": .text("101")]]) + + #expect(driver.columnFetches == 0) + #expect(driver.statements.first?.sql.contains("CAST") == false) + } +} diff --git a/apple/TableProTests/Core/Services/Masking/MaskingBridgeTests.swift b/apple/TableProTests/Core/Services/Masking/MaskingBridgeTests.swift index deb7d89f5b..330f684b5a 100644 --- a/apple/TableProTests/Core/Services/Masking/MaskingBridgeTests.swift +++ b/apple/TableProTests/Core/Services/Masking/MaskingBridgeTests.swift @@ -21,14 +21,27 @@ struct MaskingBridgeTests { ) } + // What a driver that traces its columns reports for `SELECT id, email FROM customers`. + nonisolated private static func tracedCustomers() -> QueryResult { + traced(customers(), origins: [("customers", "id"), ("customers", "email")]) + } + + nonisolated private static func traced(_ result: QueryResult, origins: [(String, String)?]) -> QueryResult { + var copy = result + copy.columnOrigins = origins.map { origin in + origin.map { PluginColumnOrigin(schema: "shop", table: $0.0, column: $0.1) } + } + return copy + } + private func install(type: DatabaseType = .mysql, sendsBatchesWhole: Bool = true) -> (ScriptAnsweringDriver, DatabaseScope) { var connection = TestFixtures.makeConnection(database: "shop", type: type) connection.maskingRules = [MaskingRule(pattern: "*.email")] let driver = ScriptAnsweringDriver( connection: connection, sendsBatchesWhole: sendsBatchesWhole, - answer: { _ in ScriptAnsweringDriver.batch([Self.customers(), Self.customers()]) }, - statementAnswer: { _ in Self.customers() } + answer: { _ in ScriptAnsweringDriver.batch([Self.tracedCustomers(), Self.tracedCustomers()]) }, + statementAnswer: { _ in Self.tracedCustomers() } ) var session = ConnectionSession(connection: connection) session.driver = driver @@ -99,7 +112,7 @@ struct MaskingBridgeTests { let other = try await bridge.runStatement( scope: scope, query: "SELECT * FROM customers", maxRows: 10, timeoutSeconds: 30, cancellation: nil, - maskingTable: "customers" + masking: .table("customers") ) let unnamed = try await bridge.runStatement( scope: scope, query: "SELECT * FROM customers", maxRows: 10, timeoutSeconds: 30, cancellation: nil @@ -108,4 +121,284 @@ struct MaskingBridgeTests { #expect(Self.leaks(other.result)) #expect(!Self.leaks(unnamed.result)) } + + private func install( + type: DatabaseType = .mysql, + answering answer: @escaping @Sendable (String) -> QueryResult + ) -> (ScriptAnsweringDriver, DatabaseScope) { + var connection = TestFixtures.makeConnection(database: "shop", type: type) + connection.maskingRules = [MaskingRule(pattern: "*.email")] + let driver = ScriptAnsweringDriver(connection: connection, statementAnswer: answer) + var session = ConnectionSession(connection: connection) + session.driver = driver + DatabaseManager.shared.injectSession(session, for: connection.id) + return (driver, DatabaseScope(connectionId: connection.id, database: "shop", schema: nil)) + } + + @Test("A column renamed with an alias is masked by the column it reads") + func aliasIsMaskedByItsOrigin() async throws { + let renamed = Self.traced( + ScriptAnsweringDriver.resultSet(columns: ["id", "e"], rows: [["1", "jane@example.com"]]), + origins: [("customers", "id"), ("customers", "email")] + ) + let (driver, scope) = install { _ in renamed } + defer { DatabaseManager.shared.removeSession(for: driver.connection.id) } + + let outcome = try await DatabaseAccessBridge().runStatement( + scope: scope, query: "SELECT id, email AS e FROM customers", maxRows: 10, timeoutSeconds: 30, cancellation: nil + ) + + #expect(!Self.leaks(outcome.result)) + #expect(outcome.result.rows.map { $0[0] } == [.text("1")]) + } + + @Test("A computed column whose origin nobody reports is masked whole") + func expressionIsMasked() async throws { + let computed = ScriptAnsweringDriver.resultSet(columns: ["lower(email)"], rows: [["jane@example.com"]]) + let (driver, scope) = install { _ in computed } + defer { DatabaseManager.shared.removeSession(for: driver.connection.id) } + + let payload = try await MCPConnectionBridge().executeQuery( + scope: scope, query: "SELECT lower(email) FROM customers", maxRows: 10, timeoutSeconds: 30, cancellation: nil + ) + + let json = payload.jsonString(prettyPrinted: false) + #expect(!json.contains("@example.com")) + #expect(json.contains("lower(email)")) + #expect(payload["status_message"] != nil) + } + + @Test("A browse of one table stays readable where the rules say so") + func browsedTableIsMatchedByName() async throws { + let (driver, scope) = install { _ in Self.customers() } + defer { DatabaseManager.shared.removeSession(for: driver.connection.id) } + + let outcome = try await DatabaseAccessBridge().runStatement( + scope: scope, query: "SELECT id, email FROM customers", maxRows: 10, timeoutSeconds: 30, cancellation: nil, + masking: .table("customers") + ) + + #expect(!Self.leaks(outcome.result)) + #expect(outcome.result.rows.map { $0[0] } == [.text("1"), .text("2")]) + } + + // Measured on MariaDB: `EXPLAIN FORMAT=JSON` wrote `a.email = 'alice@example.com'` read from a const table. + @Test("A MySQL plan comes back masked on a connection with rules, a PostgreSQL plan readable") + func planMaskingFollowsItsFormat() async throws { + let cases: [(DatabaseType, ExplainVariant, Bool)] = [ + (.mysql, ExplainVariant(id: "json", label: "JSON", sqlPrefix: "EXPLAIN FORMAT=JSON", format: .mysqlComposite), false), + (.postgresql, ExplainVariant(id: "json", label: "JSON", sqlPrefix: "EXPLAIN (FORMAT JSON)", format: .postgresJson), true), + ] + for (type, variant, readable) in cases { + let plan = ScriptAnsweringDriver.resultSet(columns: ["EXPLAIN"], rows: [["a.email = 'jane@example.com'"]]) + let (driver, scope) = install(type: type) { _ in plan } + defer { DatabaseManager.shared.removeSession(for: driver.connection.id) } + let request = try MCPConnectionBridge.explainRequest( + for: "SELECT * FROM users u JOIN a ON a.email = u.email WHERE u.id = 1", + databaseType: type, variants: [variant], variantId: nil, analyze: false + ) + let operation = try await ApprovedOperationFixture.approve( + [OperationStatement(sql: request.statement)], scope: scope, databaseType: type + ) + + let payload = try await MCPConnectionBridge().explainQuery( + operation, request: request, timeoutSeconds: 30, cancellation: nil + ) + + #expect(payload.jsonString(prettyPrinted: false).contains("@example.com") == readable, "\(type.rawValue)") + } + } + + @Test("A plan format nobody measured is masked as rows are") + func unmeasuredPlanFormatsAreMaskedAsRows() { + #expect(ExternalRowMasking.plan(in: .postgresJson) == .byName) + #expect(ExternalRowMasking.plan(in: .sqliteQueryPlan) == .byName) + #expect(ExternalRowMasking.plan(in: .mysqlComposite) == .byOrigin) + #expect(ExternalRowMasking.plan(in: .indentedText) == .byOrigin) + #expect(ExternalRowMasking.plan(in: nil) == .byOrigin) + } + + // Measured on PostgreSQL 17: `SELECT id FROM users WHERE email::int = 0` fails quoting the address it read. + @Test("A server error reaches an external caller as its code alone on a connection with rules") + func serverErrorTextIsWithheld() async throws { + for masked in [true, false] { + var connection = TestFixtures.makeConnection(database: "shop", type: .postgresql) + if masked { connection.maskingRules = [MaskingRule(pattern: "*.ssn")] } + var session = ConnectionSession( + connection: connection, + driver: PluginDriverAdapter(connection: connection, pluginDriver: EchoingDriver()) + ) + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let scope = DatabaseScope(connectionId: connection.id, database: "shop", schema: nil) + + var message: String? + do { + _ = try await DatabaseAccessBridge().runStatement( + scope: scope, query: "SELECT id FROM users WHERE email::int = 0", maxRows: 10, timeoutSeconds: 30, + cancellation: nil + ) + } catch let refusal as DatabaseAccessError { + message = refusal.message + } catch let failure as any PluginDriverError { + message = failure.pluginErrorMessage + } + + let text = try #require(message) + #expect(text.contains("jane@example.com") == !masked) + #expect(text.contains("SQLSTATE 22P02") == masked) + } + } + + @Test("The app's own refusals keep their text") + func ownErrorsKeepTheirText() { + #expect(!ServerErrorText.mayRepeatAValue(DatabaseAccessError.timeout("Query timed out after 30 seconds"))) + #expect(!ServerErrorText.mayRepeatAValue(CancellationError())) + #expect(!ServerErrorText.mayRepeatAValue(DatabaseError.notConnected)) + #expect(ServerErrorText.mayRepeatAValue(DatabaseError.queryFailed("Key (email)=(jane@example.com) already exists."))) + #expect(ServerErrorText.mayRepeatAValue(EchoingServerError())) + #expect(!ServerErrorText.mayRepeatAValue(DataWriteError.tooManyRowsAffected(table: "users", expected: 1, actual: 2))) + #expect(ServerErrorText.mayRepeatAValue( + DataWriteError.changeRefused(table: "users", kind: nil, reason: "The row could not be looked up: jane@example.com") + )) + } + + @Test("A refusal before any driver took the statement keeps its words; a driver's error does not") + func refusalBeforeTheDriverKeepsItsText() async throws { + let (driver, scope) = install() + defer { DatabaseManager.shared.removeSession(for: driver.connection.id) } + let refusal = DatabaseError.queryFailed("This tab is on shop. Switch the connection to that database to run it.") + let bridge = DatabaseAccessBridge() + + let unreached = await bridge.externalError(refusal, on: scope.connectionId, reach: DriverReach()) + let reach = DriverReach() + reach.mark() + let reached = await bridge.externalError(refusal, on: scope.connectionId, reach: reach) + + #expect((unreached as? DatabaseError)?.errorDescription == refusal.errorDescription) + #expect(reached is DatabaseAccessError) + } + + @Test("History handed to MCP holds no error text from a connection with rules") + func historyErrorTextIsWithheld() { + let entry = QueryHistoryEntry( + query: "UPDATE users SET name = NULL WHERE id = 1", + connectionId: UUID(), + databaseName: "shop", + databaseType: .postgresql, + source: .rowEdit, + executionTime: 0, + rowCount: 0, + wasSuccessful: false, + errorMessage: "null value in column \"name\" violates not-null constraint\nDETAIL: Failing row contains (1, jane@example.com, null)." + ) + + let withheld = MCPConnectionBridge.encode(historyEntry: entry, masksValues: true) + let kept = MCPConnectionBridge.encode(historyEntry: entry) + + #expect(withheld["error_message"]?.stringValue == ServerErrorText.withheldMessage) + #expect(kept["error_message"]?.stringValue?.contains("jane@example.com") == true) + } + + // Edit Document writes `replaceOne(, )` as the row edit's statement. + @Test("History handed to MCP holds no statement the app wrote from stored values, on a connection with rules") + func historyAppWrittenStatementsAreWithheld() { + func entry(_ source: QueryHistorySource) -> QueryHistoryEntry { + QueryHistoryEntry( + query: #"db.users.replaceOne({"_id": 1, "email": "jane@example.com"}, {"_id": 1, "email": "jane@example.com", "name": "J"})"#, + connectionId: UUID(), + databaseName: "shop", + databaseType: .mongodb, + source: source, + executionTime: 0, + rowCount: 1, + wasSuccessful: true + ) + } + + for source in [QueryHistorySource.rowEdit, .dataImport] { + #expect( + MCPConnectionBridge.encode(historyEntry: entry(source), masksValues: true)["query"]?.stringValue + == ServerErrorText.withheldStatementText + ) + #expect(MCPConnectionBridge.encode(historyEntry: entry(source))["query"]?.stringValue?.contains("jane@") == true) + } + #expect(MCPConnectionBridge.encode(historyEntry: entry(.editor), masksValues: true)["query"]?.stringValue?.contains("jane@") == true) + + // A search that matched the withheld text would answer for it one prefix at a time. + #expect(!MCPConnectionBridge.isSearchable(entry(.rowEdit), by: "jan", masksValues: true)) + #expect(!MCPConnectionBridge.isSearchable(entry(.dataImport), by: "jan", masksValues: true)) + #expect(MCPConnectionBridge.isSearchable(entry(.rowEdit), by: nil, masksValues: true)) + #expect(MCPConnectionBridge.isSearchable(entry(.rowEdit), by: "", masksValues: true)) + #expect(MCPConnectionBridge.isSearchable(entry(.rowEdit), by: "jan", masksValues: false)) + #expect(MCPConnectionBridge.isSearchable(entry(.editor), by: "jan", masksValues: true)) + } + + @Test("A search on a connection with rules leaves withheld sources out of the storage query, so they take no slot") + func searchLeavesWithheldSourcesOutOfTheQuery() { + let all = Set(QueryHistorySource.allCases) + let searched = MCPConnectionBridge.searchableSources(by: "jan", masksValues: true) + #expect(!searched.contains(.rowEdit)) + #expect(!searched.contains(.dataImport)) + #expect(searched == all.subtracting([.rowEdit, .dataImport])) + #expect(MCPConnectionBridge.searchableSources(by: nil, masksValues: true) == all) + #expect(MCPConnectionBridge.searchableSources(by: "", masksValues: true) == all) + #expect(MCPConnectionBridge.searchableSources(by: "jan", masksValues: false) == all) + } + + @Test("The dashboard hands MCP no session or slow statement text from a connection with rules") + func dashboardStatementTextIsWithheld() throws { + let provider = try #require(ServerDashboardQueryProviderFactory.provider(for: .postgresql)) + let session = DashboardSession( + id: "7", user: "app", database: "shop", state: "idle", durationSeconds: 1, duration: "1s", + query: "UPDATE users SET ssn = '123-45-6789' WHERE id = 1" + ) + let slow = DashboardSlowQuery(duration: "2s", query: session.query, user: "app", database: "shop") + + #expect( + MCPConnectionBridge.sessionPayload(session, provider: provider, withholdingQuery: true)["query"]?.stringValue + == ServerErrorText.withheldStatementText + ) + #expect( + MCPConnectionBridge.slowQueryPayload(slow, withholdingQuery: true)["query"]?.stringValue + == ServerErrorText.withheldStatementText + ) + #expect(MCPConnectionBridge.sessionPayload(session, provider: provider)["query"]?.stringValue == session.query) + } + + @Test("A partial commit keeps its account of what was written and loses only the server's words") + func partialCommitKeepsItsAccount() throws { + let partial = DataWritePartialCommitError( + committed: [], totalStatements: 3, engine: "PostgreSQL", disposition: .written, underlying: EchoingServerError() + ) + + let withheld = try #require(ServerErrorText.withholding(partial) as? DataWritePartialCommitError) + + #expect(withheld.totalStatements == 3) + #expect(withheld.disposition == .written) + #expect(!(withheld.errorDescription ?? "").contains("jane@example.com")) + } +} + +private struct EchoingServerError: PluginDriverError { + var pluginErrorMessage: String { #"invalid input syntax for type integer: "jane@example.com""# } + var pluginSqlState: String? { "22P02" } +} + +private final class EchoingDriver: PluginDriverStub, @unchecked Sendable { + var capabilities: PluginCapabilities { [] } + var supportsSchemas: Bool { false } + var supportsTransactions: Bool { false } + var currentSchema: String? { nil } + var serverVersion: String? { nil } + + func executeUserQuery( + query: String, + rowCap: Int?, + parameters: [PluginCellValue]?, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + throw EchoingServerError() + } } diff --git a/apple/TableProTests/Core/Services/Masking/MaskingExternalReaderTests.swift b/apple/TableProTests/Core/Services/Masking/MaskingExternalReaderTests.swift new file mode 100644 index 0000000000..4788cf5e09 --- /dev/null +++ b/apple/TableProTests/Core/Services/Masking/MaskingExternalReaderTests.swift @@ -0,0 +1,154 @@ +// +// MaskingExternalReaderTests.swift +// TableProTests +// +// The assistant's result summary, the row inspector's assistant pane, Fix with AI and AppleScript read a tab's rows +// and errors outside the bridge, so they take the bridge's masking rather than the grid's by-name match. +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +struct MaskingExternalOverlayTests { + private let policy = MaskingPolicy(rules: [MaskingRule(pattern: "*.email", style: .email)]) + private let columns = ["contact", "email", "id"] + private let types: [ColumnType] = [.text(rawType: nil), .text(rawType: nil), .integer(rawType: nil)] + + private func rows(origins: [PluginColumnOrigin?]?) -> TableRows { + TableRows.from( + queryRows: [[.text("jane@example.com"), .text("john@example.com"), .text("1")]], + columns: columns, + columnTypes: types, + masks: policy.columnMasks(table: nil, columns: columns, columnTypes: types), + columnOrigins: origins + ) + } + + private func externallyMasked(_ rows: TableRows) -> TableRows { + let external = policy.externalColumnMasks( + table: nil, columns: rows.columns, typeNames: rows.columnTypes.map(\.rawType), origins: rows.columnOrigins + ) + return rows.externallyMasked(by: rows.externalOverlay(external)) + } + + @Test("A traced alias the grid left readable is masked by the column it reads") + func tracedAliasIsMasked() { + let traced = rows(origins: [ + PluginColumnOrigin(schema: "shop", table: "users", column: "email"), + PluginColumnOrigin(schema: "shop", table: "users", column: "email"), + PluginColumnOrigin(schema: "shop", table: "users", column: "id"), + ]) + + #expect(traced.rows[0].values[0] == .text("jane@example.com")) + #expect(externallyMasked(traced).rows[0].values == [ + .text("j•••@example.com"), .text("j•••@example.com"), .text("1"), + ]) + } + + @Test("Without origins every column is masked whole, over what the grid already hid") + func untracedRowsAreMaskedWhole() { + #expect(externallyMasked(rows(origins: nil)).rows[0].values == [.text("••••"), .text("••••"), .text("••••")]) + } + + @Test("Origins reported for other columns are not used") + func staleOriginsAreDropped() { + var traced = rows(origins: [nil, nil, PluginColumnOrigin(schema: "shop", table: "users", column: "id")]) + #expect(traced.columnOrigins != nil) + traced.columns = ["id", "contact", "email"] + #expect(traced.columnOrigins == nil) + } +} + +extension GlobalStateSuites { + @MainActor + struct MaskingExternalReaderTests { + private let fixture: CoordinatorFixture = { + var connection = TestFixtures.makeConnection(database: "shop") + connection.maskingRules = [MaskingRule(pattern: "*.email", style: .email)] + return CoordinatorFixture(connection: connection) + }() + + private func load( + _ tab: QueryTab, + columns: [String], + tableName: String?, + origins: [PluginColumnOrigin?]? = nil + ) throws -> QueryTab { + let added = fixture.addTab(tab) + fixture.coordinator.applyPhase1Result( + tabId: added.id, + columns: columns, + columnTypes: columns.map { _ in ColumnType.text(rawType: "varchar") }, + rows: [columns.map { $0 == "id" ? PluginCellValue.text("1") : PluginCellValue.text("jane@example.com") }], + executionTime: 0, + rowsAffected: 0, + statusMessage: nil, + tableName: tableName, + isEditable: false, + metadata: nil, + hasSchema: false, + read: TableFreshness.Read(startedAt: .now, includesDefinition: false), + sql: tab.content.query, + connection: fixture.connection, + columnOrigins: origins + ) + return try #require(fixture.tabManager.tabs.first { $0.id == added.id }) + } + + private static func leaks(_ rows: TableRows) -> Bool { + rows.rows.contains { row in row.values.contains { $0.asText == "jane@example.com" } } + } + + @Test("A query tab's alias reaches the assistant and scripts masked, though the grid shows it") + func queryTabAliasIsMasked() throws { + let tab = try load( + QueryTab(title: "Q", query: "SELECT email AS contact, id FROM users", tabType: .query), + columns: ["contact", "id"], + tableName: nil + ) + + #expect(Self.leaks(fixture.coordinator.tabSessionRegistry.tableRows(for: tab.id))) + #expect(!Self.leaks(fixture.coordinator.externallyMaskedRows(forTab: tab.id))) + #expect(fixture.coordinator.externalOverlay(forTab: tab.id).styles == [0: .full, 1: .full]) + } + + @Test("A traced query tab keeps the columns the rules leave alone") + func tracedQueryTabKeepsUnruledColumns() throws { + let tab = try load( + QueryTab(title: "Q", query: "SELECT email AS contact, id FROM users", tabType: .query), + columns: ["contact", "id"], + tableName: nil, + origins: [ + PluginColumnOrigin(schema: "shop", table: "users", column: "email"), + PluginColumnOrigin(schema: "shop", table: "users", column: "id"), + ] + ) + + let rows = fixture.coordinator.externallyMaskedRows(forTab: tab.id) + #expect(rows.rows[0].values == [.text("j•••@example.com"), .text("1")]) + } + + @Test("A table tab is the app's own projection of that table, matched by name as the grid matches it") + func tableTabMatchesByName() throws { + let tab = try load( + QueryTab(title: "users", query: "SELECT * FROM users", tabType: .table, tableName: "users"), + columns: ["id", "email"], + tableName: "users" + ) + + #expect(fixture.coordinator.externalOverlay(forTab: tab.id).isEmpty) + #expect(fixture.coordinator.externallyMaskedRows(forTab: tab.id).rows[0].values == [ + .text("1"), .text("j•••@example.com"), + ]) + } + + @Test("Fix with AI gets the fixed sentence for an error on a connection with rules") + func errorTextIsWithheld() { + let error = #"invalid input syntax for type integer: "jane@example.com""# + #expect(fixture.coordinator.externalErrorText(error) == ServerErrorText.withheldMessage) + #expect(fixture.coordinator.externalErrorText(nil) == nil) + } + } +} diff --git a/apple/TableProTests/Core/Services/Masking/MaskingPolicyTests.swift b/apple/TableProTests/Core/Services/Masking/MaskingPolicyTests.swift index 6ba8ee668b..482b9a163b 100644 --- a/apple/TableProTests/Core/Services/Masking/MaskingPolicyTests.swift +++ b/apple/TableProTests/Core/Services/Masking/MaskingPolicyTests.swift @@ -140,4 +140,178 @@ struct MaskingPolicyTests { #expect(masked.isTruncated) #expect(masked.columns == result.columns) } + + private static func origin(_ table: String, _ column: String) -> PluginColumnOrigin { + PluginColumnOrigin(schema: "public", table: table, column: column) + } + + @Test("For an external caller an aliased column is masked by the table column it reads") + func externalAliasFollowsItsOrigin() { + let masks = emailRule.externalColumnMasks( + table: nil, columns: ["e", "id"], typeNames: [nil, nil], + origins: [Self.origin("users", "email"), Self.origin("users", "id")] + ) + #expect(masks.styles == [0: .email]) + } + + @Test("For an external caller a column nothing traced is masked whole") + func externalUntracedColumnIsMasked() { + let traced = emailRule.externalColumnMasks( + table: nil, columns: ["lower", "id"], typeNames: [nil, nil], origins: [nil, Self.origin("users", "id")] + ) + #expect(traced.styles == [0: .full]) + + let unreported = emailRule.externalColumnMasks( + table: nil, columns: ["email", "id"], typeNames: [nil, nil], origins: nil + ) + #expect(unreported.styles == [0: .full, 1: .full]) + #expect(emailRule.untracedColumns(table: nil, columns: ["email", "id"], origins: nil) == [0, 1]) + } + + // `SELECT 'x@' || ssn AS email` would otherwise come back as `x•••@123-45-6789` under an e-mail style rule. + @Test("An untraced column never takes the style of a rule its chosen name matches") + func externalUntracedNameDoesNotPickTheStyle() { + let policy = MaskingPolicy(rules: [ + MaskingRule(pattern: "*.email", style: .email), + MaskingRule(pattern: "*.card", style: .lastFour), + MaskingRule(pattern: "*.token", style: .hash), + ]) + let masks = policy.externalColumnMasks( + table: nil, columns: ["email", "card", "token"], typeNames: [nil, nil, nil], origins: [nil, nil, nil] + ) + #expect(masks.styles == [0: .full, 1: .full, 2: .full]) + } + + // Measured: PostgreSQL names the view, partition or child table read, MariaDB the view. + @Test("For an external caller a table.column rule matches a traced column whatever relation the engine names") + func externalTableRuleMatchesAnyTracedRelation() { + let policy = MaskingPolicy(rules: [MaskingRule(pattern: "users.ssn")]) + let masks = policy.externalColumnMasks( + table: nil, columns: ["x", "ssn", "id"], typeNames: [nil, nil, nil], + origins: [Self.origin("users", "ssn"), Self.origin("users_2024", "ssn"), Self.origin("users", "id")] + ) + #expect(masks.styles == [0: .full, 1: .full]) + } + + @Test("A traced column takes the style of the rule naming its relation, else the first rule for its column") + func externalTracedStylePrefersTheNamedRelation() { + let policy = MaskingPolicy(rules: [ + MaskingRule(pattern: "audit.email", style: .hash), + MaskingRule(pattern: "users.email", style: .email), + ]) + let masks = policy.externalColumnMasks( + table: nil, columns: ["email", "email"], typeNames: [nil, nil], + origins: [Self.origin("users", "email"), Self.origin("active_users", "email")] + ) + #expect(masks.styles == [0: .email, 1: .hash]) + } + + // `SELECT email AS contact FROM users` came back as `a•••@example.com` before. + @Test("A traced column takes the stricter of its origin's rule and the rule its alias matches") + func externalAliasRuleCanOnlyTightenTheStyle() { + let policy = MaskingPolicy(rules: [ + MaskingRule(pattern: "users.email", style: .email), + MaskingRule(pattern: "*.contact", style: .full), + MaskingRule(pattern: "*.handle", style: .lastFour), + MaskingRule(pattern: "*.gone", style: .null), + ]) + let masks = policy.externalColumnMasks( + table: nil, columns: ["contact", "handle", "gone", "e", "EMAIL"], typeNames: [nil, nil, nil, nil, nil], + origins: Array(repeating: Self.origin("users", "email"), count: 5) + ) + #expect(masks.styles == [0: .full, 1: .full, 2: .null, 3: .email, 4: .email]) + } + + @Test("The stricter of two styles hides what either one hides") + func strictestStyle() { + #expect(MaskingStyle.strictest(.email, nil) == .email) + #expect(MaskingStyle.strictest(nil, .hash) == .hash) + #expect(MaskingStyle.strictest(.lastFour, .lastFour) == .lastFour) + #expect(MaskingStyle.strictest(.email, .lastFour) == .full) + #expect(MaskingStyle.strictest(.hash, .email) == .full) + #expect(MaskingStyle.strictest(.full, .null) == .null) + #expect(MaskingStyle.strictest(nil, nil) == nil) + } + + @Test("A traced column the rules miss is still masked when the name it was given matches one") + func externalTracedColumnKeepsTheNameMatch() { + let masks = emailRule.externalColumnMasks( + table: nil, columns: ["email"], typeNames: [nil], origins: [Self.origin("users", "name")] + ) + #expect(masks.styles == [0: .email]) + } + + @Test("A projection the app built from one table's own columns is matched by name") + func externalAppProjectionMatchesByName() { + let masks = emailRule.externalColumnMasks( + table: "customers", columns: ["id", "email"], typeNames: [nil, nil], origins: nil + ) + #expect(masks.styles == [1: .email]) + #expect(emailRule.untracedColumns(table: "customers", columns: ["id", "email"], origins: nil).isEmpty) + } + + @Test("A column the driver withholds an origin for stays masked even in the app's own projection") + func externalWithheldOriginBeatsTheTableName() { + let masks = emailRule.externalColumnMasks( + table: "pg_stats", columns: ["attname", "histogram_bounds"], typeNames: [nil, nil], + origins: [Self.origin("pg_stats", "attname"), nil] + ) + #expect(masks.styles == [1: .full]) + + // A driver that distrusts the whole statement reports every column untraced, which proves nothing by name. + let distrusted = emailRule.externalColumnMasks( + table: "contacts_content", columns: ["c0", "c1"], typeNames: [nil, nil], origins: [nil, nil] + ) + #expect(distrusted.styles == [0: .full, 1: .full]) + } + + private static func scope(_ database: String, _ schema: String? = nil) -> DatabaseScope { + DatabaseScope(connectionId: UUID(), database: database, schema: schema) + } + + // Measured on ClickHouse 26.8: `system.query_log.exception` keeps the server message the bridge withheld. + @MainActor + @Test("The app's projection of a system table proves nothing by name; a user table's still does") + func appProjectionOfASystemTableIsUntraced() { + #expect(ExternalRowMasking.table("query_log").resolved(in: Self.scope("system"), databaseType: .clickhouse) == .byOrigin) + #expect( + ExternalRowMasking.table("system.query_log").resolved(in: Self.scope("default"), databaseType: .clickhouse) + == .byOrigin + ) + #expect(ExternalRowMasking.table("events").resolved(in: Self.scope("default"), databaseType: .clickhouse) == .table("events")) + #expect( + ExternalRowMasking.table("pg_stat_activity").resolved(in: Self.scope("shop", "pg_catalog"), databaseType: .postgresql) + == .byOrigin + ) + #expect(ExternalRowMasking.table("users").resolved(in: Self.scope("shop", "public"), databaseType: .postgresql) == .table("users")) + #expect(ExternalRowMasking.byName.resolved(in: Self.scope("system"), databaseType: .clickhouse) == .byName) + } + + @Test("A connection with no rules masks nothing for an external caller") + func externalWithoutRules() { + let masks = MaskingPolicy.none.externalColumnMasks( + table: nil, columns: ["lower"], typeNames: [nil], origins: nil + ) + #expect(masks.isEmpty) + } + + @Test("An external result says which columns were masked for want of an origin") + func externalResultNamesUntracedColumns() throws { + var result = QueryResult( + columns: ["id", "lower(email)"], + columnTypes: [.integer(rawType: "int"), .text(rawType: "text")], + rows: [[.text("1"), .text("jane@example.com")]], + rowsAffected: 0, + executionTime: 0, + error: nil + ) + result.columnOrigins = [Self.origin("users", "id"), nil] + + let masked = result.maskedForExternalCaller(by: emailRule) + + #expect(masked.rows == [[.text("1"), .text("••••")]]) + let note = try #require(masked.statusMessage) + #expect(note.contains("lower(email)")) + #expect(!note.contains("id,")) + } } diff --git a/apple/TableProTests/Core/Services/Query/UnloadedValuePlanTests.swift b/apple/TableProTests/Core/Services/Query/UnloadedValuePlanTests.swift index cd623b69c1..5e3a769548 100644 --- a/apple/TableProTests/Core/Services/Query/UnloadedValuePlanTests.swift +++ b/apple/TableProTests/Core/Services/Query/UnloadedValuePlanTests.swift @@ -162,6 +162,25 @@ struct UnloadedValuePlanTests { #expect(tableRows.unloadedByteCount(row: 0, column: 1) == 70_000) #expect(tableRows.hasUnloadedValues) } + + @Test("A folded page keeps the origins of the columns it keeps") + func foldedPageKeepsColumnOrigins() { + let id = PluginColumnOrigin(schema: "main", table: "files", column: "id") + let data = PluginColumnOrigin(schema: "main", table: "files", column: "data") + var fetched = QueryFetchResult( + columns: ["id", "data", "__tablepro_byte_length_1"], + columnTypes: [.integer(rawType: "INT"), .blob(rawType: "LONGBLOB"), .integer(rawType: "BIGINT")], + rows: [["1", .null, "70000"]], + executionTime: 0, + rowsAffected: 0, + statusMessage: nil, + isTruncated: false, + resultColumnMeta: nil + ) + fetched.columnOrigins = [id, data, nil] + let folded = fetched.foldingUnloadedValues(lengthColumns: ["__tablepro_byte_length_1"]) + #expect(folded.columnOrigins == [id, data]) + } } struct TableRowsUnloadedValueTests { diff --git a/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift b/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift index c4999b4236..9c1a9b1dd4 100644 --- a/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift +++ b/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift @@ -76,6 +76,68 @@ struct RowPasteForeignTextTests { #expect(result.pastedRows.first?.values == [.text("__DEFAULT__"), .text("Ann"), .text("ann@x.com")]) } + + @Test("A blank pasted cell takes the column's default, an empty string where NULL is refused, else NULL") + func blankCellsFollowTheColumn() { + let columns = ["id", "name", "status", "note", "born", "nickname"] + let changeManager = DataChangeManager() + changeManager.configureForTable( + tableName: "people", + columns: columns, + primaryKeyColumns: ["id"], + databaseType: .mysql, + generatedColumns: [] + ) + let manager = RowOperationsManager(changeManager: changeManager) + let clipboard = RecordingClipboard() + clipboard.text = "\tAnn\t\t\t\t\n\tBob" + var rows = TableRows.from( + queryRows: [], + columns: columns, + columnTypes: [ + .integer(rawType: "INT"), .text(rawType: "VARCHAR"), .text(rawType: "VARCHAR"), + .text(rawType: "TEXT"), .date(rawType: "DATE"), .text(rawType: "VARCHAR"), + ], + columnDefaults: ["status": "'active'"], + columnNullable: ["name": false, "status": false, "note": false, "born": false, "nickname": true], + hasAuthoritativeSchema: true + ) + + let result = manager.pasteRowsFromClipboard( + columns: columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard + ) + + let unfilled: [PluginCellValue] = [.text("__DEFAULT__"), .text(""), .null, .null] + #expect(result.pastedRows.map { $0.values } == [ + [.text("__DEFAULT__"), .text("Ann")] + unfilled, + [.text("__DEFAULT__"), .text("Bob")] + unfilled, + ]) + } + + @Test("A pasted NULL stays NULL in a column with a default") + func nullWordIsNotABlankCell() { + let columns = ["id", "status"] + let changeManager = DataChangeManager() + changeManager.configureForTable( + tableName: "people", columns: columns, primaryKeyColumns: ["id"], databaseType: .mysql, generatedColumns: [] + ) + let manager = RowOperationsManager(changeManager: changeManager) + let clipboard = RecordingClipboard() + clipboard.text = "1\tNULL" + var rows = TableRows.from( + queryRows: [], + columns: columns, + columnTypes: [.integer(rawType: "INT"), .text(rawType: "VARCHAR")], + columnDefaults: ["status": "'active'"], + hasAuthoritativeSchema: true + ) + + let result = manager.pasteRowsFromClipboard( + columns: columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard + ) + + #expect(result.pastedRows.first?.values == [.text("__DEFAULT__"), .null]) + } } @MainActor diff --git a/apple/TableProTests/Core/Storage/ConnectionSecretScopeTests.swift b/apple/TableProTests/Core/Storage/ConnectionSecretScopeTests.swift index cf0bb3c235..9c7ebc52b3 100644 --- a/apple/TableProTests/Core/Storage/ConnectionSecretScopeTests.swift +++ b/apple/TableProTests/Core/Storage/ConnectionSecretScopeTests.swift @@ -323,6 +323,41 @@ struct ConnectionSecretScopeTests { #expect(!succeeded.writeFailed) } + @Test("A save leaves a secret the form loaded and did not change as it is, even after another device changed it") + func unchangedSecretIsNotWrittenAgain() { + let connection = makeConnection(localOnly: false) + #expect(storage.addConnection(connection, password: "P1")) + let reads = storage.recordingSecretReads { + #expect(storage.loadPassword(for: connection.id) == "P1") + } + #expect(reads.loaded == [passwordKey(connection.id): "P1"]) + + keychain.seedSynced("P2", forKey: passwordKey(connection.id)) + keychain.failWrites(to: passwordKey(connection.id)) + let staged = storage.stagingSecretEdits(for: connection.id, deviceOnly: false, unchanged: reads.loaded) { + storage.savePassword("P1", for: connection.id) + } + + #expect(!staged.writeFailed) + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "P2") + } + + @Test("A save still writes a secret the form changed") + func changedSecretIsWritten() { + let connection = makeConnection(localOnly: false) + #expect(storage.addConnection(connection, password: "P1")) + let reads = storage.recordingSecretReads { + _ = storage.loadPassword(for: connection.id) + } + + let staged = storage.stagingSecretEdits(for: connection.id, deviceOnly: false, unchanged: reads.loaded) { + storage.savePassword("P3", for: connection.id) + } + + #expect(!staged.writeFailed) + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "P3") + } + @Test("Staged writes take the scope the unsaved record will have") func stagedWritesTakeTheGivenScope() { let id = UUID() diff --git a/apple/TableProTests/Core/Storage/KeychainItemScopeTests.swift b/apple/TableProTests/Core/Storage/KeychainItemScopeTests.swift index 71d6cf1c4b..e364b79c35 100644 --- a/apple/TableProTests/Core/Storage/KeychainItemScopeTests.swift +++ b/apple/TableProTests/Core/Storage/KeychainItemScopeTests.swift @@ -3,26 +3,39 @@ import Security @testable import TablePro import Testing -// Models the one rule the helper depends on: synchronizable is part of an item's identity. +// Models what the helper depends on: synchronizable is part of an item's identity, and each write dates its item. private final class ScopedItemStore: KeychainItemStore, @unchecked Sendable { struct ItemID: Hashable { let account: String let synchronizable: Bool } + private struct Item { + let data: Data + let modified: Date + } + private let lock = NSLock() - private var items: [ItemID: Data] = [:] + private var items: [ItemID: Item] = [:] + private var writes = 0 + + private func nextDate() -> Date { + writes += 1 + return Date(timeIntervalSinceReferenceDate: TimeInterval(writes)) + } func seed(_ value: String, account: String, synchronizable: Bool) { lock.lock() defer { lock.unlock() } - items[ItemID(account: account, synchronizable: synchronizable)] = Data(value.utf8) + items[ItemID(account: account, synchronizable: synchronizable)] = Item(data: Data(value.utf8), modified: nextDate()) } func value(account: String, synchronizable: Bool) -> String? { lock.lock() defer { lock.unlock() } - return items[ItemID(account: account, synchronizable: synchronizable)].map { String(decoding: $0, as: UTF8.self) } + return items[ItemID(account: account, synchronizable: synchronizable)].map { + String(decoding: $0.data, as: UTF8.self) + } } private func matches(_ query: [String: Any]) -> [ItemID] { @@ -46,7 +59,7 @@ private final class ScopedItemStore: KeychainItemStore, @unchecked Sendable { synchronizable: attributes[kSecAttrSynchronizable as String] as? Bool ?? false ) guard items[id] == nil else { return errSecDuplicateItem } - items[id] = attributes[kSecValueData as String] as? Data + items[id] = Item(data: attributes[kSecValueData as String] as? Data ?? Data(), modified: nextDate()) return errSecSuccess } @@ -60,16 +73,18 @@ private final class ScopedItemStore: KeychainItemStore, @unchecked Sendable { return errSecDuplicateItem } for id in found { - items[id] = attributes[kSecValueData as String] as? Data + items[id] = Item(data: attributes[kSecValueData as String] as? Data ?? Data(), modified: nextDate()) } return errSecSuccess } - func copyData(_ query: [String: Any]) -> (status: OSStatus, data: Data?) { + func copyItem(_ query: [String: Any]) -> KeychainCopiedItem { lock.lock() defer { lock.unlock() } - guard let id = matches(query).first else { return (errSecItemNotFound, nil) } - return (errSecSuccess, items[id]) + guard let id = matches(query).first, let item = items[id] else { + return KeychainCopiedItem(status: errSecItemNotFound, data: nil, modified: nil) + } + return KeychainCopiedItem(status: errSecSuccess, data: item.data, modified: item.modified) } func delete(_ query: [String: Any]) -> OSStatus { @@ -179,15 +194,48 @@ struct KeychainItemScopeTests { #expect(store.value(account: key, synchronizable: true) == nil) } - @Test("A read returns the item the setting selects when both exist") - func readPrefersTheSelectedItem() { - store.seed("local", account: key, synchronizable: false) + @Test("With password sync on the newer item wins; with it off this Mac's own item does") + func readReturnsTheNewerItem() { store.seed("synced", account: key, synchronizable: true) + store.seed("local", account: key, synchronizable: false) + + #expect(helper(passwordSync: true).readString(forKey: key) == "local") + #expect(helper(passwordSync: false).readString(forKey: key) == "local") - #expect(helper(passwordSync: true).readString(forKey: key) == "synced") + store.seed("newer-synced", account: key, synchronizable: true) + + #expect(helper(passwordSync: true).readString(forKey: key) == "newer-synced") #expect(helper(passwordSync: false).readString(forKey: key) == "local") } + @Test("With password sync off, a newer item another device synced never replaces this Mac's own key") + func syncOffKeepsTheLocalItemOverANewerSyncedOne() { + store.seed("this-mac-key", account: key, synchronizable: false) + store.seed("other-mac-key", account: key, synchronizable: true) + + #expect(helper(passwordSync: false).readString(forKey: key) == "this-mac-key") + #expect(helper(passwordSync: false).readStringResult(forKey: key) == .found("this-mac-key")) + } + + @Test("A password saved with password sync off is the one read once sync is back on") + func localSaveSurvivesTurningSyncBackOn() { + store.seed("from-another-device", account: key, synchronizable: true) + + #expect(helper(passwordSync: false).writeString("new", forKey: key)) + + #expect(helper(passwordSync: true).readString(forKey: key) == "new") + #expect(store.value(account: key, synchronizable: true) == "from-another-device") + } + + @Test("A credential bound to this Mac reads its own item over a newer synced one") + func boundCredentialIgnoresANewerSyncedItem() { + store.seed("local", account: key, synchronizable: false) + store.seed("synced", account: key, synchronizable: true) + + #expect(helper(passwordSync: true, followsPasswordSync: false).readString(forKey: key) == "local") + #expect(helper(passwordSync: true).readStringResult(forKey: key, deviceOnly: true) == .found("local")) + } + @Test("Delete removes both items") func deleteRemovesBothItems() { store.seed("local", account: key, synchronizable: false) diff --git a/apple/TableProTests/Core/Sync/SyncFullFetchDeletionTests.swift b/apple/TableProTests/Core/Sync/SyncFullFetchDeletionTests.swift new file mode 100644 index 0000000000..c7a6b21ec5 --- /dev/null +++ b/apple/TableProTests/Core/Sync/SyncFullFetchDeletionTests.swift @@ -0,0 +1,300 @@ +import CloudKit +import Foundation +@testable import TablePro +import TableProSyncTransport +import Testing + +@MainActor +struct SyncFullFetchDeletionTests { + private static let zoneID = SyncTestEnvironment.zoneID + private static let createdAt = Date(timeIntervalSinceReferenceDate: 800_000_000) + + private let environment: SyncTestEnvironment + private let dates = CreationDates() + + init() throws { + environment = try SyncTestEnvironment(label: "sync-full-fetch-deletion") + } + + @Test("A saved query deleted elsewhere is removed when an expired token forces a full fetch") + func expiredTokenFullFetchAppliesTheDeletion() async throws { + let kept = SQLFavorite(name: "Churn", query: "SELECT 2") + let deleted = SQLFavorite(name: "Revenue", query: "SELECT 1") + let keptRecord = Self.record(kept) + let deletedRecord = Self.record(deleted) + dates.cached[keptRecord.recordID.recordName] = Self.createdAt + dates.cached[deletedRecord.recordID.recordName] = Self.createdAt.addingTimeInterval(60) + let settings = Self.syncSettings() + AppSettingsStorage(userDefaults: environment.defaults).saveSync(settings) + _ = environment.metadata.adoptReadableRecordTypes(SyncCoordinator.readableContent(for: settings)) + let server = ExpiringZoneServer(zoneID: Self.zoneID, records: [keptRecord, deletedRecord], token: try Self.makeChangeToken()) + let coordinator = makeCoordinator(transport: server) + + #expect(await coordinator.runSyncCycle() == nil) + #expect(await environment.favorites.fetchFavorite(id: deleted.id) != nil) + + await server.serve([keptRecord]) + await server.expireToken() + #expect(await coordinator.runSyncCycle() == nil) + + #expect(await server.pulls == [.full, .expired, .full]) + #expect(await environment.favorites.fetchFavorite(id: deleted.id) == nil) + #expect(await environment.favorites.fetchFavorite(id: kept.id)?.name == "Churn") + } + + @Test("A saved query this run pushed is kept when the full fetch after the push does not list it yet") + func recordPushedThisRunSurvivesTheFullFetch() async throws { + let kept = SQLFavorite(name: "Churn", query: "SELECT 2") + let pushed = SQLFavorite(name: "Revenue", query: "SELECT 1") + let keptRecord = Self.record(kept) + let pushedName = Self.record(pushed).recordID.recordName + dates.cached[keptRecord.recordID.recordName] = Self.createdAt + dates.cached[pushedName] = Self.createdAt.addingTimeInterval(120) + let settings = Self.syncSettings() + AppSettingsStorage(userDefaults: environment.defaults).saveSync(settings) + _ = environment.metadata.adoptReadableRecordTypes(SyncCoordinator.readableContent(for: settings)) + let server = ExpiringZoneServer(zoneID: Self.zoneID, records: [keptRecord], token: try Self.makeChangeToken()) + let coordinator = makeCoordinator(transport: server) + #expect(await coordinator.runSyncCycle() == nil) + + #expect(await environment.favorites.addFavorite(pushed)) + await server.expireToken() + #expect(await coordinator.runSyncCycle() == nil) + + let pushedNames = await server.pushedRecordNames + #expect(pushedNames.contains(pushedName)) + #expect(await server.pulls == [.full, .expired, .full]) + #expect(!environment.tracker.dirtyRecords(for: .favorite).contains(pushed.id.uuidString)) + #expect(await environment.favorites.fetchFavorite(id: pushed.id) != nil) + #expect(environment.recordCache.holdsRecord(named: pushedName)) + } + + @Test("A zone made again holds only newer records, so a record it lacks is not taken as deleted") + func recreatedZoneDeletesNothing() async throws { + let kept = SQLFavorite(name: "Churn", query: "SELECT 2") + let notYetUploaded = SQLFavorite(name: "Revenue", query: "SELECT 1") + let coordinator = try await seededCoordinator(with: [kept, notYetUploaded]) + let uploadedAgain = Self.record(kept) + dates.served[ObjectIdentifier(uploadedAgain)] = Self.createdAt.addingTimeInterval(3_600) + + #expect(await coordinator.applyPullResult(Self.pull([uploadedAgain]), settings: Self.syncSettings(), isFullFetch: true)) + + #expect(await environment.favorites.fetchFavorite(id: notYetUploaded.id) != nil) + #expect(environment.recordCache.holdsRecord(named: Self.record(notYetUploaded).recordID.recordName)) + } + + @Test("A saved query edited here and missing from a full fetch is kept for its push") + func editedRecordSurvivesTheFullFetch() async throws { + let kept = SQLFavorite(name: "Churn", query: "SELECT 2") + let edited = SQLFavorite(name: "Revenue", query: "SELECT 1") + let coordinator = try await seededCoordinator(with: [kept, edited]) + environment.tracker.markDirty(.favorite, id: edited.id.uuidString) + + #expect(await coordinator.applyPullResult(Self.pull([Self.record(kept)]), settings: Self.syncSettings(), isFullFetch: true)) + + #expect(await environment.favorites.fetchFavorite(id: edited.id) != nil) + } + + @Test("A full fetch that could not read a record deletes nothing") + func unreadRecordStopsTheDeletions() async throws { + let kept = SQLFavorite(name: "Churn", query: "SELECT 2") + let unread = SQLFavorite(name: "Revenue", query: "SELECT 1") + let coordinator = try await seededCoordinator(with: [kept, unread]) + let result = PullResult( + changedRecords: [Self.record(kept)], + deletedRecordIDs: [], + newToken: nil, + unreadRecordIDs: [Self.record(unread).recordID] + ) + + #expect(await coordinator.applyPullResult(result, settings: Self.syncSettings(), isFullFetch: true)) + + #expect(await environment.favorites.fetchFavorite(id: unread.id) != nil) + } + + @Test("A delta is never read for what it does not list") + func deltaDeletesNothing() async throws { + let kept = SQLFavorite(name: "Churn", query: "SELECT 2") + let other = SQLFavorite(name: "Revenue", query: "SELECT 1") + let coordinator = try await seededCoordinator(with: [kept, other]) + + #expect(await coordinator.applyPullResult(Self.pull([Self.record(kept)]), settings: Self.syncSettings())) + + #expect(await environment.favorites.fetchFavorite(id: other.id) != nil) + } + + @Test("Only a cached copy created no earlier than the oldest fetched record is proven deleted") + func proofNeedsACreationInsideTheFetchedZone() { + let oldest = Self.createdAt + let before = SyncFullFetchDeletions.Candidate(recordID: Self.recordID("before"), cachedCreation: oldest.addingTimeInterval(-1)) + let after = SyncFullFetchDeletions.Candidate(recordID: Self.recordID("after"), cachedCreation: oldest.addingTimeInterval(1)) + let undated = SyncFullFetchDeletions.Candidate(recordID: Self.recordID("undated"), cachedCreation: nil) + let listed = SyncFullFetchDeletions.Candidate(recordID: Self.recordID("listed"), cachedCreation: oldest.addingTimeInterval(1)) + + let proven = SyncFullFetchDeletions.proven( + candidates: [before, after, undated, listed], + fetchedNames: [Self.recordID("listed").recordName], + fetchedCreations: [oldest.addingTimeInterval(5), oldest] + ) + + #expect(proven == [after.recordID]) + } + + @Test("A fetch with no dated record proves nothing") + func emptyFetchProvesNothing() { + let candidate = SyncFullFetchDeletions.Candidate(recordID: Self.recordID("gone"), cachedCreation: Self.createdAt) + + #expect(SyncFullFetchDeletions.proven(candidates: [candidate], fetchedNames: [], fetchedCreations: []).isEmpty) + } + + private func seededCoordinator(with favorites: [SQLFavorite]) async throws -> SyncCoordinator { + let coordinator = makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + let records = favorites.map(Self.record) + for (offset, record) in records.enumerated() { + dates.cached[record.recordID.recordName] = Self.createdAt.addingTimeInterval(Double(offset * 60)) + } + #expect(await coordinator.applyPullResult(Self.pull(records), settings: Self.syncSettings())) + for favorite in favorites { + #expect(await environment.favorites.fetchFavorite(id: favorite.id) != nil) + } + return coordinator + } + + private func makeCoordinator(transport: any SyncTransport) -> SyncCoordinator { + let base = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: Self.zoneID)) + let dates = dates + return SyncCoordinator( + services: base.services, + recordCache: environment.recordCache, + transport: transport, + makeNetworkMonitor: { nil }, + columnLayouts: environment.columnLayouts, + licenseGate: { nil }, + recordCreationDate: { dates.date(for: $0) } + ) + } + + private static func record(_ favorite: SQLFavorite) -> CKRecord { + SyncRecordMapper.toCKRecord(sqlFavorite: favorite, in: zoneID) + } + + private static func recordID(_ name: String) -> CKRecord.ID { + CKRecord.ID(recordName: name, zoneID: zoneID) + } + + private static func pull(_ records: [CKRecord]) -> PullResult { + PullResult(changedRecords: records, deletedRecordIDs: [], newToken: nil) + } + + private static func syncSettings() -> SyncSettings { + var settings = SyncSettings.default + settings.enabled = true + return settings + } + + private static func makeChangeToken() throws -> CKServerChangeToken { + let archiver = NSKeyedArchiver(requiringSecureCoding: true) + archiver.setClassName("CKServerChangeToken", for: FullFetchChangeTokenArchive.self) + archiver.encode(FullFetchChangeTokenArchive(), forKey: NSKeyedArchiveRootObjectKey) + archiver.finishEncoding() + let token = try NSKeyedUnarchiver.unarchivedObject(ofClass: CKServerChangeToken.self, from: archiver.encodedData) + return try #require(token) + } +} + +// The creation time CloudKit stamps on a saved record, which a record built in a test does not carry. +@MainActor +private final class CreationDates { + var cached: [String: Date] = [:] + var served: [ObjectIdentifier: Date] = [:] + + func date(for record: CKRecord) -> Date? { + served[ObjectIdentifier(record)] ?? cached[record.recordID.recordName] + } +} + +// Answers the stored token with an expiry once told to, as CloudKit does for a token it no longer keeps. +private actor ExpiringZoneServer: SyncTransport { + enum Pull: Equatable { + case full + case delta + case expired + } + + let currentZoneID: CKRecordZone.ID + private var records: [CKRecord] + private let token: CKServerChangeToken + private var tokenExpired = false + private(set) var pulls: [Pull] = [] + private(set) var pushedRecordNames: [String] = [] + + init(zoneID: CKRecordZone.ID, records: [CKRecord], token: CKServerChangeToken) { + currentZoneID = zoneID + self.records = records + self.token = token + } + + func serve(_ records: [CKRecord]) { + self.records = records + } + + func expireToken() { + tokenExpired = true + } + + func accountStatus() async throws -> CKAccountStatus { + .available + } + + func currentAccountId() async throws -> String { + ScriptedSyncTransport.accountId + } + + func ensureZoneExists() async throws {} + + // Saves without listing the record in the next fetch, as a fetch that lags the save would. + func push(records: [CKRecord], deletions: [CKRecord.ID]) async throws -> PushOutcome { + var outcome = PushOutcome() + for record in records { + pushedRecordNames.append(record.recordID.recordName) + outcome.recordSave(record) + } + for recordID in deletions { + outcome.recordDeletion(recordID) + } + return outcome + } + + func fetch(recordIDs: [CKRecord.ID]) async throws -> [CKRecord.ID: CKRecord] { + [:] + } + + func pull(since token: CKServerChangeToken?) async throws -> PullResult { + guard token != nil else { + pulls.append(.full) + tokenExpired = false + return PullResult(changedRecords: records, deletedRecordIDs: [], newToken: self.token) + } + guard !tokenExpired else { + pulls.append(.expired) + throw CKError(.changeTokenExpired) + } + pulls.append(.delta) + return PullResult(changedRecords: [], deletedRecordIDs: [], newToken: self.token) + } +} + +@objc(SyncFullFetchDeletionTestsChangeTokenArchive) +private final class FullFetchChangeTokenArchive: NSObject, NSSecureCoding { + static var supportsSecureCoding: Bool { true } + + override init() { + super.init() + } + + required init?(coder: NSCoder) { + super.init() + } + + func encode(with coder: NSCoder) {} +} diff --git a/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift b/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift index bed11927c4..17f6c3c005 100644 --- a/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift +++ b/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift @@ -286,6 +286,53 @@ struct SQLParameterExtractorTests { #expect(SQLParameterExtractor.extractParameters(from: sql, grammar: .ansi) == ["a", "b"]) } + @Test + func bindGluedToAKeywordIsAParameter() { + let extract = { (sql: String) in SQLParameterExtractor.extractParameters(from: sql, grammar: .ansi) } + #expect(extract("SELECT * FROM t LIMIT:n OFFSET:m") == ["n", "m"]) + #expect(extract("SELECT * FROM t ORDER BY id DESC LIMIT 10 OFFSET:m") == ["m"]) + #expect(extract("SELECT * FROM (SELECT 1) s LIMIT:n") == ["n"]) + #expect(extract("DELETE FROM t LIMIT:n") == ["n"]) + #expect(extract("SELECT * FROM t LIMIT ALL OFFSET:m") == ["m"]) + #expect(extract("SELECT a FROM t GROUP BY ALL LIMIT:n") == ["n"]) + #expect(extract("SELECT * FROM t WHERE id IN:ids") == ["ids"]) + #expect(extract("SELECT * FROM t WHERE a BETWEEN:lo AND:hi") == ["lo", "hi"]) + #expect(extract("SELECT CASE WHEN:c THEN:p ELSE:q END") == ["c", "p", "q"]) + #expect(extract("SELECT * FROM t WHERE name LIKE:pat OR:x") == ["pat", "x"]) + } + + @Test( + "A row-limit word in an operand's place is a column, so its colon stays a path", + arguments: [ + "SELECT limit:currency FROM raw", + "SELECT a, offset:x FROM raw", + "SELECT t.limit:x FROM raw t", + "SELECT DISTINCT limit:x FROM raw", + "SELECT * FROM raw WHERE limit:x > 1", + "SELECT src:in:x FROM raw", + "SELECT * FROM t LIMIT {limit:UInt32}", + "SELECT a FROM t WHERE a IS DISTINCT FROM limit:x", + "SELECT EXTRACT(YEAR FROM limit:ts) FROM raw", + "SELECT ALL limit:x FROM raw", + "SELECT TOP 10 limit:currency FROM raw", + ] + ) + func rowLimitWordAsAColumnKeepsItsPath(sql: String) { + #expect(SQLParameterExtractor.extractParameters(from: sql, grammar: .ansi) == []) + } + + @Test + func bindGluedToAKeywordIsSpacedFromIt() { + let result = SQLParameterExtractor.convertToNativeStyle( + sql: "SELECT * FROM t WHERE id IN(:a) LIMIT:n", + parameters: [QueryParameter(name: "a", value: "1"), QueryParameter(name: "n", value: "5")], + style: .dollar, + grammar: .ansi + ) + #expect(result.sql == "SELECT * FROM t WHERE id IN($1) LIMIT $2") + #expect(result.values.count == 2) + } + @Test func nestedCommentHidesAColon() { let sql = "/* a /* b */ :x */ SELECT :y" diff --git a/apple/TableProTests/Helpers/StubExecutionGates.swift b/apple/TableProTests/Helpers/StubExecutionGates.swift index a848d6c996..83dce282ed 100644 --- a/apple/TableProTests/Helpers/StubExecutionGates.swift +++ b/apple/TableProTests/Helpers/StubExecutionGates.swift @@ -88,7 +88,7 @@ extension DatabaseAccessBridge { maxRows: Int, timeoutSeconds: Int, cancellation: (any StatementCancellationSignal)?, - maskingTable: String? = nil + masking: ExternalRowMasking = .byOrigin ) async throws -> StatementOutcome { let statement = try await plannedStatement(scope: scope, query: query, maxRows: maxRows) let databaseType = try await ensureConnected(scope.connectionId) @@ -98,7 +98,7 @@ extension DatabaseAccessBridge { maxRows: maxRows, timeoutSeconds: timeoutSeconds, cancellation: cancellation, - maskingTable: maskingTable + masking: masking ) } diff --git a/apple/TableProTests/Views/Editor/SQLEditorPendingTextTests.swift b/apple/TableProTests/Views/Editor/SQLEditorPendingTextTests.swift new file mode 100644 index 0000000000..bb2a2a5acf --- /dev/null +++ b/apple/TableProTests/Views/Editor/SQLEditorPendingTextTests.swift @@ -0,0 +1,217 @@ +// Over 500,000 units the editor writes its text to the tab 150 ms after the last key, so a tab switch, a teardown, +// a close or a script inside that window dropped the last keys or read the tab without them. + +import AppKit +import SwiftUI +import TableProEditorKit +import TableProTestSupport +import TableProTextEngine +import Testing + +@testable import TablePro + +@MainActor +private enum LargeTabEditor { + static let text = String(repeating: "a", count: 500_001) + + static func makeWindow() -> NSWindow { + let window = NSWindow( + contentRect: NSRect(x: 0, y: 0, width: 600, height: 400), + styleMask: [.titled], + backing: .buffered, + defer: false + ) + window.isReleasedWhenClosed = false + return window + } + + // Bound to the tab by id, as the query tab binds it, and in a window, so the router can reach it. + static func mount(on coordinator: MainContentCoordinator, tabId: UUID) async throws -> (SQLEditorController, NSWindow) { + let manager = coordinator.tabManager + let editor = SQLEditorController(input: SQLEditorInput( + text: Binding( + get: { manager.tabs.first { $0.id == tabId }?.content.query ?? "" }, + set: { value in manager.mutate(tabId: tabId) { $0.content.query = value } } + ), + cursorPositions: .constant([]), + databaseType: coordinator.connection.type, + connectionId: coordinator.connectionId, + tabID: tabId, + claimFocusOnAppear: true + )) + let window = makeWindow() + window.contentView = editor.textViewController.view + coordinator.contentWindow = window + try await waitUntil { + window.contentView?.layoutSubtreeIfNeeded() + return !editor.coordinator.pendingFocusClaim + } + return (editor, window) + } + + static func typeAtStart(_ text: String, in editor: SQLEditorController, window: NSWindow) async throws { + let textView = try #require(editor.textViewController.textView) + try await waitUntil { + window.contentView?.layoutSubtreeIfNeeded() + return textView.window != nil + } + _ = window.makeFirstResponder(textView) + textView.selectionManager.setSelectedRange(NSRange(location: 0, length: 0)) + textView.replaceCharacters(in: NSRange(location: 0, length: 0), with: text) + } + + static func addFileTab(to coordinator: MainContentCoordinator, databaseName: String = "") throws -> UUID { + let manager = coordinator.tabManager + manager.addTab( + initialQuery: text, + title: "Large", + databaseName: databaseName, + sourceFileURL: URL(fileURLWithPath: "/private/tmp/sql-editor-pending-text.sql") + ) + let tabId = try #require(manager.selectedTabId) + try #require(!coordinator.hasUnsavedWork(forTab: tabId)) + return tabId + } + + static func query(ofTab tabId: UUID, in coordinator: MainContentCoordinator) -> String? { + coordinator.tabManager.tabs.first { $0.id == tabId }?.content.query + } +} + +@MainActor +struct SQLEditorPendingTextTests { + private final class Tabs { + var texts: [UUID: String] = [:] + + func binding(_ id: UUID) -> Binding { + Binding(get: { self.texts[id] ?? "" }, set: { self.texts[id] = $0 }) + } + } + + private func input(_ tabs: Tabs, tab id: UUID) -> SQLEditorInput { + SQLEditorInput(text: tabs.binding(id), cursorPositions: .constant([]), databaseType: .sqlite, tabID: id) + } + + @Test("A tab switch right after typing keeps the keys in the tab they were typed in") + func switchKeepsTheLastKeys() async throws { + let tabs = Tabs() + let first = UUID() + let second = UUID() + tabs.texts[first] = LargeTabEditor.text + tabs.texts[second] = "SELECT 2" + let editor = SQLEditorController(input: input(tabs, tab: first)) + let window = LargeTabEditor.makeWindow() + window.contentView = editor.textViewController.view + defer { editor.teardown() } + defer { window.orderOut(nil) } + try await LargeTabEditor.typeAtStart("x", in: editor, window: window) + try #require(tabs.texts[first] == LargeTabEditor.text) + + editor.apply(input(tabs, tab: second)) + + try await waitUntil { tabs.texts[first] == "x" + LargeTabEditor.text } + #expect(editor.textViewController.textView.string == "SELECT 2") + #expect(tabs.texts[second] == "SELECT 2") + } + + @Test("Closing the editor right after typing still writes the keys to the tab") + func teardownKeepsTheLastKeys() async throws { + let tabs = Tabs() + let tab = UUID() + tabs.texts[tab] = LargeTabEditor.text + var editor: SQLEditorController? = SQLEditorController(input: input(tabs, tab: tab)) + let window = LargeTabEditor.makeWindow() + window.contentView = editor?.textViewController.view + defer { window.orderOut(nil) } + try await LargeTabEditor.typeAtStart("x", in: try #require(editor), window: window) + try #require(tabs.texts[tab] == LargeTabEditor.text) + + editor?.teardown() + window.contentView = nil + editor = nil + + try await waitUntil { tabs.texts[tab] == "x" + LargeTabEditor.text } + } + + @Test("Closing a clean file tab right after typing asks to save it") + func closeRightAfterTypingAsks() async throws { + let connection = TestFixtures.makeConnection(database: "db_a") + let coordinator = SessionStateFactory.create(connection: connection, payload: nil).coordinator + defer { coordinator.teardown() } + let tabId = try LargeTabEditor.addFileTab(to: coordinator) + let (editor, window) = try await LargeTabEditor.mount(on: coordinator, tabId: tabId) + defer { editor.teardown() } + defer { window.orderOut(nil) } + try await LargeTabEditor.typeAtStart("x", in: editor, window: window) + try #require(LargeTabEditor.query(ofTab: tabId, in: coordinator) == LargeTabEditor.text) + + var selectedTables: Set = [] + var pendingTruncates: Set = [] + var pendingDeletes: Set = [] + var tableOperationOptions: [DatabaseTreeTableRef: TableOperationOptions] = [:] + let actions = MainContentCommandActions( + coordinator: coordinator, + connection: connection, + selectedTables: Binding(get: { selectedTables }, set: { selectedTables = $0 }), + pendingTruncates: Binding(get: { pendingTruncates }, set: { pendingTruncates = $0 }), + pendingDeletes: Binding(get: { pendingDeletes }, set: { pendingDeletes = $0 }), + tableOperationOptions: Binding(get: { tableOperationOptions }, set: { tableOperationOptions = $0 }), + trailingPaneState: TrailingPaneState() + ) + actions.window = window + var prompts = 0 + actions.confirmSaveChanges = { _, _ in + prompts += 1 + return .cancel + } + + await actions.closeTabAwaiting(id: tabId) + + #expect(prompts == 1) + #expect(LargeTabEditor.query(ofTab: tabId, in: coordinator) == "x" + LargeTabEditor.text) + } + + @Test("Closing a rail entry right after typing reads its tabs with the last keys") + func railEntryCloseReadsTheLastKeys() async throws { + let connection = TestFixtures.makeConnection(database: "db_a") + let coordinator = SessionStateFactory.create(connection: connection, payload: nil).coordinator + defer { coordinator.teardown() } + let tabId = try LargeTabEditor.addFileTab(to: coordinator, databaseName: "db_a") + let (editor, window) = try await LargeTabEditor.mount(on: coordinator, tabId: tabId) + defer { editor.teardown() } + defer { window.orderOut(nil) } + try await LargeTabEditor.typeAtStart("x", in: editor, window: window) + try #require(LargeTabEditor.query(ofTab: tabId, in: coordinator) == LargeTabEditor.text) + + let victims = WorkspaceCloseAction.committedTabs(in: "db_a", across: [coordinator]) + + let victim = try #require(victims.first { $0.id == tabId }) + #expect(victim.content.query == "x" + LargeTabEditor.text) + #expect(coordinator.hasUnsavedWork(in: victim)) + } +} + +extension GlobalStateSuites { + @MainActor + struct ScriptingPendingTextTests { + @Test("A script reads a large tab's text with the keys typed a moment ago") + func scriptReadsTheLastKeys() async throws { + let connection = TestFixtures.makeConnection(database: "db_a") + let saved = ConnectionStorage.shared.loadConnections() + ConnectionStorage.shared.saveConnections(saved + [connection]) + defer { ConnectionStorage.shared.saveConnections(saved) } + let coordinator = SessionStateFactory.create(connection: connection, payload: nil).coordinator + defer { coordinator.teardown() } + let tabId = try LargeTabEditor.addFileTab(to: coordinator) + let (editor, window) = try await LargeTabEditor.mount(on: coordinator, tabId: tabId) + defer { editor.teardown() } + defer { window.orderOut(nil) } + try await LargeTabEditor.typeAtStart("x", in: editor, window: window) + try #require(LargeTabEditor.query(ofTab: tabId, in: coordinator) == LargeTabEditor.text) + + let query = ScriptingSnapshot.query(ofTab: tabId, connectionId: connection.id) + + #expect(query == "x" + LargeTabEditor.text) + } + } +} diff --git a/apple/TableProTests/Views/Structure/StructureRebuildPartialFailureTests.swift b/apple/TableProTests/Views/Structure/StructureRebuildPartialFailureTests.swift new file mode 100644 index 0000000000..5c8a441c73 --- /dev/null +++ b/apple/TableProTests/Views/Structure/StructureRebuildPartialFailureTests.swift @@ -0,0 +1,178 @@ +// +// StructureRebuildPartialFailureTests.swift +// TableProTests +// +// An engine whose DDL commits statement by statement keeps what ran before a failure, so the error names it. +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class CommittingDDLDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var queries: [String] = [] + private var failuresLeft: [String: Int] + let supportsTransactions: Bool + private let state: PluginSessionTransactionState + + init( + failing failuresLeft: [String: Int], + supportsTransactions: Bool = false, + state: PluginSessionTransactionState = .idle + ) { + self.failuresLeft = failuresLeft + self.supportsTransactions = supportsTransactions + self.state = state + } + + var executed: [String] { lock.withLock { queries } } + + func sessionTransactionState() async -> PluginSessionTransactionState { state } + + func execute(query: String) async throws -> PluginQueryResult { + let fails = lock.withLock { () -> Bool in + queries.append(query) + guard let left = failuresLeft[query], left > 0 else { return false } + failuresLeft[query] = left - 1 + return true + } + if fails { throw DatabaseError.queryFailed("lost connection") } + return .empty + } + + func switchDatabase(to database: String) async throws {} +} + +@MainActor +struct StructureRebuildPartialFailureTests { + private func run(_ plan: PluginColumnReorderPlan, driver: CommittingDDLDriver) async -> String? { + let connection = TestFixtures.makeConnection(database: "app") + var session = ConnectionSession( + connection: connection, + driver: PluginDriverAdapter(connection: connection, pluginDriver: driver) + ) + session.status = .connected + session.browseDatabase = "app" + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let scope = DatabaseScope(connectionId: connection.id, database: "app", schema: nil) + + do { + try await StructureRebuildPlanRunner.execute( + StructureRebuildPlanRunner.Prepared(plan: plan, fingerprint: nil, scope: scope, tableName: "orders"), + databaseType: connection.type, + operationDescription: "Reorder Columns", + isConfirmationPreCleared: true, + gate: AlwaysAllowGate() + ) + return nil + } catch { + return error.localizedDescription + } + } + + @Test("A reorder that fails part way with nothing to undo it names the statements that stayed applied") + func namesTheStatementsThatRan() async throws { + let plan = PluginColumnReorderPlan( + statements: ["MOVE a", "MOVE b", "MOVE c"], + cost: .metadataOnly, + verifications: [] + ) + let driver = CommittingDDLDriver(failing: ["MOVE b": 1]) + + let message = try #require(await run(plan, driver: driver)) + + #expect(message.contains("lost connection")) + #expect(message.contains("MOVE a")) + #expect(!message.contains("MOVE b")) + #expect(!message.contains("MOVE c")) + #expect(driver.executed == ["MOVE a", "MOVE b"]) + } + + @Test("A compensation that fails is named, so the hidden column can be restored by hand") + func namesTheCompensationThatFailed() async throws { + let plan = PluginColumnReorderPlan( + statements: ["HIDE a", "SHOW a"], + compensation: ["SHOW a"], + cost: .metadataOnly, + verifications: [] + ) + let driver = CommittingDDLDriver(failing: ["SHOW a": 2]) + + let message = try #require(await run(plan, driver: driver)) + + #expect(message.contains("lost connection")) + #expect(message.contains("SHOW a")) + #expect(!message.contains("HIDE a")) + #expect(driver.executed == ["HIDE a", "SHOW a", "SHOW a"]) + } + + @Test("A compensation that puts everything back leaves only the failure in the message") + func restoredPlanReportsTheFailureAlone() async throws { + let plan = PluginColumnReorderPlan( + statements: ["HIDE a", "SHOW a"], + compensation: ["SHOW a"], + cost: .metadataOnly, + verifications: [] + ) + let driver = CommittingDDLDriver(failing: ["SHOW a": 1]) + + let message = try #require(await run(plan, driver: driver)) + + #expect(message == String(format: String(localized: "The change could not be applied: %@"), "lost connection")) + #expect(driver.executed == ["HIDE a", "SHOW a", "SHOW a"]) + } + + @Test("A transactional plan that joined the user's open transaction says a rollback undoes what ran") + func statementsInsideTheOpenTransactionAreNotCalledApplied() async throws { + let plan = PluginColumnReorderPlan( + statements: ["ALTER a", "ALTER b"], + isTransactional: true, + cost: .tableRebuild, + verifications: [] + ) + let driver = CommittingDDLDriver(failing: ["ALTER b": 1], supportsTransactions: true, state: .inTransaction) + + let message = try #require(await run(plan, driver: driver)) + + let pending = String( + localized: "These statements ran inside the transaction already open on this connection, so rolling it back undoes them:" + ) + #expect(message.contains(pending)) + #expect(!message.contains(String(localized: "These statements ran before the failure and were not rolled back:"))) + #expect(message.contains("ALTER a")) + #expect(driver.executed == ["ALTER a", "ALTER b"]) + } + + @Test("A transactional plan on an engine without transactions names what ran as applied") + func transactionalPlanWithoutTransactionsNamesWhatRan() async throws { + let plan = PluginColumnReorderPlan( + statements: ["ALTER a", "ALTER b"], + isTransactional: true, + cost: .tableRebuild, + verifications: [] + ) + let driver = CommittingDDLDriver(failing: ["ALTER b": 1]) + + let message = try #require(await run(plan, driver: driver)) + + #expect(message.contains(String(localized: "These statements ran before the failure and were not rolled back:"))) + #expect(message.contains("ALTER a")) + } + + @Test("A plan whose first statement fails names nothing as applied") + func firstStatementFailureNamesNothing() async throws { + let plan = PluginColumnReorderPlan( + statements: ["MOVE a", "MOVE b"], + cost: .metadataOnly, + verifications: [] + ) + let driver = CommittingDDLDriver(failing: ["MOVE a": 1]) + + let message = try #require(await run(plan, driver: driver)) + + #expect(message == String(format: String(localized: "The change could not be applied: %@"), "lost connection")) + } +} diff --git a/apple/changelog.d/4178.fixed.1.md b/apple/changelog.d/4178.fixed.1.md new file mode 100644 index 0000000000..6054872de8 --- /dev/null +++ b/apple/changelog.d/4178.fixed.1.md @@ -0,0 +1 @@ +Pasting spreadsheet rows with empty cells wrote NULL instead of the column's default value. diff --git a/apple/changelog.d/4178.fixed.10.md b/apple/changelog.d/4178.fixed.10.md new file mode 100644 index 0000000000..796567750a --- /dev/null +++ b/apple/changelog.d/4178.fixed.10.md @@ -0,0 +1 @@ +A password saved while Sync Passwords was off went unused once Sync Passwords was turned back on. diff --git a/apple/changelog.d/4178.fixed.11.md b/apple/changelog.d/4178.fixed.11.md new file mode 100644 index 0000000000..cbfa87886e --- /dev/null +++ b/apple/changelog.d/4178.fixed.11.md @@ -0,0 +1 @@ +Saving a connection rewrote passwords you did not change, overwriting a newer one saved on another device. diff --git a/apple/changelog.d/4178.fixed.12.md b/apple/changelog.d/4178.fixed.12.md new file mode 100644 index 0000000000..2c1c50d532 --- /dev/null +++ b/apple/changelog.d/4178.fixed.12.md @@ -0,0 +1 @@ +Reading a SQLite full-text search table over MCP or from the assistant failed with authorization denied. diff --git a/apple/changelog.d/4178.fixed.2.md b/apple/changelog.d/4178.fixed.2.md new file mode 100644 index 0000000000..895cb3b4bf --- /dev/null +++ b/apple/changelog.d/4178.fixed.2.md @@ -0,0 +1 @@ +Switching databases on a connection that reconnects to switch rolled back an open transaction without asking. diff --git a/apple/changelog.d/4178.fixed.3.md b/apple/changelog.d/4178.fixed.3.md new file mode 100644 index 0000000000..019d2dbf89 --- /dev/null +++ b/apple/changelog.d/4178.fixed.3.md @@ -0,0 +1 @@ +Items deleted on another Mac came back after iCloud sync had to download everything again. diff --git a/apple/changelog.d/4178.fixed.4.md b/apple/changelog.d/4178.fixed.4.md new file mode 100644 index 0000000000..fe4c7c81c4 --- /dev/null +++ b/apple/changelog.d/4178.fixed.4.md @@ -0,0 +1 @@ +Claude Agent asked for MCP access to the chat's connection on every message at Ask Once. diff --git a/apple/changelog.d/4178.fixed.5.md b/apple/changelog.d/4178.fixed.5.md new file mode 100644 index 0000000000..5dc985a2fa --- /dev/null +++ b/apple/changelog.d/4178.fixed.5.md @@ -0,0 +1 @@ +A structure save or column reorder that failed part way did not say which changes were already applied. diff --git a/apple/changelog.d/4178.fixed.6.md b/apple/changelog.d/4178.fixed.6.md new file mode 100644 index 0000000000..a8ee3392b7 --- /dev/null +++ b/apple/changelog.d/4178.fixed.6.md @@ -0,0 +1 @@ +Importing, copying or syncing rows into a MySQL or MariaDB BIT column stored the wrong value: 5 became 53. diff --git a/apple/changelog.d/4178.fixed.7.md b/apple/changelog.d/4178.fixed.7.md new file mode 100644 index 0000000000..540f3e86e7 --- /dev/null +++ b/apple/changelog.d/4178.fixed.7.md @@ -0,0 +1 @@ +Adding a Redis hash, list, set or sorted set key in the grid wrote into an existing key of that name. diff --git a/apple/changelog.d/4178.fixed.8.md b/apple/changelog.d/4178.fixed.8.md new file mode 100644 index 0000000000..1d0d1c8500 --- /dev/null +++ b/apple/changelog.d/4178.fixed.8.md @@ -0,0 +1 @@ +Editing a SurrealDB field that held both strings and numbers changed the edited value's type. diff --git a/apple/changelog.d/4178.fixed.9.md b/apple/changelog.d/4178.fixed.9.md new file mode 100644 index 0000000000..1c711e8b5d --- /dev/null +++ b/apple/changelog.d/4178.fixed.9.md @@ -0,0 +1 @@ +A MySQL or MariaDB query with no query timeout hung for about two hours when the network dropped its reply. diff --git a/apple/changelog.d/4178.fixed.md b/apple/changelog.d/4178.fixed.md new file mode 100644 index 0000000000..9b68cf7b2d --- /dev/null +++ b/apple/changelog.d/4178.fixed.md @@ -0,0 +1 @@ +Saving, closing or switching tabs right after typing in a very large script lost the last keystrokes. diff --git a/apple/changelog.d/4178.security.md b/apple/changelog.d/4178.security.md new file mode 100644 index 0000000000..06274ec54e --- /dev/null +++ b/apple/changelog.d/4178.security.md @@ -0,0 +1 @@ +MCP clients and the AI assistant could read masked values through a renamed column, a query plan or an error message.