diff --git a/.claude/rules/data-sync-storage.md b/.claude/rules/data-sync-storage.md index 7dc1f491e5..c38348fc51 100644 --- a/.claude/rules/data-sync-storage.md +++ b/.claude/rules/data-sync-storage.md @@ -181,6 +181,7 @@ This path holds user data. A silent wrong answer here looks exactly like an empt | Snowflake SSO and MFA tokens, per host and user. A bare account's `NAME.USER` item stays for 0.83.1, which reads only that key | Keychain, device-local | `SnowflakeTokenCache` | | Whether `connections.json` was ever given an integrity tag, so a deleted tag reads as modified | Keychain `com.TablePro.connectionStoreStamped.`, device-local | `ConnectionStoreIntegrity` | | Whether Local only connections' secrets were copied to device-only items | UserDefaults `com.TablePro.localOnlySecretsKeptOnDevice`, device-local, never synced | `ConnectionStorage` | +| iCloud Keychain copies of Local only secrets the keychain refused to remove, retried at launch | UserDefaults `com.TablePro.localOnlySyncedCopiesPendingRemoval`, device-local, never synced | `ConnectionStorage` | | Whether plugin secure fields moved out of `connections.json` into the Keychain, set only when every field moved and the save succeeded | UserDefaults `com.TablePro.pluginSecureFieldsMovedToKeychain`, device-local, never synced | `ConnectionStorage` | | Whether a saved-queries store replaced a damaged file and waits for a full fetch, so push treats it as unreadable | Table `sync_refill` in the saved-queries SQLite store, dropped once a full fetch applied | `SQLFavoriteStorage` | | Deletions the server reported for a sync category that is off, applied when the category is next read | UserDefaults, device-local, never synced | `SyncMetadataStorage.deferredDeletions` | diff --git a/.claude/rules/tests.md b/.claude/rules/tests.md index 6b366b0dee..cd1648a222 100644 --- a/.claude/rules/tests.md +++ b/.claude/rules/tests.md @@ -73,7 +73,7 @@ UI flows: `.claude/rules/ui-tests.md`. Running tests and reading verdicts: `.cla Use one before writing a local helper. - `TableProTestSupport` (TableProCore): `waitUntil`, `AsyncGate`, `ManualClock` (production code takes `clock: any Clock = ContinuousClock()`), `BoundedCall`, `HeldOpenWriter`, `BackgroundPipeWriter`, `BackgroundPipeReader`. -- `apple/TableProTests/Helpers/`: `GlobalStateSuites`, `EphemeralDefaults`, `.appSettings { }`, `.recordingClipboard`, `CoordinatorFixture`, `OfflineRegistry`, `InMemoryColumnLayoutStore`, `InMemoryKeychain`, `PluginDriverStub`, `TestFixtures`, `AlwaysAllowGate`, `AlwaysDenyGate`, `MCPInMemoryApprovalStore`, `MCPFixedApprovalReader`, `ArrayRowProvider`, `ScopeRecordingKeychain`. +- `apple/TableProTests/Helpers/`: `GlobalStateSuites`, `EphemeralDefaults`, `.appSettings { }`, `.recordingClipboard`, `CoordinatorFixture`, `OfflineRegistry`, `InMemoryColumnLayoutStore`, `InMemoryKeychain`, `PluginDriverStub`, `TestFixtures`, `AlwaysAllowGate`, `AlwaysDenyGate`, `CountingOpener`, `MCPInMemoryApprovalStore`, `MCPFixedApprovalReader`, `ArrayRowProvider`, `ScopeRecordingKeychain`. - `TableProMCPTestSupport` (TableProMacCore): `MCPTestClock`, `MockHttpServer`, `RecordingBridgeLogger`. `TableProTLSTestFixtures` (TableProCore), also for the iOS tests. ## Known failures and flakes diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index 7b85a5049e..a59c9a69de 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -541,7 +541,7 @@ let package = Package( ), .testTarget( name: "TableProTLSClientIdentityTests", - dependencies: ["TableProTLSClientIdentity", "TableProTLSTestFixtures"], + dependencies: ["TableProTLSClientIdentity", "TableProTLSTestFixtures", "TableProTestSupport"], path: "Tests/TableProTLSClientIdentityTests" ), .testTarget( diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLSessionStateClassifier.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLSessionStateClassifier.swift new file mode 100644 index 0000000000..9b4c09ba0d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLSessionStateClassifier.swift @@ -0,0 +1,708 @@ +import Foundation + +public struct SQLSessionStateKinds: OptionSet, Sendable, Hashable { + public let rawValue: Int + + public init(rawValue: Int) { + self.rawValue = rawValue + } + + public static let temporaryObjects = SQLSessionStateKinds(rawValue: 1 << 0) + public static let sessionSettings = SQLSessionStateKinds(rawValue: 1 << 1) + public static let userVariables = SQLSessionStateKinds(rawValue: 1 << 2) + public static let preparedStatements = SQLSessionStateKinds(rawValue: 1 << 3) + public static let heldCursors = SQLSessionStateKinds(rawValue: 1 << 4) + public static let sessionLocks = SQLSessionStateKinds(rawValue: 1 << 5) + public static let listeners = SQLSessionStateKinds(rawValue: 1 << 6) + + public static let all: SQLSessionStateKinds = [ + .temporaryObjects, .sessionSettings, .userVariables, .preparedStatements, .heldCursors, .sessionLocks, + .listeners, + ] +} + +public struct SQLSessionStateEffect: Sendable, Equatable { + public let creates: SQLSessionStateKinds + public let clears: SQLSessionStateKinds + + public init(creates: SQLSessionStateKinds = [], clears: SQLSessionStateKinds = []) { + self.creates = creates + self.clears = clears + } + + public static let none = SQLSessionStateEffect() + + public var isEmpty: Bool { + creates.isEmpty && clears.isEmpty + } + + public func followed(by next: SQLSessionStateEffect) -> SQLSessionStateEffect { + SQLSessionStateEffect( + creates: creates.subtracting(next.clears).union(next.creates), + clears: clears.union(next.clears) + ) + } + + public func applied(to kinds: SQLSessionStateKinds) -> SQLSessionStateKinds { + kinds.subtracting(clears).union(creates) + } +} + +public enum SQLSessionDialect: Sendable, Hashable { + case postgres + case mySQL + case sqlServer + case oracle + case sqlite + case duckDB +} + +// No engine reports a session's temporary tables, settings or variables, and a dead connection answers nothing, so +// the text decides. A `CALL`, `DO`, `EXEC proc` or PL/SQL block is not read: what it sets up is out of sight. +public enum SQLSessionStateClassifier: Sendable { + // Past this a text is data, an INSERT's VALUES, and reading it all would cost a pass per run. Each text is cut before + // it is split, so a setup past this point in one long text, such as a T-SQL batch with no `;`, is not seen. + private static let classifiedPrefixLength = 65_536 + private static let modifierLookahead = 5 + + private static let period = UInt16(UnicodeScalar(".").value) + private static let atSign = UInt16(UnicodeScalar("@").value) + private static let colon = UInt16(UnicodeScalar(":").value) + private static let equals = UInt16(UnicodeScalar("=").value) + + // What `skipping` names runs again on every new connection, such as a startup command, so it is never lost. + public static func effect( + ofStatements statements: [String], + dialect: SQLSessionDialect, + grammar: SQLLexicalGrammar, + skipping skipped: [String] = [] + ) -> SQLSessionStateEffect { + let skippedTexts = Set(skipped.map(comparableText)) + var effect = SQLSessionStateEffect.none + for text in statements { + let head = classifiedHead(of: text) + guard SQLSessionStateQuickCheck.mayChangeSession(head, dialect: dialect) else { continue } + for statement in SQLStatementClassifier.statements(of: head, grammar: grammar) + where SQLSessionStateQuickCheck.mayChangeSession(statement, dialect: dialect) + && !skippedTexts.contains(comparableText(statement)) { + effect = effect.followed(by: statementEffect(of: statement, dialect: dialect, grammar: grammar)) + } + } + return effect + } + + public static func effect(of sql: String, dialect: SQLSessionDialect, grammar: SQLLexicalGrammar) -> SQLSessionStateEffect { + effect(ofStatements: [sql], dialect: dialect, grammar: grammar) + } + + static func comparableText(_ statement: String) -> String { + var text = statement.trimmingCharacters(in: .whitespacesAndNewlines) + while text.hasSuffix(";") { + text = String(text.dropLast()).trimmingCharacters(in: .whitespacesAndNewlines) + } + return text + } + + // Every server reading of a `/*!` comment counts what it sets up, and only what all of them clear is cleared. + private static func statementEffect( + of statement: String, + dialect: SQLSessionDialect, + grammar: SQLLexicalGrammar + ) -> SQLSessionStateEffect { + // More versions than a reader takes one by one leaves the text as written, its executable comments unread. + let texts = SQLExecutableCommentReadings.texts(of: statement, grammar: grammar).map { Array($0) } ?? [statement] + var creates: SQLSessionStateKinds = [] + var clears: SQLSessionStateKinds? + for text in texts { + let reading = SQLStatementClassifier.runnableStatements(of: text, grammar: grammar) + .reduce(SQLSessionStateEffect.none) { partial, piece in + partial.followed(by: pieceEffect(of: piece, dialect: dialect, grammar: grammar)) + } + creates.formUnion(reading.creates) + clears = clears.map { $0.intersection(reading.clears) } ?? reading.clears + } + return SQLSessionStateEffect(creates: creates, clears: clears ?? []) + } + + private static func classifiedHead(of statement: String) -> String { + let text = statement as NSString + guard text.length > classifiedPrefixLength else { return statement } + let cut = text.rangeOfComposedCharacterSequence(at: classifiedPrefixLength).location + return text.substring(to: cut) + } + + private static func pieceEffect( + of piece: String, + dialect: SQLSessionDialect, + grammar: SQLLexicalGrammar + ) -> SQLSessionStateEffect { + guard !SQLProceduralBlock.runs(piece, grammar: grammar) else { return .none } + let code = StatementCode(piece, grammar: grammar) + guard let start = code.tokens.firstIndex(where: { $0.kind != .open }), code.tokens[start].word != nil else { + return .none + } + switch dialect { + case .postgres: + return postgresEffect(code, start: start) + case .mySQL: + return SQLSessionStateEffect(creates: mySQLKinds(code, start: start)) + case .sqlServer: + return SQLSessionStateEffect(creates: sqlServerKinds(code, start: start)) + case .oracle: + return SQLSessionStateEffect(creates: oracleKinds(code, start: start)) + case .sqlite: + return SQLSessionStateEffect(creates: sqliteKinds(code, start: start)) + case .duckDB: + return SQLSessionStateEffect(creates: duckDBKinds(code, start: start)) + } + } +} + +// MARK: - PostgreSQL + +extension SQLSessionStateClassifier { + private static let postgresObjectKinds: Set = ["TABLE", "VIEW", "SEQUENCE", "FUNCTION", "PROCEDURE", "TYPE"] + + // `SET LOCAL` lasts until the transaction ends, `SET TRANSACTION` covers one, and `SET CONSTRAINTS` the current one. + private static let postgresTransactionScopedSets: Set = ["LOCAL", "TRANSACTION", "CONSTRAINTS"] + + // KingbaseES ships each `pg_` function under a `sys_` name too. The `_xact_` forms end with the transaction. + private static let postgresSessionLockFunctions: Set = [ + "PG_ADVISORY_LOCK", "PG_ADVISORY_LOCK_SHARED", "PG_TRY_ADVISORY_LOCK", "PG_TRY_ADVISORY_LOCK_SHARED", + "SYS_ADVISORY_LOCK", "SYS_ADVISORY_LOCK_SHARED", "SYS_TRY_ADVISORY_LOCK", "SYS_TRY_ADVISORY_LOCK_SHARED", + ] + + private static func postgresEffect(_ code: StatementCode, start: Int) -> SQLSessionStateEffect { + let keyword = code.word(at: start) ?? "" + let next = code.word(at: start + 1) + // A prepared statement runs what it holds only when executed. + var kinds: SQLSessionStateKinds = keyword == "PREPARE" ? [] : postgresCallKinds(code) + switch keyword { + case "CREATE": + let temporary = createsTemporary(code, after: start, kinds: postgresObjectKinds) + || createsInSchema(code, after: start, kinds: postgresObjectKinds) { $0.hasPrefix("PG_TEMP") } + // `ON COMMIT DROP` ends the table with the transaction. + if temporary, !code.tokens.indices.contains(where: { code.spells(["ON", "COMMIT", "DROP"], at: $0) }) { + kinds.insert(.temporaryObjects) + } + case "SELECT", "WITH": + if selectsIntoPostgresTemporary(code) { + kinds.insert(.temporaryObjects) + } + case "SET": + if let next, !postgresTransactionScopedSets.contains(next) { + kinds.insert(.sessionSettings) + } + case "PREPARE": + if next != nil, next != "TRANSACTION" { + kinds.insert(.preparedStatements) + } + case "DECLARE": + if declaresHeldCursor(code, after: start) { + kinds.insert(.heldCursors) + } + case "LISTEN": + kinds.insert(.listeners) + case "DISCARD": + switch next { + case "ALL": + // `RESET ALL` is no clear: `role` and `session_authorization` are `GUC_NO_RESET_ALL`, and only this + // resets them too. + return SQLSessionStateEffect(creates: kinds, clears: .all) + case "TEMP", "TEMPORARY": + return SQLSessionStateEffect(creates: kinds, clears: .temporaryObjects) + default: + break + } + case "DEALLOCATE": + let target = next == "PREPARE" ? code.word(at: start + 2) : next + if target == "ALL" { + return SQLSessionStateEffect(creates: kinds, clears: .preparedStatements) + } + case "UNLISTEN": + if code.isSymbol(SqlLexer.star, at: start + 1) { + return SQLSessionStateEffect(creates: kinds, clears: .listeners) + } + default: + break + } + return SQLSessionStateEffect(creates: kinds) + } + + private static func postgresCallKinds(_ code: StatementCode) -> SQLSessionStateKinds { + var kinds: SQLSessionStateKinds = [] + for index in code.tokens.indices where code.opensCall(at: index) { + guard let name = code.word(at: index) else { continue } + if postgresSessionLockFunctions.contains(name) { + kinds.insert(.sessionLocks) + } else if name == "SET_CONFIG", !isTransactionLocalSetConfig(code, opening: index + 1) { + kinds.insert(.sessionSettings) + } + } + return kinds + } + + // `set_config(name, value, true)` lasts until the transaction ends. + private static func isTransactionLocalSetConfig(_ code: StatementCode, opening open: Int) -> Bool { + let outside = code.tokens[open].depth + var argument = 0 + var third: [TSQLTokens.Token] = [] + var index = open + 1 + while index < code.tokens.count { + let token = code.tokens[index] + if token.kind == .close, token.depth == outside { break } + if token.kind == .comma, token.depth == outside + 1 { + argument += 1 + } else if argument == 2 { + third.append(token) + } + index += 1 + } + return third.count == 1 && third[0].word == "TRUE" + } + + private static func selectsIntoPostgresTemporary(_ code: StatementCode) -> Bool { + code.tokens.indices.contains { index in + guard code.word(at: index) == "INTO", let target = code.word(at: index + 1) else { return false } + return target == "TEMP" || target == "TEMPORARY" + || (target.hasPrefix("PG_TEMP") && code.isSymbol(period, at: index + 2)) + } + } + + // A cursor outlives its transaction only `WITH HOLD`. + private static func declaresHeldCursor(_ code: StatementCode, after start: Int) -> Bool { + let end = code.tokens[(start + 1)...].firstIndex { $0.word == "FOR" } ?? code.tokens.count + return ((start + 1).. = ["GLOBAL", "PERSIST", "PERSIST_ONLY"] + private static let mySQLSessionScopes: Set = ["SESSION", "LOCAL"] + + private static func mySQLKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + let keyword = code.word(at: start) ?? "" + let next = code.word(at: start + 1) + var kinds: SQLSessionStateKinds = [] + switch keyword { + case "CREATE": + if createsTemporary(code, after: start, kinds: ["TABLE"]) { + kinds.insert(.temporaryObjects) + } + case "SET": + kinds.formUnion(mySQLSetKinds(code, from: start + 1)) + case "PREPARE": + return next == nil ? [] : [.preparedStatements] + case "LOCK": + if next == "TABLE" || next == "TABLES" { + kinds.insert(.sessionLocks) + } + default: + break + } + if code.tokens.indices.contains(where: { code.opensCall(at: $0) && code.word(at: $0) == "GET_LOCK" }) { + kinds.insert(.sessionLocks) + } + if assignsMySQLUserVariable(code) { + kinds.insert(.userVariables) + } + return kinds + } + + // Plain `SET TRANSACTION` covers the next transaction only, `SET PASSWORD` and `SET DEFAULT ROLE` change the + // account, and MariaDB's `SET STATEMENT ... FOR` one statement. + private static func mySQLSetKinds(_ code: StatementCode, from first: Int) -> SQLSessionStateKinds { + guard code.tokens.indices.contains(first) else { return [] } + let next = code.word(at: first) + let after = code.word(at: first + 1) + if after == "TRANSACTION", let next { + return mySQLSessionScopes.contains(next) ? [.sessionSettings] : [] + } + switch next { + case "PASSWORD", "STATEMENT", "TRANSACTION", "RESOURCE": + return [] + case "DEFAULT" where after == "ROLE": + return [] + case "NAMES", "CHARACTER", "CHARSET", "ROLE": + return [.sessionSettings] + default: + break + } + return mySQLAssignmentKinds(code, from: first) + } + + // A scope word sets the scope of the assignments after it that name none, as the manual says. + private static func mySQLAssignmentKinds(_ code: StatementCode, from first: Int) -> SQLSessionStateKinds { + let depth = code.tokens[first].depth + var kinds: SQLSessionStateKinds = [] + var sessionScope = true + var startsAssignment = true + for index in first.. Bool { + code.tokens.indices.contains { index in + guard code.isSymbol(atSign, at: index), !code.isSymbol(atSign, at: index - 1), + !code.isSymbol(atSign, at: index + 1) + else { return false } + if code.word(at: index - 1) == "INTO" { return true } + return code.isSymbol(colon, at: index + 2) && code.isSymbol(equals, at: index + 3) + } + } +} + +// MARK: - SQL Server + +extension SQLSessionStateClassifier { + private static let sqlServerTemporaryKinds: Set = ["TABLE", "PROC", "PROCEDURE"] + + private static func sqlServerKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + let keyword = code.word(at: start) ?? "" + let next = code.word(at: start + 1) + switch keyword { + case "CREATE": + return createsSQLServerTemporary(code, after: start) ? [.temporaryObjects] : [] + case "SELECT", "WITH": + let intoTemporary = code.tokens.indices.contains { index in + code.word(at: index) == "INTO" && namesSQLServerTemporary(code, at: index + 1) + } + return intoTemporary ? [.temporaryObjects] : [] + case "SET": + // A `@variable` dies with its batch. + guard let next, !next.hasPrefix("@") else { return [] } + return [.sessionSettings] + case "EXEC", "EXECUTE": + // `EXECUTE AS` impersonates until a `REVERT`. + return next == "AS" ? [.sessionSettings] : sqlServerProcedureKinds(code, nameAt: start + 1) + case "DECLARE": + return declaresGlobalCursor(code, after: start) ? [.heldCursors] : [] + default: + // A batch that opens with a procedure's name runs it. + return sqlServerProcedureKinds(code, nameAt: start) + } + } + + private static func createsSQLServerTemporary(_ code: StatementCode, after start: Int) -> Bool { + let window = (start + 1).. Bool { + guard code.tokens.indices.contains(index) else { return false } + let token = code.tokens[index] + let first: Int + switch token.kind { + case .word, .name: + first = token.location + case .quoted: + first = token.location + 1 + default: + return false + } + return first < code.text.length && code.text.character(at: first) == SqlLexer.hash + } + + private static func sqlServerProcedureKinds(_ code: StatementCode, nameAt start: Int) -> SQLSessionStateKinds { + var index = start + if code.word(at: index)?.hasPrefix("@") == true, code.isSymbol(equals, at: index + 1) { + index += 2 + } + while code.isSymbol(period, at: index + 1), code.word(at: index + 2) != nil { + index += 2 + } + switch code.word(at: index) { + // An application role holds the session's security context until `sp_unsetapprole` or the end of the session. + case "SP_SET_SESSION_CONTEXT", "SP_SETAPPROLE": + return [.sessionSettings] + case "SP_GETAPPLOCK": + return [.sessionLocks] + default: + return [] + } + } + + // A cursor is global unless declared `LOCAL`; a cursor variable `DECLARE @c CURSOR` dies with its batch. + private static func declaresGlobalCursor(_ code: StatementCode, after start: Int) -> Bool { + guard let name = code.word(at: start + 1), !name.hasPrefix("@") else { return false } + let end = code.tokens[(start + 1)...].firstIndex { $0.word == "FOR" } ?? code.tokens.count + let words = code.tokens[(start + 1).. = ["SET", "ENABLE", "DISABLE", "FORCE"] + private static let sqliteObjectKinds: Set = ["TABLE", "VIEW", "TRIGGER", "INDEX"] + private static let duckDBObjectKinds: Set = ["TABLE", "VIEW", "SEQUENCE", "MACRO", "FUNCTION", "TYPE"] + + // Rows in a global temporary table are not counted: the table itself is permanent. + private static func oracleKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + let next = code.word(at: start + 1) + switch code.word(at: start) { + case "ALTER": + let verb = code.word(at: start + 2) + return next == "SESSION" && verb.map(oracleSessionVerbs.contains) == true ? [.sessionSettings] : [] + case "SET": + return next == "ROLE" ? [.sessionSettings] : [] + case "CREATE": + // A private temporary table goes at commit unless it keeps its definition. + let isPrivate = code.spells(["PRIVATE", "TEMPORARY", "TABLE"], at: start + 1) + let preserved = code.tokens.indices.contains { code.spells(["PRESERVE", "DEFINITION"], at: $0) } + return isPrivate && preserved ? [.temporaryObjects] : [] + default: + return [] + } + } + + private static func sqliteKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + guard code.word(at: start) == "CREATE" else { return [] } + let temporary = createsTemporary(code, after: start, kinds: sqliteObjectKinds) + || createsInSchema(code, after: start, kinds: sqliteObjectKinds) { $0 == "TEMP" } + return temporary ? [.temporaryObjects] : [] + } + + private static func duckDBKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + let next = code.word(at: start + 1) + switch code.word(at: start) { + case "CREATE": + let temporary = createsTemporary(code, after: start, kinds: duckDBObjectKinds) + || createsInSchema(code, after: start, kinds: duckDBObjectKinds) { $0 == "TEMP" } + return temporary ? [.temporaryObjects] : [] + case "SET": + switch next { + case nil, "GLOBAL", "LOCAL": + return [] + case "VARIABLE": + return [.userVariables] + default: + return [.sessionSettings] + } + default: + return [] + } + } +} + +// MARK: - Shared readings + +extension SQLSessionStateClassifier { + private static let temporaryMarkers: Set = ["TEMP", "TEMPORARY"] + + // `CREATE [OR REPLACE] [GLOBAL | LOCAL] TEMP TABLE`: the marker stands before the kind, a few words in. + private static func createsTemporary(_ code: StatementCode, after start: Int, kinds: Set) -> Bool { + for index in (start + 1).., + schema isTemporarySchema: (String) -> Bool + ) -> Bool { + let window = (start + 1).. String? { + tokens.indices.contains(index) ? tokens[index].word : nil + } + + func opensCall(at index: Int) -> Bool { + tokens.indices.contains(index + 1) && tokens[index].word != nil && tokens[index + 1].kind == .open + } +} + +// Whole words, read without the grammar: a hit inside a comment or a literal only costs the full read, and a data word +// such as `offset`, `block` or `template`, or an email address, is no hit. Every rule above needs one of these. +enum SQLSessionStateQuickCheck: Sendable { + private static let atSign = UInt16(UnicodeScalar("@").value) + private static let colon = UInt16(UnicodeScalar(":").value) + private static let equals = UInt16(UnicodeScalar("=").value) + private static let shortestWord = 3 + private static let longestWord = 32 + + private static let postgresWords: Set = [ + "TEMP", "TEMPORARY", "SET", "SET_CONFIG", "PREPARE", "DECLARE", "LISTEN", "UNLISTEN", "DISCARD", "DEALLOCATE", + "PG_ADVISORY_LOCK", "PG_ADVISORY_LOCK_SHARED", "PG_TRY_ADVISORY_LOCK", "PG_TRY_ADVISORY_LOCK_SHARED", + "SYS_ADVISORY_LOCK", "SYS_ADVISORY_LOCK_SHARED", "SYS_TRY_ADVISORY_LOCK", "SYS_TRY_ADVISORY_LOCK_SHARED", + ] + private static let mySQLWords: Set = ["TEMP", "TEMPORARY", "SET", "PREPARE", "LOCK", "GET_LOCK"] + private static let sqlServerWords: Set = [ + "SET", "EXEC", "EXECUTE", "DECLARE", "SP_SET_SESSION_CONTEXT", "SP_GETAPPLOCK", "SP_SETAPPROLE", + ] + private static let oracleWords: Set = ["SESSION", "ROLE", "PRIVATE"] + private static let sqliteWords: Set = ["TEMP", "TEMPORARY"] + private static let duckDBWords: Set = ["TEMP", "TEMPORARY", "SET"] + + static func mayChangeSession(_ text: String, dialect: SQLSessionDialect) -> Bool { + let units = text as NSString + let length = units.length + let triggers = words(for: dialect) + var lastWord = "" + var lastWordEnd = -1 + var index = 0 + while index < length { + let unit = units.character(at: index) + if isWordUnit(unit) { + let start = index + while index < length, isWordUnit(units.character(at: index)) { + index += 1 + } + let count = index - start + lastWord = count >= shortestWord && count <= longestWord + ? units.substring(with: NSRange(location: start, length: count)).uppercased() + : "" + lastWordEnd = index + if triggers.contains(lastWord) || (dialect == .postgres && lastWord.hasPrefix("PG_TEMP")) { + return true + } + continue + } + if dialect == .mySQL, unit == atSign, + mayAssignMySQLVariable(units, at: index, gluedToInto: lastWord == "INTO" && lastWordEnd == index) { + return true + } + if dialect == .sqlServer, unit == SqlLexer.hash, startsSQLServerName(units, at: index) { + return true + } + index += 1 + } + return false + } + + private static func words(for dialect: SQLSessionDialect) -> Set { + switch dialect { + case .postgres: return postgresWords + case .mySQL: return mySQLWords + case .sqlServer: return sqlServerWords + case .oracle: return oracleWords + case .sqlite: return sqliteWords + case .duckDB: return duckDBWords + } + } + + // ASCII letters and `_`: a digit or `$` ends the run, so `1SET` still holds the `SET` the server reads. + private static func isWordUnit(_ unit: UInt16) -> Bool { + (0x41...0x5A).contains(unit) || (0x61...0x7A).contains(unit) || unit == 0x5F + } + + private static func isNameUnit(_ unit: UInt16) -> Bool { + isWordUnit(unit) || (0x30...0x39).contains(unit) || unit == 0x24 || unit >= 0x80 + } + + // An address glues its `@` to the name before it, as in `ana@example.com`. A variable stands apart, follows + // `INTO`, or is assigned with `:=`. A second `@` makes a system variable, which a statement only reads. + private static func mayAssignMySQLVariable(_ text: NSString, at index: Int, gluedToInto: Bool) -> Bool { + let length = text.length + if index > 0, text.character(at: index - 1) == atSign { return false } + guard index + 1 < length, text.character(at: index + 1) != atSign else { return false } + if gluedToInto { return true } + let previous = index > 0 ? text.character(at: index - 1) : SqlLexer.space + if !isNameUnit(previous), !"._%+-".utf16.contains(previous) { return true } + return assignedWithColonEquals(text, after: index) + } + + private static func assignedWithColonEquals(_ text: NSString, after index: Int) -> Bool { + let length = text.length + var cursor = index + 1 + let opener = text.character(at: cursor) + if opener == SqlLexer.singleQuote || opener == SqlLexer.doubleQuote || opener == SqlLexer.backtick { + cursor += 1 + while cursor < length, text.character(at: cursor) != opener { + cursor += 1 + } + cursor += 1 + } else { + while cursor < length, isNameUnit(text.character(at: cursor)) { + cursor += 1 + } + } + cursor = pastBlanksAndBlockComments(text, from: cursor) + return cursor + 1 < length && text.character(at: cursor) == colon && text.character(at: cursor + 1) == equals + } + + private static func pastBlanksAndBlockComments(_ text: NSString, from start: Int) -> Int { + let length = text.length + var cursor = start + while cursor < length { + let blank = StatementBlank.blankLength(in: text, at: cursor) + if blank > 0 { + cursor += blank + continue + } + guard cursor + 1 < length, text.character(at: cursor) == SqlLexer.slash, + text.character(at: cursor + 1) == SqlLexer.star + else { break } + let close = text.range( + of: "*/", options: [], range: NSRange(location: cursor + 2, length: length - cursor - 2) + ) + guard close.location != NSNotFound else { return length } + cursor = NSMaxRange(close) + } + return cursor + } + + // `#t`, `##t`, `[#t]` and `"#t"`: a `#` that begins a name. One inside a name, `a#b`, is part of it. + private static func startsSQLServerName(_ text: NSString, at index: Int) -> Bool { + let length = text.length + guard index + 1 < length else { return false } + if index > 0 { + let previous = text.character(at: index - 1) + if isNameUnit(previous) || previous == SqlLexer.hash || previous == atSign { return false } + } + let next = text.character(at: index + 1) + return isNameUnit(next) || next == SqlLexer.hash + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/LibPQSystemTrustRule.swift b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/LibPQSystemTrustRule.swift new file mode 100644 index 0000000000..d9b8f9c169 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/LibPQSystemTrustRule.swift @@ -0,0 +1,32 @@ +// +// LibPQSystemTrustRule.swift +// TableProTLSClientIdentity +// + +import Foundation + +// One rule for the driver and for pg_dump and pg_restore, so a session and its backup trust the same roots. +public enum LibPQSystemTrustRule { + // Verify Identity with no CA file, over TCP, with no root file the user named for libpq. + public static func usesSystemTrustAnchors( + verifiesHostname: Bool, + caCertificatePath: String, + host: String, + environment: [String: String], + fileExists: (String) -> Bool + ) -> Bool { + // libpq opens no TLS on a socket, and an empty host is its default socket. + guard verifiesHostname, caCertificatePath.isEmpty, !host.isEmpty, !host.hasPrefix("/") else { return false } + if let named = environment["PGSSLROOTCERT"], !named.isEmpty { return false } + return !fileExists(defaultRootCertificatePath(environment: environment)) + } + + // libpq reads HOME first and the password database only when it is unset or empty. + public static func defaultRootCertificatePath(environment: [String: String]) -> String { + let home = environment["HOME"].flatMap { $0.isEmpty ? nil : $0 } ?? NSHomeDirectory() + return URL(fileURLWithPath: home, isDirectory: true) + .appendingPathComponent(".postgresql", isDirectory: true) + .appendingPathComponent("root.crt") + .path + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/SystemTrustAnchors.swift b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/SystemTrustAnchors.swift new file mode 100644 index 0000000000..ec8d75984d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/SystemTrustAnchors.swift @@ -0,0 +1,284 @@ +#if os(macOS) +import Foundation +import os +import Security + +public enum SystemTrustDomain: Sendable { + case user + case admin + case system + + var securityDomain: SecTrustSettingsDomain { + switch self { + case .user: .user + case .admin: .admin + case .system: .system + } + } +} + +public enum SystemTrustResult: Sendable, Equatable { + case trustRoot + case trustAsRoot + case deny + case unspecified +} + +public struct SystemTrustSetting: Sendable, Equatable { + public var appliesToSSL: Bool + public var namesApplication: Bool + public var hostname: String? + // Nil when the entry carries no result, which Apple reads as TrustRoot. + public var result: SystemTrustResult? + + public init( + appliesToSSL: Bool = true, + namesApplication: Bool = false, + hostname: String? = nil, + result: SystemTrustResult? = nil + ) { + self.appliesToSSL = appliesToSSL + self.namesApplication = namesApplication + self.hostname = hostname + self.result = result + } +} + +public protocol SystemTrustSource { + // Empty when the domain has no trust settings. + func certificates(in domain: SystemTrustDomain) throws -> [SecCertificate] + // Nil when the domain holds no settings for the certificate. + func settings(for certificate: SecCertificate, in domain: SystemTrustDomain) throws -> [SystemTrustSetting]? + func anchorCertificates() throws -> [SecCertificate] +} + +public enum SystemTrustAnchorsError: Error, Sendable, Equatable { + case settingsUnreadable(OSStatus) + case noTrustedRoots + case exportFailed(OSStatus) + case fileUnwritable +} + +extension SystemTrustAnchorsError: LocalizedError { + public var message: String { + String( + localized: """ + Could not read this Mac's trusted certificates to verify the server. Choose the server's CA \ + certificate under SSL, then connect again. + """ + ) + } + + public var errorDescription: String? { + message + } +} + +// For a TLS library with no trust store of its own, such as the bundled OpenSSL. The roots come from the trust +// settings, never from evaluating each root alone: a SHA-1 self-signed root fails as a leaf and still anchors chains. +public enum SystemTrustAnchors { + public static let shared = SystemTrustAnchorCache(source: SecuritySystemTrustSource(), clock: ContinuousClock()) + + static let logger = Logger(subsystem: "com.TablePro", category: "SystemTrustAnchors") + + // Per-user settings override admin ones, which override the system's. The first domain with a result decides. + public static func trustedRoots(from source: any SystemTrustSource) throws -> [SecCertificate] { + var results: [Data: SystemTrustResult] = [:] + var decided: [(der: Data, certificate: SecCertificate)] = [] + for domain in [SystemTrustDomain.user, .admin, .system] { + let (certificates, fromAnchorList) = try certificates(in: domain, from: source) + for certificate in certificates { + let der = SecCertificateCopyData(certificate) as Data + guard results[der] == nil else { continue } + let result: SystemTrustResult? = fromAnchorList ? .trustRoot : decision(for: certificate, in: domain, from: source) + guard let result else { continue } + results[der] = result + decided.append((der, certificate)) + } + } + return decided.filter { results[$0.der] == .trustRoot || results[$0.der] == .trustAsRoot }.map(\.certificate) + } + + public static func pem(from source: any SystemTrustSource) throws -> Data { + let roots = try trustedRoots(from: source) + guard !roots.isEmpty else { throw SystemTrustAnchorsError.noTrustedRoots } + var exported: CFData? + let status = SecItemExport(roots as CFArray, .formatPEMSequence, [], nil, &exported) + guard status == errSecSuccess, let exported, CFDataGetLength(exported) > 0 else { + logger.error("Exporting the trusted roots failed: \(status, privacy: .public)") + throw SystemTrustAnchorsError.exportFailed(status) + } + return exported as Data + } + + static func effectiveResult(of settings: [SystemTrustSetting]) -> SystemTrustResult? { + guard !settings.isEmpty else { return .trustRoot } + for setting in settings where setting.appliesToSSL && !setting.namesApplication { + let result = setting.result ?? .trustRoot + // A PEM cannot carry a hostname limit, so a scoped trust is left out and a scoped deny covers every host. + if setting.hostname != nil { + if result == .deny { return .deny } + continue + } + if result != .unspecified { return result } + } + return nil + } + + // The system domain is the one that must answer. If it cannot, the anchors macOS ships stand in for it. + private static func certificates( + in domain: SystemTrustDomain, + from source: any SystemTrustSource + ) throws -> (certificates: [SecCertificate], fromAnchorList: Bool) { + do { + let certificates = try source.certificates(in: domain) + if domain == .system, certificates.isEmpty { + logger.error("The system trust settings list no certificates; using the system anchors") + return (try source.anchorCertificates(), true) + } + return (certificates, false) + } catch { + logger.error( + "Reading the \(String(describing: domain), privacy: .public) trust settings failed: \(String(describing: error), privacy: .private)" + ) + guard domain == .system else { throw error } + return (try source.anchorCertificates(), true) + } + } + + // A setting that cannot be read could be a deny, so it counts as one. + private static func decision( + for certificate: SecCertificate, + in domain: SystemTrustDomain, + from source: any SystemTrustSource + ) -> SystemTrustResult? { + do { + guard let settings = try source.settings(for: certificate, in: domain) else { return nil } + return effectiveResult(of: settings) + } catch { + logger.error("A certificate's \(String(describing: domain), privacy: .public) trust settings could not be read") + return .deny + } + } +} + +// The bytes are kept for a minute, so a root just trusted or distrusted in Keychain Access applies on a later connect +// without a relaunch. A failed build is never kept. +public final class SystemTrustAnchorCache: @unchecked Sendable where C.Duration == Duration { + private let source: any SystemTrustSource + private let clock: C + private let lifetime: Duration + private let lock = NSLock() + private var cached: (pem: Data, builtAt: C.Instant)? + + public init(source: any SystemTrustSource, clock: C, lifetime: Duration = .seconds(60)) { + self.source = source + self.clock = clock + self.lifetime = lifetime + } + + public func pem() throws -> Data { + try lock.withLock { + if let cached, cached.builtAt.duration(to: clock.now) < lifetime { + return cached.pem + } + let built = try SystemTrustAnchors.pem(from: source) + cached = (built, clock.now) + return built + } + } + + public func writeFile(in parent: URL = FileManager.default.temporaryDirectory) throws -> SystemTrustAnchorFile { + try SystemTrustAnchorFile.write(pem(), in: parent) + } +} + +// One file per use, in a folder only this user can open, removed when the use ends. +public struct SystemTrustAnchorFile: Sendable { + public let url: URL + let directory: URL + + static func write(_ pem: Data, in parent: URL) throws -> SystemTrustAnchorFile { + let directory = parent.appendingPathComponent("tablepro-trust-\(UUID().uuidString)", isDirectory: true) + let url = directory.appendingPathComponent("roots.pem") + do { + try FileManager.default.createDirectory( + at: directory, + withIntermediateDirectories: false, + attributes: [.posixPermissions: 0o700] + ) + guard FileManager.default.createFile( + atPath: url.path, + contents: pem, + attributes: [.posixPermissions: 0o600] + ) else { + throw SystemTrustAnchorsError.fileUnwritable + } + } catch { + try? FileManager.default.removeItem(at: directory) + SystemTrustAnchors.logger.error("Writing the trusted roots file failed") + throw SystemTrustAnchorsError.fileUnwritable + } + return SystemTrustAnchorFile(url: url, directory: directory) + } + + public func remove() { + try? FileManager.default.removeItem(at: directory) + } +} + +public struct SecuritySystemTrustSource: SystemTrustSource, Sendable { + public init() {} + + public func certificates(in domain: SystemTrustDomain) throws -> [SecCertificate] { + var certificates: CFArray? + let status = SecTrustSettingsCopyCertificates(domain.securityDomain, &certificates) + if status == errSecNoTrustSettings { return [] } + guard status == errSecSuccess else { throw SystemTrustAnchorsError.settingsUnreadable(status) } + return (certificates as? [SecCertificate]) ?? [] + } + + public func settings(for certificate: SecCertificate, in domain: SystemTrustDomain) throws -> [SystemTrustSetting]? { + var settings: CFArray? + let status = SecTrustSettingsCopyTrustSettings(certificate, domain.securityDomain, &settings) + if status == errSecItemNotFound || status == errSecNoTrustSettings { return nil } + guard status == errSecSuccess else { throw SystemTrustAnchorsError.settingsUnreadable(status) } + let entries = (settings as? [[String: Any]]) ?? [] + return entries.map(Self.setting(from:)) + } + + public func anchorCertificates() throws -> [SecCertificate] { + var anchors: CFArray? + let status = SecTrustCopyAnchorCertificates(&anchors) + guard status == errSecSuccess else { throw SystemTrustAnchorsError.settingsUnreadable(status) } + return (anchors as? [SecCertificate]) ?? [] + } + + public static func setting(from entry: [String: Any]) -> SystemTrustSetting { + let hostname = entry[kSecTrustSettingsPolicyString as String] as? String + return SystemTrustSetting( + appliesToSSL: entry[kSecTrustSettingsPolicy as String].map(isSSLPolicy) ?? true, + namesApplication: entry[kSecTrustSettingsApplication as String] != nil, + hostname: hostname?.isEmpty == false ? hostname : nil, + result: (entry[kSecTrustSettingsResult as String] as? NSNumber).map { result(of: $0.uint32Value) } + ) + } + + private static func isSSLPolicy(_ value: Any) -> Bool { + guard CFGetTypeID(value as CFTypeRef) == SecPolicyGetTypeID() else { return false } + let policy = unsafeDowncast(value as AnyObject, to: SecPolicy.self) + let properties = SecPolicyCopyProperties(policy) as? [String: Any] + return properties?[kSecPolicyOid as String] as? String == kSecPolicyAppleSSL as String + } + + // An unknown value could mean distrust, so it counts as a deny. + private static func result(of rawValue: UInt32) -> SystemTrustResult { + switch SecTrustSettingsResult(rawValue: rawValue) { + case .trustRoot?: .trustRoot + case .trustAsRoot?: .trustAsRoot + case .unspecified?: .unspecified + default: .deny + } + } +} +#endif diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLSessionStateClassifierTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLSessionStateClassifierTests.swift new file mode 100644 index 0000000000..20c3f6bb9d --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLSessionStateClassifierTests.swift @@ -0,0 +1,409 @@ +import Foundation +@testable import TableProSQLGrammar +import Testing + +struct SQLSessionStateClassifierTests { + private static func grammar(_ engine: String) -> SQLLexicalGrammar { + SQLLexicalReadings.resolve(databaseTypeId: engine, declared: nil, session: nil).execution + } + + private static let engines: [SQLSessionDialect: String] = [ + .postgres: "PostgreSQL", .mySQL: "MySQL", .sqlServer: "SQL Server", .oracle: "Oracle", .sqlite: "SQLite", + .duckDB: "DuckDB", + ] + + private func effect(_ sql: String, _ dialect: SQLSessionDialect) -> SQLSessionStateEffect { + SQLSessionStateClassifier.effect(of: sql, dialect: dialect, grammar: Self.grammar(Self.engines[dialect] ?? "")) + } + + private func creates(_ sql: String, _ dialect: SQLSessionDialect) -> SQLSessionStateKinds { + effect(sql, dialect).creates + } + + // MARK: - PostgreSQL + + @Test("PostgreSQL temporary objects, by marker, by pg_temp and by SELECT INTO", arguments: [ + "CREATE TEMP TABLE t (id int)", + "create temporary table t (id int)", + "CREATE GLOBAL TEMPORARY TABLE t (id int)", + "CREATE OR REPLACE TEMP VIEW v AS SELECT 1", + "CREATE TEMP SEQUENCE s", + "CREATE TEMP TABLE t (id int) ON COMMIT DELETE ROWS", + "CREATE TABLE pg_temp.t (id int)", + "CREATE TABLE IF NOT EXISTS pg_temp.t (id int)", + "CREATE FUNCTION pg_temp.f() RETURNS int AS $$ SELECT 1 $$ LANGUAGE sql", + "SELECT * INTO TEMP t FROM orders", + "SELECT * INTO TEMPORARY TABLE t FROM orders", + ]) + func postgresTemporaryObjects(sql: String) { + #expect(creates(sql, .postgres) == .temporaryObjects) + } + + @Test("PostgreSQL tables that outlive the session or end with the transaction are not session state", arguments: [ + "CREATE TABLE t (id int)", + "CREATE TABLE temp (id int)", + "CREATE UNLOGGED TABLE t (id int)", + "CREATE TEMP TABLE t (id int) ON COMMIT DROP", + "SELECT * INTO archive FROM orders", + ]) + func postgresLastingTables(sql: String) { + #expect(creates(sql, .postgres).isEmpty) + } + + @Test("PostgreSQL SET, SET SESSION and a session set_config are session settings", arguments: [ + "SET search_path TO app", + "SET SESSION statement_timeout = 0", + "SET ROLE reporting", + "SET SESSION AUTHORIZATION alice", + "SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY", + "SET app.key = 'secret'", + "SELECT set_config('app.user', '42', false)", + "SELECT pg_catalog.set_config('app.user', '42', false)", + ]) + func postgresSettings(sql: String) { + #expect(creates(sql, .postgres) == .sessionSettings) + } + + @Test("PostgreSQL settings that last one transaction are not session state", arguments: [ + "SET LOCAL search_path TO app", + "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE", + "SET CONSTRAINTS ALL DEFERRED", + "SELECT set_config('app.user', '42', true)", + ]) + func postgresTransactionSettings(sql: String) { + #expect(creates(sql, .postgres).isEmpty) + } + + @Test("PostgreSQL prepared statements, held cursors, session advisory locks and LISTEN") + func postgresOtherKinds() { + #expect(creates("PREPARE q (int) AS SELECT $1", .postgres) == .preparedStatements) + #expect(creates("PREPARE TRANSACTION 'tx1'", .postgres).isEmpty) + #expect(creates("DECLARE c CURSOR WITH HOLD FOR SELECT 1", .postgres) == .heldCursors) + #expect(creates("DECLARE c CURSOR FOR SELECT 1", .postgres).isEmpty) + #expect(creates("DECLARE c CURSOR WITHOUT HOLD FOR SELECT 1", .postgres).isEmpty) + #expect(creates("SELECT pg_advisory_lock(1)", .postgres) == .sessionLocks) + #expect(creates("SELECT pg_try_advisory_lock_shared(1, 2)", .postgres) == .sessionLocks) + #expect(creates("SELECT pg_advisory_xact_lock(1)", .postgres).isEmpty) + #expect(creates("LISTEN jobs", .postgres) == .listeners) + } + + @Test("DISCARD ALL clears every kind; DISCARD TEMP, DEALLOCATE ALL and UNLISTEN * clear their own") + func postgresClears() { + #expect(effect("DISCARD ALL", .postgres).clears == .all) + #expect(effect("DISCARD TEMP", .postgres).clears == .temporaryObjects) + #expect(effect("discard temporary", .postgres).clears == .temporaryObjects) + #expect(effect("DEALLOCATE ALL", .postgres).clears == .preparedStatements) + #expect(effect("DEALLOCATE PREPARE ALL", .postgres).clears == .preparedStatements) + #expect(effect("UNLISTEN *", .postgres).clears == .listeners) + #expect(effect("UNLISTEN jobs", .postgres) == .none) + #expect(effect("RESET search_path", .postgres) == .none) + #expect(effect("RESET ALL", .postgres) == .none) + #expect(effect("DISCARD PLANS", .postgres) == .none) + } + + @Test("RESET ALL keeps the role and the session authorization, so it clears no settings") + func resetAllKeepsTheRole() { + let grammar = Self.grammar("PostgreSQL") + let effect = SQLSessionStateClassifier.effect( + ofStatements: ["SET ROLE app_admin", "RESET ALL"], dialect: .postgres, grammar: grammar + ) + #expect(effect.applied(to: []) == .sessionSettings) + let discarded = SQLSessionStateClassifier.effect( + ofStatements: ["SET ROLE app_admin", "DISCARD ALL"], dialect: .postgres, grammar: grammar + ) + #expect(discarded.applied(to: []).isEmpty) + } + + @Test("Statements apply in order, so what a later one clears goes and what it sets up stays") + func statementsApplyInOrder() { + let grammar = Self.grammar("PostgreSQL") + let cleared = SQLSessionStateClassifier.effect( + ofStatements: ["CREATE TEMP TABLE t (id int)", "DISCARD ALL"], dialect: .postgres, grammar: grammar + ) + #expect(cleared.applied(to: []).isEmpty) + let reset = SQLSessionStateClassifier.effect( + ofStatements: ["DISCARD ALL", "SET search_path TO app"], dialect: .postgres, grammar: grammar + ) + #expect(reset.applied(to: [.temporaryObjects, .listeners]) == .sessionSettings) + let partly = SQLSessionStateClassifier.effect( + ofStatements: ["CREATE TEMP TABLE t (id int); SET search_path TO app", "DISCARD TEMP"], + dialect: .postgres, + grammar: grammar + ) + #expect(partly.applied(to: []) == .sessionSettings) + } + + @Test("Text inside a comment, a literal, a DO body or a procedure call never counts", arguments: [ + "SELECT 'SET search_path TO x'", + "-- SET search_path TO x\nSELECT 1", + "SELECT 1 /* CREATE TEMP TABLE t (id int) */", + "DO $$ BEGIN CREATE TEMP TABLE t (id int); END $$", + "CALL set_up_session()", + "SELECT $$pg_advisory_lock(1)$$", + ]) + func postgresUnseenText(sql: String) { + #expect(effect(sql, .postgres) == .none) + } + + @Test("The effect carries kinds only, never a value the statement held") + func effectHoldsNoText() { + let noted = effect("SET app.key = 'secret'", .postgres) + #expect(noted.creates == .sessionSettings) + #expect(!String(describing: noted).contains("secret")) + #expect(!String(describing: noted).contains("app.key")) + } + + @Test("A statement equal to a startup command is skipped, one that differs still counts") + func startupStatementsAreSkipped() { + let grammar = Self.grammar("PostgreSQL") + let skipped = SQLSessionStateClassifier.effect( + ofStatements: ["SET search_path TO app;", "CREATE TEMP TABLE t (id int)"], + dialect: .postgres, + grammar: grammar, + skipping: [" SET search_path TO app ;\n"] + ) + #expect(skipped.creates == .temporaryObjects) + let differs = SQLSessionStateClassifier.effect( + ofStatements: ["SET search_path TO other"], + dialect: .postgres, + grammar: grammar, + skipping: ["SET search_path TO app"] + ) + #expect(differs.creates == .sessionSettings) + } + + // MARK: - MySQL and MariaDB + + @Test("MySQL temporary tables", arguments: [ + "CREATE TEMPORARY TABLE t (id int)", + "CREATE OR REPLACE TEMPORARY TABLE t (id int)", + "CREATE TEMPORARY TABLE IF NOT EXISTS t SELECT 1", + ]) + func mySQLTemporaryTables(sql: String) { + #expect(creates(sql, .mySQL) == .temporaryObjects) + } + + @Test("MySQL session system variables, NAMES, CHARACTER SET, SESSION TRANSACTION and ROLE", arguments: [ + "SET sql_mode = ''", + "SET autocommit = 0", + "SET SESSION sql_mode = ''", + "SET LOCAL sql_mode = ''", + "SET @@sql_mode = ''", + "SET @@SESSION.sql_mode = ''", + "SET NAMES utf8mb4", + "SET CHARACTER SET utf8mb4", + "SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED", + "SET ROLE reporting", + "SET GLOBAL max_connections = 500, SESSION sql_mode = ''", + "SET GLOBAL max_connections = 500, @@SESSION.sql_mode = ''", + "/*!40101 SET NAMES utf8mb4 */", + ]) + func mySQLSettings(sql: String) { + #expect(creates(sql, .mySQL) == .sessionSettings) + } + + @Test("MySQL global settings, account changes and next-transaction settings are not session state", arguments: [ + "SET GLOBAL max_connections = 500", + "SET GLOBAL max_connections = 500, wait_timeout = 60", + "SET PERSIST max_connections = 500", + "SET PERSIST_ONLY max_connections = 500", + "SET @@GLOBAL.max_connections = 500", + "SET PASSWORD = 'x'", + "SET PASSWORD FOR app = 'x'", + "SET DEFAULT ROLE ALL TO app", + "SET TRANSACTION ISOLATION LEVEL READ COMMITTED", + "SET GLOBAL TRANSACTION READ ONLY", + "SET STATEMENT max_statement_time = 1 FOR SELECT 1", + "CREATE TABLE t (id int)", + ]) + func mySQLLastingChanges(sql: String) { + #expect(creates(sql, .mySQL).isEmpty) + } + + @Test("MySQL user variables are set by SET, := and INTO", arguments: [ + "SET @id = 42", + "SELECT @id := 42", + "SELECT id INTO @id FROM t LIMIT 1", + "SELECT id FROM t LIMIT 1 INTO @id", + "SELECT 1 INTO@id", + "SELECT@id:=1", + "SELECT @id /* note */ := 1", + "/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE */", + ]) + func mySQLUserVariables(sql: String) { + #expect(creates(sql, .mySQL) == .userVariables) + } + + @Test("MySQL prepared statements and session locks, and what only reads") + func mySQLOtherKinds() { + #expect(creates("PREPARE s FROM 'SELECT 1'", .mySQL) == .preparedStatements) + #expect(creates("SELECT GET_LOCK('job', 10)", .mySQL) == .sessionLocks) + #expect(creates("LOCK TABLES t WRITE", .mySQL) == .sessionLocks) + #expect(creates("SET @got = GET_LOCK('job', 10)", .mySQL) == [.userVariables, .sessionLocks]) + #expect(creates("SELECT @id", .mySQL).isEmpty) + #expect(creates("SELECT @@version", .mySQL).isEmpty) + #expect(creates("SELECT 'SET NAMES utf8mb4'", .mySQL).isEmpty) + #expect(creates("# SET NAMES latin1\nSELECT 1", .mySQL).isEmpty) + #expect(creates("CALL set_up_session()", .mySQL).isEmpty) + } + + // MARK: - SQL Server + + @Test("SQL Server # and ## tables, SELECT INTO a # table and a temporary procedure", arguments: [ + "CREATE TABLE #orders (id int)", + "CREATE TABLE ##orders (id int)", + "SELECT * INTO #orders FROM dbo.orders", + "CREATE PROCEDURE #p AS SELECT 1", + "SELECT 1\nCREATE TABLE #t (id int)", + "CREATE TABLE [#orders] (id int)", + "CREATE TABLE \"#orders\" (id int)", + "SELECT * INTO [#orders] FROM dbo.orders", + "CREATE TABLE #tëmp (id int)", + ]) + func sqlServerTemporaryObjects(sql: String) { + #expect(creates(sql, .sqlServer) == .temporaryObjects) + } + + @Test("SQL Server SET options, EXECUTE AS and session context", arguments: [ + "SET NOCOUNT ON", + "SET XACT_ABORT ON", + "SET TRANSACTION ISOLATION LEVEL SNAPSHOT", + "SELECT 1 SET NOCOUNT ON", + "EXECUTE AS USER = 'auditor'", + "EXEC sp_set_session_context 'tenant', 42", + "EXEC sys.sp_set_session_context @key = N'tenant', @value = 42", + "sp_set_session_context 'tenant', 42", + "EXEC sp_setapprole 'reporting', 'secret'", + "EXEC sys.sp_setapprole @rolename = 'reporting', @password = 'secret'", + "sp_setapprole 'reporting', 'secret'", + ]) + func sqlServerSettings(sql: String) { + #expect(creates(sql, .sqlServer) == .sessionSettings) + } + + @Test("SQL Server variables, procedure calls, dynamic SQL and permanent tables are not session state", arguments: [ + "SET @n = 1", + "DECLARE @n int; SET @n = 1", + "UPDATE t SET a = 1", + "EXEC dbo.rebuild", + "EXEC('SET NOCOUNT ON')", + "CREATE TABLE orders (id int)", + "SELECT * INTO archive FROM orders", + "INSERT INTO #orders VALUES (1)", + ]) + func sqlServerNotSessionState(sql: String) { + #expect(creates(sql, .sqlServer).isEmpty) + } + + @Test("SQL Server global cursors and application locks") + func sqlServerCursorsAndLocks() { + #expect(creates("DECLARE c CURSOR FOR SELECT 1", .sqlServer) == .heldCursors) + #expect(creates("DECLARE c CURSOR GLOBAL FOR SELECT 1", .sqlServer) == .heldCursors) + #expect(creates("DECLARE c INSENSITIVE CURSOR FOR SELECT 1", .sqlServer) == .heldCursors) + #expect(creates("DECLARE c CURSOR LOCAL FOR SELECT 1", .sqlServer).isEmpty) + #expect(creates("DECLARE @c CURSOR", .sqlServer).isEmpty) + #expect(creates("EXEC sp_getapplock @Resource = 'job', @LockMode = 'Exclusive', @LockOwner = 'Session'", .sqlServer) + == .sessionLocks) + #expect(creates("EXEC @rc = sp_getapplock 'job', 'Exclusive', 'Session'", .sqlServer) == .sessionLocks) + } + + // MARK: - Oracle, SQLite and DuckDB + + @Test("Oracle ALTER SESSION, SET ROLE and a private temporary table that keeps its definition") + func oracleKinds() { + #expect(creates("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD'", .oracle) == .sessionSettings) + #expect(creates("ALTER SESSION SET CURRENT_SCHEMA = app", .oracle) == .sessionSettings) + #expect(creates("ALTER SESSION ENABLE PARALLEL DML", .oracle) == .sessionSettings) + #expect(creates("SET ROLE reporting", .oracle) == .sessionSettings) + #expect(creates("SET TRANSACTION READ ONLY", .oracle).isEmpty) + #expect( + creates("CREATE PRIVATE TEMPORARY TABLE ora$ptt_t (id NUMBER) ON COMMIT PRESERVE DEFINITION", .oracle) + == .temporaryObjects + ) + #expect(creates("CREATE PRIVATE TEMPORARY TABLE ora$ptt_t (id NUMBER) ON COMMIT DROP DEFINITION", .oracle).isEmpty) + #expect(creates("CREATE GLOBAL TEMPORARY TABLE t (id NUMBER) ON COMMIT PRESERVE ROWS", .oracle).isEmpty) + #expect(creates("BEGIN DBMS_SESSION.SET_ROLE('reporting'); END;", .oracle).isEmpty) + } + + @Test("SQLite and DuckDB temporary objects, by marker and by the temp schema") + func embeddedTemporaryObjects() { + #expect(creates("CREATE TEMP TABLE t (id INTEGER)", .sqlite) == .temporaryObjects) + #expect(creates("CREATE TEMPORARY VIEW v AS SELECT 1", .sqlite) == .temporaryObjects) + #expect(creates("CREATE TABLE temp.t (id INTEGER)", .sqlite) == .temporaryObjects) + #expect(creates("CREATE TABLE t (id INTEGER)", .sqlite).isEmpty) + #expect(creates("CREATE TABLE temp (id INTEGER)", .sqlite).isEmpty) + #expect(creates("CREATE TEMP TABLE t (id INTEGER)", .duckDB) == .temporaryObjects) + #expect(creates("CREATE OR REPLACE TEMP MACRO m(a) AS a + 1", .duckDB) == .temporaryObjects) + #expect(creates("CREATE TABLE temp.t (id INTEGER)", .duckDB) == .temporaryObjects) + } + + @Test("DuckDB SET without GLOBAL is a session setting, SET VARIABLE a variable") + func duckDBSettings() { + #expect(creates("SET threads = 4", .duckDB) == .sessionSettings) + #expect(creates("SET SESSION threads = 4", .duckDB) == .sessionSettings) + #expect(creates("SET VARIABLE x = 42", .duckDB) == .userVariables) + #expect(creates("SET GLOBAL threads = 4", .duckDB).isEmpty) + } + + // MARK: - Effects + + @Test("A later clear removes what an earlier statement set up, a later setup survives an earlier clear") + func effectComposition() { + let setUp = SQLSessionStateEffect(creates: .temporaryObjects) + let discard = SQLSessionStateEffect(clears: .all) + #expect(setUp.followed(by: discard) == SQLSessionStateEffect(creates: [], clears: .all)) + let settings = SQLSessionStateEffect(creates: .sessionSettings) + #expect(discard.followed(by: settings).applied(to: .userVariables) == .sessionSettings) + } + + @Test("A statement longer than the classified head is read only to there") + func longStatementIsReadToItsHead() { + let values = String(repeating: "a", count: 100_000) + #expect(creates("INSERT INTO t VALUES ('\(values)')", .mySQL).isEmpty) + #expect(creates("SET @id = 42, @blob = '\(values)'", .mySQL) == .userVariables) + } + + @Test("A text is cut to its head before it is split, so a setup past it in the same text is not read") + func textIsCutBeforeItIsSplit() { + let values = String(repeating: "a", count: 70_000) + #expect(creates("SET NOCOUNT ON\nINSERT INTO t VALUES ('\(values)')", .sqlServer) == .sessionSettings) + #expect(creates("INSERT INTO t VALUES ('\(values)'); SET NOCOUNT ON", .sqlServer).isEmpty) + let grammar = Self.grammar("SQL Server") + let separate = SQLSessionStateClassifier.effect( + ofStatements: ["INSERT INTO t VALUES ('\(values)')", "SET NOCOUNT ON"], dialect: .sqlServer, grammar: grammar + ) + #expect(separate.creates == .sessionSettings) + } + + @Test("The quick check passes over data words and addresses", arguments: [ + ("SELECT * FROM t ORDER BY id LIMIT 10 OFFSET 20", SQLSessionDialect.postgres), + ("INSERT INTO t VALUES ('template', 'asset', 'block', 'clock')", .postgres), + ("INSERT INTO users VALUES ('ana@example.com', 'bo.li@example.org')", .mySQL), + ("INSERT INTO t VALUES ('reset', 'offset', 'blocked')", .mySQL), + ("INSERT INTO t VALUES ('Order#5', 'a#b')", .sqlServer), + ("SELECT * FROM sessions WHERE role_id = 1", .oracle), + ("SELECT pg_advisory_xact_lock(1)", .postgres), + ]) + func quickCheckSkipsData(sql: String, dialect: SQLSessionDialect) { + #expect(!SQLSessionStateQuickCheck.mayChangeSession(sql, dialect: dialect)) + } + + @Test("The quick check keeps every form a rule reads, however it is spelled", arguments: [ + ("set search_path to app", SQLSessionDialect.postgres), + ("SELECT * INTO pg_temp_3.t FROM orders", .postgres), + ("SELECT 1SET", .postgres), + ("SELECT 1 INTO@v", .mySQL), + ("SELECT 1 INTO /* v */ @v", .mySQL), + ("SELECT@v/* note */:=1", .mySQL), + ("SELECT @`v` := 1", .mySQL), + ("/*!40101 SET NAMES utf8mb4 */", .mySQL), + ("CREATE TABLE [#t] (id int)", .sqlServer), + ("SELECT * INTO ##t FROM orders", .sqlServer), + ("sp_setapprole 'reporting', 'secret'", .sqlServer), + ("ALTER SESSION SET CURRENT_SCHEMA = app", .oracle), + ("CREATE TABLE temp.t (id INTEGER)", .sqlite), + ]) + func quickCheckKeepsEveryRule(sql: String, dialect: SQLSessionDialect) { + #expect(SQLSessionStateQuickCheck.mayChangeSession(sql, dialect: dialect)) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/SystemTrustAnchorsTests.swift b/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/SystemTrustAnchorsTests.swift new file mode 100644 index 0000000000..47418aa408 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/SystemTrustAnchorsTests.swift @@ -0,0 +1,288 @@ +#if os(macOS) +import Foundation +import Security +import TableProTestSupport +import TableProTLSClientIdentity +import TableProTLSTestFixtures +import Testing + +private final class FixtureTrustSource: SystemTrustSource { + struct Entry { + let certificate: SecCertificate + let settings: [SystemTrustSetting]? + var unreadable = false + } + + var domains: [SystemTrustDomain: [Entry]] = [:] + var unreadableDomains: Set = [] + var anchors: [SecCertificate] = [] + private(set) var userDomainReads = 0 + + func add(_ certificate: SecCertificate, to domain: SystemTrustDomain, _ settings: [SystemTrustSetting]?) { + domains[domain, default: []].append(Entry(certificate: certificate, settings: settings)) + } + + func certificates(in domain: SystemTrustDomain) throws -> [SecCertificate] { + if domain == .user { userDomainReads += 1 } + guard !unreadableDomains.contains(domain) else { throw SystemTrustAnchorsError.settingsUnreadable(-1) } + return domains[domain, default: []].map(\.certificate) + } + + func settings(for certificate: SecCertificate, in domain: SystemTrustDomain) throws -> [SystemTrustSetting]? { + let der = SecCertificateCopyData(certificate) as Data + guard let entry = domains[domain, default: []].first(where: { SecCertificateCopyData($0.certificate) as Data == der }) + else { return nil } + guard !entry.unreadable else { throw SystemTrustAnchorsError.settingsUnreadable(-1) } + return entry.settings + } + + func anchorCertificates() throws -> [SecCertificate] { + anchors + } +} + +struct SystemTrustAnchorsTests { + private let rootA: SecCertificate + private let rootB: SecCertificate + + init() throws { + rootA = try #require(SecCertificateCreateWithData(nil, TLSTestFixtures.der(TLSTestFixtures.caCertificate) as CFData)) + rootB = try #require( + SecCertificateCreateWithData(nil, TLSTestFixtures.der(TLSTestFixtures.serverNameCACertificate) as CFData) + ) + } + + private func ders(_ certificates: [SecCertificate]) -> Set { + Set(certificates.map { SecCertificateCopyData($0) as Data }) + } + + private func exported(_ source: FixtureTrustSource) throws -> Set { + ders(try SystemTrustAnchors.trustedRoots(from: source)) + } + + @Test("A certificate whose trust settings are empty is trusted as a root") + func emptySettingsTrustTheRoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + + #expect(try exported(source) == ders([rootA])) + } + + @Test("A user Deny removes a root the system trusts") + func aUserDenyRemovesASystemRoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootB, to: .system, []) + source.add(rootA, to: .user, [SystemTrustSetting(result: .deny)]) + + #expect(try exported(source) == ders([rootB])) + } + + @Test("A root trusted in the admin domain joins the system roots") + func anAdminTrustAddsARoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootB, to: .admin, [SystemTrustSetting(result: .trustRoot)]) + + #expect(try exported(source) == ders([rootA, rootB])) + } + + @Test("User settings override admin settings in both directions") + func userSettingsOverrideAdminSettings() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .admin, [SystemTrustSetting(result: .deny)]) + source.add(rootA, to: .user, [SystemTrustSetting(result: .trustRoot)]) + source.add(rootB, to: .admin, [SystemTrustSetting(result: .trustRoot)]) + source.add(rootB, to: .user, [SystemTrustSetting(result: .deny)]) + + #expect(try exported(source) == ders([rootA])) + } + + @Test("A setting for another policy or for one application does not apply") + func anEntryForAnotherPolicyDoesNotApply() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootA, to: .user, [SystemTrustSetting(appliesToSSL: false, result: .deny)]) + source.add(rootB, to: .user, [SystemTrustSetting(namesApplication: true, result: .trustRoot)]) + + #expect(try exported(source) == ders([rootA])) + } + + @Test("A trust limited to one hostname is left out, because a PEM file cannot carry the limit") + func aHostnameScopedTrustIsNotExported() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootB, to: .user, [SystemTrustSetting(hostname: "db.example.com", result: .trustRoot)]) + + #expect(try exported(source) == ders([rootA])) + } + + @Test("A Deny limited to one hostname removes the root for every host") + func aHostnameScopedDenyRemovesTheRoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootA, to: .user, [ + SystemTrustSetting(hostname: "db.example.com", result: .deny), + SystemTrustSetting(result: .trustRoot) + ]) + + #expect(try exported(source).isEmpty) + } + + @Test("Unspecified moves on to the next entry, then to the next domain") + func unspecifiedFallsThroughToTheNextDomain() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootA, to: .user, [SystemTrustSetting(result: .unspecified)]) + source.add(rootB, to: .user, [SystemTrustSetting(result: .unspecified)]) + + #expect(try exported(source) == ders([rootA])) + + source.add(rootB, to: .admin, [SystemTrustSetting(result: .unspecified), SystemTrustSetting()]) + #expect(try exported(source) == ders([rootA, rootB])) + } + + @Test("A settings array the domain cannot read counts as a Deny") + func unreadableSettingsFailClosed() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.domains[.user] = [FixtureTrustSource.Entry(certificate: rootA, settings: [], unreadable: true)] + + #expect(try exported(source).isEmpty) + } + + @Test("A user or admin domain that cannot be read fails the build") + func anUnreadableUserDomainIsAnError() { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.unreadableDomains = [.admin] + + #expect(throws: SystemTrustAnchorsError.settingsUnreadable(-1)) { + try SystemTrustAnchors.trustedRoots(from: source) + } + } + + @Test("The system anchors stand in when the system domain cannot be read, and a user Deny still wins") + func theSystemAnchorsAreTheFallback() throws { + let source = FixtureTrustSource() + source.unreadableDomains = [.system] + source.anchors = [rootA, rootB] + source.add(rootB, to: .user, [SystemTrustSetting(result: .deny)]) + + #expect(try exported(source) == ders([rootA])) + + source.unreadableDomains = [] + #expect(try exported(source) == ders([rootA])) + } + + @Test("No trusted root is an error, never an empty file") + func noAnchorsIsAnError() { + let source = FixtureTrustSource() + source.add(rootA, to: .system, [SystemTrustSetting(result: .deny)]) + + #expect(throws: SystemTrustAnchorsError.noTrustedRoots) { + try SystemTrustAnchors.pem(from: source) + } + } + + @Test("The PEM holds one block per trusted root") + func thePEMHoldsEveryRoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootB, to: .system, []) + + let pem = try #require(String(bytes: SystemTrustAnchors.pem(from: source), encoding: .utf8)) + + #expect(pem.components(separatedBy: "-----BEGIN CERTIFICATE-----").count == 3) + } + + @Test("This Mac's own trust settings export as PEM") + func theRealStoreExportsPEM() throws { + let source = SecuritySystemTrustSource() + + let roots = try SystemTrustAnchors.trustedRoots(from: source) + let pem = try #require(String(bytes: SystemTrustAnchors.pem(from: source), encoding: .utf8)) + + #expect(!roots.isEmpty) + #expect(pem.hasPrefix("-----BEGIN CERTIFICATE-----")) + } + + @Test("A trust settings entry is read for its policy, hostname, application and result") + func readsATrustSettingsEntry() { + let scoped = SecuritySystemTrustSource.setting(from: [ + kSecTrustSettingsPolicy as String: SecPolicyCreateSSL(true, nil), + kSecTrustSettingsPolicyString as String: "db.example.com", + kSecTrustSettingsResult as String: NSNumber(value: 3) + ]) + #expect(scoped == SystemTrustSetting(hostname: "db.example.com", result: .deny)) + + let otherPolicy = SecuritySystemTrustSource.setting(from: [kSecTrustSettingsPolicy as String: SecPolicyCreateBasicX509()]) + #expect(otherPolicy == SystemTrustSetting(appliesToSSL: false)) + + let application = SecuritySystemTrustSource.setting(from: [kSecTrustSettingsApplication as String: "app"]) + #expect(application == SystemTrustSetting(namesApplication: true)) + } + + @Test("A result value Apple does not define counts as a Deny", arguments: [0, 9]) + func anUnknownResultCountsAsADeny(value: Int) { + let setting = SecuritySystemTrustSource.setting(from: [kSecTrustSettingsResult as String: NSNumber(value: value)]) + #expect(setting.result == .deny) + } + + @Test("The bytes are reused for a minute, then rebuilt, and a failed build is not kept") + func theCacheRebuildsAfterAMinute() throws { + let source = FixtureTrustSource() + let clock = ManualClock() + let cache = SystemTrustAnchorCache(source: source, clock: clock) + + #expect(throws: SystemTrustAnchorsError.noTrustedRoots) { try cache.pem() } + source.add(rootA, to: .system, []) + let first = try cache.pem() + #expect(source.userDomainReads == 2) + + clock.advance(by: .seconds(59)) + #expect(try cache.pem() == first) + #expect(source.userDomainReads == 2) + + clock.advance(by: .seconds(1)) + _ = try cache.pem() + #expect(source.userDomainReads == 3) + } + + @Test("Each use gets its own owner-only file in an owner-only folder, removed with its folder") + func aWrittenFileIsOwnerOnly() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + let cache = SystemTrustAnchorCache(source: source, clock: ManualClock()) + let parent = FileManager.default.temporaryDirectory.appendingPathComponent("trust-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: parent, withIntermediateDirectories: false) + defer { try? FileManager.default.removeItem(at: parent) } + + let file = try cache.writeFile(in: parent) + let other = try cache.writeFile(in: parent) + let folder = file.url.deletingLastPathComponent() + + #expect(file.url != other.url) + #expect(try Data(contentsOf: file.url) == cache.pem()) + let folderMode = try FileManager.default.attributesOfItem(atPath: folder.path)[.posixPermissions] as? NSNumber + let fileMode = try FileManager.default.attributesOfItem(atPath: file.url.path)[.posixPermissions] as? NSNumber + #expect(folderMode?.int16Value == 0o700) + #expect(fileMode?.int16Value == 0o600) + + file.remove() + #expect(!FileManager.default.fileExists(atPath: folder.path)) + #expect(FileManager.default.fileExists(atPath: other.url.path)) + } + + @Test("A file that cannot be written is an error and leaves nothing behind") + func anUnwritableFolderIsAnError() { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + let cache = SystemTrustAnchorCache(source: source, clock: ManualClock()) + let missing = FileManager.default.temporaryDirectory.appendingPathComponent("missing-\(UUID().uuidString)") + + #expect(throws: SystemTrustAnchorsError.fileUnwritable) { try cache.writeFile(in: missing) } + #expect(!FileManager.default.fileExists(atPath: missing.path)) + } +} +#endif diff --git a/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift b/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift index 42d318e2c1..7a5792e4a7 100644 --- a/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift +++ b/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift @@ -63,6 +63,39 @@ struct JSONDocumentStreamingTests { return try JSONImportParsing.parseElements(at: url, targetTable: targetTable) } + // The rows each element becomes, nil for one that is not an object. + private func rows(_ elements: [Any]) -> [[String: PluginCellValue]?] { + elements.map { try? JSONImportParsing.row(from: $0) } + } + + private func importedRows(_ data: Data, chunkSize: Int = JSONDocumentReader.defaultChunkSize) throws -> [[String: PluginCellValue]] { + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + var reader = try JSONDocumentReader(url: url, chunkSize: chunkSize, checkCancellation: {}) + let layout = try JSONDocumentOutline.rows(in: &reader, targetTable: nil) { reader, _ in try reader.skipValue() } + switch layout { + case .array(let offset, _, _): + return try drain(JSONStreamedElements(reader: reader, arrayAt: offset)).map { try JSONImportParsing.row(from: $0) } + case .singleObject: + reader.close() + return try drain(JSONImportParsing.rowElements(at: url, targetTable: nil, checkCancellation: {}).elements) + .map { try JSONImportParsing.row(from: $0) } + } + } + + private static let encodings: [String.Encoding] = [.utf16BigEndian, .utf16LittleEndian, .utf32BigEndian, .utf32LittleEndian] + + // Foundation writes no byte order mark for an encoding that names its byte order. + private static func encoded(_ text: String, _ encoding: String.Encoding, withMark: Bool) -> Data { + let mark: [UInt8] = switch encoding { + case .utf16BigEndian: [0xFE, 0xFF] + case .utf16LittleEndian: [0xFF, 0xFE] + case .utf32BigEndian: [0x00, 0x00, 0xFE, 0xFF] + default: [0xFF, 0xFE, 0x00, 0x00] + } + return (withMark ? Data(mark) : Data()) + (text.data(using: encoding) ?? Data()) + } + @Test("Elements read one at a time equal a whole-file parse, whatever the chunk size", arguments: [1, 3, 64, 1 << 20]) func streamedElementsMatchAWholeFileParse(chunkSize: Int) throws { let array = "[\n " + Self.trickyRows.joined(separator: ",\n ") + "\n]" @@ -122,9 +155,15 @@ struct JSONDocumentStreamingTests { (#"[{"a":1},]"#, nil), (#"{"t":[{"a":1} , ],}"#, nil), ] as [(String, String?)]) + // A lone object streams as a built row, so it is compared as rows; an array's elements are compared as values. func shapesMatchAWholeFileParse(json: String, targetTable: String?) throws { let imported = try importedElements(json, targetTable: targetTable) - #expect((imported as NSArray).isEqual(to: try parsedElements(json, targetTable: targetTable))) + let parsed = try parsedElements(json, targetTable: targetTable) + if imported.contains(where: { $0 is [String: PluginCellValue] }) { + #expect(rows(imported) == rows(parsed)) + } else { + #expect((imported as NSArray).isEqual(to: parsed)) + } } // A whole-file parse hands the tables over as a dictionary, whose order changes from launch to launch. @@ -157,7 +196,7 @@ struct JSONDocumentStreamingTests { } } - @Test("A UTF-16 file still reads, through JSONSerialization's own decoding") + @Test("A UTF-16 file with a byte order mark streams its rows and fields") func utf16FileStillReads() throws { let url = try writeJSON(try #require(#"{"t":[{"name":"Ada"},{"name":"Grace"}]}"#.data(using: .utf16))) defer { try? FileManager.default.removeItem(at: url) } @@ -167,6 +206,87 @@ struct JSONDocumentStreamingTests { #expect(try JSONImportParsing.detectFields(at: url, targetTable: nil).map(\.name) == ["name"]) } + // `JSONSerialization` reads UTF-32LE with a byte order mark as UTF-16LE and fails, so the oracle + // is the parse of the same text without its mark, which it reads in every encoding. + @Test("UTF-16 and UTF-32 text streams to the rows a whole-file parse gives, with or without a mark", arguments: [1, 3, 7, 1 << 20]) + func decodedTextMatchesAWholeFileParse(chunkSize: Int) throws { + let json = "[\n " + Self.trickyRows.joined(separator: ",\n ") + #",{"emoji":"😀 𝄞 a"}]"# + for encoding in Self.encodings { + let url = try writeJSON(Self.encoded(json, encoding, withMark: false)) + defer { try? FileManager.default.removeItem(at: url) } + let expected = rows(try JSONImportParsing.parseElements(at: url, targetTable: nil)) + #expect(expected.count == Self.trickyRows.count + 1) + for withMark in [true, false] { + let data = Self.encoded(json, encoding, withMark: withMark) + #expect(try importedRows(data, chunkSize: chunkSize).map(Optional.some) == expected) + } + } + } + + @Test("A decoded file holds one chunk and one element, not the file") + func decodedReaderHoldsOneChunkAndOneElement() throws { + let rows = (1...2_000).map { #"{"id":\#($0),"name":"Hà Nội \#($0)"}"# } + let data = Self.encoded("[" + rows.joined(separator: ",") + "]", .utf16LittleEndian, withMark: true) + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + let chunkSize = 128 + var reader = try JSONDocumentReader(url: url, chunkSize: chunkSize) + let layout = try JSONDocumentOutline.rows(in: &reader, targetTable: nil) { reader, _ in try reader.skipValue() } + guard case .array(let offset, let count, _) = layout else { + Issue.record("Expected the rows of an array, got \(layout)") + return + } + var elements = try JSONStreamedElements(reader: reader, arrayAt: offset) + defer { elements.close() } + var mostHeld = 0 + while try elements.next() != nil { + mostHeld = max(mostHeld, elements.heldByteCount) + } + let longestElement = rows.map(\.utf8.count).max() ?? 0 + #expect(count == 2_000) + #expect(mostHeld <= chunkSize * 2 + longestElement) + #expect(data.count > 100 * (chunkSize * 2 + longestElement)) + } + + @Test("An error in decoded text points at the byte of the file", arguments: [String.Encoding.utf16LittleEndian, .utf32BigEndian]) + func decodedErrorNamesTheFileByte(encoding: String.Encoding) throws { + let text = #"[{"a":"é😀"} x]"# + let data = Self.encoded(text, encoding, withMark: true) + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + let mark = encoding == .utf16LittleEndian ? 2 : 4 + let unitsBefore = Array(#"[{"a":"é😀"} "#.utf16).count + let expected = encoding == .utf16LittleEndian ? mark + unitsBefore * 2 : mark + (#"[{"a":"é😀"} "#.unicodeScalars.count) * 4 + #expect(throws: JSONDocumentError.unexpectedCharacter(byteOffset: expected)) { + _ = try JSONImportParsing.rowElements(at: url, targetTable: nil, checkCancellation: {}) + } + } + + @Test("Text that is not valid UTF-16 fails instead of importing", arguments: [ + Data([0xFF, 0xFE, 0x5B, 0x00, 0x00, 0xD8, 0x5D, 0x00]), + Data([0xFF, 0xFE, 0x5B, 0x00, 0x5D]), + ]) + func invalidDecodedTextFails(data: Data) throws { + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + #expect(throws: (any Error).self) { + _ = try JSONImportParsing.rowElements(at: url, targetTable: nil, checkCancellation: {}) + } + } + + @Test("A lone object is one row, read member by member, keeping the first of two members with one name") + func singleObjectStreamsItsMembers() throws { + let tags = "[" + (1...3_000).map { #""tag \#($0)""# }.joined(separator: ",") + "]" + let json = #"{"id":7,"tags":"# + tags + #","id":8,"meta":{"k":null},"ok":true}"# + let data = Data(json.utf8) + let streamed = try importedRows(data, chunkSize: 64) + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + #expect(streamed.map(Optional.some) == rows(try JSONImportParsing.parseElements(at: url, targetTable: nil))) + #expect(streamed.first?["id"] == .text("7")) + #expect(try JSONImportParsing.detectFields(at: url, targetTable: nil).map(\.name) == ["id", "meta", "ok", "tags"]) + } + @Test("A file that starts with a UTF-8 byte order mark streams") func byteOrderMarkIsSkipped() throws { let url = try writeJSON(Data([0xEF, 0xBB, 0xBF]) + Data(#"[{"a":1}]"#.utf8)) diff --git a/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift b/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift index 6d562e9811..4aa0e25e01 100644 --- a/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift +++ b/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift @@ -9,8 +9,7 @@ import TableProPluginKit enum JSONDocumentRows: Equatable { // The rows are the elements of the array that opens at `offset`. `slot` is the one `visit` was given for them. case array(offset: Int, elementCount: Int, slot: Int) - // A single object, which is one row, or text in UTF-16 or UTF-32: `JSONSerialization` reads the file whole. - case wholeDocument + case singleObject(offset: Int) } // Finds which array of a `.json` file holds the rows, by the rules `JSONImportParsing.extractElements` @@ -32,7 +31,7 @@ enum JSONDocumentOutline { targetTable: String?, visit: (inout JSONDocumentReader, _ slot: Int?) throws -> Void ) throws -> JSONDocumentRows { - guard try reader.startDocument() else { return .wholeDocument } + try reader.startDocument() guard let first = try reader.nextSignificantByte() else { throw JSONDocumentError.endsEarly } switch first { case JSONDocumentReader.openBracket: @@ -41,11 +40,12 @@ enum JSONDocumentOutline { try reader.expectEnd() return .array(offset: offset, elementCount: array.count, slot: 0) case JSONDocumentReader.openBrace: - return try memberRows(in: &reader, targetTable: targetTable, visit: visit) + let offset = reader.offset + return try memberRows(in: &reader, at: offset, targetTable: targetTable, visit: visit) case 0x22, 0x2D, 0x30...0x39, 0x66, 0x6E, 0x74: throw JSONImportParsing.unexpectedShapeError default: - throw JSONDocumentError.unexpectedCharacter(byteOffset: reader.offset) + throw JSONDocumentError.unexpectedCharacter(byteOffset: reader.fileOffset) } } @@ -69,6 +69,7 @@ enum JSONDocumentOutline { private static func memberRows( in reader: inout JSONDocumentReader, + at objectOffset: Int, targetTable: String?, visit: (inout JSONDocumentReader, _ slot: Int?) throws -> Void ) throws -> JSONDocumentRows { @@ -96,7 +97,7 @@ enum JSONDocumentOutline { } try reader.expectEnd() - guard !members.isEmpty, members.allSatisfy({ $0.holdsRows }) else { return .wholeDocument } + guard !members.isEmpty, members.allSatisfy({ $0.holdsRows }) else { return .singleObject(offset: objectOffset) } let match = targetTable.flatMap { target in members.first { $0.name.caseInsensitiveCompare(target) == .orderedSame } ?? members.first { $0.name.split(separator: ".").last?.caseInsensitiveCompare(target) == .orderedSame } diff --git a/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift b/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift index ea927d568d..8bace6ac1c 100644 --- a/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift +++ b/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift @@ -8,6 +8,7 @@ import Foundation enum JSONDocumentError: LocalizedError, Equatable { case endsEarly case unexpectedCharacter(byteOffset: Int) + case invalidText var errorDescription: String? { switch self { @@ -15,13 +16,17 @@ enum JSONDocumentError: LocalizedError, Equatable { return String(localized: "The JSON file ends before its last value is complete.") case .unexpectedCharacter(let byteOffset): return String(format: String(localized: "The JSON file is not valid at byte %@."), (byteOffset + 1).formatted()) + case .invalidText: + return String(localized: "The JSON file has text that is not valid in its encoding.") } } } -// Walks a UTF-8 JSON file one chunk at a time. It finds where each value starts and ends without +// Walks a JSON file one chunk at a time. It finds where each value starts and ends without // building it, so the file is held one chunk and one value at a time. `JSONSerialization` parses // each value a caller asks for, so every value reads exactly as it would in a whole-file parse. +// UTF-16 and UTF-32 text is turned into UTF-8 as it is read, so `offset` counts UTF-8 bytes and +// `fileOffset` the file's own. struct JSONDocumentReader { static let defaultChunkSize = 1 << 20 @@ -42,6 +47,11 @@ struct JSONDocumentReader { private var bufferStart = 0 private var valueStart: Int? private var reachedEnd = false + private var decoder: JSONTextDecoder? + // Where the text starts in the file, past any byte order mark, and how many of its code units + // the buffer has already dropped. Only a decoded file uses them. + private var textStart = 0 + private var droppedUnits = 0 init( url: URL, @@ -55,34 +65,54 @@ struct JSONDocumentReader { var offset: Int { bufferStart + position } + var fileOffset: Int { fileOffset(atBufferIndex: position) } + var heldByteCount: Int { buffer.count } func close() { try? handle.close() } - // Steps over a UTF-8 byte order mark. False for UTF-16 or UTF-32 text, which `JSONSerialization` - // also reads: it has a byte order mark of its own or a zero byte in its first two. - mutating func startDocument() throws -> Bool { + // Steps over a byte order mark and, for UTF-16 or UTF-32 text, decodes what was read so far. + mutating func startDocument() throws { while buffer.count - position < 4, try refill() {} - let head = buffer[position...].prefix(4) - if head.starts(with: [0xEF, 0xBB, 0xBF]) { - position += 3 - return true - } - if head.starts(with: [0xFE, 0xFF]) || head.starts(with: [0xFF, 0xFE]) { - return false + let detected = JSONTextEncoding.detect(buffer[position...]) + guard detected.encoding != .utf8 else { + position += detected.markLength + return } - return !head.prefix(2).contains(0) + var decoder = JSONTextDecoder(encoding: detected.encoding) + textStart = bufferStart + position + detected.markLength + let raw = Data(buffer[(position + detected.markLength)...]) + buffer = try decoder.utf8(from: raw) + if reachedEnd { try decoder.finish() } + self.decoder = decoder + bufferStart = 0 + position = 0 + droppedUnits = 0 } + // `offset` is one `self.offset` gave. Decoded text has no file position for it, so the text is + // decoded again from its start up to there. mutating func seek(to offset: Int) throws { - try handle.seek(toOffset: UInt64(offset)) buffer.removeAll(keepingCapacity: true) - bufferStart = offset position = 0 valueStart = nil reachedEnd = false + guard decoder != nil else { + try handle.seek(toOffset: UInt64(offset)) + bufferStart = offset + return + } + try handle.seek(toOffset: UInt64(textStart)) + decoder?.reset() + bufferStart = 0 + droppedUnits = 0 + while bufferStart + buffer.count < offset { + position = buffer.count + guard try refill() else { throw JSONDocumentError.endsEarly } + } + position = offset - bufferStart } // The next byte that is not JSON whitespace, left unread, or nil at the end of the file. @@ -120,11 +150,10 @@ struct JSONDocumentReader { mutating func memberName() throws -> String { guard let byte = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } - guard byte == Self.quote else { throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) } - let start = offset + guard byte == Self.quote else { throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) } let name = try readValue() guard let decoded = try JSONSerialization.jsonObject(with: name, options: .fragmentsAllowed) as? String else { - throw JSONDocumentError.unexpectedCharacter(byteOffset: start) + throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset(atBufferIndex: position - name.count)) } try consume(Self.colon) return decoded @@ -132,7 +161,7 @@ struct JSONDocumentReader { mutating func expectEnd() throws { guard try nextSignificantByte() == nil else { - throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) + throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) } } @@ -150,7 +179,7 @@ struct JSONDocumentReader { private mutating func continueAfterValue(closing: UInt8) throws -> Bool { guard let byte = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } guard byte == Self.comma || byte == closing else { - throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) + throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) } position += 1 if byte == closing { return false } @@ -161,7 +190,7 @@ struct JSONDocumentReader { private mutating func consume(_ expected: UInt8) throws { guard let byte = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } - guard byte == expected else { throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) } + guard byte == expected else { throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) } position += 1 } @@ -191,7 +220,7 @@ struct JSONDocumentReader { case 0x2D, 0x30...0x39, 0x66, 0x6E, 0x74: try passLiteral() default: - throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) + throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) } } @@ -262,7 +291,7 @@ struct JSONDocumentReader { return false } position = index - if mismatched { throw JSONDocumentError.unexpectedCharacter(byteOffset: offset - 1) } + if mismatched { throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset(atBufferIndex: position - 1)) } if closed { return } guard try refill() else { throw JSONDocumentError.endsEarly } } @@ -292,17 +321,47 @@ struct JSONDocumentReader { try checkCancellation() let keep = valueStart ?? position if keep > 0 { + if decoder != nil { + droppedUnits += codeUnits(in: buffer[0.. Int { + guard let decoder else { return bufferStart + index } + let units = droppedUnits + codeUnits(in: buffer[0..) -> Int { + let countsPairs = decoder?.encoding.unitWidth == 2 + var units = 0 + for byte in bytes where byte & 0xC0 != 0x80 { + units += countsPairs && byte >= 0xF0 ? 2 : 1 } - buffer.append(contentsOf: chunk) - return true + return units } } diff --git a/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift b/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift index 9835b41ddf..d11b26c8b3 100644 --- a/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift +++ b/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift @@ -13,7 +13,7 @@ protocol JSONElementSequence { func close() } -// The elements of one array of a UTF-8 file, each read and parsed as it is reached. +// The elements of one array of the file, each read and parsed as it is reached. struct JSONStreamedElements: JSONElementSequence { private var reader: JSONDocumentReader private var hasMore: Bool @@ -39,19 +39,17 @@ struct JSONStreamedElements: JSONElementSequence { } } -// The elements of a file `JSONSerialization` has already read whole. -struct JSONParsedElements: JSONElementSequence { - private let elements: [Any] - private var cursor = 0 +// A file whose top-level object is the one row, built member by member. +struct JSONSingleRow: JSONElementSequence { + private var row: [String: PluginCellValue]? - init(_ elements: [Any]) { - self.elements = elements + init(_ row: [String: PluginCellValue]) { + self.row = row } mutating func next() -> Any? { - guard cursor < elements.count else { return nil } - defer { cursor += 1 } - return elements[cursor] + defer { row = nil } + return row } func close() {} diff --git a/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift b/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift index 636afd94f9..f4c5acf4a8 100644 --- a/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift +++ b/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift @@ -15,10 +15,14 @@ struct JSONFieldSurvey { mutating func add(_ row: NSDictionary) { row.enumerateKeysAndObjects { key, value, _ in guard let name = key as? String else { return } - profiles[name, default: JSONFieldProfile()].add(value) + add(value, named: name) } } + mutating func add(_ value: Any, named name: String) { + profiles[name, default: JSONFieldProfile()].add(value) + } + var fields: [PluginImportField] { profiles.keys.sorted().map { name in let profile = profiles[name] ?? JSONFieldProfile() diff --git a/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift b/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift index 453890598d..e92faf3392 100644 --- a/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift +++ b/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift @@ -37,9 +37,9 @@ enum JSONImportParsing { .importFailed("The file contains multiple tables and none matches the target table") } - // The values of a `.json` file that become rows, and how many there are. A UTF-8 file is read - // one element at a time after a pass that finds its rows and checks its structure, so a broken - // file fails before the first row is imported. + // The values of a `.json` file that become rows, and how many there are. The file is read one + // element at a time after a pass that finds its rows and checks its structure, so a broken file + // fails before the first row is imported. static func rowElements( at url: URL, targetTable: String?, @@ -50,29 +50,53 @@ enum JSONImportParsing { let rows = try JSONDocumentOutline.rows(in: &reader, targetTable: targetTable) { reader, _ in try reader.skipValue() } - if case .array(let offset, let count, _) = rows { + switch rows { + case .array(let offset, let count, _): return (try JSONStreamedElements(reader: reader, arrayAt: offset), count) + case .singleObject(let offset): + defer { reader.close() } + var row: [String: PluginCellValue] = [:] + try members(ofObjectAt: offset, in: &reader) { name, value in + row[name] = cellValue(from: value) + } + return (JSONSingleRow(row), 1) } } catch { reader.close() throw error } - reader.close() - let elements = try parseElements(at: url, targetTable: targetTable) - return (JSONParsedElements(elements), elements.count) + } + + // Each member of the object at `offset`, parsed one at a time. A parsed object keeps the first + // of two members that share a name, so the later one is passed over. + static func members( + ofObjectAt offset: Int, + in reader: inout JSONDocumentReader, + _ body: (String, Any) throws -> Void + ) throws { + try reader.seek(to: offset) + guard try reader.enterObject() else { return } + var names: Set = [] + repeat { + let name = try reader.memberName() + let value = try reader.readValue() + guard names.insert(name).inserted else { continue } + try autoreleasepool { + try body(name, try JSONSerialization.jsonObject(with: value, options: .fragmentsAllowed)) + } + } while try reader.continueObject() } static func row(from element: Any) throws -> [String: PluginCellValue] { + if let row = element as? [String: PluginCellValue] { + return row + } guard let object = element as? NSDictionary else { throw PluginImportError.importFailed(String(localized: "Each array element must be a JSON object")) } return convertRow(object) } - static func parseRows(at url: URL, targetTable: String?) throws -> [NSDictionary] { - try parseElements(at: url, targetTable: targetTable).compactMap { $0 as? NSDictionary } - } - static func parseElements(at url: URL, targetTable: String?) throws -> [Any] { let data = try Data(contentsOf: url) let object = try JSONSerialization.jsonObject(with: data) @@ -181,8 +205,12 @@ enum JSONImportParsing { switch rows { case .array(_, _, let slot): return surveys[slot]?.fields ?? [] - case .wholeDocument: - return try detectFields(in: try parseRows(at: url, targetTable: targetTable)) + case .singleObject(let offset): + var survey = JSONFieldSurvey() + try members(ofObjectAt: offset, in: &reader) { name, value in + survey.add(value, named: name) + } + return survey.fields } } diff --git a/apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift b/apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift new file mode 100644 index 0000000000..6d937f2826 --- /dev/null +++ b/apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift @@ -0,0 +1,109 @@ +// +// JSONTextDecoder.swift +// JSONImportPlugin +// + +import Foundation + +// The five encodings `JSONSerialization` reads, with or without a byte order mark. +enum JSONTextEncoding: Equatable { + case utf8 + case utf16BigEndian + case utf16LittleEndian + case utf32BigEndian + case utf32LittleEndian + + var unitWidth: Int { + switch self { + case .utf8: return 1 + case .utf16BigEndian, .utf16LittleEndian: return 2 + case .utf32BigEndian, .utf32LittleEndian: return 4 + } + } + + // RFC 4627, section 3: the first two characters are ASCII, so the zero bytes in the first four + // name the encoding when no byte order mark does. + static func detect(_ head: some Collection) -> (encoding: JSONTextEncoding, markLength: Int) { + let bytes = Array(head.prefix(4)) + if bytes.starts(with: [0xEF, 0xBB, 0xBF]) { return (.utf8, 3) } + if bytes.starts(with: [0x00, 0x00, 0xFE, 0xFF]) { return (.utf32BigEndian, 4) } + if bytes.starts(with: [0xFF, 0xFE, 0x00, 0x00]) { return (.utf32LittleEndian, 4) } + if bytes.starts(with: [0xFE, 0xFF]) { return (.utf16BigEndian, 2) } + if bytes.starts(with: [0xFF, 0xFE]) { return (.utf16LittleEndian, 2) } + guard bytes.count >= 2 else { return (.utf8, 0) } + if bytes.count >= 4 { + if bytes[0] == 0, bytes[1] == 0, bytes[2] == 0 { return (.utf32BigEndian, 0) } + if bytes[1] == 0, bytes[2] == 0, bytes[3] == 0 { return (.utf32LittleEndian, 0) } + } + if bytes[0] == 0 { return (.utf16BigEndian, 0) } + if bytes[1] == 0 { return (.utf16LittleEndian, 0) } + return (.utf8, 0) + } +} + +// Turns UTF-16 or UTF-32 text into UTF-8 a chunk at a time. A code unit, or a surrogate pair, that a +// chunk cuts in two waits for the next chunk. +struct JSONTextDecoder { + let encoding: JSONTextEncoding + private var pending: [UInt8] = [] + + init(encoding: JSONTextEncoding) { + self.encoding = encoding + } + + mutating func reset() { + pending.removeAll(keepingCapacity: true) + } + + mutating func utf8(from chunk: Data) throws -> [UInt8] { + guard encoding != .utf8 else { return Array(chunk) } + pending.append(contentsOf: chunk) + var usable = pending.count - pending.count % encoding.unitWidth + var output: [UInt8] = [] + output.reserveCapacity(usable) + let failed: Bool + switch encoding { + case .utf8: + return Array(chunk) + case .utf16BigEndian, .utf16LittleEndian: + var units: [UInt16] = [] + units.reserveCapacity(usable / 2) + var index = 0 + while index < usable { + let first = UInt16(pending[index]) + let second = UInt16(pending[index + 1]) + units.append(encoding == .utf16BigEndian ? first << 8 | second : second << 8 | first) + index += 2 + } + if let last = units.last, (0xD800...0xDBFF).contains(last) { + units.removeLast() + usable -= 2 + } + failed = transcode(units.makeIterator(), from: UTF16.self, to: UTF8.self, stoppingOnError: true) { + output.append($0) + } + case .utf32BigEndian, .utf32LittleEndian: + var units: [UInt32] = [] + units.reserveCapacity(usable / 4) + var index = 0 + while index < usable { + let bytes = pending[index..<(index + 4)].map(UInt32.init) + units.append(encoding == .utf32BigEndian + ? bytes[0] << 24 | bytes[1] << 16 | bytes[2] << 8 | bytes[3] + : bytes[3] << 24 | bytes[2] << 16 | bytes[1] << 8 | bytes[0]) + index += 4 + } + failed = transcode(units.makeIterator(), from: UTF32.self, to: UTF8.self, stoppingOnError: true) { + output.append($0) + } + } + if failed { throw JSONDocumentError.invalidText } + pending.removeSubrange(0.., insertedRowIndices: Set ) throws -> [PluginRowWrite] { + // In the order received, which is edit order: a row deleted to free a unique value goes before the row taking it. var writes: [PluginRowWrite] = [] - var deletes: [PluginRowWrite] = [] for change in changes { switch change.type { case .insert: @@ -45,10 +45,10 @@ struct MSSQLStatementGenerator { case .delete: guard deletedRowIndices.contains(change.rowIndex), let statement = try deleteStatement(for: change) else { continue } - deletes.append(PluginRowWrite(statement: statement.statement, parameters: statement.parameters, rowIndices: [change.rowIndex])) + writes.append(PluginRowWrite(statement: statement.statement, parameters: statement.parameters, rowIndices: [change.rowIndex])) } } - return writes + deletes + return writes } /// The statements alone, for a host that asks without a context. With no context there is nothing to refuse. diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQConnectionString.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQConnectionString.swift index 04bb56d2ea..7cd8bc1d45 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQConnectionString.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQConnectionString.swift @@ -6,6 +6,7 @@ import Foundation import TableProPluginKit import TableProSQLQuoting +import TableProTLSClientIdentity internal struct LibPQConnectTimeout: Equatable, Sendable { static let defaultMilliseconds = 10_000 @@ -46,9 +47,8 @@ internal struct LibPQConnectTimeout: Equatable, Sendable { } } -/// One monotonic wall-clock budget for libpq's TCP/TLS/authentication and the bootstrap queries -/// that make the session usable. Keeping the absolute instant here prevents every catalog probe -/// from receiving the full timeout again. +// One absolute instant for the TCP, TLS and sign-in steps and the bootstrap queries, so no catalog probe gets the full +// timeout again. internal struct LibPQConnectDeadline: Equatable, Sendable { private let expiresAtMicroseconds: Int64 @@ -67,6 +67,13 @@ internal struct LibPQConnectDeadline: Equatable, Sendable { } } +// Where libpq takes the roots it checks the server's certificate against. +internal enum LibPQRootCertificateSource: Equatable, Sendable { + case caFile(String) + case libpqDefault + case systemTrustAnchors +} + internal enum LibPQConnectionString { static let clientEncoding = "UTF8" @@ -75,11 +82,10 @@ internal enum LibPQConnectionString { static let sessionApplicationName = "TablePro" static let metadataApplicationName = "TablePro Metadata" - /// Left to the kernel, macOS sends the first keepalive after 7,200 s of idle, while an AWS - /// Network Load Balancer drops an idle flow after 350 s and an Azure Load Balancer silently after - /// 4 minutes. These keep a parked session's flow open and let the kernel declare a dead peer - /// about 90 s after the last traffic. `tcp_user_timeout` is left out: macOS has no - /// `TCP_USER_TIMEOUT`, so libpq accepts it and changes nothing. + // Left to the kernel, macOS sends the first keepalive after 7,200 s of idle, while an AWS Network Load Balancer drops + // an idle flow after 350 s and an Azure Load Balancer silently after 4 minutes. These keep a parked session's flow + // open and find a dead peer about 90 s after the last traffic. No `tcp_user_timeout`: macOS has no + // `TCP_USER_TIMEOUT`, so libpq accepts it and changes nothing. static let keepaliveParameters: [(String, String)] = [ ("keepalives", "1"), ("keepalives_idle", "60"), @@ -98,7 +104,8 @@ internal enum LibPQConnectionString { applicationName: String? = nil, connectTimeoutSeconds: Int? = nil, kerberos: LibPQKerberosSignIn? = nil, - serverName: String? = nil + serverName: String? = nil, + systemTrustAnchorsPath: String? = nil ) throws -> String { var parameters: [(String, String)] = [] if let serverName, !serverName.isEmpty, serverName != host, isAddressLiteral(host) { @@ -142,6 +149,8 @@ internal enum LibPQConnectionString { } if sslConfig.verifiesCertificate, !sslConfig.caCertificatePath.isEmpty { parameters.append(("sslrootcert", sslConfig.caCertificatePath)) + } else if sslConfig.verifiesHostname, let systemTrustAnchorsPath { + parameters.append(("sslrootcert", systemTrustAnchorsPath)) } if !sslConfig.clientCertificatePath.isEmpty { parameters.append(("sslcert", sslConfig.clientCertificatePath)) @@ -181,6 +190,29 @@ internal enum LibPQConnectionString { // Nothing can exist under a device file, so libpq finds no file there and says nothing. static let unreadablePassfile = "/dev/null/pgpass" + // Nil when the mode checks no certificate. Verify CA never takes this Mac's roots: it would accept any server with a + // public certificate, which libpq refuses for sslrootcert=system. A `~/.postgresql/root.crt` keeps libpq's default. + static func rootCertificateSource( + for sslConfig: SSLConfiguration, + host: String, + environment: [String: String] = ProcessInfo.processInfo.environment, + fileExists: (String) -> Bool = { FileManager.default.fileExists(atPath: $0) } + ) -> LibPQRootCertificateSource? { + guard sslConfig.verifiesCertificate else { return nil } + guard sslConfig.caCertificatePath.isEmpty else { return .caFile(sslConfig.caCertificatePath) } + return LibPQSystemTrustRule.usesSystemTrustAnchors( + verifiesHostname: sslConfig.verifiesHostname, + caCertificatePath: sslConfig.caCertificatePath, + host: host, + environment: environment, + fileExists: fileExists + ) ? .systemTrustAnchors : .libpqDefault + } + + static func defaultRootCertificatePath(environment: [String: String]) -> String { + LibPQSystemTrustRule.defaultRootCertificatePath(environment: environment) + } + private static func isAddressLiteral(_ host: String) -> Bool { var ipv4 = in_addr() var ipv6 = in6_addr() @@ -191,16 +223,15 @@ internal enum LibPQConnectionString { purpose == "metadata" ? metadataApplicationName : sessionApplicationName } - /// libpq applies `fallback_application_name` over an `application_name` set in `options`, so a - /// name the user already gave the session there has to keep the fallback out entirely. + // libpq applies `fallback_application_name` over an `application_name` set in `options`, so a name the user gave the + // session there keeps the fallback out. static func namesApplication(_ options: String?) -> Bool { guard let options else { return false } return settingNames(in: options).contains("application_name") } - /// The setting each `-c name=value`, `-cname=value` and `--name=value` in `options` assigns, spelled - /// the way the server reads a name: in any case, with `-` and `_` as the same character. Only the - /// names, because a value such as `search_path=application_name` assigns something else. + // Spelled the way the server reads a name: any case, `-` and `_` alike. Only the names, because a value such as + // `search_path=application_name` assigns something else. static func settingNames(in options: String) -> [String] { let tokens = options.split(whereSeparator: \.isWhitespace).map(String.init) var names: [String] = [] diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift index 20b22f439c..a57fb8a048 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift @@ -15,6 +15,7 @@ import TableProCoreTypes import TableProKerberos import TableProPluginKit import TableProPostgreSQLCore +import TableProTLSClientIdentity private final class LibPQCancelRequest: @unchecked Sendable { private let cancelObject: OpaquePointer @@ -32,11 +33,11 @@ private final class LibPQCancelRequest: @unchecked Sendable { // MARK: - Connection Class -/// libpq allows one thread at a time on a `PGconn`, so every libpq call on `conn` runs on `queue`, -/// and a member that other files can reach and that makes such a call opens with -/// `preconditionOnQueue()`. `stateLock` guards `conn`, `serverMessages` and the underscored state, -/// is never held across a libpq call, and is taken in this file only. Cancelling a running query -/// is the one path that calls libpq off the queue. +// libpq allows one thread at a time on a `PGconn`, so every libpq call on `conn` runs on `queue`, +// and a member that other files can reach and that makes such a call opens with +// `preconditionOnQueue()`. `stateLock` guards `conn`, `serverMessages` and the underscored state, +// is never held across a libpq call, and is taken in this file only. Cancelling a running query +// is the one path that calls libpq off the queue. final class LibPQPluginConnection: @unchecked Sendable { static let logger = Logger(subsystem: "com.TablePro.PostgreSQLDriver", category: "LibPQPluginConnection") @@ -77,7 +78,7 @@ final class LibPQPluginConnection: @unchecked Sendable { private var _interruptsBlockedCallOnDisconnect = false private var serverMessages: Unmanaged? private var _standardConformingStrings = true - /// Queue-confined. Non-nil until every schema/flavor/catalog bootstrap probe has completed. + // Queue-confined. Non-nil until every schema/flavor/catalog bootstrap probe has completed. private var activeConnectDeadline: LibPQConnectDeadline? private var bootstrapTransportFailed = false // Queue-confined. Nil until a probe, sent while no block was open, showed whether the server takes savepoints: @@ -204,7 +205,10 @@ final class LibPQPluginConnection: @unchecked Sendable { KerberosTicket.discard(cachePath) } } - let conninfo = try connectionString() + // libpq reads the roots file while it polls, and every reconnect writes a new one. + let trustAnchors = try systemTrustAnchorsFile() + defer { trustAnchors?.remove() } + let conninfo = try connectionString(systemTrustAnchorsPath: trustAnchors?.url.path) let connection = try signingIn(until: deadline) { () throws -> OpaquePointer in guard let connection = conninfo.withCString({ PQconnectStart($0) }) else { throw LibPQPluginError.connectionFailed @@ -242,6 +246,18 @@ final class LibPQPluginConnection: @unchecked Sendable { adopted = true } + // A failure fails the connect: without sslrootcert libpq would look for a file that is not there. + private func systemTrustAnchorsFile() throws -> SystemTrustAnchorFile? { + guard LibPQConnectionString.rootCertificateSource(for: sslConfig, host: host) == .systemTrustAnchors else { + return nil + } + do { + return try SystemTrustAnchors.shared.writeFile() + } catch { + throw LibPQPluginError(message: error.localizedDescription, sqlState: nil, detail: nil) + } + } + // PQconnectStart can already acquire the GSS credential, so the whole start sits in the scope. private func signingIn(until deadline: LibPQConnectDeadline, _ body: () throws -> T) throws -> T { guard let kerberos else { return try body() } @@ -257,8 +273,8 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// The core calls this only after schema and flavor-specific catalog probes. Until then every - /// statement uses libpq's nonblocking API and the original absolute connect deadline. + // The core calls this only after schema and flavor-specific catalog probes. Until then every + // statement uses libpq's nonblocking API and the original absolute connect deadline. func finishConnecting() async throws { try await pluginDispatchAsync(on: queue) { [self] in try checkConnectCancellation() @@ -325,9 +341,9 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// `PGRES_POLLING_*` only says whether the socket wants a read or a write, so it cannot tell - /// a TLS handshake from an authentication exchange. `PQstatus` can, and reading it costs one - /// pointer dereference per poll slice. + // `PGRES_POLLING_*` only says whether the socket wants a read or a write, so it cannot tell + // a TLS handshake from an authentication exchange. `PQstatus` can, and reading it costs one + // pointer dereference per poll slice. private func reportHandshakeStage( of connection: OpaquePointer, last: inout ConnStatusType?, @@ -354,6 +370,9 @@ final class LibPQPluginConnection: @unchecked Sendable { private func connectionError(from connection: OpaquePointer) -> Error { let error = getError(from: connection) + if let missingRoots = LibPQSSLMapping.missingRootCertificateError(error.message, sslConfig: sslConfig) { + return missingRoots + } if let sslError = LibPQSSLClassifier.classifySSLError(error.message) { return sslError } @@ -449,9 +468,9 @@ final class LibPQPluginConnection: @unchecked Sendable { return LibPQStringConformance.isOn(String(cString: value)) } - /// Executes a bootstrap statement without ever entering libpq's blocking `PQexec` path. A - /// statement can produce more than one result, so this mirrors `PQexec` by returning the last - /// one while still polling every byte under the same connect deadline. + // Executes a bootstrap statement without ever entering libpq's blocking `PQexec` path. A + // statement can produce more than one result, so this mirrors `PQexec` by returning the last + // one while still polling every byte under the same connect deadline. private func bootstrapResult( for statement: String, on connection: OpaquePointer, @@ -582,7 +601,7 @@ final class LibPQPluginConnection: @unchecked Sendable { Self.logger.warning("standard_conforming_strings is off; string literals escape backslashes") } - private func connectionString() throws -> String { + private func connectionString(systemTrustAnchorsPath: String?) throws -> String { try LibPQConnectionString.build( host: host, port: port, @@ -594,7 +613,8 @@ final class LibPQPluginConnection: @unchecked Sendable { applicationName: applicationName, connectTimeoutSeconds: connectTimeout.nativeSeconds, kerberos: kerberos, - serverName: serverName + serverName: serverName, + systemTrustAnchorsPath: systemTrustAnchorsPath ) } @@ -700,20 +720,18 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// Runs a read whose statement changes session settings with `SET LOCAL`, and confines those - /// settings to the read whether or not the session has a transaction block open. - /// - /// With no block open, the settings and the read are one implicit transaction that ends with it. - /// Inside a block they would outlast the read, so a savepoint scopes them and is rolled back - /// straight after. The status check and every statement run in one block on this connection's - /// queue. Split into separate dispatches, another caller sharing the session could run between - /// the savepoint and its rollback, and the rollback silently undid that caller's work. PGlite has - /// no pooled connection, so a query tab and the metadata reads share one session. - /// - /// The `ROLLBACK TO` is its own `PQexec`, because `PQexec` returns only a string's last result - /// and the rows are the read's. A rollback that fails after a read that succeeded is reported - /// rather than the rows: the session would otherwise keep the read's settings for the rest of - /// the user's transaction. + // Runs a read whose statement changes session settings with `SET LOCAL`, and confines those + // settings to the read whether or not the session has a transaction block open. + // With no block open, the settings and the read are one implicit transaction that ends with it. + // Inside a block they would outlast the read, so a savepoint scopes them and is rolled back + // straight after. The status check and every statement run in one block on this connection's + // queue. Split into separate dispatches, another caller sharing the session could run between + // the savepoint and its rollback, and the rollback silently undid that caller's work. PGlite has + // no pooled connection, so a query tab and the metadata reads share one session. + // The `ROLLBACK TO` is its own `PQexec`, because `PQexec` returns only a string's last result + // and the rows are the read's. A rollback that fails after a read that succeeded is reported + // rather than the rows: the session would otherwise keep the read's settings for the rest of + // the user's transaction. func executeTransactionScopedRead(_ statement: String) async throws -> LibPQPluginQueryResult { let statementToRun = String(statement) @@ -739,9 +757,9 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// A catalog query the driver sends on its own while reading a result, on the connection's queue, scoped the way - /// `executeTransactionScopedRead` scopes one. The caller clears the result. Inside an aborted block it hands back - /// the savepoint's own `25P02` failure. + // A catalog query the driver sends on its own while reading a result, on the connection's queue, scoped the way + // `executeTransactionScopedRead` scopes one. The caller clears the result. Inside an aborted block it hands back + // the savepoint's own `25P02` failure. func pinnedCatalogSideQueryOnQueue(_ statement: String, conn: OpaquePointer) -> OpaquePointer? { preconditionOnQueue() guard pinsCatalogSearchPath else { return statement.withCString { PQexec(conn, $0) } } @@ -791,11 +809,11 @@ final class LibPQPluginConnection: @unchecked Sendable { private static let scopedReadSavepoint = "SAVEPOINT tablepro_scoped_read" private static let scopedReadRollback = "ROLLBACK TO SAVEPOINT tablepro_scoped_read; RELEASE SAVEPOINT tablepro_scoped_read" - /// Runs one of the driver's own session-setting statements so a failure cannot abort a transaction block the - /// user has open. Inside one, a savepoint holds the statement: rolled back when it fails, released when it - /// succeeds, which keeps what it set in the block. One dispatch, for the reason `executeTransactionScopedRead` - /// gives. In an aborted block the statement goes as it is and the server answers `25P02`. `requiresIdleSession` - /// refuses in the same dispatch, so no transaction can open between the check and the statement. + // Runs one of the driver's own session-setting statements so a failure cannot abort a transaction block the + // user has open. Inside one, a savepoint holds the statement: rolled back when it fails, released when it + // succeeds, which keeps what it set in the block. One dispatch, for the reason `executeTransactionScopedRead` + // gives. In an aborted block the statement goes as it is and the server answers `25P02`. `requiresIdleSession` + // refuses in the same dispatch, so no transaction can open between the check and the statement. func executeSessionSetting( _ statement: String, parameters: [PluginCellValue], @@ -829,19 +847,18 @@ final class LibPQPluginConnection: @unchecked Sendable { private static let sessionSettingRollback = "ROLLBACK TO SAVEPOINT tablepro_session_setting; RELEASE SAVEPOINT tablepro_session_setting" - /// Whether the session is inside a transaction block, including one a failed statement has - /// aborted, so a statement sent now joins it. Called on the connection's queue only: one `PGconn` - /// may not be used from two threads at once, and the lock guards the pointer rather than the call. + // Whether the session is inside a transaction block, including one a failed statement has + // aborted, so a statement sent now joins it. Called on the connection's queue only: one `PGconn` + // may not be used from two threads at once, and the lock guards the pointer rather than the call. private func isInsideTransactionBlockOnQueue() -> Bool { let state = transactionStateOnQueue() return state == .inTransaction || state == .inError } - /// What the session has open, from the `ReadyForQuery` status libpq keeps from the last reply. - /// - /// No round trip and no server work: measured against PostgreSQL 17.11, a million calls took - /// 1.5ms, and the answer is local enough to survive a backend another session terminated - /// (`PQtransactionStatus` still reported `INTRANS` with `PQstatus` OK). + // What the session has open, from the `ReadyForQuery` status libpq keeps from the last reply. + // No round trip and no server work: measured against PostgreSQL 17.11, a million calls took + // 1.5ms, and the answer is local enough to survive a backend another session terminated + // (`PQtransactionStatus` still reported `INTRANS` with `PQstatus` OK). func transactionState() async -> LibPQTransactionState { do { return try await pluginDispatchAsync(on: queue) { [self] in @@ -858,10 +875,10 @@ final class LibPQPluginConnection: @unchecked Sendable { return Self.transactionState(PQtransactionStatus(conn)) } - /// libpq has no round trip that sends no statement, so the socket is read first: that sees a - /// server that closed the session, and inside a transaction block it is the whole check - /// (`LibPQSessionCheck.sendsStatement`). A statement the server refuses still proves the - /// backend answered, so it does not fail the ping. + // libpq has no round trip that sends no statement, so the socket is read first: that sees a + // server that closed the session, and inside a transaction block it is the whole check + // (`LibPQSessionCheck.sendsStatement`). A statement the server refuses still proves the + // backend answered, so it does not fail the ping. func ping() async throws { try await pluginDispatchAsync(on: queue) { [self] in guard !isShuttingDown, let conn = connectionHandle else { throw LibPQPluginError.notConnected } @@ -1444,9 +1461,9 @@ final class LibPQPluginConnection: @unchecked Sendable { // MARK: - Pending Results - /// A statement the user did not get to see is worse than a slow one, so a COPY ended by any - /// drain is reported rather than swallowed: `INSERT INTO t VALUES (1); COPY t FROM STDIN` used - /// to come back as "INSERT 0 1" with the COPY discarded. + // A statement the user did not get to see is worse than a slow one, so a COPY ended by any + // drain is reported rather than swallowed: `INSERT INTO t VALUES (1); COPY t FROM STDIN` used + // to come back as "INSERT 0 1" with the COPY discarded. private func abandonedCopyError(_ outcome: LibPQDrainOutcome, generation: Int) -> Error? { guard let copy = outcome.abandonedCopy else { return nil } if cancellationGate.isCancelled(generation) { return CancellationError() } @@ -1468,9 +1485,9 @@ final class LibPQPluginConnection: @unchecked Sendable { LibPQPluginError(message: copy.direction.unsupportedMessage, sqlState: nil, detail: nil) } - /// Reading `COPY TO STDOUT` to its end defeats the row cap the bounded read exists for, so the - /// statement is cancelled first wherever a cancel is safe. Inside a transaction block it is not, - /// because a cancel aborts the transaction the user opened. + // Reading `COPY TO STDOUT` to its end defeats the row cap the bounded read exists for, so the + // statement is cancelled first wherever a cancel is safe. Inside a transaction block it is not, + // because a cancel aborts the transaction the user opened. private func cancelsAbandonedOutput(_ conn: OpaquePointer) -> Bool { !suppressServerSideCancel && PQtransactionStatus(conn) != PQTRANS_INTRANS } @@ -1500,9 +1517,9 @@ final class LibPQPluginConnection: @unchecked Sendable { return finishPendingResults(conn, cancellingOutput: false) } - /// The abort is polled by the producer rather than acted on from `onTermination`, because both - /// run on one serial queue: a drain enqueued from the handler sits behind the producer and runs - /// only once the whole result has been read, which is no abort at all. + // The abort is polled by the producer rather than acted on from `onTermination`, because both + // run on one serial queue: a drain enqueued from the handler sits behind the producer and runs + // only once the whole result has been read, which is no abort at all. func streamQuery(_ query: String, readOnly: Bool = false) -> AsyncThrowingStream { let queryToRun = String(query) @@ -1519,15 +1536,15 @@ final class LibPQPluginConnection: @unchecked Sendable { defer { cancellationGate.endQuery(generation) } defer { refreshStandardConformingStrings(from: conn) } - /// The consumer can go away before this block is scheduled, in which case the - /// query is never sent at all. + // The consumer can go away before this block is scheduled, in which case the + // query is never sent at all. guard !abort.isAborted else { continuation.finish() return } - /// Read before the query goes out: once it is in flight the status is - /// PQTRANS_ACTIVE, and the transaction this guard exists for is invisible. + // Read before the query goes out: once it is in flight the status is + // PQTRANS_ACTIVE, and the transaction this guard exists for is invisible. let cancelsOutput = cancelsAbandonedOutput(conn) finishAbandonedResults(conn, cancellingOutput: cancelsOutput) if let ended = sessionEndedBeforeSending(conn) { @@ -1777,12 +1794,12 @@ final class LibPQPluginConnection: @unchecked Sendable { private func sessionEndedBeforeSending(_ conn: OpaquePointer) -> LibPQConnectionLostError? { recordTransactionState(of: conn) - /// All three calls earn their place, measured against a terminated backend on 9.1.24 and - /// 17.11. One `PQconsumeInput` leaves the status `CONNECTION_OK`, so a single read never - /// sees the loss; the second one turns it `CONNECTION_BAD`. `PQisBusy` never moves the - /// status, but it is what parses the buffered message: without it every closed-session - /// case loses the server's FATAL and its SQLSTATE and reports libpq's own "server closed - /// the connection unexpectedly" instead. + // All three calls earn their place, measured against a terminated backend on 9.1.24 and + // 17.11. One `PQconsumeInput` leaves the status `CONNECTION_OK`, so a single read never + // sees the loss; the second one turns it `CONNECTION_BAD`. `PQisBusy` never moves the + // status, but it is what parses the buffered message: without it every closed-session + // case loses the server's FATAL and its SQLSTATE and reports libpq's own "server closed + // the connection unexpectedly" instead. if PQstatus(conn) == CONNECTION_OK { _ = PQconsumeInput(conn) _ = PQisBusy(conn) diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQSSLMapping.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQSSLMapping.swift index 366ba4f142..b263b12cce 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQSSLMapping.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQSSLMapping.swift @@ -11,4 +11,21 @@ enum LibPQSSLMapping { case .verifyIdentity: return "verify-full" } } + + // libpq's own text suggests sslrootcert=system, which TablePro does not offer and libpq refuses under verify-ca. + static func missingRootCertificateError(_ message: String, sslConfig: SSLConfiguration) -> LibPQPluginError? { + guard sslConfig.mode == .verifyCa, sslConfig.caCertificatePath.isEmpty else { return nil } + let lowered = message.lowercased() + guard lowered.contains("root certificate file"), lowered.contains("does not exist") else { return nil } + return LibPQPluginError( + message: String( + localized: """ + Verify CA needs the server's CA certificate. Choose it under SSL, or use Verify Identity to check \ + the server against this Mac's trusted certificates. + """ + ), + sqlState: nil, + detail: nil + ) + } } diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQConnectionStringTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQConnectionStringTests.swift index ff3a6b976b..a38c092d50 100644 --- a/apple/Plugins/PostgreSQLDriverTests/LibPQConnectionStringTests.swift +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQConnectionStringTests.swift @@ -14,7 +14,8 @@ struct LibPQConnectionStringTests { sslConfig: SSLConfiguration = SSLConfiguration(), options: String? = nil, applicationName: String? = nil, - connectTimeoutSeconds: Int? = nil + connectTimeoutSeconds: Int? = nil, + systemTrustAnchorsPath: String? = nil ) throws -> String { try LibPQConnectionString.build( host: "db.example.com", @@ -25,7 +26,8 @@ struct LibPQConnectionStringTests { sslConfig: sslConfig, options: options, applicationName: applicationName, - connectTimeoutSeconds: connectTimeoutSeconds + connectTimeoutSeconds: connectTimeoutSeconds, + systemTrustAnchorsPath: systemTrustAnchorsPath ) } @@ -260,6 +262,98 @@ struct LibPQConnectionStringTests { #expect(verifying.contains("sslrootcert='/ca.pem'")) } + @Test("Verify Identity with no CA file checks the server against this Mac's trusted roots") + func verifyIdentityWithoutACAUsesTheSystemAnchors() throws { + let conninfo = try build( + sslConfig: SSLConfiguration(mode: .verifyIdentity), + systemTrustAnchorsPath: "/tmp/roots.pem" + ) + + #expect(conninfo.contains("sslmode='verify-full'")) + #expect(conninfo.contains("sslrootcert='/tmp/roots.pem'")) + } + + @Test("A chosen CA file wins over this Mac's trusted roots") + func aCAFileWinsOverTheSystemAnchors() throws { + let conninfo = try build( + sslConfig: SSLConfiguration(mode: .verifyIdentity, caCertificatePath: "/ca.pem"), + systemTrustAnchorsPath: "/tmp/roots.pem" + ) + + #expect(conninfo.contains("sslrootcert='/ca.pem'")) + #expect(!conninfo.contains("/tmp/roots.pem")) + } + + @Test("Verify CA never checks the server against this Mac's trusted roots") + func verifyCAWithoutACAGetsNoSystemAnchors() throws { + let conninfo = try build(sslConfig: SSLConfiguration(mode: .verifyCa), systemTrustAnchorsPath: "/tmp/roots.pem") + + #expect(conninfo.contains("sslmode='verify-ca'")) + #expect(!conninfo.contains("sslrootcert")) + } + + @Test("A mode that checks no certificate gets no root file", arguments: [SSLMode.disabled, .preferred, .required]) + func requiredAndPreferredNeverGetSystemAnchors(mode: SSLMode) throws { + let conninfo = try build(sslConfig: SSLConfiguration(mode: mode), systemTrustAnchorsPath: "/tmp/roots.pem") + + #expect(!conninfo.contains("sslrootcert")) + } + + @Test("The roots come from the CA file, then libpq's own root.crt, then this Mac's trusted roots") + func rootCertificateSourceOrder() { + let home = ["HOME": "/Users/test"] + let rootCrt = "/Users/test/.postgresql/root.crt" + func source( + _ mode: SSLMode, + ca: String = "", + host: String = "db.example.com", + rootCrtExists: Bool = false + ) -> LibPQRootCertificateSource? { + LibPQConnectionString.rootCertificateSource( + for: SSLConfiguration(mode: mode, caCertificatePath: ca), + host: host, + environment: home, + fileExists: { rootCrtExists && $0 == rootCrt } + ) + } + + #expect(source(.verifyIdentity, ca: "/ca.pem", rootCrtExists: true) == .caFile("/ca.pem")) + #expect(source(.verifyCa, ca: "/ca.pem") == .caFile("/ca.pem")) + #expect(source(.verifyIdentity, rootCrtExists: true) == .libpqDefault) + #expect(source(.verifyIdentity) == .systemTrustAnchors) + #expect(source(.verifyIdentity, host: "/tmp") == .libpqDefault) + #expect(source(.verifyCa) == .libpqDefault) + #expect(source(.required) == nil) + #expect(source(.preferred) == nil) + #expect(source(.disabled) == nil) + } + + @Test("A PGSSLROOTCERT the user set, or an empty host, leaves the roots to libpq") + func namedRootFileAndDefaultSocketStayWithLibpq() { + let config = SSLConfiguration(mode: .verifyIdentity) + #expect(LibPQConnectionString.rootCertificateSource( + for: config, host: "db.example.com", environment: ["HOME": "/Users/test", "PGSSLROOTCERT": "/etc/corp-ca.pem"], + fileExists: { _ in false } + ) == .libpqDefault) + #expect(LibPQConnectionString.rootCertificateSource( + for: config, host: "", environment: ["HOME": "/Users/test"], fileExists: { _ in false } + ) == .libpqDefault) + #expect(LibPQConnectionString.rootCertificateSource( + for: config, host: "db.example.com", environment: ["HOME": "/Users/test", "PGSSLROOTCERT": ""], + fileExists: { _ in false } + ) == .systemTrustAnchors) + } + + @Test("libpq's own root file is found under HOME, and under the account's home when HOME is unset or empty") + func defaultRootCertificateFollowsHome() { + let fallback = NSHomeDirectory() + "/.postgresql/root.crt" + + #expect(LibPQConnectionString.defaultRootCertificatePath(environment: ["HOME": "/Users/test"]) + == "/Users/test/.postgresql/root.crt") + #expect(LibPQConnectionString.defaultRootCertificatePath(environment: ["HOME": ""]) == fallback) + #expect(LibPQConnectionString.defaultRootCertificatePath(environment: [:]) == fallback) + } + @Test("A client certificate and key reach libpq as sslcert and sslkey") func clientCertificateIsSent() throws { let conninfo = try build(sslConfig: SSLConfiguration( @@ -330,8 +424,8 @@ struct LibPQConnectionStringTests { #expect(try !build(applicationName: "").contains("application_name")) } - /// Measured on libpq 17: a fallback name replaces an `application_name` set in `options`, so it - /// has to stay out whenever the user named the session there, in any of the server's spellings. + // Measured on libpq 17: a fallback name replaces an `application_name` set in `options`, so it + // has to stay out whenever the user named the session there, in any of the server's spellings. @Test( "A name the user gives the session in the connection options keeps the fallback out", arguments: [ diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQSSLMappingTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQSSLMappingTests.swift index c20d11494d..4c5a1a3633 100644 --- a/apple/Plugins/PostgreSQLDriverTests/LibPQSSLMappingTests.swift +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQSSLMappingTests.swift @@ -32,4 +32,33 @@ struct LibPQSSLMappingTests { func verifyIdentity() { #expect(LibPQSSLMapping.sslmode(for: .verifyIdentity) == "verify-full") } + + private static let missingRootMessage = """ + root certificate file "/Users/test/.postgresql/root.crt" does not exist + Either provide the file, use the system's trusted roots with sslrootcert=system, or change sslmode to disable \ + server certificate verification. + """ + + @Test("Verify CA with no CA file names the missing CA and the mode that needs none, never sslrootcert=system") + func verifyCAWithoutACAExplainsTheMissingCA() throws { + let error = try #require( + LibPQSSLMapping.missingRootCertificateError(Self.missingRootMessage, sslConfig: SSLConfiguration(mode: .verifyCa)) + ) + + #expect(error.message.contains("Verify CA needs the server's CA certificate")) + #expect(error.message.contains("Verify Identity")) + #expect(!error.message.contains("sslrootcert")) + } + + @Test("libpq's text is kept for any other mode, for a chosen CA file, and for any other failure") + func otherFailuresKeepLibpqText() { + let otherModes = [SSLMode.verifyIdentity, .required, .preferred].map { SSLConfiguration(mode: $0) } + for sslConfig in otherModes { + #expect(LibPQSSLMapping.missingRootCertificateError(Self.missingRootMessage, sslConfig: sslConfig) == nil) + } + let withCA = SSLConfiguration(mode: .verifyCa, caCertificatePath: "/ca.pem") + #expect(LibPQSSLMapping.missingRootCertificateError(Self.missingRootMessage, sslConfig: withCA) == nil) + let refused = "connection to server at \"db\" failed: FATAL: password authentication failed for user \"app\"" + #expect(LibPQSSLMapping.missingRootCertificateError(refused, sslConfig: SSLConfiguration(mode: .verifyCa)) == nil) + } } diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift index 4fe2be6314..f8fb1604eb 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift @@ -22,30 +22,33 @@ public enum SurrealStatementGenerator { insertedRowIndices: Set ) throws(PluginRowWriteRefusal) -> [PluginRowWrite] { var writes: [PluginRowWrite] = [] + var insertedWritten: Set = [] let typing = FieldTyping(kinds: kinds, held: heldKinds, records: recordKinds, declared: declaredKinds) - for change in changes where change.type == .update && !insertedRowIndices.contains(change.rowIndex) { - let write = try update(table: table, scope: scope, columns: columns, typing: typing, change: change) - guard let write else { continue } - writes.append(write) - } - var defaults: [Int: Set] = [:] for change in changes where change.type == .insert { defaults[change.rowIndex, default: []].formUnion(change.defaultColumns ?? []) } - for index in insertedRowIndices.sorted() { - guard let values = insertedRowData[index] else { continue } - writes.append(try insert( - table: table, scope: scope, columns: columns, typing: typing, values: values, - defaults: defaults[index] ?? [], rowIndex: index - )) - } - for change in changes where change.type == .delete || deletedRowIndices.contains(change.rowIndex) { - guard !insertedRowIndices.contains(change.rowIndex) else { continue } - guard let write = delete(table: table, scope: scope, columns: columns, change: change) else { continue } - writes.append(write) + // In the order received, which is edit order: a record deleted to free a unique value goes before the one taking it. + for change in changes { + let index = change.rowIndex + if insertedRowIndices.contains(index) { + guard insertedWritten.insert(index).inserted, let values = insertedRowData[index] else { continue } + writes.append(try insert( + table: table, scope: scope, columns: columns, typing: typing, values: values, + defaults: defaults[index] ?? [], rowIndex: index + )) + continue + } + if change.type == .update, + let write = try update(table: table, scope: scope, columns: columns, typing: typing, change: change) { + writes.append(write) + } + if change.type == .delete || deletedRowIndices.contains(index), + let write = delete(table: table, scope: scope, columns: columns, change: change) { + writes.append(write) + } } return writes diff --git a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index 0810eb1d63..b1f03c7154 100644 --- a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -55,6 +55,7 @@ public struct PluginRowChange: Sendable { case delete } + // One row can arrive as several changes, each with its own key and the row as it stands before that change. public let rowIndex: Int public let type: ChangeType public let cellChanges: [(columnIndex: Int, columnName: String, oldValue: PluginCellValue, newValue: PluginCellValue)] diff --git a/apple/TablePro/Core/AI/AnthropicProvider.swift b/apple/TablePro/Core/AI/AnthropicProvider.swift index 0019e43fda..7106932e2f 100644 --- a/apple/TablePro/Core/AI/AnthropicProvider.swift +++ b/apple/TablePro/Core/AI/AnthropicProvider.swift @@ -405,7 +405,14 @@ final class AnthropicProvider: ChatTransport { let tokens = usage["output_tokens"] as? Int { state.outputTokens = tokens } - return [] + switch (json["delta"] as? [String: Any])?["stop_reason"] as? String { + case "max_tokens": + return [.outputLimitReached(.outputTokens)] + case "model_context_window_exceeded": + return [.outputLimitReached(.contextWindow)] + default: + return [] + } case "error": if let errorObj = json["error"] as? [String: Any], let message = errorObj["message"] as? String { diff --git a/apple/TablePro/Core/AI/Chat/ChatTransport.swift b/apple/TablePro/Core/AI/Chat/ChatTransport.swift index 5ef3b2230a..03f685965a 100644 --- a/apple/TablePro/Core/AI/Chat/ChatTransport.swift +++ b/apple/TablePro/Core/AI/Chat/ChatTransport.swift @@ -83,6 +83,12 @@ struct ChatToolSpec: Codable, Equatable, Sendable { } } +// Why a provider stopped the response before the model finished it. +enum ChatOutputLimit: String, Codable, Sendable { + case outputTokens + case contextWindow +} + enum ChatStreamEvent: Sendable { case textDelta(String) case toolUseStart(id: String, name: String, providerMetadata: [String: String]? = nil) @@ -93,6 +99,7 @@ enum ChatStreamEvent: Sendable { case reasoningStart(id: String) case reasoningDelta(id: String, text: String) case reasoningEnd(id: String, opaque: ReasoningOpaque?) + case outputLimitReached(ChatOutputLimit) } final class ToolReplyToken: Sendable { diff --git a/apple/TablePro/Core/AI/Chat/ChatTurn.swift b/apple/TablePro/Core/AI/Chat/ChatTurn.swift index b7bb4c1d5f..429a06f5b5 100644 --- a/apple/TablePro/Core/AI/Chat/ChatTurn.swift +++ b/apple/TablePro/Core/AI/Chat/ChatTurn.swift @@ -105,6 +105,7 @@ final class ChatTurn: ObservableObject, Identifiable { @Published var usage: AITokenUsage? @Published var modelId: String? @Published var providerId: String? + @Published var outputLimit: ChatOutputLimit? init( id: UUID = UUID(), @@ -113,7 +114,8 @@ final class ChatTurn: ObservableObject, Identifiable { timestamp: Date = Date(), usage: AITokenUsage? = nil, modelId: String? = nil, - providerId: String? = nil + providerId: String? = nil, + outputLimit: ChatOutputLimit? = nil ) { self.id = id self.role = role @@ -122,6 +124,7 @@ final class ChatTurn: ObservableObject, Identifiable { self.usage = usage self.modelId = modelId self.providerId = providerId + self.outputLimit = outputLimit } init(wire: ChatTurnWire) { @@ -132,6 +135,7 @@ final class ChatTurn: ObservableObject, Identifiable { self.usage = wire.usage self.modelId = wire.modelId self.providerId = wire.providerId + self.outputLimit = wire.outputLimit } var plainText: String { @@ -152,7 +156,8 @@ final class ChatTurn: ObservableObject, Identifiable { timestamp: timestamp, usage: usage, modelId: modelId, - providerId: providerId + providerId: providerId, + outputLimit: outputLimit ) } @@ -337,6 +342,7 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { var usage: AITokenUsage? var modelId: String? var providerId: String? + var outputLimit: ChatOutputLimit? init( id: UUID = UUID(), @@ -345,7 +351,8 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { timestamp: Date = Date(), usage: AITokenUsage? = nil, modelId: String? = nil, - providerId: String? = nil + providerId: String? = nil, + outputLimit: ChatOutputLimit? = nil ) { self.id = id self.role = role @@ -354,6 +361,7 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { self.usage = usage self.modelId = modelId self.providerId = providerId + self.outputLimit = outputLimit } init(from decoder: Decoder) throws { @@ -364,6 +372,10 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { usage = try container.decodeIfPresent(AITokenUsage.self, forKey: .usage) modelId = try container.decodeIfPresent(String.self, forKey: .modelId) providerId = try container.decodeIfPresent(String.self, forKey: .providerId) + // A value a later version wrote reads as no note rather than failing the conversation. + outputLimit = (try? container.decodeIfPresent(String.self, forKey: .outputLimit)).flatMap { + $0.flatMap(ChatOutputLimit.init(rawValue:)) + } if let decodedBlocks = try container.decodeIfPresent([LossyContentBlockWire].self, forKey: .blocks) { blocks = decodedBlocks.compactMap(\.block) @@ -386,6 +398,7 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { try container.encodeIfPresent(usage, forKey: .usage) try container.encodeIfPresent(modelId, forKey: .modelId) try container.encodeIfPresent(providerId, forKey: .providerId) + try container.encodeIfPresent(outputLimit, forKey: .outputLimit) } var plainText: String { @@ -399,7 +412,7 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { } private enum CodingKeys: String, CodingKey { - case id, role, blocks, timestamp, usage, modelId, providerId + case id, role, blocks, timestamp, usage, modelId, providerId, outputLimit } private enum LegacyKeys: String, CodingKey { diff --git a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift index e21593ca8f..cdf25b831a 100644 --- a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift +++ b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift @@ -51,6 +51,8 @@ final class ClaudeAgentProvider: ChatTransport { continuation.yield(.textDelta(text)) case .usage(let usage): continuation.yield(.usage(usage)) + case .outputLimit(let limit): + continuation.yield(.outputLimitReached(limit)) case .failure(let message): await toolBridge.release(grant) grant = nil @@ -90,6 +92,7 @@ final class ClaudeAgentProvider: ChatTransport { enum DecodedEvent: Equatable { case text(String) case usage(AITokenUsage) + case outputLimit(ChatOutputLimit) case failure(String) } @@ -129,6 +132,7 @@ final class ClaudeAgentProvider: ChatTransport { guard let json = decodeJSON(line), let type = json["type"] as? String else { return nil } switch type { case "stream_event": + if let limit = outputLimit(json) { return .outputLimit(limit) } guard let text = partialText(json) else { return nil } return .text(text) case "result": @@ -149,6 +153,18 @@ final class ClaudeAgentProvider: ChatTransport { return text } + // The CLI passes the Messages API stream through, so a cut response ends with its message_delta stop reason. + static func outputLimit(_ json: [String: Any]) -> ChatOutputLimit? { + guard let event = json["event"] as? [String: Any], + event["type"] as? String == "message_delta", + let delta = event["delta"] as? [String: Any] else { return nil } + switch delta["stop_reason"] as? String { + case "max_tokens": return .outputTokens + case "model_context_window_exceeded": return .contextWindow + default: return nil + } + } + static func resultEvent(_ json: [String: Any]) -> DecodedEvent? { if json["is_error"] as? Bool == true { let message = (json["result"] as? String).flatMap { $0.isEmpty ? nil : $0 } diff --git a/apple/TablePro/Core/AI/GeminiProvider.swift b/apple/TablePro/Core/AI/GeminiProvider.swift index b59e1a7663..b2197bd9cd 100644 --- a/apple/TablePro/Core/AI/GeminiProvider.swift +++ b/apple/TablePro/Core/AI/GeminiProvider.swift @@ -321,6 +321,10 @@ final class GeminiProvider: ChatTransport { } } } + if let candidates = json["candidates"] as? [[String: Any]], + candidates.first?["finishReason"] as? String == "MAX_TOKENS" { + events.append(.outputLimitReached(.outputTokens)) + } if let usageMetadata = json["usageMetadata"] as? [String: Any] { if let prompt = usageMetadata["promptTokenCount"] as? Int { state.inputTokens = prompt diff --git a/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift b/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift index 9a57205e05..03e7c264ec 100644 --- a/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift +++ b/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift @@ -175,6 +175,13 @@ final class OpenAICompatibleProvider: ChatTransport { if finishReason == "tool_calls" { events.append(contentsOf: state.flushToolUseEnds()) } + if finishReason == "length" { + events.append(.outputLimitReached(.outputTokens)) + } + } + // Ollama's own API names the cut in `done_reason`. + if (json["done"] as? Bool) == true, (json["done_reason"] as? String) == "length" { + events.append(.outputLimitReached(.outputTokens)) } if let usage = json["usage"] as? [String: Any], diff --git a/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift b/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift index 43729ee0ef..2ab633f0f4 100644 --- a/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift +++ b/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift @@ -428,15 +428,17 @@ final class OpenAIResponsesProvider: ChatTransport { return [.textDelta(delta)] } return [] - case "response.completed": - if let responseObj = json["response"] as? [String: Any], - let usage = responseObj["usage"] as? [String: Any] { + case "response.completed", "response.incomplete": + let responseObj = json["response"] as? [String: Any] + if let usage = responseObj?["usage"] as? [String: Any] { let input = usage["input_tokens"] as? Int ?? 0 let output = usage["output_tokens"] as? Int ?? 0 state.inputTokens = input state.outputTokens = output } - return [] + let details = responseObj?["incomplete_details"] as? [String: Any] + guard type == "response.incomplete", details?["reason"] as? String == "max_output_tokens" else { return [] } + return [.outputLimitReached(.outputTokens)] case "response.failed": if let responseObj = json["response"] as? [String: Any], let errorObj = responseObj["error"] as? [String: Any], diff --git a/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift b/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift index 1c6d8f757c..4ceae454c5 100644 --- a/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift +++ b/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift @@ -109,6 +109,34 @@ final class DataChangeManager: ObservableObject, ChangeManaging { private var loadedValues: [RowID: [Int: PluginCellValue]] = [:] private var coalescedOrder: [CoalescedCellKey] = [] + // Process-wide, so a run restored from another tab never shares an id with a new command. + private static var lastActionID = 0 + private var commandAction: Int? + private var typingAction: Int? + + private static func newActionID() -> Int { + lastActionID += 1 + return lastActionID + } + + // One undo step over many cells, so a row it writes more than once stays one write in the save. + func recordAsOneAction(_ body: () -> Void) { + guard commandAction == nil else { + body() + return + } + commandAction = Self.newActionID() + defer { commandAction = nil } + body() + } + + private func typingRunAction() -> Int { + if let typingAction { return typingAction } + let action = Self.newActionID() + typingAction = action + return action + } + /// Whether a typed edit is waiting to become an undo step. The Edit menu reads it, because the /// undo manager has nothing registered yet while the run is open. var hasCoalescedUndoRun: Bool { !coalescedOrder.isEmpty } @@ -323,6 +351,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { wasExpression: pending.isExpression(rowID: rowID, columnIndex: columnIndex), isExpression: isExpression ) + let action = coalescesWithPrevious ? typingRunAction() : commandAction ?? Self.newActionID() objectWillChange.send() let recorded = pending.recordCellChange( rowID: rowID, @@ -332,7 +361,8 @@ final class DataChangeManager: ObservableObject, ChangeManaging { newValue: newValue, originalRow: originalRow, absence: absence, - isExpression: isExpression + isExpression: isExpression, + action: action ) guard recorded else { hasChanges = !pending.isEmpty @@ -346,11 +376,14 @@ final class DataChangeManager: ObservableObject, ChangeManaging { ) } else { registerUndo(actionName: String(localized: "Edit Cell")) { target in - target.applyDataUndo(.cellEdit( - rowID: rowID, columnIndex: columnIndex, columnName: columnName, - previousValue: oldValue, newValue: newValue, originalRow: originalRow, - absence: absence, expression: expression - )) + target.applyDataUndo( + .cellEdit( + rowID: rowID, columnIndex: columnIndex, columnName: columnName, + previousValue: oldValue, newValue: newValue, originalRow: originalRow, + absence: absence, expression: expression + ), + actionID: action + ) } } hasChanges = !pending.isEmpty @@ -382,25 +415,28 @@ final class DataChangeManager: ObservableObject, ChangeManaging { coalescedOrder.append(key) } - /// Registers the run's cells as one undo step, in the order they were first written. - /// - /// A cell typed back to where it started contributes nothing: registering it would leave a step - /// that restores a value the cell already holds, and undoing it would record a pending change - /// over a value that already matches the server. + // A cell typed back to where it started is left out: undoing it would stage a value the server already holds. func endCoalescedUndoRun() { - guard !coalescedOrder.isEmpty else { return } + guard !coalescedOrder.isEmpty else { + typingAction = nil + return + } let edits = coalescedOrder.compactMap { coalescedEdits[$0] }.filter { !$0.isNoOp } + let action = typingAction discardCoalescedUndoRun() guard !edits.isEmpty, let undoManager = undoManagerProvider?() else { return } undoManager.beginUndoGrouping() for edit in edits { undoManager.registerUndo(withTarget: self) { target in - target.applyDataUndo(.cellEdit( - rowID: edit.rowID, columnIndex: edit.columnIndex, columnName: edit.columnName, - previousValue: edit.previousValue, newValue: edit.newValue, originalRow: edit.originalRow, - absence: edit.absence, expression: edit.expression - )) + target.applyDataUndo( + .cellEdit( + rowID: edit.rowID, columnIndex: edit.columnIndex, columnName: edit.columnName, + previousValue: edit.previousValue, newValue: edit.newValue, originalRow: edit.originalRow, + absence: edit.absence, expression: edit.expression + ), + actionID: action + ) } } undoManager.setActionName(String(localized: "Edit Cell")) @@ -410,6 +446,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { private func discardCoalescedUndoRun() { coalescedEdits.removeAll() coalescedOrder.removeAll() + typingAction = nil } private func discardUndoHistory() { @@ -560,7 +597,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { // MARK: - Core Undo Application - private func applyDataUndo(_ action: UndoAction) { + private func applyDataUndo(_ action: UndoAction, actionID: Int? = nil) { objectWillChange.send() switch action { case .cellEdit(let rowID, let columnIndex, let columnName, let previousValue, let newValue, let originalRow, @@ -568,7 +605,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { applyCellEditUndo( rowID: rowID, columnIndex: columnIndex, columnName: columnName, previousValue: previousValue, newValue: newValue, originalRow: originalRow, - absence: absence, expression: expression, action: action + absence: absence, expression: expression, action: action, actionID: actionID ) case .rowInsertion(let rowID, let image): @@ -597,20 +634,25 @@ final class DataChangeManager: ObservableObject, ChangeManaging { private func applyCellEditUndo( rowID: RowID, columnIndex: Int, columnName: String, previousValue: PluginCellValue, newValue: PluginCellValue, originalRow: [PluginCellValue]?, - absence: FieldAbsence, expression: ExpressionTransition, action: UndoAction + absence: FieldAbsence, expression: ExpressionTransition, action: UndoAction, actionID: Int? ) { + let isUndoing = undoManagerProvider?()?.isUndoing ?? false registerUndo(actionName: String(localized: "Edit Cell")) { target in - target.applyDataUndo(.cellEdit( - rowID: rowID, columnIndex: columnIndex, columnName: columnName, - previousValue: newValue, newValue: previousValue, originalRow: originalRow, - absence: absence.reversed, expression: expression.reversed - )) + target.applyDataUndo( + .cellEdit( + rowID: rowID, columnIndex: columnIndex, columnName: columnName, + previousValue: newValue, newValue: previousValue, originalRow: originalRow, + absence: absence.reversed, expression: expression.reversed + ), + actionID: actionID + ) } + // A redo is a new write, so it goes back after the other rows' changes still pending, where it was. let updateHoldsCell = pending.change(forRow: rowID, type: .update)? .cellChanges.contains { $0.columnIndex == columnIndex } ?? false - if updateHoldsCell { - pending.revertUpdateCell( + if updateHoldsCell, isUndoing { + pending.revertCellWrite( rowID: rowID, columnIndex: columnIndex, columnName: columnName, previousValue: previousValue, previousIsAbsent: absence.wasAbsent, @@ -627,7 +669,8 @@ final class DataChangeManager: ObservableObject, ChangeManaging { rowID: rowID, columnIndex: columnIndex, columnName: columnName, originalDBValue: newValue, newValue: previousValue, originalRow: originalRow, - absence: absence.reversed, expression: expression.reversed + absence: absence.reversed, expression: expression.reversed, + action: actionID ) } lastUndoResult = UndoResult( diff --git a/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift b/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift index f86e46a25c..a3910c6c8f 100644 --- a/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift +++ b/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift @@ -79,6 +79,47 @@ struct CellChange: Identifiable, Equatable { var restoresOriginal: Bool { oldValue == newValue && oldIsAbsent == newIsAbsent && oldIsExpression == newIsExpression } + + func followed(by later: CellChange) -> CellChange { + CellChange( + columnIndex: columnIndex, + columnName: later.columnName, + oldValue: oldValue, + newValue: later.newValue, + oldIsAbsent: oldIsAbsent, + newIsAbsent: later.newIsAbsent, + oldIsExpression: oldIsExpression, + newIsExpression: later.newIsExpression + ) + } +} + +// A cell's old value is the cell as this run found it, not as the row was read. +struct UpdateRun: Equatable { + let sequence: Int + let action: Int + var cellChanges: [CellChange] + + func holds(_ columnIndex: Int) -> Bool { + cellChanges.contains { $0.columnIndex == columnIndex } + } + + func writesExpression(into columns: Set) -> Bool { + cellChanges.contains { $0.newIsExpression && columns.contains($0.columnIndex) } + } + + mutating func write(_ cellChange: CellChange) { + guard let index = cellChanges.firstIndex(where: { $0.columnIndex == cellChange.columnIndex }) else { + cellChanges.append(cellChange) + return + } + let written = cellChanges[index].followed(by: cellChange) + if written.restoresOriginal { + cellChanges.remove(at: index) + } else { + cellChanges[index] = written + } + } } struct RowChange: Identifiable, Equatable { @@ -104,6 +145,9 @@ struct RowChange: Identifiable, Equatable { /// and reusing its unique value in a new one only works in that order. var sequence: Int + // An update's edits in the order they were made. Empty reads as one run at `sequence`. + var runs: [UpdateRun] + init( rowID: RowID, type: ChangeType, @@ -111,7 +155,8 @@ struct RowChange: Identifiable, Equatable { originalRow: [PluginCellValue]? = nil, absentColumns: Set = [], expressionColumns: Set = [], - sequence: Int = 0 + sequence: Int = 0, + runs: [UpdateRun] = [] ) { self.id = UUID() self.rowID = rowID @@ -121,6 +166,76 @@ struct RowChange: Identifiable, Equatable { self.absentColumns = absentColumns self.expressionColumns = expressionColumns self.sequence = sequence + self.runs = runs + } + + // A row keeps one UPDATE at its first run when a later step could not find it: with no key, a keyless match fails + // once the server rewrites a column on the first UPDATE, and an earlier run's SQL in a key is not the key it wrote. + static func saveSteps(_ changes: [RowChange], keyColumns: Set) -> [RowChange] { + var steps: [(step: RowChange, source: Int)] = [] + for (source, change) in changes.enumerated() { + guard change.type == .update, let first = change.runs.first else { + steps.append((change, source)) + continue + } + guard !keyColumns.isEmpty, change.runs.count > 1, var row = change.originalRow, + !change.runs.dropLast().contains(where: { $0.writesExpression(into: keyColumns) }) + else { + var placed = change + placed.sequence = first.sequence + steps.append((placed, source)) + continue + } + var absent = change.absentColumns + for run in change.runs { + steps.append(( + RowChange( + rowID: change.rowID, type: .update, cellChanges: run.cellChanges, originalRow: row, + absentColumns: absent, expressionColumns: change.expressionColumns, sequence: run.sequence + ), + source + )) + for cellChange in run.cellChanges { + if row.indices.contains(cellChange.columnIndex) { + row[cellChange.columnIndex] = cellChange.newValue + } + if cellChange.newIsAbsent { + absent.insert(cellChange.columnIndex) + } else { + absent.remove(cellChange.columnIndex) + } + } + } + } + + let ordered = steps.enumerated() + .sorted { ($0.element.step.sequence, $0.offset) < ($1.element.step.sequence, $1.offset) } + .map(\.element) + // Two runs with nothing left between them, once a change recorded there was cancelled, are one UPDATE. + var merged: [(step: RowChange, source: Int)] = [] + for entry in ordered { + if let last = merged.last, last.source == entry.source, entry.step.type == .update { + merged[merged.count - 1].step = last.step.followed(by: entry.step) + } else { + merged.append(entry) + } + } + return merged.map(\.step) + } + + private func followed(by later: RowChange) -> RowChange { + var cells = cellChanges + for cellChange in later.cellChanges { + if let index = cells.firstIndex(where: { $0.columnIndex == cellChange.columnIndex }) { + cells[index] = cells[index].followed(by: cellChange) + } else { + cells.append(cellChange) + } + } + return RowChange( + rowID: rowID, type: type, cellChanges: cells, originalRow: originalRow, + absentColumns: absentColumns, expressionColumns: expressionColumns, sequence: sequence + ) } // A value loaded after the change was recorded replaces the placeholder its pre-image captured. diff --git a/apple/TablePro/Core/ChangeTracking/PendingChanges.swift b/apple/TablePro/Core/ChangeTracking/PendingChanges.swift index c4882458bd..e9d21fe2bc 100644 --- a/apple/TablePro/Core/ChangeTracking/PendingChanges.swift +++ b/apple/TablePro/Core/ChangeTracking/PendingChanges.swift @@ -2,12 +2,6 @@ // PendingChanges.swift // TablePro // -// Value type holding all uncommitted edits to a result set. -// Owns the consistency invariants between `changes`, `changeIndex`, -// `deletedRowIDs`, `insertedRowIDs`, `modifiedCells`, and -// `insertedRowData`. Callers mutate through methods that maintain -// the cross-collection state. -// import Foundation import TableProPluginKit @@ -16,9 +10,7 @@ struct PendingChanges: Equatable { private(set) var changes: [RowChange] = [] private(set) var deletedRowIDs: Set = [] - /// Stamped onto every change so statement generation can recover the order the user worked in. - /// `changes` cannot carry that order itself: a cancelled change is removed by swapping the last - /// element into its slot. + // `changes` cannot carry the edit order: a cancelled change is swap-removed from it. private var nextSequence = 0 private(set) var insertedRowIDs: Set = [] private(set) var modifiedCells: [RowID: Set] = [:] @@ -108,11 +100,7 @@ struct PendingChanges: Equatable { // MARK: - Mutate (recording user edits) - /// Whether the recorded edit is a no-op (oldValue == newValue with no prior modification). - /// Returns the result so the caller can decide whether to register undo. - /// - /// A field going missing, or coming back, is a change even when the value reads `.null` on - /// both sides. So is an expression put in place of text that reads the same, or text in place of one. + // False for a no-op, so no undo is registered. A nil `action` is an undo step of its own. @discardableResult mutating func recordCellChange( rowID: RowID, @@ -122,7 +110,8 @@ struct PendingChanges: Equatable { newValue: PluginCellValue, originalRow: [PluginCellValue]? = nil, absence: FieldAbsence = FieldAbsence(), - isExpression: Bool = false + isExpression: Bool = false, + action: Int? = nil ) -> Bool { let wasExpression = self.isExpression(rowID: rowID, columnIndex: columnIndex) if oldValue == newValue && absence.wasAbsent == absence.isAbsent && wasExpression == isExpression { @@ -152,14 +141,12 @@ struct PendingChanges: Equatable { } if let updateIdx = changeIndex[RowChangeKey(rowID: rowID, type: .update)] { - mergeUpdateCell(at: updateIdx, cellChange: cellChange) + mergeUpdateCell(at: updateIdx, cellChange: cellChange, action: action) } else { - appendChange(RowChange( - rowID: rowID, type: .update, - cellChanges: [cellChange], originalRow: originalRow, - absentColumns: absence.originalRow - )) - modifiedCells[rowID, default: []].insert(columnIndex) + appendUpdate( + rowID: rowID, cellChange: cellChange, originalRow: originalRow, + absentColumns: absence.originalRow, action: action + ) } return true } @@ -246,9 +233,7 @@ struct PendingChanges: Equatable { recordRowDeletion(rowID: rowID, originalRow: originalRow, absentColumns: absentColumns) } - /// Re-apply a cell edit during undo replay (skips undo registration). - /// `originalDBValue` is the cell's value in the unmodified database row. - /// It must be preserved so that a later collapse compares correctly. + // `originalDBValue` is the cell as it stands before this write. mutating func reapplyCellChange( rowID: RowID, columnIndex: Int, @@ -257,7 +242,8 @@ struct PendingChanges: Equatable { newValue: PluginCellValue, originalRow: [PluginCellValue]?, absence: FieldAbsence = FieldAbsence(), - expression: ExpressionTransition = ExpressionTransition() + expression: ExpressionTransition = ExpressionTransition(), + action: Int? = nil ) { let cellChange = CellChange( columnIndex: columnIndex, @@ -279,14 +265,12 @@ struct PendingChanges: Equatable { } if let updateIdx = changeIndex[RowChangeKey(rowID: rowID, type: .update)] { - mergeUpdateCell(at: updateIdx, cellChange: cellChange) + mergeUpdateCell(at: updateIdx, cellChange: cellChange, action: action) } else { - appendChange(RowChange( - rowID: rowID, type: .update, - cellChanges: [cellChange], originalRow: originalRow, - absentColumns: absence.originalRow - )) - modifiedCells[rowID, default: []].insert(columnIndex) + appendUpdate( + rowID: rowID, cellChange: cellChange, originalRow: originalRow, + absentColumns: absence.originalRow, action: action + ) } } @@ -306,8 +290,8 @@ struct PendingChanges: Equatable { ) } - /// Restore a cell's value during undo replay when an existing change matches. - mutating func revertUpdateCell( + // Undo takes back the cell's latest write, which the newest run holding the cell made. + mutating func revertCellWrite( rowID: RowID, columnIndex: Int, columnName: String, @@ -316,29 +300,24 @@ struct PendingChanges: Equatable { previousIsExpression: Bool = false ) { guard let updateIdx = changeIndex[RowChangeKey(rowID: rowID, type: .update)], - let cellIdx = changes[updateIdx].cellChanges.firstIndex(where: { $0.columnIndex == columnIndex }) + let current = changes[updateIdx].cellChanges.first(where: { $0.columnIndex == columnIndex }) else { return } - let original = changes[updateIdx].cellChanges[cellIdx] - if previousValue == original.oldValue && previousIsAbsent == original.oldIsAbsent - && previousIsExpression == original.oldIsExpression { - changes[updateIdx].cellChanges.remove(at: cellIdx) - removeModifiedCell(rowID: rowID, columnIndex: columnIndex) - if changes[updateIdx].cellChanges.isEmpty { - removeChangeAt(updateIdx) - } - } else { - changes[updateIdx].cellChanges[cellIdx] = CellChange( - columnIndex: columnIndex, - columnName: columnName, - oldValue: original.oldValue, - newValue: previousValue, - oldIsAbsent: original.oldIsAbsent, - newIsAbsent: previousIsAbsent, - oldIsExpression: original.oldIsExpression, - newIsExpression: previousIsExpression - ) - } + materializeRuns(at: updateIdx) + let reverted = CellChange( + columnIndex: columnIndex, + columnName: columnName, + oldValue: current.newValue, + newValue: previousValue, + oldIsAbsent: current.newIsAbsent, + newIsAbsent: previousIsAbsent, + oldIsExpression: current.newIsExpression, + newIsExpression: previousIsExpression + ) + guard writeNetCell(reverted, at: updateIdx), + let runIdx = changes[updateIdx].runs.lastIndex(where: { $0.holds(columnIndex) }) + else { return } + write(reverted, intoRun: runIdx, at: updateIdx) } /// Insert a synthetic .insert RowChange for undo replay (e.g., after redoing a deletion's undo). @@ -430,7 +409,7 @@ struct PendingChanges: Equatable { modifiedCells = snapshot.modifiedCells insertedRowData = snapshot.insertedRowData supersededUpdates = snapshot.supersededUpdates - let sequences = changes.map(\.sequence) + supersededUpdates.values.map(\.sequence) + let sequences = (changes + supersededUpdates.values).flatMap { [$0.sequence] + $0.runs.map(\.sequence) } nextSequence = (sequences.max() ?? -1) + 1 rebuildChangeIndex() } @@ -458,6 +437,29 @@ struct PendingChanges: Equatable { changeIndex[RowChangeKey(rowID: stamped.rowID, type: stamped.type)] = changes.count - 1 } + private mutating func appendUpdate( + rowID: RowID, + cellChange: CellChange, + originalRow: [PluginCellValue]?, + absentColumns: Set, + action: Int? + ) { + let run = UpdateRun( + sequence: nextSequence, action: action ?? Self.ownAction(at: nextSequence), cellChanges: [cellChange] + ) + appendChange(RowChange( + rowID: rowID, type: .update, + cellChanges: [cellChange], originalRow: originalRow, + absentColumns: absentColumns, runs: [run] + )) + modifiedCells[rowID, default: []].insert(cellChange.columnIndex) + } + + // Below every id DataChangeManager hands out, so no other write ever joins it. + private static func ownAction(at sequence: Int) -> Int { + -(sequence + 1) + } + @discardableResult private mutating func removeChange(rowID: RowID, type: ChangeType) -> Bool { let key = RowChangeKey(rowID: rowID, type: type) @@ -528,32 +530,72 @@ struct PendingChanges: Equatable { } } - private mutating func mergeUpdateCell(at updateIdx: Int, cellChange: CellChange) { + private mutating func mergeUpdateCell(at updateIdx: Int, cellChange: CellChange, action: Int?) { + materializeRuns(at: updateIdx) + guard writeNetCell(cellChange, at: updateIdx) else { return } + + // The newest run takes the write when the same step opened it or nothing was recorded since. + if let newest = changes[updateIdx].runs.indices.last { + let run = changes[updateIdx].runs[newest] + if (action != nil && run.action == action) || run.sequence == nextSequence - 1 { + write(cellChange, intoRun: newest, at: updateIdx) + return + } + } + changes[updateIdx].runs.append(UpdateRun( + sequence: nextSequence, action: action ?? Self.ownAction(at: nextSequence), cellChanges: [cellChange] + )) + nextSequence += 1 + } + + // An update built outside this type reads as one run at its own place. + private mutating func materializeRuns(at updateIdx: Int) { + let update = changes[updateIdx] + guard update.runs.isEmpty, !update.cellChanges.isEmpty else { return } + changes[updateIdx].runs = [UpdateRun( + sequence: update.sequence, action: Self.ownAction(at: update.sequence), cellChanges: update.cellChanges + )] + } + + // False when the write left the cell as it was read, so no run writes it. + private mutating func writeNetCell(_ cellChange: CellChange, at updateIdx: Int) -> Bool { let rowID = changes[updateIdx].rowID guard let cellIdx = changes[updateIdx].cellChanges.firstIndex(where: { $0.columnIndex == cellChange.columnIndex }) else { changes[updateIdx].cellChanges.append(cellChange) modifiedCells[rowID, default: []].insert(cellChange.columnIndex) - return + return true } - let original = changes[updateIdx].cellChanges[cellIdx] - let merged = CellChange( - columnIndex: cellChange.columnIndex, - columnName: cellChange.columnName, - oldValue: original.oldValue, - newValue: cellChange.newValue, - oldIsAbsent: original.oldIsAbsent, - newIsAbsent: cellChange.newIsAbsent, - oldIsExpression: original.oldIsExpression, - newIsExpression: cellChange.newIsExpression - ) - changes[updateIdx].cellChanges[cellIdx] = merged + let merged = changes[updateIdx].cellChanges[cellIdx].followed(by: cellChange) + guard merged.restoresOriginal else { + changes[updateIdx].cellChanges[cellIdx] = merged + return true + } + removeUpdateCell(at: updateIdx, cellIdx: cellIdx) + return false + } + + private mutating func write(_ cellChange: CellChange, intoRun runIdx: Int, at updateIdx: Int) { + changes[updateIdx].runs[runIdx].write(cellChange) + if changes[updateIdx].runs[runIdx].cellChanges.isEmpty { + changes[updateIdx].runs.remove(at: runIdx) + } + } - guard merged.restoresOriginal else { return } + // A cell the grid shows unchanged leaves every run too, so no trigger fires for a value nobody changed. + private mutating func removeUpdateCell(at updateIdx: Int, cellIdx: Int) { + let rowID = changes[updateIdx].rowID + let columnIndex = changes[updateIdx].cellChanges[cellIdx].columnIndex changes[updateIdx].cellChanges.remove(at: cellIdx) - removeModifiedCell(rowID: rowID, columnIndex: cellChange.columnIndex) + removeModifiedCell(rowID: rowID, columnIndex: columnIndex) + for runIdx in changes[updateIdx].runs.indices.reversed() { + changes[updateIdx].runs[runIdx].cellChanges.removeAll { $0.columnIndex == columnIndex } + if changes[updateIdx].runs[runIdx].cellChanges.isEmpty { + changes[updateIdx].runs.remove(at: runIdx) + } + } if changes[updateIdx].cellChanges.isEmpty { removeChangeAt(updateIdx) } @@ -575,11 +617,7 @@ struct PendingChanges: Equatable { changes[updateIdx].cellChanges[cellIdx].oldIsExpression == restoredIsExpression else { return false } - changes[updateIdx].cellChanges.remove(at: cellIdx) - removeModifiedCell(rowID: rowID, columnIndex: columnIndex) - if changes[updateIdx].cellChanges.isEmpty { - removeChangeAt(updateIdx) - } + removeUpdateCell(at: updateIdx, cellIdx: cellIdx) return true } } diff --git a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift index 21df0ea117..5583026125 100644 --- a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift +++ b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift @@ -17,12 +17,12 @@ struct ParameterizedStatement: @unchecked Sendable { let parameters: [Any?] } -/// A statement plus the rows it is meant to touch, so a caller can hold the server to that count -/// and tell a change that became a statement from one that did not. +// The rows it should touch hold the server to a count; the changes it writes tell a written change from a dropped one. struct AttributedStatement: @unchecked Sendable { let statement: ParameterizedStatement let kind: RowWriteKind let rowIDs: [RowID] + let changeIDs: [UUID] var refusesSharedKey = false var rowCount: Int { rowIDs.count } @@ -125,16 +125,8 @@ struct SQLStatementGenerator { ).map(\.statement) } - /// The same statements, each carrying how many rows it is meant to touch. - /// - /// Emitted in the order the user made the changes, because that order can be load-bearing: a - /// row deleted to free a unique value, and a new row taking that value, only work if the - /// DELETE runs first. Grouping every INSERT ahead of every DELETE, which is what this used to - /// do, turned that save into a constraint violation and rolled the whole thing back. - /// - /// Deletes still batch, but only across a consecutive run of them. That keeps the case that - /// matters, selecting many rows and pressing Delete, on one statement, while a delete - /// separated from another delete by an insert stays on its own side of it. + // In edit order, since a row deleted to free a unique value must go before the row that takes it. Only a + // consecutive run of deletes batches. func generateAttributedStatements( from changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]], @@ -156,7 +148,8 @@ struct SQLStatementGenerator { flushDeleteRun() if let stmt = generateUpdateSQL(for: change) { statements.append(AttributedStatement( - statement: stmt, kind: .update, rowIDs: [change.rowID], refusesSharedKey: matchesEitherWay(change) + statement: stmt, kind: .update, rowIDs: [change.rowID], changeIDs: [change.id], + refusesSharedKey: matchesEitherWay(change) )) } case .insert: @@ -164,7 +157,9 @@ struct SQLStatementGenerator { guard insertedRowIDs.contains(change.rowID) else { continue } flushDeleteRun() if let stmt = generateInsertSQL(for: change, insertedRowData: insertedRowData) { - statements.append(AttributedStatement(statement: stmt, kind: .insert, rowIDs: [change.rowID])) + statements.append(AttributedStatement( + statement: stmt, kind: .insert, rowIDs: [change.rowID], changeIDs: [change.id] + )) } case .delete: // SAFETY: Verify the row is still marked as deleted @@ -553,12 +548,12 @@ struct SQLStatementGenerator { private func generateDeleteStatements(for changes: [RowChange]) -> [AttributedStatement] { let rowMatches = changes.compactMap { change in - deleteRowMatches(for: change).map { (rowID: change.rowID, matches: $0) } + deleteRowMatches(for: change).map { (rowID: change.rowID, changeID: change.id, matches: $0) } } guard !rowMatches.isEmpty else { return [] } var statements: [AttributedStatement] = [] - var chunk: [(rowID: RowID, matches: [DeleteColumnMatch])] = [] + var chunk: [(rowID: RowID, changeID: UUID, matches: [DeleteColumnMatch])] = [] var chunkParameterCount = 0 func flush() { @@ -567,6 +562,7 @@ struct SQLStatementGenerator { statement: deleteStatement(for: chunk.map(\.matches)), kind: .delete, rowIDs: chunk.map(\.rowID), + changeIDs: chunk.map(\.changeID), refusesSharedKey: chunk.contains { $0.matches.contains { matchesEitherWay($0.boundValue) } } ) ) diff --git a/apple/TablePro/Core/Coordinators/FindCoordinator.swift b/apple/TablePro/Core/Coordinators/FindCoordinator.swift index 146c4ac2af..0d6b7f247a 100644 --- a/apple/TablePro/Core/Coordinators/FindCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/FindCoordinator.swift @@ -19,7 +19,8 @@ final class FindCoordinator { var hasActiveFind: Bool { guard let find = model?.find else { return false } - return find.isVisible && !find.matches.isEmpty + // A typed term's search has no matches yet; Find Next waits for it rather than going to the editor. + return find.isVisible && (!find.matches.isEmpty || find.isAwaitingLoadedRows) } func showFind(replace: Bool = false) { diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift index 12236dcb7f..3072505f7c 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift @@ -71,6 +71,8 @@ private struct PrintedBatchAnswer: CarriesServerOutput { private struct BatchResultPlan { let queries: [[SQLStatementScanner.ExecutableStatement]?] let built: StatementRowsBuild + // Over every result the batches produce, in order: each result set, and one count result per batch with none. + let results: RunResultPlan } private struct BatchRun { @@ -271,6 +273,11 @@ extension QueryExecutionCoordinator { ) { batch in try await Self.runBatch(batch, driver: driver, failureOutput: failureOutput) } + await DatabaseManager.shared.noteSessionStatements( + prepared.prefix(outcome.ranCount(of: prepared.count)).flatMap { $0.batch.statements.map(\.sql) }, + ranOn: driver, + scope: scope + ) let sessionState = await driver.heldSessionTransactionState() return BatchRun(outcome: outcome, plan: plan, sessionState: sessionState, startState: startState) } @@ -522,47 +529,83 @@ extension QueryExecutionCoordinator { let grammar = parent.lexicalGrammar let firstNumber = parent.tabManager.tabs.first { $0.id == tabId }?.display.pinnedResultCount ?? 0 let reading = parent.tableNameReading + let runPlan = plan.results var builtIndex = 0 // Taken in the order `batchResultPlan` built them: batch by batch, each batch's result sets in order. func nextBuilt() -> StatementRows? { defer { builtIndex += 1 } return plan.built.statement(at: builtIndex, readBy: reading) } - var resultSets: [ResultSet] = [] + var own: [Int: ResultSet] = [:] + var parts: [Int: RunSummaryPart] = [:] + var candidate = 0 + // A result left out by the plan is never built; one the summary covers keeps only its counts and output. + func place(_ result: QueryResult?, count output: BatchOutput? = nil, build: (Int) -> ResultSet) { + defer { candidate += 1 } + if let position = runPlan.position(ofStatement: candidate) { + own[candidate] = build(firstNumber + position) + } else if runPlan.summarizes(statement: candidate) { + if let result { + parts[candidate] = RunSummaryPart(result) + } else if let output { + parts[candidate] = RunSummaryPart( + rowsAffected: output.rowsAffected, + executionTime: output.executionTime, + serverOutput: .none, + statusMessage: nil + ) + } + } + } for ((entry, output), queries) in zip(zip(prepared, outputs), plan.queries) { - let produced: [ResultSet] + let firstCandidate = candidate if let queries { - produced = zip(queries, output.resultSets).map { statement, result in - let replayable = entry.parameterValues == nil - && !BatchResultMapping.referencesLocalVariable(statement.sql, grammar: grammar) - return makeStatementResultSet( - result: result, - sql: statement.sql, - index: firstNumber + resultSets.count, - baseQuery: replayable ? statement.sql : nil, - tabId: tabId, - anchor: StatementAnchor(statement), - built: nextBuilt() - ) + for (statement, result) in zip(queries, output.resultSets) { + let built = nextBuilt() + place(result) { index in + let replayable = entry.parameterValues == nil + && !BatchResultMapping.referencesLocalVariable(statement.sql, grammar: grammar) + return makeStatementResultSet( + result: result, + sql: statement.sql, + index: index, + baseQuery: replayable ? statement.sql : nil, + tabId: tabId, + anchor: StatementAnchor(statement), + built: built + ) + } } } else if output.resultSets.isEmpty { - produced = output.errors.isEmpty - ? [countResultSet(output, anchor: entry.batch.anchor, index: firstNumber + resultSets.count)] - : [] + if output.errors.isEmpty { + place(nil, count: output) { index in countResultSet(output, anchor: entry.batch.anchor, index: index) } + } } else { - produced = output.resultSets.enumerated().map { offset, result in - numberedResultSet( - result, - anchor: entry.batch.anchor, - index: firstNumber + resultSets.count + offset, - tabId: tabId, - built: nextBuilt() - ) + for result in output.resultSets { + let built = nextBuilt() + place(result) { index in + numberedResultSet(result, anchor: entry.batch.anchor, index: index, tabId: tabId, built: built) + } } } - produced.last?.serverOutput = output.serverOutput - resultSets += produced + // The batch's printed output belongs to its last result, wherever that result went. + guard candidate > firstCandidate else { continue } + let last = candidate - 1 + if let resultSet = own[last] { + resultSet.serverOutput = output.serverOutput + } else if let part = parts[last] { + parts[last] = part.withServerOutput(output.serverOutput) + } + } + let resultSets = runPlan.entries.compactMap { entry -> ResultSet? in + switch entry { + case .statement(let index): + return own[index] + case .summary(let indices): + return makeRunSummaryResultSet(indices.compactMap { parts[$0] }) + } } + appendDroppedNotice(of: runPlan, to: resultSets) return resultSets } @@ -573,11 +616,31 @@ extension QueryExecutionCoordinator { let queries = zip(prepared, outputs).map { entry, output in Self.mappedQueries(of: entry, output: output, databaseType: databaseType, grammar: grammar) } - let sources = zip(queries, outputs).flatMap { mapped, output -> [(sql: String?, result: QueryResult)] in - guard let mapped else { return output.resultSets.map { (sql: nil, result: $0) } } - return zip(mapped, output.resultSets).map { (sql: $0.sql as String?, result: $1) } + // One entry per result in the order `batchResultSets` places them; a count result has no rows to build. + var shapes: [Bool] = [] + var sources: [(sql: String?, result: QueryResult)] = [] + var sourceCandidates: [Int] = [] + for (mapped, output) in zip(queries, outputs) { + if mapped == nil, output.resultSets.isEmpty { + if output.errors.isEmpty { + shapes.append(false) + } + continue + } + let pairs: [(sql: String?, result: QueryResult)] = mapped.map { mapped in + zip(mapped, output.resultSets).map { (sql: $0.sql as String?, result: $1) } + } ?? output.resultSets.map { (sql: nil, result: $0) } + for pair in pairs { + sourceCandidates.append(shapes.count) + shapes.append(!pair.result.columns.isEmpty) + sources.append(pair) + } + } + let results = RunResultPlan(producesRows: shapes) + let shown = zip(sources, sourceCandidates).map { source, candidate in + results.showsRows(ofStatement: candidate) ? source : nil } - return BatchResultPlan(queries: queries, built: await buildStatementRows(sources, tabId: tabId)) + return BatchResultPlan(queries: queries, built: await buildStatementRows(shown, tabId: tabId), results: results) } // The query each of a batch's result sets belongs to, or nil when the batch's shape does not pin them. diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift index f9256485cd..7006e2f157 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift @@ -14,9 +14,10 @@ struct StatementRows: Sendable { } // The names hold only while the session still reads them the same way, so the build keeps the reading it used. +// A statement whose rows no result shows has nothing built. struct StatementRowsBuild: Sendable { let reading: TableNameReading - let statements: [StatementRows] + let statements: [StatementRows?] func statement(at index: Int, readBy current: TableNameReading) -> StatementRows? { guard current == reading, statements.indices.contains(index) else { return nil } @@ -125,8 +126,8 @@ extension QueryExecutionCoordinator { return TableRows.from(prepared: prepared, columnTypes: result.columnTypes, columnOrigins: result.columnOrigins) } - // A nil statement is a result no statement names, which reads no table. - func buildStatementRows(_ statements: [(sql: String?, result: QueryResult)], tabId: UUID) async -> StatementRowsBuild { + // A nil sql is a result no statement names, which reads no table. A nil entry builds nothing. + func buildStatementRows(_ statements: [(sql: String?, result: QueryResult)?], tabId: UUID) async -> StatementRowsBuild { let reveals = parent.tabManager.tabs.first { $0.id == tabId }?.revealsMaskedValues == true return await Self.statementRows( statements, @@ -139,11 +140,13 @@ extension QueryExecutionCoordinator { // this build runs to the end. @concurrent nonisolated static func statementRows( - _ statements: [(sql: String?, result: QueryResult)], + _ statements: [(sql: String?, result: QueryResult)?], reading: TableNameReading, policy: MaskingPolicy ) async -> StatementRowsBuild { - StatementRowsBuild(reading: reading, statements: statements.map { sql, result in + StatementRowsBuild(reading: reading, statements: statements.map { entry -> StatementRows? in + guard let entry else { return nil } + let (sql, result) = entry let tableName = sql.flatMap(reading.tableName(in:)) let shape = TableRows.RowShape( columns: result.columns, @@ -160,6 +163,50 @@ extension QueryExecutionCoordinator { }) } + static let summaryMessageLimit = 5 + + // The statements of one run that returned no rows, as one result: rows affected and time added + // up, server output in the order it was printed. + func makeRunSummaryResultSet(_ parts: [RunSummaryPart]) -> ResultSet { + let count = parts.count.formatted() + let resultSet = ResultSet(label: String(format: String(localized: "%@ Statements"), count)) + resultSet.rowsAffected = parts.reduce(0) { $0 + $1.rowsAffected } + resultSet.executionTime = parts.reduce(0) { $0 + $1.executionTime } + // A driver's own message (a warning, a server reply) stays, each once, after the count; a long script's + // varied replies are cut to the first few. + var seen: Set = [] + var messages: [String] = [] + var unshown = 0 + for message in parts.compactMap(\.statusMessage) where !message.isEmpty && seen.insert(message).inserted { + if messages.count < Self.summaryMessageLimit { + messages.append(message) + } else { + unshown += 1 + } + } + if unshown == 1 { + messages.append(String(localized: "1 more message.")) + } else if unshown > 1 { + messages.append(String(format: String(localized: "%@ more messages."), unshown.formatted())) + } + resultSet.statusMessage = ([String(format: String(localized: "Ran %@ statements that returned no rows."), count)] + messages) + .joined(separator: " ") + resultSet.serverOutput = PluginServerOutput( + lines: parts.flatMap(\.serverOutput.lines), + isTruncated: parts.contains { $0.serverOutput.isTruncated } + ) + return resultSet + } + + func appendDroppedNotice(of plan: RunResultPlan, to resultSets: [ResultSet]) { + guard let notice = plan.droppedNotice else { return } + for resultSet in resultSets { + resultSet.statusMessage = [resultSet.statusMessage, notice] + .compactMap { $0?.isEmpty == false ? $0 : nil } + .joined(separator: " ") + } + } + /// Each statement in a multi-statement run targets its own table, so each result carries its /// own identity. Nothing fetches key columns for these statements, so the origin says so and /// the result is read-only until something does. Inheriting the previous run's keys made an diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift index 6143d67473..0520265783 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift @@ -215,6 +215,8 @@ extension QueryExecutionCoordinator { route: DatabaseManager.shared.executionRoute(for: scope), cancellation: .cancellableRead(lease) ) { [queryExecutor = parent.queryExecutor, boundValues] driver in + // Before it runs, so a statement that fails or a connection lost under it is still noted. + await DatabaseManager.shared.noteSessionStatements([statement.sql], ranOn: driver, scope: scope) let fetched = try await queryExecutor.executeQuery( driver: driver, sql: statement.sql, @@ -437,7 +439,13 @@ extension QueryExecutionCoordinator { case .completed(let ran), .cancelled(let ran), .failed(let ran, _, _): results = ran } - let built = await buildStatementRows(zip(sent, results).map { (sql: $0.originalSQL as String?, result: $1) }, tabId: tabId) + let resultPlan = RunResultPlan(producesRows: results.map { !$0.columns.isEmpty }) + let built = await buildStatementRows( + zip(sent, results).enumerated().map { index, pair in + resultPlan.showsRows(ofStatement: index) ? (sql: pair.0.originalSQL as String?, result: pair.1) : nil + }, + tabId: tabId + ) switch outcome { case .cancelled(let results): @@ -567,6 +575,10 @@ extension QueryExecutionCoordinator { ) } } + let sent = await running.prepared + await DatabaseManager.shared.noteSessionStatements( + sent.prefix(outcome.ranCount(of: sent.count)).map(\.sentSQL), ranOn: driver, scope: scope + ) guard sessionPlan == .sessionTransaction else { let endState: PluginSessionTransactionState = asksEndState ? await driver.heldSessionTransactionState() @@ -675,25 +687,35 @@ extension QueryExecutionCoordinator { ) } + // Only the statements the plan keeps become results, so a long script builds a few, not one per statement. private func statementResultSets(_ history: ExecutedStatementHistory, tabId: UUID, built: StatementRowsBuild) -> [ResultSet] { let firstNumber = parent.tabManager.tabs.first { $0.id == tabId }?.display.pinnedResultCount ?? 0 let reading = parent.tableNameReading - return zip(history.prepared, history.results).enumerated().map { index, pair in - let (statement, result) = pair - return makeStatementResultSet( - result: result, - sql: statement.originalSQL, - index: firstNumber + index, - baseQuery: statement.executableSQL, - baseQueryParameterValues: statement.parameterValues?.map(TypedParameterValue.kept), - namedParameterStatement: statement.parameterValues == nil - ? nil - : NamedParameterStatement(sql: statement.originalSQL, parameters: history.parameters), - tabId: tabId, - anchor: statement.anchor, - built: built.statement(at: index, readBy: reading) - ) + let ran = Array(zip(history.prepared, history.results)) + let plan = RunResultPlan(producesRows: ran.map { !$0.1.columns.isEmpty }) + let resultSets = plan.entries.enumerated().map { position, entry -> ResultSet in + switch entry { + case .summary(let indices): + return makeRunSummaryResultSet(indices.map { RunSummaryPart(ran[$0].1) }) + case .statement(let index): + let (statement, result) = ran[index] + return makeStatementResultSet( + result: result, + sql: statement.originalSQL, + index: firstNumber + position, + baseQuery: statement.executableSQL, + baseQueryParameterValues: statement.parameterValues?.map(TypedParameterValue.kept), + namedParameterStatement: statement.parameterValues == nil + ? nil + : NamedParameterStatement(sql: statement.originalSQL, parameters: history.parameters), + tabId: tabId, + anchor: statement.anchor, + built: built.statement(at: index, readBy: reading) + ) + } } + appendDroppedNotice(of: plan, to: resultSets) + return resultSets } /// A commit whose connection died before the answer leaves every statement of the batch in a diff --git a/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift b/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift index 9cf2f91330..f43a74ee09 100644 --- a/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift @@ -296,46 +296,48 @@ final class RowEditingCoordinator: ObservableObject { var edits: [(row: Int, column: Int, value: PluginCellValue)] = [] var removedCells: Set = [] - for (rowID, value) in valuesByRow { - guard let storageRow = tableRows.index(of: rowID) else { continue } - let row = tableRows.rows[storageRow] - guard row.unloadedByteCount(columnIndex) == nil else { continue } - let values = Array(row.values) - let removesField = removedRowIDs.contains(rowID) && parent.changeManager.supportsFieldRemoval - let absence = FieldAbsence( - wasAbsent: row.isAbsent(columnIndex), isAbsent: removesField, originalRow: row.absentColumns - ) - let isExpression = expressionRowIDs.contains(rowID) && !removesField - let wasExpression = parent.changeManager.isExpression(rowID: rowID, columnIndex: columnIndex) - guard values.indices.contains(columnIndex), - values[columnIndex] != value || absence.wasAbsent != absence.isAbsent - || wasExpression != isExpression else { continue } - if continuity == .typing { - parent.changeManager.recordTypedCellChange( - rowID: rowID, - columnIndex: columnIndex, - columnName: columnName, - oldValue: values[columnIndex], - newValue: value, - originalRow: values, - absence: absence, - isExpression: isExpression - ) - } else { - parent.changeManager.recordCellChange( - rowID: rowID, - columnIndex: columnIndex, - columnName: columnName, - oldValue: values[columnIndex], - newValue: value, - originalRow: values, - absence: absence, - isExpression: isExpression + parent.changeManager.recordAsOneAction { + for (rowID, value) in valuesByRow { + guard let storageRow = tableRows.index(of: rowID) else { continue } + let row = tableRows.rows[storageRow] + guard row.unloadedByteCount(columnIndex) == nil else { continue } + let values = Array(row.values) + let removesField = removedRowIDs.contains(rowID) && parent.changeManager.supportsFieldRemoval + let absence = FieldAbsence( + wasAbsent: row.isAbsent(columnIndex), isAbsent: removesField, originalRow: row.absentColumns ) - } - edits.append((row: storageRow, column: columnIndex, value: value)) - if removesField { - removedCells.insert(CellPosition(row: storageRow, column: columnIndex)) + let isExpression = expressionRowIDs.contains(rowID) && !removesField + let wasExpression = parent.changeManager.isExpression(rowID: rowID, columnIndex: columnIndex) + guard values.indices.contains(columnIndex), + values[columnIndex] != value || absence.wasAbsent != absence.isAbsent + || wasExpression != isExpression else { continue } + if continuity == .typing { + parent.changeManager.recordTypedCellChange( + rowID: rowID, + columnIndex: columnIndex, + columnName: columnName, + oldValue: values[columnIndex], + newValue: value, + originalRow: values, + absence: absence, + isExpression: isExpression + ) + } else { + parent.changeManager.recordCellChange( + rowID: rowID, + columnIndex: columnIndex, + columnName: columnName, + oldValue: values[columnIndex], + newValue: value, + originalRow: values, + absence: absence, + isExpression: isExpression + ) + } + edits.append((row: storageRow, column: columnIndex, value: value)) + if removesField { + removedCells.insert(CellPosition(row: storageRow, column: columnIndex)) + } } } guard !edits.isEmpty else { return } @@ -351,21 +353,23 @@ final class RowEditingCoordinator: ObservableObject { let tableRows = parent.tabSessionRegistry.tableRows(for: tabId) var edits: [(row: Int, column: Int, value: PluginCellValue)] = [] var removedCells: Set = [] - for edit in resolved { - guard let storageRow = tableRows.index(of: edit.rowID) else { continue } - changeManager.recordCellChange( - rowID: edit.rowID, - columnIndex: edit.dataIndex, - columnName: edit.columnName, - oldValue: edit.oldValue, - newValue: edit.newValue, - originalRow: edit.originalRow, - absence: edit.absence, - isExpression: edit.isExpression - ) - edits.append((row: storageRow, column: edit.dataIndex, value: edit.newValue)) - if edit.absence.isAbsent { - removedCells.insert(CellPosition(row: storageRow, column: edit.dataIndex)) + changeManager.recordAsOneAction { + for edit in resolved { + guard let storageRow = tableRows.index(of: edit.rowID) else { continue } + changeManager.recordCellChange( + rowID: edit.rowID, + columnIndex: edit.dataIndex, + columnName: edit.columnName, + oldValue: edit.oldValue, + newValue: edit.newValue, + originalRow: edit.originalRow, + absence: edit.absence, + isExpression: edit.isExpression + ) + edits.append((row: storageRow, column: edit.dataIndex, value: edit.newValue)) + if edit.absence.isAbsent { + removedCells.insert(CellPosition(row: storageRow, column: edit.dataIndex)) + } } } guard !edits.isEmpty else { return false } diff --git a/apple/TablePro/Core/Coordinators/RunResultPlan.swift b/apple/TablePro/Core/Coordinators/RunResultPlan.swift new file mode 100644 index 0000000000..45ec117b36 --- /dev/null +++ b/apple/TablePro/Core/Coordinators/RunResultPlan.swift @@ -0,0 +1,103 @@ +// +// RunResultPlan.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +// Which statements of a run become results of their own. Two or more statements that return no +// rows share one summary, placed where the last of them ran, so the result of the last statement +// stays the last result. Only the last `maxRowResults` results that return rows are kept. +struct RunResultPlan: Equatable, Sendable { + static let maxRowResults = 100 + + enum Entry: Equatable, Sendable { + case statement(Int) + case summary([Int]) + } + + let entries: [Entry] + let rowResultCount: Int + let keptRowResultCount: Int + private let ownPositions: [Int: Int] + private let summarized: Set + + var droppedRowResultCount: Int { rowResultCount - keptRowResultCount } + + init(producesRows: [Bool], maxRowResults: Int = Self.maxRowResults) { + let rowIndices = producesRows.indices.filter { producesRows[$0] } + let kept = Set(rowIndices.suffix(max(0, maxRowResults))) + let summarized = producesRows.indices.filter { !producesRows[$0] } + let collapses = summarized.count >= 2 + + var entries: [Entry] = [] + for index in producesRows.indices { + if producesRows[index] { + if kept.contains(index) { entries.append(.statement(index)) } + } else if !collapses { + entries.append(.statement(index)) + } else if index == summarized.last { + entries.append(.summary(summarized)) + } + } + self.entries = entries + var ownPositions: [Int: Int] = [:] + var covered: Set = [] + for (position, entry) in entries.enumerated() { + switch entry { + case .statement(let index): ownPositions[index] = position + case .summary(let indices): covered.formUnion(indices) + } + } + self.ownPositions = ownPositions + self.summarized = covered + rowResultCount = rowIndices.count + keptRowResultCount = kept.count + } + + // True when the statement's rows are shown on a result of its own, so they are worth building. + func showsRows(ofStatement index: Int) -> Bool { + ownPositions[index] != nil + } + + // Where the statement's own result sits among the run's results. + func position(ofStatement index: Int) -> Int? { + ownPositions[index] + } + + func summarizes(statement index: Int) -> Bool { + summarized.contains(index) + } + + var droppedNotice: String? { + guard droppedRowResultCount > 0 else { return nil } + return String( + format: String(localized: "Showing the last %1$@ of %2$@ results that returned rows."), + keptRowResultCount.formatted(), + rowResultCount.formatted() + ) + } +} + +struct RunSummaryPart { + let rowsAffected: Int + let executionTime: TimeInterval + let serverOutput: PluginServerOutput + let statusMessage: String? + + func withServerOutput(_ output: PluginServerOutput) -> RunSummaryPart { + RunSummaryPart(rowsAffected: rowsAffected, executionTime: executionTime, serverOutput: output, statusMessage: statusMessage) + } +} + +extension RunSummaryPart { + init(_ result: QueryResult) { + self.init( + rowsAffected: result.rowsAffected, + executionTime: result.executionTime, + serverOutput: result.serverOutput, + statusMessage: result.statusMessage + ) + } +} diff --git a/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift b/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift index d718cdff74..af13e95a3d 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift @@ -95,14 +95,7 @@ extension DataGridModel { searchDataset(revealsFirstMatch: true) return } - let source = findSource - let result = GridFindEngine.search(find.term, options: find.options, in: source) - let current = GridFindEngine.index( - atOrAfter: selection.activeCell, - in: GridFindEngine.positions(of: result.matches, in: source) - ) - find.setResults(result.matches, current: current, problem: result.problem, basis: findBasis(of: source)) - revealCurrentFindMatch() + searchLoadedRows(GridFindState.Landing(revealsFirstMatch: true, cursorAtStart: activeCellIdentity)) } // The rows changed under the bar: keep the current match when it survives and leave the selection alone. @@ -123,33 +116,24 @@ extension DataGridModel { searchDataset(revealsFirstMatch: false) return } - let source = findSource - let basis = findBasis(of: source) - let previous = find.currentMatch - let result: GridFindEngine.Result - if let changes, find.loadedRowsBasis == basis, - let refreshed = GridFindEngine.refreshed( - find.matches, - writtenCells: changes.writtenCells, - addedRows: changes.addedRows, - term: find.term, - options: find.options, - in: source - ) { - result = refreshed - } else { - result = GridFindEngine.search(find.term, options: find.options, in: source) - } - let current: Int? - if let previous, let kept = result.matches.firstIndex(of: previous) { - current = kept - } else { - current = GridFindEngine.index( - atOrAfter: selection.activeCell, - in: GridFindEngine.positions(of: result.matches, in: source) - ) + // A running search leaves no basis, so a change meanwhile searches again. + if let changes { + let source = findSource + let basis = findBasis(of: source) + if find.loadedRowsBasis == basis, + let refreshed = GridFindEngine.refreshed( + find.matches, + writtenCells: changes.writtenCells, + addedRows: changes.addedRows, + term: find.term, + options: find.options, + in: source + ) { + landLoadedRowsResult(refreshed, landing: GridFindState.Landing(), basis: basis, source: source) + return + } } - find.setResults(result.matches, current: current, problem: result.problem, basis: basis) + searchLoadedRows(GridFindState.Landing()) } // The owner's rows changed (an edit, an undo, a filter): search them again, keeping the current match. @@ -170,11 +154,11 @@ extension DataGridModel { find.stopDatasetSearch() } - // Returns once no whole-dataset search is waiting or running. + // Returns once no search of the loaded rows or the whole dataset is waiting or running. func findSearchSettled() async { - while let task = find.datasetSearch { + while let task = find.datasetSearch ?? find.loadedRowsSearch { await task.value - if find.datasetSearch == task { return } + if find.datasetSearch == task || find.loadedRowsSearch == task { return } } } @@ -215,7 +199,12 @@ extension DataGridModel { // MARK: - Replacing func replaceCurrentMatch() { - guard canReplaceInFind, find.problem == nil, let match = find.currentMatch, commitEditing() else { return } + guard canReplaceInFind else { return } + // Replace acts on the match the bar is about to show. + if find.isAwaitingLoadedRows { + searchLoadedRowsNow() + } + guard find.problem == nil, let match = find.currentMatch, commitEditing() else { return } if let finder = datasetFinder { replaceDatasetMatch(match, finder: finder) return @@ -252,10 +241,12 @@ extension DataGridModel { return nil } let source = findSource - let result = GridFindEngine.search(find.term, options: find.options, in: source) - guard result.problem == nil else { return nil } + if find.isAwaitingLoadedRows || find.loadedRowsBasis != findBasis(of: source) { + searchLoadedRowsNow() + } + guard find.problem == nil else { return nil } let replacement = GridFindEngine.replacements( - for: result.matches, + for: find.matches, term: find.term, options: find.options, template: find.replacement, @@ -308,9 +299,16 @@ extension DataGridModel { stepDatasetFind(finder, forward: forward) return } - if find.matches.isEmpty { - refreshFind() + if find.isAwaitingLoadedRows { + find.deferStep(forward: forward) + } else if find.matches.isEmpty { + searchLoadedRows(GridFindState.Landing(steps: forward ? 1 : -1)) + } else { + stepThroughFindMatches(forward: forward) } + } + + private func stepThroughFindMatches(forward: Bool) { guard !find.matches.isEmpty else { return } let index = GridFindEngine.step( from: find.currentIndex, @@ -322,6 +320,78 @@ extension DataGridModel { revealCurrentFindMatch() } + // MARK: - Loaded rows + + // Off the main actor. A term with nothing to scan is answered here, as is an invalid pattern. + private func searchLoadedRows(_ landing: GridFindState.Landing) { + let source = findSource + let basis = findBasis(of: source) + let term = find.term + let options = find.options + let generation = find.beginLoadedRowsSearch(landing) + if let answered = GridFindEngine.resultWithoutScanning(term, options: options, in: source) { + if let merged = find.endLoadedRowsSearch(generation: generation) { + landLoadedRowsResult(answered, landing: merged, basis: basis, source: source) + } + return + } + let dateFormat = DateFormattingService.shared.currentFormat + find.attachLoadedRowsSearch(Task { [weak self] in + let result = await GridFindEngine.searchOffTheMainActor(term, options: options, in: source, dateFormat: dateFormat) + self?.loadedRowsSearchDidFinish(result, basis: basis, generation: generation) + }, generation: generation) + } + + private func loadedRowsSearchDidFinish(_ result: GridFindEngine.Result?, basis: GridFindEngine.Basis, generation: Int) { + guard let landing = find.endLoadedRowsSearch(generation: generation), let result, find.isVisible else { return } + let source = findSource + // A setting the cell text follows changed while it ran, and nothing refreshed the bar. + guard findBasis(of: source) == basis else { + searchLoadedRows(landing) + return + } + landLoadedRowsResult(result, landing: landing, basis: basis, source: source) + } + + // On the main actor, landing as a running search would have. + private func searchLoadedRowsNow() { + let source = findSource + let basis = findBasis(of: source) + let generation = find.beginLoadedRowsSearch(GridFindState.Landing()) + guard let landing = find.endLoadedRowsSearch(generation: generation) else { return } + let result = GridFindEngine.search(find.term, options: find.options, in: source) + landLoadedRowsResult(result, landing: landing, basis: basis, source: source) + } + + private func landLoadedRowsResult( + _ result: GridFindEngine.Result, + landing: GridFindState.Landing, + basis: GridFindEngine.Basis, + source: GridFindEngine.Source + ) { + let current: Int? + if !landing.revealsFirstMatch, let previous = find.currentMatch, let kept = result.matches.firstIndex(of: previous) { + current = kept + } else { + current = GridFindEngine.index( + atOrAfter: selection.activeCell, + in: GridFindEngine.positions(of: result.matches, in: source) + ) + } + find.setResults(result.matches, current: current, problem: result.problem, basis: basis) + if landing.revealsFirstMatch, landing.cursorAtStart == activeCellIdentity { + revealCurrentFindMatch() + } + for _ in 0.. 0) + } + } + + // By identity, so a row change that moves the cursor's position is not a move by the user. + private var activeCellIdentity: GridFindState.Match? { + activeCell.map { GridFindState.Match(rowID: $0.rowID, columnID: $0.column.id) } + } + // The match becomes the active cell, so Copy, Return and the arrow keys act on the found column. private func revealCurrentFindMatch() { guard let match = find.currentMatch else { return } diff --git a/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift b/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift index 730159e873..0890bd9e78 100644 --- a/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift +++ b/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift @@ -120,7 +120,7 @@ struct GridFindMatcher { @MainActor enum GridFindEngine { - struct Source { + struct Source: Sendable { let rows: TableRows let order: DataGridRowOrder let columns: DataGridColumnSet @@ -129,11 +129,18 @@ enum GridFindEngine { var revealsInvisibleCharacters = false } - struct Result: Equatable { + struct Result: Equatable, Sendable { var matches: [GridFindState.Match] = [] var problem: String? } + private enum Plan { + case answered(Result) + case scan(GridFindMatcher, columns: [DataGridColumn]) + } + + nonisolated static let cancellationInterval = 4_096 + // Everything a loaded-rows search read besides the rows themselves, the settings the cell text follows included. struct Basis: Equatable { let term: String @@ -173,7 +180,7 @@ enum GridFindEngine { var cells: Int { edits.count } } - static var invalidPatternProblem: String { + nonisolated static var invalidPatternProblem: String { String(localized: "Invalid regular expression") } @@ -187,22 +194,39 @@ enum GridFindEngine { } static func search(_ term: String, options: GridFindState.Options, in source: Source) -> Result { - let matcher: GridFindMatcher? - do { - matcher = try GridFindMatcher.make(term: term, options: options) - } catch { - return Result(problem: invalidPatternProblem) + search(term, options: options, in: source, dates: DateFormattingService.shared.renderer, isCancelled: { false }) ?? Result() + } + + // Nil once cancelled, which is asked every 4,096 cells. + nonisolated static func search( + _ term: String, + options: GridFindState.Options, + in source: Source, + dates: DateDisplayRenderer, + isCancelled: () -> Bool + ) -> Result? { + let matcher: GridFindMatcher + let searched: [DataGridColumn] + switch plan(term, options: options, in: source) { + case .answered(let result): + return result + case .scan(let planned, let columns): + matcher = planned + searched = columns } - guard let matcher else { return Result() } - let searched = searchedColumns(options: options, in: source) - guard !searched.isEmpty else { return Result() } var found: [GridFindState.Match] = [] + var budget = 0 let rowCount = source.order.count(in: source.rows) for displayRow in 0.. Result? { + search( + term, + options: options, + in: source, + dates: DateDisplayRenderer(format: dateFormat), + isCancelled: { Task.isCancelled } + ) + } + + // What a search answers without reading a cell: an invalid pattern, a term that finds nothing, no column to search. + nonisolated static func resultWithoutScanning(_ term: String, options: GridFindState.Options, in source: Source) -> Result? { + guard case .answered(let result) = plan(term, options: options, in: source) else { return nil } + return result + } + + nonisolated private static func plan(_ term: String, options: GridFindState.Options, in source: Source) -> Plan { + let matcher: GridFindMatcher? + do { + matcher = try GridFindMatcher.make(term: term, options: options) + } catch { + return .answered(Result(problem: invalidPatternProblem)) + } + guard let matcher else { return .answered(Result()) } + let searched = searchedColumns(options: options, in: source) + guard !searched.isEmpty else { return .answered(Result()) } + return .scan(matcher, columns: searched) + } + // What `search` finds after only these cells (by data index) were written and these rows added, others having // left, on the same basis. Nil when the term cannot match, which `search` answers as cheaply. static func refreshed( @@ -273,7 +333,7 @@ enum GridFindEngine { } // A scope naming a column that is no longer presented searches every presented column. - static func searchedColumns(options: GridFindState.Options, in source: Source) -> [DataGridColumn] { + nonisolated static func searchedColumns(options: GridFindState.Options, in source: Source) -> [DataGridColumn] { let candidates: [DataGridColumn] if let scope = options.columnScope, let scoped = source.columns.column(id: scope) { candidates = [scoped] @@ -285,8 +345,17 @@ enum GridFindEngine { } } - // Nil for NULL, an empty cell, and bytes the column's format could not decode, which show as hex. static func searchableText(in row: Row, column: DataGridColumn, source: Source) -> String? { + searchableText(in: row, column: column, source: source, dates: DateFormattingService.shared.renderer) + } + + // Nil for NULL, an empty cell, and bytes the column's format could not decode, which show as hex. + nonisolated static func searchableText( + in row: Row, + column: DataGridColumn, + source: Source, + dates: DateDisplayRenderer + ) -> String? { let dataIndex = column.dataIndex guard dataIndex < row.values.count else { return nil } let rawValue = row.values[dataIndex] @@ -305,7 +374,8 @@ enum GridFindEngine { rawValue, columnType: column.type, displayFormat: format, - databaseType: source.databaseType + databaseType: source.databaseType, + dates: dates ) guard !shown.isEmpty else { return nil } return source.revealsInvisibleCharacters ? shown : shown.shownWithoutMarks @@ -439,7 +509,7 @@ enum GridFindEngine { return String(format: String(localized: "Replaced %1$@ in %2$@."), matchPhrase, cellPhrase) } - private static func displayFormat(for column: DataGridColumn, in source: Source) -> ValueDisplayFormat? { + nonisolated private static func displayFormat(for column: DataGridColumn, in source: Source) -> ValueDisplayFormat? { column.dataIndex < source.displayFormats.count ? source.displayFormats[column.dataIndex] : nil } diff --git a/apple/TablePro/Core/DataGrid/GridFindState.swift b/apple/TablePro/Core/DataGrid/GridFindState.swift index ba2c2f68b0..e297c1740d 100644 --- a/apple/TablePro/Core/DataGrid/GridFindState.swift +++ b/apple/TablePro/Core/DataGrid/GridFindState.swift @@ -3,18 +3,28 @@ import Foundation @MainActor final class GridFindState: ObservableObject { - struct Options: Equatable { + struct Options: Equatable, Sendable { var matchesCase = false var matchesWholeWord = false var usesRegularExpression = false var columnScope: DataGridColumnID? } - struct Match: Hashable { + struct Match: Hashable, Sendable { let rowID: RowID let columnID: DataGridColumnID } + // What a search of the loaded rows does once its matches land. + struct Landing: Equatable { + // A typed term selects the first match from the cursor; a refresh keeps the current match. + var revealsFirstMatch = false + // The active cell when the typed term's search began. A user who moved it since keeps their place. + var cursorAtStart: Match? + // Find Next presses minus Find Previous presses while the search ran. + var steps = 0 + } + @Published var isVisible = false @Published var showsReplace = false @Published var term = "" @@ -26,6 +36,8 @@ final class GridFindState: ObservableObject { @Published private(set) var isSearching = false // The user stopped a whole-dataset search before it finished. @Published private(set) var searchStopped = false + // An empty match list means nothing yet; unlike isSearching it shows no progress. + @Published private(set) var isAwaitingLoadedRows = false weak var datasetFinder: (any DataGridDatasetFinding)? private(set) var datasetQuery: GridDatasetFindQuery? @@ -33,6 +45,16 @@ final class GridFindState: ObservableObject { private var datasetGeneration = 0 // What the matches were searched on, while they came from the loaded rows; nil once anything else set them. private(set) var loadedRowsBasis: GridFindEngine.Basis? + // Runs off the main actor and sets no isSearching: a refresh keeps the matches shown until the new ones land. + private(set) var loadedRowsSearch: Task? + private var loadedRowsGeneration = 0 + private var loadedRowsLanding = Landing() + + // A closed grid's scan would otherwise run to the end and hold its rows until then. + deinit { + loadedRowsSearch?.cancel() + datasetSearch?.cancel() + } var currentMatch: Match? { guard let currentIndex, matches.indices.contains(currentIndex) else { return nil } @@ -60,6 +82,7 @@ final class GridFindState: ObservableObject { func reset() { cancelDatasetSearch() + cancelLoadedRowsSearch() datasetQuery = nil searchStopped = false term = "" @@ -70,11 +93,74 @@ final class GridFindState: ObservableObject { problem = nil } + // MARK: - Loaded-rows search + + // The search hands this generation back when it lands. Any other search stops, and a running one this + // replaces hands on what it was to do on landing. + func beginLoadedRowsSearch(_ landing: Landing) -> Int { + var merged = landing + if loadedRowsSearch != nil { + if !landing.revealsFirstMatch, loadedRowsLanding.revealsFirstMatch { + merged.revealsFirstMatch = true + merged.cursorAtStart = loadedRowsLanding.cursorAtStart + } + merged.steps += loadedRowsLanding.steps + } + cancelLoadedRowsSearch() + if datasetSearch != nil { + cancelDatasetSearch() + } + loadedRowsLanding = merged + loadedRowsBasis = nil + // The old term's count and current match would read as the new term's until the scan lands. + if merged.revealsFirstMatch { + matches = [] + currentIndex = nil + problem = nil + } + return loadedRowsGeneration + } + + func attachLoadedRowsSearch(_ task: Task, generation: Int) { + guard generation == loadedRowsGeneration else { + task.cancel() + return + } + loadedRowsSearch = task + // Set here, not in begin, so a term answered without scanning never publishes it. + isAwaitingLoadedRows = true + } + + // Nil when a later search, a reset or a dataset search took this one's place. + func endLoadedRowsSearch(generation: Int) -> Landing? { + guard generation == loadedRowsGeneration else { return nil } + let landing = loadedRowsLanding + loadedRowsSearch = nil + if isAwaitingLoadedRows { isAwaitingLoadedRows = false } + loadedRowsLanding = Landing() + loadedRowsGeneration &+= 1 + return landing + } + + func deferStep(forward: Bool) { + loadedRowsLanding.steps += forward ? 1 : -1 + } + + func cancelLoadedRowsSearch() { + loadedRowsSearch?.cancel() + loadedRowsSearch = nil + // @Published fires on every write, so skip the redraw when nothing changes. + if isAwaitingLoadedRows { isAwaitingLoadedRows = false } + loadedRowsGeneration &+= 1 + loadedRowsLanding = Landing() + } + // MARK: - Whole-dataset search // The search hands this generation back when it finishes; any later start or cancel makes it stale. func beginDatasetSearch(_ query: GridDatasetFindQuery?, clearsResults: Bool) -> Int { cancelDatasetSearch() + cancelLoadedRowsSearch() datasetQuery = query searchStopped = false problem = nil diff --git a/apple/TablePro/Core/DataWrite/DataWritePlan.swift b/apple/TablePro/Core/DataWrite/DataWritePlan.swift index 0c0e29ccdd..f67c4c5bf5 100644 --- a/apple/TablePro/Core/DataWrite/DataWritePlan.swift +++ b/apple/TablePro/Core/DataWrite/DataWritePlan.swift @@ -66,6 +66,10 @@ struct RowWriteOperation: Codable, Sendable, Equatable { // What the driver found the row by when it was read, for an update or a delete. A rewind writes to that row and // no other. Nil in a record saved before it was kept, and on an engine that keeps none. var rowLocator: String? + // Where the row's first write ran in the save. Nil in a record saved before it was kept, which runs by list order. + var order: Int? + // Only for an update written in two or more steps. An older build ignores it and rewinds the net images. + var steps: [RowWriteStep]? var isReversible: Bool { refusal == nil } @@ -85,6 +89,82 @@ struct RowWriteOperation: Codable, Sendable, Equatable { guard let image else { return [] } return Set(image.indices.filter { image[$0].isNull }) } + + // The row as each step left it, applying the steps to the pre-image in turn. + func stepImages() -> [RowWriteStepImage]? { + guard let steps, var row = preImage else { return nil } + var absent = absentColumnsBeforeWrite + return steps.map { step in + for (offset, column) in step.columns.enumerated() { + if row.indices.contains(column), step.after.indices.contains(offset) { + row[column] = step.after[offset] + } + if step.afterAbsent.contains(column) { + absent.insert(column) + } else { + absent.remove(column) + } + } + return RowWriteStepImage(step: step, after: row, absentAfter: absent) + } + } +} + +// `before` and `after` line up with `columns`, which are indices into the operation's columns. +struct RowWriteStep: Codable, Sendable, Equatable { + let order: Int + let columns: [Int] + let before: [PluginCellValue] + let after: [PluginCellValue] + let beforeAbsent: Set + let afterAbsent: Set + + init( + order: Int, + columns: [Int], + before: [PluginCellValue], + after: [PluginCellValue], + beforeAbsent: Set = [], + afterAbsent: Set = [] + ) { + self.order = order + self.columns = columns + self.before = before + self.after = after + self.beforeAbsent = beforeAbsent + self.afterAbsent = afterAbsent + } + + init(_ step: RowChange) { + self.init( + order: step.sequence, + columns: step.cellChanges.map(\.columnIndex), + before: step.cellChanges.map(\.oldValue), + after: step.cellChanges.map(\.newValue), + beforeAbsent: Set(step.cellChanges.filter(\.oldIsAbsent).map(\.columnIndex)), + afterAbsent: Set(step.cellChanges.filter(\.newIsAbsent).map(\.columnIndex)) + ) + } + + var inverted: RowWriteStep { + RowWriteStep( + order: order, columns: columns, before: after, after: before, + beforeAbsent: afterAbsent, afterAbsent: beforeAbsent + ) + } + + func reordered(_ order: Int) -> RowWriteStep { + RowWriteStep( + order: order, columns: columns, before: before, after: after, + beforeAbsent: beforeAbsent, afterAbsent: afterAbsent + ) + } +} + +struct RowWriteStepImage { + let step: RowWriteStep + let after: [PluginCellValue] + let absentAfter: Set } // A driver's `PluginRowLookup`, bound for the executor. diff --git a/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift b/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift index 3686d2ca50..dfca08ec8c 100644 --- a/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift +++ b/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift @@ -134,13 +134,23 @@ struct RewindExecutor { ) } - // In the order the restore ran them, since undoing the restore runs its record backwards. + // In the order the restore ran them, each step ordered by when the restore ran it, since undoing the restore + // runs its record backwards. static func inverseOperations(of plan: RewindPlan) -> [RowWriteOperation] { - plan.rows.filter { $0.outcome.restores }.reversed().map { row -> RowWriteOperation in - let source = row.operation + let restored = plan.rows.filter { $0.outcome.restores }.map(\.operation) + var rank: [RewindPlanner.RunEntry: Int] = [:] + for (position, entry) in RewindPlanner.runOrder(of: restored).enumerated() { + rank[entry] = position + } + return restored.enumerated().reversed().map { index, source -> RowWriteOperation in let kind = source.kind.inverted // The same stored refusal a save through a view writes, so a build without the view flag refuses too. let throughView = plan.record.writtenThroughView == true && kind == .delete + let steps = source.steps.map { sourceSteps in + sourceSteps.indices.reversed().map { stepIndex in + sourceSteps[stepIndex].inverted.reordered(rank[.init(operation: index, step: stepIndex)] ?? 0) + } + } return RowWriteOperation( kind: kind, target: source.target, @@ -152,7 +162,9 @@ struct RewindExecutor { refusal: throughView ? .identityNotPreservable : nil, preImageAbsentColumns: source.postImageAbsentColumns, postImageAbsentColumns: source.preImageAbsentColumns, - rowLocator: source.rowLocator + rowLocator: source.rowLocator, + order: steps?.first?.order ?? rank[.init(operation: index, step: nil)], + steps: steps ) } } diff --git a/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift b/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift index 361fb6bc40..7b6b7116d3 100644 --- a/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift +++ b/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift @@ -56,26 +56,12 @@ struct RewindPlanner { let byLocator = indexByLocator(currentRows) let matchesByLocator = locatorQuery() != nil || !byLocator.isEmpty - var rows: [RewindRowPlan] = [] - var restorable: [RowWriteOperation] = [] - for operation in record.operations { + var found: [FoundRow] = record.operations.map { operation in if let refusal = record.refusal(of: operation) { - rows.append( - RewindRowPlan( - id: UUID(), operation: operation, - outcome: .notReversible(refusal), currentImage: nil - ) - ) - continue + return FoundRow(outcome: .notReversible(refusal), current: nil) } guard keyIsComparable(operation) else { - rows.append( - RewindRowPlan( - id: UUID(), operation: operation, - outcome: .notReversible(.keyNotComparable), currentImage: nil - ) - ) - continue + return FoundRow(outcome: .notReversible(.keyNotComparable), current: nil) } let currentRow: RewindCurrentRow? if let locator = operation.rowLocator, matchesByLocator { @@ -84,11 +70,17 @@ struct RewindPlanner { currentRow = primaryKey(of: operation).flatMap { current[$0] } ?? rowRestoredUnderKeyBefore(operation, in: current) } - let outcome = classify(operation, current: currentRow) + return FoundRow(outcome: classify(operation, current: currentRow), current: currentRow) + } + restoreDeletesWhoseKeyAnInsertFrees(&found) + + var rows: [RewindRowPlan] = [] + var restorable: [RowWriteOperation] = [] + for (operation, row) in zip(record.operations, found) { rows.append( - RewindRowPlan(id: UUID(), operation: operation, outcome: outcome, currentImage: currentRow?.values) + RewindRowPlan(id: UUID(), operation: operation, outcome: row.outcome, currentImage: row.current?.values) ) - if outcome.restores { + if row.outcome.restores { restorable.append(operation) } } @@ -106,6 +98,36 @@ struct RewindPlanner { // MARK: - Classification + private struct FoundRow { + var outcome: RewindRowOutcome + var current: RewindCurrentRow? + } + + // A new row that took a deleted row's key is removed before the deleted row is put back, so that key is free. + private func restoreDeletesWhoseKeyAnInsertFrees(_ found: inout [FoundRow]) { + var rank: [Int: Int] = [:] + for (position, entry) in Self.runOrder(of: record.operations).enumerated() where rank[entry.operation] == nil { + rank[entry.operation] = position + } + let operations = record.operations + for (index, delete) in operations.enumerated() + where delete.kind == .delete && found[index].outcome == .rowAlreadyPresent { + guard let key = primaryKey(of: delete), + let occupant = found[index].current, + primaryKey(of: occupant.values, columns: delete.columns, keyColumns: delete.primaryKeyColumns) == key, + let deleteRank = rank[index] + else { continue } + let freed = operations.indices.contains { other in + operations[other].kind == .insert && found[other].outcome == .willRestore + && primaryKey(of: operations[other]) == key + && (rank[other] ?? .max) < deleteRank + } + if freed { + found[index] = FoundRow(outcome: .willRestore, current: nil) + } + } + } + private var readsThroughView: Bool { record.writtenThroughView == true } private func classify(_ operation: RowWriteOperation, current: RewindCurrentRow?) -> RewindRowOutcome { @@ -244,19 +266,43 @@ struct RewindPlanner { // MARK: - Statements - /// The inverse operations, newest first. - /// - /// Reverse order matters when one row in the save freed a value another row took: putting the - /// second row's change back before the first row's is the only order that does not collide. - /// The forward generator groups by verb, which loses that order, so the inverse is generated - /// one operation at a time. + struct RunEntry: Hashable { + let operation: Int + let step: Int? + } + + // Newest first, step by step, since a value one write freed may be one a later write took. Ties, and a record + // saved before orders were kept, run by list position, the later first. + static func runOrder(of operations: [RowWriteOperation]) -> [RunEntry] { + var entries: [(order: Int?, entry: RunEntry)] = [] + for (index, operation) in operations.enumerated() { + if let steps = operation.steps, !steps.isEmpty { + for (stepIndex, step) in steps.enumerated() { + entries.append((step.order, RunEntry(operation: index, step: stepIndex))) + } + } else { + entries.append((operation.order, RunEntry(operation: index, step: nil))) + } + } + return entries.sorted { lhs, rhs in + (lhs.order ?? .min, lhs.entry.operation, lhs.entry.step ?? 0) + > (rhs.order ?? .min, rhs.entry.operation, rhs.entry.step ?? 0) + }.map(\.entry) + } + private func statements(for operations: [RowWriteOperation]) throws -> RestoreStatements { var statements: [ParameterizedStatement] = [] var lookups: [RowLookup?] = [] var prologue: [String] = [] var epilogue: [String] = [] - for operation in operations.reversed() { - let inverse = try inverseStatements(for: operation) + for entry in Self.runOrder(of: operations) { + let operation = operations[entry.operation] + let inverse: RestoreStatements + if let step = entry.step { + inverse = try inverseStatements(for: operation, step: step) + } else { + inverse = try inverseStatements(for: operation) + } statements.append(contentsOf: inverse.statements) lookups.append(contentsOf: inverse.lookups) prologue.append(contentsOf: inverse.prologue.filter { !prologue.contains($0) }) @@ -265,6 +311,31 @@ struct RewindPlanner { return RestoreStatements(statements: statements, lookups: lookups, prologue: prologue, epilogue: epilogue) } + // Sets the step's columns back, keyed on the row as that step left it. + private func inverseStatements(for operation: RowWriteOperation, step index: Int) throws -> RestoreStatements { + guard let image = operation.stepImages()?[safe: index] else { return RestoreStatements(statements: []) } + var factory = factory + factory.rowLocators = operation.rowLocator.map { [.existing(0): $0] } ?? [:] + let step = image.step + let cellChanges = step.columns.enumerated().compactMap { offset, column -> CellChange? in + guard operation.columns.indices.contains(column), image.after.indices.contains(column), + step.before.indices.contains(offset) + else { return nil } + return CellChange( + columnIndex: column, columnName: operation.columns[column], + oldValue: image.after[column], newValue: step.before[offset], + oldIsAbsent: image.absentAfter.contains(column), newIsAbsent: step.beforeAbsent.contains(column) + ) + } + guard !cellChanges.isEmpty else { return RestoreStatements(statements: []) } + return try factory.lookedUpStatements( + for: [RowChange( + rowID: .existing(0), type: .update, cellChanges: cellChanges, + originalRow: image.after, absentColumns: image.absentAfter + )] + ) + } + private func inverseStatements(for operation: RowWriteOperation) throws -> RestoreStatements { var factory = factory factory.rowLocators = operation.rowLocator.map { [.existing(0): $0] } ?? [:] diff --git a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift index fb8d7ce52a..978770a650 100644 --- a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift +++ b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift @@ -131,22 +131,22 @@ struct RowChangeStatementFactory { } } - /// Every statement the changes need, or a throw naming the changes that would be left out. - /// - /// The host's statements carry the rows they touch, so the executor can hold the server to that - /// count. A driver's carry one only for a keyless update or delete of one row. + // Split into save steps first, so the refusals, every writer, the preview and the coverage see what runs. func rowWriteStatements( for changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]] = [:], deletedRowIDs: Set = [], insertedRowIDs: Set = [] ) throws -> RowWriteStatements { - try refuseServerOwnedEdits(in: changes) - try refuseDefaultOnUpdate(in: changes) - try refuseUnwritableExpressions(in: changes, insertedRowData: insertedRowData) - try refuseMisdimensionedVectors(in: changes, insertedRowData: insertedRowData) + let steps = RowChange.saveSteps( + changes, keyColumns: Set(primaryKeyColumns.compactMap { columns.firstIndex(of: $0) }) + ) + try refuseServerOwnedEdits(in: steps) + try refuseDefaultOnUpdate(in: steps) + try refuseUnwritableExpressions(in: steps, insertedRowData: insertedRowData) + try refuseMisdimensionedVectors(in: steps, insertedRowData: insertedRowData) let insertedRowData = insertedRowData.mapValues(leavingServerOwnedColumnsToTheServer) - let changes = changes.map { markingServerOwnedDefaults($0, in: insertedRowData) } + let changes = steps.map { markingServerOwnedDefaults($0, in: insertedRowData) } if let driverStatements = try pluginRowWrites( for: changes, insertedRowData: insertedRowData, @@ -399,7 +399,7 @@ struct RowChangeStatementFactory { changes, deletedRowIDs: deletedRowIDs, insertedRowIDs: insertedRowIDs, - writtenRowIDs: Set(statements.flatMap(\.rowIDs)) + writtenChangeIDs: Set(statements.flatMap(\.changeIDs)) ) if let kind = UnwrittenRowCounts(unwritten).leadingKind { throw DataWriteError.rowsNotIdentifiable(tableName, kind) @@ -584,7 +584,7 @@ struct RowChangeStatementFactory { changes, deletedRowIDs: deletedRowIDs, insertedRowIDs: insertedRowIDs, - writtenRowIDs: Set(writes.flatMap(\.rowIndices).compactMap(keyed.rowID(forIndex:))) + writtenChangeIDs: Set(writes.flatMap(\.rowIndices).compactMap { keyed.change(forIndex: $0)?.id }) ) guard unwritten.isEmpty else { throw DataWriteError.changesNotWritable(table: tableName, unwritten: UnwrittenRowCounts(unwritten)) @@ -709,13 +709,15 @@ struct RowChangeStatementFactory { } } +// One key per change, so a row written in two steps reaches the driver as two changes. struct PluginKeyedChanges { let changes: [PluginRowChange] let insertedRowData: [Int: [PluginCellValue]] let deletedRowIndices: Set let insertedRowIndices: Set - /// The row each key stands for, indexed by key. + // Indexed by key. let rowIDs: [RowID] + private let steps: [RowChange] init( changes: [RowChange], @@ -725,23 +727,27 @@ struct PluginKeyedChanges { defaultColumns: [RowID: Set] = [:] ) { let ordered = changes.sorted { $0.sequence < $1.sequence } - var keys: [RowID: Int] = [:] - var rowIDs: [RowID] = [] - for change in ordered where keys[change.rowID] == nil { - keys[change.rowID] = rowIDs.count - rowIDs.append(change.rowID) + var insertKeys: [RowID: Int] = [:] + var deleteKeys: [RowID: Int] = [:] + for (key, change) in ordered.enumerated() { + switch change.type { + case .insert: insertKeys[change.rowID] = key + case .delete: deleteKeys[change.rowID] = key + case .update: break + } } - self.rowIDs = rowIDs - self.changes = ordered.compactMap { change in - keys[change.rowID].map { PluginRowChange(change, key: $0, defaultColumns: defaultColumns[change.rowID]) } + self.steps = ordered + self.rowIDs = ordered.map(\.rowID) + self.changes = ordered.enumerated().map { key, change in + PluginRowChange(change, key: key, defaultColumns: defaultColumns[change.rowID]) } self.insertedRowData = Dictionary( uniqueKeysWithValues: insertedRowData.compactMap { rowID, values in - keys[rowID].map { ($0, values) } + insertKeys[rowID].map { ($0, values) } } ) - self.deletedRowIndices = Set(deletedRowIDs.compactMap { keys[$0] }) - self.insertedRowIndices = Set(insertedRowIDs.compactMap { keys[$0] }) + self.deletedRowIndices = Set(deletedRowIDs.compactMap { deleteKeys[$0] }) + self.insertedRowIndices = Set(insertedRowIDs.compactMap { insertKeys[$0] }) } func locators(_ byRow: [RowID: String]) -> [Int: String] { @@ -752,14 +758,14 @@ struct PluginKeyedChanges { return keyed } - /// The row a driver's `rowIndex` names, or nil when it names none of these changes. - func rowID(forIndex index: Int) -> RowID? { - rowIDs.indices.contains(index) ? rowIDs[index] : nil + // Nil when a driver's `rowIndex` names none of these changes. + func change(forIndex index: Int) -> RowChange? { + steps.indices.contains(index) ? steps[index] : nil } func writeKind(ofRowIndex index: Int) -> RowWriteKind? { - guard let change = changes.first(where: { $0.rowIndex == index }) else { return nil } - switch change.type { + guard let type = change(forIndex: index)?.type else { return nil } + switch type { case .insert: return .insert case .update: return .update case .delete: return .delete diff --git a/apple/TablePro/Core/DataWrite/RowWriteCoverage.swift b/apple/TablePro/Core/DataWrite/RowWriteCoverage.swift index fef6c92260..188f5e7e30 100644 --- a/apple/TablePro/Core/DataWrite/RowWriteCoverage.swift +++ b/apple/TablePro/Core/DataWrite/RowWriteCoverage.swift @@ -23,20 +23,21 @@ enum RowWriteCoverage { } } + // By change, not by row: a row edited around another change is written in steps, and each one has to be. static func unwrittenChanges( _ changes: [RowChange], deletedRowIDs: Set, insertedRowIDs: Set, - writtenRowIDs: Set + writtenChangeIDs: Set ) -> [RowChange] { changes.filter { change in isPending(change, deletedRowIDs: deletedRowIDs, insertedRowIDs: insertedRowIDs) - && !writtenRowIDs.contains(change.rowID) + && !writtenChangeIDs.contains(change.id) } } } -/// How many changes of each kind a save could not write, which is what the refusal names. +// How many rows of each kind a save could not write, which is what the refusal names. struct UnwrittenRowCounts: Equatable, Sendable { let updates: Int let inserts: Int @@ -49,11 +50,10 @@ struct UnwrittenRowCounts: Equatable, Sendable { } init(_ changes: [RowChange]) { - self.init( - updates: changes.count { $0.type == .update }, - inserts: changes.count { $0.type == .insert }, - deletes: changes.count { $0.type == .delete } - ) + func rows(_ type: ChangeType) -> Int { + Set(changes.lazy.filter { $0.type == type }.map(\.rowID)).count + } + self.init(updates: rows(.update), inserts: rows(.insert), deletes: rows(.delete)) } var isEmpty: Bool { updates == 0 && inserts == 0 && deletes == 0 } diff --git a/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift b/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift index 11cafebc8f..77503f51b6 100644 --- a/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift +++ b/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift @@ -34,20 +34,36 @@ enum RowWriteOperationBuilder { writesThroughView: Bool = false, rowLocators: [RowID: String] = [:] ) -> [RowWriteOperation] { + // The same steps the save runs, so the record matches what ran. + let keyColumns = Set(primaryKeyColumns.compactMap { columns.firstIndex(of: $0) }) + var stepsByRow: [RowID: [RowChange]] = [:] + for step in RowChange.saveSteps(changes, keyColumns: keyColumns) where step.type == .update { + stepsByRow[step.rowID, default: []].append(step) + } + func firstOrder(of change: RowChange) -> Int { + guard change.type == .update else { return change.sequence } + return stepsByRow[change.rowID]?.first?.sequence ?? change.sequence + } + // A rewind undoes them newest first, and `changes` stops being in edit order once a change is cancelled. - changes.sorted { $0.sequence < $1.sequence }.compactMap { change in + let ordered = changes.enumerated() + .sorted { (firstOrder(of: $0.element), $0.offset) < (firstOrder(of: $1.element), $1.offset) } + .map(\.element) + return ordered.compactMap { change -> RowWriteOperation? in + var operation: RowWriteOperation switch change.type { case .update: - var operation = update( - change, target: target, columns: columns, + let steps = stepsByRow[change.rowID] ?? [change] + operation = update( + change, steps: steps, target: target, columns: columns, primaryKeyColumns: primaryKeyColumns, generatedColumns: generatedColumns, containsTableOperation: containsTableOperation ) operation.rowLocator = rowLocators[change.rowID] - return operation + operation.steps = steps.count > 1 ? steps.map { RowWriteStep($0) } : nil case .delete: guard deletedRowIDs.contains(change.rowID) else { return nil } - var operation = delete( + operation = delete( change, target: target, columns: columns, primaryKeyColumns: primaryKeyColumns, containsTableOperation: containsTableOperation, @@ -55,21 +71,23 @@ enum RowWriteOperationBuilder { throughView: writesThroughView ) operation.rowLocator = rowLocators[change.rowID] - return operation case .insert: guard insertedRowIDs.contains(change.rowID) else { return nil } - return insert( + operation = insert( change, values: insertedRowData[change.rowID], target: target, columns: columns, primaryKeyColumns: primaryKeyColumns, containsTableOperation: containsTableOperation ) } + operation.order = firstOrder(of: change) + return operation } } private static func update( _ change: RowChange, + steps: [RowChange], target: DataWriteTarget, columns: [String], primaryKeyColumns: [String], @@ -95,16 +113,18 @@ enum RowWriteOperationBuilder { postImage: postImage, writtenColumns: writtenColumns, refusal: updateRefusal( - change, preImage: preImage, primaryKeyColumns: primaryKeyColumns, - generatedColumns: generatedColumns, containsTableOperation: containsTableOperation + writing: change.cellChanges + steps.flatMap(\.cellChanges), preImage: preImage, + primaryKeyColumns: primaryKeyColumns, generatedColumns: generatedColumns, + containsTableOperation: containsTableOperation ), preImageAbsentColumns: change.absentColumns, postImageAbsentColumns: change.absentColumnsAfterUpdate ) } + // Every step's cells count: the inverse of a later step would write an earlier step's SQL back as a literal. private static func updateRefusal( - _ change: RowChange, + writing cellChanges: [CellChange], preImage: [PluginCellValue]?, primaryKeyColumns: [String], generatedColumns: Set, @@ -112,14 +132,14 @@ enum RowWriteOperationBuilder { ) -> RewindRefusal? { if containsTableOperation { return .destructiveTableOperation } if primaryKeyColumns.isEmpty || preImage == nil { return .noPrimaryKey } - if change.cellChanges.contains(where: \.newIsExpression) { + if cellChanges.contains(where: \.newIsExpression) { return .serverComputedValue } - if change.cellChanges.contains(where: { generatedColumns.contains($0.columnName) }) { + if cellChanges.contains(where: { generatedColumns.contains($0.columnName) }) { return .serverComputedValue } if let preImage, preImage.contains(where: exceedsCapture) { return .valueTooLarge } - if change.cellChanges.contains(where: { exceedsCapture($0.newValue) }) { return .valueTooLarge } + if cellChanges.contains(where: { exceedsCapture($0.newValue) }) { return .valueTooLarge } return nil } diff --git a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift index fb726e0bd5..dc9e221372 100644 --- a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift +++ b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift @@ -134,9 +134,16 @@ extension DatabaseAccessBridge { onLateCompletion: { CatalogChangeService.post(statementsRan) } ) { driver in reach.mark() - return try await ScriptBatchRun.run( - batches, startLines: startLines, rowCap: rowCap, driver: driver, progress: progress, context: context - ) + do { + let answered = try await ScriptBatchRun.run( + batches, startLines: startLines, rowCap: rowCap, driver: driver, progress: progress, context: context + ) + await manager.noteSessionStatements(Self.ranStatements(of: batches, progress), ranOn: driver, scope: scope) + return answered + } catch { + await manager.noteSessionStatements(Self.ranStatements(of: batches, progress), ranOn: driver, scope: scope) + throw error + } } } catch { if classification.tier != .safe { @@ -151,6 +158,11 @@ extension DatabaseAccessBridge { return run.outcome(executionTimeMs: (CFAbsoluteTimeGetCurrent() - startTime) * 1_000) } + // The batch a script stopped at ran too, and can have created a temporary table before its error. + private static func ranStatements(of batches: [ExecutableBatch], _ progress: ScriptBatchProgress) -> [String] { + batches.prefix(progress.completedBatchCount + 1).flatMap { $0.statements.map(\.sql) } + } + private static func postSucceededBatches( of batches: [ExecutableBatch], progress: ScriptBatchProgress, diff --git a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift index d3577f51c1..58b00eb116 100644 --- a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift +++ b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift @@ -415,6 +415,8 @@ internal actor DatabaseAccessBridge { onLateCompletion: { CatalogChangeService.post(statementRan) } ) { driver in reach.mark() + // Before it runs, so a statement that fails or a connection lost under it is still noted. + await manager.noteSessionStatements([sql], ranOn: driver, scope: scope) let answer = try await driver.executeUserQuery( query: sql, rowCap: fetchCap, diff --git a/apple/TablePro/Core/Database/DatabaseManager+CredentialLease.swift b/apple/TablePro/Core/Database/DatabaseManager+CredentialLease.swift index 164eba3a84..a0d45461fe 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+CredentialLease.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+CredentialLease.swift @@ -48,15 +48,19 @@ extension DatabaseManager { } } - // The lane reopens on its next use, signing in with a new lease. A transaction it held is reported as lost. + // The lane reopens on its next use, signing in with a new lease. A transaction or session state it held is + // reported as lost. private func closeLane(_ database: String, for ending: CredentialLeaseEnding) async { let connectionId = ending.connectionId guard let driver = sessionLanes.parkedDriver(for: connectionId, database: database) else { return } let held = await driver.heldSessionTransactionState() guard credentialLeaseAction(for: ending) == .closeLane(database: database) else { return } - if !held.permitsAppTransaction { - sessionLanes.markTransactionLost(database: database, for: connectionId) - } + sessionLanes.markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: sessionStateLedger.state(heldBy: driver, for: connectionId), + for: connectionId + ) Self.logger.info("Closing a lane whose Vault lease is ending for \(connectionId, privacy: .public)") sessionLanes.close(database: database, for: connectionId, supersedingOpens: false) } @@ -73,11 +77,13 @@ extension DatabaseManager { guard credentialLeaseAction(for: ending) == .reconnectSession else { return } try? await sessionDriverGate.withExclusiveAccess(browsedGateKey(for: connectionId)) { guard credentialLeaseAction(for: ending) == .reconnectSession else { return } - if holdsTransaction(connectionId, driver: ending.driver) { - // Reported to the next statement on this database, as a lane's lost transaction is. - let database = activeSessions[connectionId]?.resolvedBrowseDatabase ?? "" - sessionLanes.markTransactionLost(database: database, for: connectionId) - } + // Reported to the next statement on this database, as a lane's lost transaction is. What the session set up + // is noted by the reconnect, just before it closes this driver. + sessionLanes.markLost( + database: activeSessions[connectionId]?.resolvedBrowseDatabase ?? "", + transaction: holdsTransaction(connectionId, driver: ending.driver), + for: connectionId + ) Self.logger.info("Vault lease ending, reconnecting \(connectionId, privacy: .public) with a new one") let outcome = await performHealthMonitorReconnect(connectionId: connectionId, failedDriver: ending.driver) guard case .retry = outcome, credentialLeaseAction(for: ending) == .reconnectSession else { return } diff --git a/apple/TablePro/Core/Database/DatabaseManager+Health.swift b/apple/TablePro/Core/Database/DatabaseManager+Health.swift index 25bde8326d..a82634ef22 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Health.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Health.swift @@ -186,17 +186,25 @@ extension DatabaseManager { } } - // Read before the driver is closed and replaced, which prunes the hold. The user's next statement there - // is refused once, as on a lane, instead of running outside the transaction it was written for. - internal func noteTransactionLost(with driver: DatabaseDriver?, for connectionId: UUID) { + // Read before the driver is closed and replaced, which prunes the hold and the ledger. The user's next statement + // there is refused once, as on a lane, instead of running without the transaction or session it was written for. + // A switch the user asked for passes `includingState: false`: they chose to leave that session. + internal func noteSessionLost(with driver: DatabaseDriver?, for connectionId: UUID, includingState: Bool = true) { guard let driver, let session = activeSessions[connectionId] else { return } let lost = sessionTransactions.holds(for: connectionId).filter { sessionTransactions.driver(for: $0) === driver } for hold in lost { - sessionLanes.markTransactionLost( + sessionLanes.markLost( database: hold.key.lane ?? session.resolvedBrowseDatabase, + transaction: true, for: connectionId ) } + guard includingState else { return } + sessionLanes.markLost( + database: session.resolvedBrowseDatabase, + state: sessionStateLedger.state(heldBy: driver, for: connectionId), + for: connectionId + ) } internal func reconnectVerdict(for error: Error) -> ConnectionFailureClassifier.ReconnectVerdict { @@ -259,7 +267,7 @@ extension DatabaseManager { generation: Int ) async throws -> ReconnectResult? { let connectionId = session.connection.id - noteTransactionLost(with: session.driver, for: connectionId) + noteSessionLost(with: session.driver, for: connectionId) session.driver?.disconnect() // Resolved as the connect resolved it; a reference that lost its approval needs a person. let resolved = try await envVarReferenceConsent.resolve(session.connection, trigger: .background) @@ -415,7 +423,8 @@ extension DatabaseManager { // A debug target dialed through this transport would be left holding its locks once it goes. await RoutineDebugSessions.shared.stopAll(connectionId: sessionId) MetadataConnectionPool.shared.beginTransportReplacement(connectionId: sessionId) - await closeTransportLanes(for: sessionId) { self.isCurrentAttempt(generation, for: sessionId) } + // The user asked for this reconnect, so what the lanes' sessions set up is not reported as lost. + await closeTransportLanes(for: sessionId, includingState: false) { self.isCurrentAttempt(generation, for: sessionId) } } defer { if replacesPooledTransport { @@ -467,7 +476,9 @@ extension DatabaseManager { if movesPooledEndpoint { await RoutineDebugSessions.shared.stopAll(connectionId: sessionId) MetadataConnectionPool.shared.beginTransportReplacement(connectionId: sessionId) - await closeTransportLanes(for: sessionId) { self.isCurrentAttempt(generation, for: sessionId) } + await closeTransportLanes(for: sessionId, includingState: false) { + self.isCurrentAttempt(generation, for: sessionId) + } } defer { if movesPooledEndpoint { diff --git a/apple/TablePro/Core/Database/DatabaseManager+Recovery.swift b/apple/TablePro/Core/Database/DatabaseManager+Recovery.swift index 59fab36098..3d657c0891 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Recovery.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Recovery.swift @@ -1,5 +1,6 @@ import Foundation import os +import TableProSQLGrammar import TableProUtilities extension DatabaseManager { @@ -239,7 +240,12 @@ extension DatabaseManager { // Closes only the lanes parked when it starts, each while still parked as the same driver and only while // `owns` holds: a connect that took the session over parks lanes, and transactions, that are not this teardown's. - internal func closeTransportLanes(for connectionId: UUID, while owns: () -> Bool) async { + // A reconnect the user asked for passes `includingState: false`, as their own database switch does. + internal func closeTransportLanes( + for connectionId: UUID, + includingState: Bool = true, + while owns: () -> Bool + ) async { guard owns() else { return } let lanes = sessionLanes.parkedDatabases(for: connectionId).compactMap { database in sessionLanes.parkedDriver(for: connectionId, database: database).map { (database, $0) } @@ -250,9 +256,12 @@ extension DatabaseManager { let held = await driver.heldSessionTransactionState() guard owns() else { return } guard sessionLanes.parkedDriver(for: connectionId, database: database) === driver else { continue } - if !held.permitsAppTransaction { - sessionLanes.markTransactionLost(database: database, for: connectionId) - } + sessionLanes.markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: includingState ? sessionStateLedger.state(heldBy: driver, for: connectionId) : [], + for: connectionId + ) sessionLanes.close(database: database, for: connectionId, supersedingOpens: false) } } diff --git a/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift b/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift index ad1ba9c387..2189070fbf 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift @@ -5,6 +5,7 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar /// Where an operation bound to a `DatabaseScope` runs. enum ScopedDriverRoute: Equatable { @@ -76,16 +77,15 @@ extension DatabaseManager { else { return .sessionDriver } - /// A database the user has browsed keeps its own session connection, so its tabs carry on in - /// the transaction, temp tables and settings they left there. - /// A database whose connection ended with a transaction still open takes the session path - /// once more, where that loss is reported before anything runs on a replacement. + // A database the user has browsed keeps its own session connection, so its tabs carry on in the transaction, + // temp tables and settings they left there. A database whose connection ended holding a transaction or session + // state takes the session path once more, where that loss is reported before anything runs on a replacement. if usesDatabaseLanes(session) { if sessionLanes.parkedDriver(for: scope.connectionId, database: scope.database) != nil { return .sessionDriver } - // A table read depends on no transaction, so a lost one is not its to hear about. - if opensLane, sessionLanes.hasTransactionLoss(database: scope.database, for: scope.connectionId) { + // A table read depends on no transaction or session state, so a lost one is not its to hear about. + if opensLane, sessionLanes.hasLoss(database: scope.database, for: scope.connectionId) { return .sessionDriver } // A temporary table lives in the session that made it, which a new connection is not. @@ -106,10 +106,9 @@ extension DatabaseManager { return .pooled } - /// `cancellation` decides whether Stop, or a navigation that supersedes a tab, can reach the - /// leased driver. The parameter has no default on purpose: a new call site must state which it - /// is, because getting it wrong silently either makes a query uncancellable or makes a commit - /// killable. + // `cancellation` decides whether Stop, or a navigation that supersedes a tab, can reach the leased driver. The + // parameter has no default on purpose: a new call site must state which it is, because getting it wrong silently + // either makes a query uncancellable or makes a commit killable. func withScopedDriver( scope: DatabaseScope, route: ScopedDriverRoute, @@ -395,9 +394,9 @@ extension DatabaseManager { } } - /// `ownsSessionState` is false for a table read or a metadata read, which depend on nothing the - /// session holds. Such a turn neither opens a session connection nor takes the report of a lost - /// transaction, which is kept for the user's next statement. + // `ownsSessionState` is false for a table read or a metadata read, which depend on nothing the session holds. Such + // a turn neither opens a session connection nor takes the report of a lost session, which is kept for the next + // turn that owns session state. That turn is refused once with the whole loss, whatever work it is. private func withSessionDriverTurn( scope: DatabaseScope, ownsSessionState: Bool, @@ -406,20 +405,20 @@ extension DatabaseManager { let connectionId = scope.connectionId let lane = activeSessions[connectionId].flatMap { laneDatabase(for: scope, in: $0) } let startsOnBrowsed = lane == nil || lane == activeSessions[connectionId]?.resolvedBrowseDatabase - /// Decided before the turn is queued. Work written against a connection that was there when - /// it queued and is gone when its turn comes is refused, never moved onto a new one: the - /// transaction it meant to continue ended with the connection it was on. + // Decided before the turn is queued. Work written against a connection that was there when it queued and is + // gone when its turn comes is refused, never moved onto a new one: the transaction it meant to continue ended + // with the connection it was on. let startsWithoutLane = !startsOnBrowsed && !scope.isSessionBound && lane.flatMap { sessionLanes.parkedDriver(for: connectionId, database: $0) } == nil if startsOnBrowsed { - /// Outside the gate on purpose. A verification that has to reconnect runs the whole - /// reconnect, which restores the schema and the database on the new driver, and doing that - /// while holding the gate would deadlock the very thing waiting to be pinned. + // Outside the gate on purpose. A verification that has to reconnect runs the whole reconnect, which + // restores the schema and the database on the new driver, and doing that while holding the gate would + // deadlock the very thing waiting to be pinned. await verifyBeforeUse(connectionId) } - /// A check that failed and could not recover left the driver installed and disconnected, - /// so the presence of a driver below is not enough. Refusing here is the point of checking - /// at all: without it the user's own work runs on a handle the app already knows is dead. + // A check that failed and could not recover left the driver installed and disconnected, so the presence of a + // driver below is not enough. Refusing here is the point of checking at all: without it the user's own work + // runs on a handle the app already knows is dead. guard isUsable(connectionId) else { throw DatabaseError.notConnected } @@ -428,19 +427,19 @@ extension DatabaseManager { ) { try await trackOperation(sessionId: connectionId) { try Task.checkCancellation() - /// Asked again once the lease has its turn, because a database switch that held the - /// gate can have left the driver the same way while this lease waited, or moved the - /// database this turn belongs to from the browsed connection to a parked one. - /// A database whose own connection is gone, its entry closed, falls back to the browsed - /// driver only to be turned away: a table read re-routes to the pool and anything else - /// is refused by `pin` before it runs a statement there. + // Asked again once the lease has its turn, because a database switch that held the gate can have left + // the driver the same way while this lease waited, or moved the database this turn belongs to from the + // browsed connection to a parked one. A database whose own connection is gone, its entry closed, falls + // back to the browsed driver only to be turned away: a table read re-routes to the pool and anything + // else is refused by `pin` before it runs a statement there. guard isUsable(connectionId), let browsedDriver = activeSessions[connectionId]?.driver else { throw DatabaseError.notConnected } guard let lane else { - // A driver replaced while it held a transaction, by a reconnect or a Vault lease rotation. - if ownsSessionState, let lost = sessionLanes.takeAnyTransactionLoss(for: connectionId) { - throw DatabaseError.connectionFailed(Self.parkedTransactionLostMessage(database: lost)) + // A driver replaced while it held a transaction or session state, by a reconnect or a Vault lease + // rotation. + if ownsSessionState, let lost = sessionLanes.takeAnyLoss(for: connectionId) { + throw DatabaseError.connectionFailed(Self.sessionLostMessage(database: lost.database, loss: lost.loss)) } return try await recordingTransaction(on: browsedDriver, for: .init(connectionId: connectionId, lane: nil)) { try await turn(browsedDriver) @@ -451,7 +450,7 @@ extension DatabaseManager { driver = held } else if ownsSessionState, !scope.isSessionBound, startsWithoutLane || sessionLanes.wasClosedIdle(database: lane, for: connectionId), - !sessionLanes.hasTransactionLoss(database: lane, for: connectionId) { + !sessionLanes.hasLoss(database: lane, for: connectionId) { driver = try await openedLaneDriver(database: lane, for: connectionId) } else { driver = browsedDriver @@ -459,8 +458,8 @@ extension DatabaseManager { let laneDriver = sessionLanes.isParked(driver, for: connectionId) ? try await verifiedParkedDriver(driver, database: lane, for: connectionId) : driver - if ownsSessionState, sessionLanes.takeTransactionLoss(database: lane, for: connectionId) { - throw DatabaseError.connectionFailed(Self.parkedTransactionLostMessage(database: lane)) + if ownsSessionState, let loss = sessionLanes.takeLoss(database: lane, for: connectionId) { + throw DatabaseError.connectionFailed(Self.sessionLostMessage(database: lane, loss: loss)) } sessionLanes.noteOpenedUse(database: lane, for: connectionId) defer { sessionLanes.noteOpenedUse(database: lane, for: connectionId) } @@ -514,6 +513,12 @@ extension DatabaseManager { return scope.isServerScoped ? session.resolvedBrowseDatabase : scope.database } + // The key a scope's turn records its transaction under, which the session state ledger shares. + internal func sessionKey(for scope: DatabaseScope) -> SessionTransactionTracker.Key? { + guard let session = activeSessions[scope.connectionId] else { return nil } + return SessionTransactionTracker.Key(connectionId: scope.connectionId, lane: laneDatabase(for: scope, in: session)) + } + internal func sessionDriver(holding scope: DatabaseScope) -> DatabaseDriver? { guard let session = activeSessions[scope.connectionId] else { return nil } return sessionDriver(for: scope.connectionId, laneDatabase: laneDatabase(for: scope, in: session)) @@ -528,18 +533,17 @@ extension DatabaseManager { return sessionLanes.parkedDriver(for: connectionId, database: laneDatabase) } - /// A parked connection nobody watched may have died while it sat idle. One that no longer - /// answers is replaced before the work runs, and if it was holding a transaction the work is - /// refused once, because running it on the new connection would carry on as if that transaction - /// had not just been rolled back by the server. + // A parked connection nobody watched may have died while it sat idle. One that no longer answers is replaced + // before the work runs, and if it held a transaction or session state the work is refused once, because running it + // on the new connection would carry on as if the server had not just dropped them. private func verifiedParkedDriver( _ driver: DatabaseDriver, database: String, for connectionId: UUID ) async throws -> DatabaseDriver { if sessionLanes.isFresh(driver) { return driver } - /// Read before the ping: a ping that finds the socket closed leaves libpq reporting no - /// transaction state at all, which would read as nothing to lose. + // Read before the ping: a ping that finds the socket closed leaves libpq reporting no transaction state at + // all, which would read as nothing to lose. let held = await driver.heldSessionTransactionState() let wasOpened = sessionLanes.isOpened(database: database, for: connectionId) sessionLanes.beginVerifying(driver) @@ -549,12 +553,15 @@ extension DatabaseManager { sessionLanes.markVerified(driver) return driver } catch { + // Recorded before the close, which prunes the ledger, and before the reopen, which can fail or be + // superseded: the session is gone either way, and whatever runs on this database next has to hear of it. + sessionLanes.markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: sessionStateLedger.state(heldBy: driver, for: connectionId), + for: connectionId + ) sessionLanes.close(database: database, for: connectionId, supersedingOpens: false) - /// Recorded before the reopen, which can fail or be superseded: the transaction is gone - /// either way, and whatever runs on this database next has to hear about it. - if !held.permitsAppTransaction { - sessionLanes.markTransactionLost(database: database, for: connectionId) - } let generation = sessionLanes.generation(for: connectionId) let reopened = try await sessionLanes.open( DatabaseScope(connectionId: connectionId, database: database, schema: nil) @@ -617,10 +624,9 @@ extension DatabaseManager { throw DatabaseError.notConnected } - /// Closes the connections opened for SQL on databases the user never browsed that nobody has - /// used for `idleFor`. One that is running work or holds a transaction stays. Whatever else the - /// session held goes with it unreported: on PostgreSQL its temp tables, `SET` values and - /// advisory locks. + // Closes the connections opened for SQL on databases the user never browsed that nobody has used for `idleFor`. + // One that is running work, holds a transaction or holds session state the user's statements set up stays until + // disconnect, as a browsed one does. What a routine set up is out of sight and goes with the close. internal func closeIdleOpenedLanes( for connectionId: UUID, idleFor: TimeInterval = SessionLanes.openedIdleTimeout, @@ -631,8 +637,8 @@ extension DatabaseManager { } } - /// Asked inside the lane's own turn, so no statement runs between the answer and the close. Only - /// a driver that says it is idle is closed: one that cannot say may be holding a transaction. + // Asked inside the lane's own turn, so no statement runs between the answer and the close. Only a driver that says + // it is idle is closed: one that cannot say may be holding a transaction. private func closeOpenedLaneIfIdle( _ database: String, driver: DatabaseDriver, @@ -640,10 +646,12 @@ extension DatabaseManager { ) async -> Bool { let key = SessionDriverGate.Key(connectionId: connectionId, database: database) guard !sessionDriverGate.isHeld(key) else { return false } + let ledgerKey = SessionTransactionTracker.Key(connectionId: connectionId, lane: database) let closed = try? await sessionDriverGate.withExclusiveAccess(key) { () async -> Bool in guard sessionLanes.isOpened(database: database, for: connectionId), sessionLanes.parkedDriver(for: connectionId, database: database) === driver, - !sessionTransactions.holdsTransaction(connectionId: connectionId, container: database) + !sessionTransactions.holdsTransaction(connectionId: connectionId, container: database), + sessionStateLedger.state(for: ledgerKey, heldBy: driver).isEmpty else { return false } let state = await Self.boundedTransactionState(of: driver, clock: ContinuousClock()) guard state == .idle, diff --git a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift index 66dcd2fad0..36c19de2ac 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift @@ -574,21 +574,21 @@ extension DatabaseManager { ) } - /// The connection parked for `database`, if it still answers. One that died is closed, and if it - /// was holding a transaction, the next work on that database is told the server rolled it back. - /// A turn running on it right now is answer enough, and asking would wait for that turn. + // The connection parked for `database`, if it still answers. One that died is closed, and if it held a transaction + // or session state, the next work on that database is told what the server dropped. A turn running on it right now + // is answer enough, and asking would wait for that turn. private func answeringParkedDriver(database: String, for connectionId: UUID) async -> DatabaseDriver? { let key = SessionDriverGate.Key(connectionId: connectionId, database: database) guard var parked = sessionLanes.parkedDriver(for: connectionId, database: database) else { return nil } - /// A check already running decides for this switch too: once it ends it has either confirmed - /// the driver or parked a replacement, and that is what gets promoted. + // A check already running decides for this switch too: once it ends it has either confirmed the driver or + // parked a replacement, and that is what gets promoted. if sessionLanes.isVerifying(parked) { await sessionLanes.waitForVerification(of: parked) guard let settled = sessionLanes.parkedDriver(for: connectionId, database: database) else { return nil } parked = settled } if sessionLanes.isFresh(parked) || sessionDriverGate.isHeld(key) { return parked } - /// Read before the ping, which on a closed socket leaves libpq reporting no state at all. + // Read before the ping, which on a closed socket leaves libpq reporting no state at all. let held = await parked.heldSessionTransactionState() do { try await sessionDriverGate.withExclusiveAccess(key) { @@ -598,10 +598,13 @@ extension DatabaseManager { return parked } catch { guard sessionLanes.parkedDriver(for: connectionId, database: database) === parked else { return nil } + sessionLanes.markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: sessionStateLedger.state(heldBy: parked, for: connectionId), + for: connectionId + ) sessionLanes.close(database: database, for: connectionId, supersedingOpens: false) - if !held.permitsAppTransaction { - sessionLanes.markTransactionLost(database: database, for: connectionId) - } return nil } } @@ -657,7 +660,7 @@ extension DatabaseManager { let previousSavedSchema = appSettingsStorage.loadLastSchema(for: connectionId) // Noted under the database being left, before the session names the target. - noteTransactionLost(with: previous.driver, for: connectionId) + noteSessionLost(with: previous.driver, for: connectionId, includingState: false) updateSession(connectionId) { session in session.connection.database = database session.browseDatabase = database @@ -1014,6 +1017,7 @@ extension DatabaseManager { activeSessions[connectionId] = session connectionStatusVersions[connectionId, default: 0] &+= 1 sessionTransactions.prune(connectionId) + sessionStateLedger.prune(connectionId) AppEvents.shared.connectionStatusChanged.send( ConnectionStatusChange(connectionId: connectionId, status: session.reportedStatus) ) diff --git a/apple/TablePro/Core/Database/DatabaseManager.swift b/apple/TablePro/Core/Database/DatabaseManager.swift index 5c4db15f3c..275998de5a 100644 --- a/apple/TablePro/Core/Database/DatabaseManager.swift +++ b/apple/TablePro/Core/Database/DatabaseManager.swift @@ -9,6 +9,7 @@ import Combine import Foundation import os import TableProPluginKit +import TableProSQLGrammar /// Manages database connections and active drivers @MainActor @@ -143,6 +144,11 @@ final class DatabaseManager: ObservableObject { self?.isLiveSessionDriver(driver, for: key) ?? false } + // What the user's statements set up on each session connection, read before a reconnect closes one. + internal lazy var sessionStateLedger = SessionStateLedger { [weak self] key, driver in + self?.isLiveSessionDriver(driver, for: key) ?? false + } + /// The driver each connection's health monitor last pinged, so the reconnect that follows a /// failed ping acts only on that driver and never on one a database switch put in its place. /// Held, not just identified: a closed driver that was freed would hand its address, and with it @@ -212,6 +218,10 @@ final class DatabaseManager: ObservableObject { self.recoveryClock = recoveryClock sessionLanes.onClose = { [weak self] connectionId in self?.sessionTransactions.prune(connectionId) + self?.sessionStateLedger.prune(connectionId) + } + sessionLanes.recordedState = { [weak self] driver, connectionId in + self?.sessionStateLedger.state(heldBy: driver, for: connectionId) ?? [] } observeConnectionUpdates() observeHealthCheckSetting() diff --git a/apple/TablePro/Core/Database/NativeDumpDescriptor.swift b/apple/TablePro/Core/Database/NativeDumpDescriptor.swift index 2d93bf3683..c48f7f9e47 100644 --- a/apple/TablePro/Core/Database/NativeDumpDescriptor.swift +++ b/apple/TablePro/Core/Database/NativeDumpDescriptor.swift @@ -6,34 +6,26 @@ import Foundation import TableProPluginKit -/// Everything one engine's own dump and restore need, as data. -/// -/// The three engines that had this hard-coded were all PostgreSQL. Generalizing it is not a matter -/// of parameterizing a binary name: `mysqldump` writes SQL to standard output where `pg_dump -Fc` -/// writes an archive to a path, `mongodump` takes a URI rather than host and port flags, `sqlite3` -/// needs no network arguments at all, and DuckDB runs no binary because its dump is a statement its -/// own in-process engine executes. So a descriptor names its mechanism and supplies its own -/// arguments or statements, rather than filling slots in one shared command shape. +// The three engines that had this hard-coded were all PostgreSQL. Generalizing it is not a matter +// of parameterizing a binary name: `mysqldump` writes SQL to standard output where `pg_dump -Fc` +// writes an archive to a path, `mongodump` takes a URI rather than host and port flags, `sqlite3` +// needs no network arguments at all, and DuckDB runs no binary because its dump is a statement its +// own in-process engine executes. So a descriptor names its mechanism and supplies its own +// arguments or statements, rather than filling slots in one shared command shape. struct NativeDumpDescriptor: Sendable { - /// How the tool hands back what it produced. This is the difference that no shared argument - /// list can paper over. enum OutputDelivery: Sendable, Equatable { - /// The tool is told where to write, and writes there itself. case toolWritesFile - /// The tool writes to standard output and the caller redirects it to the file. case standardOutput } - /// What the caller offers as a file name, and what a restore will accept back. - /// - /// `id` exists because one engine can offer two: DuckDB writes either a single `.duckdb` file - /// or a directory of Parquet, and the sheet has to name which one it is about to write. + // `id` exists because one engine can offer two: DuckDB writes either a single `.duckdb` file + // or a directory of Parquet, and the sheet has to name which one it is about to write. struct ArchiveFormat: Sendable, Equatable, Identifiable { let id: String let fileExtension: String let contentDescription: String - /// A destination the user chooses with an open panel rather than a save panel, because the - /// engine writes a folder of files rather than one file. + // A destination the user chooses with an open panel rather than a save panel, because the + // engine writes a folder of files rather than one file. let producesDirectory: Bool init( @@ -56,25 +48,19 @@ struct NativeDumpDescriptor: Sendable { let password: String? let scope: NativeDumpScope - /// The engine's own name for the database in front of it, read from the live connection. - /// - /// Only DuckDB needs one, and it needs it badly: `COPY FROM DATABASE TO ` names - /// a catalog, and DuckDB derives that catalog from the file's basename. Measured, a file - /// `my-weird.name.db` attaches as `my-weird`, and `COPY FROM DATABASE main` fails with - /// `Catalog "main" does not exist`. So neither the file name nor a constant will do. + // Only DuckDB needs one, and it needs it badly: `COPY FROM DATABASE TO ` names + // a catalog, and DuckDB derives that catalog from the file's basename. Measured, a file + // `my-weird.name.db` attaches as `my-weird`, and `COPY FROM DATABASE main` fails with + // `Catalog "main" does not exist`. So neither the file name nor a constant will do. let currentCatalog: String? - /// The path a file-backed driver actually opens. - /// - /// SQLite keeps it in `database`, while DuckDB and libSQL keep it in a plugin-declared - /// additional field and leave `database` empty. Resolved by the caller through - /// `LocalFilePathField` rather than guessed here, because reading `connection.database` for - /// libSQL handed `sqlite3` an empty path, which exits 0 and writes a 52-byte file that the - /// result sheet then reports as a successful backup. + // SQLite keeps the file's path in `database`; DuckDB and libSQL keep it in a plugin-declared field and leave + // `database` empty. Resolved by the caller through `LocalFilePathField`: reading `connection.database` for libSQL + // handed `sqlite3` an empty path, which exits 0 and writes a 52-byte file the result sheet reports as a backup. let localFilePath: String? - /// What the live session's driver reports the server to be, so a tool can be given the - /// flags that server needs. `mysqldump` 8.0 reads a table no MariaDB server has. + // What the live session's driver reports the server to be, so a tool can be given the + // flags that server needs. `mysqldump` 8.0 reads a table no MariaDB server has. let serverVersion: String? // Whether the server's greeting offered TLS, for a tool whose SSL options depend on it. Nil when not read. @@ -83,6 +69,9 @@ struct NativeDumpDescriptor: Sendable { // Set by `NativeDumpService.buildCommand` before the arguments are built. var clientPluginFolder: MariaDBClientPluginFolder? + // Set by `NativeDumpService` when the tool's `needsSystemTrustAnchors` says so, and removed when the tool exits. + var systemTrustAnchorsFile: URL? + init( connection: DatabaseConnection, database: String, @@ -120,25 +109,24 @@ struct NativeDumpDescriptor: Sendable { let value: String } - /// A tool on the user's Mac that the app spawns. struct CommandLineTool: Sendable { - /// The candidate names, in the order they are tried. More than one because a tool can ship - /// under two names: MariaDB renamed `mysqldump` to `mariadb-dump` in 11.0 and keeps the old - /// name only as a symlink that some builds omit. + // The candidate names, in the order they are tried. More than one because a tool can ship + // under two names: MariaDB renamed `mysqldump` to `mariadb-dump` in 11.0 and keeps the old + // name only as a symlink that some builds omit. let backupBinaries: [String] let restoreBinaries: [String] - /// What to tell the user to install when neither name resolves. There is no portable - /// answer, so each engine names its own package. + // What to tell the user to install when neither name resolves. There is no portable + // answer, so each engine names its own package. let installHint: String let backupDelivery: OutputDelivery let restoreDelivery: OutputDelivery - /// True when the tool has no channel for a password but its own argument list, which every - /// process on the machine can read through `ps`. Only `sqlpackage` is in that position; - /// every other engine here takes one from the environment or a file. The flows that touch - /// such a tool say so rather than leaving the user to find out. + // True when the tool has no channel for a password but its own argument list, which every + // process on the machine can read through `ps`. Only `sqlpackage` is in that position; + // every other engine here takes one from the environment or a file. The flows that touch + // such a tool say so rather than leaving the user to find out. let exposesPasswordInArguments: Bool let configurationFileEntries: @Sendable (Request) -> [ConfigurationEntry] @@ -147,14 +135,14 @@ struct NativeDumpDescriptor: Sendable { internal let requiresUntranslatedMessages: Bool - /// Picks the binary to run for a known server version, when the engine's tools refuse - /// some servers. Nil leaves the plain PATH lookup in place. + // Picks the binary to run for a known server version, when the engine's tools refuse + // some servers. Nil leaves the plain PATH lookup in place. let toolForServer: (@Sendable (_ binary: String, _ serverVersion: String?) -> NativeDumpToolSelection)? - /// Says what the resolved binary actually is, for an engine whose tools forked their - /// options. It runs a process, so it runs once per resolution rather than once per - /// argument list. Nil leaves the tool unidentified, which is what every engine but MySQL - /// and MariaDB wants. + // Says what the resolved binary actually is, for an engine whose tools forked their + // options. It runs a process, so it runs once per resolution rather than once per + // argument list. Nil leaves the tool unidentified, which is what every engine but MySQL + // and MariaDB wants. let identifyExecutable: (@Sendable (_ name: String, _ path: String) -> NativeDumpResolvedTool)? let backupArguments: @Sendable (Request, NativeDumpResolvedTool) throws -> [String] @@ -166,6 +154,9 @@ struct NativeDumpDescriptor: Sendable { let clientPluginFolder: @Sendable (Request, NativeDumpResolvedTool) throws -> MariaDBClientPluginFolder? + // True when the tool has no trust store of its own to check the server with, so it gets a file of this Mac's. + let needsSystemTrustAnchors: @Sendable (Request) -> Bool + init( backupBinaries: [String], restoreBinaries: [String], @@ -182,7 +173,8 @@ struct NativeDumpDescriptor: Sendable { environment: @escaping @Sendable (Request) -> [String: String] = { _ in [:] }, configurationFileEntries: @escaping @Sendable (Request) -> [ConfigurationEntry] = { _ in [] }, prepare: @escaping @Sendable (Request, NativeDumpResolvedTool) async throws -> Request = { request, _ in request }, - clientPluginFolder: @escaping @Sendable (Request, NativeDumpResolvedTool) throws -> MariaDBClientPluginFolder? = { _, _ in nil } + clientPluginFolder: @escaping @Sendable (Request, NativeDumpResolvedTool) throws -> MariaDBClientPluginFolder? = { _, _ in nil }, + needsSystemTrustAnchors: @escaping @Sendable (Request) -> Bool = { _ in false } ) { self.backupBinaries = backupBinaries self.restoreBinaries = restoreBinaries @@ -200,6 +192,7 @@ struct NativeDumpDescriptor: Sendable { self.configurationFileEntries = configurationFileEntries self.prepare = prepare self.clientPluginFolder = clientPluginFolder + self.needsSystemTrustAnchors = needsSystemTrustAnchors } func binaries(for kind: NativeDumpKind) -> [String] { @@ -214,8 +207,8 @@ struct NativeDumpDescriptor: Sendable { try kind == .backup ? backupArguments(request, resolved) : restoreArguments(request, resolved) } - /// What the app knows about the binary it resolved. An engine that declares no - /// `identifyExecutable` gets the plain answer, which is what its arguments already assume. + // What the app knows about the binary it resolved. An engine that declares no + // `identifyExecutable` gets the plain answer, which is what its arguments already assume. func identify(name: String, path: String) -> NativeDumpResolvedTool { identifyExecutable?(name, path) ?? NativeDumpResolvedTool(name: name, path: path) } @@ -229,19 +222,17 @@ struct NativeDumpDescriptor: Sendable { } } - /// Statements the engine already in front of the app runs for itself. - /// - /// There is no binary to find, no password to hand over and nothing to reap, which is why this - /// cannot be a `CommandLineTool` with an empty argument list. An embedded engine also has no - /// server to write on the user's behalf, so the file lands where the panel picked. + // There is no binary to find, no password to hand over and nothing to reap, which is why this + // cannot be a `CommandLineTool` with an empty argument list. An embedded engine also has no + // server to write on the user's behalf, so the file lands where the panel picked. struct EngineStatements: Sendable { - /// The statements for one direction, in order. `alias` is a name for the attached backup, - /// unique per run so a `DETACH` that could not run leaves nothing blocking the next one. + // The statements for one direction, in order. `alias` is a name for the attached backup, + // unique per run so a `DETACH` that could not run leaves nothing blocking the next one. let backupStatements: @Sendable (Request, String) -> [String] let restoreStatements: @Sendable (Request, String) -> [String] - /// Run after a failure and ignored. A `DETACH` inside an aborted transaction fails too, and - /// the caller has the engine's real error to report rather than this one. + // Run after a failure and ignored. A `DETACH` inside an aborted transaction fails too, and + // the caller has the engine's real error to report rather than this one. let cleanupStatements: @Sendable (String) -> [String] // Set when the archive's own SQL must not run on the session: the driver imports it into a new file of this @@ -271,9 +262,9 @@ struct NativeDumpDescriptor: Sendable { let objectScope: NativeDumpObjectScope let restoreSemantics: RestoreSemantics - /// True when the tool opens the database file itself and so cannot reach a connection that has - /// no local file. libSQL claims the SQLite descriptor and reaches either a file or a Turso URL, - /// and only the file can be handed to `sqlite3`. + // True when the tool opens the database file itself and so cannot reach a connection that has + // no local file. libSQL claims the SQLite descriptor and reaches either a file or a Turso URL, + // and only the file can be handed to `sqlite3`. let requiresLocalFile: Bool init( diff --git a/apple/TablePro/Core/Database/NativeDumpRegistry.swift b/apple/TablePro/Core/Database/NativeDumpRegistry.swift index d1ec365aa5..88421a5b11 100644 --- a/apple/TablePro/Core/Database/NativeDumpRegistry.swift +++ b/apple/TablePro/Core/Database/NativeDumpRegistry.swift @@ -6,21 +6,19 @@ import Foundation import TableProPluginKit import TableProSQLQuoting - -/// Which engines can dump and restore, and how to drive them. -/// -/// Curated here rather than declared by each driver plugin, because the facts are about tools on -/// the user's Mac rather than about the driver's own behaviour, and adding a `DriverPlugin` static -/// for them would mean an ABI change and a re-release of every registry plugin for data no plugin -/// produces. -/// -/// A password never reaches the argument list. Anything in `argv` is readable by every process on -/// the machine through `ps`, so each engine's own out-of-band channel is used instead: `PGPASSWORD` -/// and `MYSQL_PWD` for the two that read the environment, and a `0600` config file for MongoDB, -/// whose tools read neither. +import TableProTLSClientIdentity + +// Curated here rather than declared by each driver plugin, because the facts are about tools on +// the user's Mac rather than about the driver's own behaviour, and adding a `DriverPlugin` static +// for them would mean an ABI change and a re-release of every registry plugin for data no plugin +// produces. +// A password never reaches the argument list. Anything in `argv` is readable by every process on +// the machine through `ps`, so each engine's own out-of-band channel is used instead: `PGPASSWORD` +// and `MYSQL_PWD` for the two that read the environment, and a `0600` config file for MongoDB, +// whose tools read neither. enum NativeDumpRegistry { - /// The formats one engine can write, in the order the sheet offers them. The first is the - /// default. Only DuckDB has more than one. + // The formats one engine can write, in the order the sheet offers them. The first is the + // default. Only DuckDB has more than one. static func formats(for type: DatabaseType) -> [NativeDumpDescriptor.ArchiveFormat] { switch type { case .duckdb: @@ -58,12 +56,11 @@ enum NativeDumpRegistry { descriptor(for: type) != nil } - /// Whether this particular connection can be dumped, which is not always the same question as - /// whether its engine can. - /// - /// libSQL reaches either a local file or a Turso URL, and only the file can be handed to - /// `sqlite3`. Answering by type alone is how a remote libSQL connection produced a 52-byte - /// backup the result sheet reported as a success. + // Whether this particular connection can be dumped, which is not always the same question as + // whether its engine can. + // libSQL reaches either a local file or a Turso URL, and only the file can be handed to + // `sqlite3`. Answering by type alone is how a remote libSQL connection produced a 52-byte + // backup the result sheet reported as a success. static func supports(_ connection: DatabaseConnection, localFilePath: String?) -> Bool { guard let descriptor = descriptor(for: connection.type) else { return false } guard descriptor.requiresLocalFile else { return true } @@ -73,8 +70,8 @@ enum NativeDumpRegistry { // MARK: - PostgreSQL - /// Redshift reports 8.0.2, which no `pg_dump` release claims to support, so only PostgreSQL - /// itself is matched to the server. + // Redshift reports 8.0.2, which no `pg_dump` release claims to support, so only PostgreSQL + // itself is matched to the server. static let postgresToolForServer: @Sendable (String, String?) -> NativeDumpToolSelection = { binary, serverVersion in PostgreSQLDumpToolLocator.select(binary: binary, serverVersion: serverVersion) } @@ -121,10 +118,14 @@ enum NativeDumpRegistry { environment["PGHOSTADDR"] = hostaddr } environment.merge( - postgresSSLEnvironment(request.connection.transportSSLConfiguration) + postgresSSLEnvironment( + request.connection.transportSSLConfiguration, + systemTrustAnchorsFile: request.systemTrustAnchorsFile + ) ) { _, new in new } return environment - } + }, + needsSystemTrustAnchors: { request in postgresNeedsSystemTrustAnchors(request) } ) ), archiveFormat: NativeDumpDescriptor.ArchiveFormat( @@ -186,18 +187,23 @@ enum NativeDumpRegistry { } } - /// The whole of the connection's SSL configuration, not just its mode. - /// - /// libpq falls back to `~/.postgresql/root.crt` when no root certificate is named, and measured - /// with pg_dump 17.11 a `verify-ca` connection with no such file fails before connecting with - /// `root certificate file "..." does not exist`. So a Verify CA connection that opens in the - /// app could never be dumped. The rule matches what `LibPQConnectionString` already sends on - /// the live connection, down to the CA being tied to the modes that verify. - static func postgresSSLEnvironment(_ ssl: SSLConfiguration) -> [String: String] { + // What `LibPQConnectionString` sends on the live connection. Measured with pg_dump 17.11, a verifying mode with no + // root file named and no `~/.postgresql/root.crt` fails before connecting with `root certificate file "..." does not + // exist`. + static func postgresSSLEnvironment( + _ ssl: SSLConfiguration, + systemTrustAnchorsFile: URL? = nil, + inherited: [String: String] = ProcessInfo.processInfo.environment + ) -> [String: String] { guard ssl.isEnabled, let mode = postgresSSLMode(ssl.mode) else { return [:] } var environment = ["PGSSLMODE": mode] if ssl.verifiesCertificate, !ssl.caCertificatePath.isEmpty { environment["PGSSLROOTCERT"] = ssl.caCertificatePath + } else if ssl.verifiesHostname, let systemTrustAnchorsFile { + environment["PGSSLROOTCERT"] = systemTrustAnchorsFile.path + } else if ssl.verifiesCertificate, let named = inherited["PGSSLROOTCERT"], !named.isEmpty { + // The tool starts from a minimal environment, so a root file the user named for libpq is carried to it. + environment["PGSSLROOTCERT"] = named } if !ssl.clientCertificatePath.isEmpty { environment["PGSSLCERT"] = ssl.clientCertificatePath @@ -208,10 +214,32 @@ enum NativeDumpRegistry { return environment } + // The driver's rule, `LibPQConnectionString.rootCertificateSource`: Verify Identity with no CA file checks this Mac's + // trusted roots, unless the server is a socket or libpq's own `~/.postgresql/root.crt` is there. The tool inherits + // HOME, so it looks for that file where this does. + static func postgresNeedsSystemTrustAnchors( + _ request: NativeDumpDescriptor.Request, + environment: [String: String] = ProcessInfo.processInfo.environment, + fileExists: (String) -> Bool = { FileManager.default.fileExists(atPath: $0) } + ) -> Bool { + let ssl = request.connection.transportSSLConfiguration + return LibPQSystemTrustRule.usesSystemTrustAnchors( + verifiesHostname: ssl.verifiesHostname, + caCertificatePath: ssl.caCertificatePath, + host: postgresEndpoint(request).host, + environment: environment, + fileExists: fileExists + ) + } + + static func postgresDefaultRootCertificatePath(environment: [String: String]) -> String { + LibPQSystemTrustRule.defaultRootCertificatePath(environment: environment) + } + // MARK: - MySQL and MariaDB - /// MariaDB 11.0 renamed every client, keeping the `mysql`-prefixed names as symlinks that some - /// builds leave out, so both spellings are tried. + // MariaDB 11.0 renamed every client, keeping the `mysql`-prefixed names as symlinks that some + // builds leave out, so both spellings are tried. private static var mysql: NativeDumpDescriptor { NativeDumpDescriptor( mechanism: .commandLineTool( @@ -269,12 +297,12 @@ enum NativeDumpRegistry { ) } - /// `--` before the database, because `my_getopt` does not stop parsing options at the first - /// positional argument and every name after it is the user's. Measured with mysqldump 8.4.11 - /// and 12.3.2, a database named `--no-data` with the terminator behind it dumped a *different* - /// database with no rows and exited 0, which the result sheet reports as a successful backup; - /// a table named `--ssl-mode=DISABLED` in the same slot sends the whole dump in cleartext. With - /// `--` in front of the database, both are names again. + // `--` before the database, because `my_getopt` does not stop parsing options at the first + // positional argument and every name after it is the user's. Measured with mysqldump 8.4.11 + // and 12.3.2, a database named `--no-data` with the terminator behind it dumped a *different* + // database with no rows and exited 0, which the result sheet reports as a successful backup; + // a table named `--ssl-mode=DISABLED` in the same slot sends the whole dump in cleartext. With + // `--` in front of the database, both are names again. private static func mysqlObjectArguments(_ request: NativeDumpDescriptor.Request) -> [String] { ["--", request.database] + request.scope.objects.map(\.name) } @@ -390,9 +418,9 @@ enum NativeDumpRegistry { // MARK: - MongoDB - /// `mongodump` reads no password from the environment, and a password in `argv` is readable by - /// every process on the machine. Its `--config` file is the only channel left, so the caller - /// writes one at mode `0600` and passes its path; `NativeDumpService` removes it afterwards. + // `mongodump` reads no password from the environment, and a password in `argv` is readable by + // every process on the machine. Its `--config` file is the only channel left, so the caller + // writes one at mode `0600` and passes its path; `NativeDumpService` removes it afterwards. private static var mongodb: NativeDumpDescriptor { NativeDumpDescriptor( mechanism: .commandLineTool( @@ -432,8 +460,8 @@ enum NativeDumpRegistry { // The tools check the name in the URI, 127.0.0.1 through a tunnel, and no option names another. Verify CA skips it. private static let mongoModesThroughTunnel: [SSLMode] = [.disabled, .preferred, .required, .verifyCa] - /// A whole-database dump names the database; a narrowed one names each collection, because - /// `--db` plus `--collection` takes only one collection at a time while `--nsInclude` repeats. + // A whole-database dump names the database; a narrowed one names each collection, because + // `--db` plus `--collection` takes only one collection at a time while `--nsInclude` repeats. private static func mongoNamespaceFlags(_ request: NativeDumpDescriptor.Request) -> [String] { guard !request.scope.isWholeDatabase else { return ["--db=\(request.database)"] } return request.scope.objects.map { object in @@ -458,13 +486,12 @@ enum NativeDumpRegistry { // MARK: - SQL Server - /// `sqlpackage` is Microsoft's own client-side export, and the only one that writes a whole - /// database to a local file. `bcp` moves one table at a time and produces no schema, so it - /// cannot stand in for a dump. - /// - /// Not verified against a live server here: the arguments come from Microsoft's documented CLI - /// surface rather than from a run, so a mistake in them surfaces as the tool's own usage error - /// rather than a wrong file. + // `sqlpackage` is Microsoft's own client-side export, and the only one that writes a whole + // database to a local file. `bcp` moves one table at a time and produces no schema, so it + // cannot stand in for a dump. + // Not verified against a live server here: the arguments come from Microsoft's documented CLI + // surface rather than from a run, so a mistake in them surfaces as the tool's own usage error + // rather than a wrong file. private static var sqlServer: NativeDumpDescriptor { NativeDumpDescriptor( mechanism: .commandLineTool( @@ -522,10 +549,10 @@ enum NativeDumpRegistry { return "\(schema).\(object.name)" } - /// `sqlpackage` takes no password flag and no password environment variable: the connection - /// string is the only channel it has. That puts the password in `argv`, where `ps` can read it, - /// so the string is built here and `NativeDumpService` keeps SQL Server out of the flows that - /// promise otherwise. Anything better needs Entra or integrated auth, which have no password. + // `sqlpackage` takes no password flag and no password environment variable: the connection + // string is the only channel it has. That puts the password in `argv`, where `ps` can read it, + // so the string is built here and `NativeDumpService` keeps SQL Server out of the flows that + // promise otherwise. Anything better needs Entra or integrated auth, which have no password. private static func sqlServerConnectionString(_ request: NativeDumpDescriptor.Request) throws -> String { let quoted = NativeDumpArgumentQuoting.sqlServerConnectionStringValue var parts = try [ @@ -559,12 +586,11 @@ enum NativeDumpRegistry { // MARK: - SQLite - /// The database is a file the tool opens directly, so there is no host, port or password, and - /// `.dump` writes SQL to standard output. - /// - /// The path comes from `localFilePath` rather than from `connection.database`. libSQL claims - /// this descriptor too and keeps its path in a plugin-declared additional field, so reading - /// `database` handed `sqlite3` an empty string, which exits 0 after writing a 52-byte file. + // The database is a file the tool opens directly, so there is no host, port or password, and + // `.dump` writes SQL to standard output. + // The path comes from `localFilePath` rather than from `connection.database`. libSQL claims + // this descriptor too and keeps its path in a plugin-declared additional field, so reading + // `database` handed `sqlite3` an empty string, which exits 0 after writing a 52-byte file. private static var sqlite: NativeDumpDescriptor { NativeDumpDescriptor( mechanism: .commandLineTool( diff --git a/apple/TablePro/Core/Database/NativeDumpService.swift b/apple/TablePro/Core/Database/NativeDumpService.swift index 33b825cf0a..6e8bee1bfa 100644 --- a/apple/TablePro/Core/Database/NativeDumpService.swift +++ b/apple/TablePro/Core/Database/NativeDumpService.swift @@ -14,25 +14,24 @@ import os import TableProConnectionLibrary import TableProPluginKit import TableProProcess +import TableProTLSClientIdentity import Yams // MARK: - Public Types -/// What the service is doing: dump (back up) a database or restore a dump file. enum NativeDumpKind: Equatable, Sendable { case backup case restore } -/// Observable state of a backup or restore. enum NativeDumpState: Equatable { case idle case running(database: String, fileURL: URL, bytesProcessed: Int64, totalBytes: Int64?) case cancelling case finished(database: String, fileURL: URL, bytesProcessed: Int64, skippedSettings: [String] = []) - /// A restore that fails part way through has already replayed some of the dump, and the - /// target is left in whatever state that reached. A backup writes only to its own file, which - /// is removed, so nothing of the user's is touched. + // A restore that fails part way through has already replayed some of the dump, and the + // target is left in whatever state that reached. A backup writes only to its own file, which + // is removed, so nothing of the user's is touched. case failed(message: String, targetMayBeModified: Bool) case cancelled } @@ -48,6 +47,7 @@ enum NativeDumpError: LocalizedError, Equatable { case unreachableServer(message: String) case nulInArgument case uncheckedThroughTunnel(tool: String, mode: SSLMode, supportedModes: [SSLMode]) + case systemTrustUnavailable(message: String) var errorDescription: String? { switch self { @@ -67,7 +67,7 @@ enum NativeDumpError: LocalizedError, Equatable { return String(localized: "The selected backup file is not readable.") case .engineStatementUnavailable: return String(localized: "TablePro could not read the database's own name from the connection.") - case .incompatibleTool(let message), .unreachableServer(let message): + case .incompatibleTool(let message), .unreachableServer(let message), .systemTrustUnavailable(let message): return message case .nulInArgument: return String(localized: "The database name or a sign-in value holds a NUL character, which a command line cannot carry.") @@ -98,29 +98,28 @@ internal enum NativeDumpExitPolicy: Equatable, Sendable { } } -/// Parameters for a single backup or restore subprocess. struct NativeDumpCommand: Equatable { let executable: URL let arguments: [String] let environment: [String: String] let stderrByteCap: Int - /// Where the tool's output goes. `pg_dump -Fc` is told a path and writes it itself, while - /// `mysqldump` and `sqlite3 .dump` write SQL to standard output for the caller to redirect. + // Where the tool's output goes. `pg_dump -Fc` is told a path and writes it itself, while + // `mysqldump` and `sqlite3 .dump` write SQL to standard output for the caller to redirect. let delivery: NativeDumpDescriptor.OutputDelivery - /// The file standard output is redirected to, or read from on a restore. Nil when the tool - /// handles the file itself. + // The file standard output is redirected to, or read from on a restore. Nil when the tool + // handles the file itself. let redirectedFileURL: URL? - /// A credentials file written for tools that read a password from neither the environment nor - /// standard input. Removed once the process exits, however it exits. + // A credentials file written for tools that read a password from neither the environment nor + // standard input. Removed once the process exits, however it exits. let temporaryCredentialsFileURL: URL? let clientPluginFolder: MariaDBClientPluginFolder? - /// Which direction a redirected file goes: a restore feeds it in as standard input, a backup - /// takes standard output and writes it out. + // Which direction a redirected file goes: a restore feeds it in as standard input, a backup + // takes standard output and writes it out. let isRestore: Bool internal let exitPolicy: NativeDumpExitPolicy @@ -150,11 +149,10 @@ struct NativeDumpCommand: Equatable { } } -/// Parameters for a dump the engine runs for itself. struct NativeDumpStatementJob: Sendable, Equatable { let statements: [String] - /// Run after a failure and ignored. A `DETACH` inside a transaction the engine already aborted - /// fails too, and the caller has the real error to report instead of this one. + // Run after a failure and ignored. A `DETACH` inside a transaction the engine already aborted + // fails too, and the caller has the real error to report instead of this one. let cleanupStatements: [String] let scope: DatabaseScope var isolatedImport: NativeDumpIsolatedImport? @@ -168,8 +166,8 @@ struct NativeDumpIsolatedImport: Sendable, Equatable { let file: URL } -/// What a run is, so the service can pick a runner without either runner accepting a shape it -/// cannot execute. +// What a run is, so the service can pick a runner without either runner accepting a shape it +// cannot execute. enum NativeDumpJob { case process(NativeDumpCommand) case statements(NativeDumpStatementJob) @@ -201,7 +199,6 @@ internal extension NativeDumpJob { } } -/// Captured terminal state of a finished/cancelled run. struct NativeDumpRunResult: Equatable { let exitCode: Int32 let stderr: String @@ -209,14 +206,12 @@ struct NativeDumpRunResult: Equatable { var targetUntouched = false } -/// Runs one backup or restore and reports how it ended. Abstracted so the state machine can be -/// tested without launching a process or reaching a database. +// Runs one backup or restore and reports how it ended. Abstracted so the state machine can be +// tested without launching a process or reaching a database. protocol NativeDumpRunner: AnyObject { - /// Begins the work. Throws synchronously if it cannot be started at all. func start() throws - /// Asks for cancellation. Safe to call more than once. + // Safe to call more than once. func cancel() - /// Resolves once the work has ended, however it ended. var result: NativeDumpRunResult { get async } // How far a restore has read its file, or nil when the runner cannot tell. var bytesRead: Int64? { get } @@ -244,6 +239,8 @@ final class NativeDumpService: ObservableObject { private var preparation: Task? // The Vault lease the tool signs in with, held until the tool exits. private var credentialLease: VaultLease? + // The roots file the tool checks the server with, removed when the tool exits. + private var systemTrustAnchors: SystemTrustAnchorFile? private let runningWork: RunningWorkRegistry private let reportCompletion: @MainActor (OperationCompletion) -> Void private var trackedRestore: TrackedRestore? @@ -273,13 +270,13 @@ final class NativeDumpService: ObservableObject { state = newState switch newState { case .finished: - releaseCredentialLease() + releaseRunResources() finishTrackedRestore(.succeeded(OperationSummary())) case .failed(let message, _): - releaseCredentialLease() + releaseRunResources() finishTrackedRestore(.failed(reason: message)) case .cancelled: - releaseCredentialLease() + releaseRunResources() finishTrackedRestore(.cancelled) case .idle, .running, .cancelling: break @@ -417,12 +414,12 @@ final class NativeDumpService: ObservableObject { effective.username = username } let password = signIn.password - releaseCredentialLease() + releaseRunResources() credentialLease = signIn.lease signIn.lease?.start { _ in } var jobStarted = false defer { - if !jobStarted { releaseCredentialLease() } + if !jobStarted { releaseRunResources() } } let localFilePath = Self.localFilePath(for: effective) let catalog = descriptor.engineStatements == nil @@ -453,6 +450,7 @@ final class NativeDumpService: ObservableObject { return } request = prepared + request.systemTrustAnchorsFile = try await holdSystemTrustAnchors(for: request, tool: tool) let command = try Self.buildCommand( kind: kind, tool: tool, @@ -480,9 +478,29 @@ final class NativeDumpService: ObservableObject { Self.logger.info("\(self.toolName, privacy: .public) started db=\(database, privacy: .private(mask: .hash))") } - private func releaseCredentialLease() { + private func releaseRunResources() { credentialLease?.release() credentialLease = nil + systemTrustAnchors?.remove() + systemTrustAnchors = nil + } + + // Off the main actor: the first build in a minute reads every trust setting, 150 to 275 ms measured. + private func holdSystemTrustAnchors( + for request: NativeDumpDescriptor.Request, + tool: NativeDumpDescriptor.CommandLineTool + ) async throws -> URL? { + let file = try await Task.detached { () throws -> SystemTrustAnchorFile? in + guard tool.needsSystemTrustAnchors(request) else { return nil } + do { + return try SystemTrustAnchors.shared.writeFile() + } catch { + throw NativeDumpError.systemTrustUnavailable(message: error.localizedDescription) + } + }.value + systemTrustAnchors?.remove() + systemTrustAnchors = file + return file?.url } // Nil when a cancel arrived while it ran: the checks it was making stop, and the run reads as cancelled rather than @@ -561,12 +579,11 @@ final class NativeDumpService: ObservableObject { } } - /// A cancel that arrives before the tool is running is latched rather than dropped. - /// - /// `start` finds the binary and asks it what it is before there is anything to cancel, and both - /// steps spawn a process, so the window is real: a Cancel taken during it used to be discarded - /// and the tool launched afterwards, which on a restore means writing to the target database - /// the user just said to leave alone. + // A cancel that arrives before the tool is running is latched rather than dropped. + // `start` finds the binary and asks it what it is before there is anything to cancel, and both + // steps spawn a process, so the window is real: a Cancel taken during it used to be discarded + // and the tool launched afterwards, which on a restore means writing to the target database + // the user just said to leave alone. func cancel() { cancelRequested = true preparation?.cancel() @@ -634,21 +651,18 @@ final class NativeDumpService: ObservableObject { return refusal.map { .incompatible($0) } ?? .missing } - /// The path a file-backed driver actually opens, read from wherever that driver keeps it. - /// SQLite uses `database`; DuckDB and libSQL use a plugin-declared additional field. + // SQLite uses `database`; DuckDB and libSQL use a plugin-declared additional field. static func localFilePath(for connection: DatabaseConnection) -> String? { guard let field = PluginManager.shared.localFilePathField(for: connection.type) else { return nil } let path = connection.localFilePath(in: field) return path.isEmpty ? nil : path } - /// The engine's own name for the database being dumped. - /// - /// `COPY FROM DATABASE` names a catalog, and DuckDB derives a catalog name from the file's - /// basename: measured, `my-weird.name.db` attaches as `my-weird`, and `COPY FROM DATABASE main` - /// fails with `Catalog "main" does not exist`. DuckDB also reaches more than one catalog at once - /// through `ATTACH`, so the answer is the picked database when the engine lists it, and only - /// otherwise the one the session is on. + // `COPY FROM DATABASE` names a catalog, and DuckDB derives a catalog name from the file's + // basename: measured, `my-weird.name.db` attaches as `my-weird`, and `COPY FROM DATABASE main` + // fails with `Catalog "main" does not exist`. DuckDB also reaches more than one catalog at once + // through `ATTACH`, so the answer is the picked database when the engine lists it, and only + // otherwise the one the session is on. static func resolvedCatalog(connectionId: UUID, database: String) async -> String? { guard let scope = DatabaseManager.shared.resolvedScope( database: database, schema: nil, for: connectionId @@ -664,9 +678,9 @@ final class NativeDumpService: ObservableObject { } } - /// Pure so the choice is testable without an engine. The picked name wins when the engine says - /// it exists, because a connection with two catalogs attached has to dump the one that was - /// ticked rather than whichever the session is parked on. + // Pure so the choice is testable without an engine. The picked name wins when the engine says + // it exists, because a connection with two catalogs attached has to dump the one that was + // ticked rather than whichever the session is parked on. nonisolated static func catalogName( picked: String, reported: [String], @@ -716,18 +730,18 @@ final class NativeDumpService: ObservableObject { ) } - /// Unique per run, so a `DETACH` that could not execute leaves nothing behind that blocks the - /// next attempt with "database with name already exists". + // Unique per run, so a `DETACH` that could not execute leaves nothing behind that blocks the + // next attempt with "database with name already exists". static func backupAlias() -> String { "tablepro_backup_\(UUID().uuidString.replacingOccurrences(of: "-", with: "").prefix(12))" } // MARK: - Command Construction - /// The descriptor supplies the arguments and the environment; this adds the parts every tool - /// shares. A password never joins the argument list, because `argv` is world readable through - /// `ps`: it goes in the environment where the tool reads one, and in a `0600` file where it - /// does not. + // The descriptor supplies the arguments and the environment; this adds the parts every tool + // shares. A password never joins the argument list, because `argv` is world readable through + // `ps`: it goes in the environment where the tool reads one, and in a `0600` file where it + // does not. nonisolated static func buildCommand( kind: NativeDumpKind, tool: NativeDumpDescriptor.CommandLineTool, @@ -773,9 +787,9 @@ final class NativeDumpService: ObservableObject { ) } - /// `mongodump` and `mongorestore` read a password from neither the environment nor standard - /// input, and one in `argv` is readable by every process on the machine. Their `--config` file - /// is the remaining channel, so it is written owner-only and removed when the process exits. + // `mongodump` and `mongorestore` read a password from neither the environment nor standard + // input, and one in `argv` is readable by every process on the machine. Their `--config` file + // is the remaining channel, so it is written owner-only and removed when the process exits. nonisolated static func writeMongoConfigurationFile( _ entries: [NativeDumpDescriptor.ConfigurationEntry] ) throws -> URL { @@ -799,10 +813,10 @@ final class NativeDumpService: ObservableObject { return try Yams.serialize(node: .mapping(Node.Mapping(pairs)), width: -1, allowUnicode: true) } - /// The locale variables a spawned tool reads, named rather than spelled at each use site. The - /// preference-key guard scans this target's sources for a quoted string passed to a `forKey:` - /// label and reports anything outside the `com.TablePro` namespace, so a locale variable - /// written that way reads as a stray `UserDefaults` key. Comments are scanned too. + // The locale variables a spawned tool reads, named rather than spelled at each use site. The + // preference-key guard scans this target's sources for a quoted string passed to a `forKey:` + // label and reports anything outside the `com.TablePro` namespace, so a locale variable + // written that way reads as a stray `UserDefaults` key. Comments are scanned too. nonisolated internal enum LocaleEnvironmentKey { static let everyCategory = "LC_ALL" static let characterHandling = "LC_CTYPE" @@ -875,9 +889,9 @@ final class NativeDumpService: ObservableObject { Self.logger.error("\(self.toolName, privacy: .public) failed code=\(result.exitCode) db=\(database, privacy: .private(mask: .hash)) stderr=\(result.stderr)") } - /// A DuckDB Parquet backup is a folder, so its size is the sum of what is inside it and its - /// removal is recursive. `removeItem` already handles a directory; `attributesOfItem` does not - /// sum one. + // A DuckDB Parquet backup is a folder, so its size is the sum of what is inside it and its + // removal is recursive. `removeItem` already handles a directory; `attributesOfItem` does not + // sum one. nonisolated static func sizeOfDestination(_ url: URL) -> Int64 { let manager = FileManager.default var isDirectory: ObjCBool = false @@ -937,13 +951,12 @@ final class NativeDumpService: ObservableObject { // MARK: - Database Size Helper extension NativeDumpService { - /// Best-effort estimate of the database's on-disk size. Used as an upper bound for the backup - /// progress bar; the dump file is typically much smaller because of compression, so the bar - /// tops out at the size and then jumps when the tool exits. Returns nil if the query fails. - /// - /// Routed through `withMetadataDriver` rather than taken straight off `driver(for:)`. On the - /// session driver this query queues behind whatever the user is running and joins whatever - /// transaction a query tab left open, which is the hazard `metadataRoute` exists to avoid. + // Best-effort estimate of the database's on-disk size. Used as an upper bound for the backup + // progress bar; the dump file is typically much smaller because of compression, so the bar + // tops out at the size and then jumps when the tool exits. + // Routed through `withMetadataDriver` rather than taken straight off `driver(for:)`. On the + // session driver this query queues behind whatever the user is running and joins whatever + // transaction a query tab left open, which is the hazard `metadataRoute` exists to avoid. static func estimatedDatabaseSize( connection: DatabaseConnection, database: String @@ -959,8 +972,8 @@ extension NativeDumpService { return Int64(text) } - /// An engine with no cheap size answer returns nil, which leaves the progress bar - /// indeterminate rather than showing a percentage of a number nobody measured. + // An engine with no cheap size answer returns nil, which leaves the progress bar + // indeterminate rather than showing a percentage of a number nobody measured. nonisolated static func sizeQuery(for type: DatabaseType) -> String? { switch type { case .postgresql, .redshift: diff --git a/apple/TablePro/Core/Database/SessionLanes.swift b/apple/TablePro/Core/Database/SessionLanes.swift index ebcd95ee09..b1c291d62b 100644 --- a/apple/TablePro/Core/Database/SessionLanes.swift +++ b/apple/TablePro/Core/Database/SessionLanes.swift @@ -5,6 +5,18 @@ import Foundation import os +import TableProSQLGrammar + +// What a connection replaced or closed without the user asking took with it: an open transaction, what the user's +// own statements set up, or both. +internal struct SessionLoss: Equatable, Sendable { + var transaction: Bool + var state: SQLSessionStateKinds + + func merging(_ other: SessionLoss) -> SessionLoss { + SessionLoss(transaction: transaction || other.transaction, state: state.union(other.state)) + } +} /// The user connections a session keeps open on databases other than the one it is browsing. /// @@ -33,9 +45,9 @@ internal final class SessionLanes { /// Drivers being checked before use. A turn held on one of these is not a query proving the /// driver answers, so a switch back waits for the check instead of promoting it on trust. private var verifying: [ObjectIdentifier: [CheckedContinuation]] = [:] - /// Databases whose connection died holding a transaction and was replaced. The next turn there - /// is refused once, so work meant to continue that transaction does not carry on without it. - private var lostTransactions: [UUID: Set] = [:] + // Databases whose connection died holding a transaction or session state and was replaced. The next turn there is + // refused once, so work meant for that session does not carry on without it. + private var losses: [UUID: [String: SessionLoss]] = [:] private struct Opened { weak var driver: AnyObject? @@ -46,8 +58,8 @@ internal final class SessionLanes { /// switched away from, nothing on screen closes one, so it is closed once idle. Browsing its /// database takes it off this list. private var opened: [UUID: [String: Opened]] = [:] - /// Databases whose opened connection was closed for being idle. Work queued for one reopens it: - /// no transaction was lost. The session's temp tables, `SET` values and advisory locks were. + // Databases whose opened connection was closed for being idle. Work queued for one reopens it. One holding a + // transaction or session state the user's statements set up is never closed idle. private var closedIdle: [UUID: Set] = [:] internal static let openedIdleTimeout: TimeInterval = MetadataConnectionPool.idleTimeout @@ -59,6 +71,9 @@ internal final class SessionLanes { /// Told after a parked connection closes, which ends any transaction it held. internal var onClose: (@MainActor (UUID) -> Void)? + // What the user's statements set up on a driver, from the ledger the session owns. + internal var recordedState: (@MainActor (DatabaseDriver, UUID) -> SQLSessionStateKinds)? + internal init(opener: Opener? = nil) { self.opener = opener ?? { scope in try await MetadataConnectionPool.openServerDriver(for: scope, purpose: .session) @@ -122,13 +137,13 @@ internal final class SessionLanes { /// Closes every parked connection of a session. A disconnect ends them, and so does a rebuilt /// transport: they were dialed through the tunnel port that is going away. Opens still in flight /// lose their generation, so one that lands afterwards closes its own driver. - internal func closeAll(for connectionId: UUID, keepingLostTransactions: Bool = false) { + internal func closeAll(for connectionId: UUID, keepingLosses: Bool = false) { generations[connectionId, default: 0] &+= 1 let drivers = parked.removeValue(forKey: connectionId) ?? [:] opened.removeValue(forKey: connectionId) closedIdle.removeValue(forKey: connectionId) - if !keepingLostTransactions { - lostTransactions.removeValue(forKey: connectionId) + if !keepingLosses { + losses.removeValue(forKey: connectionId) } for driver in drivers.values { verifiedAt.removeValue(forKey: ObjectIdentifier(driver)) @@ -196,35 +211,54 @@ internal final class SessionLanes { } } - internal func markTransactionLost(database: String, for connectionId: UUID) { - lostTransactions[connectionId, default: []].insert(database) + internal func markLost( + database: String, + transaction: Bool = false, + state: SQLSessionStateKinds = [], + for connectionId: UUID + ) { + guard transaction || !state.isEmpty else { return } + let loss = SessionLoss(transaction: transaction, state: state) + losses[connectionId, default: [:]][database] = losses[connectionId]?[database]?.merging(loss) ?? loss + Self.logger.info( + "noted a lost session connId=\(connectionId, privacy: .public) transaction=\(transaction) kinds=\(state.rawValue)" + ) } - internal func hasTransactionLoss(database: String, for connectionId: UUID) -> Bool { - lostTransactions[connectionId]?.contains(database) ?? false + internal func hasLoss(database: String, for connectionId: UUID) -> Bool { + losses[connectionId]?[database] != nil } - /// Closes every parked connection when the transport under them is rebuilt, first noting each - /// one that was holding a transaction, so the next work on that database is told the server - /// rolled it back instead of quietly starting over on a new connection. - internal func closeAllNotingLostTransactions(for connectionId: UUID) async { + // Closes every parked connection when the transport under them is rebuilt, first noting each one that held a + // transaction or session state, so the next work on that database is told what the server dropped instead of + // quietly starting over on a new connection. + internal func closeAllNotingLosses(for connectionId: UUID) async { for (database, driver) in parked[connectionId] ?? [:] { let held = await driver.heldSessionTransactionState() - if !held.permitsAppTransaction { - lostTransactions[connectionId, default: []].insert(database) - } + markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: recordedState?(driver, connectionId) ?? [], + for: connectionId + ) } - closeAll(for: connectionId, keepingLostTransactions: true) + closeAll(for: connectionId, keepingLosses: true) } - /// Reports a lost transaction once, clearing it. - internal func takeTransactionLoss(database: String, for connectionId: UUID) -> Bool { - lostTransactions[connectionId]?.remove(database) != nil + // Reported once, whole, to the first turn that owns session state, as a lost transaction always was. + internal func takeLoss(database: String, for connectionId: UUID) -> SessionLoss? { + let loss = losses[connectionId]?.removeValue(forKey: database) + if losses[connectionId]?.isEmpty == true { + losses.removeValue(forKey: connectionId) + } + return loss } - // A connection with one driver for every database lost the transaction whichever name it was noted under. - internal func takeAnyTransactionLoss(for connectionId: UUID) -> String? { - lostTransactions.removeValue(forKey: connectionId)?.min() + // A connection with one driver for every database lost the session whichever name it was noted under. + internal func takeAnyLoss(for connectionId: UUID) -> (database: String, loss: SessionLoss)? { + guard let noted = losses.removeValue(forKey: connectionId), let database = noted.keys.min() else { return nil } + let loss = noted.values.reduce(SessionLoss(transaction: false, state: [])) { $0.merging($1) } + return (database: database, loss: loss) } internal func markOpened(_ driver: DatabaseDriver, database: String, for connectionId: UUID, at date: Date = Date()) { diff --git a/apple/TablePro/Core/Database/SessionStateLedger.swift b/apple/TablePro/Core/Database/SessionStateLedger.swift new file mode 100644 index 0000000000..f248ee458c --- /dev/null +++ b/apple/TablePro/Core/Database/SessionStateLedger.swift @@ -0,0 +1,171 @@ +// +// SessionStateLedger.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar + +// What the user's own statements set up on each session connection, as kinds only: a value a `SET` carried is kept +// nowhere. A reconnect reads it for the driver it is about to close, since the server cannot be asked once it is gone. +@MainActor +internal final class SessionStateLedger { + private struct Entry { + // Weak, and compared by identity: a closed driver that was freed hands its address to the next one allocated. + weak var driver: AnyObject? + var state: SQLSessionStateKinds + } + + private var entries: [SessionTransactionTracker.Key: Entry] = [:] + private let isLive: @MainActor (SessionTransactionTracker.Key, DatabaseDriver) -> Bool + + internal init(isLive: @escaping @MainActor (SessionTransactionTracker.Key, DatabaseDriver) -> Bool) { + self.isLive = isLive + } + + internal func record(_ effect: SQLSessionStateEffect, on driver: DatabaseDriver, for key: SessionTransactionTracker.Key) { + let current = entries[key].flatMap { $0.driver === driver ? $0.state : nil } ?? [] + // A turn can outlive its session's teardown; its late answer must not land on the replacement. + guard isLive(key, driver) else { + if entries[key]?.driver === driver { entries.removeValue(forKey: key) } + return + } + let state = effect.applied(to: current) + entries[key] = state.isEmpty ? nil : Entry(driver: driver, state: state) + } + + internal func state(for key: SessionTransactionTracker.Key, heldBy driver: DatabaseDriver) -> SQLSessionStateKinds { + guard let entry = entries[key], entry.driver === driver else { return [] } + return entry.state + } + + internal func state(heldBy driver: DatabaseDriver, for connectionId: UUID) -> SQLSessionStateKinds { + entries.reduce(into: SQLSessionStateKinds()) { state, item in + if item.key.connectionId == connectionId, item.value.driver === driver { + state.formUnion(item.value.state) + } + } + } + + // Drops what a closed, replaced or disconnected driver held, as the transaction tracker drops its holds. + internal func prune(_ connectionId: UUID) { + for (key, entry) in entries where key.connectionId == connectionId { + if let driver = entry.driver as? DatabaseDriver, isLive(key, driver) { continue } + entries.removeValue(forKey: key) + } + } +} + +private struct SessionStateReading: Sendable { + let key: SessionTransactionTracker.Key + let dialect: SQLSessionDialect + let grammar: SQLLexicalGrammar + let startup: [String] +} + +extension DatabaseManager { + // Called inside the user's own lease, where the driver is the one the text ran on. A statement that failed counts: + // it can have created a temporary table before its error. + internal func noteSessionStatements(_ statements: [String], ranOn driver: DatabaseDriver, scope: DatabaseScope) async { + guard !statements.isEmpty, + let session = activeSessions[scope.connectionId], + let dialect = SQLSessionDialect(TransactionEngineFamily.of(session.connection.type)), + let key = sessionKey(for: scope), + isLiveSessionDriver(driver, for: key) + else { return } + let databaseType = session.connection.type + let grammar = SQLLexicalResolver.readings(for: databaseType, session: driver.sessionLexicalState).execution + // A reconnect runs the startup commands again, so what they set up is never lost. + let startup = Self.startupStatements( + from: session.connection.startupCommands, databaseType: databaseType, grammar: grammar + ) + let reading = SessionStateReading(key: key, dialect: dialect, grammar: grammar, startup: startup) + let effect = await Self.sessionStateEffect(of: statements, reading: reading) + guard !effect.isEmpty else { return } + sessionStateLedger.record(effect, on: driver, for: key) + } + + // A Run All of a long script reads every statement, which would hold the main actor. + @concurrent + nonisolated private static func sessionStateEffect( + of statements: [String], + reading: SessionStateReading + ) async -> SQLSessionStateEffect { + SQLSessionStateClassifier.effect( + ofStatements: statements, dialect: reading.dialect, grammar: reading.grammar, skipping: reading.startup + ) + } + + // Kinds only, never a statement, a name or a value: a `SET` can carry a secret. + static func sessionLostMessage(database: String, loss: SessionLoss) -> String { + guard !loss.state.isEmpty else { return parkedTransactionLostMessage(database: database) } + let kinds = ListFormatter.localizedString(byJoining: loss.state.localizedNames) + guard loss.transaction else { + return String( + format: String( + // swiftlint:disable:next line_length + localized: "The connection to %1$@ was lost while it waited and has been reconnected. The new session does not have what the old one set up: %2$@. Set it up again, then run the statements." + ), + database, kinds + ) + } + return String( + format: String( + // swiftlint:disable:next line_length + localized: "The connection to %1$@ was lost while it waited, and the server rolled back its open transaction. The new session does not have what the old one set up either: %2$@. Set it up again, then run the statements." + ), + database, kinds + ) + } +} + +extension SQLSessionDialect { + // Every engine left out records nothing, as before the ledger existed. + init?(_ family: TransactionEngineFamily) { + switch family { + case .postgres, .redshift, .cockroach: + self = .postgres + case .mysql: + self = .mySQL + case .sqlServer: + self = .sqlServer + case .oracle: + self = .oracle + case .sqlite: + self = .sqlite + case .duckdb: + self = .duckDB + case .redis, .other: + return nil + } + } +} + +extension SQLSessionStateKinds { + // In bit order, so a message names the kinds the same way every time. + var localizedNames: [String] { + var names: [String] = [] + if contains(.temporaryObjects) { names.append(String(localized: "temporary tables")) } + if contains(.sessionSettings) { names.append(String(localized: "session settings")) } + if contains(.userVariables) { names.append(String(localized: "user variables")) } + if contains(.preparedStatements) { names.append(String(localized: "prepared statements")) } + if contains(.heldCursors) { names.append(String(localized: "cursors")) } + if contains(.sessionLocks) { names.append(String(localized: "session locks")) } + if contains(.listeners) { names.append(String(localized: "LISTEN registrations")) } + return names + } +} + +extension BatchStatementOutcome { + // The statement that failed ran too, as `CatalogEvent.statementsRan` counts it. A Stop ran only what it returned. + func ranCount(of totalCount: Int) -> Int { + switch self { + case .completed: + return totalCount + case .cancelled(let results): + return results.count + case .failed(let results, let failure, _): + return failure.ranStatementCount(executedCount: results.count, totalCount: totalCount) + } + } +} diff --git a/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift b/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift index 549259fa11..ac55d9e63a 100644 --- a/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift +++ b/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift @@ -9,23 +9,16 @@ import SwiftUI import TableProAutocomplete import TableProPluginKit -/// Everything a `DatabaseType` knows by asking the plugin registry. -/// -/// It lives apart from the type itself because these are the members that make naming a database -/// type cost the whole plugin system at compile time. `DatabaseType` is a string a connection -/// carries and a value the display layer compares against a constant; keeping the lookups here -/// means a file that only does that does not compile against the registry, the snapshot table and -/// the theme engine behind them. +// Apart from `DatabaseType`, so a file that only names or compares a type does not compile against the registry, the +// snapshot table and the theme engine behind them. extension DatabaseType { - /// All registered database types, derived dynamically from the plugin metadata registry. static var allKnownTypes: [DatabaseType] { PluginMetadataRegistry.shared.allRegisteredTypeIds().map { DatabaseType(rawValue: $0) } } } extension DatabaseType { - /// Returns nil if rawValue doesn't match any registered type. init?(validating rawValue: String) { guard PluginMetadataRegistry.shared.hasType(rawValue) else { return nil } self.rawValue = rawValue @@ -33,14 +26,11 @@ extension DatabaseType { } extension DatabaseType { - /// Plugin type ID used for PluginManager lookup, resolved via the registry. var pluginTypeId: String { PluginMetadataRegistry.shared.pluginTypeId(for: rawValue) } - /// Genuinely a fact about the plugin binary rather than the database, so it asks by - /// `pluginTypeId`: Redshift is served by the bundled PostgreSQL plugin and there is nothing - /// of its own to download. + // Asks by `pluginTypeId`: Redshift is served by the bundled PostgreSQL plugin and has nothing of its own to download. var isDownloadablePlugin: Bool { PluginMetadataRegistry.shared.snapshot(forRegisteredTypeId: pluginTypeId)?.isDownloadable ?? false } @@ -49,8 +39,7 @@ extension DatabaseType { PluginMetadataRegistry.shared.snapshot(for: self)?.iconName ?? "cylinder" } - /// Returns the correct SwiftUI Image for this database type, handling both - /// SF Symbol names (e.g. "cylinder.fill") and asset catalog names (e.g. "mysql-icon"). + // `iconName` is an SF Symbol (`cylinder.fill`) or an asset catalog name (`mysql-icon`). var iconImage: Image { let name = iconName if NSImage(systemSymbolName: name, accessibilityDescription: nil) != nil { @@ -88,6 +77,10 @@ extension DatabaseType { PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.verifiesServerWithSystemTrust ?? false } + var requestedTLSVerifiesIdentity: Bool { + PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.requestedTLSVerifiesIdentity ?? false + } + var supportsPerConnectionCertificatePaths: Bool { PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.supportsPerConnectionCertificatePaths ?? true } @@ -98,7 +91,7 @@ extension DatabaseType { } var sslModeWhenTLSEnabled: SSLMode { - verifiesServerWithSystemTrust ? .verifyIdentity : .required + requestedTLSVerifiesIdentity ? .verifyIdentity : .required } func impliedSSLMode(forPort port: Int) -> SSLMode? { @@ -261,8 +254,8 @@ extension DatabaseType { .capabilities.supportsUserDefinedTypeBrowse ?? false } - /// The object kinds this engine fetches. The sidebar waits on these fetches before calling a - /// container empty, and keeps a section for one that failed; it lists a section only for rows. + // The object kinds this engine fetches. The sidebar waits on these fetches before calling a + // container empty, and keeps a section for one that failed; it lists a section only for rows. var declaredObjectKinds: Set { var kinds: Set = [] if supportsRoutines { diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift index 298a4c1a62..410a02df9b 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift @@ -8,17 +8,16 @@ import TableProPluginKit import TableProSQLVocabulary import TableProUtilities -/// What the app knows about a database type before its plugin loads. -/// -/// The primary type ids here are overwritten by `buildMetadataSnapshot` the moment the plugin -/// registers, so these are the pre-load answer for those, apart from an explain list the plugin -/// leaves empty, which keeps the one here. For a variant id they are the whole -/// answer: `registerVariant` keeps the curated entry and ignores the plugin's own statics, which -/// is the only reason MariaDB, TiDB, Databend, OceanBase, Redshift, CockroachDB and PGlite can differ from -/// the plugin that drives them. +// What the app knows about a database type before its plugin loads. +// The primary type ids here are overwritten by `buildMetadataSnapshot` the moment the plugin +// registers, so these are the pre-load answer for those, apart from an explain list the plugin +// leaves empty, which keeps the one here. For a variant id they are the whole +// answer: `registerVariant` keeps the curated entry and ignores the plugin's own statics, which +// is the only reason MariaDB, TiDB, Databend, OceanBase, Redshift, CockroachDB and PGlite can differ from +// the plugin that drives them. extension PluginMetadataRegistry { - /// SQLite's own extensions. libSQL and Turso name a new local file with them too, since the file - /// their driver creates is a SQLite database. + // SQLite's own extensions. libSQL and Turso name a new local file with them too, since the file + // their driver creates is a SQLite database. static let sqliteFileExtensions = ["db", "db3", "s3db", "sl3", "sqlite", "sqlite3", "sqlitedb"] // swiftlint:disable:next function_body_length @@ -192,8 +191,8 @@ extension PluginMetadataRegistry { ) ] - /// Cloud SQL signs IAM principals in to MySQL and PostgreSQL only, so SQL Server and the - /// MySQL and PostgreSQL forks keep the AWS-only picker. + // Cloud SQL signs IAM principals in to MySQL and PostgreSQL only, so SQL Server and the + // MySQL and PostgreSQL forks keep the AWS-only picker. let cloudIAMFields = GoogleCloudSQLAuthFields.standardWithAWS() + [AWSAuthFields.rdsEndpointField()] let postgresqlAuthFields = KerberosAuthFields.standardWithAWSAndGoogleCloud() + [ @@ -201,8 +200,8 @@ extension PluginMetadataRegistry { AWSAuthFields.rdsEndpointField() ] - /// The MySQL plugin's types only. PostgreSQL shares the IAM fields and must not pick this up: - /// its driver holds no releasable resource, so the setting would do nothing. + // The MySQL plugin's types only. PostgreSQL shares the IAM fields and must not pick this up: + // its driver holds no releasable resource, so the setting would do nothing. let mysqlIdleReleaseField = ConnectionField( id: "mysqlIdleReleaseMinutes", label: String(localized: "Release the Server Connection After (minutes, 0 to keep it)"), @@ -409,6 +408,7 @@ extension PluginMetadataRegistry { supportsDatabaseTriggerBrowse: true, supportsUserDefinedTypeBrowse: true, defaultSSLMode: .preferred, + verifiesServerWithSystemTrust: true, readOnlyEnforcement: .sessionDefault ), schema: PluginMetadataSnapshot.SchemaInfo( @@ -470,7 +470,8 @@ extension PluginMetadataRegistry { supportsDropSchema: true, supportsCreateSchema: true, supportsSchemaOwner: true, - defaultSSLMode: .preferred + defaultSSLMode: .preferred, + verifiesServerWithSystemTrust: true ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "public", @@ -548,6 +549,7 @@ extension PluginMetadataRegistry { supportsCheckConstraintEditing: false, supportsGeneratedColumns: true, defaultSSLMode: .preferred, + verifiesServerWithSystemTrust: true, readOnlyEnforcement: .sessionDefault ), schema: PluginMetadataSnapshot.SchemaInfo( diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+DuckDBDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+DuckDBDefaults.swift index d8106a1092..6e29479517 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+DuckDBDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+DuckDBDefaults.swift @@ -8,7 +8,6 @@ import TableProPluginKit import TableProUtilities extension PluginMetadataRegistry { - /// The curated snapshot for DuckDB, alongside its connection fields in the sibling file. func duckdbPluginDefaults( dialect: SQLDialectDescriptor, columnTypes: [String: [String]] @@ -47,6 +46,7 @@ extension PluginMetadataRegistry { // httpfs checks a server against the system CA bundle when no CA is given, so a URL asking // for TLS with no mode means a verified connection, never Required's unchecked one. verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, supportsConnectionPooling: false, localFilePathField: .additionalField("duckdbFilePath"), serverEndpointFields: ServerEndpointFields(hostFieldId: "duckdbHost", portFieldId: "duckdbPort"), @@ -64,16 +64,15 @@ extension PluginMetadataRegistry { systemDatabaseNames: ["system", "temp"], systemSchemaNames: [], fileExtensions: ["duckdb", "ddb", "parquet", "csv", "tsv", "json", "ndjson"], - /// Only DuckDB's own storage. The data formats above are recognised by name - /// alone, because `duckdb_open` picks their reader from the extension and - /// refuses a Parquet file called anything else. - /// - /// `DUCK` sits behind the header's eight-byte checksum, followed by the storage - /// version as a little-endian `uint64`. Four bytes on their own are not enough - /// to name a format, and `SELECT 'DUCK';` spells them at exactly that offset, - /// so the version's high six bytes have to be zero as well. Storage versions - /// are still in the sixties, and a version past 65535 would cost recognition by - /// content rather than break it. + // Only DuckDB's own storage. The data formats above are recognised by name + // alone, because `duckdb_open` picks their reader from the extension and + // refuses a Parquet file called anything else. + // `DUCK` sits behind the header's eight-byte checksum, followed by the storage + // version as a little-endian `uint64`. Four bytes on their own are not enough + // to name a format, and `SELECT 'DUCK';` spells them at exactly that offset, + // so the version's high six bytes have to be zero as well. Storage versions + // are still in the sixties, and a version past 65535 would cost recognition by + // content rather than break it. fileSignatures: [.magic("DUCK", at: 8).andZeroes(at: 14, count: 6)], databaseGroupingStrategy: .bySchema, structureColumnFields: [.name, .type, .nullable, .defaultValue, .autoIncrement, .comment] diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+HanaDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+HanaDefaults.swift index 16e6804ba0..4bbd5fd4e0 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+HanaDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+HanaDefaults.swift @@ -41,7 +41,8 @@ extension PluginMetadataRegistry { defaultSSLMode: .verifyIdentity, supportsOpportunisticTLS: false, tlsImpliedPorts: [443], - verifiesServerWithSystemTrust: true + verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "", diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift index 2905785999..aded48a792 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift @@ -7,13 +7,11 @@ import Foundation import TableProPluginKit extension PluginMetadataRegistry { - /// The curated snapshot for Kafka. - /// - /// A registry-only driver is not installed until someone connects with it, so without an - /// entry here the type is absent from the New Connection picker and there is no way to - /// create the connection that would trigger the install. The connection fields are - /// duplicated from `KafkaConnectionField` for the same reason: the plugin that owns them - /// is not loaded yet. + // A registry-only driver is not installed until someone connects with it, so without an + // entry here the type is absent from the New Connection picker and there is no way to + // create the connection that would trigger the install. The connection fields are + // duplicated from `KafkaConnectionField` for the same reason: the plugin that owns them + // is not loaded yet. func kafkaPluginDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { [ ("Kafka", PluginMetadataSnapshot( @@ -43,6 +41,7 @@ extension PluginMetadataRegistry { defaultSSLMode: .disabled, supportsOpportunisticTLS: false, verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, supportsColumnSort: false, supportsRawFilter: false, statementPolicyInDriver: true @@ -92,8 +91,8 @@ private let kafkaColumnTypes: [String: [String]] = [ "Coordinates": ["INTEGER", "BIGINT", "TIMESTAMP"] ] -/// Kept identical to `KafkaConnectionField.fields()` in the plugin. The two cannot share code: -/// this list has to exist before the bundle that owns it is downloaded. +// Kept identical to `KafkaConnectionField.fields()` in the plugin. The two cannot share code: +// this list has to exist before the bundle that owns it is downloaded. private func kafkaConnectionFields() -> [ConnectionField] { [ ConnectionField( diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift index 95c510298b..6286e8fee2 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift @@ -33,6 +33,7 @@ extension PluginMetadataRegistry { supportsModifyPrimaryKey: false, supportsOpportunisticTLS: false, verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, columnsAreSampled: true, supportsPrincipalConnectionLimit: false ), diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift index 6adb6fe2d9..dd3765b193 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift @@ -7,9 +7,9 @@ import Foundation import TableProPluginKit extension PluginMetadataRegistry { - /// SQL Server refuses `=` on these (Msg 402 for `ntext`, `text`, `image` and `xml`, 403 for the spatial types), so - /// a keyless match compares them through a cast, which is exact for each. A `hierarchyid` reads as its bytes, which - /// the cast to `varbinary(max)` compares without an implicit conversion back to the type. + // SQL Server refuses `=` on these (Msg 402 for `ntext`, `text`, `image` and `xml`, 403 for the spatial types), so + // a keyless match compares them through a cast, which is exact for each. A `hierarchyid` reads as its bytes, which + // the cast to `varbinary(max)` compares without an implicit conversion back to the type. static let mssqlRowMatchTextTypePrefixes = ["NTEXT", "TEXT", "XML", "IMAGE", "GEOGRAPHY", "GEOMETRY", "HIERARCHYID"] // Measured on ClickHouse 25.8: a JSON value or a NaN never equals the text it was read as, and concat() of it does. @@ -17,13 +17,13 @@ extension PluginMetadataRegistry { "JSON", "OBJECT(", "DYNAMIC", "FLOAT", "BFLOAT16", "NULLABLE(FLOAT", "NULLABLE(BFLOAT16", ] - /// `sql_variant` refuses `=` too (Msg 206), and no cast keeps its base type: the integer `1` and the string `N'1'` - /// cast to the same text, so a keyless match on it could pick either row. + // `sql_variant` refuses `=` too (Msg 206), and no cast keeps its base type: the integer `1` and the string `N'1'` + // cast to the same text, so a keyless match on it could pick either row. static let mssqlRowMatchExcludedTypePrefixes = ["SQL_VARIANT"] - /// Oracle cannot compare a LOB, `LONG`, `LONG RAW`, `BFILE`, `XMLTYPE`, `JSON`, `VECTOR` or spatial value with `=` - /// (ORA-22848 on a CLOB), so a keyless match leaves them out. Matched against the bare type name, which an - /// owner-qualified column such as `"MDSYS"."SDO_GEOMETRY"` classifies by. + // Oracle cannot compare a LOB, `LONG`, `LONG RAW`, `BFILE`, `XMLTYPE`, `JSON`, `VECTOR` or spatial value with `=` + // (ORA-22848 on a CLOB), so a keyless match leaves them out. Matched against the bare type name, which an + // owner-qualified column such as `"MDSYS"."SDO_GEOMETRY"` classifies by. static let oracleRowMatchExcludedTypePrefixes = [ "CLOB", "NCLOB", "BLOB", "LONG", "BFILE", "XMLTYPE", "JSON", "VECTOR", "SDO_" ] @@ -602,6 +602,7 @@ extension PluginMetadataRegistry { supportsOpportunisticTLS: false, tlsImpliedPorts: [8_443, 443], verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, readOnlyEnforcement: .engineEnforced ), schema: PluginMetadataSnapshot.SchemaInfo( diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TrinoDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TrinoDefaults.swift index 974471c1f2..d147b1d00d 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TrinoDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TrinoDefaults.swift @@ -37,7 +37,8 @@ extension PluginMetadataRegistry { defaultSSLMode: .disabled, supportsOpportunisticTLS: false, tlsImpliedPorts: [443], - verifiesServerWithSystemTrust: true + verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true ), schema: Self.trinoSchemaInfo, editor: PluginMetadataSnapshot.EditorConfig( @@ -88,6 +89,7 @@ extension PluginMetadataRegistry { supportsOpportunisticTLS: false, tlsImpliedPorts: [443], verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, // OFFSET is off by default (offset_clause_enabled), and the grammar takes neither // FETCH NEXT nor LIMIT before OFFSET. pagination: .leadingRowsOnly(maximumRows: nil) diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift index c50ece4e4d..196da8f824 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift @@ -73,93 +73,96 @@ struct PluginMetadataSnapshot: Sendable { var defaultSSLMode: SSLMode = .disabled var supportsOpportunisticTLS: Bool = true var tlsImpliedPorts: [Int] = [] + // Verify Identity with no CA file checks the server against the system trust store. var verifiesServerWithSystemTrust: Bool = false + // A URL or port that asks for TLS without naming a mode gets Verify Identity rather than Required. Kept apart from + // `verifiesServerWithSystemTrust`: a libpq server on a private CA that connects with `ssl=true` today would stop. + var requestedTLSVerifiesIdentity: Bool = false var supportsPerConnectionCertificatePaths: Bool = true var supportsClientKeyPassphrase: Bool = false var supportsConnectionPooling: Bool = true - /// Whether two pooled drivers for the same connection sit on one physical session. Snowflake - /// keys its session on the account and role and deliberately not on the database, so every - /// pooled scope shares one mutable `currentDatabase`: a statement that selects a database on - /// behalf of one entry selects it for all of them. + // Whether two pooled drivers for the same connection sit on one physical session. Snowflake + // keys its session on the account and role and deliberately not on the database, so every + // pooled scope shares one mutable `currentDatabase`: a statement that selects a database on + // behalf of one entry selects it for all of them. var pooledDriversShareOneSession: Bool = false var authenticationIsDatabaseScoped: Bool = false - /// Whether a connection that names no database has nothing to browse until one is chosen. A - /// MySQL session opened without one has no current database: `SHOW TABLES` there is - /// `ERROR 1046 No database selected`. An engine whose session falls back to a default - /// database, such as ClickHouse's `default`, leaves this false. + // Whether a connection that names no database has nothing to browse until one is chosen. A + // MySQL session opened without one has no current database: `SHOW TABLES` there is + // `ERROR 1046 No database selected`. An engine whose session falls back to a default + // database, such as ClickHouse's `default`, leaves this false. var browsingRequiresSelectedDatabase: Bool = false var pagination: PaginationCapability = .offset - /// Whether an exact count reads the whole table with no estimate to stand in for it, and only the driver - /// knows how to count it. DynamoDB bills a `Scan` of every item and has no `COUNT(*)`; Cassandra reads - /// every partition, because `LIMIT` does not bound `COUNT(*)`, and a filtered count needs CQL the host - /// cannot write. Such an engine is counted only when the user asks, and only by its driver. + // Whether an exact count reads the whole table with no estimate to stand in for it, and only the driver + // knows how to count it. DynamoDB bills a `Scan` of every item and has no `COUNT(*)`; Cassandra reads + // every partition, because `LIMIT` does not bound `COUNT(*)`, and a filtered count needs CQL the host + // cannot write. Such an engine is counted only when the user asks, and only by its driver. var exactRowCountIsFullScan: Bool = false - /// Whether a table can be ordered by any of its columns. CQL orders rows only by clustering columns inside - /// one partition, so a Cassandra table offers no column sort and a query result sorts the rows it holds. + // Whether a table can be ordered by any of its columns. CQL orders rows only by clustering columns inside + // one partition, so a Cassandra table offers no column sort and a query result sorts the rows it holds. var supportsColumnSort: Bool = true - /// Whether filters can ask for rows matching any one of several conditions. CQL has no `OR`. + // Whether filters can ask for rows matching any one of several conditions. CQL has no `OR`. var supportsMatchAnyFilters: Bool = true - /// Whether a filter row can hold a condition written in the engine's own query language. A - /// driver that answers one with an error, such as etcd or DynamoDB, or reads it as a column - /// name, is not offered the row at all. + // Whether a filter row can hold a condition written in the engine's own query language. A + // driver that answers one with an error, such as etcd or DynamoDB, or reads it as a column + // name, is not offered the row at all. var supportsRawFilter: Bool = true - /// Whether a table's columns are a sample of its rows rather than a declared schema. A - /// MongoDB collection lists the fields found in its first documents, so a field missing from - /// one side's list says nothing about whether that side holds it. + // Whether a table's columns are a sample of its rows rather than a declared schema. A + // MongoDB collection lists the fields found in its first documents, so a field missing from + // one side's list says nothing about whether that side holds it. var columnsAreSampled: Bool = false - /// Whether a contains, starts with or ends with filter matches a scalar column of any type by the text it shows, - /// so a search can send number and date columns too. MongoDB's builder converts the value before it matches. + // Whether a contains, starts with or ends with filter matches a scalar column of any type by the text it shows, + // so a search can send number and date columns too. MongoDB's builder converts the value before it matches. var patternFiltersReadAnyColumn: Bool = false - /// Whether the driver types a filter's values by the kinds it learned from rows it read, so a count of a - /// filtered table runs on the driver that read the page. A pooled driver read nothing and would type the same - /// filter another way: a date compared as a string counts no rows. + // Whether the driver types a filter's values by the kinds it learned from rows it read, so a count of a + // filtered table runs on the driver that read the page. A pooled driver read nothing and would type the same + // filter another way: a date compared as a string counts no rows. var filtersTypedByReadRows: Bool = false var isEngineReadOnly: Bool = false - /// Which connection field carries the path of the local database file this driver opens, - /// for the types that open one. Nil for every driver that reaches its database over the - /// network, which is what makes it the test for "can this connection name a remote file". + // Which connection field carries the path of the local database file this driver opens, + // for the types that open one. Nil for every driver that reaches its database over the + // network, which is what makes it the test for "can this connection name a remote file". var localFilePathField: LocalFilePathField? - /// The plugin fields that name the server, for a type the built-in Host and Port do not - /// reach. Nil for every type whose driver dials `config.host`, which is what keeps a tunnel - /// forwarding and rewriting the built-in endpoint there. + // The plugin fields that name the server, for a type the built-in Host and Port do not + // reach. Nil for every type whose driver dials `config.host`, which is what keeps a tunnel + // forwarding and rewriting the built-in endpoint there. var serverEndpointFields: ServerEndpointFields? - /// Fields a driver older than this app ignores to the user's cost. A connection that sets one - /// away from its default, while it is in use, is refused when the loaded driver does not - /// declare it: DuckDB's SSL Mode read by a driver from before it existed would connect as - /// Automatic, which is plain HTTP on a loopback host, however the user set it. + // Fields a driver older than this app ignores to the user's cost. A connection that sets one + // away from its default, while it is in use, is refused when the loaded driver does not + // declare it: DuckDB's SSL Mode read by a driver from before it existed would connect as + // Automatic, which is plain HTTP on a loopback host, however the user set it. var fieldsRequiringDriverSupport: [String] = [] - /// The extensions a new database file can be named with, the first one preferred. Empty - /// when the driver refuses a path with nothing at it, as Beancount does, so the form offers - /// no way to name one. DuckDB lists only its own format: a missing Parquet or CSV path is - /// refused rather than created. + // The extensions a new database file can be named with, the first one preferred. Empty + // when the driver refuses a path with nothing at it, as Beancount does, so the form offers + // no way to name one. DuckDB lists only its own format: a missing Parquet or CSV path is + // refused rather than created. var newDatabaseFileExtensions: [String] = [] var supportsSOCKSProxy: Bool { supportsSSH } - /// `cloudflared access tcp` forwards a loopback port just as an SSH tunnel does. A stored flag - /// drifted: it defaulted to true in the built-in snapshots, so a registry type the plugin - /// declares without SSH offered Cloudflare Tunnel until that plugin was installed. + // `cloudflared access tcp` forwards a loopback port just as an SSH tunnel does. A stored flag + // drifted: it defaulted to true in the built-in snapshots, so a registry type the plugin + // declares without SSH offered Cloudflare Tunnel until that plugin was installed. var supportsCloudflareTunnel: Bool { supportsSSH } - /// A tunnel command forwards a loopback port to the server the connection names, so it - /// applies wherever an SSH tunnel would. Computed for the same reason `supportsSOCKSProxy` - /// is: a stored flag would need an opt-out line in every hand-written snapshot. + // A tunnel command forwards a loopback port to the server the connection names, so it + // applies wherever an SSH tunnel would. Computed for the same reason `supportsSOCKSProxy` + // is: a stored flag would need an opt-out line in every hand-written snapshot. var supportsTunnelCommand: Bool { supportsSSH } - /// Whether this type may point at a file on an SSH server instead of a local one. - /// - /// Deliberately not derived from `localFilePathField`. Beancount opens a local file and must - /// still be excluded: a ledger is a graph of files reached through `include`, so one file - /// out of it either fails to load or presents incomplete accounts, which is worse. + // Whether this type may point at a file on an SSH server instead of a local one. + // Deliberately not derived from `localFilePathField`. Beancount opens a local file and must + // still be excluded: a ledger is a graph of files reached through `include`, so one file + // out of it either fails to load or presents incomplete accounts, which is worse. var supportsRemoteDatabaseFile: Bool = false - /// Whether this type can run its statements on an SSH server against a live database file, - /// rather than only fetching a read-only copy. True for SQLite alone today, because the - /// remote agent opens the file with the server's `libsqlite3`. + // Whether this type can run its statements on an SSH server against a live database file, + // rather than only fetching a read-only copy. True for SQLite alone today, because the + // remote agent opens the file with the server's `libsqlite3`. var supportsRemoteDatabaseSession: Bool = false var supportsPrincipalConnectionLimit: Bool = true @@ -211,24 +214,24 @@ struct PluginMetadataSnapshot: Sendable { let systemDatabaseNames: [String] let systemSchemaNames: [String] let fileExtensions: [String] - /// Curated in the app rather than declared by the plugin: claiming a format from the - /// system also needs a `CFBundleDocumentTypes` entry only the app bundle can make. + // Curated in the app rather than declared by the plugin: claiming a format from the + // system also needs a `CFBundleDocumentTypes` entry only the app bundle can make. let fileSignatures: [DatabaseFileSignature] let databaseGroupingStrategy: GroupingStrategy let structureColumnFields: [StructureColumnField] let implicitSchemaName: String? let rowMatchExcludedTypePrefixes: [String] - /// Column types whose keyless row match has to compare the server's own text rendering - /// rather than the value itself. A grid reads every cell back as text, and on these types - /// that text does not compare equal to what it was read from: measured on MySQL 8.4.11 and - /// OceanBase 4.4.2.1, a `FLOAT` holding 1.1 reads back `1.1` and `WHERE f = '1.1'` matches - /// no row, because the server widens the stored single to a different double than it parses - /// the literal into. `JSON` never compares equal to a string at all. Listing a type here - /// wraps only the keyless comparison, and only for the types measured to need it: `CONCAT` - /// over a `BIT` column returns its raw bytes and breaks a match that works today. + // Column types whose keyless row match has to compare the server's own text rendering + // rather than the value itself. A grid reads every cell back as text, and on these types + // that text does not compare equal to what it was read from: measured on MySQL 8.4.11 and + // OceanBase 4.4.2.1, a `FLOAT` holding 1.1 reads back `1.1` and `WHERE f = '1.1'` matches + // no row, because the server widens the stored single to a different double than it parses + // the literal into. `JSON` never compares equal to a string at all. Listing a type here + // wraps only the keyless comparison, and only for the types measured to need it: `CONCAT` + // over a `BIT` column returns its raw bytes and breaks a match that works today. let rowMatchTextTypePrefixes: [String] - /// Teradata and Snowflake compare database names without regard to case, so `dbc` is the - /// system database `DBC`. Every other engine keeps case: MySQL can hold a user database `MySQL`. + // Teradata and Snowflake compare database names without regard to case, so `dbc` is the + // system database `DBC`. Every other engine keeps case: MySQL can hold a user database `MySQL`. let systemDatabaseNamesIgnoreCase: Bool // Nil for an engine that keeps the case of an unquoted name, or whose rule is not curated yet. let identifierRules: SQLIdentifierRules? @@ -313,10 +316,10 @@ struct PluginMetadataSnapshot: Sendable { let category: DatabaseCategory let tagline: String let hidesBuiltInPassword: Bool - /// The driver takes no container name on the connection, so the built-in field would - /// be a second, meaningless place to type one: an embedded engine reads it from the - /// file it opens, Redis numbers its databases through its own field, and a key-value - /// store may have no container at all. + // The driver takes no container name on the connection, so the built-in field would + // be a second, meaningless place to type one: an embedded engine reads it from the + // file it opens, Redis numbers its databases through its own field, and a key-value + // store may have no container at all. let hidesBuiltInDatabase: Bool let defaultUnixSocketPath: String? // Non-nil means the form offers Connect Using > Socket. @@ -505,11 +508,10 @@ final class PluginMetadataRegistry: @unchecked Sendable { registerBuiltInDefaults() } - /// The curated table the app ships, before any plugin or test registers over it. - /// - /// The live snapshots are not that table: a loaded plugin replaces its entry, and a test can - /// register a synthetic engine into the same shared registry. Anything that means to check the - /// table itself reads this instead of iterating `allRegisteredTypeIds()`. + // The curated table the app ships, before any plugin or test registers over it. + // The live snapshots are not that table: a loaded plugin replaces its entry, and a test can + // register a synthetic engine into the same shared registry. Anything that means to check the + // table itself reads this instead of iterating `allRegisteredTypeIds()`. func builtInDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { Self.curatedDefaults() + registryPluginDefaults() } @@ -542,12 +544,12 @@ final class PluginMetadataRegistry: @unchecked Sendable { registerLocked(snapshot: snapshot, forTypeId: typeId, preserveIcon: preserveIcon) } - /// Registers an additional database type served by a multi-type plugin (Redshift, - /// CockroachDB, PGlite on the PostgreSQL plugin). A plugin's statics are per-class, so - /// they cannot express per-type facts like PGlite's disabled SSL or single-connection limit. - /// The curated built-in entry is therefore authoritative for a variant; the plugin fills the - /// EXPLAIN variants the curated entry leaves open, and supplies the editor config. A variant - /// with no curated entry falls back to deriving its snapshot from the plugin. + // Registers an additional database type served by a multi-type plugin (Redshift, + // CockroachDB, PGlite on the PostgreSQL plugin). A plugin's statics are per-class, so + // they cannot express per-type facts like PGlite's disabled SSL or single-connection limit. + // The curated built-in entry is therefore authoritative for a variant; the plugin fills the + // EXPLAIN variants the curated entry leaves open, and supplies the editor config. A variant + // with no curated entry falls back to deriving its snapshot from the plugin. func registerVariant( pluginSnapshot: PluginMetadataSnapshot, forTypeId typeId: String, @@ -611,33 +613,30 @@ final class PluginMetadataRegistry: @unchecked Sendable { } } - /// A raw lookup by the exact id a snapshot was registered under, for the callers that hold an - /// id rather than a type: registration itself, and iteration over `allRegisteredTypeIds()`. - /// - /// It is deliberately not named `snapshot(forTypeId:)` any more. That spelling read as the - /// way to ask about a `DatabaseType`, so 62 call sites passed it `databaseType.pluginTypeId` - /// and every variant was answered with its primary's facts. Asking about a type is - /// `snapshot(for:)`; this overload cannot be reached from a `DatabaseType` without first - /// choosing which id you mean, which is the point. + // A raw lookup by the exact id a snapshot was registered under, for the callers that hold an + // id rather than a type: registration itself, and iteration over `allRegisteredTypeIds()`. + // It is deliberately not named `snapshot(forTypeId:)` any more. That spelling read as the + // way to ask about a `DatabaseType`, so 62 call sites passed it `databaseType.pluginTypeId` + // and every variant was answered with its primary's facts. Asking about a type is + // `snapshot(for:)`; this overload cannot be reached from a `DatabaseType` without first + // choosing which id you mean, which is the point. func snapshot(forRegisteredTypeId typeId: String) -> PluginMetadataSnapshot? { lock.lock() defer { lock.unlock() } return snapshots[typeId] } - /// The snapshot describing a database type, which for a variant is its own curated entry - /// rather than the entry of the plugin that serves it. Reaching a snapshot through - /// `pluginTypeId` asks the primary instead, which is how a Redshift tab came to be told - /// PostgreSQL's `ILIKE` folds non-ASCII, and how PGlite came to be offered the SSH, SSL, - /// Cloudflare Tunnel and SOCKS panes its own entry declares it does not support. - /// - /// The fallback covers a type registered by a plugin with no curated entry of its own, where - /// the primary's snapshot is the only one there is. - /// - /// This is the only way to read a snapshot for a `DatabaseType`. `pluginTypeId` answers a - /// different question, "which plugin serves this type", and belongs to driver lookup alone. - /// The entry the app ships for a type, kept after a plugin registers over it: what a driver - /// built with this app declares, against which an older loaded driver can be told apart. + // The snapshot describing a database type, which for a variant is its own curated entry + // rather than the entry of the plugin that serves it. Reaching a snapshot through + // `pluginTypeId` asks the primary instead, which is how a Redshift tab came to be told + // PostgreSQL's `ILIKE` folds non-ASCII, and how PGlite came to be offered the SSH, SSL, + // Cloudflare Tunnel and SOCKS panes its own entry declares it does not support. + // The fallback covers a type registered by a plugin with no curated entry of its own, where + // the primary's snapshot is the only one there is. + // This is the only way to read a snapshot for a `DatabaseType`. `pluginTypeId` answers a + // different question, "which plugin serves this type", and belongs to driver lookup alone. + // The entry the app ships for a type, kept after a plugin registers over it: what a driver + // built with this app declares, against which an older loaded driver can be told apart. func curatedSnapshot(for databaseType: DatabaseType) -> PluginMetadataSnapshot? { lock.lock() defer { lock.unlock() } @@ -665,42 +664,37 @@ final class PluginMetadataRegistry: @unchecked Sendable { // MARK: - Dynamic Type Registration - /// Registers an alias type ID that maps to a primary type ID. - /// Used for multi-type plugins (e.g., MariaDB → MySQL, Redshift → PostgreSQL). + // For multi-type plugins: MariaDB to MySQL, Redshift to PostgreSQL. func registerTypeAlias(_ aliasTypeId: String, primaryTypeId: String) { lock.lock() defer { lock.unlock() } reverseTypeIndex[aliasTypeId] = primaryTypeId } - /// The inverse of `registerTypeAlias`. `unregister(typeId:)` drops a snapshot and leaves the - /// alias behind, so a test that registers one has nothing to undo it with. + // The inverse of `registerTypeAlias`. `unregister(typeId:)` drops a snapshot and leaves the + // alias behind, so a test that registers one has nothing to undo it with. func removeTypeAlias(_ aliasTypeId: String) { lock.lock() defer { lock.unlock() } reverseTypeIndex.removeValue(forKey: aliasTypeId) } - /// Returns all registered type IDs (sorted for deterministic UI ordering). func allRegisteredTypeIds() -> [String] { lock.lock() defer { lock.unlock() } return Array(snapshots.keys).sorted() } - /// Resolves a database type raw value to its plugin type ID for driver lookup. - /// For multi-type plugins (MySQL serves MariaDB), maps the alias to the primary. - /// - /// This is the answer to "which plugin serves this type", never "what is this type like". - /// A snapshot read for a `DatabaseType` belongs in `snapshot(for:)`, which asks the variant - /// first; `snapshot(forTypeId:)` is for a bare id that is already the one you want. + // For multi-type plugins (MySQL serves MariaDB), maps the alias to the primary. + // This is the answer to "which plugin serves this type", never "what is this type like". + // A snapshot read for a `DatabaseType` belongs in `snapshot(for:)`, which asks the variant + // first; `snapshot(forTypeId:)` is for a bare id that is already the one you want. func pluginTypeId(for rawValue: String) -> String { lock.lock() defer { lock.unlock() } return reverseTypeIndex[rawValue] ?? rawValue } - /// Checks if a type ID is registered (has a snapshot). func hasType(_ typeId: String) -> Bool { lock.lock() defer { lock.unlock() } @@ -709,8 +703,7 @@ final class PluginMetadataRegistry: @unchecked Sendable { // MARK: - Snapshot Builder - /// Builds a PluginMetadataSnapshot from a DriverPlugin's protocol properties. - /// Used by PluginManager to self-register plugins at load time. + // PluginManager registers each plugin with this at load time. func buildMetadataSnapshot( from driverType: any DriverPlugin.Type, isDownloadable: Bool = false @@ -802,6 +795,8 @@ final class PluginMetadataRegistry: @unchecked Sendable { tlsImpliedPorts: existingSnapshot?.capabilities.tlsImpliedPorts ?? [], verifiesServerWithSystemTrust: existingSnapshot?.capabilities .verifiesServerWithSystemTrust ?? false, + requestedTLSVerifiesIdentity: existingSnapshot?.capabilities + .requestedTLSVerifiesIdentity ?? false, supportsPerConnectionCertificatePaths: existingSnapshot?.capabilities .supportsPerConnectionCertificatePaths ?? true, supportsClientKeyPassphrase: existingSnapshot?.capabilities.supportsClientKeyPassphrase ?? false, diff --git a/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift b/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift index 41f93a634b..5b2e70e303 100644 --- a/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift +++ b/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift @@ -9,18 +9,13 @@ import Foundation import TableProPluginKit enum BlobDisplayContext { - /// Data grid cell: compact single-line "0x48656C6C6F..." case grid - /// Sidebar detail view: full multi-line hex dump case detail - /// Copy to clipboard: compact hex case copy - /// Editable hex in sidebar: space-separated hex bytes "48 65 6C 6C 6F" case edit } -@MainActor -final class BlobFormattingService { +final class BlobFormattingService: Sendable { static let shared = BlobFormattingService() private init() {} @@ -68,12 +63,10 @@ final class BlobFormattingService { return String(data: data, encoding: .isoLatin1) } - /// Whether the given column type requires BLOB formatting. func requiresFormatting(columnType: ColumnType) -> Bool { columnType.isBlobType } - /// Format a value if the column type is a BLOB type; otherwise return the original value. func formatIfNeeded(_ value: String, columnType: ColumnType?, for context: BlobDisplayContext) -> String { guard let columnType, requiresFormatting(columnType: columnType) else { return value diff --git a/apple/TablePro/Core/Services/Formatting/CellDisplayFormatter.swift b/apple/TablePro/Core/Services/Formatting/CellDisplayFormatter.swift index 7f1945b61e..bd42bef09e 100644 --- a/apple/TablePro/Core/Services/Formatting/CellDisplayFormatter.swift +++ b/apple/TablePro/Core/Services/Formatting/CellDisplayFormatter.swift @@ -9,10 +9,10 @@ import Foundation import TableProPluginKit -@MainActor enum CellDisplayFormatter { static let maxDisplayLength = 10_000 + @MainActor static func format( _ rawValue: PluginCellValue, columnType: ColumnType?, @@ -22,12 +22,39 @@ enum CellDisplayFormatter { display(rawValue, columnType: columnType, displayFormat: displayFormat, databaseType: databaseType).text } - // `isFormatted` is false wherever the cell falls back to how its column reads without the display format. + static func format( + _ rawValue: PluginCellValue, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dates: DateDisplayRenderer + ) -> String? { + display(rawValue, columnType: columnType, displayFormat: displayFormat, databaseType: databaseType, dates: dates).text + } + + @MainActor static func display( _ rawValue: PluginCellValue, columnType: ColumnType?, displayFormat: ValueDisplayFormat?, databaseType: DatabaseType? + ) -> (text: String?, isFormatted: Bool) { + display( + rawValue, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: DateFormattingService.shared.renderer + ) + } + + // `isFormatted` is false wherever the cell falls back to how its column reads without the display format. + static func display( + _ rawValue: PluginCellValue, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dates: DateDisplayRenderer ) -> (text: String?, isFormatted: Bool) { switch rawValue { case .null: @@ -43,16 +70,13 @@ enum CellDisplayFormatter { guard !value.isEmpty else { return (value, false) } if let displayFormat, displayFormat != .raw, displayFormat.isApplicable(to: columnType, databaseType: databaseType), - let formatted = ValueDisplayFormatter.apply(value, format: displayFormat, columnType: columnType) { + let formatted = ValueDisplayFormatter.apply(value, format: displayFormat, columnType: columnType, dates: dates) { return (fitToCell(formatted), rendered(formatted, from: value, by: displayFormat)) } var displayValue = value if let columnType { if columnType.isDateType { - if let formatted = DateFormattingService.shared.format( - dateString: displayValue, - columnType: columnType - ) { + if let formatted = dates.format(dateString: displayValue, columnType: columnType) { displayValue = formatted } } else if columnType.isVectorType { diff --git a/apple/TablePro/Core/Services/Formatting/DateDisplayRenderer.swift b/apple/TablePro/Core/Services/Formatting/DateDisplayRenderer.swift new file mode 100644 index 0000000000..6b37d06584 --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/DateDisplayRenderer.swift @@ -0,0 +1,118 @@ +import Foundation + +// One column shape's formatters: the pattern as the user chose it, and the same pattern with a slot +// for the sub-second digits a value may carry, used only for a value that arrived with a fraction. +private struct TemporalFormatters { + let plain: DateFormatter + let withFractionalSeconds: DateFormatter? +} + +// Not Sendable: each value sets its formatter's zone, so one renderer serves one thread. Work off the +// main actor builds its own from the format option. +final class DateDisplayRenderer { + let format: DateFormatOption + + private let dateTimeFormatters: TemporalFormatters + private let dateOnlyFormatters: TemporalFormatters + private let timeOnlyFormatters: TemporalFormatters + + // The only formatter that renders an instant rather than a wall clock the value carries, so its zone + // is the reader's. `.autoupdatingCurrent` follows a system zone change; a captured `.current` is + // frozen for the life of the process, measured. + private let instantFormatter: DateFormatter + + private let formatCache = NSCache() + + init(format: DateFormatOption) { + self.format = format + dateTimeFormatters = Self.createFormatters(for: format, components: .dateAndTime) + dateOnlyFormatters = Self.createFormatters(for: format, components: .dateOnly) + timeOnlyFormatters = Self.createFormatters(for: format, components: .timeOnly) + instantFormatter = Self.createFormatter(pattern: format.formatString(for: .dateAndTime)) + instantFormatter.timeZone = .autoupdatingCurrent + formatCache.countLimit = 100_000 + } + + func format(_ date: Date) -> String { + instantFormatter.string(from: date) + } + + // Nil when the text does not parse. The value renders in the zone it was written in, not the + // reader's, so the grid and the cell editor agree on the day. A value with an offset prints that + // offset as the literal text it arrived with: a pattern token prints the formatter's zone, which + // would give a naive `DATETIME` an offset the database never held. The cache key needs no zone, + // since the zone is read off the value. + func format(dateString: String, columnType: ColumnType?) -> String? { + let components = temporalComponents(for: columnType) + let cacheKey = "\(components.rawValue)|\(dateString)" as NSString + if let cached = formatCache.object(forKey: cacheKey) { + return cached.length == 0 ? nil : cached as String + } + + guard let parsed = DatabaseDateParser.parse(dateString) else { + formatCache.setObject("" as NSString, forKey: cacheKey) + return nil + } + var result = render(parsed, with: formatters(for: components)) + if self.format.rendersTime(for: components), let suffix = parsed.layout.timeZoneSuffix { + result += suffix + } + formatCache.setObject(result as NSString, forKey: cacheKey) + return result + } + + // A column with no type reads as a full timestamp, the grid's fallback before the editor and the + // formatter shared this vocabulary. + private func temporalComponents(for columnType: ColumnType?) -> TemporalComponents { + guard let columnType else { return .dateAndTime } + return DateEditingService.components(for: columnType) + } + + private func formatters(for components: TemporalComponents) -> TemporalFormatters { + switch components { + case .dateOnly: return dateOnlyFormatters + case .timeOnly: return timeOnlyFormatters + case .dateAndTime: return dateTimeFormatters + } + } + + // The value's own sub-second digits are spliced in, not rendered, so `.789012` stays `.789012` and two + // rows a microsecond apart never share a value filter entry. A fraction of only zeros is padding + // (MySQL writes one on every `DATETIME(6)` row) and is dropped. + private func render(_ parsed: ParsedTemporalValue, with formatters: TemporalFormatters) -> String { + guard let fractionalSeconds = parsed.layout.fractionalSeconds, + fractionalSeconds.contains(where: { $0 != "0" && $0 != "." }), + let marked = formatters.withFractionalSeconds + else { + formatters.plain.timeZone = parsed.timeZone + return formatters.plain.string(from: parsed.wholeSecond) + } + marked.timeZone = parsed.timeZone + return marked.string(from: parsed.wholeSecond) + .replacingOccurrences(of: String(DateFormatOption.fractionalSecondsMarker), with: fractionalSeconds) + } + + // A fixed pattern needs a fixed locale and calendar: the user's region would render timestamps in + // the Buddhist or Japanese calendar and change the value the grid writes back to SQL. + private static let fixedPatternLocale = Locale(identifier: "en_US_POSIX") + + private static func createFormatters( + for format: DateFormatOption, + components: TemporalComponents + ) -> TemporalFormatters { + TemporalFormatters( + plain: createFormatter(pattern: format.formatString(for: components)), + withFractionalSeconds: format.fractionalSecondsFormatString(for: components) + .map(createFormatter(pattern:)) + ) + } + + private static func createFormatter(pattern: String) -> DateFormatter { + let formatter = DateFormatter() + formatter.dateFormat = pattern + formatter.locale = fixedPatternLocale + formatter.calendar = Calendar(identifier: .gregorian) + formatter.timeZone = TimeZone.current + return formatter + } +} diff --git a/apple/TablePro/Core/Services/Formatting/DateFormattingService.swift b/apple/TablePro/Core/Services/Formatting/DateFormattingService.swift index 25e5751407..67449a233a 100644 --- a/apple/TablePro/Core/Services/Formatting/DateFormattingService.swift +++ b/apple/TablePro/Core/Services/Formatting/DateFormattingService.swift @@ -2,73 +2,33 @@ // DateFormattingService.swift // TablePro // -// Centralized date formatting service that respects user settings. -// Thread-safe singleton that formats dates according to DataGridSettings.dateFormat. -// import Combine import Foundation -/// The pair of formatters one column shape needs: the pattern as the user chose it, and the same -/// pattern with a slot for the sub-second digits a value may carry. -private struct TemporalFormatters { - let plain: DateFormatter - let withFractionalSeconds: DateFormatter? -} - -/// Centralized date formatting service that respects user settings @MainActor final class DateFormattingService { static let shared = DateFormattingService() - // MARK: - Properties - - /// Cached formatters for the current user-selected format. - /// - /// Two per column shape, not one: the marked variant carries a quoted slot after the seconds - /// and is used only for a value that arrived with a fraction, so a value without one pays - /// neither the second format nor the splice. - private var dateTimeFormatters: TemporalFormatters - private var dateOnlyFormatters: TemporalFormatters - private var timeOnlyFormatters: TemporalFormatters - - /// The one formatter that renders an instant rather than a wall clock the value already - /// carries, so it is the only one whose zone is the reader's. - /// - /// It holds `.autoupdatingCurrent`, which follows a system time zone change with no - /// reassignment; a captured `.current` is frozen for the life of the process, measured. Its - /// pattern still tracks the user's chosen format, so `updateFormat` rewrites that and never - /// the zone. - private let instantFormatter: DateFormatter + // Replaced on a format change rather than reconfigured, so no text cached under the old format survives. + private(set) var renderer: DateDisplayRenderer private var systemTimeZoneObserver: NSObjectProtocol? - /// Rises each time the Mac's zone moves, so a display cache built in the old one compares - /// unequal and is replaced rather than repaired. + // Rises each time the Mac's zone moves, so a display cache built in the old one compares unequal and + // is replaced rather than repaired. private(set) var systemTimeZoneGeneration = 0 - /// Current date format option - private(set) var currentFormat: DateFormatOption - - /// Cache for formatted date strings to avoid repeated parsing - private let formatCache = NSCache() - - // MARK: - Initialization + var currentFormat: DateFormatOption { renderer.format } + // AppSettingsManager sets the user's format once it has loaded. private init() { - // Will be updated by AppSettingsManager after it completes initialization - self.currentFormat = .iso8601 - self.dateTimeFormatters = Self.createFormatters(for: .iso8601, components: .dateAndTime) - self.dateOnlyFormatters = Self.createFormatters(for: .iso8601, components: .dateOnly) - self.timeOnlyFormatters = Self.createFormatters(for: .iso8601, components: .timeOnly) - self.instantFormatter = Self.createFormatter(pattern: DateFormatOption.iso8601.formatString(for: .dateAndTime)) - self.instantFormatter.timeZone = .autoupdatingCurrent - formatCache.countLimit = 100_000 + renderer = DateDisplayRenderer(format: .iso8601) observeSystemTimeZone() } - /// `NSSystemTimeZoneDidChange` documents no posting thread, and this app has already shipped a - /// crash from assuming one, so the observer names the main queue and hops explicitly. + // `NSSystemTimeZoneDidChange` documents no posting thread, and this app has already shipped a crash + // from assuming one, so the observer names the main queue and hops explicitly. private func observeSystemTimeZone() { systemTimeZoneObserver = NotificationCenter.default.addObserver( forName: .NSSystemTimeZoneDidChange, @@ -82,122 +42,16 @@ final class DateFormattingService { } } - // MARK: - Public Methods - - /// Update the date format (called by AppSettingsManager when settings change) func updateFormat(_ format: DateFormatOption) { guard format != currentFormat else { return } - currentFormat = format - dateTimeFormatters = Self.createFormatters(for: format, components: .dateAndTime) - dateOnlyFormatters = Self.createFormatters(for: format, components: .dateOnly) - timeOnlyFormatters = Self.createFormatters(for: format, components: .timeOnly) - instantFormatter.dateFormat = format.formatString(for: .dateAndTime) - // Clear cache when format changes since all cached values are now stale - formatCache.removeAllObjects() + renderer = DateDisplayRenderer(format: format) } - /// Format a date using current user settings - /// - Parameter date: The date to format - /// - Returns: Formatted date string func format(_ date: Date) -> String { - instantFormatter.string(from: date) + renderer.format(date) } - /// Format a string date value (parse then format). - /// - /// The value is rendered in the zone it was written in, not the reader's. A value carrying an - /// offset names an instant in that offset, and the cell editor opens it there, so formatting it - /// somewhere else made the grid and the picker disagree about which day the row held. A value - /// with no offset is naive and parses in GMT, so it prints the wall clock it was written with - /// whatever zone the reader is in. - /// - /// The offset itself is printed with it, because a wall clock alone does not name an instant. - /// It is the literal text the value arrived with, off the same layout the write-back reads. A - /// pattern token cannot serve: it prints the formatter's own zone, which for a value carrying - /// no offset is the reader's, so a `DATETIME` would gain an offset the database never held. - /// - /// The cache key needs no zone: the zone is read off the value, so the string determines it. - /// - Parameter dateString: Date string from database (ISO 8601, MySQL timestamp, etc.) - /// - Parameter columnType: Column type, used to pick date-only / time-only / datetime variant - /// - Returns: Formatted date string, or nil if unparseable func format(dateString: String, columnType: ColumnType? = nil) -> String? { - let components = temporalComponents(for: columnType) - let cacheKey = "\(components.rawValue)|\(dateString)" as NSString - if let cached = formatCache.object(forKey: cacheKey) { - return cached.length == 0 ? nil : cached as String - } - - guard let parsed = DatabaseDateParser.parse(dateString) else { - formatCache.setObject("" as NSString, forKey: cacheKey) - return nil - } - var result = render(parsed, with: formatters(for: components)) - if currentFormat.rendersTime(for: components), let suffix = parsed.layout.timeZoneSuffix { - result += suffix - } - formatCache.setObject(result as NSString, forKey: cacheKey) - return result - } - - /// A column with no type at all is read as a full timestamp, which is what the grid's own - /// fallback did before the editor and the formatter shared this vocabulary. - private func temporalComponents(for columnType: ColumnType?) -> TemporalComponents { - guard let columnType else { return .dateAndTime } - return DateEditingService.components(for: columnType) - } - - private func formatters(for components: TemporalComponents) -> TemporalFormatters { - switch components { - case .dateOnly: return dateOnlyFormatters - case .timeOnly: return timeOnlyFormatters - case .dateAndTime: return dateTimeFormatters - } - } - - /// The value's own sub-second digits are spliced in, not rendered, so `.5` stays `.5` and - /// `.789012` stays `.789012`. Two rows one microsecond apart used to draw the same text, which - /// also collapsed them into one entry in the column's value filter. - /// - /// A fraction of nothing but zeros is padding rather than precision, and MySQL writes one on - /// every row of a `DATETIME(6)` where PostgreSQL trims it at the server. It is dropped, so such - /// a column reads exactly as it did before and only a value that carries real digits grows. - private func render(_ parsed: ParsedTemporalValue, with formatters: TemporalFormatters) -> String { - guard let fractionalSeconds = parsed.layout.fractionalSeconds, - fractionalSeconds.contains(where: { $0 != "0" && $0 != "." }), - let marked = formatters.withFractionalSeconds - else { - formatters.plain.timeZone = parsed.timeZone - return formatters.plain.string(from: parsed.wholeSecond) - } - marked.timeZone = parsed.timeZone - return marked.string(from: parsed.wholeSecond) - .replacingOccurrences(of: String(DateFormatOption.fractionalSecondsMarker), with: fractionalSeconds) - } - - // MARK: - Private Helper Methods - - /// `dateFormat` is a fixed pattern, so it must be evaluated against a fixed locale and - /// calendar. Following the user's region renders database timestamps in the Buddhist or - /// Japanese calendar and reformats the very value the grid round-trips back to SQL. - private static let fixedPatternLocale = Locale(identifier: "en_US_POSIX") - - private static func createFormatters( - for format: DateFormatOption, - components: TemporalComponents - ) -> TemporalFormatters { - TemporalFormatters( - plain: createFormatter(pattern: format.formatString(for: components)), - withFractionalSeconds: format.fractionalSecondsFormatString(for: components) - .map(createFormatter(pattern:)) - ) - } - - private static func createFormatter(pattern: String) -> DateFormatter { - let formatter = DateFormatter() - formatter.dateFormat = pattern - formatter.locale = fixedPatternLocale - formatter.calendar = Calendar(identifier: .gregorian) - formatter.timeZone = TimeZone.current - return formatter + renderer.format(dateString: dateString, columnType: columnType) } } diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift index 3732e7d5b0..96a4ee5e54 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift @@ -12,14 +12,23 @@ import TableProUtilities // per cell, and it has to stay reachable without dragging the preference layer behind it. // Resolving *which* format a column carries is a separate job with separate dependencies, and it // lives in `ValueDisplayFormatService`. -@MainActor enum ValueDisplayFormatter { + @MainActor + static func apply( + _ rawValue: String, + format: ValueDisplayFormat, + columnType: ColumnType? = nil + ) -> String? { + apply(rawValue, format: format, columnType: columnType, dates: DateFormattingService.shared.renderer) + } + // nil where the format cannot render this value, so the caller falls back to how the column // would have read without it rather than showing a value the format did not actually produce. static func apply( _ rawValue: String, format: ValueDisplayFormat, - columnType: ColumnType? = nil + columnType: ColumnType?, + dates: DateDisplayRenderer ) -> String? { switch format { case .raw: @@ -31,9 +40,9 @@ enum ValueDisplayFormatter { case .uuidSwappedTime: return formatAsUuid(rawValue, order: .swappedTime) case .unixTimestamp: - return formatAsTimestamp(rawValue, divideBy: 1) + return formatAsTimestamp(rawValue, divideBy: 1, dates: dates) case .unixTimestampMillis: - return formatAsTimestamp(rawValue, divideBy: 1_000) + return formatAsTimestamp(rawValue, divideBy: 1_000, dates: dates) case .json, .phpSerialized: return rawValue case .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: @@ -105,11 +114,11 @@ enum ValueDisplayFormatter { return "\(p1)-\(p2)-\(p3)-\(p4)-\(p5)" } - private static func formatAsTimestamp(_ rawValue: String, divideBy divisor: Double) -> String { + private static func formatAsTimestamp(_ rawValue: String, divideBy divisor: Double, dates: DateDisplayRenderer) -> String { guard let numericValue = Double(rawValue) else { return rawValue } let seconds = numericValue / divisor let date = Date(timeIntervalSince1970: seconds) - return DateFormattingService.shared.format(date) + return dates.format(date) } } diff --git a/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService.swift b/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService.swift index f7e894e09b..dd23d2eac5 100644 --- a/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService.swift +++ b/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService.swift @@ -58,8 +58,12 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { let commit: ReferenceWritableKeyPath> let dedup: OnceTask, Value> let label: String + // What a read uses while nothing is on screen; a reread of rows already shown always goes bulk. + let workload: MetadataConnectionPool.Workload } + private typealias ListFetch = @Sendable (MetadataConnectionPool.Workload) async throws -> Value + @Published private(set) var databaseList: [UUID: MetadataLoadState<[DatabaseMetadata]>] = [:] @Published private(set) var schemaList: [DatabaseKey: MetadataLoadState<[String]>] = [:] @Published private(set) var tablesState: [ObjectsKey: MetadataLoadState<[TableInfo]>] = [:] @@ -98,27 +102,27 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { private init() {} private var schemaLists: TreeList { - TreeList(state: \.schemaList, commit: \.schemaListCommit, dedup: schemaDedup, label: "schemas") + TreeList(state: \.schemaList, commit: \.schemaListCommit, dedup: schemaDedup, label: "schemas", workload: .interactive) } private var tableLists: TreeList { - TreeList(state: \.tablesState, commit: \.tablesCommit, dedup: tablesDedup, label: "tables") + TreeList(state: \.tablesState, commit: \.tablesCommit, dedup: tablesDedup, label: "tables", workload: .interactive) } private var routineLists: TreeList { - TreeList(state: \.routinesState, commit: \.routinesCommit, dedup: routinesDedup, label: "routines") + TreeList(state: \.routinesState, commit: \.routinesCommit, dedup: routinesDedup, label: "routines", workload: .bulk) } private var triggerLists: TreeList { - TreeList(state: \.triggersState, commit: \.triggersCommit, dedup: triggersDedup, label: "triggers") + TreeList(state: \.triggersState, commit: \.triggersCommit, dedup: triggersDedup, label: "triggers", workload: .bulk) } private var typeLists: TreeList { - TreeList(state: \.typesState, commit: \.typesCommit, dedup: typesDedup, label: "types") + TreeList(state: \.typesState, commit: \.typesCommit, dedup: typesDedup, label: "types", workload: .bulk) } private var partitionLists: TreeList { - TreeList(state: \.partitionsState, commit: \.partitionsCommit, dedup: partitionsDedup, label: "partitions") + TreeList(state: \.partitionsState, commit: \.partitionsCommit, dedup: partitionsDedup, label: "partitions", workload: .interactive) } // MARK: - Reads @@ -467,10 +471,14 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { } } - private func fetchDatabaseList(connectionId: UUID, databaseType: DatabaseType) async throws -> [DatabaseMetadata] { + private func fetchDatabaseList( + connectionId: UUID, + databaseType: DatabaseType, + workload: MetadataConnectionPool.Workload = .interactive + ) async throws -> [DatabaseMetadata] { let systemNames = Set(PluginManager.shared.systemDatabaseNames(for: databaseType)) return try await databaseDedup.execute(key: connectionId) { [self] in - try await withDriver(connectionId: connectionId, database: nil) { driver in + try await withDriver(connectionId: connectionId, database: nil, workload: workload) { driver in try await driver.fetchDatabases().sorted().map { DatabaseMetadata.minimal(name: $0, isSystem: systemNames.contains($0)) } @@ -550,7 +558,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { } guard isConnected(connectionId) else { return } do { - let list = try await fetchDatabaseList(connectionId: connectionId, databaseType: databaseType) + let list = try await fetchDatabaseList(connectionId: connectionId, databaseType: databaseType, workload: .bulk) guard databaseListFence.isCurrent(token, for: connectionId) else { return } databaseList[connectionId] = .loaded(list) } catch is CancellationError { @@ -648,7 +656,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { connectionId: UUID, in list: TreeList, token explicitToken: Int? = nil, - fetch: @Sendable @escaping () async throws -> Value + fetch: @escaping ListFetch ) async { guard isConnected(connectionId) else { return } let token = explicitToken ?? self[keyPath: list.commit].fence.token(for: key) @@ -656,15 +664,24 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { !self[keyPath: list.commit].freshness.isCurrent(key) else { return } let revision = self[keyPath: list.commit].freshness.revision(for: key) self[keyPath: list.commit].freshness.noteFetchStarted(revision, for: key) + // Rows on screen mean nobody waits on this read, so completion goes ahead of it. + let workload: MetadataConnectionPool.Workload switch self[keyPath: list.state][key] ?? .idle { - case .loaded, .loading: break - case .idle, .failed: self[keyPath: list.state][key] = .loading + case .loaded: + workload = .bulk + case .loading: + workload = list.workload + case .idle, .failed: + workload = list.workload + self[keyPath: list.state][key] = .loading } let outcome: MetadataFetchOutcome var failure: Error? do { - outcome = .fetched(try await list.dedup.execute(key: ListFetchKey(key: key, revision: revision), work: fetch)) + outcome = .fetched(try await list.dedup.execute(key: ListFetchKey(key: key, revision: revision)) { + try await fetch(workload) + }) } catch is CancellationError { outcome = .cancelled } catch { @@ -713,7 +730,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { _ key: Key, connectionId: UUID, in list: TreeList, - fetch: @Sendable @escaping () async throws -> Value + fetch: @escaping ListFetch ) async { let token = self[keyPath: list.commit].fence.supersede(key) self[keyPath: list.commit].freshness.markChanged(key) @@ -722,55 +739,55 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { await read(key, connectionId: connectionId, in: list, token: token, fetch: fetch) } - private func schemaListFetch(_ key: DatabaseKey) -> @Sendable () async throws -> [String] { - { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database) { driver in + private func schemaListFetch(_ key: DatabaseKey) -> ListFetch<[String]> { + { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchSchemas() } } } - private func tableListFetch(_ key: ObjectsKey) -> @Sendable () async throws -> [TableInfo] { + private func tableListFetch(_ key: ObjectsKey) -> ListFetch<[TableInfo]> { let schema = key.schema - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchTables(schema: schema) } } } - private func routineListFetch(_ key: ObjectsKey) -> @Sendable () async throws -> [RoutineInfo] { + private func routineListFetch(_ key: ObjectsKey) -> ListFetch<[RoutineInfo]> { let schema = key.schema - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database, workload: .bulk) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchRoutines(schema: schema) } } } - private func triggerListFetch(_ key: ObjectsKey) -> @Sendable () async throws -> [TriggerInfo] { + private func triggerListFetch(_ key: ObjectsKey) -> ListFetch<[TriggerInfo]> { let schema = key.schema - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database, workload: .bulk) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchAllTriggers(schema: schema) } } } - private func typeListFetch(_ key: ObjectsKey) -> @Sendable () async throws -> [UserDefinedTypeInfo] { + private func typeListFetch(_ key: ObjectsKey) -> ListFetch<[UserDefinedTypeInfo]> { let schema = key.schema - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database, workload: .bulk) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchUserDefinedTypes(schema: schema) } } } - private func partitionListFetch(_ key: PartitionsKey) -> @Sendable () async throws -> [PartitionInfo] { + private func partitionListFetch(_ key: PartitionsKey) -> ListFetch<[PartitionInfo]> { let schema = key.schema let table = key.table - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchPartitionDetails(table: table, schema: schema) } } diff --git a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift index c7a274f88f..9de54b604c 100644 --- a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift +++ b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift @@ -4,6 +4,7 @@ // import Foundation +import os import TableProPluginKit // The lease a task's work runs under. That work keeps the lease until a lease it asks for returns, so such a @@ -11,10 +12,13 @@ import TableProPluginKit @MainActor private final class PooledLease { let connectionId: UUID + // A lease nested in this one never takes this entry: it would queue behind the work that waits on it. + let entryId: ObjectIdentifier var isOpen = true - init(connectionId: UUID) { + init(connectionId: UUID, entryId: ObjectIdentifier) { self.connectionId = connectionId + self.entryId = entryId } } @@ -31,43 +35,81 @@ final class MetadataConnectionPool { case bulk } + // A scope with no schema reads wherever the connection's own settings put it, so its entries never switch. + private enum Slot: Hashable, Sendable { + case home + case followsSchema + } + + // No schema in the key: an entry moves to each lease's schema, so the schemas of one database share connections. private struct Key: Hashable, Sendable { - let scope: DatabaseScope + let connectionId: UUID + let database: String let workload: Workload + let slot: Slot + + init(scope: DatabaseScope, workload: Workload) { + connectionId = scope.connectionId + database = scope.database + self.workload = workload + slot = scope.schema == nil ? .home : .followsSchema + } + } + + private struct OpenKey: Hashable, Sendable { + let key: Key + let schema: String? + // Set for a lease inside another lease's work, so its open is its own and nobody queues on it. + let owner: UUID? + } + + private enum LeaseOutcome: Sendable { + case ran(T) + // The entry was left in a transaction and is closing, so the lease takes another. + case retired } @MainActor private final class Entry { let driver: DatabaseDriver - var lastUsed: Date - var inFlightCount: Int - var closeWhenIdle: Bool + var lastUsed = Date() + var inFlightCount = 0 + var closeWhenIdle = false + // The schema the last lease moved it to, used only to pick an entry; nil until a lease has moved it. + var schema: String? + // Only leases on one schema queue on a busy entry, so this is the schema its queue is on. + var queuedSchema: String? + // A startup command can move the search path after the driver read it, so the driver's cache is not trusted. + var forcesNextSwitch: Bool + var hasServedALease = false private var tail: Task = Task {} - init(driver: DatabaseDriver) { + init(driver: DatabaseDriver, schema: String?, forcesNextSwitch: Bool) { self.driver = driver - self.lastUsed = Date() - self.inFlightCount = 0 - self.closeWhenIdle = false + self.schema = schema + self.queuedSchema = schema + self.forcesNextSwitch = forcesNextSwitch } - /// The work runs in its own task so the next caller can queue behind it, so - /// cancelling the caller has to be forwarded explicitly or a stopped query - /// would keep running with nobody waiting on it. + // The work runs in its own task so the next caller can queue behind it, so a cancel is forwarded by hand. func runSerially( connectionId: UUID, + prepare: @escaping @MainActor @Sendable (Entry) async throws -> Bool, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T - ) async throws -> T { + ) async throws -> LeaseOutcome { let previous = tail let driver = self.driver - let work = Task { @MainActor () async throws -> T in + let work = Task { @MainActor [self] () async throws -> LeaseOutcome in await previous.value try Task.checkCancellation() - let lease = PooledLease(connectionId: connectionId) + guard try await prepare(self) else { return .retired } + hasServedALease = true + let lease = PooledLease(connectionId: connectionId, entryId: ObjectIdentifier(self)) defer { lease.isOpen = false } - return try await PooledLeaseContext.$open.withValue(PooledLeaseContext.open + [lease]) { + let value = try await PooledLeaseContext.$open.withValue(PooledLeaseContext.open + [lease]) { try await body(driver) } + return .ran(value) } tail = Task { @MainActor in _ = try? await work.value } return try await withTaskCancellationHandler( @@ -79,27 +121,22 @@ final class MetadataConnectionPool { typealias DriverOpener = @MainActor (DatabaseScope) async throws -> DatabaseDriver - /// Why an open still in progress was taken away from the callers waiting on it. + // Why an open still in progress was taken away from the callers waiting on it. private enum Withdrawal { - /// The transport it was dialing is being rebuilt, so its callers take a connection again - /// once the replacement is in place. + // The transport it was dialing is being rebuilt, so its callers take a connection again after the replacement. case transportReplaced - /// What it was opened for is going away, so nothing may open it again. + // What it was opened for is going away, so nothing may open it again. case closed } - /// One open in progress, shared by every caller that asks for its key while it runs. It stays - /// listed until the last of those callers has taken its entry, because an entry nobody has - /// taken yet looks idle and would otherwise be the first one closed. + // Shared by every caller that asks for its key and schema while it runs. It stays listed until the last of them + // has taken its entry, because an entry nobody has taken yet looks idle and would be the first one closed. @MainActor private final class PendingOpen { - let task: Task + var task: Task? + var entry: Entry? var withdrawal: Withdrawal? var waiterCount = 0 - - init(task: Task) { - self.task = task - } } private struct TransportWaiter { @@ -111,22 +148,27 @@ final class MetadataConnectionPool { @MainActor private final class RoomTicket { let key: Key + let schema: String? + let held: Set var continuation: CheckedContinuation? var isWithdrawn = false - init(key: Key) { + init(key: Key, schema: String?, held: Set) { self.key = key + self.schema = schema + self.held = held } } - private var entries: [Key: Entry] = [:] - private var pending: [Key: PendingOpen] = [:] + private var entries: [Key: [Entry]] = [:] + private var pending: [OpenKey: PendingOpen] = [:] private var transportReplacements: [UUID: Int] = [:] private var transportWaiters: [UUID: [TransportWaiter]] = [:] private var roomQueues: [UUID: [RoomTicket]] = [:] private let openDriver: DriverOpener static let maxPerConnection = 6 private var sweeper: Task? + private static let logger = Logger(subsystem: "com.TablePro", category: "MetadataConnectionPool") /// How long a pooled connection may sit unused before it is handed back. /// @@ -156,38 +198,47 @@ final class MetadataConnectionPool { workload: Workload = .interactive, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T { - let entry = try await acquireEntry(scope: scope, workload: workload) - entry.inFlightCount += 1 - entry.lastUsed = Date() - defer { releaseEntry(entry, connectionId: scope.connectionId) } - return try await entry.runSerially(connectionId: scope.connectionId, body) + let key = Key(scope: scope, workload: workload) + while true { + let entry = try await acquireEntry(forKey: key, schema: scope.schema, scope: scope) + if case .ran(let value) = try await run(body, on: entry, inKey: key, schema: scope.schema) { + return value + } + } + } + + private func run( + _ body: @Sendable @escaping (DatabaseDriver) async throws -> T, + on entry: Entry, + inKey key: Key, + schema: String? + ) async throws -> LeaseOutcome { + defer { releaseEntry(entry, connectionId: key.connectionId) } + return try await entry.runSerially( + connectionId: key.connectionId, + prepare: { [self] entry in try await moveEntry(entry, inKey: key, to: schema) }, + body + ) } /// Closes only the leases attached to one database, which is what a rename of that database /// needs: PostgreSQL refuses `ALTER DATABASE ... RENAME` while any backend is connected to it, /// and an expanded row or a tab that ran a query there leaves one here. func closeAll(connectionId: UUID, database: String) { - closeEntries(withdrawingOpensAs: .closed) { scope in - scope.connectionId == connectionId && scope.database == database - } + closeEntries(withdrawingOpensAs: .closed) { $0 == connectionId && $1 == database } } func closeAll(connectionId: UUID) { - closeEntries(withdrawingOpensAs: .closed) { $0.connectionId == connectionId } + closeEntries(withdrawingOpensAs: .closed) { id, _ in id == connectionId } } - /// Holds a connection's pooled work while its transport is rebuilt. - /// - /// A pooled entry stands on the session's effective connection plus its own database, so it only - /// has to go when that endpoint does: a tunnel rebuilt on a new local port, or a reconnect - /// recovering a connection that stopped answering. An open already dialing is withdrawn, and once - /// it has returned the callers waiting on it take a connection again after `endTransportReplacement` - /// rather than failing a load nobody cancelled. A new caller waits for the same moment instead of - /// dialing an endpoint about to close. Replacements nest, and pooled work resumes when the last one - /// ends. Closing the connection, or one of its databases, fails the callers waiting for it. + // Holds a connection's pooled work while its transport is rebuilt: a tunnel on a new local port, or a reconnect of + // a connection that stopped answering. An open already dialing is withdrawn and its callers take a connection again + // after `endTransportReplacement`; a new caller waits for that too. Replacements nest. Closing the connection, or + // one of its databases, fails the callers waiting for it. func beginTransportReplacement(connectionId: UUID) { transportReplacements[connectionId, default: 0] += 1 - closeEntries(withdrawingOpensAs: .transportReplaced) { $0.connectionId == connectionId } + closeEntries(withdrawingOpensAs: .transportReplaced) { id, _ in id == connectionId } } /// Has to follow every `beginTransportReplacement` on every exit, cancellation included, or the @@ -205,29 +256,35 @@ final class MetadataConnectionPool { } #if DEBUG - /// Seeds a connected driver as the pooled connection for `scope`, so a test can observe - /// what runs on the pool without a plugin to open a real connection. + // Seeds a connected driver as a pooled connection already on `scope`'s schema, for a test with no plugin to open one. internal func injectEntry( _ driver: DatabaseDriver, scope: DatabaseScope, workload: Workload = .interactive, lastUsed: Date = Date() ) { - let entry = Entry(driver: driver) + let entry = Entry(driver: driver, schema: scope.schema, forcesNextSwitch: false) entry.lastUsed = lastUsed - entries[Key(scope: scope, workload: workload)] = entry + entry.hasServedALease = true + entries[Key(scope: scope, workload: workload), default: []].append(entry) } internal func pooledDriverCount(for connectionId: UUID) -> Int { - entries.keys.filter { $0.scope.connectionId == connectionId }.count + pooled(on: connectionId).count } internal func heldConnectionCount(for connectionId: UUID) -> Int { heldCount(for: connectionId) } + internal func leaseCount(for connectionId: UUID) -> Int { + pooled(on: connectionId).reduce(0) { $0 + $1.entry.inFlightCount } + } + internal func markInFlight(scope: DatabaseScope, workload: Workload = .interactive) { - entries[Key(scope: scope, workload: workload)]?.inFlightCount += 1 + let list = entries[Key(scope: scope, workload: workload)] ?? [] + let entry = list.first(where: { $0.schema == scope.schema }) ?? list.first + entry?.inFlightCount += 1 } internal var hasSweeper: Bool { @@ -275,33 +332,43 @@ final class MetadataConnectionPool { handRoom(on: connectionId) } - private func closeOrDeferEntry(forKey key: Key) { - guard let entry = entries.removeValue(forKey: key) else { return } + @discardableResult + private func detach(_ entry: Entry, fromKey key: Key) -> Bool { + guard var list = entries[key], let index = list.firstIndex(where: { $0 === entry }) else { return false } + list.remove(at: index) + entries[key] = list.isEmpty ? nil : list + return true + } + + private func removeEntry(_ entry: Entry, fromKey key: Key) { + guard detach(entry, fromKey: key) else { return } if entry.inFlightCount == 0 { entry.driver.disconnect() } else { entry.closeWhenIdle = true } - handRoom(on: key.scope.connectionId) + handRoom(on: key.connectionId) } private func closeEntries( withdrawingOpensAs withdrawal: Withdrawal, - where matches: (DatabaseScope) -> Bool + where matches: (UUID, String) -> Bool ) { var withdrawnFrom: Set = [] - for key in pending.keys where matches(key.scope) { - guard let open = pending.removeValue(forKey: key) else { continue } + for openKey in pending.keys where matches(openKey.key.connectionId, openKey.key.database) { + guard let open = pending.removeValue(forKey: openKey) else { continue } open.withdrawal = withdrawal - open.task.cancel() - withdrawnFrom.insert(key.scope.connectionId) + open.task?.cancel() + withdrawnFrom.insert(openKey.key.connectionId) } if withdrawal == .closed { failTransportWaiters(where: matches) withdrawRoomTickets(where: matches) } - for key in entries.keys where matches(key.scope) { - closeOrDeferEntry(forKey: key) + for (key, list) in entries where matches(key.connectionId, key.database) { + for entry in list { + removeEntry(entry, fromKey: key) + } } for connectionId in withdrawnFrom { handRoom(on: connectionId) @@ -309,14 +376,12 @@ final class MetadataConnectionPool { stopSweeperIfEmpty() } - /// A caller parked for a replacement is waiting on its connection as much as one waiting on a - /// pending open, so closing what it waits for fails it too. Left parked, it would wake when the - /// replacement ends and open a connection for a session, or a database, that has gone. - private func failTransportWaiters(where matches: (DatabaseScope) -> Bool) { + // A parked caller would otherwise wake when the replacement ends and open a connection for what has gone. + private func failTransportWaiters(where matches: (UUID, String) -> Bool) { for (connectionId, waiters) in transportWaiters { - let failing = waiters.filter { matches($0.scope) } + let failing = waiters.filter { matches($0.scope.connectionId, $0.scope.database) } guard !failing.isEmpty else { continue } - let remaining = waiters.filter { !matches($0.scope) } + let remaining = waiters.filter { !matches($0.scope.connectionId, $0.scope.database) } transportWaiters[connectionId] = remaining.isEmpty ? nil : remaining for waiter in failing { waiter.continuation.resume(throwing: CancellationError()) @@ -324,24 +389,29 @@ final class MetadataConnectionPool { } } - private func acquireEntry(scope: DatabaseScope, workload: Workload) async throws -> Entry { - let key = Key(scope: scope, workload: workload) + // In order: an idle entry on the schema, any idle entry, the busy entry on the schema, an open for the schema + // already dialing, a new entry if there is room, else a place in the room line. A lease inside another lease's + // work skips the busy entry and the open: two such leases nesting in opposite directions would wait on each other. + private func acquireEntry(forKey key: Key, schema: String?, scope: DatabaseScope) async throws -> Entry { + let held = entriesHeldByThisTask() + let openKey = OpenKey(key: key, schema: schema, owner: held.isEmpty ? nil : UUID()) while true { try await waitForTransport(for: scope) - if let entry = reusableEntry(forKey: key) { + if let entry = claimEntry(forKey: key, schema: schema, excluding: held) { return entry } - if pending[key] == nil { - try await waitForRoom(forKey: key) - // A wait can outlast its reason: the key opened meanwhile, or the transport is being rebuilt. - if pending[key] != nil || entries[key] != nil || isReplacingTransport(for: scope.connectionId) { + if pending[openKey] == nil { + try await waitForRoom(forKey: key, schema: schema, held: held) + // A wait can outlast its reason: an entry came free, the schema's open started, or the transport is being rebuilt. + if pending[openKey] != nil || placeWithoutOpening(forKey: key, schema: schema, excluding: held) != nil + || isReplacingTransport(for: key.connectionId) { continue } } - let open = try pending[key] ?? startOpen(forKey: key) - let failure = await completion(of: open, forKey: key) - /// A withdrawn open can end either way: a driver whose connect honours cancellation - /// throws, and one that finished first returns without keeping its entry. + let open = try pending[openKey] ?? startOpen(openKey) + let failure = await completion(of: open, for: openKey) + // A withdrawn open can end either way: a driver whose connect honours cancellation throws, and one that + // finished first returns without keeping its entry. if open.withdrawal == .transportReplaced { try Task.checkCancellation() continue @@ -349,72 +419,170 @@ final class MetadataConnectionPool { if let failure { throw failure } - if let entry = entries[key] { + if let entry = open.entry, isListed(entry, inKey: key) { + claim(entry, for: schema) return entry } if open.withdrawal == .closed { throw DatabaseError.notConnected } - forgetSpentOpen(open, forKey: key) + forgetSpentOpen(open, for: openKey) } } - /// An open whose entry went before this caller could take it has nothing left to hand out. - /// Joining it again returns at once, without a suspension, so a caller that did would spin on - /// the main actor and starve the other callers still waiting to resume from it. - private func forgetSpentOpen(_ open: PendingOpen, forKey key: Key) { - guard pending[key] === open else { return } - pending.removeValue(forKey: key) - handRoom(on: key.scope.connectionId) + private func entriesHeldByThisTask() -> Set { + Set(PooledLeaseContext.open.filter { $0.isOpen }.map { $0.entryId }) } - /// A cached entry is only worth reusing while it is both connected and recent. The - /// staleness half matters even with the sweeper running, because a Mac that slept comes - /// back with entries the sweeper never got to and sockets the server has long since - /// closed. The old entry is closed rather than left behind: overwriting `entries[key]` - /// with a fresh one used to leak the driver it replaced. - private func reusableEntry(forKey key: Key) -> Entry? { - guard let entry = entries[key] else { return nil } - if entry.driver.status == .connected, !entry.driver.hasLostConnection, !Self.isStale(entry.lastUsed) { - return entry + private func claimEntry(forKey key: Key, schema: String?, excluding held: Set) -> Entry? { + // A Mac that slept comes back with entries the sweeper never reached and sockets the server already closed. + for entry in entries[key] ?? [] where !isUsable(entry) { + removeEntry(entry, fromKey: key) } - closeOrDeferEntry(forKey: key) - return nil + guard let entry = placeWithoutOpening(forKey: key, schema: schema, excluding: held) else { return nil } + claim(entry, for: schema) + return entry } - private func startOpen(forKey key: Key) throws -> PendingOpen { - guard DatabaseManager.shared.session(for: key.scope.connectionId) != nil else { + // At most one busy entry per schema, so two schemas share a connection only while one of them is idle. + private func placeWithoutOpening( + forKey key: Key, + schema: String?, + excluding held: Set + ) -> Entry? { + let candidates = (entries[key] ?? []).filter { !held.contains(ObjectIdentifier($0)) && isUsable($0) } + let idle = candidates.filter { isIdle($0) } + if let onSchema = idle.first(where: { $0.schema == schema }) { + return onSchema + } + if let any = idle.min(by: { $0.lastUsed < $1.lastUsed }) { + return any + } + guard held.isEmpty else { return nil } + return candidates.first { $0.inFlightCount > 0 && $0.queuedSchema == schema } + } + + private func claim(_ entry: Entry, for schema: String?) { + entry.inFlightCount += 1 + entry.lastUsed = Date() + entry.queuedSchema = schema + } + + // Runs on the entry's queue before the lease's work, so the switch lands where the work reads. + private func moveEntry(_ entry: Entry, inKey key: Key, to schema: String?) async throws -> Bool { + guard key.slot == .followsSchema, let schema, let switchable = entry.driver as? SchemaSwitchable else { + return true + } + let unconditionally = entry.forcesNextSwitch + // Asked on every lease: Snowflake's drivers for one connection share a session, so another one may have moved it. + if !unconditionally, switchable.currentSchema == schema { + entry.schema = schema + return true + } + // A SET sent inside a transaction is undone by its rollback while the driver keeps the new name. A fresh + // entry's transaction came from the startup commands, and every replacement would carry one too. + if entry.hasServedALease { + let state = await entry.driver.sessionTransactionState() + if state == .inTransaction || state == .abortedTransaction { + Self.logger.info("Closing a pooled connection left in a transaction instead of moving its schema") + removeEntry(entry, fromKey: key) + return false + } + } + do { + try await Self.switchSchema( + entry.driver, + to: schema, + timeoutSeconds: Self.stepTimeoutSeconds(for: key.connectionId), + unconditionally: unconditionally + ) + } catch { + // The server may have run a statement the client gave up on, so the next lease sends its own. + entry.schema = nil + entry.forcesNextSwitch = true + if !Self.isConnected(entry.driver) { + removeEntry(entry, fromKey: key) + } + throw error + } + entry.schema = schema + entry.forcesNextSwitch = false + return true + } + + private static func stepTimeoutSeconds(for connectionId: UUID) -> Double? { + guard let seconds = DatabaseManager.shared.session(for: connectionId)?.effectiveQueryTimeoutSeconds, + seconds > 0 + else { return nil } + return Double(seconds) + } + + // Joining an open that has already finished returns without a suspension, so a caller that rejoined one whose entry + // went would spin on the main actor and starve the callers still waiting to resume from it. + private func forgetSpentOpen(_ open: PendingOpen, for openKey: OpenKey) { + guard pending[openKey] === open else { return } + pending.removeValue(forKey: openKey) + handRoom(on: openKey.key.connectionId) + } + + private static func isConnected(_ driver: DatabaseDriver) -> Bool { + driver.status == .connected && !driver.hasLostConnection + } + + private func isUsable(_ entry: Entry) -> Bool { + Self.isConnected(entry.driver) && !Self.isStale(entry.lastUsed) + } + + private func isListed(_ entry: Entry, inKey key: Key) -> Bool { + entries[key]?.contains { $0 === entry } == true + } + + private func isAwaitingItsOpeners(_ entry: Entry) -> Bool { + pending.values.contains { $0.entry === entry } + } + + private func isIdle(_ entry: Entry) -> Bool { + entry.inFlightCount == 0 && !isAwaitingItsOpeners(entry) + } + + // A new entry opens on the connection's own default and each lease moves it, so it can serve every schema. + private func startOpen(_ openKey: OpenKey) throws -> PendingOpen { + let key = openKey.key + guard let session = DatabaseManager.shared.session(for: key.connectionId) else { throw DatabaseError.notConnected } - evictIdleIfNeeded(for: key.scope.connectionId) - let task = Task { [self] in - let entry = Entry(driver: try await openDriver(key.scope)) + evictIdleIfNeeded(for: key.connectionId) + let scope = DatabaseScope(connectionId: key.connectionId, database: key.database, schema: nil) + let forcesFirstSwitch = key.slot == .followsSchema + && DatabaseManager.hasStartupCommands(session.connection.startupCommands) + let open = PendingOpen() + open.task = Task { [self] in + let entry = Entry(driver: try await openDriver(scope), schema: nil, forcesNextSwitch: forcesFirstSwitch) if Task.isCancelled { entry.driver.disconnect() return } - entries[key] = entry + entries[key, default: []].append(entry) + open.entry = entry startSweeperIfNeeded() } - let open = PendingOpen(task: task) - pending[key] = open + pending[openKey] = open return open } - /// Waits for an open to finish, takes it off the pending list once the last caller waiting on it - /// has, and returns how it failed. - private func completion(of open: PendingOpen, forKey key: Key) async -> Error? { + // Waits for an open to finish, takes it off the pending list once its last caller has, and returns how it failed. + private func completion(of open: PendingOpen, for openKey: OpenKey) async -> Error? { open.waiterCount += 1 defer { open.waiterCount -= 1 - if open.waiterCount == 0, pending[key] === open { - pending.removeValue(forKey: key) + if open.waiterCount == 0, pending[openKey] === open { + pending.removeValue(forKey: openKey) // A failed open frees its place, and a taken one leaves an entry that is now evictable once idle. - handRoom(on: key.scope.connectionId) + handRoom(on: openKey.key.connectionId) } } do { - try await open.task.value + _ = try await open.task?.value return nil } catch { return error @@ -464,10 +632,10 @@ final class MetadataConnectionPool { } // First come first served, interactive ahead of bulk; a caller arriving while others wait goes behind them. - private func waitForRoom(forKey key: Key) async throws { - let connectionId = key.scope.connectionId + private func waitForRoom(forKey key: Key, schema: String?, held: Set) async throws { + let connectionId = key.connectionId guard !PooledLeaseContext.open.contains(where: { $0.isOpen && $0.connectionId == connectionId }) else { return } - let ticket = RoomTicket(key: key) + let ticket = RoomTicket(key: key, schema: schema, held: held) roomQueues[connectionId, default: []].append(ticket) defer { leaveRoomQueue(ticket) } while !mayOpen(ticket) { @@ -477,11 +645,18 @@ final class MetadataConnectionPool { } } - private func mayOpen(_ ticket: RoomTicket) -> Bool { + private func proceedsWithoutAPlace(_ ticket: RoomTicket) -> Bool { let key = ticket.key - let connectionId = key.scope.connectionId - if pending[key] != nil || entries[key] != nil || isReplacingTransport(for: connectionId) { return true } - return nextInLine(on: connectionId) === ticket && hasRoom(for: key.workload, on: connectionId) + let joinsAnOpen = ticket.held.isEmpty && pending[OpenKey(key: key, schema: ticket.schema, owner: nil)] != nil + return joinsAnOpen + || placeWithoutOpening(forKey: key, schema: ticket.schema, excluding: ticket.held) != nil + || isReplacingTransport(for: key.connectionId) + } + + private func mayOpen(_ ticket: RoomTicket) -> Bool { + if proceedsWithoutAPlace(ticket) { return true } + let connectionId = ticket.key.connectionId + return nextInLine(on: connectionId) === ticket && hasRoom(for: ticket.key.workload, on: connectionId) } private func nextInLine(on connectionId: UUID) -> RoomTicket? { @@ -489,18 +664,24 @@ final class MetadataConnectionPool { return line.first { $0.key.workload == .interactive } ?? line.first } - // Wakes only the caller next in line, so a freed place goes to it rather than to whoever resumes first. + // A freed place goes only to the caller next in line. Callers that need no new place go too, and first, so an + // idle entry is reused before the next in line can evict it. private func handRoom(on connectionId: UUID) { - guard let next = nextInLine(on: connectionId), - let continuation = next.continuation, - mayOpen(next) - else { return } - next.continuation = nil - continuation.resume() + let parked = (roomQueues[connectionId] ?? []).filter { $0.continuation != nil } + var woken = parked.filter { proceedsWithoutAPlace($0) } + if let next = nextInLine(on: connectionId), next.continuation != nil, + !woken.contains(where: { $0 === next }), mayOpen(next) { + woken.append(next) + } + for ticket in woken { + guard let continuation = ticket.continuation else { continue } + ticket.continuation = nil + continuation.resume() + } } private func leaveRoomQueue(_ ticket: RoomTicket) { - let connectionId = ticket.key.scope.connectionId + let connectionId = ticket.key.connectionId roomQueues[connectionId]?.removeAll { $0 === ticket } if roomQueues[connectionId]?.isEmpty == true { roomQueues.removeValue(forKey: connectionId) @@ -536,9 +717,9 @@ final class MetadataConnectionPool { } // Same reason as `failTransportWaiters`: a parked caller would otherwise open what was just closed. - private func withdrawRoomTickets(where matches: (DatabaseScope) -> Bool) { + private func withdrawRoomTickets(where matches: (UUID, String) -> Bool) { for line in roomQueues.values { - for ticket in line where matches(ticket.key.scope) { + for ticket in line where matches(ticket.key.connectionId, ticket.key.database) { ticket.isWithdrawn = true failRoomTicket(ticket) } @@ -660,14 +841,23 @@ final class MetadataConnectionPool { } } - static func switchSchema(_ driver: DatabaseDriver, to schema: String, timeoutSeconds: Double?) async throws { + static func switchSchema( + _ driver: DatabaseDriver, + to schema: String, + timeoutSeconds: Double?, + unconditionally: Bool = false + ) async throws { guard let switchable = driver as? SchemaSwitchable else { return } try await bounded( driver: driver, timeoutSeconds: timeoutSeconds, timeoutMessage: String(format: String(localized: "Switching to schema '%@' timed out."), schema) ) { - try await switchable.switchSchemaIfNeeded(to: schema) + if unconditionally { + try await switchable.switchSchema(to: schema) + } else { + try await switchable.switchSchemaIfNeeded(to: schema) + } } } @@ -740,15 +930,16 @@ final class MetadataConnectionPool { sweeper = nil } - /// Hands back every connection nobody has used for `idleTimeout`. An entry with work on it is - /// left alone whatever its age: `lastUsed` is stamped when the work starts, so a long export - /// would otherwise have its own connection closed underneath it. + // An entry with work on it stays whatever its age: `lastUsed` is stamped when the work starts, so a long export + // would otherwise have its own connection closed underneath it. internal func sweepIdleEntries(now: Date = Date()) { var sweptFrom: Set = [] - for (key, entry) in entries where entry.inFlightCount == 0 && Self.isStale(entry.lastUsed, now: now) { - entries.removeValue(forKey: key) - entry.driver.disconnect() - sweptFrom.insert(key.scope.connectionId) + for (key, list) in entries { + for entry in list where entry.inFlightCount == 0 && Self.isStale(entry.lastUsed, now: now) { + detach(entry, fromKey: key) + entry.driver.disconnect() + sweptFrom.insert(key.connectionId) + } } for connectionId in sweptFrom { handRoom(on: connectionId) @@ -765,22 +956,34 @@ final class MetadataConnectionPool { // Bulk never takes the last place, so a completion read or a sidebar expand never waits behind a burst of bulk work. private func hasRoom(for workload: Workload, on connectionId: UUID) -> Bool { let placeIsFree = heldCount(for: connectionId) < Self.maxPerConnection - || leastRecentlyUsedIdleKey(for: connectionId) != nil + || leastRecentlyUsedIdleEntry(for: connectionId) != nil guard workload == .bulk else { return placeIsFree } return placeIsFree && busyCount(for: connectionId) < Self.maxPerConnection - 1 } - private func busyCount(for connectionId: UUID) -> Int { - var keys = Set(pending.keys.filter { $0.scope.connectionId == connectionId }) - for (key, entry) in entries where key.scope.connectionId == connectionId && entry.inFlightCount > 0 { - keys.insert(key) + private func pooled(on connectionId: UUID) -> [(key: Key, entry: Entry)] { + var found: [(key: Key, entry: Entry)] = [] + for (key, list) in entries where key.connectionId == connectionId { + found += list.map { (key: key, entry: $0) } } - return keys.count + return found + } + + // An open still dialing holds a place. One that landed holds it through its entry, and counts once. + private func openingCount(for connectionId: UUID) -> Int { + pending.filter { openKey, open in + guard openKey.key.connectionId == connectionId else { return false } + guard let landed = open.entry else { return true } + return !isListed(landed, inKey: openKey.key) + }.count } - /// Keeps a connection at the limit once its work ends. Closing only one idle entry per open, as - /// the pool used to, left a burst's high-water mark standing until the idle sweep: a tree Refresh - /// across many databases held every connection it opened for ten minutes (#3103). + private func busyCount(for connectionId: UUID) -> Int { + pooled(on: connectionId).filter { !isIdle($0.entry) }.count + openingCount(for: connectionId) + } + + // Closing one idle entry per open, as the pool used to, left a burst's high-water mark standing until the idle + // sweep: a tree Refresh across many databases held every connection it opened for ten minutes (#3103). private func trimIdleEntries(for connectionId: UUID) { while heldCount(for: connectionId) > Self.maxPerConnection { guard evictLeastRecentlyUsedIdleEntry(for: connectionId) else { break } @@ -788,29 +991,22 @@ final class MetadataConnectionPool { stopSweeperIfEmpty() } - /// Every key the connection holds a connection for or is opening one for. Counted by key, so an - /// open that has finished but is still listed until its callers take the entry counts once. private func heldCount(for connectionId: UUID) -> Int { - var keys = Set(entries.keys.filter { $0.scope.connectionId == connectionId }) - keys.formUnion(pending.keys.filter { $0.scope.connectionId == connectionId }) - return keys.count + pooled(on: connectionId).count + openingCount(for: connectionId) } @discardableResult private func evictLeastRecentlyUsedIdleEntry(for connectionId: UUID) -> Bool { - guard let key = leastRecentlyUsedIdleKey(for: connectionId), - let entry = entries.removeValue(forKey: key) + guard let victim = leastRecentlyUsedIdleEntry(for: connectionId), + detach(victim.entry, fromKey: victim.key) else { return false } - entry.driver.disconnect() + victim.entry.driver.disconnect() return true } - /// An entry whose open is still listed has callers that have not taken it yet, so it is not idle - /// however it looks. - private func leastRecentlyUsedIdleKey(for connectionId: UUID) -> Key? { - let idle = entries.filter { key, entry in - key.scope.connectionId == connectionId && entry.inFlightCount == 0 && pending[key] == nil - } - return idle.min(by: { $0.value.lastUsed < $1.value.lastUsed })?.key + private func leastRecentlyUsedIdleEntry(for connectionId: UUID) -> (key: Key, entry: Entry)? { + pooled(on: connectionId) + .filter { isIdle($0.entry) } + .min { $0.entry.lastUsed < $1.entry.lastUsed } } } diff --git a/apple/TablePro/Core/Storage/ConnectionStorage.swift b/apple/TablePro/Core/Storage/ConnectionStorage.swift index 8fadc3d6d4..80ef70f90b 100644 --- a/apple/TablePro/Core/Storage/ConnectionStorage.swift +++ b/apple/TablePro/Core/Storage/ConnectionStorage.swift @@ -789,6 +789,7 @@ final class ConnectionStorage { "cloudflaretokenid", "cloudflaretokensecret", "cloudsqlproxyserviceaccountkey", "socksproxypassword" ] private static let localOnlySecretsMigratedKey = "com.TablePro.localOnlySecretsKeptOnDevice" + private static let pendingSyncedCopyRemovalsKey = "com.TablePro.localOnlySyncedCopiesPendingRemoval" private func secretKeys(for connection: DatabaseConnection) -> [String] { let id = connection.id.uuidString @@ -854,16 +855,27 @@ final class ConnectionStorage { staging?.edits.deletedKeys.append(key) } - // Rewrites each stored secret into the scope the connection now has. Going device-only keeps the copy iCloud - // Keychain holds: removing it would remove it from every device. False when a secret could not be read or written. + // Rewrites each stored secret into the scope the connection now has. removingSyncedCopy then removes the iCloud + // Keychain copy, from every device, but only of a secret this Mac now holds itself. False when a secret could not + // be read or written. @discardableResult - private func rewriteSecrets(of connection: DatabaseConnection, readingOwnItemFirst: Bool, deviceOnly: Bool) -> Bool { + private func rewriteSecrets( + of connection: DatabaseConnection, + readingOwnItemFirst: Bool, + deviceOnly: Bool, + removingSyncedCopy: Bool = false + ) -> Bool { var complete = true for key in secretKeys(for: connection) { switch keychain.readStringResult(forKey: key, deviceOnly: readingOwnItemFirst) { case .found(let value): - if !keychain.writeString(value, forKey: key, deviceOnly: deviceOnly) { + guard keychain.writeString(value, forKey: key, deviceOnly: deviceOnly) else { complete = false + continue + } + if removingSyncedCopy, !keychain.removeSyncedCopy(forKey: key) { + Self.logger.warning("A Local only secret kept its iCloud Keychain copy: \(key, privacy: .public)") + notePendingSyncedCopyRemoval(key, connectionId: connection.id) } case .notFound: break @@ -874,7 +886,8 @@ final class ConnectionStorage { return complete } - // Each direction reads the value the connection was using, so the toggle never changes a secret. + // Each direction reads the value the connection was using, so the toggle never changes a secret. Becoming Local + // only also takes the secrets off iCloud Keychain, so the other devices ask for them again. private func moveSecretsWhoseScopeChanged(from previous: [DatabaseConnection], to current: [DatabaseConnection]) { let wasSynced = Dictionary( previous.map { ($0.id, $0.participatesInSync) }, @@ -882,12 +895,42 @@ final class ConnectionStorage { ) for connection in current { guard let synced = wasSynced[connection.id], synced != connection.participatesInSync else { continue } - rewriteSecrets(of: connection, readingOwnItemFirst: !synced, deviceOnly: synced) + rewriteSecrets(of: connection, readingOwnItemFirst: !synced, deviceOnly: synced, removingSyncedCopy: synced) + } + } + + private func notePendingSyncedCopyRemoval(_ key: String, connectionId: UUID) { + var pending = defaults.dictionary(forKey: Self.pendingSyncedCopyRemovalsKey) as? [String: String] ?? [:] + pending[key] = connectionId.uuidString + defaults.set(pending, forKey: Self.pendingSyncedCopyRemovalsKey) + } + + // A removal the user was promised and the keychain refused is tried again at launch, while the connection is + // still Local only. One that is back in sync, or gone, wants its synced copy kept or already removed it. + func removePendingSyncedCopies() { + guard let pending = defaults.dictionary(forKey: Self.pendingSyncedCopyRemovalsKey) as? [String: String], + !pending.isEmpty else { return } + let connections = loadConnections() + guard !lastLoadFailed else { return } + let byId = Dictionary(connections.map { ($0.id.uuidString, $0) }, uniquingKeysWith: { first, _ in first }) + var remaining: [String: String] = [:] + for (key, idString) in pending { + guard let connection = byId[idString], !connection.participatesInSync else { continue } + if !keychain.removeSyncedCopy(forKey: key) { + remaining[key] = idString + } + } + if remaining.isEmpty { + defaults.removeObject(forKey: Self.pendingSyncedCopyRemovalsKey) + } else { + defaults.set(remaining, forKey: Self.pendingSyncedCopyRemovalsKey) } } - // Builds before this one wrote a Local only connection's secrets wherever the password sync setting pointed. + // Builds before this one wrote a Local only connection's secrets wherever the password sync setting pointed. Only + // copied: the user saw no note saying the other devices lose them. func keepLocalOnlySecretsOnDeviceIfNeeded() { + removePendingSyncedCopies() guard !defaults.bool(forKey: Self.localOnlySecretsMigratedKey) else { return } let connections = loadConnections() guard !lastLoadFailed else { return } diff --git a/apple/TablePro/Core/Storage/KeychainHelper.swift b/apple/TablePro/Core/Storage/KeychainHelper.swift index de76462df9..f62a632250 100644 --- a/apple/TablePro/Core/Storage/KeychainHelper.swift +++ b/apple/TablePro/Core/Storage/KeychainHelper.swift @@ -37,6 +37,9 @@ protocol KeychainStoring: Sendable { func delete(forKey key: String) // deviceOnly removes this Mac's own item and leaves the one iCloud Keychain holds, which every device shares. func delete(forKey key: String, deviceOnly: Bool) + // Removes the iCloud Keychain copy, which removes it from every device. Refused while this Mac has no item of its own. + @discardableResult + func removeSyncedCopy(forKey key: String) -> Bool } extension KeychainStoring { @@ -52,6 +55,12 @@ extension KeychainStoring { func writeString(_ value: String, forKey key: String, deviceOnly: Bool) -> Bool { writeString(value, forKey: key) } + + // A store with one item per key has no synced copy apart from the one it holds. + @discardableResult + func removeSyncedCopy(forKey key: String) -> Bool { + true + } } struct KeychainCopiedItem: Sendable { @@ -251,6 +260,21 @@ final class KeychainHelper: KeychainStoring { } } + @discardableResult + func removeSyncedCopy(forKey key: String) -> Bool { + let own = items.copyItem(itemQuery(forKey: key, synchronizable: false)) + guard own.status == errSecSuccess, own.data != nil else { + log(status: own.status, operation: "find the local copy before removing the synced one", key: key) + return false + } + let status = items.delete(itemQuery(forKey: key, synchronizable: true)) + guard status == errSecSuccess || status == errSecItemNotFound else { + log(status: status, operation: "remove the synced copy", key: key) + return false + } + return true + } + // MARK: - String Convenience @discardableResult diff --git a/apple/TablePro/Models/Query/GridDisplayOrderResolver.swift b/apple/TablePro/Models/Query/GridDisplayOrderResolver.swift index e87b1d2e84..25d3fe0501 100644 --- a/apple/TablePro/Models/Query/GridDisplayOrderResolver.swift +++ b/apple/TablePro/Models/Query/GridDisplayOrderResolver.swift @@ -6,15 +6,12 @@ import Foundation import TableProPluginKit -/// Turns a per-column value filter into the order the grid displays. -/// -/// Pure, so both the grid and the coordinator that has to answer while the grid is unmounted can -/// call it and cannot disagree. Matching is done on the displayed string rather than the raw value, -/// because that is what the filter popover offered the user to pick from. +// Turns a per-column value filter into the order the grid displays. Pure, so the grid and the +// coordinator that answers while the grid is unmounted cannot disagree. Matching is on the displayed +// string rather than the raw value, because that is what the filter popover offered. @MainActor internal enum GridDisplayOrderResolver { - /// - Returns: the displayed row ids, or nil when no filter is active and the display order is - /// the storage order. + // Nil when no filter is active and the display order is the storage order. internal static func resolve( tableRows: TableRows, valueFilter: GridValueFilterState, @@ -38,27 +35,42 @@ internal enum GridDisplayOrderResolver { return result } - /// The displayed form of a cell, which is what a value filter stores and matches against. + // The displayed form of a cell, which is what a value filter stores and matches against. internal static func displayValue( _ rawValue: PluginCellValue, columnType: ColumnType?, displayFormat: ValueDisplayFormat?, databaseType: DatabaseType? + ) -> String { + displayValue( + rawValue, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: DateFormattingService.shared.renderer + ) + } + + nonisolated internal static func displayValue( + _ rawValue: PluginCellValue, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dates: DateDisplayRenderer ) -> String { CellDisplayFormatter.format( rawValue, columnType: columnType, displayFormat: displayFormat, - databaseType: databaseType + databaseType: databaseType, + dates: dates ) ?? rawValue.asText ?? "" } - /// Drops filters whose column has moved or disappeared since they were set. - /// - /// A filter is keyed by column index and remembers the name it was set on, so a result set of a - /// different shape would otherwise apply one column's chosen values to another column's cells. - /// Reading it out rather than mutating keeps this callable from a getter; the mutating - /// `GridValueFilterState.prune` still tidies the stored state at the points that replace rows. + // Drops filters whose column has moved or disappeared since they were set. A filter is keyed by + // column index and remembers the name it was set on, so a result of a different shape would apply + // one column's chosen values to another's cells. Reading it out rather than mutating keeps this + // callable from a getter; `GridValueFilterState.prune` tidies the stored state where rows are replaced. private static func activeFilters( _ valueFilter: GridValueFilterState, columns: [String] diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index de042569e1..59ea7fab79 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -4339,6 +4339,9 @@ }, "%@ May Still Be Running" : { + }, + "%@ more messages." : { + }, "%@ more, in the script beside this" : { "localizations" : { @@ -5419,6 +5422,9 @@ }, "%@ statements" : { + }, + "%@ Statements" : { + }, "%@ stops being a computed column" : { "localizations" : { @@ -13586,6 +13592,9 @@ } } } + }, + "1 more message." : { + }, "1 new column" : { @@ -54557,6 +54566,10 @@ }, "Could not read the table." : { + }, + "Could not read this Mac's trusted certificates to verify the server. Choose the server's CA certificate under SSL, then connect again." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Could not read which rows are already in the table" : { @@ -57661,6 +57674,9 @@ } } } + }, + "cursors" : { + }, "CURTIME()" : { "extractionState" : "stale", @@ -75774,6 +75790,9 @@ } } } + }, + "Exclude from iCloud Sync?" : { + }, "Exclude the AUTO_INCREMENT counter" : { "extractionState" : "stale", @@ -105427,6 +105446,9 @@ }, "Listen for Notifications" : { + }, + "LISTEN registrations" : { + }, "Listens on all interfaces (0.0.0.0), reachable from your local network." : { "localizations" : { @@ -137056,6 +137078,9 @@ } } } + }, + "prepared statements" : { + }, "Preparing Import…" : { "localizations" : { @@ -143018,6 +143043,9 @@ } } } + }, + "Ran %@ statements that returned no rows." : { + }, "ran %@×" : { "comment" : "Run count in a detail line, %@ is a count", @@ -158255,6 +158283,9 @@ } } } + }, + "Saved passwords are removed from iCloud Keychain, and your other devices will need them entered again." : { + }, "Saved passwords can only be exported to an encrypted file." : { "extractionState" : "manual", @@ -158451,6 +158482,9 @@ } } } + }, + "Saving removes its passwords from iCloud Keychain. Your other devices will need them entered again." : { + }, "Saving your choice" : { @@ -164447,6 +164481,12 @@ } } } + }, + "session locks" : { + + }, + "session settings" : { + }, "Session SID %1$d, SERIAL# %2$d. %3$@ A user with ALTER SYSTEM can end it now with %4$@." : { "extractionState" : "manual", @@ -168098,6 +168138,9 @@ } } } + }, + "Showing the last %1$@ of %2$@ results that returned rows." : { + }, "Shown beside each key" : { "localizations" : { @@ -183891,6 +183934,9 @@ }, "Temporary Table" : { + }, + "temporary tables" : { + }, "Teradata uses TLS on its HTTPS port, which the tunnel does not forward. Set SSL Mode to Disabled or Preferred to connect through the tunnel, or turn the tunnel off." : { "extractionState" : "manual", @@ -187376,6 +187422,12 @@ } } } + }, + "The connection to %1$@ was lost while it waited and has been reconnected. The new session does not have what the old one set up: %2$@. Set it up again, then run the statements." : { + + }, + "The connection to %1$@ was lost while it waited, and the server rolled back its open transaction. The new session does not have what the old one set up either: %2$@. Set it up again, then run the statements." : { + }, "The connection to Redis dropped after EXEC was sent, so whether the block ran is unknown." : { "extractionState" : "manual", @@ -189567,6 +189619,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The JSON file has text that is not valid in its encoding." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The JSON file is not valid at byte %@." : { "extractionState" : "manual", "generatesSymbol" : false @@ -192130,6 +192186,12 @@ } } } + }, + "The response stopped at the model's output token limit." : { + + }, + "The response stopped because the conversation filled the model's context window." : { + }, "The REST endpoint is not a valid URL." : { "extractionState" : "manual", @@ -211982,6 +212044,9 @@ } } } + }, + "user variables" : { + }, "User-Defined" : { "localizations" : { @@ -213780,6 +213845,10 @@ } } }, + "Verify CA needs the server's CA certificate. Choose it under SSL, or use Verify Identity to check the server against this Mac's trusted certificates." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Verify certificate and hostname" : { "localizations" : { "fr" : { diff --git a/apple/TablePro/ViewModels/AIChatViewModel+MessageEditing.swift b/apple/TablePro/ViewModels/AIChatViewModel+MessageEditing.swift index 414d5a6e4c..0eac80c09b 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel+MessageEditing.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel+MessageEditing.swift @@ -51,7 +51,8 @@ extension AIChatViewModel { timestamp: snapshot.timestamp, usage: snapshot.usage, modelId: snapshot.modelId, - providerId: snapshot.providerId + providerId: snapshot.providerId, + outputLimit: snapshot.outputLimit ) } diff --git a/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift b/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift index 0e3eca4c1d..b24a0a9a6a 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift @@ -23,6 +23,7 @@ extension AIChatViewModel { let toolUseNames: [String: String] let toolUseInputs: [String: String] let toolUseMetadata: [String: [String: String]] + let outputLimit: ChatOutputLimit? let cancelled: Bool } @@ -221,7 +222,8 @@ extension AIChatViewModel { order: round.toolUseOrder, names: round.toolUseNames, inputs: round.toolUseInputs, - metadata: round.toolUseMetadata + metadata: round.toolUseMetadata, + outputLimit: round.outputLimit ) let context = await MainActor.run { ChatToolContext( @@ -470,6 +472,7 @@ extension AIChatViewModel { var toolUseNames: [String: String] = [:] var toolUseInputs: [String: String] = [:] var toolUseMetadata: [String: [String: String]] = [:] + var outputLimit: ChatOutputLimit? var hasRenderedFirstText = false for try await event in stream { @@ -519,6 +522,9 @@ extension AIChatViewModel { opaque: opaque, idMap: &reasoningIDMap.value ) + case .outputLimitReached(let limit): + outputLimit = limit + turn(withID: assistantID)?.outputLimit = limit } } @@ -527,6 +533,7 @@ extension AIChatViewModel { toolUseNames: toolUseNames, toolUseInputs: toolUseInputs, toolUseMetadata: toolUseMetadata, + outputLimit: outputLimit, cancelled: Task.isCancelled ) } @@ -672,21 +679,29 @@ extension AIChatViewModel { response hit the output token limit. Call the tool again with complete arguments. """ + nonisolated static let cutToolArgumentsContent = """ + This tool call did not run: the response hit the output limit before its arguments were complete. \ + Call the tool again with complete arguments, and keep them shorter if you can. + """ + // A call whose arguments do not parse comes back already denied, so it never reaches approval or execution. + // A response cut at its limit was cut in its last call, whatever arguments a provider that parses them hands over. nonisolated static func assembleToolUseBlocks( order: [String], names: [String: String], inputs: [String: String], - metadata: [String: [String: String]] = [:] + metadata: [String: [String: String]] = [:], + outputLimit: ChatOutputLimit? = nil ) -> [ToolUseBlock] { order.compactMap { id -> ToolUseBlock? in guard let name = names[id] else { return nil } - guard let arguments = toolArguments(from: inputs[id] ?? "") else { + let isCut = outputLimit != nil && id == order.last + guard !isCut, let arguments = toolArguments(from: inputs[id] ?? "") else { return ToolUseBlock( id: id, name: name, input: .object([:]), - approvalState: .denied(reason: unparsableToolArgumentsContent), + approvalState: .denied(reason: isCut ? cutToolArgumentsContent : unparsableToolArgumentsContent), providerMetadata: metadata[id] ) } diff --git a/apple/TablePro/ViewModels/WelcomeViewModel+Library.swift b/apple/TablePro/ViewModels/WelcomeViewModel+Library.swift index 9ac471d344..676d4aa587 100644 --- a/apple/TablePro/ViewModels/WelcomeViewModel+Library.swift +++ b/apple/TablePro/ViewModels/WelcomeViewModel+Library.swift @@ -282,6 +282,21 @@ extension WelcomeViewModel { loadConnections() } + // Leaving sync takes the secrets off iCloud Keychain on every device, so it asks first. + func requestExcludeFromSync(_ ids: [UUID]) { + let targets = ids.compactMap { connectionsById[$0] }.filter(\.participatesInSync) + guard !targets.isEmpty else { return } + connectionsToExcludeFromSync = targets + showExcludeFromSyncConfirmation = true + } + + func confirmExcludeFromSync() { + let ids = connectionsToExcludeFromSync.map(\.id) + connectionsToExcludeFromSync = [] + guard !ids.isEmpty else { return } + setIncludedInSync(ids, included: false) + } + func duplicateConnection(_ id: UUID) { guard let connection = connectionsById[id] else { return } guard let duplicate = storage.duplicateConnection(connection) else { @@ -460,7 +475,11 @@ extension WelcomeViewModel { case .moveToNewGroup(let ids): requestNewGroup(parentId: nil, movingConnectionIds: ids) case .setIncludedInSync(let ids, let included): - setIncludedInSync(ids, included: included) + if included { + setIncludedInSync(ids, included: true) + } else { + requestExcludeFromSync(ids) + } case .deleteConnections(let ids): requestDeleteConnections(ids) case .removeFromRecent(let ids): diff --git a/apple/TablePro/ViewModels/WelcomeViewModel.swift b/apple/TablePro/ViewModels/WelcomeViewModel.swift index ebaefbff7e..1153d64200 100644 --- a/apple/TablePro/ViewModels/WelcomeViewModel.swift +++ b/apple/TablePro/ViewModels/WelcomeViewModel.swift @@ -119,6 +119,8 @@ final class WelcomeViewModel: ObservableObject { @Published var presentsWelcomeSheet = false @Published var connectionsToDelete: [DatabaseConnection] = [] @Published var showDeleteConfirmation = false + @Published var connectionsToExcludeFromSync: [DatabaseConnection] = [] + @Published var showExcludeFromSyncConfirmation = false @Published var pendingDeleteHasFavorites = false private var deleteRequestToken = UUID() @Published var showDeleteGroupConfirmation = false diff --git a/apple/TablePro/Views/AIChat/AIChatMessageView.swift b/apple/TablePro/Views/AIChat/AIChatMessageView.swift index 82960082ce..6622c98621 100644 --- a/apple/TablePro/Views/AIChat/AIChatMessageView.swift +++ b/apple/TablePro/Views/AIChat/AIChatMessageView.swift @@ -105,6 +105,13 @@ struct AIChatMessageView: View, Equatable { } } .padding(.horizontal, 8) + + if let limit = message.outputLimit { + Label(limit.note, systemImage: "exclamationmark.triangle") + .font(.caption) + .foregroundStyle(.secondary) + .padding(.horizontal, 8) + } } if let onRetry { @@ -245,3 +252,14 @@ struct ChatTypingIndicatorView: View { .accessibilityLabel(String(localized: "Responding")) } } + +private extension ChatOutputLimit { + var note: String { + switch self { + case .outputTokens: + return String(localized: "The response stopped at the model's output token limit.") + case .contextWindow: + return String(localized: "The response stopped because the conversation filled the model's context window.") + } + } +} diff --git a/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift b/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift index 3be561720c..963a6e019b 100644 --- a/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift +++ b/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift @@ -57,6 +57,11 @@ final class ConnectionFormCoordinator: ObservableObject { @Published var pluginInstallConnection: DatabaseConnection? @Published var pluginDiagnostic: PluginDiagnosticItem? + // A connection that syncs today and is saved Local only has its secrets taken off iCloud Keychain. + var removesSyncedSecretsOnSave: Bool { + advanced.localOnly && originalConnection?.participatesInSync == true + } + @Published var saveError: String? @Published var isChoosingType: Bool = false diff --git a/apple/TablePro/Views/ConnectionForm/Panes/OptionsPaneView.swift b/apple/TablePro/Views/ConnectionForm/Panes/OptionsPaneView.swift index 84c5296869..c188a6dffb 100644 --- a/apple/TablePro/Views/ConnectionForm/Panes/OptionsPaneView.swift +++ b/apple/TablePro/Views/ConnectionForm/Panes/OptionsPaneView.swift @@ -441,9 +441,14 @@ struct OptionsPaneView: View { } header: { Text(String(localized: "iCloud Sync")) } footer: { - Text("This connection won't sync to other devices via iCloud.") - .font(.caption) - .foregroundStyle(.secondary) + VStack(alignment: .leading, spacing: 4) { + Text("This connection won't sync to other devices via iCloud.") + if coordinator.removesSyncedSecretsOnSave { + Text("Saving removes its passwords from iCloud Keychain. Your other devices will need them entered again.") + } + } + .font(.caption) + .foregroundStyle(.secondary) } } } diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift index e4a9b39fb1..e56a98cd32 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift @@ -9,10 +9,69 @@ import TableProPluginKit // Each loaded value as the cell shows it, counted, in the order first seen. struct ColumnValueTally: Sendable { + static let cancellationInterval = 4_096 + var counts: [String: Int] = [:] var order: [String] = [] var nullCount = 0 + // Nil once cancelled, which is asked every 4,096 rows. + static func tally( + _ rows: TableRows, + dataIndex: Int, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dates: DateDisplayRenderer, + isCancelled: () -> Bool + ) -> ColumnValueTally? { + var counted = ColumnValueTally() + guard dataIndex >= 0, dataIndex < rows.columns.count else { return counted } + for (offset, row) in rows.rows.enumerated() { + if offset % cancellationInterval == 0, isCancelled() { return nil } + guard dataIndex < row.values.count else { continue } + let raw = row.values[dataIndex] + if case .null = raw { + counted.nullCount += 1 + continue + } + let display = GridDisplayOrderResolver.displayValue( + raw, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: dates + ) + if counted.counts[display] == nil { + counted.order.append(display) + } + counted.counts[display, default: 0] += 1 + } + return counted + } + + // A column of unique values costs a format per row and n log n locale-aware comparisons, too slow for the + // main actor. A renderer is never shared across threads, so this one builds its own from the format. + @concurrent + static func distinctValuesOffTheMainActor( + _ rows: TableRows, + dataIndex: Int, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dateFormat: DateFormatOption + ) async -> [ColumnDistinctValue]? { + tally( + rows, + dataIndex: dataIndex, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: DateDisplayRenderer(format: dateFormat), + isCancelled: { Task.isCancelled } + )?.sorted() + } + func sorted() -> [ColumnDistinctValue] { var values = order .sorted { $0.localizedStandardCompare($1) == .orderedAscending } @@ -22,47 +81,20 @@ struct ColumnValueTally: Sendable { } return values } - - // A column of unique values makes n log n locale-aware comparisons, too slow for the main actor. - @concurrent - func sortedOffTheMainActor() async -> [ColumnDistinctValue] { - sorted() - } } extension DataGridCommands { // Every loaded value as the cell shows it, which is what a value filter stores and matches; NULL first. func distinctValues(for column: DataGridColumn) -> [ColumnDistinctValue] { - valueTally(for: column).sorted() - } - - // Formatting reads main-actor state, so the tally is taken here and only the sort leaves the main actor. - func valueTally(for column: DataGridColumn) -> ColumnValueTally { - let rows = model.rows - let dataIndex = column.dataIndex - guard dataIndex >= 0, dataIndex < rows.columns.count else { return ColumnValueTally() } - let format = model.displayFormat(for: column) - let databaseType = configuration.traits.databaseType - var tally = ColumnValueTally() - for row in rows.rows { - guard dataIndex < row.values.count else { continue } - let raw = row.values[dataIndex] - if case .null = raw { - tally.nullCount += 1 - continue - } - let display = GridDisplayOrderResolver.displayValue( - raw, - columnType: column.type, - displayFormat: format, - databaseType: databaseType - ) - if tally.counts[display] == nil { - tally.order.append(display) - } - tally.counts[display, default: 0] += 1 - } - return tally + ColumnValueTally.tally( + model.rows, + dataIndex: column.dataIndex, + columnType: column.type, + displayFormat: model.displayFormat(for: column), + databaseType: configuration.traits.databaseType, + dates: DateFormattingService.shared.renderer, + isCancelled: { false } + )?.sorted() ?? [] } } @@ -80,15 +112,25 @@ extension DataGridViewController { }) else { return } guard model.anchor(for: column) != nil else { return } - let tally = gridCommands.valueTally(for: column) + let columnType = column.type + let displayFormat = model.displayFormat(for: column) + let databaseType = model.configuration.traits.databaseType + let dateFormat = DateFormattingService.shared.currentFormat let initialFilter = model.store.valueFilter.filter(forColumn: dataIndex) let loadedRowCount = rows.count let listedModel = model dismissValueFilterPopover() - // A reload, insert or removal while the list sorts cancels it, as it would close the open popover. + // A reload, insert or removal while the list builds cancels it, as it would close the open popover. menuState.valueFilterListing = Task { [weak self] in - let values = await tally.sortedOffTheMainActor() - guard !Task.isCancelled, let self else { return } + let listed = await ColumnValueTally.distinctValuesOffTheMainActor( + rows, + dataIndex: dataIndex, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dateFormat: dateFormat + ) + guard let values = listed, !Task.isCancelled, let self else { return } self.menuState.valueFilterListing = nil // A grid that took another tab's model meanwhile shows other rows. guard self.model === listedModel, diff --git a/apple/TablePro/Views/DataGrid/DataGridFindBar.swift b/apple/TablePro/Views/DataGrid/DataGridFindBar.swift index 2879242aea..2e4752a173 100644 --- a/apple/TablePro/Views/DataGrid/DataGridFindBar.swift +++ b/apple/TablePro/Views/DataGrid/DataGridFindBar.swift @@ -172,11 +172,21 @@ struct DataGridFindBar: View { // MARK: - State private var counterText: String { + Self.counterText(status: status, model: model) + } + + private var showsSearchAllRows: Bool { + Self.showsSearchAllRows(offered: capabilities.offersSearchAllRows, model: model) + } + + static func counterText(status: String?, model: DataGridModel) -> String { + let find = model.find if let status { return status } if let problem = find.problem { return problem } guard model.hasFindTerm else { return "" } // A search that refreshes keeps showing the count it is about to replace. if find.matches.isEmpty { + if find.isAwaitingLoadedRows { return "" } if find.isSearching { return String(localized: "Searching…") } if find.searchStopped { return String(localized: "Search stopped") } } @@ -188,9 +198,10 @@ struct DataGridFindBar: View { } // Only when this page has no match, so the bar stays short while matches are on screen. - private var showsSearchAllRows: Bool { - capabilities.offersSearchAllRows && model.searchAllRowsKeepsTheOptions && model.hasFindTerm - && find.problem == nil && find.matches.isEmpty + static func showsSearchAllRows(offered: Bool, model: DataGridModel) -> Bool { + let find = model.find + return offered && model.searchAllRowsKeepsTheOptions && model.hasFindTerm + && find.problem == nil && find.matches.isEmpty && !find.isAwaitingLoadedRows } // Use Selection for Find searches on its own, so only typing goes through the delay. diff --git a/apple/TablePro/Views/Main/MainContentCoordinator.swift b/apple/TablePro/Views/Main/MainContentCoordinator.swift index 2c2c39815a..d603b2f471 100644 --- a/apple/TablePro/Views/Main/MainContentCoordinator.swift +++ b/apple/TablePro/Views/Main/MainContentCoordinator.swift @@ -1292,7 +1292,9 @@ final class MainContentCoordinator: ObservableObject { scope: scope, isTableTab: isTableTab, lease: lease - ) { [queryExecutor] driver in + ) { [queryExecutor, manager = services.databaseManager] driver in + // Before it runs, so a statement that fails or a connection lost under it is still noted. + if !isAutoLoad { await manager.noteSessionStatements([statement.sql], ranOn: driver, scope: scope) } let fetched = try await queryExecutor.executeQuery( driver: driver, sql: statement.sql, diff --git a/apple/TablePro/Views/Welcome/WelcomePresentations.swift b/apple/TablePro/Views/Welcome/WelcomePresentations.swift index 4a1379516b..3913fea692 100644 --- a/apple/TablePro/Views/Welcome/WelcomePresentations.swift +++ b/apple/TablePro/Views/Welcome/WelcomePresentations.swift @@ -137,6 +137,19 @@ private struct WelcomeDeletionAlerts: ViewModifier { } message: { deletionMessage } + .alert( + String(localized: "Exclude from iCloud Sync?"), + isPresented: $vm.showExcludeFromSyncConfirmation + ) { + Button(String(localized: "Exclude"), role: .destructive) { + vm.confirmExcludeFromSync() + } + Button(String(localized: "Cancel"), role: .cancel) { + vm.connectionsToExcludeFromSync = [] + } + } message: { + Text("Saved passwords are removed from iCloud Keychain, and your other devices will need them entered again.") + } } @ViewBuilder diff --git a/apple/TableProMobile/TableProMobile/Localizable.xcstrings b/apple/TableProMobile/TableProMobile/Localizable.xcstrings index 8a20343c3c..8032af820e 100644 --- a/apple/TableProMobile/TableProMobile/Localizable.xcstrings +++ b/apple/TableProMobile/TableProMobile/Localizable.xcstrings @@ -7217,6 +7217,10 @@ } } }, + "Could not read this Mac's trusted certificates to verify the server. Choose the server's CA certificate under SSL, then connect again." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Could not run the credential_process command for profile \"%@\": %@" : { "extractionState" : "manual", "generatesSymbol" : false, diff --git a/apple/TableProTests/Core/AI/OutputLimitSignalTests.swift b/apple/TableProTests/Core/AI/OutputLimitSignalTests.swift new file mode 100644 index 0000000000..cf91ae70ea --- /dev/null +++ b/apple/TableProTests/Core/AI/OutputLimitSignalTests.swift @@ -0,0 +1,177 @@ +// +// OutputLimitSignalTests.swift +// TableProTests +// + +import Foundation +import TableProMCP +@testable import TablePro +import Testing + +private func limits(_ events: [ChatStreamEvent]) -> [ChatOutputLimit] { + events.compactMap { event in + if case .outputLimitReached(let limit) = event { return limit } + return nil + } +} + +struct OutputLimitSignalTests { + @Test("Anthropic message_delta stop reasons that cut the reply are reported", arguments: [ + ("max_tokens", [ChatOutputLimit.outputTokens]), + ("model_context_window_exceeded", [.contextWindow]), + ("end_turn", []), + ("tool_use", []), + ] as [(String, [ChatOutputLimit])]) + func anthropicStopReason(reason: String, expected: [ChatOutputLimit]) throws { + var state = AnthropicStreamState() + let events = try AnthropicProvider.parseChunk([ + "type": "message_delta", + "delta": ["stop_reason": reason], + "usage": ["output_tokens": 12], + ], state: &state) + #expect(limits(events) == expected) + #expect(state.outputTokens == 12) + } + + @Test("Chat Completions finish_reason length is reported, stop is not", arguments: [ + ("length", [ChatOutputLimit.outputTokens]), + ("stop", []), + ("tool_calls", []), + ] as [(String, [ChatOutputLimit])]) + func chatCompletionsFinishReason(reason: String, expected: [ChatOutputLimit]) { + var state = OpenAIStreamState() + let result = OpenAICompatibleProvider.parseChunk([ + "choices": [["delta": [:] as [String: Any], "finish_reason": reason]], + ], state: &state) + #expect(limits(result.events) == expected) + } + + @Test("Ollama's done_reason length is reported") + func ollamaDoneReason() { + var state = OpenAIStreamState() + let result = OpenAICompatibleProvider.parseChunk([ + "message": ["content": ""], + "done": true, + "done_reason": "length", + ], state: &state) + #expect(limits(result.events) == [.outputTokens]) + } + + @Test("A Responses stream that ends incomplete at max_output_tokens is reported, with its usage") + func responsesIncomplete() throws { + var state = ResponsesStreamState() + let events = try OpenAIResponsesProvider.parseEvent([ + "type": "response.incomplete", + "response": [ + "status": "incomplete", + "incomplete_details": ["reason": "max_output_tokens"], + "usage": ["input_tokens": 30, "output_tokens": 64], + ], + ], state: &state) + #expect(limits(events) == [.outputTokens]) + #expect(state.inputTokens == 30) + #expect(state.outputTokens == 64) + } + + @Test("A Responses stream cut by its content filter is not an output limit") + func responsesContentFilter() throws { + var state = ResponsesStreamState() + let events = try OpenAIResponsesProvider.parseEvent([ + "type": "response.incomplete", + "response": ["incomplete_details": ["reason": "content_filter"]], + ], state: &state) + #expect(limits(events).isEmpty) + } + + @Test("Gemini finishReason MAX_TOKENS is reported, STOP is not", arguments: [ + ("MAX_TOKENS", [ChatOutputLimit.outputTokens]), + ("STOP", []), + ] as [(String, [ChatOutputLimit])]) + func geminiFinishReason(reason: String, expected: [ChatOutputLimit]) { + var state = GeminiStreamState() + let events = GeminiProvider.parseChunk([ + "candidates": [["content": ["parts": [["text": "partial"]]], "finishReason": reason]], + ], state: &state, idGenerator: { "id" }) + #expect(limits(events) == expected) + } + + @Test("Claude Code's passed-through message_delta reports the cut") + func claudeAgentStopReason() { + let line = #"{"type":"stream_event","event":{"type":"message_delta","delta":{"stop_reason":"max_tokens"}}}"# + #expect(ClaudeAgentProvider.decodeEvent(line) == .outputLimit(.outputTokens)) + let ended = #"{"type":"stream_event","event":{"type":"message_delta","delta":{"stop_reason":"end_turn"}}}"# + #expect(ClaudeAgentProvider.decodeEvent(ended) == nil) + } + + @Test("At the output limit the last call is denied even with no arguments yet, and complete calls before it stand") + func cutLastCallIsDenied() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1", "call_2"], + names: ["call_1": "list_tables", "call_2": "execute_query"], + inputs: ["call_1": #"{"schema":"public"}"#, "call_2": ""], + outputLimit: .outputTokens + ) + #expect(blocks.count == 2) + #expect(blocks[0].approvalState == .approved) + #expect(blocks[0].input == .object(["schema": .string("public")])) + #expect(blocks[1].approvalState == .denied(reason: AIChatViewModel.cutToolArgumentsContent)) + } + + @Test("At the output limit cut arguments name the limit in the error the model gets") + func cutArgumentsNameTheLimit() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1"], + names: ["call_1": "execute_query"], + inputs: ["call_1": #"{"query":"SELECT * FR"#], + outputLimit: .outputTokens + ) + #expect(blocks.first?.approvalState == .denied(reason: AIChatViewModel.cutToolArgumentsContent)) + } + + @Test("At the output limit the last call is denied even when its arguments parse, as Gemini and Ollama hand them over") + func cutLastCallIsDeniedWithParsedArguments() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1"], + names: ["call_1": "execute_query"], + inputs: ["call_1": "{}"], + outputLimit: .outputTokens + ) + #expect(blocks.first?.approvalState == .denied(reason: AIChatViewModel.cutToolArgumentsContent)) + } + + @Test("At the output limit a malformed call before the last keeps the plain reason") + func earlierMalformedCallKeepsThePlainReason() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1", "call_2"], + names: ["call_1": "describe_table", "call_2": "list_tables"], + inputs: ["call_1": #"{"table":"#, "call_2": "{}"], + outputLimit: .outputTokens + ) + #expect(blocks[0].approvalState == .denied(reason: AIChatViewModel.unparsableToolArgumentsContent)) + #expect(blocks[1].approvalState == .denied(reason: AIChatViewModel.cutToolArgumentsContent)) + } + + @Test("Without the limit a call with no arguments still runs with none") + func noLimitKeepsEmptyArguments() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1"], + names: ["call_1": "list_tables"], + inputs: ["call_1": ""] + ) + #expect(blocks.first?.approvalState == .approved) + } + + @Test("A turn keeps its output limit through the saved conversation, and an unknown value reads as none") + func outputLimitPersists() throws { + let wire = ChatTurnWire(role: .assistant, blocks: [.text("cut")], outputLimit: .contextWindow) + let data = try JSONEncoder().encode(wire) + #expect(try JSONDecoder().decode(ChatTurnWire.self, from: data).outputLimit == .contextWindow) + + var object = try #require(try JSONSerialization.jsonObject(with: data) as? [String: Any]) + object["outputLimit"] = "somethingNew" + let future = try JSONSerialization.data(withJSONObject: object) + let decoded = try JSONDecoder().decode(ChatTurnWire.self, from: future) + #expect(decoded.outputLimit == nil) + #expect(decoded.plainText == "cut") + } +} diff --git a/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift b/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift index ad9c23b09f..f7bd3cfbb5 100644 --- a/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift +++ b/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift @@ -387,11 +387,15 @@ final class MockDatabaseDriver: DatabaseDriver, SchemaSwitchable, @unchecked Sen func commitTransaction() async throws { commitTransactionCallCount += 1 } func rollbackTransaction() async throws {} + // A switch to this schema fails, the way a server refuses a schema it does not have. + var switchSchemaError: (schema: String, error: any Error)? + func switchSchema(to schema: String) async throws { await onSwitchSchema?() if switchSchemaDelaySeconds > 0 { try await Task.sleep(nanoseconds: UInt64(switchSchemaDelaySeconds * 1_000_000_000)) } + if let failure = switchSchemaError, failure.schema == schema { throw failure.error } switchSchemaCallCount += 1 currentSchema = schema } diff --git a/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift b/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift index 5abc551188..fff4ca6e57 100644 --- a/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift @@ -249,4 +249,359 @@ struct UpdateSaveOrderTests { #expect(pending.change(forRow: .existing(0), type: .update)?.sequence == sequence) #expect(pending.isCellModified(rowID: .existing(0), columnIndex: 1)) } + + @Test("A row edited again after another row's delete is written after that delete") + func laterEditRunsAfterTheDeleteBeforeIt() throws { + let (manager, _) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + + let statements = try manager.generateSQL() + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE", "UPDATE"]) + try #require(statements.count == 3) + #expect(EditAroundADelete.strings(statements[0]) == ["Cyd", "3"]) + #expect(EditAroundADelete.strings(statements[1]) == ["2"]) + #expect(EditAroundADelete.strings(statements[2]) == ["b@test.com", "3"]) + } + + @Test("A swap through a temporary value saves each step in the order it was made") + func swapThroughATemporaryValueSavesInOrder() throws { + let (manager, _) = EditAroundADelete.manager() + let rowA: [PluginCellValue] = ["1", "Al", "a@test.com"] + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "email", + oldValue: "a@test.com", newValue: "tmp", originalRow: rowA + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 2, columnName: "email", + oldValue: "b@test.com", newValue: "a@test.com", originalRow: EditAroundADelete.rowB + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "email", + oldValue: "tmp", newValue: "b@test.com", originalRow: ["1", "Al", "tmp"] + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "UPDATE", "UPDATE"]) + #expect(bound == [["tmp", "1"], ["a@test.com", "2"], ["b@test.com", "1"]]) + } + + @Test("A later edit of a row keys on the key an earlier edit of it wrote") + func laterRunKeysOnTheKeyTheEarlierRunWrote() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 0, columnName: "id", + oldValue: "3", newValue: "6", originalRow: EditAroundADelete.rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: ["6", "Cy", "c@test.com"] + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE", "UPDATE"]) + #expect(bound == [["6", "3"], ["2"], ["Cyd", "6"]]) + } + + @Test("A tab switch keeps the place of every edit of a row, so a later change sorts after them") + func aTabSwitchKeepsEveryRunsPlace() throws { + let (manager, _) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + let state = manager.saveState() + + manager.restoreState(from: state, tableName: "users", databaseType: .mysql, generatedColumns: []) + manager.recordRowInsertion(rowID: .inserted(UUID()), values: ["9", "Di", "d@test.com"]) + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["UPDATE", "DELETE", "UPDATE", "INSERT"]) + } + + @Test("Undoing the delete of a row edited around another delete brings back both of its edits in place") + func undoingADeleteBringsBackEveryRun() throws { + let (manager, _) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + manager.recordRowDeletion(rowID: .existing(2), originalRow: ["3", "Cyd", "b@test.com"]) + + manager.undoRowDeletion(rowID: .existing(2)) + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["UPDATE", "DELETE", "UPDATE"]) + } + + @Test("Two edits of a row with nothing between them stay one UPDATE") + func editsWithNothingBetweenStayOneUpdate() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: EditAroundADelete.rowC + ) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "c@test.com", newValue: "x@test.com", originalRow: ["3", "Cyd", "c@test.com"] + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE"]) + #expect(bound.first == ["Cyd", "x@test.com", "3"]) + } + + @Test("A cell typed back to its value as read is not written, whatever came between") + func aCellTypedBackToItsValueAsReadIsNotWritten() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "c@test.com", newValue: "x@test.com", originalRow: EditAroundADelete.rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "x@test.com", newValue: "c@test.com", originalRow: ["3", "Cy", "x@test.com"] + ) + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["DELETE"]) + } + + @Test("Edits of a row around a change that was then cancelled are one UPDATE") + func aChangeCancelledBetweenRunsLeavesOneUpdate() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: EditAroundADelete.rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.undoRowDeletion(rowID: .existing(1)) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "c@test.com", newValue: "x@test.com", originalRow: ["3", "Cyd", "c@test.com"] + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE"]) + #expect(bound == [["Cyd", "x@test.com", "3"]]) + } + + @Test("A row whose key an earlier edit set to SQL stays one UPDATE keyed on the key as read") + func aKeySetToSQLBeforeAnotherChangeKeepsTheRowWhole() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 0, columnName: "id", oldValue: "3", newValue: "NOW()", + originalRow: EditAroundADelete.rowC, absence: FieldAbsence(), isExpression: true + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: ["NOW()", "Cy", "c@test.com"] + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE"]) + #expect(statements.first?.sql.contains("= NOW()") == true) + #expect(bound.first == ["Cyd", "3"]) + } + + @Test("A key set to SQL by a row's last edit still saves that edit in its own place") + func aKeySetToSQLInTheLastEditStillSplits() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: EditAroundADelete.rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 0, columnName: "id", oldValue: "3", newValue: "NOW()", + originalRow: ["3", "Cyd", "c@test.com"], absence: FieldAbsence(), isExpression: true + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE", "UPDATE"]) + #expect(bound == [["Cyd", "3"], ["2"], ["3"]]) + } + + @Test("A row of a table with no primary key stays one UPDATE at its first edit's place") + func aKeylessRowStaysOneUpdate() throws { + let (manager, _) = EditAroundADelete.manager(primaryKeyColumns: []) + EditAroundADelete.record(on: manager) + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["UPDATE", "DELETE"]) + } +} + +@MainActor +struct UpdateRunActionTests { + @Test("One command over several rows writes each row once, whatever its other rows' changes") + func oneCommandOverSeveralRowsIsOneUpdatePerRow() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordRowDeletion(rowID: .existing(3), originalRow: ["4", "Di", "d@test.com"]) + manager.recordAsOneAction { + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Al", newValue: "Alf", originalRow: ["1", "Al", "a@test.com"] + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 1, columnName: "name", + oldValue: "Bo", newValue: "Bob", originalRow: EditAroundADelete.rowB + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "email", + oldValue: "a@test.com", newValue: "alf@test.com", originalRow: ["1", "Alf", "a@test.com"] + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 2, columnName: "email", + oldValue: "b@test.com", newValue: "bob@test.com", originalRow: ["2", "Bob", "b@test.com"] + ) + } + manager.recordRowDeletion(rowID: .existing(4), originalRow: ["5", "Ed", "e@test.com"]) + + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["DELETE", "UPDATE", "UPDATE", "DELETE"]) + } + + @Test("Typing into several rows at once writes each row once") + func typingIntoSeveralRowsIsOneUpdatePerRow() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordTypedCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Al", newValue: "A", originalRow: ["1", "Al", "a@test.com"] + ) + manager.recordTypedCellChange( + rowID: .existing(1), columnIndex: 1, columnName: "name", + oldValue: "Bo", newValue: "A", originalRow: EditAroundADelete.rowB + ) + manager.recordTypedCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "A", newValue: "Ab", originalRow: ["1", "A", "a@test.com"] + ) + manager.endCoalescedUndoRun() + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["UPDATE", "UPDATE"]) + } +} + +@MainActor +struct UpdateRunUndoTests { + @Test("Redoing a row's later edit puts it back after the delete it followed") + func redoPutsTheLaterEditBackAfterTheDelete() throws { + let (manager, undoManager) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + + undoManager.undo() + undoManager.redo() + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE", "UPDATE"]) + #expect(bound.last == ["b@test.com", "3"]) + } + + @Test("Undoing a row's later edit leaves its earlier one where it was") + func undoingTheLaterEditLeavesTheEarlierUpdate() throws { + let (manager, undoManager) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + + undoManager.undo() + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE"]) + #expect(bound.first == ["Cyd", "3"]) + } + + @Test("Undoing and redoing a command over two rows still writes each row once") + func undoAndRedoOfACommandKeepOneUpdatePerRow() throws { + let (manager, undoManager) = EditAroundADelete.manager() + manager.recordRowDeletion(rowID: .existing(3), originalRow: ["4", "Di", "d@test.com"]) + undoManager.beginUndoGrouping() + manager.recordAsOneAction { + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Al", newValue: "Alf", originalRow: ["1", "Al", "a@test.com"] + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 1, columnName: "name", + oldValue: "Bo", newValue: "Bob", originalRow: EditAroundADelete.rowB + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "email", + oldValue: "a@test.com", newValue: "alf@test.com", originalRow: ["1", "Alf", "a@test.com"] + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 2, columnName: "email", + oldValue: "b@test.com", newValue: "bob@test.com", originalRow: ["2", "Bob", "b@test.com"] + ) + } + undoManager.endUndoGrouping() + + undoManager.undo() + let undone = try EditAroundADelete.verbs(manager.generateSQL()) + undoManager.redo() + let redone = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(undone == ["DELETE"]) + #expect(redone == ["DELETE", "UPDATE", "UPDATE"]) + } +} + +// Edit C, delete B, then give C the email B freed: each step is valid on a unique `email`. +@MainActor +enum EditAroundADelete { + static let columns = ["id", "name", "email"] + static let rowB: [PluginCellValue] = ["2", "Bo", "b@test.com"] + static let rowC: [PluginCellValue] = ["3", "Cy", "c@test.com"] + + static func manager( + table: String = "users", + primaryKeyColumns: [String] = ["id"], + databaseType: DatabaseType = .mysql + ) -> (DataChangeManager, UndoManager) { + let manager = DataChangeManager() + let undoManager = UndoManager() + undoManager.groupsByEvent = false + manager.undoManagerProvider = { undoManager } + manager.configureForTable( + tableName: table, + columns: columns, + primaryKeyColumns: primaryKeyColumns, + databaseType: databaseType, + generatedColumns: [] + ) + return (manager, undoManager) + } + + static func record(on manager: DataChangeManager) { + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "c@test.com", newValue: "b@test.com", originalRow: ["3", "Cyd", "c@test.com"] + ) + } + + nonisolated static func verbs(_ statements: [ParameterizedStatement]) -> [String] { + statements.map { String($0.sql.prefix(while: { $0 != " " })) } + } + + nonisolated static func strings(_ statement: ParameterizedStatement) -> [String] { + statement.parameters.compactMap { $0 as? String } + } } diff --git a/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift b/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift index 32b9aea657..a965934eae 100644 --- a/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift +++ b/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift @@ -95,12 +95,14 @@ struct FindCoordinatorRoutingTests { } @Test("Find Next and Find Previous step the selected tab's matches") - func stepsTheSelectedTabsMatches() throws { + func stepsTheSelectedTabsMatches() async throws { let (coordinator, first, _) = makeTabs() let model = coordinator.gridModel(forTab: first) coordinator.findCoordinator.showFind() model.find.term = "ali" model.findQueryDidChange() + #expect(coordinator.findCoordinator.hasActiveFind, "Find Next waits for the running search instead of going to the editor") + await model.findSearchSettled() #expect(model.find.matches.count == 2) #expect(coordinator.findCoordinator.hasActiveFind) let start = try #require(model.find.currentIndex) diff --git a/apple/TableProTests/Core/Coordinators/RunResultPlanTests.swift b/apple/TableProTests/Core/Coordinators/RunResultPlanTests.swift new file mode 100644 index 0000000000..89855fefbd --- /dev/null +++ b/apple/TableProTests/Core/Coordinators/RunResultPlanTests.swift @@ -0,0 +1,110 @@ +// +// RunResultPlanTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import TableProTestSupport +@testable import TablePro +import Testing + +struct RunResultPlanTests { + @Test("Statements that return no rows share one summary placed where the last of them ran") + func columnlessStatementsCollapse() { + let plan = RunResultPlan(producesRows: [false, false, true, false, true]) + #expect(plan.entries == [.statement(2), .summary([0, 1, 3]), .statement(4)]) + #expect(plan.droppedNotice == nil) + } + + @Test("A single statement without rows keeps its own result") + func singleColumnlessStatementStays() { + let plan = RunResultPlan(producesRows: [false, true]) + #expect(plan.entries == [.statement(0), .statement(1)]) + } + + @Test("A run that ends on a statement without rows ends on the summary") + func summaryIsLastWhenTheRunEndsWithoutRows() { + let plan = RunResultPlan(producesRows: [true, false, false]) + #expect(plan.entries.last == .summary([1, 2])) + } + + @Test("Only the last results that return rows are kept, and only their rows are built") + func rowResultsAreCapped() { + let plan = RunResultPlan(producesRows: Array(repeating: true, count: 5), maxRowResults: 2) + #expect(plan.entries == [.statement(3), .statement(4)]) + #expect(plan.droppedRowResultCount == 3) + #expect(plan.showsRows(ofStatement: 4)) + #expect(!plan.showsRows(ofStatement: 0)) + #expect(plan.droppedNotice != nil) + } + + @Test("A long script of writes becomes one result") + func longWriteScriptIsOneResult() { + let plan = RunResultPlan(producesRows: Array(repeating: false, count: 10_000)) + #expect(plan.entries.count == 1) + #expect(!plan.showsRows(ofStatement: 0)) + } +} + +@MainActor +struct RunAllSummaryResultTests { + @Test("Run All of statements that return no rows shows one summary result") + func runAllCollapsesWrites() async throws { + let connection = TestFixtures.makeConnection() + let driver = MockDatabaseDriver(connection: connection) + var session = ConnectionSession(connection: connection, driver: driver) + session.status = .connected + session.browseDatabase = connection.database + DatabaseManager.shared.injectSession(session, for: connection.id) + let tabManager = QueryTabManager() + let coordinator = MainContentCoordinator( + connection: connection, + tabManager: tabManager, + changeManager: DataChangeManager(), + toolbarState: ConnectionToolbarState() + ) + defer { + coordinator.cancelAllQueryTasks() + coordinator.teardown() + DatabaseManager.shared.removeSession(for: connection.id) + } + let script = "CREATE TABLE a (x int);\nINSERT INTO a VALUES (1);\nUPDATE a SET x = 2;" + let tab = QueryTab(title: "Query", query: script, tabType: .query) + tabManager.tabs.append(tab) + tabManager.selectedTabId = tab.id + + coordinator.runAllStatements() + try await waitUntil { + !coordinator.tabExecution.isExecuting(tab.id) && tabManager.tabs.first?.display.resultSets.isEmpty == false + } + + let results = try #require(tabManager.tabs.first?.display.resultSets) + #expect(results.count == 1) + let summary = try #require(results.first) + #expect(summary.resultColumns.isEmpty) + #expect(summary.statusMessage?.contains("3") == true) + #expect(summary.statementAnchor == nil) + } + + @Test("The summary adds up rows affected and time, keeps each driver message once and the output in order") + func summaryCarriesWhatTheStatementsReported() { + let fixture = CoordinatorFixture() + let parts = [ + RunSummaryPart(rowsAffected: 2, executionTime: 0.5, serverOutput: PluginServerOutput(lines: ["a"], isTruncated: false), statusMessage: "INSERT: 2 rows"), + RunSummaryPart(rowsAffected: 3, executionTime: 0.25, serverOutput: PluginServerOutput(lines: ["b"], isTruncated: true), statusMessage: "Still creating"), + RunSummaryPart(rowsAffected: 0, executionTime: 0.25, serverOutput: .none, statusMessage: "Still creating"), + ] + + let summary = fixture.coordinator.queryExecutionCoordinator.makeRunSummaryResultSet(parts) + + #expect(summary.rowsAffected == 5) + #expect(summary.executionTime == 1) + #expect(summary.serverOutput == PluginServerOutput(lines: ["a", "b"], isTruncated: true)) + let status = summary.statusMessage ?? "" + #expect(status.contains("INSERT: 2 rows")) + #expect(status.components(separatedBy: "Still creating").count == 2) + #expect(summary.resultColumns.isEmpty) + #expect(summary.origin == nil) + } +} diff --git a/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift b/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift index 35cc1d7d7f..1017472355 100644 --- a/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift +++ b/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift @@ -294,6 +294,125 @@ struct GridFindEngineTests { #expect(search("8950", in: source).matches.isEmpty) } + // MARK: - Off the main actor + + private static let shownValues = TableRows( + rows: [ + Row(id: .existing(0), values: [ + .text("2024-03-15 09:30:00.123456"), + .text("2024-12-31 23:59:59+07"), + .text("2024-03-15"), + .bytes(Data("signup".utf8)), + .bytes(Data((0..<16).map { UInt8($0) })), + .text("1700000000"), + .text("1700000000000"), + .text("a\rb") + ]), + Row(id: .existing(1), values: [ + .text("2023-11-14 22:13:20"), + .null, + .text("1999-01-01"), + .bytes(Data([0x89, 0x50, 0x4E, 0x47])), + .bytes(Data(repeating: 0xAB, count: 16)), + .text("1700003600"), + .text("1700003600000"), + .text("plain") + ]) + ], + columns: ["created", "updated", "born", "payload", "token", "stamp", "stampMillis", "note"], + columnTypes: [ + .datetime(rawType: "DATETIME(6)"), + .timestamp(rawType: "TIMESTAMPTZ"), + .date(rawType: "DATE"), + .blob(rawType: "VARBINARY(255)"), + .blob(rawType: "BINARY(16)"), + .integer(rawType: "BIGINT"), + .integer(rawType: "BIGINT"), + .text(rawType: "TEXT") + ] + ) + + private static let shownFormats: [ValueDisplayFormat?] = [nil, nil, nil, .text, .uuid, .unixTimestamp, .unixTimestampMillis, nil] + + private static let shownTerms = [ + "2024", "2023", "09:30:00.123456", "+07", "/", "PM", "signup", "00010203-0405", "abab", "a b", "b", "11" + ] + + // Both sides use the same renderer code, so this pins the same cells on and off the main actor, not the shown text. + @Test("The same search finds the same cells on and off the main actor, under every date format") + func searchOffTheMainActorMatchesTheMainActor() async { + for revealsInvisibleCharacters in [false, true] { + let source = self.source(Self.shownValues, formats: Self.shownFormats, revealsInvisibleCharacters: revealsInvisibleCharacters) + for format in DateFormatOption.allCases { + for term in Self.shownTerms { + let onMain = GridFindEngine.search( + term, + options: GridFindState.Options(), + in: source, + dates: DateDisplayRenderer(format: format), + isCancelled: { false } + ) + let offMain = await GridFindEngine.searchOffTheMainActor( + term, + options: GridFindState.Options(), + in: source, + dateFormat: format + ) + #expect(offMain == onMain, "same cells on and off the main actor: \(format) \(term) reveals: \(revealsInvisibleCharacters)") + } + } + for term in Self.shownTerms { + let shared = await GridFindEngine.searchOffTheMainActor( + term, + options: GridFindState.Options(), + in: source, + dateFormat: DateFormattingService.shared.currentFormat + ) + #expect(shared == search(term, in: source), "\(term)") + } + } + } + + @Test("The off-main search renders dates, bytes and timestamps the way the cells show them") + func searchOffTheMainActorReadsTheShownText() async { + let source = self.source(Self.shownValues, formats: Self.shownFormats) + func column(_ name: String) -> DataGridColumnID { DataGridColumnID(name: name, occurrence: 0) } + func found(_ term: String, _ format: DateFormatOption = .iso8601) async -> [GridFindState.Match]? { + await GridFindEngine.searchOffTheMainActor(term, options: GridFindState.Options(), in: source, dateFormat: format)?.matches + } + + #expect(await found("15/03/2024", .euLong) == [match(0, column("created")), match(0, column("born"))]) + #expect(await found("09:30:00.123456 AM", .usLong) == [match(0, column("created"))]) + #expect(await found("23:59:59+07") == [match(0, column("updated"))]) + #expect(await found("signup") == [match(0, column("payload"))]) + #expect(await found("00010203-0405-0607-0809-0a0b0c0d0e0f") == [match(0, column("token"))]) + #expect(await found("a b") == [match(0, column("note"))]) + #expect(await found("8950")?.isEmpty == true) + } + + @Test("A search asks whether to stop every 4,096 cells and gives up when told to") + func searchChecksForCancellation() { + let rows = TableRows.from( + queryRows: (0..<10_000).map { [PluginCellValue.text("row \($0)")] }, + columns: ["v"], + columnTypes: [.text(rawType: "TEXT")] + ) + let dates = DateDisplayRenderer(format: .iso8601) + var checks = 0 + + let finished = GridFindEngine.search("row", options: GridFindState.Options(), in: source(rows), dates: dates) { + checks += 1 + return false + } + let stopped = GridFindEngine.search( + "row", options: GridFindState.Options(), in: source(rows), dates: dates, isCancelled: { true } + ) + + #expect(finished?.matches.count == 10_000) + #expect(checks == 3) + #expect(stopped == nil) + } + // MARK: - Replacing private static let contacts = TableRows.from( @@ -474,18 +593,19 @@ struct DataGridModelFindTests { GridSelectionReducer.single(GridCoord(row: row, displayColumn: column)) } - private func search(_ model: DataGridModel, _ term: String) { + private func search(_ model: DataGridModel, _ term: String) async { model.showFind() model.find.term = term model.findQueryDidChange() + await model.findSearchSettled() } @Test("A match becomes the active cell, column included") - func matchSetsActiveCell() { + func matchSetsActiveCell() async { let (model, owner) = mount() model.setSelection(cell(2, 0)) - search(model, "acme") + await search(model, "acme") #expect(model.find.matches.count == 4) #expect(model.find.currentIndex == 1) @@ -495,9 +615,9 @@ struct DataGridModelFindTests { } @Test("Find Next walks the matches and wraps") - func findNextWraps() { + func findNextWraps() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") #expect(model.selection.activeCell == GridCoord(row: 1, displayColumn: 2)) model.findNext() @@ -512,9 +632,9 @@ struct DataGridModelFindTests { } @Test("Find Previous starts from where the user moved the cursor") - func findPreviousFromCursor() { + func findPreviousFromCursor() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") model.setSelection(cell(3, 0)) model.findPrevious() @@ -524,14 +644,15 @@ struct DataGridModelFindTests { } @Test("A refresh keeps the current match by identity and leaves the selection alone") - func refreshKeepsCurrentMatch() { + func refreshKeepsCurrentMatch() async { let (model, owner) = mount() model.setSelection(cell(2, 0)) - search(model, "acme") + await search(model, "acme") owner.tableRows.rows[0].values[2] = .text("Acme Town") model.reloadRows() model.refreshFind() + await model.findSearchSettled() #expect(model.find.matches.count == 5) #expect(model.find.currentMatch == GridFindState.Match(rowID: .existing(2), columnID: Self.company)) @@ -540,9 +661,9 @@ struct DataGridModelFindTests { } @Test("Replace commits the current match as one edit and moves to the next match") - func replaceCurrent() { + func replaceCurrent() async { let (model, owner) = mount { $0.isEditable = true } - search(model, "acme") + await search(model, "acme") model.find.replacement = "Umbrella" model.replaceCurrentMatch() @@ -556,12 +677,12 @@ struct DataGridModelFindTests { } @Test("Replace All is one commit and leaves a column it may not write") - func replaceAllOneCommit() throws { + func replaceAllOneCommit() async throws { let (model, owner) = mount { $0.isEditable = true $0.lockedColumns = ["city"] } - search(model, "acme") + await search(model, "acme") model.find.replacement = "Umbrella" let outcome = try #require(model.replaceAllMatches()) @@ -588,11 +709,12 @@ struct DataGridModelFindTests { } @Test("Hiding the bar clears the term, the matches, the scope and the replace row") - func hideClears() { + func hideClears() async { let (model, owner) = mount { $0.isEditable = true } model.findInColumn(Self.company, replace: true) model.find.term = "acme" model.findQueryDidChange() + await model.findSearchSettled() #expect(model.find.matches.count == 2) model.hideFind() @@ -606,24 +728,24 @@ struct DataGridModelFindTests { } @Test("Search All Rows hands the trimmed term to the owner only where the grid offers it") - func searchAllRows() { + func searchAllRows() async { let (plain, plainOwner) = mount() - search(plain, " zzz ") + await search(plain, " zzz ") plain.searchAllRows() #expect(plainOwner.searchedTerms.isEmpty) #expect(plain.find.isVisible) let (offered, owner) = mount { $0.commands = [.searchAllRows] } - search(offered, " zzz ") + await search(offered, " zzz ") offered.searchAllRows() #expect(owner.searchedTerms == ["zzz"]) #expect(offered.find.isVisible == false) } @Test("Search All Rows waits for plain options, since the server matches the term as typed in any column") - func searchAllRowsNeedsPlainOptions() { + func searchAllRowsNeedsPlainOptions() async { let (model, owner) = mount { $0.commands = [.searchAllRows] } - search(model, "^zzz$") + await search(model, "^zzz$") model.find.options.usesRegularExpression = true model.searchAllRows() @@ -636,9 +758,9 @@ struct DataGridModelFindTests { } @Test("New rows under an open bar are searched again") - func reloadSearchesTheNewRows() { + func reloadSearchesTheNewRows() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") #expect(model.find.matches.count == 4) owner.tableRows = TableRows.from( @@ -647,6 +769,7 @@ struct DataGridModelFindTests { columnTypes: [.integer(rawType: "INT"), .text(rawType: "TEXT"), .text(rawType: "TEXT")] ) model.reloadRows() + await model.findSearchSettled() #expect(model.find.matches.count == 1) } @@ -657,9 +780,9 @@ struct DataGridModelFindTests { } @Test("Edits under an open bar keep the matches a full search finds") - func editsKeepTheMatchesInStep() { + func editsKeepTheMatchesInStep() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") owner.tableRows.rows[2].values[1] = .text("Umbrella") model.apply(.cellChanged(row: 2, column: 1)) @@ -672,9 +795,9 @@ struct DataGridModelFindTests { } @Test("Rows added or removed under an open bar keep the matches a full search finds") - func insertsAndRemovalsKeepTheMatchesInStep() { + func insertsAndRemovalsKeepTheMatchesInStep() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") let inserted = owner.tableRows.appendInsertedRow(values: [.text("5"), .text("Acme East"), .text("Oslo")]) model.apply(inserted) @@ -696,7 +819,7 @@ struct DataGridModelFindTests { // An edit keeps its row on screen (#2667); an insert hides it, and once undone and inserted again it shows again. @Test("A row a value filter shows again after an insert is searched again") - func rowBackInTheFilteredOrderIsSearched() { + func rowBackInTheFilteredOrderIsSearched() async { let (model, owner) = mount() var filter = GridValueFilterState() filter.set( @@ -705,7 +828,7 @@ struct DataGridModelFindTests { forColumn: 2 ) model.setValueFilter(filter) - search(model, "acme") + await search(model, "acme") #expect(model.find.matches == [match(1, Self.city), match(2, Self.company), match(3, Self.company), match(3, Self.city)]) owner.tableRows.rows[2].values[2] = .text("Lyon") @@ -726,13 +849,14 @@ struct DataGridModelFindTests { } @Test("A term typed but not yet searched makes the next refresh search every row") - func changedTermSearchesEveryRow() { + func changedTermSearchesEveryRow() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") model.find.term = "paris" owner.tableRows.rows[0].values[2] = .text("Paris") model.apply(.cellChanged(row: 3, column: 1)) + await model.findSearchSettled() #expect(model.find.matches == [match(0, Self.city), match(2, Self.city)]) } @@ -748,16 +872,218 @@ struct DataGridModelFindTests { } @Test("Use Selection for Find takes the cell's text, escaped for a regular expression") - func useSelectionForFind() { + func useSelectionForFind() async { let (model, owner) = mount() model.setSelection(cell(0, 2)) model.find.options.usesRegularExpression = true model.useSelectionForFind() + await model.findSearchSettled() #expect(model.find.isVisible) #expect(model.find.term == NSRegularExpression.escapedPattern(for: "St. Louis")) #expect(model.find.matches == [GridFindState.Match(rowID: .existing(0), columnID: DataGridColumnID(name: "city", occurrence: 0))]) #expect(owner.commits.isEmpty) } + + // MARK: - A search off the main actor + + // Each test ends on `owner`: the model holds its data source weakly, and without it the grid has no rows. + + @Test("A typed term drops the old term's matches at once, and a refresh keeps its matches until its search lands") + func typedTermDropsTheOldMatches() async { + let (model, owner) = mount() + await search(model, "paris") + + model.find.term = "acme" + model.findQueryDidChange() + + #expect(model.find.loadedRowsSearch != nil) + #expect(model.find.isSearching == false) + #expect(model.find.matches.isEmpty) + #expect(model.find.currentIndex == nil) + + await model.findSearchSettled() + + #expect(model.find.loadedRowsSearch == nil) + #expect(model.find.matches.count == 4) + #expect(model.selection.activeCell == GridCoord(row: 3, displayColumn: 1)) + + model.refreshFind() + + #expect(model.find.loadedRowsSearch != nil) + #expect(model.find.matches.count == 4) + #expect(model.find.currentIndex == 2) + + await model.findSearchSettled() + #expect(model.find.currentIndex == 2) + #expect(owner.commits.isEmpty) + } + + @Test("A typed term drops the old term's invalid-pattern message at once") + func typedTermDropsTheOldProblem() async { + let (model, owner) = mount() + model.showFind() + model.find.options.usesRegularExpression = true + model.find.term = "[" + model.findQueryDidChange() + #expect(model.find.problem != nil) + + model.find.term = "acme" + model.findQueryDidChange() + + #expect(model.find.loadedRowsSearch != nil) + #expect(model.find.problem == nil) + + await model.findSearchSettled() + #expect(model.find.problem == nil) + #expect(owner.commits.isEmpty) + } + + @Test("Of two terms typed in quick succession only the second one's matches land") + func onlyTheLastTermLands() async throws { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + let first = try #require(model.find.loadedRowsSearch) + + model.find.term = "paris" + model.findQueryDidChange() + await model.findSearchSettled() + + #expect(first.isCancelled) + #expect(model.find.matches == [match(2, Self.city)]) + #expect(model.selection.activeCell == GridCoord(row: 2, displayColumn: 2)) + #expect(owner.commits.isEmpty) + } + + @Test("A row added while a search runs is among the matches that land") + func rowAddedDuringASearchIsFound() async throws { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + let running = try #require(model.find.loadedRowsSearch) + + model.apply(owner.tableRows.appendInsertedRow(values: [.text("5"), .text("Acme East"), .text("Oslo")])) + await model.findSearchSettled() + + let insertedID = owner.tableRows.rows[4].id + #expect(running.isCancelled) + #expect(model.find.matches.last == GridFindState.Match(rowID: insertedID, columnID: Self.company)) + #expect(model.find.matches == fullSearch(model, owner)) + #expect(model.selection.activeCell == GridCoord(row: 1, displayColumn: 2), "the typed term still lands on its first match") + } + + @Test("A typed term's matches land without moving a cursor the user moved during the search") + func landingKeepsAMovedCursor() async { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + + model.setSelection(cell(0, 0)) + await model.findSearchSettled() + + #expect(model.find.matches.count == 4) + #expect(model.find.currentIndex == 0) + #expect(model.selection.activeCell == GridCoord(row: 0, displayColumn: 0)) + #expect(owner.commits.isEmpty) + } + + @Test("Find Next pressed while a search runs steps once its matches land") + func findNextWaitsForTheSearch() async { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + + model.findNext() + #expect(model.selection.activeCell == nil) + + await model.findSearchSettled() + + #expect(model.find.currentIndex == 1) + #expect(model.selection.activeCell == GridCoord(row: 2, displayColumn: 1)) + #expect(owner.commits.isEmpty) + } + + @Test("Every Find Next and Find Previous pressed while a search runs counts once its matches land") + func deferredStepsAreCounted() async { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + + model.findNext() + model.findNext() + model.findNext() + model.findPrevious() + await model.findSearchSettled() + + #expect(model.find.currentIndex == 2) + #expect(model.selection.activeCell == GridCoord(row: 3, displayColumn: 1)) + #expect(owner.commits.isEmpty) + } + + @Test("Replace pressed while a search runs replaces the match that search lands on") + func replaceWaitsForTheRunningSearch() async { + let (model, owner) = mount { $0.isEditable = true } + await search(model, "paris") + model.find.term = "acme" + model.findQueryDidChange() + model.find.replacement = "Umbrella" + + model.replaceCurrentMatch() + + #expect(model.find.loadedRowsSearch == nil) + #expect(owner.commits.first?.edits.map { $0.rowID } == [.existing(3)]) + #expect(owner.commits.first?.edits.map { $0.newValue } == [.text("Umbrella Corp")]) + } + + @Test("The bar says No matches and offers Search All Rows only once the search lands") + func noMatchesWaitsForTheSearch() async { + let (model, owner) = mount { $0.commands = [.searchAllRows] } + model.showFind() + model.find.term = "zzz" + model.findQueryDidChange() + + #expect(model.find.isAwaitingLoadedRows) + #expect(DataGridFindBar.counterText(status: nil, model: model).isEmpty) + #expect(!DataGridFindBar.showsSearchAllRows(offered: model.offersSearchAllRows, model: model)) + + await model.findSearchSettled() + + #expect(!model.find.isAwaitingLoadedRows) + #expect(DataGridFindBar.counterText(status: nil, model: model) == String(localized: "No matches")) + #expect(DataGridFindBar.showsSearchAllRows(offered: model.offersSearchAllRows, model: model)) + + model.find.term = "" + model.findQueryDidChange() + + #expect(!model.find.isAwaitingLoadedRows, "an empty term is answered without a search") + #expect(model.find.loadedRowsSearch == nil) + #expect(owner.commits.isEmpty) + } + + @Test("A grid that goes away cancels the search still running") + func releasedModelCancelsTheSearch() throws { + let owner: FindGridOwner + let running: Task + weak var released: DataGridModel? + do { + let (model, mounted) = mount() + owner = mounted + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + running = try #require(model.find.loadedRowsSearch) + released = model + } + + #expect(released == nil) + #expect(running.isCancelled) + #expect(owner.commits.isEmpty) + } } diff --git a/apple/TableProTests/Core/DataWrite/PluginKeyedChangesTests.swift b/apple/TableProTests/Core/DataWrite/PluginKeyedChangesTests.swift index e09362a589..697ed53c68 100644 --- a/apple/TableProTests/Core/DataWrite/PluginKeyedChangesTests.swift +++ b/apple/TableProTests/Core/DataWrite/PluginKeyedChangesTests.swift @@ -10,7 +10,7 @@ import Testing @testable import TablePro struct PluginKeyedChangesTests { - @Test("Every row gets its own key, and the key agrees across the changes and the sets") + @Test("Every change gets its own key, and the key agrees across the changes and the sets") func keysAgreeAcrossCollections() { let insertedA = RowID.inserted(UUID()) let insertedB = RowID.inserted(UUID()) @@ -93,4 +93,31 @@ struct PluginKeyedChangesTests { #expect(crossed?.cellChanges.first?.newValue == "b") #expect(crossed?.originalRow == ["3", "a"]) } + + @MainActor + @Test("A row edited around another change reaches the driver once per edit, each with the row as it then stands") + func aRowEditedAroundAnotherChangeReachesTheDriverTwice() throws { + let handed = HandedChanges() + let driver = RowWriteStubDriver { changes, _, _, _ in + handed.changes = changes + return changes.map { PluginRowWrite(statement: "write(\($0.rowIndex))", rowIndices: [$0.rowIndex]) } + } + let (manager, _) = EditAroundADelete.manager(table: "items", databaseType: DatabaseType(rawValue: "MongoDB")) + manager.pluginDriver = driver + EditAroundADelete.record(on: manager) + + _ = try manager.generateSQL() + let types = handed.changes.map { $0.type } + let keys = Set(handed.changes.map { $0.rowIndex }) + let laterColumns = handed.changes.last?.cellChanges.map { $0.columnName } + + #expect(types == [.update, .delete, .update]) + #expect(keys.count == 3) + #expect(handed.changes.last?.originalRow == ["3", "Cyd", "c@test.com"]) + #expect(laterColumns == ["email"]) + } +} + +private final class HandedChanges { + var changes: [PluginRowChange] = [] } diff --git a/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift b/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift index 10d189f39a..b27435f4c6 100644 --- a/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift +++ b/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift @@ -358,6 +358,139 @@ struct RewindPlannerTests { #expect(undo.statements.map { String($0.sql.prefix(6)) } == ["DELETE", "INSERT"]) } + private func recordedOperations(_ edits: (inout PendingChanges) -> Void) -> [RowWriteOperation] { + var pending = PendingChanges() + edits(&pending) + return RowWriteOperationBuilder.operations( + from: pending.changes, + insertedRowData: pending.insertedRowData, + deletedRowIDs: pending.deletedRowIDs, + insertedRowIDs: pending.insertedRowIDs, + target: target, + columns: columns, + primaryKeyColumns: ["id"], + generatedColumns: [], + containsTableOperation: false + ) + } + + // A.name a to tmp, B.name b to a, A.name tmp to b: valid step by step on a unique `name`. + private func swapOperations() -> [RowWriteOperation] { + recordedOperations { pending in + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "a", newValue: "tmp", + originalRow: ["k1", "a"] + ) + pending.recordCellChange( + rowID: .existing(1), columnIndex: 1, columnName: "name", oldValue: "b", newValue: "a", + originalRow: ["k2", "b"] + ) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "tmp", newValue: "b", + originalRow: ["k1", "tmp"] + ) + } + } + + private func bound(_ statements: [ParameterizedStatement]) -> [[String]] { + statements.map { statement in statement.parameters.compactMap { $0 as? String } } + } + + @Test("A swap rewinds its newest step first, so no step takes a value another still holds") + func aSwapRewindsNewestStepFirst() throws { + let plan = try planner(operations: swapOperations()).plan(currentRows: [ + RewindCurrentRow(values: ["k1", "b"]), RewindCurrentRow(values: ["k2", "a"]) + ]) + + #expect(plan.rows.map { $0.outcome } == [.willRestore, .willRestore]) + #expect(bound(plan.statements) == [["tmp", "k1"], ["b", "k2"], ["a", "k1"]]) + } + + @Test("Undoing the rewind of a swap replays the save in its own order") + func undoingASwapRewindReplaysTheSave() throws { + let rewind = try planner(operations: swapOperations()).plan(currentRows: [ + RewindCurrentRow(values: ["k1", "b"]), RewindCurrentRow(values: ["k2", "a"]) + ]) + + let undo = try planner(operations: RewindExecutor.inverseOperations(of: rewind)).plan(currentRows: [ + RewindCurrentRow(values: ["k1", "a"]), RewindCurrentRow(values: ["k2", "b"]) + ]) + + #expect(bound(undo.statements) == [["tmp", "k1"], ["a", "k2"], ["b", "k1"]]) + } + + @Test("A key moved in an earlier step is rewound under the key each step left") + func aKeyMovedInTheFirstStepRewindsUnderEachStepsKey() throws { + let operations = recordedOperations { pending in + pending.recordCellChange( + rowID: .existing(0), columnIndex: 0, columnName: "id", oldValue: "k5", newValue: "k6", + originalRow: ["k5", "Ada"] + ) + pending.recordRowDeletion(rowID: .existing(1), originalRow: ["k7", "Bo"]) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "Ada", newValue: "Grace", + originalRow: ["k6", "Ada"] + ) + } + + let plan = try planner(operations: operations).plan(currentRows: [RewindCurrentRow(values: ["k6", "Grace"])]) + let verbs = plan.statements.map { String($0.sql.prefix(6)) } + + #expect(plan.rows.map { $0.outcome } == [.willRestore, .willRestore]) + #expect(verbs == ["UPDATE", "INSERT", "UPDATE"]) + #expect(bound(plan.statements).first == ["Ada", "k6"]) + #expect(bound(plan.statements).last == ["k5", "k6"]) + } + + @Test("A step that wrote SQL refuses the row, though a later step replaced it with a value") + func aStepThatWroteSQLRefusesTheRow() throws { + let operations = recordedOperations { pending in + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "Ada", newValue: "now()", + originalRow: ["k1", "Ada"], isExpression: true + ) + pending.recordRowDeletion(rowID: .existing(1), originalRow: ["k2", "Bo"]) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "now()", newValue: "Grace", + originalRow: ["k1", "now()"] + ) + } + + let update = try #require(operations.first { $0.kind == .update }) + #expect(update.refusal == .serverComputedValue) + let plan = try planner(operations: operations).plan(currentRows: [RewindCurrentRow(values: ["k1", "Grace"])]) + #expect(plan.rows.first?.outcome == .notReversible(.serverComputedValue)) + } + + @Test("A record saved before steps were kept rewinds newest first by list position") + func aRecordWithoutStepsRewindsAsBefore() throws { + let first = operation(kind: .update, preImage: ["k1", "Ada"], postImage: ["k1", "Grace"], writtenColumns: ["name"]) + let second = operation(kind: .update, preImage: ["k2", "Bo"], postImage: ["k2", "Cy"], writtenColumns: ["name"]) + + let plan = try planner(operations: [first, second]).plan(currentRows: [ + RewindCurrentRow(values: ["k1", "Grace"]), RewindCurrentRow(values: ["k2", "Cy"]) + ]) + + #expect(bound(plan.statements) == [["Bo", "k2"], ["Ada", "k1"]]) + } + + @Test("A deleted row comes back once the new row that took its key is removed, and not while that row was changed") + func aDeletedRowComesBackAfterTheNewRowThatTookItsKey() throws { + let operations = recordedOperations { pending in + pending.recordRowDeletion(rowID: .existing(0), originalRow: ["k5", "Ada"]) + pending.recordRowInsertion(rowID: .inserted(UUID()), values: ["k5", "Bo"]) + } + + let plan = try planner(operations: operations).plan(currentRows: [RewindCurrentRow(values: ["k5", "Bo"])]) + let verbs = plan.statements.map { String($0.sql.prefix(6)) } + #expect(plan.rows.map { $0.outcome } == [.willRestore, .willRestore]) + #expect(verbs == ["DELETE", "INSERT"]) + + let changed = try planner(operations: operations).plan(currentRows: [RewindCurrentRow(values: ["k5", "Cy"])]) + #expect(changed.rows.map { $0.outcome } == [.rowAlreadyPresent, .changedSinceSave]) + #expect(changed.statements.isEmpty) + } + @Test("The inverse runs newest first, so a value one row freed is not taken before the other gives it up") func inverseRunsInReverseOrder() throws { let first = operation(kind: .delete, preImage: ["7", "Ada"], postImage: nil, writtenColumns: columns) diff --git a/apple/TableProTests/Core/DataWrite/RewindRecordIdentityDecodingTests.swift b/apple/TableProTests/Core/DataWrite/RewindRecordIdentityDecodingTests.swift index 5480185201..87d1d1b8d0 100644 --- a/apple/TableProTests/Core/DataWrite/RewindRecordIdentityDecodingTests.swift +++ b/apple/TableProTests/Core/DataWrite/RewindRecordIdentityDecodingTests.swift @@ -5,6 +5,7 @@ import Foundation @testable import TablePro +import TableProPluginKit import Testing struct RewindRecordIdentityDecodingTests { @@ -42,6 +43,62 @@ struct RewindRecordIdentityDecodingTests { } } +struct RewindRecordStepDecodingTests { + private func record() -> RewindRecord { + let target = DataWriteTarget(database: "shop", schema: nil, table: "users") + let operation = RowWriteOperation( + kind: .update, + target: target, + columns: ["id", "name"], + primaryKeyColumns: ["id"], + preImage: ["1", "a"], + postImage: ["1", "b"], + writtenColumns: ["name"], + refusal: nil, + order: 3, + steps: [ + RowWriteStep(order: 3, columns: [1], before: ["a"], after: ["tmp"]), + RowWriteStep(order: 5, columns: [1], before: ["tmp"], after: ["b"], afterAbsent: [1]) + ] + ) + return RewindRecord( + id: UUID(), + historyId: nil, + connectionId: UUID(), + databaseType: .postgresql, + target: target, + capturedAt: Date(timeIntervalSince1970: 1_700_000_000), + generatedColumns: [], + identityColumns: [], + operations: [operation] + ) + } + + @Test + func aRecordSavedBeforeStepsDecodes() throws { + let encoded = try JSONEncoder().encode(record()) + var object = try #require(try JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + var operations = try #require(object["operations"] as? [[String: Any]]) + operations[0].removeValue(forKey: "order") + operations[0].removeValue(forKey: "steps") + object["operations"] = operations + let legacy = try JSONSerialization.data(withJSONObject: object) + + let decoded = try JSONDecoder().decode(RewindRecord.self, from: legacy) + let operation = try #require(decoded.operations.first) + #expect(operation.order == nil) + #expect(operation.steps == nil) + #expect(operation.postImage == ["1", "b"]) + } + + @Test + func stepsRoundTrip() throws { + let original = record() + let decoded = try JSONDecoder().decode(RewindRecord.self, from: JSONEncoder().encode(original)) + #expect(decoded.operations == original.operations) + } +} + struct RewindRecordColumnTypeDecodingTests { private func record(columnTypeNames: [String: String]?) -> RewindRecord { RewindRecord( diff --git a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift index 570e20367b..c6c74fa0fc 100644 --- a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift @@ -371,6 +371,40 @@ struct RowChangeStatementFactoryCoverageTests { #expect(statements.allSatisfy { $0.keylessRowCount == nil }) } + @Test("A driver that writes only the first edit of a row edited around a delete refuses the save") + func aDriverThatWritesOneRunOfARowRefusesTheSave() { + let driver = RowWriteStubDriver { changes, _, _, _ in + let firstUpdate = changes.first { $0.type == .update }?.rowIndex + return changes.compactMap { change in + guard change.type != .update || change.rowIndex == firstUpdate else { return nil } + return PluginRowWrite(statement: "write(\(change.rowIndex))", rowIndices: [change.rowIndex]) + } + } + let (manager, _) = EditAroundADelete.manager(table: "items", databaseType: DatabaseType(rawValue: "MongoDB")) + manager.pluginDriver = driver + EditAroundADelete.record(on: manager) + + #expect(throws: DataWriteError.changesNotWritable(table: "items", unwritten: UnwrittenRowCounts(updates: 1))) { + _ = try manager.generateSQL() + } + } + + @Test("A refusal counts the rows it cannot write, not the edits of each") + func aRefusalCountsRowsNotRuns() { + let driver = RowWriteStubDriver { changes, _, _, _ in + changes.filter { $0.type == .delete }.map { + PluginRowWrite(statement: "deleteOne(\($0.rowIndex))", rowIndices: [$0.rowIndex]) + } + } + let (manager, _) = EditAroundADelete.manager(table: "items", databaseType: DatabaseType(rawValue: "MongoDB")) + manager.pluginDriver = driver + EditAroundADelete.record(on: manager) + + #expect(throws: DataWriteError.changesNotWritable(table: "items", unwritten: UnwrittenRowCounts(updates: 1))) { + _ = try manager.generateSQL() + } + } + @Test("Preview SQL and Save both refuse the same mixed set") func buildRowWritesRefusesWhatGenerateSQLRefuses() { let manager = DataChangeManager() diff --git a/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift b/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift index fdf82b33af..c98576f909 100644 --- a/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift @@ -209,6 +209,37 @@ struct RowWriteOperationBuilderTests { #expect(result.map { $0.kind } == [.delete, .insert]) } + @Test("A row edited around another change is one operation with its net images and both steps") + func aRowEditedInTwoRunsIsOneOperationWithBothSteps() throws { + var pending = PendingChanges() + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Ada", newValue: "Grace", originalRow: ["7", "Ada", "2026-01-01"] + ) + pending.recordRowDeletion(rowID: .existing(1), originalRow: ["8", "Bo", "2026-01-01"]) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "updated_at", + oldValue: "2026-01-01", newValue: "2026-02-02", originalRow: ["7", "Grace", "2026-01-01"] + ) + + let result = operations( + changes: pending.changes, + deletedRowIDs: pending.deletedRowIDs + ) + + #expect(result.map { $0.kind } == [.update, .delete]) + let update = try #require(result.first) + #expect(update.preImage == ["7", "Ada", "2026-01-01"]) + #expect(update.postImage == ["7", "Grace", "2026-02-02"]) + #expect(update.order == 0) + let steps = try #require(update.steps) + #expect(steps.map { $0.order } == [0, 2]) + #expect(steps.map { $0.columns } == [[1], [2]]) + #expect(steps.map { $0.after } == [["Grace"], ["2026-02-02"]]) + #expect(result.last?.order == 1) + #expect(result.last?.steps == nil) + } + @Test("An insert carrying its own key is reversible") func userSuppliedKeyIsReversible() { let result = operations( diff --git a/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift b/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift index 81ec4f95e9..6eb56754f3 100644 --- a/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift +++ b/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift @@ -5,6 +5,8 @@ import Foundation @testable import TablePro +import TableProPluginKit +import TableProSQLGrammar import Testing @Suite("Credential lease routing", .serialized) @@ -60,6 +62,35 @@ struct CredentialLeaseRoutingTests { #expect(manager.activeSessions[connection.id]?.driver === home) } + @Test("A lane closed for its ending lease reports the session state the user set up on it") + func vaultRotationReportsSessionState() async { + let connection = TestFixtures.makeConnection(database: "app") + let home = MockDatabaseDriver(connection: connection) + let lane = MockDatabaseDriver(connection: connection) + lane.sessionTransactionStateToReturn = .idle + var session = ConnectionSession(connection: connection) + session.driver = home + session.status = .connected + let manager = DatabaseManager.shared + manager.injectSession(session, for: connection.id) + manager.sessionLanes.park(lane, database: "logs", for: connection.id) + defer { + manager.sessionLanes.closeAll(for: connection.id) + manager.removeSession(for: connection.id) + } + let key = SessionTransactionTracker.Key(connectionId: connection.id, lane: "logs") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .preparedStatements), on: lane, for: key) + + await manager.credentialLeaseEnding(ending(connection.id, lane)) + + #expect(lane.disconnectCallCount == 1) + #expect(manager.sessionStateLedger.state(heldBy: lane, for: connection.id).isEmpty) + #expect( + manager.sessionLanes.takeLoss(database: "logs", for: connection.id) + == SessionLoss(transaction: false, state: .preparedStatements) + ) + } + @Test("A browsed driver replaced while it held a transaction reports the loss to the next statement, once") func reportsLostBrowsedTransaction() async throws { let connection = TestFixtures.makeConnection(database: "app", type: .mysql) @@ -69,7 +100,7 @@ struct CredentialLeaseRoutingTests { let manager = DatabaseManager.shared manager.injectSession(session, for: connection.id) defer { manager.removeSession(for: connection.id) } - manager.sessionLanes.markTransactionLost(database: "app", for: connection.id) + manager.sessionLanes.markLost(database: "app", transaction: true, for: connection.id) let scope = DatabaseScope(connectionId: connection.id, database: "app", schema: nil) let error = await #expect(throws: DatabaseError.self) { @@ -89,14 +120,14 @@ struct CredentialLeaseRoutingTests { defer { manager.removeSession(for: connection.id) } let scope = DatabaseScope(connectionId: connection.id, database: "", schema: nil) #expect(manager.metadataRoute(for: scope) == .sessionDriver) - manager.sessionLanes.markTransactionLost(database: "", for: connection.id) + manager.sessionLanes.markLost(database: "", transaction: true, for: connection.id) try await manager.withMetadataDriver(scope: scope) { _ in } - #expect(manager.sessionLanes.hasTransactionLoss(database: "", for: connection.id)) + #expect(manager.sessionLanes.hasLoss(database: "", for: connection.id)) await #expect(throws: DatabaseError.self) { try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .protectedWrite) { _ in } } - #expect(!manager.sessionLanes.hasTransactionLoss(database: "", for: connection.id)) + #expect(!manager.sessionLanes.hasLoss(database: "", for: connection.id)) } } diff --git a/apple/TableProTests/Core/Database/SessionLanesTests.swift b/apple/TableProTests/Core/Database/SessionLanesTests.swift index 4b4e31a158..0baef7ec88 100644 --- a/apple/TableProTests/Core/Database/SessionLanesTests.swift +++ b/apple/TableProTests/Core/Database/SessionLanesTests.swift @@ -10,6 +10,7 @@ import Foundation @testable import TablePro import TableProPluginKit +import TableProSQLGrammar import TableProTestSupport import Testing @@ -299,6 +300,24 @@ struct SessionLanesTests { #expect(manager.sessionLanes.parkedDatabases(for: harness.connection.id).sorted() == ["audit", "logs"]) } + @Test("An opened connection holding session state the user set up is not closed for being idle") + func openedLaneHoldingSessionStateIsNotClosedIdle() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + _ = try await run(harness, on: "logs", leaving: .idle) + let logs = try #require(harness.opener.opened["logs"]) + let lane = SessionTransactionTracker.Key(connectionId: connectionId, lane: "logs") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .sessionSettings), on: logs, for: lane) + + await manager.closeIdleOpenedLanes(for: connectionId, idleFor: 0) + + #expect(logs.disconnectCallCount == 0) + #expect(manager.sessionLanes.parkedDriver(for: connectionId, database: "logs") === logs) + #expect(manager.sessionLanes.isOpened(database: "logs", for: connectionId)) + } + @Test("The connection of a database the user browsed and left is not closed for being idle") func browsedConnectionIsNotClosedWhenIdle() async throws { let harness = makeHarness() @@ -634,7 +653,7 @@ struct SessionLanesTests { harness.home.sessionTransactionStateToReturn = .inTransaction let scope = DatabaseScope(connectionId: harness.connection.id, database: "app", schema: nil) - await DatabaseManager.shared.sessionLanes.closeAllNotingLostTransactions(for: harness.connection.id) + await DatabaseManager.shared.sessionLanes.closeAllNotingLosses(for: harness.connection.id) #expect(harness.home.disconnectCallCount == 1) #expect(DatabaseManager.shared.executionRoute(for: scope) == .sessionDriver) @@ -645,7 +664,110 @@ struct SessionLanesTests { cancellation: .untracked ) { _ in () } } - #expect(!DatabaseManager.shared.sessionLanes.hasTransactionLoss(database: "app", for: harness.connection.id)) + #expect(!DatabaseManager.shared.sessionLanes.hasLoss(database: "app", for: harness.connection.id)) + } + + @Test("A rebuilt transport reports what the user's statements set up on a parked connection, once") + func transportRebuildReportsALanesSessionState() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + try await manager.switchDatabase(to: "logs", for: connectionId, persist: false) + harness.home.sessionTransactionStateToReturn = .idle + let lane = SessionTransactionTracker.Key(connectionId: connectionId, lane: "app") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .temporaryObjects), on: harness.home, for: lane) + let scope = DatabaseScope(connectionId: connectionId, database: "app", schema: nil) + + await manager.closeTransportLanes(for: connectionId) { true } + + #expect(harness.home.disconnectCallCount == 1) + #expect(manager.executionRoute(for: scope) == .sessionDriver) + let error = await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver( + scope: scope, route: manager.executionRoute(for: scope), cancellation: .untracked + ) { _ in () } + } + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .temporaryObjects) + ) + #expect(error?.localizedDescription.contains(expected) == true) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: connectionId)) + } + + @Test("A parked connection that died holding session state the user set up is replaced and the work refused once") + func deadParkedLaneReportsItsSessionState() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + try await manager.switchDatabase(to: "logs", for: connectionId, persist: false) + harness.home.pingError = DatabaseError.connectionFailed("server closed the connection") + harness.home.sessionTransactionStateToReturn = .idle + let lane = SessionTransactionTracker.Key(connectionId: connectionId, lane: "app") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .userVariables), on: harness.home, for: lane) + let scope = DatabaseScope(connectionId: connectionId, database: "app", schema: nil) + + let error = await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver( + scope: scope, route: .sessionDriver, cancellation: .untracked + ) { _ in () } + } + + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .userVariables) + ) + #expect(error?.localizedDescription.contains(expected) == true) + let replacement = try #require(harness.opener.opened["app"]) + #expect(harness.home.disconnectCallCount == 1) + let ranOn = try await manager.withScopedDriver( + scope: scope, route: .sessionDriver, cancellation: .untracked + ) { driver in ObjectIdentifier(driver) } + #expect(ranOn == ObjectIdentifier(replacement)) + } + + @Test("A grid save on a database whose lane died with session state is refused once, and the next one runs") + func firstOwningTurnOnALostLaneTakesTheReport() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + try await manager.switchDatabase(to: "logs", for: connectionId, persist: false) + harness.home.sessionTransactionStateToReturn = .idle + let lane = SessionTransactionTracker.Key(connectionId: connectionId, lane: "app") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .temporaryObjects), on: harness.home, for: lane) + await manager.closeTransportLanes(for: connectionId) { true } + let scope = DatabaseScope(connectionId: connectionId, database: "app", schema: nil) + + let error = await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver( + scope: scope, route: manager.executionRoute(for: scope), cancellation: .protectedWrite + ) { _ in () } + } + + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .temporaryObjects) + ) + #expect(error?.localizedDescription.contains(expected) == true) + #expect(harness.opener.opened["app"] == nil) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: connectionId)) + let savedOn = try await manager.withScopedDriver( + scope: scope, route: manager.executionRoute(for: scope), cancellation: .protectedWrite + ) { driver in ObjectIdentifier(driver) } + let reopened = try #require(harness.opener.opened["app"]) + #expect(savedOn == ObjectIdentifier(reopened)) + } + + @Test("A run notes what ran: every statement, through the one that failed, or only those a Stop let run") + func ranCountFollowsTheOutcome() { + let completed = BatchStatementOutcome.completed(results: [1, 2, 3]) + let stopped = BatchStatementOutcome.cancelled(results: [1]) + let failed = BatchStatementOutcome.failed( + results: [1], failure: .statement(sql: "SET x = 1"), errorDescription: "failed" + ) + #expect(completed.ranCount(of: 3) == 3) + #expect(stopped.ranCount(of: 3) == 1) + #expect(failed.ranCount(of: 3) == 2) } @Test("A table read leaves a lost transaction's report for the user's next statement") @@ -653,7 +775,7 @@ struct SessionLanesTests { let harness = makeHarness() defer { cleanUp(harness) } let manager = DatabaseManager.shared - manager.sessionLanes.markTransactionLost(database: "app", for: harness.connection.id) + manager.sessionLanes.markLost(database: "app", transaction: true, for: harness.connection.id) let scope = DatabaseScope(connectionId: harness.connection.id, database: "app", schema: nil) let ranOn = try await manager.withTableReadDriver(scope: scope, cancellation: .untracked) { driver in @@ -661,7 +783,7 @@ struct SessionLanesTests { } #expect(ranOn == ObjectIdentifier(harness.home)) - #expect(manager.sessionLanes.hasTransactionLoss(database: "app", for: harness.connection.id)) + #expect(manager.sessionLanes.hasLoss(database: "app", for: harness.connection.id)) await #expect(throws: DatabaseError.self) { try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in () } } diff --git a/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift b/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift index d076adf949..3ce71ed1b5 100644 --- a/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift +++ b/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift @@ -2,6 +2,7 @@ import Foundation @testable import TablePro import TableProMCP import TableProPluginKit +import TableProSQLGrammar import TableProSSHConfig import TableProTestSupport import Testing @@ -314,12 +315,12 @@ struct SessionRecoveryLoopTests { #expect(fixture.session?.driver !== fixture.driver) #expect(!manager.sessionTransactions.holdsTransaction(connectionId: fixture.id, container: "app")) - #expect(manager.sessionLanes.hasTransactionLoss(database: "app", for: fixture.id)) + #expect(manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) let error = await #expect(throws: DatabaseError.self) { try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in () } } #expect(error?.localizedDescription.contains(DatabaseManager.parkedTransactionLostMessage(database: "app")) == true) - #expect(!manager.sessionLanes.hasTransactionLoss(database: "app", for: fixture.id)) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) await cleanUp(fixture) } @@ -337,7 +338,200 @@ struct SessionRecoveryLoopTests { await loop.value #expect(fixture.session?.driver !== fixture.driver) - #expect(!manager.sessionLanes.hasTransactionLoss(database: "app", for: fixture.id)) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + await cleanUp(fixture) + } + + // Inside the user's own lease, as a query tab notes what it runs. + private func runUserStatements(_ statements: [String], on fixture: Fixture) async throws { + let manager = fixture.manager + let scope = DatabaseScope(connectionId: fixture.id, database: "app", schema: nil) + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { driver in + await manager.noteSessionStatements(statements, ranOn: driver, scope: scope) + } + } + + private func recover(_ fixture: Fixture) async throws { + fixture.manager.beginRecovery(for: fixture.id, cause: .stoppedResponding) + let loop = try loopTask(fixture) + await runNextAttempt(fixture) + await loop.value + } + + private func runNextStatement(_ fixture: Fixture, ownsSessionState: Bool = true) async throws { + let scope = DatabaseScope(connectionId: fixture.id, database: "app", schema: nil) + try await fixture.manager.withScopedDriver( + scope: scope, route: .sessionDriver, cancellation: .untracked, ownsSessionState: ownsSessionState + ) { _ in () } + } + + private func nextStatementRefusal(_ fixture: Fixture) async -> DatabaseError? { + do { + try await runNextStatement(fixture) + return nil + } catch { + return error as? DatabaseError + } + } + + @Test("A recovery that replaces a driver whose session set something up refuses the next statement once") + func recoveryReportsTheSessionStateItEnded() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["CREATE TABLE #orders (id int)"], on: fixture) + #expect(manager.sessionStateLedger.state(heldBy: fixture.driver, for: fixture.id) == .temporaryObjects) + + try await recover(fixture) + + #expect(fixture.session?.driver !== fixture.driver) + let refusal = await nextStatementRefusal(fixture) + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .temporaryObjects) + ) + #expect(refusal?.localizedDescription.contains(expected) == true) + try await runNextStatement(fixture) + await cleanUp(fixture) + } + + @Test("A recovery that ends a transaction and what the session set up names both in one message") + func recoveryReportsTransactionAndStateInOneMessage() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .inTransaction + try await runUserStatements(["SET XACT_ABORT ON"], on: fixture) + #expect(manager.sessionTransactions.holdsTransaction(connectionId: fixture.id, container: "app")) + + try await recover(fixture) + + let refusal = await nextStatementRefusal(fixture) + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: true, state: .sessionSettings) + ) + #expect(refusal?.localizedDescription.contains(expected) == true) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + await cleanUp(fixture) + } + + // A grid save, an Explain or an MCP statement takes the report as the user's statement would, as a lost + // transaction's report always went. + @Test("The first turn that owns the session after the loss is refused once, whatever work it is, and the next runs") + func firstOwningTurnTakesTheStateReport() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["CREATE TABLE #orders (id int)"], on: fixture) + + try await recover(fixture) + + let scope = DatabaseScope(connectionId: fixture.id, database: "app", schema: nil) + let error = await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .protectedWrite) { _ in () } + } + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .temporaryObjects) + ) + #expect(error?.localizedDescription.contains(expected) == true) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + try await runNextStatement(fixture) + await cleanUp(fixture) + } + + @Test("A reconnect the user asks for reports a lane's lost transaction but not what its session set up") + func manualReconnectReportsNoSessionState() async throws { + let fixture = makeFixture() + let manager = fixture.manager + let logs = MockDatabaseDriver(connection: fixture.connection) + logs.sessionTransactionStateToReturn = .idle + let audit = MockDatabaseDriver(connection: fixture.connection) + audit.sessionTransactionStateToReturn = .inTransaction + manager.sessionLanes.park(logs, database: "logs", for: fixture.id) + manager.sessionLanes.park(audit, database: "audit", for: fixture.id) + let logsLane = SessionTransactionTracker.Key(connectionId: fixture.id, lane: "logs") + let auditLane = SessionTransactionTracker.Key(connectionId: fixture.id, lane: "audit") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .temporaryObjects), on: logs, for: logsLane) + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .sessionSettings), on: audit, for: auditLane) + // A session that stopped answering moves the pooled transport, as a rebuilt tunnel does. + manager.updateSession(fixture.id) { $0.liveness = .unreachable(nil) } + + try? await manager.reconnectSession(fixture.id) + + #expect(logs.disconnectCallCount == 1) + #expect(audit.disconnectCallCount == 1) + #expect(!manager.sessionLanes.hasLoss(database: "logs", for: fixture.id)) + #expect( + manager.sessionLanes.takeLoss(database: "audit", for: fixture.id) + == SessionLoss(transaction: true, state: []) + ) + await cleanUp(fixture) + } + + @Test("A recovery of a session whose statements set nothing up refuses nothing") + func recoveryOfASessionThatSetNothingUpReportsNothing() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["SELECT * FROM orders", "UPDATE orders SET total = 0 WHERE id = 1"], on: fixture) + #expect(manager.sessionStateLedger.state(heldBy: fixture.driver, for: fixture.id).isEmpty) + + try await recover(fixture) + + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + try await runNextStatement(fixture) + await cleanUp(fixture) + } + + @Test("A startup command the user runs again is not reported, since the reconnect runs it too") + func startupCommandRerunByTheUserIsNotReported() async throws { + let fixture = makeFixture { $0.startupCommands = "SET NOCOUNT ON" } + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["SET NOCOUNT ON;"], on: fixture) + #expect(manager.sessionStateLedger.state(heldBy: fixture.driver, for: fixture.id).isEmpty) + + try await recover(fixture) + + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + try await runNextStatement(fixture) + await cleanUp(fixture) + } + + @Test("A metadata read on the session driver leaves the lost session state's report for the user's statement") + func metadataReadLeavesTheStateReport() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["DECLARE c CURSOR FOR SELECT 1"], on: fixture) + + try await recover(fixture) + try await runNextStatement(fixture, ownsSessionState: false) + + #expect(manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + let refusal = await nextStatementRefusal(fixture) + #expect(refusal != nil) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + await cleanUp(fixture) + } + + @Test("The report names kinds only, never a statement, a name or a value it carried") + func stateMessageNamesNoStatementText() async throws { + let fixture = makeFixture() + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements( + ["SET app.key = 'secret'", "EXEC sp_set_session_context 'api_token', 'hunter2'"], on: fixture + ) + + try await recover(fixture) + + let refusal = await nextStatementRefusal(fixture) + let message = refusal?.localizedDescription ?? "" + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .sessionSettings) + ) + #expect(message.contains(expected)) + for leaked in ["secret", "app.key", "api_token", "hunter2", "sp_set_session_context"] { + #expect(!message.contains(leaked)) + } await cleanUp(fixture) } @@ -544,7 +738,7 @@ struct SessionRecoveryLoopTests { #expect(fixture.manager.sessionLanes.parkedDriver(for: fixture.id, database: "logs") == nil) #expect(lane.disconnectCallCount == 1) - #expect(fixture.manager.sessionLanes.hasTransactionLoss(database: "logs", for: fixture.id)) + #expect(fixture.manager.sessionLanes.hasLoss(database: "logs", for: fixture.id)) #expect(fixture.session?.driver === fixture.driver) await cleanUp(fixture) } @@ -587,7 +781,7 @@ struct SessionRecoveryLoopTests { #expect(manager.sessionLanes.parkedDriver(for: id, database: "logs") === replacementLane) #expect(replacementLane.disconnectCallCount == 0) - #expect(!manager.sessionLanes.hasTransactionLoss(database: "logs", for: id)) + #expect(!manager.sessionLanes.hasLoss(database: "logs", for: id)) #expect(fixture.session?.driver !== fixture.driver) #expect(fixture.session?.liveness == .live) await cleanUp(fixture) diff --git a/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift b/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift index 1d8ab421c1..2e741ec1a0 100644 --- a/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift +++ b/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift @@ -110,7 +110,7 @@ struct SwitchDatabasePooledConnectionTests { #expect(driver.disconnectCallCount == 0) #expect(manager.sessionTransactions.holdsTransaction(connectionId: connection.id, container: "app")) - #expect(!manager.sessionLanes.hasTransactionLoss(database: "app", for: connection.id)) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: connection.id)) #expect(manager.session(for: connection.id)?.resolvedBrowseDatabase == "app") #expect(manager.session(for: connection.id)?.status == .connected) } @@ -145,7 +145,7 @@ struct SwitchDatabasePooledConnectionTests { try? await manager.switchDatabase(to: "unreachable", for: connection.id, persist: false) #expect(driver.disconnectCallCount >= 1) - #expect(manager.sessionLanes.hasTransactionLoss(database: "app", for: connection.id)) + #expect(manager.sessionLanes.hasLoss(database: "app", for: connection.id)) } @Test("A switch that reconnects a connection that stopped answering closes its pooled connections") diff --git a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift index e8b611a013..4a5284387e 100644 --- a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift +++ b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift @@ -158,6 +158,22 @@ private final class MockTrinoPlugin: NSObject, TableProPlugin, DriverPlugin { } } +private final class MockPostgreSQLPlugin: NSObject, TableProPlugin, DriverPlugin { + static let pluginName = "Mock PostgreSQL" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Stands in for the bundled PostgreSQL plugin" + static let capabilities: [PluginCapability] = [.databaseDriver] + + static let databaseTypeId = "PostgreSQL" + static let databaseDisplayName = "PostgreSQL" + static let iconName = "postgresql-icon" + static let defaultPort = 5_432 + + func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { + fatalError("Not used in tests") + } +} + private final class MockClickHousePlugin: NSObject, TableProPlugin, DriverPlugin { static let pluginName = "Mock ClickHouse" static let pluginVersion = "1.0.0" @@ -369,8 +385,8 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(snapshot.capabilities.supportsRawFilter) } - /// None of these has a `DriverPlugin` static, so a plugin that registers would reset them to the struct - /// defaults: counts on every open that read every partition, a header sort CQL refuses, and Match Any. + // None of these has a `DriverPlugin` static, so a plugin that registers would reset them to the struct + // defaults: counts on every open that read every partition, a header sort CQL refuses, and Match Any. @Test("Cassandra keeps its count, sort and filter limits when its plugin registers") func cassandraKeepsItsCuratedQueryLimits() { let built = PluginMetadataRegistry.shared.buildMetadataSnapshot(from: MockCassandraPlugin.self) @@ -433,8 +449,8 @@ struct PluginMetadataRegistryCuratedCapabilityTests { } } - /// Measured: `ADD … NOT NULL` and `COMMENT ON COLUMN` are syntax errors, `ALTER … TYPE` is refused, and only a - /// primary key column can be renamed, so the Structure tab offers adding and dropping a column and nothing else. + // Measured: `ADD … NOT NULL` and `COMMENT ON COLUMN` are syntax errors, `ALTER … TYPE` is refused, and only a + // primary key column can be renamed, so the Structure tab offers adding and dropping a column and nothing else. @Test("Cassandra and ScyllaDB offer only the column edits CQL can run", arguments: ["Cassandra", "ScyllaDB"]) func cassandraStructureEdits(typeId: String) throws { let snapshot = try #require(PluginMetadataRegistry.shared.snapshot(forRegisteredTypeId: typeId)) @@ -480,8 +496,8 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.connection.defaultLocalSocketPath == MySQLLocalSocket.defaultPath) } - /// A MySQL-protocol session opened with no database has none at all, so it lists nothing until - /// one is chosen. Databend's session falls back to its `default` database instead. + // A MySQL-protocol session opened with no database has none at all, so it lists nothing until + // one is chosen. Databend's session falls back to its `default` database instead. @Test("Only the MySQL engines with no default database require a selected database to browse") func browsingRequiresSelectedDatabasePerEngine() { let registry = PluginMetadataRegistry.shared @@ -591,6 +607,49 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.capabilities.verifiesServerWithSystemTrust == true) } + @Test("PostgreSQL keeps system trust, and TLS asked for without a mode stays Required, when its plugin registers") + func postgreSQLKeepsItsTLSCapabilities() { + let built = PluginMetadataRegistry.shared.buildMetadataSnapshot(from: MockPostgreSQLPlugin.self) + + #expect(built.capabilities.verifiesServerWithSystemTrust == true) + #expect(built.capabilities.requestedTLSVerifiesIdentity == false) + } + + @Test("Trino and ClickHouse keep Verify Identity for TLS asked for without a mode when their plugins register") + func requestedTLSSurvivesRegistration() { + let registry = PluginMetadataRegistry.shared + + #expect(registry.buildMetadataSnapshot(from: MockTrinoPlugin.self).capabilities.requestedTLSVerifiesIdentity) + #expect(registry.buildMetadataSnapshot(from: MockClickHousePlugin.self).capabilities.requestedTLSVerifiesIdentity) + #expect(registry.buildMetadataSnapshot(from: MockHanaPlugin.self).capabilities.requestedTLSVerifiesIdentity) + } + + @Test( + "The libpq engines check the system trust store, while TLS asked for without a mode stays Required", + arguments: ["PostgreSQL", "Redshift", "CockroachDB", "KingbaseES"] + ) + func libpqEnginesSplitTheTwoTLSFlags(typeId: String) throws { + let curated = try #require( + PluginMetadataRegistry.shared.builtInDefaults().first { $0.typeId == typeId }?.snapshot + ) + + #expect(curated.capabilities.verifiesServerWithSystemTrust == true) + #expect(curated.capabilities.requestedTLSVerifiesIdentity == false) + } + + @Test( + "The engines that already verified the system trust store keep both TLS flags", + arguments: ["SAP HANA", "Kafka", "Trino", "Presto", "Qdrant", "DuckDB", "ClickHouse"] + ) + func systemTrustEnginesKeepBothTLSFlags(typeId: String) throws { + let curated = try #require( + PluginMetadataRegistry.shared.builtInDefaults().first { $0.typeId == typeId }?.snapshot + ) + + #expect(curated.capabilities.verifiesServerWithSystemTrust == true) + #expect(curated.capabilities.requestedTLSVerifiesIdentity == true) + } + @Test("etcd offers no generic SSL Mode, since its driver reads only its own TLS Mode field") @MainActor func etcdDeclaresNoGenericSSL() { @@ -624,6 +683,7 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.capabilities.filtersTypedByReadRows == false) #expect(built.capabilities.tlsImpliedPorts.isEmpty) #expect(built.capabilities.verifiesServerWithSystemTrust == false) + #expect(built.capabilities.requestedTLSVerifiesIdentity == false) #expect(built.capabilities.supportsPerConnectionCertificatePaths == true) #expect(built.capabilities.newDatabaseFileExtensions.isEmpty) #expect(built.schema.implicitSchemaName == nil) diff --git a/apple/TableProTests/Core/Services/DateFormattingServiceTests.swift b/apple/TableProTests/Core/Services/DateFormattingServiceTests.swift index ecec6574f7..1fc6accabd 100644 --- a/apple/TableProTests/Core/Services/DateFormattingServiceTests.swift +++ b/apple/TableProTests/Core/Services/DateFormattingServiceTests.swift @@ -369,3 +369,62 @@ struct DateFormattingServiceTimeZoneTests { #expect(written == wire) } } + +// Work off the main actor builds its own renderer, so it has to print exactly what the service prints. +@MainActor +struct DateDisplayRendererTests { + private struct Sample: Sendable { + let text: String + let columnType: ColumnType? + } + + private static var samples: [Sample] { + let shapes: [ColumnType?] = [ + .datetime(rawType: "DATETIME"), + .datetime(rawType: "DATETIME(6)"), + .date(rawType: "DATE"), + .timestamp(rawType: "TIME"), + .timestamp(rawType: "TIMETZ"), + .timestamp(rawType: "TIMESTAMPTZ"), + nil + ] + let texts = DatabaseDateParserTests.wireSpellings + [ + "2024-12-31 23:59:59-03:30", + "2024-12-31 23:59:59+05:45", + "2024-12-31 23:59:59Z", + "2024-03-15 09:30:00.000000", + "2024-03-15 09:30:00.999999999", + "09:30:00.123456+05:30", + "not-a-date" + ] + return texts.flatMap { text in shapes.map { Sample(text: text, columnType: $0) } } + } + + private static let instants: [Date] = [0, 1_700_000_000, 1_709_294_400.5, -86_400].map(Date.init(timeIntervalSince1970:)) + + // The service hands its calls to a renderer, so this pins the same text on and off the main actor; the literal tests above pin the text. + @Test("A renderer off the main actor prints the same text as the service on the main actor, under every format") + func rendererMatchesTheService() async { + let samples = Self.samples + let original = DateFormattingService.shared.currentFormat + var printed: [DateFormatOption: [String?]] = [:] + for format in DateFormatOption.allCases { + DateFormattingService.shared.updateFormat(format) + printed[format] = samples.map { DateFormattingService.shared.format(dateString: $0.text, columnType: $0.columnType) } + + Self.instants.map { Optional(DateFormattingService.shared.format($0)) } + } + DateFormattingService.shared.updateFormat(original) + + for format in DateFormatOption.allCases { + let rendered = await Self.render(samples, instants: Self.instants, format: format) + #expect(rendered == printed[format], "\(format)") + } + } + + @concurrent + nonisolated private static func render(_ samples: [Sample], instants: [Date], format: DateFormatOption) async -> [String?] { + let renderer = DateDisplayRenderer(format: format) + return samples.map { renderer.format(dateString: $0.text, columnType: $0.columnType) } + + instants.map { Optional(renderer.format($0)) } + } +} diff --git a/apple/TableProTests/Core/Services/Query/DatabaseTreeMetadataServiceTests.swift b/apple/TableProTests/Core/Services/Query/DatabaseTreeMetadataServiceTests.swift index 9513e89f5e..456bc825e2 100644 --- a/apple/TableProTests/Core/Services/Query/DatabaseTreeMetadataServiceTests.swift +++ b/apple/TableProTests/Core/Services/Query/DatabaseTreeMetadataServiceTests.swift @@ -628,4 +628,66 @@ struct DatabaseTreeMetadataServicePooledScopeTests { #expect(MetadataConnectionPool.shared.pooledDriverCount(for: connection.id) == 1) await tearDown(connection) } + + // Completion reads on the interactive connection, so a Refresh of rows already on screen must not queue there. + private func twoWorkloadSession() -> (DatabaseConnection, interactive: MockDatabaseDriver, bulk: MockDatabaseDriver) { + let connection = TestFixtures.makeConnection(database: "shop", type: .postgresql) + var session = ConnectionSession(connection: connection, driver: MockDatabaseDriver(connection: connection)) + session.status = .connected + session.browseDatabase = "shop" + session.browseSchema = "public" + DatabaseManager.shared.injectSession(session, for: connection.id) + let interactive = MockDatabaseDriver(connection: connection) + let bulk = MockDatabaseDriver(connection: connection) + for (driver, workload) in [(interactive, MetadataConnectionPool.Workload.interactive), (bulk, .bulk)] { + MetadataConnectionPool.shared.injectEntry( + driver, scope: DatabaseScope(connectionId: connection.id, database: "shop", schema: "public"), workload: workload + ) + MetadataConnectionPool.shared.injectEntry( + driver, scope: DatabaseScope(connectionId: connection.id, database: "shop", schema: nil), workload: workload + ) + } + return (connection, interactive, bulk) + } + + @Test("A Refresh of tables already on screen reads on the bulk connection") + func refreshOfLoadedTablesReadsOnTheBulkConnection() async { + let (connection, interactive, bulk) = twoWorkloadSession() + let service = DatabaseTreeMetadataService.shared + + await service.loadTables(connectionId: connection.id, database: "shop", schema: "public") + #expect(interactive.fetchScopedTablesCallCount == 1) + await service.refreshTableObjects(connectionId: connection.id, database: "shop", schema: "public") + + #expect(interactive.fetchScopedTablesCallCount == 1) + #expect(bulk.fetchScopedTablesCallCount == 1) + await tearDown(connection) + } + + @Test("A Refresh of a schema list already on screen reads on the bulk connection") + func refreshOfLoadedSchemaListReadsOnTheBulkConnection() async { + let (connection, interactive, bulk) = twoWorkloadSession() + let service = DatabaseTreeMetadataService.shared + + await service.loadSchemas(connectionId: connection.id, database: "shop") + let firstReads = interactive.fetchSchemasCallCount + #expect(firstReads == 1) + await service.refreshSchemas(connectionId: connection.id, database: "shop") + + #expect(interactive.fetchSchemasCallCount == firstReads) + #expect(bulk.fetchSchemasCallCount == 1) + await tearDown(connection) + } + + @Test("A Retry with nothing on screen stays on the interactive connection") + func retryWithNothingOnScreenStaysInteractive() async { + let (connection, interactive, bulk) = twoWorkloadSession() + let service = DatabaseTreeMetadataService.shared + + await service.refreshTableObjects(connectionId: connection.id, database: "shop", schema: "public") + + #expect(interactive.fetchScopedTablesCallCount == 1) + #expect(bulk.fetchScopedTablesCallCount == 0) + await tearDown(connection) + } } diff --git a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift index 739be16695..3e3be93641 100644 --- a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift +++ b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift @@ -14,40 +14,6 @@ import Foundation import TableProTestSupport import Testing -/// Stands in for opening a pooled connection, remembering each driver by the database it was -/// opened for. -@MainActor -private final class CountingOpener { - private let connectDelaySeconds: Double - private(set) var opened: [MockDatabaseDriver] = [] - private(set) var openedByDatabase: [String: MockDatabaseDriver] = [:] - private(set) var openOrder: [String] = [] - private(set) var peakOpenCount = 0 - var afterOpen: ((DatabaseScope) -> Void)? - var deadOnArrival: Set = [] - - init(connectDelaySeconds: Double = 0) { - self.connectDelaySeconds = connectDelaySeconds - } - - var openCount: Int { - opened.filter { $0.disconnectCallCount == 0 }.count - } - - func open(_ scope: DatabaseScope) async throws -> DatabaseDriver { - let driver = MockDatabaseDriver() - driver.connectDelaySeconds = connectDelaySeconds - driver.hasLostConnection = deadOnArrival.remove(scope.database) != nil - opened.append(driver) - openedByDatabase[scope.database] = driver - openOrder.append(scope.database) - peakOpenCount = max(peakOpenCount, openCount) - try await driver.connect() - afterOpen?(scope) - return driver - } -} - @MainActor private final class StartedTaskBox { var task: Task? diff --git a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolSchemaTests.swift b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolSchemaTests.swift new file mode 100644 index 0000000000..5726f59284 --- /dev/null +++ b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolSchemaTests.swift @@ -0,0 +1,399 @@ +// +// MetadataConnectionPoolSchemaTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +private struct SchemaSighting: Equatable, Sendable { + let driver: ObjectIdentifier + let schema: String? + + init(_ driver: DatabaseDriver) { + self.driver = ObjectIdentifier(driver) + schema = (driver as? MockDatabaseDriver)?.currentSchema + } +} + +private struct NestedSighting: Sendable { + let outer: SchemaSighting + let inner: SchemaSighting +} + +@MainActor +private final class SchemaLeaseLog { + private(set) var entries: [String] = [] + + func record(_ entry: String) { + entries.append(entry) + } +} + +// Each schema of a database held its own pooled connection, so work across three schemas at the cap evicted and redialed. +@Suite("MetadataConnectionPool schemas", .serialized) +@MainActor +struct MetadataConnectionPoolSchemaTests { + private let limit = MetadataConnectionPool.maxPerConnection + + private func makeSession(startupCommands: String? = nil) -> DatabaseConnection { + var connection = TestFixtures.makeConnection(database: "shop") + connection.startupCommands = startupCommands + var session = ConnectionSession(connection: connection, driver: MockDatabaseDriver(connection: connection)) + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + return connection + } + + private func scope(_ connection: DatabaseConnection, _ schema: String?) -> DatabaseScope { + DatabaseScope(connectionId: connection.id, database: "shop", schema: schema) + } + + private func probe(_ pool: MetadataConnectionPool, _ scope: DatabaseScope) async throws -> SchemaSighting { + try await pool.withDriver(scope: scope) { driver in SchemaSighting(driver) } + } + + @Test("Leases on three schemas of one database, one after another, open one connection") + func schemasOfOneDatabaseShareAnIdleConnection() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + for schema in ["a", "b", "c"] { + _ = try await probe(pool, scope(connection, schema)) + } + + #expect(opener.opened.count == 1) + #expect(pool.pooledDriverCount(for: connection.id) == 1) + } + + @Test("Each lease reads on its own schema, so a shared connection is moved before every change of schema") + func leaseRunsOnItsOwnSchema() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + var sightings: [SchemaSighting] = [] + for schema in ["a", "b", "a"] { + sightings.append(try await probe(pool, scope(connection, schema))) + } + let schemas = sightings.map { $0.schema } + let drivers = Set(sightings.map { $0.driver }) + + #expect(schemas == ["a", "b", "a"]) + #expect(drivers.count == 1) + #expect(opener.opened.first?.switchSchemaCallCount == 3) + } + + // One connection per database would make a schema's reads wait behind another schema's long lease. + @Test("A lease on another schema opens its own connection while the first schema's lease still runs", .timeLimit(.minutes(1))) + func busySchemaOpensASiblingForAnotherSchema() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + let gate = AsyncGate() + let held = scope(connection, "a") + let holding = Task { @MainActor in + try await pool.withDriver(scope: held) { driver in + await gate.wait() + return ObjectIdentifier(driver) + } + } + await gate.arrived(1) + + let sibling = try await probe(pool, scope(connection, "b")) + #expect(opener.opened.count == 2) + + await gate.open() + let heldOn = try await holding.value + #expect(sibling.driver != heldOn) + #expect(sibling.schema == "b") + } + + @Test("Two leases on one schema at once queue on one connection") + func sameSchemaStillQueuesOnItsConnection() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + let gate = AsyncGate() + let log = SchemaLeaseLog() + let place = scope(connection, "a") + let first = Task { @MainActor in + try await pool.withDriver(scope: place) { driver in + await gate.wait() + await log.record("first") + return ObjectIdentifier(driver) + } + } + await gate.arrived(1) + let second = Task { @MainActor in + try await pool.withDriver(scope: place) { driver in + await log.record("second") + return ObjectIdentifier(driver) + } + } + try await waitUntil { pool.leaseCount(for: connection.id) == 2 } + #expect(opener.opened.count == 1) + + await gate.open() + let firstRanOn = try await first.value + let secondRanOn = try await second.value + + #expect(firstRanOn == secondRanOn) + #expect(log.entries == ["first", "second"]) + #expect(opener.opened.count == 1) + } + + // Data Compare and Object Copy lease one side inside the other, and both sides stream at once. + @Test( + "A lease nested in another schema's lease gets its own connection, past the limit when it is full", + .timeLimit(.minutes(1)) + ) + func nestedLeaseOnAnotherSchemaGetsItsOwnConnection() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + let gate = AsyncGate() + let fills = (1.. NativeDumpCommand { let tool = try #require(NativeDumpRegistry.descriptor(for: type)?.commandLineTool) let effective = overrideConnection ?? connection(type: type) + var request = NativeDumpDescriptor.Request( + connection: effective, + database: database, + fileURL: fileURL, + password: password, + scope: scope, + localFilePath: localFilePath ?? effective.database, + serverVersion: serverVersion + ) + request.systemTrustAnchorsFile = systemTrustAnchorsFile let built = try NativeDumpService.buildCommand( kind: kind, tool: tool, @@ -60,15 +71,7 @@ struct NativeDumpRegistryTests { flavor: flavor, versionText: toolVersionText ), - request: NativeDumpDescriptor.Request( - connection: effective, - database: database, - fileURL: fileURL, - password: password, - scope: scope, - localFilePath: localFilePath ?? effective.database, - serverVersion: serverVersion - ) + request: request ) if let folder = built.clientPluginFolder { try? FileManager.default.removeItem(at: folder.url) @@ -125,8 +128,8 @@ struct NativeDumpRegistryTests { #expect(try #require(NativeDumpRegistry.descriptor(for: .sqlite)).archiveFormat.fileExtension == "sql") } - /// Anything in `argv` is readable by every process on the machine through `ps`, so no - /// descriptor may put a password there. + // Anything in `argv` is readable by every process on the machine through `ps`, so no + // descriptor may put a password there. @Test("No engine puts the password in the argument list") func passwordNeverReachesArgv() throws { for type in [DatabaseType.postgresql, .mysql, .mongodb, .sqlite] { @@ -149,7 +152,7 @@ struct NativeDumpRegistryTests { #expect(built.arguments.last == "sales") } - /// `mysqldump` writes SQL to standard output, so the caller has to redirect it to the file. + // `mysqldump` writes SQL to standard output, so the caller has to redirect it to the file. @Test("MySQL is redirected through standard output in both directions") func mysqlRedirects() throws { let backup = try command(.mysql, kind: .backup) @@ -163,7 +166,7 @@ struct NativeDumpRegistryTests { #expect(!restore.arguments.contains("--single-transaction")) } - /// `pg_dump -Fc` is told the path and writes it itself, so nothing is redirected. + // `pg_dump -Fc` is told the path and writes it itself, so nothing is redirected. @Test("PostgreSQL writes its own file") func postgresWritesItsOwnFile() throws { let built = try command(.postgresql) @@ -171,8 +174,8 @@ struct NativeDumpRegistryTests { #expect(built.redirectedFileURL == nil) } - /// `mongodump` reads a password from neither the environment nor standard input, so the only - /// channel left is a config file, which must be owner-only and must not survive the process. + // `mongodump` reads a password from neither the environment nor standard input, so the only + // channel left is a config file, which must be owner-only and must not survive the process. @Test("MongoDB writes an owner-only credentials file and points at it") func mongoUsesACredentialsFile() throws { let built = try command(.mongodb) @@ -352,7 +355,7 @@ struct NativeDumpRegistryTests { ) } - /// The database is a file the tool opens, so there is nothing to authenticate to. + // The database is a file the tool opens, so there is nothing to authenticate to. @Test("SQLite passes the file path and no network arguments") func sqliteUsesTheFilePath() throws { let file = connection(type: .sqlite, host: "", port: 0, database: "/tmp/app.sqlite", username: "") @@ -390,9 +393,9 @@ struct NativeDumpRegistryTests { #expect(!FileManager.default.fileExists(atPath: marker.path)) } - /// libSQL claims the SQLite descriptor and keeps its path in a plugin-declared additional - /// field, leaving `database` empty. Reading `database` handed `sqlite3` an empty path, which - /// exits 0 after writing a 52-byte file the result sheet then reported as a backup. + // libSQL claims the SQLite descriptor and keeps its path in a plugin-declared additional + // field, leaving `database` empty. Reading `database` handed `sqlite3` an empty path, which + // exits 0 after writing a 52-byte file the result sheet then reported as a backup. @Test("libSQL dumps the file its driver opens, not the empty database field") func libsqlUsesItsResolvedPath() throws { var remote = connection(type: .libsql, host: "", port: 0, database: "", username: "") @@ -437,8 +440,8 @@ struct NativeDumpRegistryTests { } } - /// A libSQL connection to a Turso URL has no local file, so `sqlite3` cannot reach it at all. - /// Answering by type alone is what offered Backup Dump on one and wrote nothing. + // A libSQL connection to a Turso URL has no local file, so `sqlite3` cannot reach it at all. + // Answering by type alone is what offered Backup Dump on one and wrote nothing. @Test("A connection with no local file is not offered a dump") func remoteFileBackedConnectionIsUnsupported() { var remote = connection(type: .libsql, host: "", port: 0, database: "", username: "") @@ -595,9 +598,9 @@ struct NativeDumpRegistryTests { #expect(fromDirectory.arguments.starts(with: ["--no-password", "-h", "/var/run/postgresql", "-p", "5434"])) } - /// Measured, MariaDB 12.3.3 answers any `--ssl-mode` with `unknown variable` and exit 7, and - /// MySQL 8.4.11 answers `--ssl` with `unknown option` and exit 2, so neither spelling may reach - /// the other family's tool in either direction (#3046). + // Measured, MariaDB 12.3.3 answers any `--ssl-mode` with `unknown variable` and exit 7, and + // MySQL 8.4.11 answers `--ssl` with `unknown option` and exit 2, so neither spelling may reach + // the other family's tool in either direction (#3046). @Test("Neither client family is ever handed the other one's SSL flags", arguments: [ NativeDumpKind.backup, .restore ]) @@ -614,8 +617,8 @@ struct NativeDumpRegistryTests { #expect(!mysql.arguments.contains("--skip-ssl")) } - /// SSL off is the other half of the same defect: the old code sent `--ssl-mode=DISABLED`, which - /// MariaDB rejects exactly as it rejects the rest. + // SSL off is the other half of the same defect: the old code sent `--ssl-mode=DISABLED`, which + // MariaDB rejects exactly as it rejects the rest. @Test("An SSL-off connection is disabled in the tool's own spelling") func sslDisabledFollowsTheResolvedTool() throws { let maria = try command(.mysql, flavor: .mariadb) @@ -626,8 +629,8 @@ struct NativeDumpRegistryTests { #expect(mysql.arguments.contains("--ssl-mode=DISABLED")) } - /// `--ssl-cert` implies `--ssl` on MariaDB, so a connection whose SSL is off must not carry the - /// certificate the form still holds. + // `--ssl-cert` implies `--ssl` on MariaDB, so a connection whose SSL is off must not carry the + // certificate the form still holds. @Test("Certificate paths reach the tool, and only while SSL is on") func certificatePathsFollowTheMode() throws { var secured = connection(type: .mysql, sslMode: .verifyCa, sslEnabled: true) @@ -647,8 +650,8 @@ struct NativeDumpRegistryTests { #expect(!disabled.arguments.contains { $0.hasPrefix("--ssl-key") }) } - /// A tool that answered nothing is not guessed at once the connection asks for encryption: - /// MariaDB accepts `--loose-ssl-mode=REQUIRED` and ignores it, which is a cleartext dump. + // A tool that answered nothing is not guessed at once the connection asks for encryption: + // MariaDB accepts `--loose-ssl-mode=REQUIRED` and ignores it, which is a cleartext dump. @Test("An unidentified client refuses an encrypted connection rather than guessing") func unidentifiedToolRefusesEncryptedModes() throws { for mode in [SSLMode.required, .verifyCa, .verifyIdentity] { @@ -662,8 +665,8 @@ struct NativeDumpRegistryTests { #expect(!built.arguments.contains { $0.hasPrefix("--ssl") }) } - /// `mysqldump` 8.0 reads a table no MariaDB server and no MySQL before 8.0 has, and exits 2 - /// after writing part of the file. The flag that skips it is MySQL's own and backup only. + // `mysqldump` 8.0 reads a table no MariaDB server and no MySQL before 8.0 has, and exits 2 + // after writing part of the file. The flag that skips it is MySQL's own and backup only. @Test("A MySQL 8 dump tool skips column statistics on a server that has none") func columnStatisticsFlagFollowsTheServer() throws { let mysql8 = "mysqldump Ver 8.4.11 for macos26.6 on arm64 (Homebrew)" @@ -691,9 +694,9 @@ struct NativeDumpRegistryTests { #expect(!mariaTool.arguments.contains("--skip-column-statistics")) } - /// `my_getopt` parses past a positional argument, so a database named `--no-data` was read as - /// the option: measured on MySQL 8.4.11 and MariaDB 12.3.3, that dumped a different database - /// with no rows and exited 0, which the result sheet reports as a successful backup. + // `my_getopt` parses past a positional argument, so a database named `--no-data` was read as + // the option: measured on MySQL 8.4.11 and MariaDB 12.3.3, that dumped a different database + // with no rows and exited 0, which the result sheet reports as a successful backup. @Test("The option terminator comes before the database, in both directions", arguments: [ NativeDumpKind.backup, .restore ]) @@ -715,9 +718,9 @@ struct NativeDumpRegistryTests { #expect(Array(built.arguments[terminator...]) == ["--", "sales", "orders", "customers"]) } - /// libpq falls back to `~/.postgresql/root.crt` when no root certificate is named, so a - /// Verify CA connection that opens in the app could never be dumped: measured with pg_dump - /// 17.11, it fails with `root certificate file "..." does not exist`. + // libpq falls back to `~/.postgresql/root.crt` when no root certificate is named, so a + // Verify CA connection that opens in the app could never be dumped: measured with pg_dump + // 17.11, it fails with `root certificate file "..." does not exist`. @Test("PostgreSQL sends the whole SSL configuration, not just the mode") func postgresSendsCertificatePaths() throws { var secured = connection(type: .postgresql, sslMode: .verifyCa, sslEnabled: true) @@ -739,8 +742,8 @@ struct NativeDumpRegistryTests { #expect(disabled.environment["PGSSLKEY"] == nil) } - /// The CA belongs to the modes that verify one, which is the rule the live connection already - /// follows in `LibPQConnectionString`. + // The CA belongs to the modes that verify one, which is the rule the live connection already + // follows in `LibPQConnectionString`. @Test("A required connection sends its client certificate but not a CA it does not check") func postgresRequiredKeepsTheClientCertificate() throws { var secured = connection(type: .postgresql, sslMode: .required, sslEnabled: true) @@ -752,8 +755,96 @@ struct NativeDumpRegistryTests { #expect(built.environment["PGSSLCERT"] == "/certs/client.pem") } - /// MariaDB 11.0 renamed every client and some builds ship no `mysql`-prefixed symlink, so both - /// spellings have to be tried before reporting the tool missing. + private func dumpRequest(_ connection: DatabaseConnection) -> NativeDumpDescriptor.Request { + NativeDumpDescriptor.Request( + connection: connection, + database: "sales", + fileURL: URL(fileURLWithPath: "/tmp/out.dump"), + password: nil + ) + } + + @Test("Verify Identity with no CA file hands pg_dump this Mac's trusted roots") + func verifyIdentityWithoutACAGetsTheSystemAnchors() throws { + let verifying = connection(type: .postgresql, sslMode: .verifyIdentity, sslEnabled: true) + let tool = try #require(NativeDumpRegistry.descriptor(for: .postgresql)?.commandLineTool) + let roots = URL(fileURLWithPath: "/tmp/tablepro-trust-test/roots.pem") + + #expect(tool.needsSystemTrustAnchors(dumpRequest(verifying)) == NativeDumpRegistry.postgresNeedsSystemTrustAnchors( + dumpRequest(verifying) + )) + let built = try command(.postgresql, connection: verifying, systemTrustAnchorsFile: roots) + #expect(built.environment["PGSSLMODE"] == "verify-full") + #expect(built.environment["PGSSLROOTCERT"] == roots.path) + + var withCA = verifying + withCA.sslConfig.caCertificatePath = "/certs/ca.pem" + let chosen = try command(.postgresql, connection: withCA, systemTrustAnchorsFile: roots) + #expect(chosen.environment["PGSSLROOTCERT"] == "/certs/ca.pem") + } + + @Test("Verify CA with no CA file still names no root file, as libpq refuses this Mac's roots there") + func verifyCAWithoutACAStillSetsNoRootCertificate() throws { + let verifyCA = connection(type: .postgresql, sslMode: .verifyCa, sslEnabled: true) + let roots = URL(fileURLWithPath: "/tmp/tablepro-trust-test/roots.pem") + + let built = try command(.postgresql, connection: verifyCA, systemTrustAnchorsFile: roots) + + #expect(built.environment["PGSSLMODE"] == "verify-ca") + #expect(built.environment["PGSSLROOTCERT"] == nil) + #expect(!NativeDumpRegistry.postgresNeedsSystemTrustAnchors(dumpRequest(verifyCA), fileExists: { _ in false })) + } + + @Test("pg_dump gets this Mac's roots only for Verify Identity over TCP with no CA file and no root.crt of libpq's own") + func postgresSystemAnchorsFollowTheDriverRule() { + let home = ["HOME": "/Users/test"] + let rootCrt = "/Users/test/.postgresql/root.crt" + func needs(_ connection: DatabaseConnection, rootCrtExists: Bool = false) -> Bool { + NativeDumpRegistry.postgresNeedsSystemTrustAnchors( + dumpRequest(connection), + environment: home, + fileExists: { rootCrtExists && $0 == rootCrt } + ) + } + let verifying = connection(type: .postgresql, sslMode: .verifyIdentity, sslEnabled: true) + var withCA = verifying + withCA.sslConfig.caCertificatePath = "/certs/ca.pem" + var socket = verifying + socket.additionalFields[MySQLLocalSocket.fieldKey] = "/tmp/.s.PGSQL.5432" + + #expect(needs(verifying)) + #expect(needs(connection(type: .redshift, sslMode: .verifyIdentity, sslEnabled: true))) + #expect(!needs(verifying, rootCrtExists: true)) + #expect(!needs(withCA)) + #expect(!needs(socket)) + for mode in [SSLMode.disabled, .preferred, .required, .verifyCa] { + #expect(!needs(connection(type: .postgresql, sslMode: mode, sslEnabled: mode != .disabled))) + } + #expect(!NativeDumpRegistry.postgresNeedsSystemTrustAnchors( + dumpRequest(verifying), + environment: ["HOME": "/Users/test", "PGSSLROOTCERT": "/etc/corp-ca.pem"], + fileExists: { _ in false } + )) + #expect(NativeDumpRegistry.postgresSSLEnvironment( + verifying.sslConfig, inherited: ["PGSSLROOTCERT": "/etc/corp-ca.pem"] + )["PGSSLROOTCERT"] == "/etc/corp-ca.pem") + #expect(NativeDumpRegistry.postgresDefaultRootCertificatePath(environment: home) == rootCrt) + #expect(NativeDumpRegistry.postgresDefaultRootCertificatePath(environment: [:]) + == NSHomeDirectory() + "/.postgresql/root.crt") + } + + @Test("Only the PostgreSQL tools ask for this Mac's trusted roots", arguments: [ + DatabaseType.mysql, .mariadb, .mongodb, .sqlite, .mssql + ]) + func otherToolsNeverAskForTheSystemAnchors(type: DatabaseType) throws { + let tool = try #require(NativeDumpRegistry.descriptor(for: type)?.commandLineTool) + let verifying = connection(type: type, sslMode: .verifyIdentity, sslEnabled: true) + + #expect(!tool.needsSystemTrustAnchors(dumpRequest(verifying))) + } + + // MariaDB 11.0 renamed every client and some builds ship no `mysql`-prefixed symlink, so both + // spellings have to be tried before reporting the tool missing. @Test("MySQL tries both the mysql and mariadb tool names") func mysqlTriesBothToolNames() throws { let tool = try #require(NativeDumpRegistry.descriptor(for: .mysql)?.commandLineTool) @@ -786,8 +877,8 @@ struct NativeDumpRegistryTests { #expect(try configurationValue("password", in: contents) == #"p\nuri: mongodb://evil.example.com"#) } - /// A progress bar showing a percentage of a number nobody measured is worse than an - /// indeterminate one, so an engine with no cheap size answer returns nil. + // A progress bar showing a percentage of a number nobody measured is worse than an + // indeterminate one, so an engine with no cheap size answer returns nil. @Test("Only the engines with a cheap size query offer a determinate progress bar") func sizeQueryCoverage() { #expect(NativeDumpService.sizeQuery(for: .postgresql) != nil) diff --git a/apple/TableProTests/Helpers/CountingOpener.swift b/apple/TableProTests/Helpers/CountingOpener.swift new file mode 100644 index 0000000000..400640dd95 --- /dev/null +++ b/apple/TableProTests/Helpers/CountingOpener.swift @@ -0,0 +1,42 @@ +// +// CountingOpener.swift +// TableProTests +// + +import Foundation +@testable import TablePro + +// Stands in for opening a pooled connection, remembering each driver by the database it was opened for. +@MainActor +final class CountingOpener { + private let connectDelaySeconds: Double + private(set) var opened: [MockDatabaseDriver] = [] + private(set) var openedByDatabase: [String: MockDatabaseDriver] = [:] + private(set) var openOrder: [String] = [] + private(set) var peakOpenCount = 0 + var configure: ((MockDatabaseDriver) -> Void)? + var afterOpen: ((DatabaseScope) -> Void)? + var deadOnArrival: Set = [] + + init(connectDelaySeconds: Double = 0) { + self.connectDelaySeconds = connectDelaySeconds + } + + var openCount: Int { + opened.filter { $0.disconnectCallCount == 0 }.count + } + + func open(_ scope: DatabaseScope) async throws -> DatabaseDriver { + let driver = MockDatabaseDriver() + driver.connectDelaySeconds = connectDelaySeconds + driver.hasLostConnection = deadOnArrival.remove(scope.database) != nil + configure?(driver) + opened.append(driver) + openedByDatabase[scope.database] = driver + openOrder.append(scope.database) + peakOpenCount = max(peakOpenCount, openCount) + try await driver.connect() + afterOpen?(scope) + return driver + } +} diff --git a/apple/TableProTests/Helpers/ScopeRecordingKeychain.swift b/apple/TableProTests/Helpers/ScopeRecordingKeychain.swift index 12d980e421..c433bd475c 100644 --- a/apple/TableProTests/Helpers/ScopeRecordingKeychain.swift +++ b/apple/TableProTests/Helpers/ScopeRecordingKeychain.swift @@ -9,6 +9,7 @@ final class ScopeRecordingKeychain: KeychainStoring, @unchecked Sendable { private var local: [String: String] = [:] private var unreadableKeys: Set = [] private var failingKeys: Set = [] + private var failingSyncedRemovals: Set = [] func seedSynced(_ value: String, forKey key: String) { lock.lock() @@ -40,6 +41,18 @@ final class ScopeRecordingKeychain: KeychainStoring, @unchecked Sendable { failingKeys.insert(key) } + func failSyncedRemoval(of key: String) { + lock.lock() + defer { lock.unlock() } + failingSyncedRemovals.insert(key) + } + + func allowSyncedRemoval(of key: String) { + lock.lock() + defer { lock.unlock() } + failingSyncedRemovals.remove(key) + } + func allowWrites(to key: String) { lock.lock() defer { lock.unlock() } @@ -102,4 +115,14 @@ final class ScopeRecordingKeychain: KeychainStoring, @unchecked Sendable { synced[key] = nil } } + + // As the real keychain does, refused while no local value exists, so the secret is never lost on this Mac. + @discardableResult + func removeSyncedCopy(forKey key: String) -> Bool { + lock.lock() + defer { lock.unlock() } + guard local[key] != nil, !failingSyncedRemovals.contains(key) else { return false } + synced[key] = nil + return true + } } diff --git a/apple/TableProTests/Models/DatabaseTypeTests.swift b/apple/TableProTests/Models/DatabaseTypeTests.swift index 33b8797bea..fa788e1d7a 100644 --- a/apple/TableProTests/Models/DatabaseTypeTests.swift +++ b/apple/TableProTests/Models/DatabaseTypeTests.swift @@ -145,7 +145,7 @@ struct DatabaseTypeTests { func testCACertificateRequirement() { #expect(DatabaseType.trino.requiresCACertificate(for: .verifyIdentity) == false) #expect(DatabaseType.clickhouse.requiresCACertificate(for: .verifyIdentity) == false) - #expect(DatabaseType.postgresql.requiresCACertificate(for: .verifyIdentity) == true) + #expect(DatabaseType.postgresql.requiresCACertificate(for: .verifyIdentity) == false) #expect(DatabaseType.mysql.requiresCACertificate(for: .verifyIdentity) == true) #expect(DatabaseType(rawValue: "FutureDB").requiresCACertificate(for: .verifyIdentity) == true) } diff --git a/apple/TableProTests/ViewModels/ConnectionFormChildObservationTests.swift b/apple/TableProTests/ViewModels/ConnectionFormChildObservationTests.swift index 4d0b82aeca..1672d26628 100644 --- a/apple/TableProTests/ViewModels/ConnectionFormChildObservationTests.swift +++ b/apple/TableProTests/ViewModels/ConnectionFormChildObservationTests.swift @@ -76,4 +76,12 @@ struct ConnectionFormChildObservationTests { replacement.type = .postgresql #expect(counter.sends == 1) } + + @Test("a new connection saved Local only has nothing in iCloud Keychain to remove") + func newLocalOnlyConnectionRemovesNothing() { + let coordinator = ConnectionFormCoordinator(connectionId: nil) + coordinator.advanced.localOnly = true + + #expect(!coordinator.removesSyncedSecretsOnSave) + } } diff --git a/apple/TableProTests/ViewModels/SSLPaneViewModelTests.swift b/apple/TableProTests/ViewModels/SSLPaneViewModelTests.swift index 5a3e12f451..9cff10a01a 100644 --- a/apple/TableProTests/ViewModels/SSLPaneViewModelTests.swift +++ b/apple/TableProTests/ViewModels/SSLPaneViewModelTests.swift @@ -55,9 +55,13 @@ struct SSLPaneViewModelTests { } private func verifyIdentityIssues(for type: DatabaseType, caCertPath: String = "") -> [String] { + issues(for: type, mode: .verifyIdentity, caCertPath: caCertPath) + } + + private func issues(for type: DatabaseType, mode: SSLMode, caCertPath: String = "") -> [String] { let coordinator = ConnectionFormCoordinator(connectionId: nil) coordinator.network.type = type - coordinator.ssl.select(.verifyIdentity) + coordinator.ssl.select(mode) coordinator.ssl.caCertPath = caCertPath return coordinator.ssl.validationIssues } @@ -72,9 +76,20 @@ struct SSLPaneViewModelTests { @Test("Verify Identity without a CA file is refused for engines that need one") func fileTrustEnginesStillRequireACAFile() { - #expect(!DatabaseType.postgresql.verifiesServerWithSystemTrust) - #expect(verifyIdentityIssues(for: .postgresql).count == 1) - #expect(verifyIdentityIssues(for: .postgresql, caCertPath: "/tmp/ca.pem").isEmpty) + #expect(!DatabaseType.mysql.verifiesServerWithSystemTrust) + #expect(verifyIdentityIssues(for: .mysql).count == 1) + #expect(verifyIdentityIssues(for: .mysql, caCertPath: "/tmp/ca.pem").isEmpty) + } + + @Test( + "The libpq engines save Verify Identity without a CA file and still need one for Verify CA", + arguments: [DatabaseType.postgresql, .redshift, .cockroachdb, .kingbasees] + ) + func libpqEnginesNeedACAFileOnlyForVerifyCA(type: DatabaseType) { + #expect(type.verifiesServerWithSystemTrust) + #expect(verifyIdentityIssues(for: type).isEmpty) + #expect(issues(for: type, mode: .verifyCa).count == 1) + #expect(issues(for: type, mode: .verifyCa, caCertPath: "/tmp/ca.pem").isEmpty) } @Test("A Trino port of 443 escalates the type default to Verify Identity, and another port reverts it") diff --git a/apple/TableProTests/ViewModels/WelcomeViewModelTests.swift b/apple/TableProTests/ViewModels/WelcomeViewModelTests.swift index e4afcb5559..6404c8dd2c 100644 --- a/apple/TableProTests/ViewModels/WelcomeViewModelTests.swift +++ b/apple/TableProTests/ViewModels/WelcomeViewModelTests.swift @@ -332,6 +332,42 @@ final class WelcomeViewModelTests { #expect(viewModel.listState == .noFilterMatch) } + @Test func testExcludingFromSyncAsksFirst() throws { + let prod = DatabaseConnection(name: "Prod", type: .postgresql) + save([prod]) + viewModel.loadConnections() + + viewModel.perform(.setIncludedInSync([prod.id], false)) + + #expect(viewModel.showExcludeFromSyncConfirmation) + #expect(viewModel.connectionsToExcludeFromSync.map(\.id) == [prod.id]) + #expect(try #require(connectionStorage.loadConnection(id: prod.id)).localOnly == false) + } + + @Test func testConfirmingExcludeFromSyncMarksTheConnectionLocalOnly() throws { + let prod = DatabaseConnection(name: "Prod", type: .postgresql) + save([prod]) + viewModel.loadConnections() + viewModel.perform(.setIncludedInSync([prod.id], false)) + + viewModel.confirmExcludeFromSync() + + #expect(viewModel.connectionsToExcludeFromSync.isEmpty) + #expect(try #require(connectionStorage.loadConnection(id: prod.id)).localOnly) + } + + @Test func testIncludingInSyncNeedsNoConfirmation() throws { + var prod = DatabaseConnection(name: "Prod", type: .postgresql) + prod.localOnly = true + save([prod]) + viewModel.loadConnections() + + viewModel.perform(.setIncludedInSync([prod.id], true)) + + #expect(!viewModel.showExcludeFromSyncConfirmation) + #expect(try #require(connectionStorage.loadConnection(id: prod.id)).localOnly == false) + } + @Test func testDeletingTheLastConnectionClearsTheSearchItCanNoLongerRun() { let only = DatabaseConnection(name: "Only", type: .mysql, sortOrder: 0) save([only]) diff --git a/apple/TableProTests/Views/DataGrid/DataGridCommandTests.swift b/apple/TableProTests/Views/DataGrid/DataGridCommandTests.swift index f2f53cd346..565e4eba21 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridCommandTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridCommandTests.swift @@ -798,6 +798,67 @@ struct DataGridHeaderMenuTests { ]) } + @Test("The list Filter Values builds off the main actor is the one the main actor builds") + func distinctValuesOffTheMainActor() async throws { + let rows = TableRows.from( + queryRows: [ + [.text("2024-03-15 09:30:00.123456"), .text("1700000000"), .bytes(Data("signup".utf8)), .text("b")], + [.text("2024-03-15 09:30:00.123457"), .text("1700000000"), .bytes(Data([0x89, 0x50])), .null], + [.text("2024-03-15 09:30:00.123456"), .null, .bytes(Data("signup".utf8)), .text("a\rb")], + ], + columns: ["created", "stamp", "payload", "note"], + columnTypes: [ + .datetime(rawType: "DATETIME(6)"), + .integer(rawType: "BIGINT"), + .blob(rawType: "VARBINARY(255)"), + .text(rawType: "TEXT"), + ] + ) + let grid = CommandGrid(rows) + grid.model.setDisplayFormat(.unixTimestamp, for: try #require(grid.model.columns.column(at: 1))) + grid.model.setDisplayFormat(.text, for: try #require(grid.model.columns.column(at: 2))) + let created = try #require(grid.model.columns.column(at: 0)) + #expect(grid.commands.distinctValues(for: created).map { $0.count } == [2, 1]) + + for column in grid.model.columns.columns { + let offMain = await ColumnValueTally.distinctValuesOffTheMainActor( + grid.model.rows, + dataIndex: column.dataIndex, + columnType: column.type, + displayFormat: grid.model.displayFormat(for: column), + databaseType: grid.model.configuration.traits.databaseType, + dateFormat: DateFormattingService.shared.currentFormat + ) + #expect(offMain == grid.commands.distinctValues(for: column), "\(column.name)") + } + } + + @Test("Filter Values asks whether to stop every 4,096 rows and gives up when told to") + func valueTallyChecksForCancellation() { + let rows = TableRows.from( + queryRows: (0..<10_000).map { [PluginCellValue.text("\($0 % 7)")] }, + columns: ["v"], + columnTypes: [.text(rawType: "TEXT")] + ) + let dates = DateDisplayRenderer(format: .iso8601) + var checks = 0 + + let counted = ColumnValueTally.tally( + rows, dataIndex: 0, columnType: .text(rawType: "TEXT"), displayFormat: nil, databaseType: nil, dates: dates + ) { + checks += 1 + return false + } + let stopped = ColumnValueTally.tally( + rows, dataIndex: 0, columnType: .text(rawType: "TEXT"), displayFormat: nil, databaseType: nil, dates: dates, + isCancelled: { true } + ) + + #expect(counted?.order.count == 7) + #expect(checks == 3) + #expect(stopped == nil) + } + @Test("Fill Column is offered on a writable column that is not a key") func fillColumnOffer() { var configuration = CommandGrid.editable() diff --git a/apple/TableProTests/Views/DataGrid/DataGridControllerTests.swift b/apple/TableProTests/Views/DataGrid/DataGridControllerTests.swift index e423818a3e..abcad752eb 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridControllerTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridControllerTests.swift @@ -222,7 +222,7 @@ struct DataGridControllerTests { // MARK: - Find @Test("A find match moves the active cell to the match's column, so the arrow keys continue from it") - func findMatchSetsActiveColumn() { + func findMatchSetsActiveColumn() async { let grid = HostedDataGrid(rows: GridFixtureRows.people) defer { grid.close() } grid.model.setSelection(cell(0, 0)) @@ -230,6 +230,7 @@ struct DataGridControllerTests { grid.model.showFind() grid.model.find.term = "dee" grid.model.findQueryDidChange() + await grid.model.findSearchSettled() #expect(grid.model.selection.activeCell == GridCoord(row: 3, displayColumn: 1)) #expect(grid.main.selectedRowIndexes == IndexSet(integer: 3)) diff --git a/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift b/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift index ce095016eb..cee195562e 100644 --- a/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift +++ b/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift @@ -160,6 +160,19 @@ struct ResultRowsBuildOffMainTests { #expect(result.tableRows.value(at: 1, column: 1) == .text("j•••@example.com")) } + @Test("A batch with more results than a run keeps shows the last ones and says how many there were") + func batchKeepsTheLastResults() async throws { + let count = RunResultPlan.maxRowResults + 2 + let answered = (1...count).map { ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["\($0)", "x@example.com"]]) } + + let results = try await runBatch(answeredWith: answered) + + #expect(results.count == RunResultPlan.maxRowResults) + #expect(results.first?.tableRows.value(at: 0, column: 0) == .text("3")) + #expect(results.last?.tableRows.value(at: 0, column: 0) == .text("\(count)")) + #expect(results.allSatisfy { $0.statusMessage?.contains(count.formatted()) == true }) + } + @Test("A batch's numbered result sets are each masked and keep their order") func batchNumberedResultsAreMasked() async throws { let first = ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["1", "jane@example.com"]]) diff --git a/apple/TableProTests/Views/Results/InvisibleCharacterSettingTests.swift b/apple/TableProTests/Views/Results/InvisibleCharacterSettingTests.swift index 511e3343f5..9dbd8a9d6c 100644 --- a/apple/TableProTests/Views/Results/InvisibleCharacterSettingTests.swift +++ b/apple/TableProTests/Views/Results/InvisibleCharacterSettingTests.swift @@ -39,23 +39,23 @@ extension GlobalStateSuites { "Find matches a lone CR as the space the cell shows while marks are off", .appSettings { $0.dataGrid.showInvisibleCharacters = false } ) - func findReadsTheUnmarkedText() { + func findReadsTheUnmarkedText() async { let (model, owner) = makeModel(values: ["a\rb"]) defer { withExtendedLifetime(owner) {} } - #expect(find("a b", in: model).count == 1) + #expect(await find("a b", in: model).count == 1) } @Test( "Find does not read a marked CR as a space", .appSettings { $0.dataGrid.showInvisibleCharacters = true } ) - func findReadsTheMarkedText() { + func findReadsTheMarkedText() async { let (model, owner) = makeModel(values: ["a\rb"]) defer { withExtendedLifetime(owner) {} } - #expect(find("a b", in: model).isEmpty) - #expect(find("b", in: model).count == 1) + #expect(await find("a b", in: model).isEmpty) + #expect(await find("b", in: model).count == 1) } // The model holds its owner weakly, so the caller keeps the owner alive. @@ -74,10 +74,11 @@ extension GlobalStateSuites { return (model, owner) } - private func find(_ term: String, in model: DataGridModel) -> [GridFindState.Match] { + private func find(_ term: String, in model: DataGridModel) async -> [GridFindState.Match] { model.showFind() model.find.term = term model.findQueryDidChange() + await model.findSearchSettled() return model.find.matches } } diff --git a/apple/changelog.d/4200.added.md b/apple/changelog.d/4200.added.md new file mode 100644 index 0000000000..5ea239390d --- /dev/null +++ b/apple/changelog.d/4200.added.md @@ -0,0 +1 @@ +PostgreSQL connections set to Verify Identity check the server against this Mac's trusted roots without a CA file. diff --git a/apple/changelog.d/4200.changed.md b/apple/changelog.d/4200.changed.md new file mode 100644 index 0000000000..0cebd82ff0 --- /dev/null +++ b/apple/changelog.d/4200.changed.md @@ -0,0 +1 @@ +Run All shows statements that return no rows as one result, and keeps the last 100 results that return rows. diff --git a/apple/changelog.d/4200.fixed.1.md b/apple/changelog.d/4200.fixed.1.md new file mode 100644 index 0000000000..c57c00f578 --- /dev/null +++ b/apple/changelog.d/4200.fixed.1.md @@ -0,0 +1 @@ +Filter Values on a large column froze the window while it listed the values. diff --git a/apple/changelog.d/4200.fixed.2.md b/apple/changelog.d/4200.fixed.2.md new file mode 100644 index 0000000000..bd442f13c9 --- /dev/null +++ b/apple/changelog.d/4200.fixed.2.md @@ -0,0 +1 @@ +Grid edits to a row after another row's change saved out of order, failing saves such as a swap of unique values. diff --git a/apple/changelog.d/4200.fixed.3.md b/apple/changelog.d/4200.fixed.3.md new file mode 100644 index 0000000000..eb198d3b4d --- /dev/null +++ b/apple/changelog.d/4200.fixed.3.md @@ -0,0 +1 @@ +A sidebar Refresh held up autocomplete, and each schema of a database kept its own background connection. diff --git a/apple/changelog.d/4200.fixed.4.md b/apple/changelog.d/4200.fixed.4.md new file mode 100644 index 0000000000..9922a5daf1 --- /dev/null +++ b/apple/changelog.d/4200.fixed.4.md @@ -0,0 +1 @@ +After a dropped connection reconnected, statements ran without the temporary tables or settings the old session had. diff --git a/apple/changelog.d/4200.fixed.5.md b/apple/changelog.d/4200.fixed.5.md new file mode 100644 index 0000000000..ffb422e736 --- /dev/null +++ b/apple/changelog.d/4200.fixed.5.md @@ -0,0 +1 @@ +JSON import read UTF-16 and UTF-32 files, and files holding a single object, whole into memory. diff --git a/apple/changelog.d/4200.fixed.6.md b/apple/changelog.d/4200.fixed.6.md new file mode 100644 index 0000000000..cf51aee607 --- /dev/null +++ b/apple/changelog.d/4200.fixed.6.md @@ -0,0 +1 @@ +A chat reply cut off at the model's output limit gave no sign of it, and a tool call cut with it could run. diff --git a/apple/changelog.d/4200.fixed.md b/apple/changelog.d/4200.fixed.md new file mode 100644 index 0000000000..e2f2be9b9e --- /dev/null +++ b/apple/changelog.d/4200.fixed.md @@ -0,0 +1 @@ +Typing a Find term in a large grid froze the window while every cell was searched. diff --git a/apple/changelog.d/4200.security.md b/apple/changelog.d/4200.security.md new file mode 100644 index 0000000000..8497b232cf --- /dev/null +++ b/apple/changelog.d/4200.security.md @@ -0,0 +1 @@ +Marking a connection Local only left its passwords in iCloud Keychain on your other devices. diff --git a/apple/project.yml b/apple/project.yml index 41c0ec9abe..7c0b67e1d3 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -188,7 +188,7 @@ targetTemplates: PostgreSQLLinkage: dependencies: - package: TableProCore - products: [TableProCoreTypes, TableProGeometry, TableProKerberos, TableProLogRedaction, TableProPostgreSQLCore, TableProSQLVocabulary] + products: [TableProCoreTypes, TableProGeometry, TableProKerberos, TableProLogRedaction, TableProPostgreSQLCore, TableProSQLVocabulary, TableProTLSClientIdentity] settings: base: HEADER_SEARCH_PATHS: @@ -412,7 +412,7 @@ targets: - package: Yams - package: PostHog - package: TableProCore - products: [TableProAutocomplete, TableProConnectionLibrary, TableProCoreTypes, TableProEtcdCore, TableProFirebaseCore, TableProGeometry, TableProGoogleCloud, TableProGzip, TableProHTTP, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProNumberFormatting, TableProProcess, TableProQdrantCore, TableProRedisCore, TableProSQLGrammar, TableProSQLQuoting, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTabular, TableProTabularIO, TableProTheme, TableProWeaviateCore] + products: [TableProAutocomplete, TableProConnectionLibrary, TableProCoreTypes, TableProEtcdCore, TableProFirebaseCore, TableProGeometry, TableProGoogleCloud, TableProGzip, TableProHTTP, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProNumberFormatting, TableProProcess, TableProQdrantCore, TableProRedisCore, TableProSQLGrammar, TableProSQLQuoting, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTLSClientIdentity, TableProTabular, TableProTabularIO, TableProTheme, TableProWeaviateCore] - package: TableProMacCore products: [TableProCloudAPI, TableProMCP, TableProSSHConfig, TableProUtilities] # Writes nothing for a version CHANGELOG.md has no section for, so a 0.0.0-dev build ships no notes. diff --git a/apple/scripts/probes/check-system-trust-anchors.sh b/apple/scripts/probes/check-system-trust-anchors.sh new file mode 100755 index 0000000000..075a0c1fb5 --- /dev/null +++ b/apple/scripts/probes/check-system-trust-anchors.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# +# Whether the PEM that SystemTrustAnchors builds from this Mac's trust settings lets the bundled OpenSSL verify a live +# server the way libpq's verify-full does: SSL_VERIFY_PEER plus SSL_set1_host. The bundled OpenSSL's own default store +# is checked too. Its OPENSSLDIR is the libs builder's temporary folder, so that check is expected to fail with +# "unable to get local issuer certificate". +# +# Builds the shipping SystemTrustAnchors.swift with a small main that writes the PEM, and a TLS client linked against +# apple/Libs/libssl.a and libcrypto.a (run apple/scripts/download-libs.sh first, or set LIBS_DIR). Needs the network. +# A host is any TLS server whose chain should verify against this Mac's roots; port 443 unless one is given. +# +# Usage: apple/scripts/probes/check-system-trust-anchors.sh [host[:port]]... (default: www.postgresql.org) +# Exit 0: every host verifies with the exported roots. Exit 1: one does not, or the probe could not run. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +LIBS="${LIBS_DIR:-$ROOT/Libs}" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +for lib in libssl.a libcrypto.a; do + if [[ ! -f "$LIBS/$lib" ]]; then + echo "Missing $LIBS/$lib. Run apple/scripts/download-libs.sh first." >&2 + exit 1 + fi +done + +cat > "$WORK/main.swift" << 'EOF' +import Foundation + +let source = SecuritySystemTrustSource() +let start = Date() +let pem = try SystemTrustAnchors.pem(from: source) +let elapsed = Int(Date().timeIntervalSince(start) * 1_000) +try pem.write(to: URL(fileURLWithPath: CommandLine.arguments[1])) +let roots = try SystemTrustAnchors.trustedRoots(from: source).count +print("exported \(roots) roots, \(pem.count) bytes, \(elapsed) ms") +EOF + +cat > "$WORK/check.c" << 'EOF' +#include +#include +#include +#include +#include + +const void *TLS_client_method(void); +void *SSL_CTX_new(const void *method); +void SSL_CTX_free(void *ctx); +int SSL_CTX_set_default_verify_paths(void *ctx); +int SSL_CTX_load_verify_locations(void *ctx, const char *file, const char *dir); +void SSL_CTX_set_verify(void *ctx, int mode, void *callback); +void *SSL_new(void *ctx); +void SSL_free(void *ssl); +int SSL_set_fd(void *ssl, int fd); +long SSL_ctrl(void *ssl, int cmd, long larg, void *parg); +int SSL_set1_host(void *ssl, const char *host); +int SSL_connect(void *ssl); +long SSL_get_verify_result(const void *ssl); +const char *X509_verify_cert_error_string(long n); + +// SSL_set_tlsext_host_name is a macro over SSL_ctrl; 55 is SSL_CTRL_SET_TLSEXT_HOSTNAME. +static int check(const char *label, const char *pem, const char *host, const char *port) { + void *ctx = SSL_CTX_new(TLS_client_method()); + if (pem) { + if (SSL_CTX_load_verify_locations(ctx, pem, NULL) != 1) { + printf(" %-17s could not load %s\n", label, pem); + return 0; + } + } else { + SSL_CTX_set_default_verify_paths(ctx); + } + SSL_CTX_set_verify(ctx, 1, NULL); + + struct addrinfo hints = {0}, *res = NULL; + hints.ai_socktype = SOCK_STREAM; + if (getaddrinfo(host, port, &hints, &res) != 0) { + printf(" %-17s cannot resolve %s\n", label, host); + return 0; + } + int fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + if (connect(fd, res->ai_addr, res->ai_addrlen) != 0) { + printf(" %-17s cannot connect to %s:%s\n", label, host, port); + freeaddrinfo(res); + return 0; + } + freeaddrinfo(res); + void *ssl = SSL_new(ctx); + SSL_set_fd(ssl, fd); + SSL_ctrl(ssl, 55, 0, (void *)host); + SSL_set1_host(ssl, host); + int handshake = SSL_connect(ssl); + long verify = SSL_get_verify_result(ssl); + printf(" %-17s handshake %s, verify %ld (%s)\n", label, handshake == 1 ? "ok" : "failed", verify, + X509_verify_cert_error_string(verify)); + SSL_free(ssl); + close(fd); + SSL_CTX_free(ctx); + return handshake == 1 && verify == 0; +} + +int main(int argc, char **argv) { + if (argc < 4) return 2; + check("default store", NULL, argv[2], argv[3]); + return check("exported roots", argv[1], argv[2], argv[3]) ? 0 : 1; +} +EOF + +arch="$(uname -m)" +xcrun swiftc -swift-version 6 -target "$arch-apple-macos13.0" -o "$WORK/export-roots" \ + "$ROOT/Packages/TableProCore/Sources/TableProTLSClientIdentity/SystemTrustAnchors.swift" "$WORK/main.swift" +clang -arch "$arch" -mmacosx-version-min=13.0 -o "$WORK/check" "$WORK/check.c" \ + "$LIBS/libssl.a" "$LIBS/libcrypto.a" -lz + +"$WORK/export-roots" "$WORK/roots.pem" + +status=0 +for target in "${@:-www.postgresql.org}"; do + host="${target%%:*}" + port="443" + [[ "$target" == *:* ]] && port="${target##*:}" + echo "== $host:$port" + "$WORK/check" "$WORK/roots.pem" "$host" "$port" || status=1 +done +exit "$status"