From 5f6332f97d482caf234e2dc57eb2fa47902030cc Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 11 Nov 2026 14:39:10 +0700 Subject: [PATCH 1/5] fix(macos): keep T-SQL blocks and rewinds whole, stop CSV sanitizing by default, and fix Stop, defaults and gate cancels across drivers --- .../CassandraConditionalWrite.swift | 41 +++ .../RedisCommandSplitter.swift | 11 + .../SQLParenthesizedSemicolons.swift | 42 +++ .../SQLRoutineBodyTracker.swift | 124 +++++++-- .../SQLUnterminatedStatements.swift | 25 +- .../SqlBlockStructure.swift | 37 ++- .../Sources/TableProTeradataCore/Parcel.swift | 2 + .../TeradataConnection.swift | 204 +++++++++++++- .../TeradataMessages.swift | 8 + .../CassandraConditionalWriteTests.swift | 61 +++++ .../PluginParameterEscapingTests.swift | 11 + .../RedisCommandSplitterTests.swift | 12 + .../SQLParenthesizedSemicolonTests.swift | 77 ++++++ .../SQLStatementCursorIndexTests.swift | 13 +- .../TSQLControlFlowBlockTests.swift | 153 +++++++++++ .../TeradataStopTests.swift | 249 ++++++++++++++++++ .../TextViewController+EmphasizeBracket.swift | 17 +- .../Controller/TextViewControllerTests.swift | 18 ++ .../BeancountPluginDriver.swift | 6 +- .../BigtablePluginDriver+Execution.swift | 6 +- .../CSVExportPlugin/CSVExportModels.swift | 3 +- .../CSVExportOptionsView.swift | 6 +- .../CSVExportPlugin/CSVExportPlugin.swift | 95 +++++-- .../CSVImportPlugin/CSVImportPlugin.swift | 30 ++- .../CassandraPlugin.swift | 4 +- .../CassandraRowWriter.swift | 39 +-- .../DamengDriverPlugin/DamengPlugin.swift | 7 +- .../DynamoDBPluginDriver+Execution.swift | 4 +- .../EtcdDriverPlugin/EtcdPluginDriver.swift | 8 +- .../CSVExportFormulaAndDecimalTests.swift | 174 ++++++++++++ .../ExportImportRoundTripTests.swift | 17 +- .../FormatSettingsChangedMidRunTests.swift | 185 +++++++++++++ .../JSONExportPlugin/JSONExportPlugin.swift | 13 +- .../MarkdownExportPlugin.swift | 17 +- .../MongoDBConnection+SyncHelpers.swift | 5 +- .../MongoDBPluginDriver.swift | 7 +- .../MariaDBPluginConnection.swift | 36 ++- .../MariaDBStopThreadIdTests.swift | 38 +++ .../ParquetExportPlugin.swift | 9 +- .../RedisPluginDriver+Operations.swift | 5 +- .../RedisDriverPlugin/RedisPluginDriver.swift | 6 +- .../RedisConditionalWriteTests.swift | 19 ++ .../SQLImportPlugin/SQLImportPlugin.swift | 14 +- .../PluginDatabaseDriver.swift | 10 +- .../PluginExportUtilities.swift | 66 +++++ .../TableProPluginKit/PluginRowWriters.swift | 23 +- .../TeradataAsyncConnection.swift | 18 +- .../XLSXExportPlugin/XLSXExportPlugin.swift | 30 ++- .../Plugins/XLSXExportPlugin/XLSXWriter.swift | 63 +---- .../XLSXImportPlugin/XLSXImportPlugin.swift | 24 +- .../ChangeTracking/DataChangeModels.swift | 8 +- .../Core/ChangeTracking/DefaultedCells.swift | 38 +++ .../SQLStatementGenerator.swift | 7 +- .../Core/Compare/CompareSQLLiteral.swift | 22 +- .../Core/Compare/DataDiffEngine.swift | 26 +- .../Core/Compare/StreamingRowProvider.swift | 20 +- .../Coordinators/RowEditingCoordinator.swift | 6 +- .../DataWrite/Rewind/RewindExecutor.swift | 3 +- .../Core/DataWrite/Rewind/RewindPlanner.swift | 96 +++++-- .../DataWrite/RowChangeStatementFactory.swift | 111 +++++++- .../DataWrite/RowWriteOperationBuilder.swift | 9 +- .../DatabaseManager+ContainerDDL.swift | 36 +-- .../Database/DatabaseManager+Health.swift | 42 ++- .../Database/DatabaseManager+Principals.swift | 12 +- .../Database/DatabaseManager+Schema.swift | 14 +- .../Database/DatabaseManager+Sessions.swift | 5 +- .../Database/DatabaseManager+Startup.swift | 102 ++++++- .../Core/Database/DatabaseManager.swift | 4 + .../Core/Database/TriggerEditing.swift | 19 +- apple/TablePro/Core/LSP/LSPTransport.swift | 72 ++++- .../Core/ObjectCopy/ObjectCopyRowCopier.swift | 12 +- .../Core/ObjectCopy/ObjectCopyRunner.swift | 16 +- .../Plugins/ExportDataSourceAdapter.swift | 8 +- .../TablePro/Core/Plugins/ExportRowCut.swift | 11 + .../Execution/GatedOperationFailure.swift | 39 +++ .../Query/MetadataConnectionPool.swift | 18 +- .../Services/Query/RowOperationsManager.swift | 59 +++-- .../Core/Services/Query/RowParser.swift | 10 +- .../Query/SelectionSummaryCalculator.swift | 14 +- .../Shortcuts/Shared/PayloadRowPlanner.swift | 50 +++- .../Core/Shortcuts/Shared/RowPayload.swift | 20 +- .../Core/Shortcuts/ShortcutsRowWriter.swift | 5 +- .../UsersRoles/PrincipalChangeManager.swift | 17 +- .../JavaScript/QueryStatementModel.swift | 52 +++- .../Utilities/SQL/DisplayedResultReader.swift | 13 +- .../Core/Utilities/SQL/JsonRowConverter.swift | 2 +- .../Core/Utilities/SQL/QueryClassifier.swift | 13 +- .../Utilities/SQL/ResultJsonSerializer.swift | 2 + .../SQL/SQLRowToStatementConverter.swift | 14 +- .../Core/Vim/VimTextBufferAdapter.swift | 59 ----- .../PluginCellValue+DefaultMarker.swift | 19 +- .../ClickHouse/ClickHousePartInfo.swift | 4 +- .../Models/Query/SelectionSummary.swift | 10 +- .../Schema/ClickHousePartStatements.swift | 46 ++-- .../Models/UI/MultiRowEditState.swift | 8 +- .../TablePro/Resources/Localizable.xcstrings | 41 +++ .../ViewModels/SchemaEditorViewModel.swift | 2 +- .../ViewModels/UserTypeEditorModel.swift | 2 +- .../UsersRolesViewModel+Changes.swift | 2 +- .../Views/Compare/CompareRowDiffPane.swift | 15 ++ .../DataGrid/Cells/DataGridCellContent.swift | 3 +- .../Cells/DataGridCellPresenter.swift | 2 + .../Commands/DataGridCommands+Copy.swift | 1 + .../Commands/DataGridCommands+SetValue.swift | 4 +- .../DataGrid/DataGridViewController.swift | 3 +- .../CreateDatabaseSheet.swift | 2 +- .../Views/Editor/StatementRunController.swift | 50 +++- .../Views/Main/Child/DataTabSQLExporter.swift | 10 +- .../Main/Child/MainEditorContentView.swift | 1 + .../MainContentCoordinator+Navigation.swift | 11 +- ...ntentCoordinator+SafeModePromptRetry.swift | 28 ++ .../Main/MainContentCommandActions.swift | 67 +++-- .../Views/Main/MainContentCoordinator.swift | 9 +- .../ObjectCopy/CopyObjectsResultView.swift | 5 + .../UserType/CreateUserTypeSheet.swift | 9 +- .../Views/Results/ResultsJsonView.swift | 9 + .../Views/Sidebar/EditSequenceSheet.swift | 2 +- .../Views/Sidebar/ObjectCommentSheet.swift | 4 +- .../Views/Sidebar/SetSequenceValueSheet.swift | 2 +- .../Views/Structure/ClickHousePartsView.swift | 217 +++++++++------ .../Views/Structure/CreateTableDraft.swift | 15 +- .../StructureEditingSession+Apply.swift | 51 +--- ...tructureEditingSession+ColumnReorder.swift | 4 +- .../Views/Structure/TriggerDetailView.swift | 2 +- .../Views/Structure/TriggerEditorView.swift | 18 +- .../Completion/QueryCompletionCatalog.swift | 51 +++- .../QueryCompletionCatalogTests.swift | 29 ++ .../SQLStatementGeneratorBitColumnTests.swift | 4 +- .../SQLStatementGeneratorRowMatchTests.swift | 6 +- .../SQLStatementGeneratorTests.swift | 69 ++++- .../Compare/CompareDriverRowCutTests.swift | 90 +++++++ .../Core/Compare/CompareSQLLiteralTests.swift | 27 +- .../Compare/DataSyncScriptBuilderTests.swift | 2 +- .../RowEditingCoordinatorUUIDKeyTests.swift | 2 +- .../Core/DataWrite/RewindPlannerTests.swift | 81 ++++++ ...wChangeStatementFactoryCoverageTests.swift | 4 +- ...hangeStatementFactoryExpressionTests.swift | 84 +++++- .../RowWriteOperationBuilderTests.swift | 41 ++- .../Core/Database/DatabaseManagerTests.swift | 5 +- .../Database/SchemaWriteGateDenialTests.swift | 86 ++++++ .../Database/StartupCommandWarningTests.swift | 135 ++++++++++ .../Database/TriggerInfoMappingTests.swift | 86 ++++++ .../LSP/LSPTransportBrokenPipeTests.swift | 48 ++++ .../ObjectCopy/ObjectCopyRowCutTests.swift | 107 ++++++++ .../Query/MetadataConnectionPoolTests.swift | 21 +- .../SelectionSummaryCalculatorTests.swift | 17 +- .../RowOperationsFieldAbsenceTests.swift | 4 +- .../RowOperationsManagerPasteTests.swift | 18 +- .../Services/RowOperationsManagerTests.swift | 44 +++- .../RowOperationsManagerUUIDKeyTests.swift | 25 +- .../Services/RowPasteForeignTextTests.swift | 16 +- .../PayloadRowPlannerHeaderMatchTests.swift | 59 +++++ .../Shortcuts/RowPayloadFormatTests.swift | 60 +++++ .../PrincipalApplyOpenTransactionTests.swift | 44 ++++ .../PrincipalChangeManagerTests.swift | 40 +++ .../Utilities/JsonRowConverterTests.swift | 18 +- .../SQL/QueryClassifierTSQLBlockTests.swift | 39 +++ .../SQLRowToStatementConverterTests.swift | 31 ++- .../Vim/VimTextBufferAdapterPerfTests.swift | 103 ++------ .../Helpers/StubExecutionGates.swift | 9 +- .../Models/MultiRowEditStateTests.swift | 23 +- .../ClickHousePartStatementsTests.swift | 81 ++++-- .../Editor/StatementCaretTableTests.swift | 109 ++++++++ .../TableLoadAfterSafeModePromptTests.swift | 40 +++ .../WindowCloseSaveEveryConnectionTests.swift | 129 +++++++++ .../Views/Results/ResultsJsonViewTests.swift | 24 ++ .../SelectionSummaryTrackerTests.swift | 2 +- .../Structure/ClickHousePartsGridTests.swift | 101 ++++++- .../StructureListGridAccessibilityTests.swift | 3 +- .../StructureSaveConfirmationTests.swift | 10 +- .../TriggerEditorFontSizeTests.swift | 25 ++ core/crates/core/src/export/csv.rs | 55 ++-- core/crates/core/src/export/csv_options.rs | 4 +- core/crates/core/src/export/encoder/tests.rs | 6 +- core/crates/mcp/src/export_tools.rs | 10 +- .../storage/src/query_history/export.rs | 6 +- .../src/conformance/cases/large_table.rs | 9 +- .../src/conformance/cases/round_trip.rs | 8 +- .../crates/app/src/ui/export/encoding_sink.rs | 2 +- .../app/src/ui/export/export_ledger/tests.rs | 12 +- linux/crates/platform/tests/settings.rs | 2 +- .../data/app.tablepro.TablePro.gschema.xml.in | 2 +- linux/docs/storage.md | 2 +- 183 files changed, 4958 insertions(+), 1068 deletions(-) create mode 100644 apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLParenthesizedSemicolons.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLParenthesizedSemicolonTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift create mode 100644 apple/Plugins/FormatPluginTests/CSVExportFormulaAndDecimalTests.swift create mode 100644 apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift create mode 100644 apple/Plugins/MySQLDriverTests/MariaDBStopThreadIdTests.swift create mode 100644 apple/TablePro/Core/ChangeTracking/DefaultedCells.swift create mode 100644 apple/TablePro/Core/Services/Execution/GatedOperationFailure.swift create mode 100644 apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SafeModePromptRetry.swift create mode 100644 apple/TableProTests/Core/Compare/CompareDriverRowCutTests.swift create mode 100644 apple/TableProTests/Core/Database/SchemaWriteGateDenialTests.swift create mode 100644 apple/TableProTests/Core/Database/StartupCommandWarningTests.swift create mode 100644 apple/TableProTests/Core/LSP/LSPTransportBrokenPipeTests.swift create mode 100644 apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCutTests.swift create mode 100644 apple/TableProTests/Core/Shortcuts/PayloadRowPlannerHeaderMatchTests.swift create mode 100644 apple/TableProTests/Core/Shortcuts/RowPayloadFormatTests.swift create mode 100644 apple/TableProTests/Core/Utilities/SQL/QueryClassifierTSQLBlockTests.swift create mode 100644 apple/TableProTests/Views/Editor/StatementCaretTableTests.swift create mode 100644 apple/TableProTests/Views/Main/TableLoadAfterSafeModePromptTests.swift create mode 100644 apple/TableProTests/Views/Main/WindowCloseSaveEveryConnectionTests.swift create mode 100644 apple/TableProTests/Views/Structure/TriggerEditorFontSizeTests.swift diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift new file mode 100644 index 0000000000..a4772dda72 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift @@ -0,0 +1,41 @@ +// +// CassandraConditionalWrite.swift +// TableProCassandraCore +// + +import Foundation + +// A grid write names its row by key and is conditioned on it: an INSERT is an upsert, and an UPDATE writes its cells +// whether or not the row is there, so either would otherwise replace or recreate a row the grid never showed. +// A condition that fails answers one row whose `[applied]` is false, which the server reports as a success. +public enum CassandraConditionalWrite { + public static let appliedColumn = "[applied]" + public static let insertCondition = "IF NOT EXISTS" + public static let updateCondition = "IF EXISTS" + + public enum Refusal: Sendable, Equatable { + case keyTaken + case rowMissing + } + + public static func insertStatement(target: String, names: [String], values: [String]) -> String { + "INSERT INTO \(target) (\(names.joined(separator: ", "))) VALUES (\(values.joined(separator: ", "))) " + + insertCondition + } + + public static func updateStatement(target: String, assignments: [String], keyCondition: String) -> String { + "UPDATE \(target) SET \(assignments.joined(separator: ", ")) WHERE \(keyCondition) \(updateCondition)" + } + + public static func refusal(statement: String, firstColumn: String?, firstValue: String?) -> Refusal? { + guard firstColumn == appliedColumn, firstValue?.lowercased() == "false" else { return nil } + let normalized = statement.uppercased() + if normalized.hasPrefix("INSERT"), normalized.hasSuffix(" " + insertCondition) { + return .keyTaken + } + if normalized.hasPrefix("UPDATE"), normalized.hasSuffix(" " + updateCondition) { + return .rowMissing + } + return nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisCommandSplitter.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisCommandSplitter.swift index 8d42794c8d..d92e7bc4bc 100644 --- a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisCommandSplitter.swift +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisCommandSplitter.swift @@ -32,6 +32,17 @@ public enum RedisCommandSplitter { return nil } + // Every command with the last caret offset `command(at:in:)` gives it, for a caller that keeps them between + // caret moves. + public static func commandsWithCaretReach(in text: String) -> [(command: Command, reach: Int)] { + var lexer = RedisCommandLexer(text, endsCommands: true, decodesArguments: false) + var commands: [(command: Command, reach: Int)] = [] + while let found = lexer.next() { + commands.append((command: command(found, in: text), reach: found.reach)) + } + return commands + } + private static func command(_ found: RedisCommandLexer.Command, in text: String) -> Command { let range = NSRange(location: found.start, length: found.end - found.start) return Command(text: (text as NSString).substring(with: range), range: range) diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLParenthesizedSemicolons.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLParenthesizedSemicolons.swift new file mode 100644 index 0000000000..d5d1607083 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLParenthesizedSemicolons.swift @@ -0,0 +1,42 @@ +import Foundation + +// The scanner keeps the `;`s of a PostgreSQL rule's `DO ALSO (...; ...)` in its statement. A client that cuts at every +// `;` runs each piece, so a gate tiers them too: `DO ALSO (SELECT 1; DELETE FROM t)` by its `DELETE`. +enum SQLParenthesizedSemicolons { + static func pieces(in statement: String, grammar: SQLLexicalGrammar) -> [String] { + guard statement.utf16.contains(SqlLexer.semicolon) else { return [] } + let text = statement as NSString + let length = text.length + var cuts: [Int] = [] + var depth = 0 + var index = 0 + while index < length { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + index = max(span.end, index + 1) + continue + } + switch text.character(at: index) { + case SqlLexer.openParen: + depth += 1 + case SqlLexer.closeParen: + depth = max(0, depth - 1) + case SqlLexer.semicolon where depth > 0: + cuts.append(index) + default: + break + } + index += 1 + } + guard !cuts.isEmpty else { return [] } + var pieces: [String] = [] + var start = 0 + for cut in cuts + [length] { + let piece = StatementBlank.trimming(text.substring(with: NSRange(location: start, length: cut - start))) + if !piece.isEmpty { + pieces.append(piece) + } + start = cut + 1 + } + return pieces + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift index f1596e7ceb..d4abc7cc10 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift @@ -2,13 +2,26 @@ import Foundation /// Statement boundaries for the dialects whose only `;`-holding construct is a routine body written `BEGIN ... END`. /// -/// A `BEGIN` opens a body only inside a statement that defines a routine, for the safety reason recorded on +/// Outside T-SQL a `BEGIN` opens a body only inside a statement that defines a routine, for the safety reason recorded on /// ``SqlBlockStructure/opensRoutineDefinition(_:)``. The `;` belongs to the statement only after a T-SQL `MERGE`, which /// ``SQLLexicalGrammar/terminatedMergeStatements`` asks for: these engines accept every other statement without it. /// /// `BEGIN`, `CASE` and `END` count only where a keyword can stand: an engine that does not reserve them takes them as -/// names, and a block a name opened would never close. A text left inside one is refused by the external gate. +/// names, and a block a name opened would never close. T-SQL reserves all three, so there a block also opens outside +/// a routine, as in `IF @x = 1 BEGIN ... END` and `BEGIN TRY ... END TRY BEGIN CATCH ... END CATCH`. A text left +/// inside one is refused by the external gate. +/// +/// A `;` inside parentheses ends the statement, so a stray `(` cannot swallow what follows it, except in a PostgreSQL +/// rule's action list, `DO ALSO (INSERT ...; INSERT ...)`, which holds its commands together. public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { + // A PostgreSQL `CREATE RULE`: its header, the `DO [ALSO | INSTEAD]` before the action list, and the list itself. + private enum RuleAction: Equatable { + case none + case header + case afterDo + case list(depth: Int) + } + private enum Header { case opener /// After `CREATE`, `ALTER` or `REPLACE`. `accountWords` is what is left of an unquoted `DEFINER = user@host`. @@ -25,6 +38,11 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { private static let statementIntroducers: Set = ["THEN", "ELSE", "DO", "LOOP", "REPEAT"] private let grammar: SQLLexicalGrammar + private let readsBlocksEverywhere: Bool + // T-SQL outside a routine: where a block's `BEGIN` can stand, as the statement's first word, after `IF`, `WHILE` + // or `ELSE`, or after a block the statement already opened. Anywhere else `begin` is in a name's place. + private var followsControlFlow = false + private var pendingBeginMayOpen = false private var header = Header.opener private var definesRoutine = false private var openedBlockBody = false @@ -34,6 +52,8 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { /// whatever word follows it. private var openCaseExpressions = 0 private var parenDepth = 0 + private var ruleAction = RuleAction.none + private var ruleActionHeldSemicolon = false private var pendingBegin = false private var pendingEnd = false private var followsName = false @@ -50,12 +70,15 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public init(grammar: SQLLexicalGrammar) { self.grammar = grammar + self.readsBlocksEverywhere = grammar.contains(.unterminatedStatements) self.merge = grammar.contains(.terminatedMergeStatements) ? SQLMergeStatementTracker() : nil } public var needsWords: Bool { + // Inside parentheses no word opens or closes a block, and a `MERGE` there needs no `;`. + if readsBlocksEverywhere { return parenDepth == 0 } if case .read = header { - return definesRoutine || merge?.needsWords == true + return definesRoutine || ruleAction == .header || ruleAction == .afterDo || merge?.needsWords == true } return true } @@ -70,7 +93,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { // An `END` still waiting for the word after it closes a block when the text ends. public var holdsOpenBlock: Bool { - depth > (pendingEnd ? 1 : 0) + depth > (pendingEnd ? 1 : 0) || ruleActionHeldSemicolon } public mutating func observeWord(_ word: String) { @@ -87,10 +110,27 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { startsBodyStatement = false let statementPosition = atStatementPosition observeStatementPosition(after: word, wasAtStatementPosition: statementPosition) - guard !isMember, readHeader(word), definesRoutine, parenDepth == 0 else { return } + guard !isMember else { return } + var isFirstWord = false + if case .opener = header { + isFirstWord = true + } + let pastHeader = readHeader(word) + if pastHeader, parenDepth == 0 { + observeRuleWord(word) + } + guard parenDepth == 0, readsBlocksEverywhere || (pastHeader && definesRoutine) else { return } switch word { - case "BEGIN" where opensNestedBlock(atStatementPosition: statementPosition): + case "BEGIN" where readsBlocksEverywhere: + pendingBegin = true + pendingBeginMayOpen = opensNestedBlock(atStatementPosition: statementPosition, isFirstWord: isFirstWord) + case "BEGIN" where opensNestedBlock(atStatementPosition: statementPosition, isFirstWord: isFirstWord): pendingBegin = true + pendingBeginMayOpen = true + case "IF" where readsBlocksEverywhere, "WHILE" where readsBlocksEverywhere: + followsControlFlow = true + case "ELSE" where readsBlocksEverywhere && openCaseExpressions == 0: + followsControlFlow = true case "CASE" where !grammar.hasSingleBodyForm: depth += 1 openCaseExpressions += 1 @@ -103,6 +143,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeSymbol(_ symbol: UInt16) { merge?.observeSymbol(symbol) + if ruleAction == .afterDo, symbol != SqlLexer.openParen { + ruleAction = .none + } let opensBlock = symbol == SqlLexer.openParen && !grammar.contains(.atomicRoutineBodies) settlePendingBeforeNonWord(opensBlock: opensBlock) followsDot = followsName && symbol == Self.dot @@ -112,9 +155,16 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { switch symbol { case SqlLexer.openParen: parenDepth += 1 + if ruleAction == .afterDo { + ruleAction = .list(depth: parenDepth) + } endName() case SqlLexer.closeParen: parenDepth = max(0, parenDepth - 1) + if case .list(let listDepth) = ruleAction, parenDepth < listDepth { + ruleAction = .none + ruleActionHeldSemicolon = false + } default: break } @@ -122,6 +172,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeOpaqueToken() { merge?.observeOpaqueToken() + if ruleAction == .afterDo { + ruleAction = .none + } settlePendingBeforeNonWord(opensBlock: !grammar.contains(.atomicRoutineBodies)) followsName = true followsDot = false @@ -140,13 +193,27 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { pendingEnd = false closeBlock() } + if readsBlocksEverywhere { + // A T-SQL `CASE` is only an expression, which no `;` can stand inside, so one still open was never closed. + depth = max(0, depth - openCaseExpressions) + } openCaseExpressions = 0 - parenDepth = 0 followsName = false followsDot = false - let endsStatement = depth == 0 - startsBodyStatement = !endsStatement - atStatementPosition = !endsStatement + let heldByRuleAction: Bool + if case .list(let listDepth) = ruleAction { + heldByRuleAction = depth == 0 && parenDepth >= listDepth + } else { + heldByRuleAction = false + } + if heldByRuleAction { + ruleActionHeldSemicolon = true + } else { + parenDepth = 0 + } + let endsStatement = depth == 0 && !heldByRuleAction + startsBodyStatement = depth > 0 + atStatementPosition = depth > 0 labelMayFollow = false inDeclaration = false inHandlerDeclaration = false @@ -175,6 +242,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { } return false case .kind(let accountWords): + if word == "RULE", grammar.contains(.atomicRoutineBodies) { + ruleAction = .header + } if SqlBlockStructure.isRoutineKind(word, grammar: grammar) { definesRoutine = true header = .name @@ -207,11 +277,33 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { } } - private func opensNestedBlock(atStatementPosition: Bool) -> Bool { - guard depth > 0, !grammar.hasSingleBodyForm, !grammar.contains(.unterminatedStatements) else { return true } + private func opensNestedBlock(atStatementPosition: Bool, isFirstWord: Bool) -> Bool { + if readsBlocksEverywhere { + return depth > 0 || definesRoutine || isFirstWord || followsControlFlow || openedBlockBody + } + guard depth > 0, !grammar.hasSingleBodyForm else { return true } return atStatementPosition || inHandlerDeclaration } + private mutating func observeRuleWord(_ word: String) { + switch ruleAction { + case .header where word == "DO": + ruleAction = .afterDo + case .afterDo where word == "ALSO" || word == "INSTEAD": + break + case .afterDo: + ruleAction = .none + default: + break + } + } + + // `BEGIN TRY` and `BEGIN CATCH` are blocks wherever they stand, and nothing else follows a name that way. + private func beginOpensBlock(followedBy word: String) -> Bool { + if readsBlocksEverywhere, word == "TRY" || word == "CATCH" { return true } + return pendingBeginMayOpen && !SqlBlockStructure.beginStartsStatement(followedBy: word, grammar: grammar) + } + private mutating func endName() { if case .name = header { header = .read @@ -227,7 +319,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { openBlockBody() return true } - if !SqlBlockStructure.beginStartsTransaction(followedBy: word) { + if beginOpensBlock(followedBy: word) { openBlockBody() } } @@ -237,7 +329,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { closeBlock() return false } - switch SqlBlockStructure.endingFollowedBy(word, closesCaseExpression: openCaseExpressions > 0) { + switch SqlBlockStructure.endingFollowedBy(word, closesCaseExpression: openCaseExpressions > 0, grammar: grammar) { case .closesControlFlow: return true case .closesCaseStatement: @@ -254,7 +346,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { private mutating func settlePendingBeforeNonWord(opensBlock: Bool) { if pendingBegin { pendingBegin = false - if opensBlock { + // A T-SQL `begin(` outside a routine is a name, as in `CREATE TABLE begin(a int)`. + let opensAfterName = !readsBlocksEverywhere || definesRoutine || depth > 0 + if opensBlock, pendingBeginMayOpen, opensAfterName { openBlockBody() } } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift index 38f254c5bd..851741b05f 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift @@ -20,11 +20,13 @@ public enum SQLUnterminatedStatements { /// Every text a gate has to tier for `statement`: the statement itself, then, when `grammar` needs no terminator, /// the text of each statement that begins inside it, less the statements nested in that one, which come on their /// own. A grammar that takes a statement in place of a value or a table adds what ``SQLEmbeddedStatements`` finds, - /// and one with a pipe operator each statement ``SQLPipeOperatorChain`` finds. + /// and one with a pipe operator each statement ``SQLPipeOperatorChain`` finds. A grammar that needs a terminator + /// also adds the pieces a `;` inside the statement's parentheses separates, from ``SQLParenthesizedSemicolons``. public static func runnable(in statement: String, grammar: SQLLexicalGrammar) -> [String] { guard grammar.contains(.unterminatedStatements) else { return [statement] + SQLEmbeddedStatements.texts(in: statement, grammar: grammar) + SQLPipeOperatorChain.statements(in: statement, grammar: grammar) + + SQLParenthesizedSemicolons.pieces(in: statement, grammar: grammar) } let text = statement as NSString let tokens = TSQLTokens.read(text, grammar: grammar) @@ -32,6 +34,27 @@ public enum SQLUnterminatedStatements { return [statement] + ownTexts(of: statementSpans(in: tokens, length: text.length), in: text) } + // A scanner statement on T-SQL keeps a `;` outside parentheses only when a block holds it, so anything after one but + // the block's `END`, an `ELSE` or another `;` is a second statement the block runs. A stored routine runs nothing. + public static func blockHoldsSeveralStatements(_ statement: String, grammar: SQLLexicalGrammar) -> Bool { + guard grammar.contains(.unterminatedStatements) else { return false } + let tokens = TSQLTokens.read(statement as NSString, grammar: grammar) + guard !definesRoutine(tokens) else { return false } + return tokens.indices.dropLast().contains { index in + guard tokens[index].kind == .terminator, tokens[index].depth == 0 else { return false } + let next = tokens[index + 1] + guard next.kind != .terminator else { return false } + switch next.word { + case "ELSE": + return false + case "END": + return tokens.indices.contains(index + 2) && tokens[index + 2].word == "CONVERSATION" + default: + return true + } + } + } + /// The reserved keywords that begin a T-SQL statement. `ELSE` and `FETCH` are left out because they also continue /// a `CASE` and an `OFFSET` clause, and the statement after an `ELSE` begins with a keyword of its own. `WITH` is /// left out because a common table expression cannot follow a statement without a `;`, and the word also opens diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift index 0a11e84e6d..6968b11fef 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift @@ -23,7 +23,8 @@ public enum SqlBlockStructure { /// What an `END` means, given the word after it. public enum EndFollower: Equatable, Sendable { - /// `END IF`, `END LOOP` and their kin close a construct that never opened a block, and the word is theirs. + /// `END IF`, `END LOOP` and their kin close a construct that never opened a block, and T-SQL's + /// `END CONVERSATION` is a statement that closes nothing. Either way the word is theirs. case closesControlFlow /// `END CASE` closes the `CASE` that opened a block, and the word is theirs. case closesCaseStatement @@ -48,6 +49,10 @@ public enum SqlBlockStructure { /// second opener. private static let controlFlowFollowers: Set = ["IF", "LOOP", "WHILE", "FOR", "REPEAT"] + // T-SQL's `BEGIN DIALOG`, `BEGIN CONVERSATION TIMER` and `END CONVERSATION` are statements, not a block's ends. + private static let serviceBrokerFollowers: Set = ["DIALOG", conversationKeyword] + private static let conversationKeyword = "CONVERSATION" + /// The keywords a statement has to open with before a `BEGIN` inside it is read as a routine body. /// /// Only ``SQLRoutineBodyTracker`` consults this, and the reason is safety rather than grammar. An anonymous @@ -56,7 +61,9 @@ public enum SqlBlockStructure { /// single statement whose first word is `BEGIN`, and `QueryClassifier` tiers a statement by its leading keyword, /// so a swallowed `DROP` would be tiered a plain write and skip the destructive confirmation. Folding cannot /// execute anything, so it takes every block. Oracle, whose anonymous blocks are the everyday form, has its own - /// grammar in ``PLSQLUnitTracker`` and a classifier that reads the whole block. + /// grammar in ``PLSQLUnitTracker`` and a classifier that reads the whole block. T-SQL reserves `BEGIN`, so its + /// blocks open outside a routine too, and the gate tiers every statement inside one through + /// ``SQLUnterminatedStatements``. private static let routineDefinitionOpeners: Set = ["CREATE", "ALTER", "REPLACE"] /// What such a statement has to define, for the same reason. `BEGIN` is not reserved on PostgreSQL, MySQL or @@ -111,6 +118,28 @@ public enum SqlBlockStructure { return transactionFollowers.contains(keyword) } + public static func beginStartsStatement(followedBy keyword: String?, grammar: SQLLexicalGrammar) -> Bool { + if beginStartsTransaction(followedBy: keyword) { return true } + return readsTSQLBlocks(grammar) && keyword.map(serviceBrokerFollowers.contains) == true + } + + // T-SQL closes every block with a bare `END`: in `END IF @x = 1 ...` the `IF` begins the next statement. + public static func endingFollowedBy( + _ keyword: String, + closesCaseExpression: Bool, + grammar: SQLLexicalGrammar + ) -> EndFollower { + guard readsTSQLBlocks(grammar) else { + return endingFollowedBy(keyword, closesCaseExpression: closesCaseExpression) + } + return !closesCaseExpression && keyword == conversationKeyword ? .closesControlFlow : .closesBlock + } + + // The grammar whose statements need no `;` between them is T-SQL's, which reserves `BEGIN`, `CASE` and `END`. + private static func readsTSQLBlocks(_ grammar: SQLLexicalGrammar) -> Bool { + grammar.contains(.unterminatedStatements) + } + /// What an `END` followed by `keyword` closes. /// /// - Parameter closesCaseExpression: whether a `CASE` is open that has held no `;`. Only the statement form can @@ -260,7 +289,7 @@ public enum SqlBlockStructure { let cursor = skipTrivia(from: offset, in: text, length: length, grammar: grammar) guard cursor < length else { return true } guard text.character(at: cursor) != SqlLexer.semicolon else { return true } - return beginStartsTransaction(followedBy: readKeyword(text, at: cursor, length: length).text) + return beginStartsStatement(followedBy: readKeyword(text, at: cursor, length: length).text, grammar: grammar) } /// Goes by the word after the `END` alone: only folding asks, and it keeps no record of which `CASE` is open. @@ -273,7 +302,7 @@ public enum SqlBlockStructure { let cursor = skipTrivia(from: offset, in: text, length: length, grammar: grammar) guard cursor < length else { return .closesBlock(resumeAt: offset) } let follower = readKeyword(text, at: cursor, length: length) - switch endingFollowedBy(follower.text, closesCaseExpression: false) { + switch endingFollowedBy(follower.text, closesCaseExpression: false, grammar: grammar) { case .closesControlFlow: return .none case .closesCaseStatement: diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift index e00a7551a5..cb810a30f5 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift @@ -4,6 +4,8 @@ import TableProWireIO enum ParcelFlavor: UInt16 { case request = 1 case response = 4 + case abort = 6 + case cancel = 7 case success = 8 case failure = 9 case record = 10 diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift index ed5608fe9f..440de15bef 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift @@ -1,7 +1,8 @@ import Darwin import Foundation -public final class TeradataConnection { +// One queue drives it. Only `cancel()` comes from elsewhere, and what it reads and sends is under socketLock or wireLock. +public final class TeradataConnection: @unchecked Sendable { private let config: TeradataConnectionConfig private let socketLock = NSLock() private var _socket: (any TeradataTransport)? @@ -20,13 +21,56 @@ public final class TeradataConnection { private var gssSequence: UInt64 = 0 private var confidentialityBypassed = false + // Orders every post-logon send with the counters it consumes, since Stop sends its abort from another thread. + private let wireLock = NSLock() + // Under socketLock, which is never held across I/O. + private var _phase = RequestPhase.idle + private var _stopRequested = false + private var _operationActive = false + // The request a stop is waiting on the server for, so the stop's deadline closes nothing that came after it. + private var _stoppingRequest: UInt32? + + // 3110: "The transaction was aborted by the user." + static let userAbortCode = 3_110 + private static let abortLockWait: TimeInterval = 0.25 + // A server or network that stops answering would otherwise hold a stop for the whole 20-second receive timeout. + static let defaultStopGrace: TimeInterval = 5 + private let stopGrace: TimeInterval + private let stopDeadlineQueue = DispatchQueue(label: "com.TablePro.teradata.stop-deadline") + + private enum RequestPhase: Equatable { + case idle + case connecting + case awaitingResponse(UInt32) + case fetching(UInt32) + } + + private enum StopAction { + case nothing + case abort(UInt32) + case endOutput(UInt32) + case close + } + + private enum Exchange { + case answered([Parcel]) + case notSent + case outputEnded + } + public init(config: TeradataConnectionConfig) { self.config = config + stopGrace = Self.defaultStopGrace } - init(config: TeradataConnectionConfig, transport: any TeradataTransport) { + init( + config: TeradataConnectionConfig, + transport: any TeradataTransport, + stopGrace: TimeInterval = TeradataConnection.defaultStopGrace + ) { self.config = config self._socket = transport + self.stopGrace = stopGrace confidentialityBypassed = true } @@ -49,6 +93,7 @@ public final class TeradataConnection { socketLock.withLock { _socket = socket _hasLostConnection = false + _phase = .connecting } var established = false defer { @@ -57,6 +102,7 @@ public final class TeradataConnection { self.socket = nil connectDeadline = nil } + socketLock.withLock { _phase = .idle } } serverIP = Self.resolveAddress(config.host) ?? config.host _ = try deadline.remainingMilliseconds() @@ -91,28 +137,91 @@ public final class TeradataConnection { let body = TeradataMessages.optionsParcel(returnStatementInfo: false).encoded() + TeradataMessages.requestParcel(sql: sql).encoded() + TeradataMessages.responseParcel().encoded() - let parcels: [Parcel] + let outcome: Exchange do { - parcels = try exchange(body, requestNumber: requestNumber) + outcome = try exchange(body, requestNumber: requestNumber) } catch { + endRequest() // The reply still on its way would be read as the next request's. dropTransport() throw error } - return try TeradataResultParser.parse(parcels) + let stopped = endRequest() + guard case .answered(let parcels) = outcome else { throw TeradataWireError.cancelled } + do { + return try TeradataResultParser.parse(parcels) + } catch TeradataWireError.server(let code, _) where stopped && code == Self.userAbortCode { + // The server rolled back the transaction the request ran in; the host has no way yet to say so. + throw TeradataWireError.cancelled + } } - private func exchange(_ body: [UInt8], requestNumber: UInt32) throws -> [Parcel] { - try sendEncrypted(kind: .start, body: body, requestNumber: requestNumber) + private func exchange(_ body: [UInt8], requestNumber: UInt32) throws -> Exchange { + let sent = try wireLock.withLock { () -> Bool in + // A stop that landed before the request went out refuses it, and the session stays as it was. + let refused = socketLock.withLock { () -> Bool in + guard !_stopRequested else { return true } + _phase = .awaitingResponse(requestNumber) + return false + } + guard !refused else { return false } + try sendEncryptedLocked(kind: .start, body: body, requestNumber: requestNumber) + return true + } + guard sent else { return .notSent } var parcels: [Parcel] = [] while true { let batch = try readReply() parcels.append(contentsOf: batch) - if Self.responseIsComplete(batch) { return parcels } + if Self.responseIsComplete(batch) { return .answered(parcels) } + // Once the first reply is out an abort no longer takes effect, so a stop ends the output instead. + let stopRequested = socketLock.withLock { () -> Bool in + _phase = .fetching(requestNumber) + return _stopRequested + } + if stopRequested { + try endOutput(requestNumber: requestNumber) + return .outputEnded + } try sendContinue(requestNumber: requestNumber) } } + // The Cancel parcel in place of the Response parcel deletes the request's spool, and the server answers with + // End Request. One that does not throws, and the caller gives the transport up. + private func endOutput(requestNumber: UInt32) throws { + try sendEncrypted(kind: .cont, body: TeradataMessages.cancelParcel().encoded(), requestNumber: requestNumber) + guard Self.responseIsComplete(try readReply()) else { + throw TeradataWireError.malformed("the server kept answering a cancelled request") + } + } + + // A caller's operation, so a stop that lands before its request goes out still reaches it. + public func beginOperation() { + socketLock.withLock { + _operationActive = true + _stopRequested = false + } + } + + public func endOperation() { + socketLock.withLock { + _operationActive = false + _stopRequested = false + } + } + + @discardableResult + private func endRequest() -> Bool { + socketLock.withLock { + let stopped = _stopRequested + _phase = .idle + _stopRequested = false + _stoppingRequest = nil + return stopped + } + } + private func dropTransport() { let dropped = socketLock.withLock { let transport = _socket @@ -129,19 +238,80 @@ public final class TeradataConnection { return } if sessionNumber != 0 { - let message = LanMessage( - kind: .logoff, body: TeradataMessages.logoffParcel().encoded(), - sessionNumber: sessionNumber, byteVar: 0, - authentication: TeradataMessages.authBytes(nextMessage()), hostCharSet: charsetCode) - try? socket.send(message.encoded()) + wireLock.withLock { + let message = LanMessage( + kind: .logoff, body: TeradataMessages.logoffParcel().encoded(), + sessionNumber: sessionNumber, byteVar: 0, + authentication: TeradataMessages.authBytes(nextMessage()), hostCharSet: charsetCode) + try? socket.send(message.encoded()) + } } socket.close() self.socket = nil connectDeadline = nil } + // A request the server is still working on is aborted on the session, which rolls back its transaction, answers it + // with failure 3110 and stays logged on. Past the first reply the request's own loop ends the output instead, and a + // request not yet sent is refused. A connect has no request to stop, so it gives the transport up, as does an + // abort that cannot be sent or a server that has not ended the request within the grace. public func cancel() { - socket?.cancel() + let action: StopAction = socketLock.withLock { + switch _phase { + case .idle: + if _operationActive { _stopRequested = true } + return .nothing + case .connecting: + return .close + case .awaitingResponse(let request): + _stopRequested = true + _stoppingRequest = request + return .abort(request) + case .fetching(let request): + _stopRequested = true + _stoppingRequest = request + return .endOutput(request) + } + } + switch action { + case .nothing: + return + case .close: + socket?.cancel() + case .abort(let request): + guard sendAbort(for: request) else { + socket?.cancel() + return + } + closeUnlessEnded(request) + case .endOutput(let request): + closeUnlessEnded(request) + } + } + + private func closeUnlessEnded(_ request: UInt32) { + stopDeadlineQueue.asyncAfter(deadline: .now() + stopGrace) { [weak self] in + guard let self else { return } + // Shut down under the lock, so a request that ended meanwhile cannot hand the transport to a new one. + socketLock.withLock { + guard _stoppingRequest == request else { return } + _socket?.cancel() + } + } + } + + // A long start still going out holds the lock, and the caller may be the main thread. + private func sendAbort(for request: UInt32) -> Bool { + guard wireLock.lock(before: Date(timeIntervalSinceNow: Self.abortLockWait)) else { return false } + defer { wireLock.unlock() } + // Past the first reply the request's own loop ends the output; a request already answered needs nothing. + guard socketLock.withLock({ _phase == .awaitingResponse(request) }) else { return true } + do { + try sendEncryptedLocked(kind: .abort, body: TeradataMessages.abortParcel().encoded(), requestNumber: request) + return true + } catch { + return false + } } static func responseIsComplete(_ parcels: [Parcel]) -> Bool { @@ -252,6 +422,12 @@ public final class TeradataConnection { } private func sendEncrypted(kind: MessageKind, body: [UInt8], requestNumber: UInt32) throws { + try wireLock.withLock { + try sendEncryptedLocked(kind: kind, body: body, requestNumber: requestNumber) + } + } + + private func sendEncryptedLocked(kind: MessageKind, body: [UInt8], requestNumber: UInt32) throws { let message = LanMessage( kind: kind, body: body, sessionNumber: sessionNumber, requestNumber: requestNumber, byteVar: 0, authentication: TeradataMessages.authBytes(nextMessage()), diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataMessages.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataMessages.swift index 042633063c..d0ec9d359e 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataMessages.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataMessages.swift @@ -119,6 +119,14 @@ enum TeradataMessages { Parcel(.logoff) } + static func abortParcel() -> Parcel { + Parcel(.abort) + } + + static func cancelParcel() -> Parcel { + Parcel(.cancel) + } + static func charsetCode(fromConfigResponse body: [UInt8], name: String) -> UInt8? { let needle = Array(name.utf8) guard !needle.isEmpty else { return nil } diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift new file mode 100644 index 0000000000..b02a53181e --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift @@ -0,0 +1,61 @@ +// +// CassandraConditionalWriteTests.swift +// TableProCassandraCoreTests +// + +import TableProCassandraCore +import Testing + +struct CassandraConditionalWriteTests { + private static let update = CassandraConditionalWrite.updateStatement( + target: "\"shop\".\"orders\"", assignments: ["\"note\" = ?"], keyCondition: "\"id\" = ?" + ) + private static let insert = CassandraConditionalWrite.insertStatement( + target: "\"shop\".\"orders\"", names: ["\"id\"", "\"note\""], values: ["?", "?"] + ) + + // Without the condition CQL writes the cell anyway and recreates the row with every other column null. + @Test("A grid edit only writes a row that still exists") + func updateIsConditionedOnTheRow() { + #expect(Self.update == "UPDATE \"shop\".\"orders\" SET \"note\" = ? WHERE \"id\" = ? IF EXISTS") + #expect(Self.insert == "INSERT INTO \"shop\".\"orders\" (\"id\", \"note\") VALUES (?, ?) IF NOT EXISTS") + } + + @Test("An edit of a row deleted since it was read is refused, not reported as saved") + func updateOfAMissingRowIsRefused() { + #expect(CassandraConditionalWrite.refusal( + statement: Self.update, firstColumn: "[applied]", firstValue: "false" + ) == .rowMissing) + } + + @Test("A new row whose key is taken is refused") + func insertOfATakenKeyIsRefused() { + #expect(CassandraConditionalWrite.refusal( + statement: Self.insert, firstColumn: "[applied]", firstValue: "false" + ) == .keyTaken) + } + + @Test("A write the condition let through is no refusal") + func appliedWriteIsNotRefused() { + #expect(CassandraConditionalWrite.refusal( + statement: Self.update, firstColumn: "[applied]", firstValue: "true" + ) == nil) + #expect(CassandraConditionalWrite.refusal( + statement: Self.insert, firstColumn: "[applied]", firstValue: "true" + ) == nil) + } + + // A condition on a column answers with the row's current values, which the console shows as the server sent them. + @Test("An unapplied write with a condition of its own is left to the result") + func otherConditionsAreLeftToTheResult() { + #expect(CassandraConditionalWrite.refusal( + statement: "UPDATE t SET a = 1 WHERE k = 1 IF a = 0", firstColumn: "[applied]", firstValue: "false" + ) == nil) + #expect(CassandraConditionalWrite.refusal( + statement: "DELETE FROM t WHERE k = 1 IF EXISTS", firstColumn: "[applied]", firstValue: "false" + ) == nil) + #expect(CassandraConditionalWrite.refusal( + statement: "UPDATE t SET a = 1 WHERE k = 1", firstColumn: "a", firstValue: "false" + ) == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginParameterEscapingTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginParameterEscapingTests.swift index 4f2d558573..1f17a3ac4b 100644 --- a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginParameterEscapingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginParameterEscapingTests.swift @@ -107,6 +107,17 @@ struct IsNumericLiteralTests { #expect(!StubDriver.isNumericLiteral("12abc")) #expect(!StubDriver.isNumericLiteral("1.2.3")) } + + // Written bare, a zip or a product code is compared as a number and matches `123` and `0123` too. + @Test("Numeric-shaped text with padding zeros is bound as a string", arguments: ["00123", "-007", "0042", "00.5"]) + func paddedNumbersAreQuoted(value: String) { + #expect(StubDriver().escapedParameterValue(value).unicodeScalars.first == "'") + } + + @Test("A number that opens with a single zero is bound bare", arguments: ["0", "0.5", "-0.5", "0e3", "+0", "100"]) + func singleLeadingZeroStaysBare(value: String) { + #expect(StubDriver().escapedParameterValue(value) == value) + } } // MARK: - escapedParameterValue diff --git a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisCommandSplitterTests.swift b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisCommandSplitterTests.swift index 515d4828cd..5dee906b01 100644 --- a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisCommandSplitterTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisCommandSplitterTests.swift @@ -103,6 +103,18 @@ struct RedisCommandSplitterTests { } } + @Test("Each command's caret reach gives every caret the command command(at:) gives it", arguments: [ + "GET a; GET b\n\nGET c \n", "GET a\n", "GET a; ", "SET k \"a\nb\"\nGET k", " GET a ;", "SET šŸ˜€ 1\r\nGET šŸ˜€", "", + ]) + func caretReachMatchesCommandAt(script: String) { + let reached = RedisCommandSplitter.commandsWithCaretReach(in: script) + #expect(reached.map { $0.command } == RedisCommandSplitter.commands(in: script)) + for caret in 0...((script as NSString).length + 1) { + let viaReach = reached.first { caret <= $0.reach }?.command + #expect(viaReach == RedisCommandSplitter.command(at: caret, in: script), "caret \(caret)") + } + } + @Test("A caret after the last command, or in text with none, has no command") func caretPastEveryCommand() { #expect(command(at: 6, in: "GET a\n") == nil) diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLParenthesizedSemicolonTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLParenthesizedSemicolonTests.swift new file mode 100644 index 0000000000..2f9cf1257c --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLParenthesizedSemicolonTests.swift @@ -0,0 +1,77 @@ +import Foundation +import TableProSQLGrammar +import Testing + +struct SQLParenthesizedSemicolonTests { + private static func grammar(_ typeId: String) -> SQLLexicalGrammar { + SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil).execution + } + + private func sent(_ sql: String, _ grammar: SQLLexicalGrammar) -> [String] { + SQLStatementScanner.executableStatements(in: sql, grammar: grammar).map(\.sql) + } + + @Test("A PostgreSQL rule's multi-command action list stays one statement", arguments: [ + "CREATE RULE r AS ON INSERT TO t DO ALSO (INSERT INTO a VALUES (1); INSERT INTO b VALUES (2))", + "CREATE OR REPLACE RULE r AS ON UPDATE TO t WHERE (NEW.a > 0) DO INSTEAD (UPDATE a SET n = 1; NOTIFY t)", + "CREATE RULE r AS ON DELETE TO t DO (DELETE FROM a; DELETE FROM b)", + ]) + func ruleActionStaysWhole(rule: String) { + let grammar = Self.grammar("PostgreSQL") + let sql = rule + ";\nSELECT 9;" + + #expect(sent(sql, grammar) == [rule, "SELECT 9"]) + #expect(SQLStatementScanner.navigableStatements(in: sql, grammar: grammar).count == 2) + #expect(!SQLStatementScanner.endsInsideOpenBlock(sql, grammar: grammar)) + } + + @Test("A rule whose action is one command ends at its ;") + func ruleWithoutAnActionList() { + let grammar = Self.grammar("PostgreSQL") + + #expect(sent("CREATE RULE r AS ON INSERT TO t DO ALSO INSERT INTO log VALUES (NEW.id);\nSELECT 2;", grammar).count == 2) + #expect(sent("CREATE RULE r AS ON INSERT TO t DO INSTEAD NOTHING;\nSELECT 2;", grammar).count == 2) + } + + @Test("A stray ( ends at its ; and leaves every statement after it on its own", arguments: [ + "PostgreSQL", "MySQL", "SQLite", "SQL Server", "DuckDB", "ClickHouse", + ]) + func strayParenthesisEndsAtItsSemicolon(typeId: String) { + let grammar = Self.grammar(typeId) + let sql = "SELECT 1;\nSELECT count(id FROM a;\nSELECT 2;\nSELECT 3;" + let caret = (sql as NSString).range(of: "SELECT 3").location + + #expect(sent(sql, grammar) == ["SELECT 1", "SELECT count(id FROM a", "SELECT 2", "SELECT 3"]) + #expect(SQLStatementScanner.statementAtCursor(in: sql, cursorPosition: caret, grammar: grammar) == "SELECT 3") + #expect(!SQLStatementScanner.endsInsideOpenBlock(sql, grammar: grammar)) + } + + @Test("A rule's action list left open holds the rest of the text, which the gate reads as more than one statement") + func openActionListLeavesTheTextInsideABlock() { + let grammar = Self.grammar("PostgreSQL") + let sql = "CREATE RULE r AS ON INSERT TO t DO ALSO (INSERT INTO a VALUES (1);\nDROP TABLE users;" + + #expect(sent(sql, grammar).count == 1) + #expect(SQLStatementScanner.endsInsideOpenBlock(sql, grammar: grammar)) + } + + @Test("The gate tiers each command a rule's action list holds") + func gateTiersThePieces() { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "PostgreSQL", declared: nil, session: nil) + + #expect(SQLStatementClassifier.deletesEveryRow( + "CREATE RULE r AS ON INSERT TO t DO ALSO (SELECT 1; DELETE FROM audit)", + databaseTypeId: "PostgreSQL", + readings: readings + )) + #expect(SQLStatementClassifier.classify( + "SELECT (1; DROP TABLE users; SELECT 1)", databaseTypeId: "PostgreSQL", readings: readings + ).tier == .destructive) + } + + @Test("Inside a routine body a parenthesis one statement leaves open does not hide the body's END") + func bodyStatementParenthesisDoesNotHideTheEnd() { + let sql = "CREATE PROCEDURE p()\nBEGIN\n SELECT (1;\n SELECT 2;\nEND;\nSELECT 3;" + #expect(sent(sql, Self.grammar("MySQL")).last == "SELECT 3") + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift index c63a3d7bbe..79bcaf44ed 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift @@ -20,6 +20,7 @@ private let cursorIndexFixtures = [ CREATE FUNCTION f() RETURNS int AS $$ BEGIN RETURN 1; END; $$ LANGUAGE plpgsql; /* c; */ SELECT E'x' 'y';; + CREATE RULE r AS ON INSERT TO t DO ALSO (INSERT INTO a VALUES (1); INSERT INTO b VALUES (2)); SELECT 2 """), CursorIndexFixture(typeId: "MySQL", text: """ @@ -44,6 +45,15 @@ private let cursorIndexFixtures = [ GO SELECT [a;b] GO + IF @x = 1 + BEGIN + SELECT 1; + SELECT CASE WHEN @x = 1 THEN 2 END; + END + ELSE BEGIN TRY SELECT 3; END TRY BEGIN CATCH SELECT 4; END CATCH; + SELECT (5; 6); + SET XACT_ABORT ON + BEGIN TRY SELECT 7; END TRY BEGIN CATCH THROW; END CATCH """), CursorIndexFixture(typeId: "Oracle", text: """ BEGIN @@ -70,7 +80,8 @@ private let cursorIndexFixtures = [ ] private let cursorIndexInsertions = [ - ";", "'", "\nGO\n", "\n/\n", " ", "x", "$$", "BEGIN ", "END;", "--", "/*", "*/", "\n", "GO", "/", + ";", "'", "\nGO\n", "\n/\n", " ", "x", "$$", "BEGIN ", "END;", "--", "/*", "*/", "\n", "GO", "/", "(", ")", + "CASE ", "END ", ] private let longScriptRow = "SELECT id, name FROM users WHERE active = true;\n" diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift new file mode 100644 index 0000000000..271bc67937 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift @@ -0,0 +1,153 @@ +import Foundation +import TableProSQLGrammar +import Testing + +// T-SQL reserves BEGIN, so a BEGIN ... END block holds its `;`s wherever it stands, not only in a routine's body. +struct TSQLControlFlowBlockTests { + private static let sqlServer = SQLLexicalReadings.resolve(databaseTypeId: "SQL Server", declared: nil, session: nil) + .execution + + private static let merge = "MERGE dbo.t AS t USING dbo.s AS s ON t.id = s.id WHEN MATCHED THEN DELETE" + + private func sent(_ sql: String) -> [String] { + SQLStatementScanner.executableStatements(in: sql, grammar: Self.sqlServer).map(\.sql) + } + + private func atCursor(_ sql: String, on marker: String) -> String { + let offset = (sql as NSString).range(of: marker).location + return SQLStatementScanner.statementAtCursor(in: sql, cursorPosition: offset, grammar: Self.sqlServer) + } + + @Test("A block an IF, a WHILE or a statement of its own opens stays one statement", arguments: [ + ("IF 1=1 BEGIN SELECT 1; SELECT 2; END; SELECT 3", ["IF 1=1 BEGIN SELECT 1; SELECT 2; END", "SELECT 3"]), + ( + "WHILE @i < 3\nBEGIN\n SET @i += 1;\n PRINT @i;\nEND;\nSELECT @i;", + ["WHILE @i < 3\nBEGIN\n SET @i += 1;\n PRINT @i;\nEND", "SELECT @i"] + ), + ("BEGIN\n SELECT 1;\n SELECT 2;\nEND;\nSELECT 3;", ["BEGIN\n SELECT 1;\n SELECT 2;\nEND", "SELECT 3"]), + ( + "IF @a = 1 BEGIN IF @b = 1 BEGIN SELECT 1; END ELSE BEGIN SELECT 2; END; SELECT 3; END; SELECT 4", + ["IF @a = 1 BEGIN IF @b = 1 BEGIN SELECT 1; END ELSE BEGIN SELECT 2; END; SELECT 3; END", "SELECT 4"] + ), + ( + "BEGIN TRY\n DELETE FROM a;\nEND TRY\nBEGIN CATCH\n SELECT ERROR_MESSAGE();\nEND CATCH;\nSELECT 9;", + ["BEGIN TRY\n DELETE FROM a;\nEND TRY\nBEGIN CATCH\n SELECT ERROR_MESSAGE();\nEND CATCH", "SELECT 9"] + ), + ]) + func blockHoldsItsSemicolons(sql: String, expected: [String]) { + #expect(sent(sql) == expected) + } + + @Test("Run Current anywhere in a conditional block runs the whole block") + func caretInsideTheBlockRunsTheBlock() { + let block = "IF @env = 'dev'\nBEGIN\n DELETE FROM a;\n DELETE FROM b;\nEND" + let sql = "SELECT 0;\n\(block);\nSELECT 9;" + + #expect(atCursor(sql, on: "IF @env") == block) + #expect(atCursor(sql, on: "DELETE FROM b") == block) + #expect(SQLStatementScanner.navigableStatements(in: sql, grammar: Self.sqlServer).count == 3) + } + + @Test("An END is never followed by more of its block, so the IF after it is the next statement") + func endBeforeTheNextStatementClosesTheBlock() { + let sql = "IF @a = 1 BEGIN SELECT 1; END\nIF @b = 1 BEGIN SELECT 2; END;\nSELECT 3;" + + #expect(sent(sql) == ["IF @a = 1 BEGIN SELECT 1; END\nIF @b = 1 BEGIN SELECT 2; END", "SELECT 3"]) + #expect(!SQLStatementScanner.endsInsideOpenBlock(sql, grammar: Self.sqlServer)) + } + + @Test("A transaction's BEGIN is a statement, not a block", arguments: ["TRAN", "TRANSACTION", "DISTRIBUTED TRANSACTION"]) + func transactionBeginIsAStatement(opener: String) { + #expect(sent("BEGIN \(opener);\nUPDATE t SET a = 1;\nCOMMIT;") == ["BEGIN \(opener)", "UPDATE t SET a = 1", "COMMIT"]) + } + + @Test("Service Broker's BEGIN DIALOG, BEGIN CONVERSATION TIMER and END CONVERSATION are statements") + func serviceBrokerStatementsAreNotBlocks() { + let script = """ + BEGIN DIALOG CONVERSATION @h FROM SERVICE [s] TO SERVICE 't' ON CONTRACT [c]; + BEGIN CONVERSATION TIMER (@h) TIMEOUT = 60; + END CONVERSATION @h; + SELECT 1; + """ + #expect(sent(script).count == 4) + + let inBlock = "IF @x = 1 BEGIN END CONVERSATION @h; SELECT 1; END; SELECT 2" + #expect(sent(inBlock) == ["IF @x = 1 BEGIN END CONVERSATION @h; SELECT 1; END", "SELECT 2"]) + } + + @Test("A CASE expression closes at its own END, and one left open at a ; does not hold the next statement") + func caseExpressions() { + #expect(sent("SELECT CASE WHEN a = 1 THEN 'x' END AS c FROM t;\nSELECT 2;").count == 2) + #expect(sent("IF @x = 1 BEGIN SELECT CASE WHEN @x = 1 THEN 1 END; SELECT 2; END; SELECT 3").count == 2) + #expect(sent("SELECT CASE WHEN a = 1 THEN 1;\nSELECT 2;") == ["SELECT CASE WHEN a = 1 THEN 1", "SELECT 2"]) + } + + @Test("A MERGE inside a block keeps its ; and the block's own ; is a separator") + func mergeInsideABlock() { + let block = "IF 1 = 1 BEGIN \(Self.merge); END" + #expect(sent(block + ";\nSELECT 1;") == [block, "SELECT 1"]) + } + + @Test("A block left open runs to its GO line and no further") + func goLineEndsAnOpenBlock() { + let sql = "IF 1 = 1 BEGIN SELECT 1;\nGO\nSELECT 2" + #expect(sent(sql) == ["IF 1 = 1 BEGIN SELECT 1", "SELECT 2"]) + #expect(SQLStatementScanner.endsInsideOpenBlock("IF 1 = 1 BEGIN SELECT 1;", grammar: Self.sqlServer)) + } + + @Test("The gate still tiers every statement a block holds") + func gateTiersEachStatementInsideTheBlock() { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "SQL Server", declared: nil, session: nil) + let sql = "IF @env = 'dev' BEGIN SELECT 1; DROP TABLE dbo.t; END" + + #expect(SQLStatementClassifier.classify(sql, databaseTypeId: "SQL Server", readings: readings).tier == .destructive) + #expect(SQLStatementClassifier.deletesEveryRow( + "IF 1 = 1 BEGIN DELETE FROM dbo.t; END", databaseTypeId: "SQL Server", readings: readings + )) + } + + @Test("A TRY block after a statement written without a ; is still one block") + func tryBlockAfterAnUnterminatedStatement() { + let script = "SET XACT_ABORT ON\nBEGIN TRY\n DELETE FROM a;\nEND TRY\nBEGIN CATCH\n THROW;\nEND CATCH;\nSELECT 1;" + #expect(sent(script).count == 2) + #expect(sent(script).last == "SELECT 1") + } + + @Test("A begin where a name stands opens no block", arguments: [ + ("CREATE TABLE begin(a int); DROP TABLE users", 2), + ("INSERT INTO begin VALUES (1); DROP TABLE users", 2), + ("DO BEGIN SELECT 1 FROM DUMMY; SELECT 2 FROM DUMMY; END;", 3), + ]) + func nameInBeginsPlaceOpensNothing(sql: String, count: Int) { + #expect(sent(sql).count == count) + } + + @Test("A block holding more than one statement is told apart from one statement", arguments: [ + ("IF x BEGIN UPDATE a SET c = 1; UPDATE b SET c = 1; END", true), + ("BEGIN MERGE dbo.t AS t USING dbo.s AS s ON t.id = s.id WHEN MATCHED THEN DELETE; SELECT 1; END", true), + ("IF x BEGIN SELECT 1; WITH c AS (SELECT id FROM t) DELETE FROM c; END", true), + ("IF x BEGIN SELECT 1; DISABLE TRIGGER trg ON t; END", true), + ("BEGIN TRY SELECT 1; THROW 50000, 'x', 1; END TRY BEGIN CATCH SELECT 2; END CATCH", true), + ("IF x BEGIN SELECT 1; done: SELECT 2; END", true), + ("IF x BEGIN SELECT 1; (SELECT 2); END", true), + ("IF x BEGIN SELECT 1; END CONVERSATION @h; END", true), + ("IF x SELECT 1", false), + ("IF x BEGIN SELECT 1; END", false), + ("IF x BEGIN SELECT 1;; END", false), + ("BEGIN IF x SELECT 1; ELSE SELECT 2; END", false), + ("BEGIN TRY SELECT 1; END TRY BEGIN CATCH THROW; END CATCH", false), + ("CREATE PROCEDURE p AS BEGIN SELECT 1; SELECT 2; END", false), + ]) + func blockHoldingSeveralStatements(sql: String, holdsSeveral: Bool) { + #expect(SQLUnterminatedStatements.blockHoldsSeveralStatements(sql, grammar: Self.sqlServer) == holdsSeveral) + let postgreSQL = SQLLexicalReadings.resolve(databaseTypeId: "PostgreSQL", declared: nil, session: nil).execution + #expect(!SQLUnterminatedStatements.blockHoldsSeveralStatements(sql, grammar: postgreSQL)) + } + + @Test("Other engines still read BEGIN outside a routine as a word", arguments: ["PostgreSQL", "MySQL", "SQLite"]) + func otherEnginesAreUnchanged(typeId: String) { + let grammar = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil).execution + let statements = SQLStatementScanner.executableStatements(in: "BEGIN\nDELETE FROM t;\nSELECT 1;", grammar: grammar) + #expect(statements.map { $0.sql } == ["BEGIN\nDELETE FROM t", "SELECT 1"]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift new file mode 100644 index 0000000000..4f19e1b292 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift @@ -0,0 +1,249 @@ +import Foundation +@testable import TableProTeradataCore +import Testing + +// Replies are read in order; a hook keyed by a reply's index runs just before that reply is read, which is where a +// request is waiting on the server when Stop is pressed. +private final class StoppableTransport: TeradataTransport { + private var replies: [[UInt8]] + private var stream: [UInt8] = [] + private var repliesRead = 0 + var beforeReply: [Int: () -> Void] = [:] + private(set) var sent: [LanMessage] = [] + private(set) var shutDown = false + private(set) var closed = false + + var sentKinds: [UInt8] { + sent.map { $0.kind } + } + + init(replies: [[UInt8]]) { + self.replies = replies + } + + func send(_ bytes: [UInt8]) throws { + if shutDown || closed { throw TeradataWireError.cancelled } + sent.append(LanMessage.decode( + header: Array(bytes.prefix(LanMessage.headerLength)), + body: Array(bytes.dropFirst(LanMessage.headerLength)) + )) + } + + func receive(_ count: Int) throws -> [UInt8] { + if stream.isEmpty { + beforeReply.removeValue(forKey: repliesRead)?() + if shutDown || closed { throw TeradataWireError.cancelled } + guard !replies.isEmpty else { throw TeradataWireError.truncated("no reply") } + stream = replies.removeFirst() + repliesRead += 1 + } + guard stream.count >= count else { throw TeradataWireError.truncated("closed") } + defer { stream.removeFirst(count) } + return Array(stream.prefix(count)) + } + + func finishConnecting() {} + + func cancel() { + shutDown = true + } + + func close() { + closed = true + } +} + +// A server that never answers: a receive waits until the transport is shut down, which the stop's deadline does from +// its own queue. +private final class SilentTransport: TeradataTransport, @unchecked Sendable { + private let lock = NSLock() + private let shutDown = DispatchSemaphore(value: 0) + private var sent: [UInt8] = [] + private var shut = false + private var received = false + var onFirstReceive: (() -> Void)? + + var sentKinds: [UInt8] { lock.withLock { sent } } + var wasShutDown: Bool { lock.withLock { shut } } + + func send(_ bytes: [UInt8]) throws { + try lock.withLock { + if shut { throw TeradataWireError.cancelled } + sent.append(bytes[2]) + } + } + + func receive(_ count: Int) throws -> [UInt8] { + let first = lock.withLock { () -> Bool in + defer { received = true } + return !received + } + if first { onFirstReceive?() } + _ = shutDown.wait(timeout: .now() + 10) + throw TeradataWireError.cancelled + } + + func finishConnecting() {} + + func cancel() { + lock.withLock { shut = true } + shutDown.signal() + } + + func close() { + cancel() + } +} + +struct TeradataStopTests { + private static let config = TeradataConnectionConfig(host: "td.example", username: "dbc", password: "dbc") + private static let start = MessageKind.start.rawValue + private static let cont = MessageKind.cont.rawValue + private static let abort = MessageKind.abort.rawValue + + private static func reply(_ parcels: [Parcel]) -> [UInt8] { + LanMessage(kind: .start, body: parcels.flatMap { $0.encoded() }).encoded() + } + + private static func failure(code: Int, _ message: String) -> Parcel { + let text = Array(message.utf8) + let body = [UInt8](repeating: 0, count: 8) + + [UInt8(code >> 8), UInt8(code & 0xFF), UInt8(text.count >> 8), UInt8(text.count & 0xFF)] + text + return Parcel(.failure, body: body) + } + + private static let finished = reply([Parcel(.success), Parcel(.endRequest)]) + + private static func thrown(_ body: () throws -> Void) -> Error? { + do { + try body() + return nil + } catch { + return error + } + } + + private static func isCancelled(_ error: Error?) -> Bool { + guard let wire = error as? TeradataWireError, case .cancelled = wire else { return false } + return true + } + + @Test("Stop while the server works on a request aborts it on the session, which stays logged on") + func stopAbortsTheRequestAndKeepsTheSession() throws { + let aborted = Self.reply([Self.failure(code: 3_110, "The transaction was aborted by the user.")]) + let transport = StoppableTransport(replies: [aborted, Self.finished]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[0] = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders CROSS JOIN sales.items") } + + #expect(Self.isCancelled(error)) + #expect(transport.sentKinds == [Self.start, Self.abort]) + #expect(transport.sent.last?.requestNumber == transport.sent.first?.requestNumber) + #expect(transport.sent.last?.body == TeradataMessages.abortParcel().encoded()) + #expect(!transport.shutDown) + #expect(!transport.closed) + #expect(connection.isConnected) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.abort, Self.start]) + } + + @Test("A request that failed on its own before the abort landed keeps its own error") + func failureOtherThanTheAbortIsReported() throws { + let missing = Self.reply([Self.failure(code: 3_807, "Object 'sales.orderz' does not exist.")]) + let transport = StoppableTransport(replies: [missing]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[0] = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orderz") } + + guard let wire = error as? TeradataWireError, case .server(let code, _) = wire else { + Issue.record("expected the server's error, got \(String(describing: error))") + return + } + #expect(code == 3_807) + #expect(connection.isConnected) + } + + // An export Stop lands here: the abort does not take effect once the server has started answering. + @Test("Stop while a long answer is being fetched ends the output and keeps the session") + func stopDuringFetchEndsTheOutput() throws { + let batch = Self.reply([Parcel(.success)]) + let ended = Self.reply([Parcel(.endRequest)]) + let transport = StoppableTransport(replies: [batch, batch, ended, Self.finished]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[1] = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders") } + + #expect(Self.isCancelled(error)) + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont]) + #expect(transport.sent.last?.body == TeradataMessages.cancelParcel().encoded()) + #expect(!transport.shutDown) + #expect(!transport.closed) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont, Self.start]) + } + + @Test("An output the server keeps sending after the Cancel parcel gives the session up") + func unendedOutputGivesTheSessionUp() throws { + let batch = Self.reply([Parcel(.success)]) + let transport = StoppableTransport(replies: [batch, batch, batch]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[1] = { connection.cancel() } + + #expect(Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders") } != nil) + #expect(transport.closed) + #expect(connection.hasLostConnection) + } + + @Test("Stop in a running call before its request goes out refuses the request and keeps the session") + func stopBeforeTheRequestIsSentRefusesIt() throws { + let transport = StoppableTransport(replies: [Self.finished]) + let connection = TeradataConnection(config: Self.config, transport: transport) + + connection.beginOperation() + connection.cancel() + let error = Self.thrown { _ = try connection.execute("DELETE FROM sales.orders") } + connection.endOperation() + + #expect(Self.isCancelled(error)) + #expect(transport.sent.isEmpty) + #expect(connection.isConnected) + + connection.beginOperation() + _ = try connection.execute("SELECT 1") + connection.endOperation() + #expect(transport.sentKinds == [Self.start]) + } + + @Test("A stopped request the server never answers gives the session up after the grace", .timeLimit(.minutes(1))) + func unansweredStopGivesTheSessionUp() throws { + let transport = SilentTransport() + let connection = TeradataConnection(config: Self.config, transport: transport, stopGrace: 0.05) + transport.onFirstReceive = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders CROSS JOIN sales.items") } + + #expect(Self.isCancelled(error)) + #expect(transport.sentKinds == [Self.start, Self.abort]) + #expect(transport.wasShutDown) + #expect(connection.hasLostConnection) + } + + @Test("Stop with no request running sends nothing and keeps the session") + func stopWhileIdleDoesNothing() throws { + let transport = StoppableTransport(replies: [Self.finished]) + let connection = TeradataConnection(config: Self.config, transport: transport) + + connection.cancel() + _ = try connection.execute("SELECT 1") + + #expect(transport.sentKinds == [Self.start]) + #expect(!transport.shutDown) + } +} diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+EmphasizeBracket.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+EmphasizeBracket.swift index 0d390a3be1..04429cdc55 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+EmphasizeBracket.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+EmphasizeBracket.swift @@ -37,7 +37,7 @@ extension TextViewController { pair.0, pair.1, from: range.location, - limit: min((textView.visibleTextRange ?? .zero).max + 4_096, textView.documentRange.max), + limit: min(max(range.location, (textView.visibleTextRange ?? .zero).max) + 4_096, textView.documentRange.max), reverse: false ) { emphasizeCharacter(characterIndex) @@ -52,7 +52,10 @@ extension TextViewController { pair.1, pair.0, from: range.location - 1, - limit: max((textView.visibleTextRange?.location ?? 0) - 4_096, textView.documentRange.location), + limit: max( + min(range.location - 1, textView.visibleTextRange?.location ?? 0) - 4_096, + textView.documentRange.location + ), reverse: true ) { emphasizeCharacter(characterIndex) @@ -91,12 +94,16 @@ extension TextViewController { if reverse { options = options.union(.reverse) } + // A caret outside the visible range can put the limit on the wrong side of `from`, and NSString raises + // an exception nothing can catch for a range past the text. + let length = textView.textStorage.length + let lower = max(0, min(reverse ? limit : from, length)) + let upper = max(0, min(reverse ? from : limit, length)) + guard upper > lower else { return nil } var closeCount = 0 var index: Int? textView.textStorage.mutableString.enumerateSubstrings( - in: reverse ? - NSRange(location: limit, length: from - limit) : - NSRange(location: from, length: limit - from), + in: NSRange(location: lower, length: upper - lower), options: options, using: { substring, range, _, stop in if substring == close { diff --git a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/TextViewControllerTests.swift b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/TextViewControllerTests.swift index ff714958bf..8542ff6869 100644 --- a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/TextViewControllerTests.swift +++ b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/TextViewControllerTests.swift @@ -330,6 +330,24 @@ final class TextViewControllerTests: XCTestCase { ) } + func test_findClosingPairWithALimitOnTheWrongSideOrPastTheText() { + controller.textView.string = "{ Lorem }" + + XCTAssertNil(controller.findClosingPair("{", "}", from: 5, limit: 2, reverse: false)) + XCTAssertNil(controller.findClosingPair("}", "{", from: 2, limit: 5, reverse: true)) + XCTAssertEqual(controller.findClosingPair("{", "}", from: 1, limit: 10_000, reverse: false), 8) + } + + func test_bracketEmphasisFarPastTheVisibleTextFindsThePair() { + controller.configuration.appearance.bracketPairEmphasis = .flash + controller.setText(String(repeating: "x", count: 60_000) + "{ a }") + + controller.setCursorPositions([CursorPosition(range: NSRange(location: 60_001, length: 0))]) + + XCTAssertEqual(controller.cursorPositions.first?.range.location, 60_001) + XCTAssertEqual(controller.findClosingPair("{", "}", from: 60_001, limit: 60_005, reverse: false), 60_004) + } + // MARK: Set Text func test_setText() { diff --git a/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift b/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift index 4d776c917f..661f20f618 100644 --- a/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift +++ b/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift @@ -460,11 +460,13 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } return try executeSQLite(query: query, parameters: []) } - continuation.yield(.header(PluginStreamHeader( + var header = PluginStreamHeader( columns: result.columns, columnTypeNames: result.columnTypeNames, estimatedRowCount: result.rows.count - ))) + ) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) continuation.yield(.rows(result.rows)) continuation.finish() } catch { diff --git a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift index 1b3c890c81..e362dfd30d 100644 --- a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift +++ b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift @@ -156,9 +156,11 @@ extension BigtablePluginDriver { try await self.export(spec, continuation: continuation) } else { let result = try await self.execute(query: query) - continuation.yield(.header(PluginStreamHeader( + var header = PluginStreamHeader( columns: result.columns, columnTypeNames: result.columnTypeNames, estimatedRowCount: nil - ))) + ) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } } continuation.finish() diff --git a/apple/Plugins/CSVExportPlugin/CSVExportModels.swift b/apple/Plugins/CSVExportPlugin/CSVExportModels.swift index 2fdbd570a2..cd7942fb4b 100644 --- a/apple/Plugins/CSVExportPlugin/CSVExportModels.swift +++ b/apple/Plugins/CSVExportPlugin/CSVExportModels.swift @@ -79,7 +79,8 @@ public struct CSVExportOptions: Equatable, Codable, Sendable { public var quoteHandling: CSVQuoteHandling = .asNeeded public var lineBreak: CSVLineBreak = .lf public var decimalFormat: CSVDecimalFormat = .period - public var sanitizeFormulas: Bool = true + // Off: RFC 4180 carries a field as written, and OWASP scopes the quote prefix to files meant for a spreadsheet. + public var sanitizeFormulas: Bool = false /// What the file's bytes are written in. UTF-8 keeps every existing export byte for byte. public var encoding: PluginTextEncoding = .utf8 diff --git a/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift b/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift index a35bd63f80..f38791eced 100644 --- a/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift +++ b/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift @@ -23,7 +23,11 @@ struct CSVExportOptionsView: View { Toggle("Sanitize formula-like values", isOn: $plugin.settings.sanitizeFormulas) .toggleStyle(.checkbox) - .help("Prevent CSV formula injection by prefixing values starting with =, +, -, @ with a single quote") + .help(""" + For a file opened in a spreadsheet. Values that would run as formulas, such as those starting \ + with =, +, - or @, get a leading single quote, which stays in the value for anything that \ + imports the file. + """) } Divider() diff --git a/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift b/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift index 6f11d384b2..f4f2bf17b3 100644 --- a/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift +++ b/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift @@ -59,6 +59,7 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let lineBreak = options.lineBreak.value let encoding = options.encoding var report = CSVEncodingReport() + var neutralisedCount = 0 if options.writesByteOrderMark, !encoding.byteOrderMark.isEmpty { try fileHandle.write(contentsOf: Data(encoding.byteOrderMark)) @@ -80,7 +81,7 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } var isFirstBatch = true - var columns: [String] = [] + var columnNumbers: [PluginColumnNumbers] = [] let stream = dataSource.streamRows(for: table) for try await element in stream { @@ -88,17 +89,27 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi switch element { case .header(let header): - columns = header.columns + columnNumbers = header.columnTypeNames.map { PluginColumnNumbers(typeName: $0) } if isFirstBatch && options.includeFieldNames { - let headerLine = columns - .map { escapeCSVField($0, options: options) } + if options.sanitizeFormulas { + neutralisedCount += header.columns.filter(PluginRowWriters.isFormulaLike).count + } + let headerLine = header.columns + .map { escapeCSVField($0, options: options, sanitizes: options.sanitizeFormulas) } .joined(separator: options.delimiter.actualValue) try write(headerLine + lineBreak, to: fileHandle, encoding: encoding, report: &report) } isFirstBatch = false case .rows(let rows): for row in rows { - try writeCSVRow(row, options: options, to: fileHandle, report: &report) + try writeCSVRow( + row, + columnNumbers: columnNumbers, + options: options, + to: fileHandle, + report: &report, + neutralisedCount: &neutralisedCount + ) progress.incrementRow() } } @@ -114,7 +125,22 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi try PluginExportUtilities.commitAtomicWrite(from: tempURL, to: destination) committed = true progress.finalizeTable() - return ExportFormatResult(warnings: report.warnings(for: encoding)) + return ExportFormatResult(warnings: report.warnings(for: encoding) + Self.neutralisedWarnings(neutralisedCount)) + } + + // The quote is part of the value for anything that imports the file, so the export says how many it changed. + static func neutralisedWarnings(_ count: Int) -> [String] { + guard count > 0 else { return [] } + return [String( + format: String( + localized: """ + Values a spreadsheet would read as formulas, written with a leading single quote: %lld. \ + Turn off Sanitize formula-like values to export them unchanged. + """, + bundle: .main + ), + Int64(count) + )] } // MARK: - Private @@ -143,26 +169,31 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi try fileHandle.write(contentsOf: encoded.data) } + // A text column holding "1.10" is a version or a code; a column with no usable type keeps the decimal comma. private func writeCSVRow( _ row: [PluginCellValue], + columnNumbers: [PluginColumnNumbers], options: CSVExportOptions, to fileHandle: FileHandle, - report: inout CSVEncodingReport + report: inout CSVEncodingReport, + neutralisedCount: inout Int ) throws { - let delimiter = options.delimiter.actualValue - let lineBreak = options.lineBreak.value + var fields: [String] = [] + fields.reserveCapacity(row.count) - let rowLine = row.map { cell -> String in + for (index, cell) in row.enumerated() { let val: String switch cell { case .null: - return options.convertNullToEmpty ? "" : "NULL" + fields.append(options.convertNullToEmpty ? "" : "NULL") + continue case .text(let s): val = s case .bytes(let d): val = "0x" + d.map { String(format: "%02X", $0) }.joined() } + let takesDecimalComma = index >= columnNumbers.count || columnNumbers[index] != .text var processed = val // `\r\n` is one Character, so only a scalar test sees a Windows line break. let hadLineBreaks = val.unicodeScalars.contains { $0 == "\n" || $0 == "\r" } @@ -174,40 +205,54 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi .replacingOccurrences(of: "\n", with: " ", options: .literal) } - if options.decimalFormat == .comma { + var isCommaDecimal = false + if options.decimalFormat == .comma, takesDecimalComma { let range = NSRange(processed.startIndex..., in: processed) if Self.decimalFormatRegex.firstMatch(in: processed, range: range) != nil { processed = processed.replacingOccurrences(of: ".", with: ",") + isCommaDecimal = true } } - return escapeCSVField(processed, options: options, originalHadLineBreaks: hadLineBreaks) - }.joined(separator: delimiter) + // `-1,5` is still a number, and the quote would turn it into text. + let sanitizes = options.sanitizeFormulas && !isCommaDecimal + if sanitizes, PluginRowWriters.isFormulaLike(processed) { + neutralisedCount += 1 + } + fields.append(escapeCSVField( + processed, options: options, sanitizes: sanitizes, originalHadLineBreaks: hadLineBreaks + )) + } - try write(rowLine + lineBreak, to: fileHandle, encoding: options.encoding, report: &report) + try write( + fields.joined(separator: options.delimiter.actualValue) + options.lineBreak.value, + to: fileHandle, + encoding: options.encoding, + report: &report + ) } - /// Escaping and quoting live in `PluginRowWriters`, so this format, the other export formats - /// and the MCP tool spell a value the same way. Only the option mapping is this plugin's own. - /// - /// `originalHadLineBreaks` says the value's breaks were already replaced with spaces upstream, - /// and it still forces quoting: the source text spanned lines, and a reader that splits on the - /// delimiter has no way to know the space it now sees was one. - private func escapeCSVField(_ field: String, options: CSVExportOptions, originalHadLineBreaks: Bool = false) -> String { - let escaped = PluginRowWriters.csvField(field, options: writeOptions(options)) + // `originalHadLineBreaks` still forces quoting after the breaks became spaces: the source text spanned lines. + private func escapeCSVField( + _ field: String, + options: CSVExportOptions, + sanitizes: Bool, + originalHadLineBreaks: Bool = false + ) -> String { + let escaped = PluginRowWriters.csvField(field, options: writeOptions(options, sanitizes: sanitizes)) guard originalHadLineBreaks, options.quoteHandling == .asNeeded, escaped.unicodeScalars.first != "\"" else { return escaped } return "\"\(escaped.replacingOccurrences(of: "\"", with: "\"\"", options: .literal))\"" } - private func writeOptions(_ options: CSVExportOptions) -> PluginCsvWriteOptions { + private func writeOptions(_ options: CSVExportOptions, sanitizes: Bool) -> PluginCsvWriteOptions { PluginCsvWriteOptions( delimiter: options.delimiter.actualValue, quoteHandling: quoteHandling(options.quoteHandling), lineEnding: options.lineBreak.value, nullAsEmpty: true, - sanitizesFormulas: options.sanitizeFormulas, + sanitizesFormulas: sanitizes, flattensLineBreaks: false ) } diff --git a/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift b/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift index 0a3b2d9ae3..c9716c5229 100644 --- a/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift +++ b/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift @@ -29,13 +29,13 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi var fieldDetectionSignature: String { settings.detectionSignature } - var rowImportConfiguration: RowImportRunner.Configuration? { runConfiguration } + var rowImportConfiguration: RowImportRunner.Configuration? { Self.runConfiguration(settings) } - private var runConfiguration: RowImportRunner.Configuration { + private static func runConfiguration(_ options: CSVImportOptions) -> RowImportRunner.Configuration { RowImportRunner.Configuration( - errorHandling: settings.errorHandling, - wrapInTransaction: settings.wrapInTransaction, - deleteExistingRows: settings.deleteExistingRows + errorHandling: options.errorHandling, + wrapInTransaction: options.wrapInTransaction, + deleteExistingRows: options.deleteExistingRows ) } @@ -58,10 +58,12 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi ) async throws -> PluginImportResult { let startTime = Date() let url = source.fileURL() + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-import. + let options = settings let text: CSVImportText do { - text = try CSVImportText.contents(of: url, encoding: settings.encoding, isCancelled: { progress.isCancelled }) + text = try CSVImportText.contents(of: url, encoding: options.encoding, isCancelled: { progress.isCancelled }) } catch is TabularCancellation { throw PluginImportCancellationError() } catch { @@ -80,9 +82,9 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi } let data = text.data - let dialect = CSVImportParsing.resolveDialect(in: data, options: settings) + let dialect = CSVImportParsing.resolveDialect(in: data, options: options) let parser = CSVStreamingParser(dialect: dialect) - let hasHeader = settings.hasHeaderRow + let hasHeader = options.hasHeaderRow let (dataRanges, columnNames) = indexRowsAndNames(in: data, parser: parser, hasHeader: hasHeader) guard !columnNames.isEmpty else { @@ -93,7 +95,7 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi let lineOffset = hasHeader ? 2 : 1 var cursor = 0 - let configuration = runConfiguration + let configuration = Self.runConfiguration(options) var pendingFailure: PluginImportError? var malformedCount = 0 var malformedErrors: [PluginImportResult.ImportStatementError] = [] @@ -111,7 +113,8 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi ranges: dataRanges[cursor..>, startIndex: Int, lineOffset: Int, - columnNames: [String] + columnNames: [String], + options: CSVImportOptions ) -> (rows: [RowImportRunner.Entry], malformed: [MalformedRow]) { data.withUnsafeBytes { raw -> (rows: [RowImportRunner.Entry], malformed: [MalformedRow]) in guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return ([], []) } @@ -185,12 +189,12 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi if CSVImportParsing.isBlank(fields) { continue } let line = startIndex + offset + lineOffset if let error = CSVImportParsing.shapeError( - fields: fields, columnCount: columnNames.count, options: settings + fields: fields, columnCount: columnNames.count, options: options ) { malformed.append((line, error)) continue } - out.append((line, CSVImportParsing.row(fields: fields, columnNames: columnNames, options: settings))) + out.append((line, CSVImportParsing.row(fields: fields, columnNames: columnNames, options: options))) } return (out, malformed) } diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift index f59b0a903d..ff8717faec 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift @@ -324,7 +324,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen let rawResult = try await connectionActor.executeQuery( query, consistency: consistency, cancellation: cancellation ) - if let refusal = CassandraRowWriter.unappliedInsertRefusal( + if let refusal = CassandraRowWriter.unappliedWriteRefusal( statement: query, columns: rawResult.columns, rows: rawResult.rows ) { throw CassandraPluginError.queryFailed(refusal) @@ -371,7 +371,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen let rawResult = try await connectionActor.executePrepared( query, parameters: parameters, consistency: consistency ) - if let refusal = CassandraRowWriter.unappliedInsertRefusal( + if let refusal = CassandraRowWriter.unappliedWriteRefusal( statement: query, columns: rawResult.columns, rows: rawResult.rows ) { throw CassandraPluginError.queryFailed(refusal) diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift b/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift index 0c7ce00b73..ca4edf9125 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift @@ -4,16 +4,15 @@ // import Foundation +import TableProCassandraCore import TableProPluginKit -/// The CQL a grid save writes: one statement per row, each naming the row by its whole primary key. -/// -/// CQL has no `OR` and no parenthesised groups, so several rows cannot share one `DELETE`, and an `INSERT` is an -/// upsert, so a new row whose key is taken would silently replace the row already there. A new row is therefore -/// written `IF NOT EXISTS`, and anything CQL cannot do to a row is refused by name rather than sent. +// One statement per row, each naming the row by its whole primary key: CQL has no `OR` and no parenthesised groups. +// A new row is written `IF NOT EXISTS` and an edit `IF EXISTS` (CassandraConditionalWrite), and anything CQL cannot +// do to a row is refused by name rather than sent. enum CassandraRowWriter { static let defaultSentinel = "__DEFAULT__" - static let unappliedMarker = "[applied]" + static let unappliedMarker = CassandraConditionalWrite.appliedColumn static func rowWrites( keyspace: String?, @@ -54,15 +53,18 @@ enum CassandraRowWriter { } } - /// An `INSERT … IF NOT EXISTS` that found its key taken answers with `[applied] = false` and the row already - /// there, which the server reports as a success. - static func unappliedInsertRefusal(statement: String, columns: [String], rows: [[PluginCellValue]]) -> String? { - let normalized = statement.uppercased() - guard normalized.hasPrefix("INSERT"), normalized.hasSuffix("IF NOT EXISTS"), - columns.first == unappliedMarker, - rows.first?.first?.asText?.lowercased() == "false" - else { return nil } - return String(localized: "A row with this primary key already exists. Edit that row instead.") + static func unappliedWriteRefusal(statement: String, columns: [String], rows: [[PluginCellValue]]) -> String? { + let refusal = CassandraConditionalWrite.refusal( + statement: statement, firstColumn: columns.first, firstValue: rows.first?.first?.asText + ) + switch refusal { + case .keyTaken: + return String(localized: "A row with this primary key already exists. Edit that row instead.") + case .rowMissing: + return String(localized: "No row has this primary key, so nothing was updated. Another client may have deleted it.") + case nil: + return nil + } } private struct Writer { @@ -100,8 +102,7 @@ enum CassandraRowWriter { )) } } - let statement = "INSERT INTO \(target) (\(names.joined(separator: ", "))) " - + "VALUES (\(placeholders.joined(separator: ", "))) IF NOT EXISTS" + let statement = CassandraConditionalWrite.insertStatement(target: target, names: names, values: placeholders) return PluginRowWrite(statement: statement, parameters: parameters, rowIndices: [change.rowIndex]) } @@ -133,7 +134,9 @@ enum CassandraRowWriter { } } let key = try keyCondition(for: change) - let statement = "UPDATE \(target) SET \(assignments.joined(separator: ", ")) WHERE \(key.cql)" + let statement = CassandraConditionalWrite.updateStatement( + target: target, assignments: assignments, keyCondition: key.cql + ) return PluginRowWrite(statement: statement, parameters: parameters + key.values, rowIndices: [change.rowIndex]) } diff --git a/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift b/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift index cc80a87298..70b89896ec 100644 --- a/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift +++ b/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift @@ -323,10 +323,9 @@ final class DamengPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let task = Task { do { let result = try await self.execute(query: query, context: context) - continuation.yield(.header(PluginStreamHeader( - columns: result.columns, - columnTypeNames: result.columnTypeNames - ))) + var header = PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } diff --git a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift index ee013d3fe9..72389cdf12 100644 --- a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift +++ b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift @@ -373,7 +373,9 @@ extension DynamoDBPluginDriver { source = try await spoolSource(for: statement, session: session) } catch DynamoDBError.invalidStatement(let message) where message == Self.notAReadMarker { let result = try await runStatement(text, parameters: [], rowCap: PluginRowLimits.emergencyMax) - continuation.yield(.header(PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames))) + var header = PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } return } diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift index e8d3c50f61..038ed7dbcc 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift @@ -242,11 +242,11 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation: AsyncThrowingStream.Continuation ) async throws { let matcher = try filter.matcher() - continuation.yield(.header(PluginStreamHeader( + var header = PluginStreamHeader( columns: Self.columns, columnTypeNames: Self.columnTypeNames, estimatedRowCount: nil - ))) + ) let range = Self.prefixRange(prefix) guard sortAscending else { @@ -256,6 +256,9 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { req.sortTarget = "KEY" let response = try await client.rangeRequest(req) let rows = (response.kvs ?? []).map(Self.filterRow).filter(matcher.matches).map(\.cells) + // `more` says the range held keys past the limit. + if response.more == true { header.rowLimit = rows.count } + continuation.yield(.header(header)) if !rows.isEmpty { continuation.yield(.rows(rows)) } @@ -263,6 +266,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return } + continuation.yield(.header(header)) do { try await scanKeyValues(in: range, client: client) { kvs in let rows = kvs.map(Self.filterRow).filter(matcher.matches).map(\.cells) diff --git a/apple/Plugins/FormatPluginTests/CSVExportFormulaAndDecimalTests.swift b/apple/Plugins/FormatPluginTests/CSVExportFormulaAndDecimalTests.swift new file mode 100644 index 0000000000..db59df3638 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/CSVExportFormulaAndDecimalTests.swift @@ -0,0 +1,174 @@ +// +// CSVExportFormulaAndDecimalTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +private final class TypedCSVSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId = "PostgreSQL" + private let header: PluginStreamHeader + private let rows: [PluginRow] + + init(columns: [String], columnTypeNames: [String], rows: [PluginRow]) { + header = PluginStreamHeader(columns: columns, columnTypeNames: columnTypeNames) + self.rows = rows + } + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + let header = header + let rows = rows + return AsyncThrowingStream { continuation in + continuation.yield(.header(header)) + continuation.yield(.rows(rows)) + continuation.finish() + } + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { "" } + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { "\"\(identifier)\"" } + + func escapeStringLiteral(_ value: String) -> String { value } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { nil } +} + +struct CSVExportFormulaAndDecimalTests { + private func export( + _ options: CSVExportOptions, + columns: [String], + types: [String], + rows: [PluginRow] + ) async throws -> (text: String, warnings: [String]) { + let output = try await CSVExportHarness.shared.bytes( + options: options, + tables: [PluginExportTable(name: "users", databaseName: "", tableType: "table", schema: nil, kind: .table)], + dataSource: TypedCSVSource(columns: columns, columnTypeNames: types, rows: rows) + ) + return (String(decoding: output.data, as: UTF8.self), output.result.warnings) + } + + // A database export is read back by importers, which keep the quote as part of the value. + @Test("The stock export writes a value that starts like a formula as it is") + func stockExportKeepsValues() async throws { + let output = try await export( + CSVExportOptions(), + columns: ["handle"], + types: ["text"], + rows: [["@handle"], ["+84 90 123 4567"], ["-pending"]] + ) + #expect(output.text == "handle\n@handle\n+84 90 123 4567\n-pending\n") + #expect(output.warnings.isEmpty) + } + + @Test("Sanitizing says how many values it gave a leading quote") + func sanitizingCountsWhatItChanged() async throws { + var options = CSVExportOptions() + options.sanitizeFormulas = true + let output = try await export( + options, + columns: ["handle"], + types: ["text"], + rows: [["@handle"], ["plain"], ["=1+1"], ["-5"], ["-pending"]] + ) + #expect(output.text == "handle\n\"'@handle\"\nplain\n\"'=1+1\"\n-5\n\"'-pending\"\n") + #expect(output.warnings == CSVExportPlugin.neutralisedWarnings(3)) + } + + @Test("Nothing changed means no warning about changes") + func noCountWithoutChanges() { + #expect(CSVExportPlugin.neutralisedWarnings(0).isEmpty) + } + + // A text column holding "1.10" is a version or a code, not a number. + @Test("A decimal comma leaves text columns alone and keeps columns with no usable type") + func decimalCommaSkipsTextColumns() async throws { + var options = CSVExportOptions() + options.decimalFormat = .comma + options.delimiter = .semicolon + let output = try await export( + options, + columns: ["version", "amount", "untyped"], + types: ["varchar(16)", "numeric(10,2)", ""], + rows: [["1.10", "1.10", "1.10"]] + ) + #expect(output.text == "version;amount;untyped\n1.10;1,10;1,10\n") + } + + @Test("A decimal comma reaches every engine's numeric type", arguments: [ + "Nullable(Float64)", "Nullable(Decimal(10, 2))", "money", "smallmoney", "BINARY_DOUBLE", "BINARY_FLOAT", + "BIGNUMERIC", "double precision", "NUMBER(10,2)" + ]) + func decimalCommaReachesNumericTypes(typeName: String) async throws { + var options = CSVExportOptions() + options.decimalFormat = .comma + options.delimiter = .semicolon + let output = try await export(options, columns: ["n"], types: [typeName], rows: [["1.5"]]) + #expect(output.text == "n\n1,5\n") + } + + @Test("A decimal comma never reaches a text type", arguments: [ + "varchar(16)", "String", "LowCardinality(String)", "nvarchar(10)", "TEXT", "jsonb" + ]) + func decimalCommaSkipsTextTypes(typeName: String) async throws { + var options = CSVExportOptions() + options.decimalFormat = .comma + options.delimiter = .semicolon + let output = try await export(options, columns: ["n"], types: [typeName], rows: [["1.5"]]) + #expect(output.text == "n\n1.5\n") + } + + @Test("A header name sanitizing changes is counted with the values") + func headerNamesAreCounted() async throws { + var options = CSVExportOptions() + options.sanitizeFormulas = true + let output = try await export(options, columns: ["=total"], types: ["text"], rows: [["=1"]]) + #expect(output.text == "\"'=total\"\n\"'=1\"\n") + #expect(output.warnings == CSVExportPlugin.neutralisedWarnings(2)) + } + + // `'-1,5` is text to a spreadsheet, so a negative number written with a decimal comma lost its value. + @Test("A number written with a decimal comma is never given a quote") + func commaDecimalsAreNotSanitized() async throws { + var options = CSVExportOptions() + options.sanitizeFormulas = true + options.decimalFormat = .comma + options.delimiter = .semicolon + let output = try await export( + options, + columns: ["amount", "note"], + types: ["numeric", "text"], + rows: [["-1.5", "-1.5x"]] + ) + #expect(output.text == "amount;note\n-1,5;\"'-1.5x\"\n") + #expect(output.warnings == CSVExportPlugin.neutralisedWarnings(1)) + } +} + +struct PluginRowWritersFormulaLeadTests { + // OWASP lists a leading line feed and the full-width operators, which a Japanese locale reads as formulas. + @Test("A leading line feed or a full-width operator counts as a formula lead") + func owaspLeadsAreNeutralised() { + let options = PluginCsvWriteOptions.default + #expect(PluginRowWriters.csvField("\n=1+1", options: options).hasPrefix("\"'")) + for lead in ["\u{FF1D}", "\u{FF0B}", "\u{FF0D}", "\u{FF20}"] { + #expect(PluginRowWriters.csvField("\(lead)SUM(A1)", options: options) == "\"'\(lead)SUM(A1)\"") + } + } + + @Test("Only a value that would be prefixed is formula-like") + func formulaLikeMatchesTheWriter() { + #expect(PluginRowWriters.isFormulaLike("=1+1")) + #expect(PluginRowWriters.isFormulaLike("\t=1+1")) + #expect(!PluginRowWriters.isFormulaLike("-5")) + #expect(!PluginRowWriters.isFormulaLike("plain")) + #expect(!PluginRowWriters.isFormulaLike("")) + } +} diff --git a/apple/Plugins/FormatPluginTests/ExportImportRoundTripTests.swift b/apple/Plugins/FormatPluginTests/ExportImportRoundTripTests.swift index 29503ea388..ed0cae7764 100644 --- a/apple/Plugins/FormatPluginTests/ExportImportRoundTripTests.swift +++ b/apple/Plugins/FormatPluginTests/ExportImportRoundTripTests.swift @@ -85,6 +85,14 @@ struct ExportImportRoundTripTests { try RoundTrip.expectSame(target, RoundTripFixture.table, FormatLimits.csv.carried(RoundTripFixture.rows)) } + @Test("A CSV file sanitized for a spreadsheet imports back with the quote it gave formula-like values") + func csvSanitizedImportsBackWithItsQuotes() async throws { + var options = CSVExportOptions() + options.sanitizeFormulas = true + let target = try await RoundTrip.csv(options) + try RoundTrip.expectSame(target, RoundTripFixture.table, FormatLimits.csvSanitized.carried(RoundTripFixture.rows)) + } + @Test("A CSV file written with a byte order mark imports back") func csvWithByteOrderMarkImportsBack() async throws { var options = CSVExportOptions() @@ -166,9 +174,14 @@ struct ExportImportRoundTripTests { extension FormatLimits { // CSV has no NULL and no binary: both NULL and '' are written as an empty field and read as - // NULL, and a binary value is written as hex text. A value a spreadsheet would run as a - // formula gains a leading quote. + // NULL, and a binary value is written as hex text. static let csv = FormatLimits( + emptyTextIsNull: true, + binary: { .text(FormatLimits.hex($0)) } + ) + + // Sanitizing for a spreadsheet gives a value that would run as a formula a leading quote, which the import keeps. + static let csvSanitized = FormatLimits( emptyTextIsNull: true, binary: { .text(FormatLimits.hex($0)) }, text: { FormatLimits.csvNeutralised[$0] ?? $0 } diff --git a/apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift b/apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift new file mode 100644 index 0000000000..f995008f40 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift @@ -0,0 +1,185 @@ +// +// FormatSettingsChangedMidRunTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +// One plugin instance serves every window, so a dialog open in another window changes `settings` while a run reads. +// Each source here changes them between its first and second batch, the way that dialog would. +private final class TwoBatchSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId = "SQLite" + private let header: PluginStreamHeader + private let batches: [[PluginRow]] + private let betweenBatches: @Sendable () -> Void + + init(columns: [String], batches: [[PluginRow]], betweenBatches: @escaping @Sendable () -> Void) { + header = PluginStreamHeader(columns: columns, columnTypeNames: columns.map { _ in "TEXT" }) + self.batches = batches + self.betweenBatches = betweenBatches + } + + // Pulled, so the change lands only after the plugin has written the first batch. + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + let header = header + let batches = batches + let betweenBatches = betweenBatches + let pulls = Counter() + return AsyncThrowingStream(unfolding: { + let pull = pulls.next() + if pull == 0 { return .header(header) } + guard pull <= batches.count else { return nil } + if pull == 2 { betweenBatches() } + return .rows(batches[pull - 1]) + }) + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { "" } + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { "\"\(identifier)\"" } + + func escapeStringLiteral(_ value: String) -> String { value } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { nil } +} + +private final class Counter: @unchecked Sendable { + private let lock = NSLock() + private var value = 0 + + func next() -> Int { + lock.lock() + defer { lock.unlock() } + defer { value += 1 } + return value + } +} + +private final class ForeignKeyRecordingSink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "SQLite" + var onExecute: () -> Void = {} + private(set) var disabled = 0 + private(set) var enabled = 0 + + func execute(statement: String) async throws { onExecute() } + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} + func disableForeignKeyChecks() async throws { disabled += 1 } + func enableForeignKeyChecks() async throws { enabled += 1 } +} + +private final class OneStatementSource: PluginImportSource, @unchecked Sendable { + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + AsyncThrowingStream { continuation in + continuation.yield((statement: "INSERT INTO t VALUES (1);", lineNumber: 1)) + continuation.finish() + } + } + + func fileURL() -> URL { URL(fileURLWithPath: "/dev/null") } + + func fileSizeBytes() -> Int64 { 0 } +} + +@Suite(.serialized) +struct FormatSettingsChangedMidRunTests { + private let table = PluginExportTable(name: "t", databaseName: "", tableType: "table", schema: nil, kind: .table) + private let rowWithNull: PluginRow = [.text("1"), .null] + + private func export( + _ plugin: any ExportFormatPlugin, + from source: TwoBatchSource, + fileExtension: String + ) async throws -> Data { + let destination = FileManager.default.temporaryDirectory + .appendingPathComponent("\(UUID().uuidString).\(fileExtension)") + defer { try? FileManager.default.removeItem(at: destination) } + _ = try await plugin.export( + tables: [table], + dataSource: source, + destination: destination, + progress: PluginExportProgress(progress: Progress(totalUnitCount: 1)) + ) + return try Data(contentsOf: destination) + } + + @Test("JSON keeps writing null keys after another window turns Include NULL values off") + func jsonKeepsItsNullChoice() async throws { + let plugin = JSONExportPlugin() + let stored = plugin.settings + defer { plugin.settings = stored } + var options = JSONExportOptions() + options.layout = .newlineDelimited + options.includeNullValues = true + plugin.settings = options + + let source = TwoBatchSource(columns: ["id", "note"], batches: [[rowWithNull], [rowWithNull]]) { + plugin.settings.includeNullValues = false + } + let text = String(decoding: try await export(plugin, from: source, fileExtension: "ndjson"), as: UTF8.self) + #expect(text == "{\"id\": \"1\", \"note\": null}\n{\"id\": \"1\", \"note\": null}\n") + } + + @Test("Markdown keeps its NULL placeholder after another window changes it") + func markdownKeepsItsPlaceholder() async throws { + let plugin = MarkdownExportPlugin() + let stored = plugin.settings + defer { plugin.settings = stored } + var options = MarkdownExportOptions() + options.alignsColumns = false + options.includesTableNames = false + options.nullPlaceholder = "NULL" + plugin.settings = options + + let source = TwoBatchSource(columns: ["id", "note"], batches: [[rowWithNull], [rowWithNull]]) { + plugin.settings.nullPlaceholder = "(none)" + } + let text = String(decoding: try await export(plugin, from: source, fileExtension: "md"), as: UTF8.self) + #expect(!text.contains("(none)")) + #expect(text.components(separatedBy: "| 1 | NULL |").count == 3) + } + + @Test("XLSX keeps writing NULL as an empty cell after another window turns the option off") + func xlsxKeepsItsNullChoice() async throws { + let plugin = XLSXExportPlugin() + let stored = plugin.settings + defer { plugin.settings = stored } + var options = XLSXExportOptions() + options.convertNullToEmpty = true + plugin.settings = options + + let source = TwoBatchSource(columns: ["id", "note"], batches: [[rowWithNull], [rowWithNull]]) { + plugin.settings.convertNullToEmpty = false + } + let written = try await export(plugin, from: source, fileExtension: "xlsx") + let text = try #require(String(bytes: written, encoding: .isoLatin1)) + #expect(!text.contains("NULL")) + } + + @Test("A SQL import turns foreign key checks back on after another window unticks the option") + func sqlImportRestoresTheChecksItTurnedOff() async throws { + let plugin = SQLImportPlugin() + let stored = plugin.settings + defer { plugin.settings = stored } + var options = SQLImportOptions() + options.disableForeignKeyChecks = true + plugin.settings = options + + let sink = ForeignKeyRecordingSink() + sink.onExecute = { plugin.settings.disableForeignKeyChecks = false } + _ = try await plugin.performImport( + source: OneStatementSource(), + sink: sink, + progress: PluginImportProgress(progress: Progress()) + ) + #expect(sink.disabled == 1) + #expect(sink.enabled == 1) + } +} diff --git a/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift b/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift index cf23a29b92..60adecb627 100644 --- a/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift +++ b/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift @@ -53,10 +53,11 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug } } - /// NDJSON is one row per line by definition, so the pretty-print setting cannot apply to it - /// and the wrapping object has no place to go. - let isNewlineDelimited = settings.layout == .newlineDelimited - let prettyPrint = settings.prettyPrint && !isNewlineDelimited + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-export. + let options = settings + // NDJSON is one row per line by definition, so pretty-printing and the wrapping object do not apply. + let isNewlineDelimited = options.layout == .newlineDelimited + let prettyPrint = options.prettyPrint && !isNewlineDelimited let indent = prettyPrint ? " " : "" let newline = prettyPrint ? "\n" : "" @@ -104,7 +105,7 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug for (colIndex, column) in columns.enumerated() { if colIndex < row.count { let value = row[colIndex] - if settings.includeNullValues || !value.isNull { + if options.includeNullValues || !value.isNull { if !isFirstField { rowString += ", " } @@ -117,7 +118,7 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug let jsonValue = formatJSONValue( value, columnTypeName: colTypeName, - preserveAsString: settings.preserveAllAsStrings + preserveAsString: options.preserveAllAsStrings ) rowString += "\"\(escapedKey)\": \(jsonValue)" } diff --git a/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift b/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift index 2ee2f17664..8fc08e0fd3 100644 --- a/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift +++ b/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift @@ -54,13 +54,15 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable if !committed { PluginExportUtilities.rollbackAtomicWrite(at: tempURL) } } + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-export. + let options = settings for (index, table) in tables.enumerated() { try progress.checkCancellation() progress.setCurrentTable(table.qualifiedName, index: index + 1) - if settings.includesTableNames { + if options.includesTableNames { try fileHandle.write(contentsOf: "## \(table.qualifiedName)\n\n".toUTF8Data()) } - try await writeTable(table, dataSource: dataSource, to: fileHandle, progress: progress) + try await writeTable(table, options: options, dataSource: dataSource, to: fileHandle, progress: progress) try fileHandle.write(contentsOf: "\n".toUTF8Data()) } @@ -73,6 +75,7 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable private func writeTable( _ table: PluginExportTable, + options: MarkdownExportOptions, dataSource: any PluginExportDataSource, to fileHandle: FileHandle, progress: PluginExportProgress @@ -89,9 +92,9 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable columns = header.columns case .rows(let rows): for row in rows { - let cells = row.map { cellText($0) } + let cells = row.map { cellText($0, nullPlaceholder: options.nullPlaceholder) } progress.incrementRow() - guard settings.alignsColumns, !wroteHeader else { + guard options.alignsColumns, !wroteHeader else { if !wroteHeader { try writeHeader(columns, widths: nil, to: fileHandle) wroteHeader = true @@ -115,7 +118,7 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable } if !wroteHeader { - widths = settings.alignsColumns + widths = options.alignsColumns ? MarkdownTableRenderer.widths(header: columns, sample: buffered) : nil try writeHeader(columns, widths: widths, to: fileHandle) @@ -138,10 +141,10 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable ).toUTF8Data()) } - private func cellText(_ value: PluginCellValue) -> String { + private func cellText(_ value: PluginCellValue, nullPlaceholder: String) -> String { switch value { case .null: - return MarkdownTableRenderer.cell(settings.nullPlaceholder) + return MarkdownTableRenderer.cell(nullPlaceholder) case .bytes(let data): return MarkdownTableRenderer.cell("0x\(data.map { String(format: "%02X", $0) }.joined())") case .text(let text): diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift index 3be35d7dd1..55c8df7418 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift @@ -518,7 +518,10 @@ nonisolated extension MongoDBConnection { let documents = try MongoExportDocument.read( MongoReadDocuments(dictionaries: sample, texts: texts), representation: uuidRepresentation ) - continuation.yield(.header(projection.header)) + var header = projection.header + // iterateCursorStreaming stops there, so a stream that reaches it was cut. + header.rowLimit = PluginRowLimits.emergencyMax + continuation.yield(.header(header)) if !sample.isEmpty { let rows = BsonDocumentFlattener.flatten( documents: sample, columns: projection.columns, kinds: projection.kinds, diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 5e10afbea4..05bfefbd69 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -1031,10 +1031,9 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se } ) if !result.columns.isEmpty { - continuation.yield(.header(PluginStreamHeader( - columns: result.columns, - columnTypeNames: result.columnTypeNames - ))) + var header = PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) } if !result.rows.isEmpty { continuation.yield(.rows(result.rows, documents: result.exportDocuments)) diff --git a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift index edea147522..ba69a63dc3 100644 --- a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift +++ b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift @@ -88,7 +88,19 @@ struct MariaDBPluginQueryResult { // MARK: - Connection Class final class MariaDBPluginConnection: @unchecked Sendable { - private var mysql: UnsafeMutablePointer? + // Under stateLock, which disconnect() clears it under. Only `queue` may call into the handle: the close is queued there. + private var _mysql: UnsafeMutablePointer? + private var _serverThreadId: UInt = 0 + + private var mysql: UnsafeMutablePointer? { + stateLock.withLock { _mysql } + } + + // Read from the snapshot taken at connect: Stop runs off `queue`, where the handle may already be freed. + private var connectedThreadId: UInt? { + stateLock.withLock { _mysql == nil ? nil : _serverThreadId } + } + private let queue = DispatchQueue(label: "com.TablePro.mariadb.plugin", qos: .userInitiated) /// Serial, and separate from `queue`, which the read being cancelled is sitting on. Serial so a @@ -342,9 +354,9 @@ final class MariaDBPluginConnection: @unchecked Sendable { if let cachePath = kerberos?.cachePath { KerberosTicket.discard(cachePath) } - let handle = mysql + let handle = _mysql let cleanupQueue = queue - mysql = nil + _mysql = nil if let handle = handle { cleanupQueue.async { mysql_close(handle) @@ -390,8 +402,10 @@ final class MariaDBPluginConnection: @unchecked Sendable { self.baselineSelectLimit = nil self.hasCapturedBaselineSelectLimit = false + let threadId = mysql_thread_id(handle) self.stateLock.lock() - self.mysql = handle + self._mysql = handle + self._serverThreadId = threadId self._isConnected = true self._errorLanguage = session.errorLanguage self._sessionEndedByKill = false @@ -600,10 +614,10 @@ final class MariaDBPluginConnection: @unchecked Sendable { KerberosTicket.discard(cachePath) } - let handle = mysql - mysql = nil - stateLock.lock() + let handle = _mysql + _mysql = nil + _serverThreadId = 0 _isConnected = false stateLock.unlock() @@ -635,7 +649,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { let target = killTarget guard mayEndSession || !target.endsSession else { return } - guard let mysql = mysql, let statement = target.statement(threadId: mysql_thread_id(mysql)) else { return } + guard let threadId = connectedThreadId, let statement = target.statement(threadId: threadId) else { return } cancelQueue.async { [self] in guard deliverKill(statement) else { return } if target.endsSession { @@ -657,11 +671,9 @@ final class MariaDBPluginConnection: @unchecked Sendable { recordKillDelivered(generation: generation) } - /// The server thread this connection is on, read before a statement goes out so a kill still - /// has somewhere to go once the handle itself is unusable. + // Read before a statement goes out, so a kill still has somewhere to go once the handle itself is unusable. internal var currentThreadId: UInt { - guard let mysql = self.mysql else { return 0 } - return mysql_thread_id(mysql) + connectedThreadId ?? 0 } /// The kill has to reach the server the query is running on, which means repeating the transport diff --git a/apple/Plugins/MySQLDriverTests/MariaDBStopThreadIdTests.swift b/apple/Plugins/MySQLDriverTests/MariaDBStopThreadIdTests.swift new file mode 100644 index 0000000000..52bb160f9a --- /dev/null +++ b/apple/Plugins/MySQLDriverTests/MariaDBStopThreadIdTests.swift @@ -0,0 +1,38 @@ +// +// MariaDBStopThreadIdTests.swift +// MySQLDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +// Stop runs off the statement queue, where a disconnect's queued close can free the handle under it, so the kill's +// thread id is the one the handshake gave, held beside the handle and cleared with it. +@Suite(.timeLimit(.minutes(1))) +struct MariaDBStopThreadIdTests { + @Test("The kill's thread id is the session's own, and none is left once the session is closed") + func threadIdFollowsTheSession() async throws { + let server = try PlaintextMySQLServer(asksForCleartext: false) + defer { server.close() } + async let session = server.serveOneSession() + let connection = MariaDBPluginConnection( + transport: .tcp(host: "127.0.0.1", port: UInt32(server.port)), + user: "app", + password: "app-password", + database: "", + sslConfig: SSLConfiguration(mode: .disabled), + enableCleartextPlugin: false + ) + + try await connection.connect() + let signedIn = connection.currentThreadId + connection.disconnect() + let closed = connection.currentThreadId + _ = await session + + // The fake server's handshake names connection 1. + #expect(signedIn == 1) + #expect(closed == 0) + } +} diff --git a/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift b/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift index 90eeda01e8..2495d396c6 100644 --- a/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift +++ b/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift @@ -59,10 +59,12 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP progress: PluginExportProgress ) async throws -> ExportFormatResult { guard !tables.isEmpty else { return ExportFormatResult() } + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-export. + let options = settings var warnings: [String] = [] try await ParquetTableFiles.write(tables, destination: destination, progress: progress) { table, fileURL in - warnings += try await writeTable(table, dataSource: dataSource, to: fileURL, progress: progress) + warnings += try await writeTable(table, options: options, dataSource: dataSource, to: fileURL, progress: progress) } if tables.count > 1 { @@ -76,6 +78,7 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP private func writeTable( _ table: PluginExportTable, + options: ParquetExportOptions, dataSource: any PluginExportDataSource, to fileURL: URL, progress: PluginExportProgress @@ -123,8 +126,8 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP } let fallbacks = try staging.copyToParquet( fileURL: fileURL, - compression: settings.compression.duckDBValue, - rowGroupSize: settings.rowGroupSize + compression: options.compression.duckDBValue, + rowGroupSize: options.rowGroupSize ) return fallbacks.map { fallback in String( diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift index c2f93088de..6ac131744d 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift @@ -141,8 +141,9 @@ extension RedisPluginDriver { if opts.xx { args.append("XX".redisArgument) } if opts.keepTTL { args.append("KEEPTTL".redisArgument) } } - try await conn.run(args) - return buildStatusResult("OK", startTime: startTime) + // NX or XX that stops the write answers nil, not an error. + let reply = try await conn.run(args) + return buildStatusResult(reply.displayText, startTime: startTime) case .del(let keys): let args = ["DEL"] + keys diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift index 43d70ee2b9..4d4ffcb41b 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift @@ -561,11 +561,13 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { default: let startTime = Date() let result = try await executeOperation(operation, connection: conn, startTime: startTime) - continuation.yield(.header(PluginStreamHeader( + var header = PluginStreamHeader( columns: result.columns, columnTypeNames: result.columnTypeNames, estimatedRowCount: nil - ))) + ) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } diff --git a/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift b/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift index 72e4aa3bd3..4402cf7e01 100644 --- a/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift @@ -180,6 +180,25 @@ struct RedisConditionalWriteTests { #expect(result.rows.first?.first?.asText == "OK") } + @Test("A SET that NX or XX kept from writing shows nil, as the server answered") + func skippedSetShowsNil() async throws { + let channel = StubRedisChannel([.null]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("SET lock:job 1 NX"), connection: channel, startTime: Date() + ) + #expect(result.rows.first?.first?.asText == "(nil)") + #expect(channel.sentCommands == [["SET", "lock:job", "1", "NX"]]) + } + + @Test("A SET that wrote the key shows OK") + func appliedSetShowsOK() async throws { + let channel = StubRedisChannel([.status("OK")]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("SET lock:job 1 XX EX 60"), connection: channel, startTime: Date() + ) + #expect(result.rows.first?.first?.asText == "OK") + } + @Test("A conditional write the server refused outright keeps the server's message") func execKeepsServerErrorForConditionalWrite() { let reply = RedisReply.array([.error("ERR no such key")]) diff --git a/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift b/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift index 91b6a9c205..137d0a7ac3 100644 --- a/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift +++ b/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift @@ -51,15 +51,17 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi var errors: [PluginImportResult.ImportStatementError] = [] let maxErrors = 1_000 - let errorMode = settings.errorHandling - let useTransaction = settings.wrapInTransaction && errorMode != .skipAndContinue + // One plugin instance serves every window: a checkbox unticked elsewhere mid-import left the checks off. + let options = settings + let errorMode = options.errorHandling + let useTransaction = options.wrapInTransaction && errorMode != .skipAndContinue let fileSizeBytes = source.fileSizeBytes() let estimatedTotal = max(1, Int(fileSizeBytes / 500)) progress.setEstimatedTotal(estimatedTotal) do { - if settings.disableForeignKeyChecks { + if options.disableForeignKeyChecks { try await sink.disableForeignKeyChecks() } @@ -99,7 +101,7 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi Self.logger.warning("Failed to commit partial import") } } - if settings.disableForeignKeyChecks { + if options.disableForeignKeyChecks { do { try await sink.enableForeignKeyChecks() } catch { @@ -134,7 +136,7 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi didCommit = true } - if settings.disableForeignKeyChecks { + if options.disableForeignKeyChecks { do { try await sink.enableForeignKeyChecks() } catch { @@ -156,7 +158,7 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi } } - if settings.disableForeignKeyChecks { + if options.disableForeignKeyChecks { do { try await sink.enableForeignKeyChecks() } catch { diff --git a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index dc8a667c2c..53d2af6929 100644 --- a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -1567,7 +1567,8 @@ public extension PluginDatabaseDriver { } func escapedParameterValue(_ value: String) -> String { - if Self.isNumericLiteral(value) { + // No numeric column reads as `00123` or `-007`, so that text is a code or a zip and stays a string. + if Self.isNumericLiteral(value), !opensWithPaddingZero(value) { return value } return PluginSQLQuoting.stringLiteral(value, escapes: PluginSQLQuoting.Escapes(features: literalFeatures)) @@ -1611,3 +1612,10 @@ extension PluginDatabaseDriver { return backslash ? [.backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes] : [] } } + +// `007`, `-0042`: an integer part that opens with 0 and goes on with another digit. +private func opensWithPaddingZero(_ value: String) -> Bool { + let unsigned = value.unicodeScalars.drop { $0 == "-" || $0 == "+" } + guard unsigned.first == "0", let next = unsigned.dropFirst().first else { return false } + return ("0"..."9").contains(next) +} diff --git a/apple/Plugins/TableProPluginKit/PluginExportUtilities.swift b/apple/Plugins/TableProPluginKit/PluginExportUtilities.swift index 726b45de69..a89463cd38 100644 --- a/apple/Plugins/TableProPluginKit/PluginExportUtilities.swift +++ b/apple/Plugins/TableProPluginKit/PluginExportUtilities.swift @@ -109,6 +109,72 @@ public enum PluginExportUtilities { } } +// Shared so XLSX number cells and the CSV decimal comma read a column's declared type the same way. +public enum PluginColumnNumbers: Equatable, Sendable { + case text + case exact + case approximate + case untyped + + public init(typeName: String, bitsAreStrings: Bool = false) { + let base = Self.baseName(typeName) + if base.isEmpty { + self = .untyped + } else if Self.bitTypes.contains(base) { + self = bitsAreStrings ? .text : .exact + } else if Self.approximateTypes.contains(base) { + self = .approximate + } else if Self.exactTypes.contains(base) { + self = .exact + } else if Self.textTypes.contains(base) || Self.textMarkers.contains(where: { base.contains($0) }) { + self = .text + } else { + self = .untyped + } + } + + public var isNumeric: Bool { + self == .exact || self == .approximate + } + + // `Nullable(UInt64)` and `LowCardinality(Nullable(Int32))` name the type inside. The base is the + // first word without its width: `int4`, `Int64` and `INT UNSIGNED` are all `int`. + private static func baseName(_ typeName: String) -> String { + var name = typeName.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + while name.hasSuffix(")"), let open = name.firstIndex(of: "("), + ["nullable", "lowcardinality"].contains(String(name[.. = [ + "int", "uint", "integer", "bigint", "smallint", "tinyint", "mediumint", "hugeint", + "uinteger", "ubigint", "usmallint", "utinyint", "uhugeint", "decimal", "dec", "numeric", + "number", "bignumeric", "serial", "bigserial", "smallserial", "money", "smallmoney", "year", "oid" + ] + + private static let approximateTypes: Set = [ + "float", "double", "real", "binary_float", "binary_double" + ] + + private static let textTypes: Set = ["enum", "set", "json", "jsonb", "name"] + + private static let bitTypes: Set = ["bit", "varbit"] + + // PostgreSQL's `bit(n)` is a string of 0 and 1. SQL Server's is a flag and MySQL's reads as a number. + public static func bitsAreStrings(databaseTypeId: String) -> Bool { + enginesWithBitStrings.contains(databaseTypeId) + } + + private static let enginesWithBitStrings: Set = [ + "PostgreSQL", "Redshift", "CockroachDB", "PGlite", "AlloyDB", "Citus", "Greenplum", "KingbaseES" + ] + + private static let textMarkers = ["char", "text", "string", "clob"] +} + public extension String { func toUTF8Data() throws -> Data { guard let data = self.data(using: .utf8) else { diff --git a/apple/Plugins/TableProPluginKit/PluginRowWriters.swift b/apple/Plugins/TableProPluginKit/PluginRowWriters.swift index e92617afb2..d3c0be5caa 100644 --- a/apple/Plugins/TableProPluginKit/PluginRowWriters.swift +++ b/apple/Plugins/TableProPluginKit/PluginRowWriters.swift @@ -18,9 +18,7 @@ public struct PluginCsvWriteOptions: Sendable, Equatable { public let lineEnding: String public let nullAsEmpty: Bool - /// Prefixes a value opening with `=`, `+`, `-` or `@` with a quote. A spreadsheet treats a cell - /// starting with one of those as a formula, and the values in an export come from the database - /// rather than from the person opening the file. + // The quote stays in the value for any reader but a spreadsheet, so it suits files meant for one. public let sanitizesFormulas: Bool /// Replaces a line break inside a value with a space. Off means the value keeps its break and is @@ -73,9 +71,7 @@ public enum PluginRowWriters { .replacingOccurrences(of: "\r", with: " ", options: .literal) } var wasNeutralised = false - // Scalars, not Characters: `\r\n` is one Character, so a Character test misses a Windows line break. - if options.sanitizesFormulas, let first = processed.unicodeScalars.first, - isFormulaLead(Character(first)), !isPlainNumber(processed) { + if options.sanitizesFormulas, isFormulaLike(processed) { processed = "'" + processed wasNeutralised = true } @@ -95,12 +91,19 @@ public enum PluginRowWriters { private static let quotedScalars: Set = ["\"", "\n", "\r", "\t"] - /// A leading tab or carriage return is treated as a formula lead too: Excel strips it before - /// parsing the cell, so `\t=1+1` reaches the formula engine exactly as `=1+1` would. - private static func isFormulaLead(_ character: Character) -> Bool { - formulaPrefixes.contains(character) || character == "\t" || character == "\r" + // Public so a format plugin can count the values sanitizing changes. + public static func isFormulaLike(_ value: String) -> Bool { + // Scalars, not Characters: `\r\n` is one Character, so a Character test misses a Windows line break. + guard let first = value.unicodeScalars.first, formulaLeads.contains(first) else { return false } + return !isPlainNumber(value) } + // OWASP's list: the four operators, the whitespace Excel strips before parsing (so `\t=1+1` runs as `=1+1`), + // and the full-width operators a Japanese locale reads as formulas. The same set as the shared core's `FORMULA_LEADS`. + private static let formulaLeads: Set = [ + "=", "+", "-", "@", "\t", "\r", "\n", "\u{FF1D}", "\u{FF0B}", "\u{FF0D}", "\u{FF20}" + ] + // A number is never a formula, and `'-5` is text to every reader, a spreadsheet included. // The same rule as the Linux app's `is_plain_number`. private static func isPlainNumber(_ value: String) -> Bool { diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift index ef3166e4c1..ea42a44623 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift @@ -4,6 +4,8 @@ import TableProTeradataCore final class TeradataAsyncConnection: @unchecked Sendable { private let connection: TeradataConnection private let queue = DispatchQueue(label: "com.TablePro.teradata.connection") + private let runningLock = NSLock() + private var running: TeradataAsyncCancellation? init(config: TeradataConnectionConfig) { connection = TeradataConnection(config: config) @@ -41,6 +43,13 @@ final class TeradataAsyncConnection: @unchecked Sendable { return try await withTaskCancellationHandler { try await withCheckedThrowingContinuation { continuation in queue.async { + // Begun before the call is marked running, so a Stop either side of the mark reaches it. + self.connection.beginOperation() + self.runningLock.withLock { self.running = cancellation } + defer { + self.runningLock.withLock { self.running = nil } + self.connection.endOperation() + } do { try cancellation.throwIfCancelled() let value = try body(self.connection) @@ -53,9 +62,12 @@ final class TeradataAsyncConnection: @unchecked Sendable { } } onCancel: { cancellation.cancel() - // TeradataConnection snapshots its transport under a lock; the transport's cancel - // path uses shutdown rather than disconnecting concurrently with protocol cleanup. - connection.cancel() + // A call still queued never starts; stopping the connection would stop the request of the one running. + // Held across the stop, so the call cannot end and hand the session to the next one meanwhile. + runningLock.withLock { + guard running === cancellation else { return } + connection.cancel() + } } } } diff --git a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift index 63a467ef35..f773973f4a 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift @@ -43,6 +43,8 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug destination: URL, progress: PluginExportProgress ) async throws -> ExportFormatResult { + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-export. + let options = settings let writer = XLSXWriter(databaseTypeId: dataSource.databaseTypeId) var didSplitSheets = false @@ -56,7 +58,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug var currentSheetRowCount = 0 var columns: [String] = [] var columnTypeNames: [String] = [] - let headerRowCount = settings.includeHeaderRow ? 1 : 0 + let headerRowCount = options.includeHeaderRow ? 1 : 0 let stream = dataSource.streamRows(for: table) for try await element in stream { @@ -70,8 +72,8 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug name: table.name, columns: columns, columnTypeNames: columnTypeNames, - includeHeader: settings.includeHeaderRow, - convertNullToEmpty: settings.convertNullToEmpty + includeHeader: options.includeHeaderRow, + convertNullToEmpty: options.convertNullToEmpty ) currentSheetRowCount = headerRowCount isFirstBatch = false @@ -83,7 +85,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug let remaining = Self.maxRowsPerSheet - currentSheetRowCount if rowBatch.count <= remaining { try autoreleasepool { - try writer.addRows(rowBatch, convertNullToEmpty: settings.convertNullToEmpty) + try writer.addRows(rowBatch, convertNullToEmpty: options.convertNullToEmpty) } currentSheetRowCount += rowBatch.count } else { @@ -91,7 +93,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug let overflow = Array(rowBatch.dropFirst(remaining)) if !fitting.isEmpty { try autoreleasepool { - try writer.addRows(fitting, convertNullToEmpty: settings.convertNullToEmpty) + try writer.addRows(fitting, convertNullToEmpty: options.convertNullToEmpty) } currentSheetRowCount += fitting.count } @@ -99,14 +101,14 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug baseName: table.name, columns: columns, columnTypeNames: columnTypeNames, - includeHeader: settings.includeHeaderRow, - convertNullToEmpty: settings.convertNullToEmpty + includeHeader: options.includeHeaderRow, + convertNullToEmpty: options.convertNullToEmpty ) didSplitSheets = true currentSheetRowCount = headerRowCount if !overflow.isEmpty { try autoreleasepool { - try writer.addRows(overflow, convertNullToEmpty: settings.convertNullToEmpty) + try writer.addRows(overflow, convertNullToEmpty: options.convertNullToEmpty) } currentSheetRowCount += overflow.count } @@ -124,7 +126,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug let remaining = Self.maxRowsPerSheet - currentSheetRowCount if rowBatch.count <= remaining { try autoreleasepool { - try writer.addRows(rowBatch, convertNullToEmpty: settings.convertNullToEmpty) + try writer.addRows(rowBatch, convertNullToEmpty: options.convertNullToEmpty) } currentSheetRowCount += rowBatch.count } else { @@ -132,21 +134,21 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug let overflow = Array(rowBatch.dropFirst(remaining)) if !fitting.isEmpty { try autoreleasepool { - try writer.addRows(fitting, convertNullToEmpty: settings.convertNullToEmpty) + try writer.addRows(fitting, convertNullToEmpty: options.convertNullToEmpty) } } try writer.continueSheet( baseName: table.name, columns: columns, columnTypeNames: columnTypeNames, - includeHeader: settings.includeHeaderRow, - convertNullToEmpty: settings.convertNullToEmpty + includeHeader: options.includeHeaderRow, + convertNullToEmpty: options.convertNullToEmpty ) didSplitSheets = true currentSheetRowCount = headerRowCount if !overflow.isEmpty { try autoreleasepool { - try writer.addRows(overflow, convertNullToEmpty: settings.convertNullToEmpty) + try writer.addRows(overflow, convertNullToEmpty: options.convertNullToEmpty) } currentSheetRowCount += overflow.count } @@ -163,7 +165,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug name: table.name, columns: [], includeHeader: false, - convertNullToEmpty: settings.convertNullToEmpty + convertNullToEmpty: options.convertNullToEmpty ) try writer.finishSheet() } diff --git a/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift b/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift index c404199eac..cdb19eb612 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift @@ -46,66 +46,7 @@ final class XLSXWriter { case empty } - enum ColumnNumbers: Equatable { - case text - case exact - case approximate - case untyped - - init(typeName: String, bitsAreStrings: Bool = false) { - let base = Self.baseName(typeName) - if base.isEmpty { - self = .untyped - } else if Self.bitTypes.contains(base) { - self = bitsAreStrings ? .text : .exact - } else if Self.approximateTypes.contains(base) { - self = .approximate - } else if Self.exactTypes.contains(base) { - self = .exact - } else if Self.textTypes.contains(base) || Self.textMarkers.contains(where: { base.contains($0) }) { - self = .text - } else { - self = .untyped - } - } - - // `Nullable(UInt64)` and `LowCardinality(Nullable(Int32))` name the type inside. The base is the - // first word without its width: `int4`, `Int64` and `INT UNSIGNED` are all `int`. - private static func baseName(_ typeName: String) -> String { - var name = typeName.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() - while name.hasSuffix(")"), let open = name.firstIndex(of: "("), - ["nullable", "lowcardinality"].contains(String(name[.. = [ - "int", "uint", "integer", "bigint", "smallint", "tinyint", "mediumint", "hugeint", - "uinteger", "ubigint", "usmallint", "utinyint", "uhugeint", "decimal", "dec", "numeric", - "number", "bignumeric", "serial", "bigserial", "smallserial", "money", "smallmoney", "year", "oid" - ] - - private static let approximateTypes: Set = [ - "float", "double", "real", "binary_float", "binary_double" - ] - - private static let textTypes: Set = ["enum", "set", "json", "jsonb", "name"] - - private static let bitTypes: Set = ["bit", "varbit"] - - // PostgreSQL's `bit(n)` is a string of 0 and 1. SQL Server's is a flag and MySQL's reads as a number. - static func bitsAreStrings(databaseTypeId: String) -> Bool { - enginesWithBitStrings.contains(databaseTypeId) - } - - private static let enginesWithBitStrings: Set = [ - "PostgreSQL", "Redshift", "CockroachDB", "PGlite", "AlloyDB", "Citus", "Greenplum", "KingbaseES" - ] - - private static let textMarkers = ["char", "text", "string", "clob"] - } + typealias ColumnNumbers = PluginColumnNumbers // Excel keeps 15 significant digits of a number. private static let maximumExactDigits = 15 @@ -225,6 +166,8 @@ final class XLSXWriter { let isPlain = literal == text && !literal.contains("e") let digitCount = literal.filter { $0.isNumber }.count return isPlain && digitCount <= maximumExactDigits ? .number(literal) : .string(text) + @unknown default: + return .string(text) } } diff --git a/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift b/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift index 6f7592ab3a..d74bafbfba 100644 --- a/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift +++ b/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift @@ -39,13 +39,13 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug var fieldDetectionSignature: String { settings.detectionSignature } - var rowImportConfiguration: RowImportRunner.Configuration? { runConfiguration } + var rowImportConfiguration: RowImportRunner.Configuration? { Self.runConfiguration(settings) } - private var runConfiguration: RowImportRunner.Configuration { + private static func runConfiguration(_ options: XLSXImportOptions) -> RowImportRunner.Configuration { RowImportRunner.Configuration( - errorHandling: settings.errorHandling, - wrapInTransaction: settings.wrapInTransaction, - deleteExistingRows: settings.deleteExistingRows + errorHandling: options.errorHandling, + wrapInTransaction: options.wrapInTransaction, + deleteExistingRows: options.deleteExistingRows ) } @@ -78,11 +78,13 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug progress: PluginImportProgress ) async throws -> PluginImportResult { let started = Date() + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-import. + let options = settings let sheet: XLSXImportSheet do { sheet = try XLSXImportSheet.read( at: source.fileURL(), - hasHeaderRow: settings.hasHeaderRow, + hasHeaderRow: options.hasHeaderRow, isCancelled: { progress.isCancelled } ) } catch is TabularCancellation { @@ -96,7 +98,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug progress.setEstimatedTotal(max(1, sheet.rowCount)) var cursor = 0 let outcome = try await RowImportRunner.run( - configuration: runConfiguration, + configuration: Self.runConfiguration(options), sink: sink, progress: progress ) { @@ -108,7 +110,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug if filled.count < rows.count { progress.incrementStatement(by: rows.count - filled.count) } - return filled.map { (line: $0.line, row: self.row($0.values, header: header)) } + return filled.map { (line: $0.line, row: Self.row($0.values, header: header, options: options)) } } return PluginImportResult( @@ -121,7 +123,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug // MARK: - Private - private func row(_ values: [String?], header: [String]) -> [String: PluginCellValue] { + private static func row(_ values: [String?], header: [String], options: XLSXImportOptions) -> [String: PluginCellValue] { var mapped: [String: PluginCellValue] = [:] for (index, column) in header.enumerated() { let raw = values[safe: index] ?? nil @@ -129,10 +131,10 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug mapped[column] = .null continue } - let trimmed = settings.trimWhitespace + let trimmed = options.trimWhitespace ? raw.trimmingCharacters(in: .whitespacesAndNewlines) : raw - if settings.emptyAsNull, trimmed.isEmpty { + if options.emptyAsNull, trimmed.isEmpty { mapped[column] = .null } else { mapped[column] = .text(trimmed) diff --git a/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift b/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift index 7a795ff87d..f86e46a25c 100644 --- a/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift +++ b/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift @@ -49,8 +49,8 @@ struct CellChange: Identifiable, Equatable { let oldIsAbsent: Bool let newIsAbsent: Bool let oldIsExpression: Bool - /// The new value is `.text` holding SQL from `CellExpressionVocabulary`, written unquoted. Never set with - /// `newIsAbsent`. + /// The new value is `.text` holding SQL from `CellExpressionVocabulary`, or `PluginCellValue.columnDefault`, + /// written unquoted. Never set with `newIsAbsent`. let newIsExpression: Bool init( @@ -92,8 +92,8 @@ struct RowChange: Identifiable, Equatable { /// stands for an insert. Empty on every engine that cannot tell a missing field from NULL. var absentColumns: Set - /// An insert's cells that hold a SQL expression, as the row stands. An update keeps its own on each - /// `CellChange`. + /// An insert's cells that hold a SQL expression or the column default, as the row stands. An update keeps its own + /// on each `CellChange`. var expressionColumns: Set /// The order the user made this change in. diff --git a/apple/TablePro/Core/ChangeTracking/DefaultedCells.swift b/apple/TablePro/Core/ChangeTracking/DefaultedCells.swift new file mode 100644 index 0000000000..bdbb48fc6b --- /dev/null +++ b/apple/TablePro/Core/ChangeTracking/DefaultedCells.swift @@ -0,0 +1,38 @@ +// +// DefaultedCells.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +// The cells a pending insert or edit set to the column's default, by row and data index. Only those rows can hold one. +enum DefaultedCells { + static func marked( + in rows: TableRows, + rowIDs: Set, + expressionColumns: (RowID) -> Set + ) -> [RowID: Set] { + var cells: [RowID: Set] = [:] + for rowID in rowIDs { + guard let row = rows.row(withID: rowID) else { continue } + let defaulted = expressionColumns(rowID).filter { + row.values.indices.contains($0) && row.values[$0].isColumnDefault(isExpression: true) + } + if !defaulted.isEmpty { cells[rowID] = defaulted } + } + return cells + } +} + +extension ChangeManaging { + func defaultedCells(in rows: TableRows) -> [RowID: Set] { + DefaultedCells.marked(in: rows, rowIDs: insertedRowIDs.union(modifiedCells.keys)) { expressionColumns(forRow: $0) } + } +} + +extension AnyChangeManager { + func defaultedCells(in rows: TableRows) -> [RowID: Set] { + DefaultedCells.marked(in: rows, rowIDs: insertedRowIDs.union(modifiedCells.keys)) { expressionColumns(forRow: $0) } + } +} diff --git a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift index 17f1871fa9..21df0ea117 100644 --- a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift +++ b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift @@ -257,7 +257,8 @@ struct SQLStatementGenerator { var bindParameters: [Any?] = [] for (index, value) in values.enumerated() { - if case .text(let s) = value, s == "__DEFAULT__" { continue } + // Left out rather than written as DEFAULT, which SQLite does not take in VALUES. + if value.isColumnDefault(isExpression: expressionColumns.contains(index)) { continue } guard index < columns.count else { continue } let columnName = columns[index] @@ -383,7 +384,7 @@ struct SQLStatementGenerator { guard !change.cellChanges.isEmpty else { return nil } let nonDefaultChanges = change.cellChanges.filter { - $0.newValue != .text("__DEFAULT__") && !generatedColumns.contains($0.columnName) + !$0.newValue.isColumnDefault(isExpression: $0.newIsExpression) && !generatedColumns.contains($0.columnName) } guard !nonDefaultChanges.isEmpty else { return nil } @@ -419,8 +420,6 @@ struct SQLStatementGenerator { var parameters: [Any?] = [] let setClauses = writableChanges.map { cellChange -> String in switch cellChange.newValue { - case .text(let s) where s == "__DEFAULT__": - return "\(quoteIdentifierFn(cellChange.columnName)) = DEFAULT" case .text(let expression) where cellChange.newIsExpression: return "\(quoteIdentifierFn(cellChange.columnName)) = \(expression)" default: diff --git a/apple/TablePro/Core/Compare/CompareSQLLiteral.swift b/apple/TablePro/Core/Compare/CompareSQLLiteral.swift index 24cc1a74a8..40fc624f42 100644 --- a/apple/TablePro/Core/Compare/CompareSQLLiteral.swift +++ b/apple/TablePro/Core/Compare/CompareSQLLiteral.swift @@ -77,17 +77,20 @@ internal enum CompareSQLLiteral { return SQLStringLiteralPrefix.forDatabaseType(databaseType) + literal } + // Shared by Compare & Sync and Copy as SQL. `SQLExportBinaryLiteral` in the SQL export plugin holds the + // measurements behind each spelling; the plugin cannot link app code, so it keeps its own copy. internal static func binaryLiteral(for data: Data, databaseType: DatabaseType) -> String? { - let hex = data.map { String(format: "%02X", $0) }.joined() + let hex = data.uppercaseHex() switch binaryStyle(for: databaseType) { case .bitString: return "X'\(hex)'" case .postgresBytea: - return "'\\x\(hex.lowercased())'::bytea" + return postgresFamilyBinaryLiteral(hex: hex, databaseType: databaseType) case .zeroX: return "0x\(hex)" case .hexToRaw: - return "HEXTORAW('\(hex)')" + // `HEXTORAW('')` stores NULL, which a `BLOB NOT NULL` refuses. + return data.isEmpty ? "EMPTY_BLOB()" : "HEXTORAW('\(hex)')" case .unhexFunction: return "unhex('\(hex)')" case .unknown: @@ -95,6 +98,19 @@ internal enum CompareSQLLiteral { } } + // `'\x…'::bytea` follows `standard_conforming_strings`: with it off the server stores the text's bytes. KingbaseES + // documents `decode` only as Oracle's DECODE, so it keeps the bytea literal; Redshift has no bytea and reads VARBYTE. + private static func postgresFamilyBinaryLiteral(hex: String, databaseType: DatabaseType) -> String { + switch databaseType { + case .kingbasees: + return "'\\x\(hex)'::bytea" + case .redshift: + return "from_hex('\(hex)')" + default: + return "decode('\(hex)', 'hex')" + } + } + internal enum BinaryStyle { case bitString case postgresBytea diff --git a/apple/TablePro/Core/Compare/DataDiffEngine.swift b/apple/TablePro/Core/Compare/DataDiffEngine.swift index 882c746b62..df96ff5b92 100644 --- a/apple/TablePro/Core/Compare/DataDiffEngine.swift +++ b/apple/TablePro/Core/Compare/DataDiffEngine.swift @@ -30,10 +30,16 @@ internal struct DataRow: Hashable, Sendable { internal protocol DataRowProviding: AnyObject { var endedAtRowLimit: Bool { get } + // The limit the driver stopped this side's stream at, once it has ended there. + var driverRowCut: Int? { get } func nextRow() async throws -> DataRow? func drain() async throws } +internal extension DataRowProviding { + var driverRowCut: Int? { nil } +} + internal protocol OneSidedRowResolving: AnyObject { func rows(matching keys: [[PluginCellValue]], on side: ComparisonSide) async throws -> [DataRow] } @@ -111,6 +117,10 @@ internal struct DataDiffSummary: Hashable, Sendable { /// as `id > ` resumes from without skipping a key or comparing one twice. internal let resumeKey: [PluginCellValue]? + // The limit a driver stopped a side's stream at, so the answer covers only the rows read before it. + internal let sourceRowCut: Int? + internal let targetRowCut: Int? + /// One comparison's own identity. Two runs that produce the same counts and the same digest are /// still two answers, and a view that caches on the answer has to be able to tell them apart: /// matching rows are counted rather than digested, so both can move while the digest stands. @@ -130,6 +140,8 @@ internal struct DataDiffSummary: Hashable, Sendable { stoppedAtRowLimit: Bool = false, differenceDigest: String = "", resumeKey: [PluginCellValue]? = nil, + sourceRowCut: Int? = nil, + targetRowCut: Int? = nil, runIdentity: UUID = UUID() ) { self.insertCount = insertCount @@ -145,6 +157,8 @@ internal struct DataDiffSummary: Hashable, Sendable { self.stoppedAtRowLimit = stoppedAtRowLimit self.differenceDigest = differenceDigest self.resumeKey = resumeKey + self.sourceRowCut = sourceRowCut + self.targetRowCut = targetRowCut self.runIdentity = runIdentity } @@ -359,7 +373,9 @@ internal struct DataDiffEngine { return accumulator.summary( comparedKeyCount: positions, stoppedAtRowLimit: stoppedAtRowLimit, - resumeKey: stoppedAtRowLimit ? lastClassifiedKey : nil + resumeKey: stoppedAtRowLimit ? lastClassifiedKey : nil, + sourceRowCut: source.driverRowCut, + targetRowCut: target.driverRowCut ) } @@ -536,7 +552,9 @@ internal extension DataDiffEngine { func summary( comparedKeyCount: Int, stoppedAtRowLimit: Bool, - resumeKey: [PluginCellValue]? = nil + resumeKey: [PluginCellValue]? = nil, + sourceRowCut: Int? = nil, + targetRowCut: Int? = nil ) -> DataDiffSummary { DataDiffSummary( insertCount: counts[.insert] ?? 0, @@ -551,7 +569,9 @@ internal extension DataDiffEngine { comparedKeyCount: comparedKeyCount, stoppedAtRowLimit: stoppedAtRowLimit, differenceDigest: digest.finalize().map { String(format: "%02x", $0) }.joined(), - resumeKey: resumeKey + resumeKey: resumeKey, + sourceRowCut: sourceRowCut, + targetRowCut: targetRowCut ) } diff --git a/apple/TablePro/Core/Compare/StreamingRowProvider.swift b/apple/TablePro/Core/Compare/StreamingRowProvider.swift index 979cabe6fb..54a4ac107f 100644 --- a/apple/TablePro/Core/Compare/StreamingRowProvider.swift +++ b/apple/TablePro/Core/Compare/StreamingRowProvider.swift @@ -13,6 +13,7 @@ // import Foundation +import os import TableProPluginKit internal struct RowReadContext: Sendable { @@ -55,6 +56,7 @@ internal final class StreamingRowProvider: DataRowProviding { private var deliveredCount = 0 private var isFinished = false private var sawRowPastTheLimit = false + private let rowCut: OSAllocatedUnfairLock internal init( stream: AsyncThrowingStream, @@ -62,17 +64,25 @@ internal final class StreamingRowProvider: DataRowProviding { rowLimit: Int? = nil, context: RowReadContext? = nil ) { - self.iterator = stream.makeAsyncIterator() + let rowCut = OSAllocatedUnfairLock(initialState: nil) + self.rowCut = rowCut + self.iterator = ExportRowCut.reporting(stream, engineCeiling: nil) { limit in + rowCut.withLock { $0 = limit } + }.makeAsyncIterator() self.columns = columns self.rowLimit = rowLimit self.context = context } - /// The read asks for one row more than the limit and never hands that row out, so this is the - /// server's own answer to whether anything was left rather than a guess from the row count: a - /// table holding exactly the limit is not a truncated read. + // The read asks for one row past the limit, so a table holding exactly the limit is not a cut read. A stream the + // driver stopped at a limit it declares ends here too: a key only the other side holds was never read on this one. internal var endedAtRowLimit: Bool { - sawRowPastTheLimit + sawRowPastTheLimit || driverRowCut != nil + } + + internal var driverRowCut: Int? { + guard isFinished else { return nil } + return rowCut.withLock { $0 } } internal func nextRow() async throws -> DataRow? { diff --git a/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift b/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift index ec3657620b..9cf2f91330 100644 --- a/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift @@ -218,7 +218,7 @@ final class RowEditingCoordinator: ObservableObject { private static func sampleText(atColumn index: Int, in tableRows: TableRows) -> String? { for row in tableRows.rows where row.values.indices.contains(index) { - if case .text(let text) = row.values[index], text != "__DEFAULT__" { return text } + if case .text(let text) = row.values[index], row.values[index] != .columnDefault { return text } } return nil } @@ -430,10 +430,12 @@ final class RowEditingCoordinator: ObservableObject { self?.copyRowsAsJson(rowIDs) }) else { return } let model = parent.gridModel(forTab: tab.id) + let tableRows = parent.tabSessionRegistry.tableRows(for: tab.id) let output = ResultJsonSerializer.serialize( - tableRows: parent.tabSessionRegistry.tableRows(for: tab.id), + tableRows: tableRows, displayIDs: model.displayedRowIDs, selectedDisplayIndices: Set(rowIDs.compactMap { model.displayIndex(of: $0) }), + defaultedCells: parent.changeManager.defaultedCells(in: tableRows), columns: VisibleColumnProjection(indices: presentedColumnIndices(tabId: tab.id)) ) guard output.rowCount > 0 else { return } diff --git a/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift b/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift index a4c255ea3b..3686d2ca50 100644 --- a/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift +++ b/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift @@ -134,8 +134,9 @@ struct RewindExecutor { ) } + // In the order the restore ran them, since undoing the restore runs its record backwards. static func inverseOperations(of plan: RewindPlan) -> [RowWriteOperation] { - plan.rows.filter { $0.outcome.restores }.map { row -> RowWriteOperation in + plan.rows.filter { $0.outcome.restores }.reversed().map { row -> RowWriteOperation in let source = row.operation let kind = source.kind.inverted // The same stored refusal a save through a view writes, so a build without the view flag refuses too. diff --git a/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift b/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift index 0438224191..361fb6bc40 100644 --- a/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift +++ b/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift @@ -82,6 +82,7 @@ struct RewindPlanner { currentRow = byLocator[Array(locator.utf8)] } else { currentRow = primaryKey(of: operation).flatMap { current[$0] } + ?? rowRestoredUnderKeyBefore(operation, in: current) } let outcome = classify(operation, current: currentRow) rows.append( @@ -135,7 +136,7 @@ struct RewindPlanner { } } - /// Whether the row holds what the image says on every column the save wrote: the same value, + /// Whether the row holds what the image says on every compared column: the same value, /// and the field there or missing as it was. Set NULL on a missing field and Remove Field on a /// NULL one leave the value alone, so comparing values alone reads either as already restored. /// @@ -147,7 +148,7 @@ struct RewindPlanner { absent expectedAbsent: Set?, on operation: RowWriteOperation ) -> Bool { - operation.writtenColumns.allSatisfy { column in + comparedColumns(of: operation).allSatisfy { column in guard let index = operation.columns.firstIndex(of: column), index < current.values.count, index < expected.count, current.values[index] == expected[index] @@ -157,6 +158,14 @@ struct RewindPlanner { } } + // A row found by a key the save wrote holds that key whatever row it is, so a moved row is compared on every + // column the server does not compute; a trigger's stamp then reads as changed, which leaves the row alone. + private func comparedColumns(of operation: RowWriteOperation) -> [String] { + guard keyBeforeMove(of: operation) != nil else { return operation.writtenColumns } + let generated = Set(record.generatedColumns) + return operation.columns.filter { !generated.contains($0) } + } + // MARK: - Reading private func indexByKey(_ rows: [RewindCurrentRow]) -> [RewindRowKey: RewindCurrentRow] { @@ -194,9 +203,11 @@ struct RewindPlanner { guard !keyColumns.isEmpty else { return [] } if keyColumns.count == 1, let keyColumn = keyColumns.first { - let filters = operations.compactMap { operation -> TableFilter? in - guard let value = keyValue(of: operation, column: keyColumn) else { return nil } - return TableFilter(columnName: keyColumn, filterOperator: .equal, value: value) + let filters = operations.flatMap { operation in + readImages(of: operation).compactMap { image -> TableFilter? in + guard let value = keyValue(in: image, of: operation, column: keyColumn) else { return nil } + return TableFilter(columnName: keyColumn, filterOperator: .equal, value: value) + } } guard !filters.isEmpty, let query = queryBuilder.buildFilteredQuery( @@ -211,21 +222,23 @@ struct RewindPlanner { return [query] } - return operations.compactMap { operation in - let filters = keyColumns.compactMap { column -> TableFilter? in - guard let value = keyValue(of: operation, column: column) else { return nil } - return TableFilter(columnName: column, filterOperator: .equal, value: value) + return operations.flatMap { operation in + readImages(of: operation).compactMap { image -> String? in + let filters = keyColumns.compactMap { column -> TableFilter? in + guard let value = keyValue(in: image, of: operation, column: column) else { return nil } + return TableFilter(columnName: column, filterOperator: .equal, value: value) + } + guard filters.count == keyColumns.count else { return nil } + return queryBuilder.buildFilteredQuery( + tableName: record.target.table, + schemaName: record.target.schema, + filters: filters, + logicMode: .and, + columns: columns, + selectColumns: columns, + limit: 1 + ) } - guard filters.count == keyColumns.count else { return nil } - return queryBuilder.buildFilteredQuery( - tableName: record.target.table, - schemaName: record.target.schema, - filters: filters, - logicMode: .and, - columns: columns, - selectColumns: columns, - limit: 1 - ) } } @@ -293,12 +306,40 @@ struct RewindPlanner { // MARK: - Keys + // Where the save left the row, or for a delete where it was: an edit that rewrote the key moved it. + private func savedImage(of operation: RowWriteOperation) -> [PluginCellValue]? { + operation.postImage ?? operation.preImage + } + private func primaryKey(of operation: RowWriteOperation) -> RewindRowKey? { - let source = operation.preImage ?? operation.postImage - guard let source else { return nil } + guard let source = savedImage(of: operation) else { return nil } return primaryKey(of: source, columns: operation.columns, keyColumns: operation.primaryKeyColumns) } + private func keyBeforeMove(of operation: RowWriteOperation) -> RewindRowKey? { + guard operation.kind == .update, let preImage = operation.preImage else { return nil } + let before = primaryKey(of: preImage, columns: operation.columns, keyColumns: operation.primaryKeyColumns) + return before == primaryKey(of: operation) ? nil : before + } + + // A rewind that already ran put a moved row back under its old key, where it reads as restored. + private func readImages(of operation: RowWriteOperation) -> [[PluginCellValue]] { + guard let saved = savedImage(of: operation) else { return [] } + guard keyBeforeMove(of: operation) != nil, let preImage = operation.preImage else { return [saved] } + return [saved, preImage] + } + + // Only a row holding the old values, so another row that took the old key since is not mistaken for this one. + private func rowRestoredUnderKeyBefore( + _ operation: RowWriteOperation, + in current: [RewindRowKey: RewindCurrentRow] + ) -> RewindCurrentRow? { + guard let key = keyBeforeMove(of: operation), let row = current[key], let preImage = operation.preImage, + matches(row, preImage, absent: operation.preImageAbsentColumns, on: operation) + else { return nil } + return row + } + private func primaryKey( of row: [PluginCellValue], columns: [String], @@ -319,16 +360,17 @@ struct RewindPlanner { /// compare against, and a delete would be reported as restorable with no conflict check at all. private func keyIsComparable(_ operation: RowWriteOperation) -> Bool { !operation.primaryKeyColumns.isEmpty - && operation.primaryKeyColumns.allSatisfy { keyValue(of: operation, column: $0) != nil } + && operation.primaryKeyColumns.allSatisfy { + keyValue(in: savedImage(of: operation), of: operation, column: $0) != nil + } } - private func keyValue(of operation: RowWriteOperation, column: String) -> String? { - let source = operation.preImage ?? operation.postImage - guard let source, + private func keyValue(in image: [PluginCellValue]?, of operation: RowWriteOperation, column: String) -> String? { + guard let image, let index = operation.columns.firstIndex(of: column), - index < source.count + index < image.count else { return nil } - return source[index].asText + return image[index].asText } } diff --git a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift index 968b67c8df..19517845a3 100644 --- a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift +++ b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift @@ -146,7 +146,7 @@ struct RowChangeStatementFactory { try refuseUnwritableExpressions(in: changes, insertedRowData: insertedRowData) try refuseMisdimensionedVectors(in: changes, insertedRowData: insertedRowData) let insertedRowData = insertedRowData.mapValues(leavingServerOwnedColumnsToTheServer) - let changes = changes.map(droppingServerOwnedExpressions) + let changes = changes.map { markingServerOwnedDefaults($0, in: insertedRowData) } if let driverStatements = try pluginRowWrites( for: changes, insertedRowData: insertedRowData, @@ -190,7 +190,9 @@ struct RowChangeStatementFactory { private func refuseDefaultOnUpdate(in changes: [RowChange]) throws { guard !CellExpressionVocabulary.updateTakesDefault(databaseType) else { return } for change in changes where change.type == .update { - guard let defaulted = change.cellChanges.first(where: { $0.newValue.isDefaultMarker }) else { continue } + guard let defaulted = change.cellChanges.first(where: { + $0.newValue.isColumnDefault(isExpression: $0.newIsExpression) + }) else { continue } throw DataWriteError.changeRefused( table: tableName, kind: .update, @@ -209,9 +211,10 @@ struct RowChangeStatementFactory { in changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]] ) throws { + // A default reaches a driver in the spelling it reads, so only SQL proper has to be SQL this engine runs. let marked = changes.flatMap { change in Self.markedCells(of: change, insertedRowData: insertedRowData, columns: columns) - } + }.filter { $0.value != .columnDefault } guard !marked.isEmpty else { return } for cell in marked { guard case .text(let sql) = cell.value, CellExpressionVocabulary.isCanonical(sql, for: databaseType) @@ -309,21 +312,58 @@ struct RowChangeStatementFactory { guard !generatedColumns.isEmpty else { return values } return values.enumerated().map { index, value in guard columns.indices.contains(index), generatedColumns.contains(columns[index]) else { return value } - return Self.defaultMarker + return .columnDefault } } - /// The cell now holds the default marker, so it is no longer SQL to write. - private func droppingServerOwnedExpressions(_ change: RowChange) -> RowChange { - guard change.type == .insert, !change.expressionColumns.isEmpty, !generatedColumns.isEmpty else { return change } - var kept = change - kept.expressionColumns = change.expressionColumns.filter { - !(columns.indices.contains($0) && generatedColumns.contains(columns[$0])) + // Whatever SQL the cell held, it now holds the default, which only the mark makes one. + private func markingServerOwnedDefaults(_ change: RowChange, in insertedRowData: [RowID: [PluginCellValue]]) -> RowChange { + guard change.type == .insert, !generatedColumns.isEmpty else { return change } + let owned = Set(columns.indices.filter { generatedColumns.contains(columns[$0]) }) + var marked = change + marked.expressionColumns.subtract(owned) + if let values = insertedRowData[change.rowID] { + marked.expressionColumns.formUnion(owned.filter { values.indices.contains($0) }) } - return kept + return marked } - private static let defaultMarker = PluginCellValue.text("__DEFAULT__") + // A driver reads its own spelling for a default, not the mark, so a default reaches it spelled that way and unmarked. + private static func spellingDefaultsForDrivers( + _ changes: [RowChange], + insertedRowData: [RowID: [PluginCellValue]] + ) -> (changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]]) { + var rows = insertedRowData + let spelledChanges = changes.map { change -> RowChange in + var spelled = change + if change.type == .insert, var values = rows[change.rowID] { + for index in change.expressionColumns where values.indices.contains(index) && values[index] == .columnDefault { + values[index] = .pluginDefaultMarker + spelled.expressionColumns.remove(index) + } + rows[change.rowID] = values + } + let defaulted = change.cellChanges.filter { $0.newValue.isColumnDefault(isExpression: $0.newIsExpression) } + guard !defaulted.isEmpty else { return spelled } + spelled.cellChanges = change.cellChanges.map { cell in + guard cell.newValue.isColumnDefault(isExpression: cell.newIsExpression) else { return cell } + return CellChange( + columnIndex: cell.columnIndex, + columnName: cell.columnName, + oldValue: cell.oldValue, + newValue: .pluginDefaultMarker, + oldIsAbsent: cell.oldIsAbsent, + newIsAbsent: cell.newIsAbsent, + oldIsExpression: cell.oldIsExpression + ) + } + if change.type == .insert, rows[change.rowID] == nil { + spelled.expressionColumns.subtract(defaulted.map(\.columnIndex)) + } + return spelled + } + return (spelledChanges, rows) + } private func rowWriteContext(for keyed: PluginKeyedChanges) -> PluginRowWriteContext { var context = PluginRowWriteContext() @@ -386,6 +426,7 @@ struct RowChangeStatementFactory { }, context: restoreContext(rowCount: rows.count) ) { + try refuseRestoredTextDriversReadAsDefault(rows, restorable: restorable) try refuseUnfilteredWrites(restored.map { $0.statement }) return RestoreStatements(statements: restored.map { ParameterizedStatement(sql: $0.statement, parameters: $0.parameters.map(\.asAny)) @@ -489,9 +530,10 @@ struct RowChangeStatementFactory { insertedRowIDs: Set ) throws -> [DriverWrittenStatement]? { guard let pluginDriver else { return nil } + let spelled = Self.spellingDefaultsForDrivers(changes, insertedRowData: insertedRowData) let keyed = PluginKeyedChanges( - changes: changes, - insertedRowData: insertedRowData, + changes: spelled.changes, + insertedRowData: spelled.insertedRowData, deletedRowIDs: deletedRowIDs, insertedRowIDs: insertedRowIDs ) @@ -524,6 +566,7 @@ struct RowChangeStatementFactory { throw DataWriteError.changeRefused(table: tableName, kind: nil, reason: error.localizedDescription) } + try refuseTextDriversReadAsDefault(in: changes, insertedRowData: insertedRowData) try refuseUnfilteredWrites(writes.map(\.statement)) let unwritten = RowWriteCoverage.unwrittenChanges( changes, @@ -547,6 +590,46 @@ struct RowChangeStatementFactory { } } + // A driver may read the text `__DEFAULT__` as the column default, and none can say whether it does until PluginKit + // carries the mark beside the value, so the text is never handed to one as a value. + private func refuseTextDriversReadAsDefault( + in changes: [RowChange], + insertedRowData: [RowID: [PluginCellValue]] + ) throws { + for change in changes where change.type != .delete { + var written = change.cellChanges.filter { !$0.newIsExpression }.map { (column: $0.columnName, value: $0.newValue) } + if change.type == .insert, let stored = insertedRowData[change.rowID] { + written = zip(columns, stored).enumerated() + .filter { !change.expressionColumns.contains($0.offset) } + .map { (column: $0.element.0, value: $0.element.1) } + } + guard let literal = written.first(where: { $0.value == .pluginDefaultMarker }) else { continue } + throw textReadAsDefault(in: literal.column, kind: change.type == .insert ? .insert : .update) + } + } + + // A rewind puts a deleted row back with the values it held, which can be that text. + private func refuseRestoredTextDriversReadAsDefault(_ rows: [[PluginCellValue]], restorable: [Int]) throws { + for row in rows { + guard let index = restorable.first(where: { row.indices.contains($0) && row[$0] == .pluginDefaultMarker }) + else { continue } + throw textReadAsDefault(in: columns[index], kind: .insert) + } + } + + private func textReadAsDefault(in column: String, kind: RowWriteKind) -> DataWriteError { + .changeRefused( + table: tableName, + kind: kind, + reason: String( + format: String( + localized: "This database's driver may read the text __DEFAULT__ in %@ as the column default, so it cannot be saved as text." + ), + column + ) + ) + } + // The host's own generator never writes one, so only a driver's statements are read. private func refuseUnfilteredWrites(_ statements: [String]) throws { guard PluginManager.shared.editorLanguage(for: databaseType) == .sql else { return } diff --git a/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift b/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift index 457424223b..11cafebc8f 100644 --- a/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift +++ b/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift @@ -34,7 +34,8 @@ enum RowWriteOperationBuilder { writesThroughView: Bool = false, rowLocators: [RowID: String] = [:] ) -> [RowWriteOperation] { - changes.compactMap { change in + // A rewind undoes them newest first, and `changes` stops being in edit order once a change is cancelled. + changes.sorted { $0.sequence < $1.sequence }.compactMap { change in switch change.type { case .update: var operation = update( @@ -111,7 +112,7 @@ enum RowWriteOperationBuilder { ) -> RewindRefusal? { if containsTableOperation { return .destructiveTableOperation } if primaryKeyColumns.isEmpty || preImage == nil { return .noPrimaryKey } - if change.cellChanges.contains(where: { $0.newIsExpression || $0.newValue.isDefaultMarker }) { + if change.cellChanges.contains(where: \.newIsExpression) { return .serverComputedValue } if change.cellChanges.contains(where: { generatedColumns.contains($0.columnName) }) { @@ -172,11 +173,11 @@ enum RowWriteOperationBuilder { return .serverAssignedKey } let value = values[index] - if value.isNull || value.isDefaultMarker || change.expressionColumns.contains(index) { + if value.isNull || change.expressionColumns.contains(index) { return .serverAssignedKey } } - if values.contains(where: \.isDefaultMarker) || !change.expressionColumns.isEmpty { + if !change.expressionColumns.isEmpty { return .serverComputedValue } if values.contains(where: exceedsCapture) { return .valueTooLarge } diff --git a/apple/TablePro/Core/Database/DatabaseManager+ContainerDDL.swift b/apple/TablePro/Core/Database/DatabaseManager+ContainerDDL.swift index 147c37dfaf..f4550da094 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+ContainerDDL.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+ContainerDDL.swift @@ -59,10 +59,7 @@ extension DatabaseManager { } } - /// Runs a container operation the driver performs itself, with no statement to show. - /// - /// `kind` is the caller's to state because there is no text to classify: a drop is - /// `.destructiveQuery` and everything else is `.schemaMutation`. + // No statement to classify, so the caller states `kind`: a drop is destructive, anything else a schema mutation. func runContainerOperation( description: String, kind: OperationKind, @@ -70,6 +67,7 @@ extension DatabaseManager { databaseType: DatabaseType, event: CatalogEvent?, isConfirmationPreCleared: Bool = false, + gate: any ExecutionGate = ExecutionGateProvider.shared, perform: @Sendable @escaping (any DatabaseDriver) async throws -> Void ) async throws { try await authorizeContainerOperation( @@ -78,7 +76,8 @@ extension DatabaseManager { connectionId: scope.connectionId, databaseType: databaseType, description: description, - isConfirmationPreCleared: isConfirmationPreCleared + isConfirmationPreCleared: isConfirmationPreCleared, + gate: gate ) try await withScopedDriver( scope: scope, @@ -90,16 +89,14 @@ extension DatabaseManager { if let event { CatalogChangeService.post(event) } } - /// Runs a container operation the driver expressed as statements. - /// - /// The kind comes from the statements rather than from the action's name, so an edit that - /// revokes a privilege is authorized at the destructive tier the same way a drop is. + // The kind comes from the statements, so an edit that revokes a privilege is authorized like a drop. func runContainerStatements( _ plan: ContainerDDLPlan, description: String, scope: DatabaseScope, databaseType: DatabaseType, - event: CatalogEvent? + event: CatalogEvent?, + gate: any ExecutionGate = ExecutionGateProvider.shared ) async throws { let statements = plan.statements guard !statements.isEmpty else { return } @@ -113,7 +110,8 @@ extension DatabaseManager { connectionId: scope.connectionId, databaseType: databaseType, description: description, - isConfirmationPreCleared: false + isConfirmationPreCleared: false, + gate: gate ) let route = schemaChangeRoute(for: scope) @@ -139,21 +137,19 @@ extension DatabaseManager { if let event { CatalogChangeService.post(event) } } - /// `isConfirmationPreCleared` is for a caller that has already shown the user a destructive - /// confirmation of its own. Without it the sidebar's Drop dialog is followed by the gate's, - /// asking twice for one gesture and once per target for a batch. It clears the confirmation - /// only: Touch ID, the read-only refusal and the audit record all still apply. + // Pre-cleared skips only a confirmation the caller already showed; Touch ID, Read-Only and the audit still apply. private func authorizeContainerOperation( sql: String?, kind: OperationKind, connectionId: UUID, databaseType: DatabaseType, description: String, - isConfirmationPreCleared: Bool + isConfirmationPreCleared: Bool, + gate: any ExecutionGate ) async throws { var capabilities = CallerCapabilities.interactiveUser if isConfirmationPreCleared { capabilities.insert(.confirmationPreCleared) } - let decision = await ExecutionGateProvider.shared.authorize( + let decision = await gate.authorize( OperationRequest( connectionId: connectionId, databaseType: databaseType, @@ -164,11 +160,7 @@ extension DatabaseManager { operationDescription: description ) ) - guard case .authorized = decision else { - throw DatabaseError.queryFailed( - decision.deniedReason ?? String(localized: "This change was not authorized") - ) - } + if let denial = decision.denialError { throw denial } } private static func execute( diff --git a/apple/TablePro/Core/Database/DatabaseManager+Health.swift b/apple/TablePro/Core/Database/DatabaseManager+Health.swift index b42ed9eae7..3f9fc5feb2 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Health.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Health.swift @@ -149,10 +149,17 @@ extension DatabaseManager { result.driver.disconnect() return .abort } + let warning = openingWarning( + driver: result.driver, + startupFailures: result.startupFailures, + restoreWarning: result.restoreWarning, + for: connectionId + ) updateSession(connectionId) { session in session.driver = result.driver session.effectiveConnection = result.effectiveConnection session.status = .connected + session.connectionWarning = warning if let schemaDriver = result.driver as? SchemaSwitchable { session.browseSchema = schemaDriver.currentSchema } @@ -233,6 +240,8 @@ extension DatabaseManager { let driver: DatabaseDriver let effectiveConnection: DatabaseConnection let cachedPassword: String? + let startupFailures: [StartupCommandFailure] + let restoreWarning: String? } // Rebuilds the tunnel for a tunneled session before connecting the new driver. Tunnels are keyed by @@ -282,13 +291,13 @@ extension DatabaseManager { throw CancellationError() } - await applyTimeoutAndStartupCommands( + let startupFailures = await applyTimeoutAndStartupCommands( on: driver, queryTimeoutSeconds: session.effectiveQueryTimeoutSeconds, startupCommands: session.connection.startupCommands, connectionName: session.connection.name ) - await restoreSchemaAndDatabase( + let restoreWarning = await restoreSchemaAndDatabase( on: driver, savedSchema: session.browseSchema, savedDatabase: databaseSwitchRequiresReconnect(session.connection) ? nil : session.browseDatabase @@ -297,7 +306,9 @@ extension DatabaseManager { return ReconnectResult( driver: driver, effectiveConnection: connectionForDriver, - cachedPassword: connectResult.cachedPassword + cachedPassword: connectResult.cachedPassword, + startupFailures: startupFailures, + restoreWarning: restoreWarning ) } @@ -306,7 +317,7 @@ extension DatabaseManager { queryTimeoutSeconds: Int, startupCommands: String?, connectionName: String - ) async { + ) async -> [StartupCommandFailure] { do { try await driver.applyQueryTimeout(queryTimeoutSeconds) } catch { @@ -315,7 +326,7 @@ extension DatabaseManager { ) } - await executeStartupCommands(startupCommands, on: driver, connectionName: connectionName) + return await executeStartupCommands(startupCommands, on: driver, connectionName: connectionName) } private func databaseSwitchRequiresReconnect(_ connection: DatabaseConnection) -> Bool { @@ -323,17 +334,23 @@ extension DatabaseManager { .capabilities.requiresReconnectForDatabaseSwitch ?? false } + // The reconnected session stays on the server's default when a restore fails, which the window would not show. func restoreSchemaAndDatabase( on driver: DatabaseDriver, savedSchema: String?, savedDatabase: String? - ) async { + ) async -> String? { + var warnings: [String?] = [] // The schema lives inside the database, and a database switch can leave the driver on none. if let savedDatabase, let adapter = driver as? PluginDriverAdapter { do { try await adapter.switchDatabase(to: savedDatabase) } catch { Self.logger.warning("Failed to restore database '\(savedDatabase)' on reconnect: \(error.localizedDescription)") + warnings.append(String( + format: String(localized: "Reconnecting could not reopen the database %1$@: %2$@"), + savedDatabase, error.localizedDescription + )) } } @@ -342,8 +359,13 @@ extension DatabaseManager { try await schemaDriver.switchSchema(to: savedSchema) } catch { Self.logger.warning("Failed to restore schema '\(savedSchema)' on reconnect: \(error.localizedDescription)") + warnings.append(String( + format: String(localized: "Reconnecting could not reopen the schema %1$@: %2$@"), + savedSchema, error.localizedDescription + )) } } + return Self.joinedWarning(warnings) } internal func stopHealthMonitor(for connectionId: UUID) async { @@ -440,13 +462,13 @@ extension DatabaseManager { } let driver = connectResult.driver - await applyTimeoutAndStartupCommands( + let startupFailures = await applyTimeoutAndStartupCommands( on: driver, queryTimeoutSeconds: session.effectiveQueryTimeoutSeconds, startupCommands: session.connection.startupCommands, connectionName: session.connection.name ) - await restoreSchemaAndDatabase( + let restoreWarning = await restoreSchemaAndDatabase( on: driver, savedSchema: activeSessions[sessionId]?.browseSchema, savedDatabase: databaseSwitchRequiresReconnect(session.connection) ? nil : activeSessions[sessionId]?.browseDatabase @@ -456,10 +478,14 @@ extension DatabaseManager { throw CancellationError() } + let warning = openingWarning( + driver: driver, startupFailures: startupFailures, restoreWarning: restoreWarning, for: sessionId + ) updateSession(sessionId) { session in session.driver = driver session.status = .connected session.effectiveConnection = effectiveConnection + session.connectionWarning = warning if let schemaDriver = driver as? SchemaSwitchable { session.browseSchema = schemaDriver.currentSchema } diff --git a/apple/TablePro/Core/Database/DatabaseManager+Principals.swift b/apple/TablePro/Core/Database/DatabaseManager+Principals.swift index 7cf505ddcd..12680f123e 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Principals.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Principals.swift @@ -14,7 +14,8 @@ extension DatabaseManager { func executePrincipalChanges( changes: [PrincipalChange], databaseType: DatabaseType, - connectionId: UUID + connectionId: UUID, + gate: any ExecutionGate = ExecutionGateProvider.shared ) async throws { guard let driver = driver(for: connectionId) else { throw DatabaseError.notConnected @@ -39,7 +40,7 @@ extension DatabaseManager { ? .destructiveQuery : .schemaMutation - let authorization = await ExecutionGateProvider.shared.authorize( + let authorization = await gate.authorize( OperationRequest( connectionId: connectionId, databaseType: databaseType, @@ -50,12 +51,7 @@ extension DatabaseManager { operationDescription: String(localized: "Apply User and Role Changes") ) ) - guard case .authorized = authorization else { - throw DatabaseError.queryFailed( - authorization.deniedReason - ?? String(localized: "User and role changes were not authorized") - ) - } + if let denial = authorization.denialError { throw denial } /// Reported whether or not every statement ran: an engine that cannot roll principal /// statements back keeps the ones before a failure, and `DROP OWNED` among them drops objects. diff --git a/apple/TablePro/Core/Database/DatabaseManager+Schema.swift b/apple/TablePro/Core/Database/DatabaseManager+Schema.swift index fd4b884352..01ab285459 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Schema.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Schema.swift @@ -226,13 +226,7 @@ extension DatabaseManager { } } - /// Run a Create Table draft's statements, on the same isolated route and in the same shape as - /// `executeSchemaChanges`. - /// - /// The route is what matters. The view used to reach for `driver(for:)`, which is the session - /// driver, so a `BEGIN` opened here joined whatever transaction a query tab had left open and - /// the matching `COMMIT` took that tab's uncommitted work with it. `schemaChangeRoute` exists - /// for exactly that, and the app's own DDL has no business on the user's connection. + // On the schema change route: a BEGIN on the session driver would join a query tab's open transaction. func executeCreateTable( statements: [String], databaseType: DatabaseType, @@ -254,11 +248,7 @@ extension DatabaseManager { operationDescription: String(localized: "Create Table") ) ) - guard case .authorized = authorization else { - throw DatabaseError.queryFailed( - authorization.deniedReason ?? String(localized: "Operation not permitted") - ) - } + if let denial = authorization.denialError { throw denial } let executionTimes: [TimeInterval] do { diff --git a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift index 2e0bfa525e..23c6f022e6 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift @@ -136,7 +136,7 @@ extension DatabaseManager { attempt: attempt, for: connection.id ) - await applyTimeoutAndStartupCommands( + let startupFailures = await applyTimeoutAndStartupCommands( on: driver, queryTimeoutSeconds: effectiveQueryTimeoutSeconds, startupCommands: resolvedConnection.startupCommands, @@ -167,7 +167,7 @@ extension DatabaseManager { session.driver = driver session.status = driver.status session.effectiveConnection = effectiveConnection - session.connectionWarning = driver.connectionWarning + session.connectionWarning = openingWarning(driver: driver, startupFailures: startupFailures, for: connection.id) // A connect answers whatever went wrong before it, including a reconnect that gave up on this entry. session.liveness = .live session.recovery = nil @@ -1172,6 +1172,7 @@ extension DatabaseManager { monitorPingedDrivers.removeValue(forKey: connectionId) sessionDriverGate.drain(connectionId: connectionId) connectionStatusVersions.removeValue(forKey: connectionId) + reportedStartupFailures.removeValue(forKey: connectionId) forgetVerification(for: connectionId) aiAccessApprovals.revoke(connectionId) AppEvents.shared.connectionStatusChanged.send( diff --git a/apple/TablePro/Core/Database/DatabaseManager+Startup.swift b/apple/TablePro/Core/Database/DatabaseManager+Startup.swift index 5f68e9c7a7..3804867891 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Startup.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Startup.swift @@ -61,26 +61,116 @@ extension DatabaseManager { return statements } - @discardableResult nonisolated internal func executeStartupCommands( _ commands: String?, on driver: DatabaseDriver, connectionName: String - ) async -> [(statement: String, error: String)] { - var failures: [(statement: String, error: String)] = [] + ) async -> [StartupCommandFailure] { + var failures: [StartupCommandFailure] = [] let databaseType = driver.connection.type let grammar = SQLLexicalResolver.readings(for: databaseType, session: driver.sessionLexicalState).execution for statement in Self.startupStatements(from: commands, databaseType: databaseType, grammar: grammar) { do { _ = try await driver.execute(query: statement) Self.startupLogger.info( - "Startup command succeeded for '\(connectionName)': \(statement)" + "Startup command succeeded for '\(connectionName, privacy: .private)': \(statement, privacy: .private)" ) } catch { Self.startupLogger.warning( - "Startup command failed for '\(connectionName)': \(statement) — \(error.localizedDescription)" + "Startup command failed for '\(connectionName, privacy: .private)': \(statement, privacy: .private): \(error.localizedDescription, privacy: .private)" ) - failures.append((statement: statement, error: error.localizedDescription)) + failures.append(StartupCommandFailure(statement: statement, error: error)) } } return failures } + + // A startup command can be a safety rail, `SET ROLE app_ro` or a read-only default, so a session that opened + // without one says so on the connection's warning rather than only in the log. + nonisolated internal static func startupWarning(for failures: [StartupCommandFailure]) -> String? { + guard let first = failures.first else { return nil } + guard failures.count > 1 else { + guard let code = first.code else { + return String( + format: String(localized: "The startup command %@ failed, so this session runs without it."), + shownStatement(first.statement) + ) + } + return String( + format: String(localized: "The startup command %1$@ failed (%2$@), so this session runs without it."), + shownStatement(first.statement), code + ) + } + let lines = failures.map { failure in + failure.code.map { "\(shownStatement(failure.statement)) (\($0))" } ?? shownStatement(failure.statement) + } + return String( + format: String(localized: "Startup commands failed, so this session runs without them:\n%@"), + lines.joined(separator: "\n") + ) + } + + // A startup command can carry a secret, a password after IDENTIFIED BY or a key in a literal, so the banner shows + // it only up to its first quote or credential clause. + nonisolated internal static func shownStatement(_ statement: String) -> String { + let quote = statement.firstIndex(where: { $0 == "'" || $0 == "\"" || $0 == "$" }) + let stops = [quote] + ["IDENTIFIED", "PASSWORD"].map { statement.range(of: $0, options: .caseInsensitive)?.lowerBound } + let cut = stops.compactMap { $0 }.min() ?? statement.endIndex + let kept = statement[.. limit else { return kept } + return String(kept.prefix(limit)) + "…" + } + + nonisolated internal static func joinedWarning(_ parts: [String?]) -> String? { + let present = parts.compactMap { $0 }.filter { !$0.isEmpty } + return present.isEmpty ? nil : present.joined(separator: "\n\n") + } + + // The warning a session opens with: the login's own, then what its setup could not do. It replaces the previous + // one, so a reconnect that ran every startup command drops a banner about the session before it, and the record + // of named failures starts over. + internal func openingWarning( + driver: DatabaseDriver, + startupFailures: [StartupCommandFailure], + restoreWarning: String? = nil, + for connectionId: UUID + ) -> String? { + reportedStartupFailures[connectionId] = Set(startupFailures.map { $0.statement }) + return Self.joinedWarning([driver.connectionWarning, Self.startupWarning(for: startupFailures), restoreWarning]) + } + + // A pool entry or a database lane opens a session of its own, which can fail a command the window's session ran. + // Each failing statement is named once per connect: the pool keeps opening entries while the window is open. + internal func reportStartupFailures(_ failures: [StartupCommandFailure], for connectionId: UUID) { + guard activeSessions[connectionId] != nil else { return } + var reported = reportedStartupFailures[connectionId] ?? [] + let unreported = failures.filter { reported.insert($0.statement).inserted } + reportedStartupFailures[connectionId] = reported + guard let warning = Self.startupWarning(for: unreported) else { return } + updateSession(connectionId) { session in + session.connectionWarning = Self.joinedWarning([session.connectionWarning, warning]) + } + } +} + +// What the banner may show of a failed startup command: the statement and the server's code, never its message, which +// can echo the values the statement carried. The message goes to the log, private. +struct StartupCommandFailure: Sendable, Equatable { + let statement: String + let code: String? + + init(statement: String, code: String?) { + self.statement = statement + self.code = code + } + + init(statement: String, error: Error) { + self.init(statement: statement, code: Self.code(of: error)) + } + + static func code(of error: Error) -> String? { + guard let driverError = error as? any PluginDriverError else { return nil } + if let state = driverError.pluginSqlState, !state.isEmpty { return state } + if let code = driverError.pluginErrorCode, code != 0 { return String(code) } + return nil + } } diff --git a/apple/TablePro/Core/Database/DatabaseManager.swift b/apple/TablePro/Core/Database/DatabaseManager.swift index 15072aca99..2dec865754 100644 --- a/apple/TablePro/Core/Database/DatabaseManager.swift +++ b/apple/TablePro/Core/Database/DatabaseManager.swift @@ -126,6 +126,10 @@ final class DatabaseManager: ObservableObject { /// for the same reason every other shared write here is generation-checked. internal var connectionStages: [UUID: ConnectionStage] = [:] + // The startup statements a connection's warning has already named since its connect, so a pool entry or lane + // opened later names only a failure the window was not told about. + internal var reportedStartupFailures: [UUID: Set] = [:] + /// Orders operations that move the shared driver, so two windows cannot interleave /// their pins and each run against the other's database. internal let sessionDriverGate = SessionDriverGate() diff --git a/apple/TablePro/Core/Database/TriggerEditing.swift b/apple/TablePro/Core/Database/TriggerEditing.swift index c3744ad7e4..8a427c1bfd 100644 --- a/apple/TablePro/Core/Database/TriggerEditing.swift +++ b/apple/TablePro/Core/Database/TriggerEditing.swift @@ -14,7 +14,6 @@ import TableProSQLGrammar enum TriggerEditingError: LocalizedError { case notConnected - case denied(String) case dropUnavailable case notCreateTrigger case definitionUnavailable @@ -23,7 +22,6 @@ enum TriggerEditingError: LocalizedError { var errorDescription: String? { switch self { case .notConnected: String(localized: "Not connected to database") - case let .denied(reason): reason case .dropUnavailable: String(localized: "This database cannot drop triggers") case .notCreateTrigger: String( @@ -121,9 +119,7 @@ enum TriggerApplyStrategy: Equatable { enum TriggerEditing { nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "TriggerEditing") - /// Runs on the schema change route, never on the session driver: the trigger's own BEGIN - /// would join a transaction a query tab left open, and its COMMIT or ROLLBACK would take - /// that tab's uncommitted work with it. + // On the schema change route: a BEGIN on the session driver would join a query tab's open transaction. static func apply( scope: DatabaseScope, connection: DatabaseConnection, @@ -162,9 +158,7 @@ enum TriggerEditing { statements: plan.gatedStatements(for: sql) ) ) - guard case .authorized = decision else { - throw TriggerEditingError.denied(decision.deniedReason ?? String(localized: "Operation not permitted")) - } + if let denial = decision.denialError { throw denial } let startedAt = Date() defer { @@ -230,10 +224,7 @@ enum TriggerEditing { name: String, gate: any ExecutionGate = ExecutionGateProvider.shared ) async throws { - /// Built on a driver in the scope that will run it, the way `apply` builds its own inside - /// the lease. The session driver is wherever the sidebar last went, and an engine that - /// qualifies its DDL with the connection's current database writes that name into a - /// statement the pooled connection then runs somewhere else. + // Built in the scope that runs it: the session driver may qualify the DDL with another database. let generated = try await DatabaseManager.shared.withMetadataDriver(scope: scope) { driver in driver.generateDropTriggerSQL(name: name, table: tableName) } @@ -252,9 +243,7 @@ enum TriggerEditing { operationDescription: String(localized: "Drop Trigger") ) ) - guard case .authorized = decision else { - throw TriggerEditingError.denied(decision.deniedReason ?? String(localized: "Operation not permitted")) - } + if let denial = decision.denialError { throw denial } let startedAt = Date() try await withSchemaChangeDriver(scope: scope) { driver in diff --git a/apple/TablePro/Core/LSP/LSPTransport.swift b/apple/TablePro/Core/LSP/LSPTransport.swift index 47e274cc6a..6677bb3748 100644 --- a/apple/TablePro/Core/LSP/LSPTransport.swift +++ b/apple/TablePro/Core/LSP/LSPTransport.swift @@ -32,6 +32,45 @@ enum LSPTransportError: Error, LocalizedError { } } +// MARK: - LSPMessageWriter + +// Writes on its own queue, so a server that stops reading blocks the queue and never the transport. +final class LSPMessageWriter: @unchecked Sendable { + private let handle: FileHandle + private let queue = DispatchQueue(label: "com.TablePro.LSPTransport.writer") + private var isClosed = false + + init(_ handle: FileHandle) throws { + // Without it, a write to a server that has exited raises SIGPIPE and ends the app. + guard fcntl(handle.fileDescriptor, F_SETNOSIGPIPE, 1) == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + self.handle = handle + } + + func write(_ frame: Data, onFailure: @escaping @Sendable (any Error) -> Void) { + queue.async { [self] in + guard !isClosed else { + onFailure(LSPTransportError.processNotRunning) + return + } + do { + try handle.write(contentsOf: frame) + } catch { + onFailure(error) + } + } + } + + func close() { + queue.async { [self] in + guard !isClosed else { return } + isClosed = true + try? handle.close() + } + } +} + // MARK: - LSPTransport actor LSPTransport { @@ -39,6 +78,7 @@ actor LSPTransport { private var process: Process? private var stdinPipe: Pipe? + private var writer: LSPMessageWriter? private var stdoutPipe: Pipe? private var stderrPipe: Pipe? private var stderrReader: PipeReader? @@ -66,6 +106,7 @@ actor LSPTransport { proc.standardOutput = stdout proc.standardError = stderr + self.writer = try LSPMessageWriter(stdin.fileHandleForWriting) self.stdinPipe = stdin self.stdoutPipe = stdout self.stderrPipe = stderr @@ -105,9 +146,8 @@ actor LSPTransport { readerTask?.cancel() readerTask = nil - if let stdinHandle = stdinPipe?.fileHandleForWriting { - try? stdinHandle.close() - } + writer?.close() + writer = nil if let stdoutHandle = stdoutPipe?.fileHandleForReading { try? stdoutHandle.close() } @@ -144,7 +184,7 @@ actor LSPTransport { return try await withCheckedThrowingContinuation { continuation in pendingRequests[requestID] = continuation do { - try writeMessage(data) + try writeMessage(data, failingRequest: requestID) } catch { pendingRequests.removeValue(forKey: requestID) continuation.resume(throwing: error) @@ -212,19 +252,22 @@ actor LSPTransport { // MARK: - Private - private func writeMessage(_ data: Data) throws { - guard let stdinPipe else { + private func writeMessage(_ data: Data, failingRequest requestID: Int? = nil) throws { + guard let writer else { throw LSPTransportError.processNotRunning } - - let header = "Content-Length: \(data.count)\r\n\r\n" - guard let headerData = header.data(using: .utf8) else { - throw LSPTransportError.invalidResponse + let frame = Data("Content-Length: \(data.count)\r\n\r\n".utf8) + data + writer.write(frame) { [weak self] error in + Task { [weak self] in + await self?.writeFailed(error, requestID: requestID) + } } + } - let handle = stdinPipe.fileHandleForWriting - handle.write(headerData) - handle.write(data) + private func writeFailed(_ error: any Error, requestID: Int?) { + Self.logger.warning("LSP write failed: \(error.localizedDescription)") + guard let requestID, let continuation = pendingRequests.removeValue(forKey: requestID) else { return } + continuation.resume(throwing: error) } private func runReadLoop(handle: FileHandle) async { @@ -341,6 +384,9 @@ actor LSPTransport { private func handleProcessExit(code: Int32) { Self.logger.info("LSP process exited with code \(code)") + writer?.close() + writer = nil + stdinPipe = nil let pending = pendingRequests pendingRequests.removeAll() for (_, continuation) in pending { diff --git a/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift b/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift index 164f6c3613..5b6028255f 100644 --- a/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift +++ b/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift @@ -38,6 +38,8 @@ internal struct ObjectCopyRowCopier: Sendable { internal struct Outcome: Sendable { internal let inserted: Int internal let cancelled: Bool + // The limit the source's stream stopped at, so the table holds only its leading rows. + internal var rowCut: Int? } /// A table that failed after some of its batches were written. Outside a transaction those stay @@ -94,7 +96,13 @@ internal struct ObjectCopyRowCopier: Sendable { let generator = try makeGenerator(targetDriver: targetDriver) let budget = SQLWriteBatchBudget(columnCount: step.columns.count, generator: generator) var filler = SQLWriteBatchFiller<[PluginCellValue]>(budget: budget) - var stream = sourceDriver.streamRows(query: step.sourceQuery).makeAsyncIterator() + let cut = OSAllocatedUnfairLock(initialState: nil) + // The copy's SELECT states no engine ceiling, so only a limit the driver declares applies, as for its own export query. + var stream = ExportRowCut.reporting( + sourceDriver.streamRows(query: step.sourceQuery), engineCeiling: nil + ) { limit in + cut.withLock { $0 = limit } + }.makeAsyncIterator() var inserted = 0 @@ -121,7 +129,7 @@ internal struct ObjectCopyRowCopier: Sendable { inserted += try await write(batch, generator: generator, driver: targetDriver) onProgress(inserted) } - return Outcome(inserted: inserted, cancelled: Task.isCancelled) + return Outcome(inserted: inserted, cancelled: Task.isCancelled, rowCut: cut.withLock { $0 }) } catch is CancellationError { return Outcome(inserted: inserted, cancelled: true) } catch where inserted > 0 { diff --git a/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift b/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift index 6cef58ed51..f8c04fd656 100644 --- a/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift +++ b/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift @@ -22,6 +22,7 @@ internal struct ObjectCopyObjectOutcome: Identifiable, Sendable { internal let selection: ObjectCopySelection internal let rowsCopied: Int internal let error: String? + internal var warning: String? internal var id: String { selection.id } internal var succeeded: Bool { error == nil } @@ -67,6 +68,8 @@ internal struct ObjectCopyRunResult: Sendable { internal var firstError: String? { outcomes.compactMap(\.error).first } + internal var warnings: [String] { outcomes.compactMap(\.warning) } + /// Every failure, each with an identity of its own. /// /// One object fails once per phase it reaches, so a table whose `CREATE` failed under Skip and @@ -507,9 +510,14 @@ internal struct ObjectCopyRunner { /// would send a retry over them. var rowsCopied = 0 - mutating func recordCopy(of step: ObjectCopyTableStep, inserted: Int) { + mutating func recordCopy(of step: ObjectCopyTableStep, inserted: Int, rowCut: Int? = nil) { rowsCopied += inserted - outcomes.append(ObjectCopyObjectOutcome(selection: step.selection, rowsCopied: inserted, error: nil)) + outcomes.append(ObjectCopyObjectOutcome( + selection: step.selection, + rowsCopied: inserted, + error: nil, + warning: rowCut.map { StreamRowCutWarning.text(table: step.selection.qualifiedName, limit: $0) } + )) } /// Stop reaching the driver mid-stream is the user stopping, not the copy failing, and the @@ -702,7 +710,7 @@ internal struct ObjectCopyRunner { result.recordStop(of: step, inserted: outcome.inserted, progress: progress) break } - result.recordCopy(of: step, inserted: outcome.inserted) + result.recordCopy(of: step, inserted: outcome.inserted, rowCut: outcome.rowCut) } catch is CancellationError { result.recordStop(of: step, inserted: 0, progress: progress) break @@ -748,7 +756,7 @@ internal struct ObjectCopyRunner { result.recordStop(of: step, inserted: outcome.inserted, progress: progress) break } - result.recordCopy(of: step, inserted: outcome.inserted) + result.recordCopy(of: step, inserted: outcome.inserted, rowCut: outcome.rowCut) } catch is CancellationError { result.recordStop(of: step, inserted: 0, progress: progress) break diff --git a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift index 4c67711e74..5878dcda37 100644 --- a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift +++ b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift @@ -61,13 +61,7 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable // One line per table whose rows stopped at a ceiling, so a partial copy is never reported as the whole table. var cappedTableWarnings: [String] { - cappedTables.withLock { $0 }.map { capped in - String( - format: String(localized: "%1$@: only the first %2$lld rows were read, the most this database returns from one query."), - capped.table, - capped.limit - ) - } + cappedTables.withLock { $0 }.map { StreamRowCutWarning.text(table: $0.table, limit: $0.limit) } } func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { diff --git a/apple/TablePro/Core/Plugins/ExportRowCut.swift b/apple/TablePro/Core/Plugins/ExportRowCut.swift index 04efb07dc6..a3be667953 100644 --- a/apple/TablePro/Core/Plugins/ExportRowCut.swift +++ b/apple/TablePro/Core/Plugins/ExportRowCut.swift @@ -58,6 +58,17 @@ final class ExportRowCut: @unchecked Sendable { } } +// One wording for every reader of a cut stream: export, Table Transfer, Object Copy and Data Compare. +enum StreamRowCutWarning { + static func text(table: String, limit: Int) -> String { + String( + format: String(localized: "%1$@: only the first %2$lld rows were read, the most this database returns from one query."), + table, + limit + ) + } +} + extension ExportRowCut where Element == PluginStreamElement { static func reporting( _ upstream: AsyncThrowingStream, diff --git a/apple/TablePro/Core/Services/Execution/GatedOperationFailure.swift b/apple/TablePro/Core/Services/Execution/GatedOperationFailure.swift new file mode 100644 index 0000000000..f0e6bdaeae --- /dev/null +++ b/apple/TablePro/Core/Services/Execution/GatedOperationFailure.swift @@ -0,0 +1,39 @@ +// +// GatedOperationFailure.swift +// TablePro +// + +import Foundation + +// Cancel at the gate's confirmation is the user's own choice: it shows nothing and the sheet keeps its edits. +internal enum GatedOperationFailure: Equatable { + case cancelledByUser + case refused(String) + case failed(String) + + internal init(_ error: any Error) { + guard let gateError = error as? ExecutionGateError else { + self = .failed(error.localizedDescription) + return + } + switch gateError { + case .cancelledByUser: + self = .cancelledByUser + case .denied(let reason): + self = .refused(reason) + } + } + + internal var message: String? { + switch self { + case .cancelledByUser: nil + case .refused(let reason), .failed(let reason): reason + } + } + + // A refusal and a Cancel both stop before the first statement: nothing to report or reload. + internal var reportsFailure: Bool { + if case .failed = self { return true } + return false + } +} diff --git a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift index 2d85197c26..c7a274f88f 100644 --- a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift +++ b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift @@ -591,13 +591,14 @@ final class MetadataConnectionPool { ) do { try await driver.connect() - try await Self.prepareSession( + let startupFailures = try await Self.prepareSession( driver, queryTimeoutSeconds: queryTimeout, startupCommands: session.connection.startupCommands, connectionName: session.connection.name, timeoutSeconds: operationTimeoutSeconds ) + DatabaseManager.shared.reportStartupFailures(startupFailures, for: scope.connectionId) if let database = plan.switchDatabase { try await Self.switchDatabase(driver, to: database, timeoutSeconds: operationTimeoutSeconds) } @@ -680,14 +681,14 @@ final class MetadataConnectionPool { startupCommands: String?, connectionName: String, timeoutSeconds: Double? - ) async throws { + ) async throws -> [StartupCommandFailure] { try await bounded( driver: driver, timeoutSeconds: timeoutSeconds, timeoutMessage: String(localized: "Preparing the metadata connection timed out.") ) { try? await driver.applyQueryTimeout(queryTimeoutSeconds) - await DatabaseManager.shared.executeStartupCommands( + return await DatabaseManager.shared.executeStartupCommands( startupCommands, on: driver, connectionName: connectionName ) } @@ -695,18 +696,17 @@ final class MetadataConnectionPool { /// Disconnects the driver when the deadline fires so a driver call that /// ignores task cancellation still completes and the timeout can propagate. - private static func bounded( + private static func bounded( driver: DatabaseDriver, timeoutSeconds: Double?, timeoutMessage: String, - _ operation: @escaping @Sendable () async throws -> Void - ) async throws { + _ operation: @escaping @Sendable () async throws -> T + ) async throws -> T { guard let timeoutSeconds, timeoutSeconds > 0 else { - try await operation() - return + return try await operation() } do { - try await withTimeout( + return try await withTimeout( seconds: timeoutSeconds, onTimeout: { driver.disconnect() }, operation: operation diff --git a/apple/TablePro/Core/Services/Query/RowOperationsManager.swift b/apple/TablePro/Core/Services/Query/RowOperationsManager.swift index 66f5ebf5e4..2a33425139 100644 --- a/apple/TablePro/Core/Services/Query/RowOperationsManager.swift +++ b/apple/TablePro/Core/Services/Query/RowOperationsManager.swift @@ -49,10 +49,12 @@ final class RowOperationsManager { func addNewRow(tableRows: inout TableRows, uuidKeyForms: [Int: UUIDCellForm] = [:]) -> AddNewRowResult? { var newRowValues: [PluginCellValue] = [] var absentColumns: Set = [] + var defaultColumns: Set = [] var prefilled: Set = [] for (index, column) in tableRows.columns.enumerated() { if tableRows.generatedColumns.contains(column) || tableRows.serverAssignsValue(forColumn: column) { - newRowValues.append(.text("__DEFAULT__")) + newRowValues.append(.columnDefault) + defaultColumns.insert(index) } else if let form = uuidKeyForms[index] { newRowValues.append(Self.freshUUID(form)) prefilled.insert(index) @@ -62,7 +64,11 @@ final class RowOperationsManager { } } return appendInsertedRow( - values: newRowValues, absentColumns: absentColumns, prefilledColumns: prefilled, to: &tableRows + values: newRowValues, + absentColumns: absentColumns, + expressionColumns: defaultColumns, + prefilledColumns: prefilled, + to: &tableRows ) } @@ -86,14 +92,15 @@ final class RowOperationsManager { + Array(tableRows.columnIdentity.keys) for resetColumn in resetColumns { if let index = tableRows.columns.firstIndex(of: resetColumn), index < newValues.count { + absentColumns.remove(index) if let form = uuidKeyForms[index] { newValues[index] = Self.freshUUID(form) prefilled.insert(index) + expressionColumns.remove(index) } else { - newValues[index] = .text("__DEFAULT__") + newValues[index] = .columnDefault + expressionColumns.insert(index) } - absentColumns.remove(index) - expressionColumns.remove(index) } } return appendInsertedRow( @@ -406,9 +413,9 @@ final class RowOperationsManager { } if let pkIndex = schema.primaryKeyIndex, pkIndex < values.count { - values[pkIndex] = .text("__DEFAULT__") + values[pkIndex] = .columnDefault absentColumns.remove(pkIndex) - expressionColumns.remove(pkIndex) + expressionColumns.insert(pkIndex) } return ParsedRow( @@ -438,8 +445,8 @@ final class RowOperationsManager { return .entered(text, columnType: tableRows.editingColumnType(at: column)) } if let pkIndex = schema.primaryKeyIndex, pkIndex < values.count { - values[pkIndex] = .text("__DEFAULT__") - expressionColumns.remove(pkIndex) + values[pkIndex] = .columnDefault + expressionColumns.insert(pkIndex) } return ParsedRow(values: values, sourceLineNumber: index + 1, expressionColumns: expressionColumns) } @@ -447,28 +454,39 @@ final class RowOperationsManager { // A blank cell of pasted text was left unfilled, so it takes what a new row's cell does: the server's value where // the column has a default, else NULL. A NOT NULL text column with no default takes the empty string it can hold. - private static func unfilledCellValues(columns: [String], tableRows: TableRows) -> [PluginCellValue] { - columns.map { name in + private static func unfilledCellValues( + columns: [String], + tableRows: TableRows + ) -> (values: [PluginCellValue], defaults: Set) { + var defaults: Set = [] + let values = columns.enumerated().map { position, name -> PluginCellValue in guard let index = tableRows.columns.firstIndex(of: name) else { return .null } if tableRows.generatedColumns.contains(name) || tableRows.serverAssignsValue(forColumn: name) { - return .text("__DEFAULT__") + defaults.insert(position) + return .columnDefault } if tableRows.refusesNull(columnAt: index), tableRows.editingColumnType(at: index)?.hasEmptyValue == true { return .text("") } return .null } + return (values, defaults) } - private static func fillingBlankCells(_ row: ParsedRow, with unfilled: [PluginCellValue]) -> ParsedRow { + private static func fillingBlankCells( + _ row: ParsedRow, + with unfilled: (values: [PluginCellValue], defaults: Set) + ) -> ParsedRow { guard !row.blankColumns.isEmpty else { return row } var values = row.values - for index in row.blankColumns where index < values.count && index < unfilled.count { - values[index] = unfilled[index] + var expressionColumns = row.expressionColumns + for index in row.blankColumns where index < values.count && index < unfilled.values.count { + values[index] = unfilled.values[index] + if unfilled.defaults.contains(index) { expressionColumns.insert(index) } } var filled = ParsedRow(values: values, sourceLineNumber: row.sourceLineNumber) filled.absentColumns = row.absentColumns - filled.expressionColumns = row.expressionColumns + filled.expressionColumns = expressionColumns return filled } @@ -509,7 +527,7 @@ final class RowOperationsManager { return containsComma ? CSVRowParser() : TSVRowParser() } - // Every paste path resets the key to __DEFAULT__ first; an unfilled UUID key then takes a fresh UUID per row. + // Every paste path resets the key to its default first; an unfilled UUID key then takes a fresh UUID per row. private func insertParsedRows( _ parsedRows: [ParsedRow], uuidKeyForms: [Int: UUIDCellForm], @@ -531,11 +549,12 @@ final class RowOperationsManager { var absentColumns = parsedRow.absentColumns var expressionColumns = parsedRow.expressionColumns.filter { $0 < rowValues.count } for index in serverOwned where index < rowValues.count { - rowValues[index] = .text("__DEFAULT__") + rowValues[index] = .columnDefault absentColumns.remove(index) - expressionColumns.remove(index) + expressionColumns.insert(index) } - for (index, form) in uuidKeyForms where index < rowValues.count && rowValues[index] == .text("__DEFAULT__") { + for (index, form) in uuidKeyForms where index < rowValues.count + && rowValues[index].isColumnDefault(isExpression: expressionColumns.contains(index)) { rowValues[index] = Self.freshUUID(form) absentColumns.remove(index) expressionColumns.remove(index) diff --git a/apple/TablePro/Core/Services/Query/RowParser.swift b/apple/TablePro/Core/Services/Query/RowParser.swift index cdd13c09a6..64179f3039 100644 --- a/apple/TablePro/Core/Services/Query/RowParser.swift +++ b/apple/TablePro/Core/Services/Query/RowParser.swift @@ -37,13 +37,17 @@ private func fittedRow( blankColumns.insert(values.count) values.append(nil) } - if let pkIndex = schema.primaryKeyIndex, pkIndex < values.count { - values[pkIndex] = "__DEFAULT__" + var cells = values.map(PluginCellValue.fromOptional) + var defaultColumns: Set = [] + if let pkIndex = schema.primaryKeyIndex, pkIndex < cells.count { + cells[pkIndex] = .columnDefault + defaultColumns.insert(pkIndex) blankColumns.remove(pkIndex) } return ParsedRow( - values: values.map(PluginCellValue.fromOptional), + values: cells, sourceLineNumber: lineNumber, + expressionColumns: defaultColumns, blankColumns: blankColumns ) } diff --git a/apple/TablePro/Core/Services/Query/SelectionSummaryCalculator.swift b/apple/TablePro/Core/Services/Query/SelectionSummaryCalculator.swift index 7470c95bb8..4d031fd612 100644 --- a/apple/TablePro/Core/Services/Query/SelectionSummaryCalculator.swift +++ b/apple/TablePro/Core/Services/Query/SelectionSummaryCalculator.swift @@ -61,11 +61,8 @@ actor SelectionSummaryCalculator { } private struct Tally { - private static let defaultMarkerLength = PluginCellValue.defaultMarkerText.utf8.count - let emptyTextIsEmpty: Bool - let insertedRowIDs: Set - let modifiedCells: [RowID: Set] + let defaultedCells: [RowID: Set] var valueCount = 0 var emptyCount = 0 var notANumberCount = 0 @@ -73,8 +70,7 @@ private struct Tally { init(input: SelectionSummaryInput) { emptyTextIsEmpty = input.policy.emptyTextIsEmpty - insertedRowIDs = input.insertedRowIDs - modifiedCells = input.modifiedCells + defaultedCells = input.defaultedCells } mutating func add(_ value: PluginCellValue, column: Int, rule: SelectionSummaryColumnRule, rowID: RowID) { @@ -108,13 +104,9 @@ private struct Tally { return numbers.add(text.trimmingCharacters(in: .whitespaces)) } - // The default marker is the grid's placeholder only where an edit put it; elsewhere it is text the table holds. private func isEmpty(_ text: String, column: Int, rowID: RowID) -> Bool { if text.isEmpty { return emptyTextIsEmpty } - guard text.utf8.count == Self.defaultMarkerLength, text == PluginCellValue.defaultMarkerText else { - return false - } - return insertedRowIDs.contains(rowID) || modifiedCells[rowID]?.contains(column) == true + return defaultedCells[rowID]?.contains(column) == true } func summary(coversWholeColumn: Bool) -> SelectionSummary { diff --git a/apple/TablePro/Core/Shortcuts/Shared/PayloadRowPlanner.swift b/apple/TablePro/Core/Shortcuts/Shared/PayloadRowPlanner.swift index 47d4e2da17..0065a4e3d0 100644 --- a/apple/TablePro/Core/Shortcuts/Shared/PayloadRowPlanner.swift +++ b/apple/TablePro/Core/Shortcuts/Shared/PayloadRowPlanner.swift @@ -16,17 +16,17 @@ nonisolated enum PayloadRowPlanner { dropsEmptyPrimaryKey: Bool = true ) throws -> [[(column: String, value: String?)]] { guard !columns.isEmpty else { throw IntentDataError.noColumns(table) } - let columnNames = Set(columns.map(\.name)) + let columnNames = columns.map(\.name) let primaryKeys = Set(columns.filter(\.isPrimaryKey).map(\.name)) return try rows.compactMap { row in - let unknown = row.keys.filter { !columnNames.contains($0) } - guard unknown.isEmpty else { throw IntentDataError.unknownColumns(unknown.sorted(), table) } + let matched = keysByColumn(row.keys, columns: columnNames) + guard matched.unknown.isEmpty else { throw IntentDataError.unknownColumns(matched.unknown.sorted(), table) } var values: [(column: String, value: String?)] = [] for column in columns { guard !column.isServerOwned else { continue } - guard let value = row.value(for: column.name) else { continue } + guard let key = matched.byColumn[column.name], let value = row.value(for: key) else { continue } if dropsEmptyPrimaryKey, primaryKeys.contains(column.name), value.isEmptyOrNull { continue } values.append((column: column.name, value: value.sqlValue)) } @@ -34,4 +34,46 @@ nonisolated enum PayloadRowPlanner { return values } } + + // As the import sheet matches a header: its exact spelling claims a column first, then a case-insensitive match. + // There is no sheet to map a header by hand, so SQL Server's `[name]` also finds `name`. + static func keysByColumn(_ keys: [String], columns: [String]) -> (byColumn: [String: String], unknown: [String]) { + var byColumn: [String: String] = [:] + var loose: [String] = [] + let spelled = Set(columns) + for key in keys.sorted() { + if spelled.contains(key) { + byColumn[key] = key + } else { + loose.append(key) + } + } + var unknown: [String] = [] + for key in loose { + let claimed = byColumn + guard let column = matchingColumn(for: key, in: columns, unclaimed: { claimed[$0] == nil }) else { + unknown.append(key) + continue + } + byColumn[column] = key + } + return (byColumn, unknown) + } + + private static func matchingColumn(for key: String, in columns: [String], unclaimed: (String) -> Bool) -> String? { + for name in [key, unbracketed(key)].compactMap({ $0 }) { + if columns.contains(name), unclaimed(name) { return name } + let folded = columns.filter { $0.caseInsensitiveCompare(name) == .orderedSame && unclaimed($0) } + if folded.count == 1 { return folded[0] } + } + return nil + } + + // `]` inside the brackets is doubled, so a lone one means the key is not a bracketed name. + private static func unbracketed(_ key: String) -> String? { + guard key.count > 2, key.hasPrefix("["), key.hasSuffix("]") else { return nil } + let inner = String(key.dropFirst().dropLast()) + guard !inner.replacingOccurrences(of: "]]", with: "", options: .literal).contains("]") else { return nil } + return inner.replacingOccurrences(of: "]]", with: "]", options: .literal) + } } diff --git a/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift b/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift index 99fd7d2c56..95736a027e 100644 --- a/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift +++ b/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift @@ -11,15 +11,29 @@ nonisolated enum RowPayload { let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { throw IntentDataError.emptyPayload } - let rows = trimmed.hasPrefix("{") || trimmed.hasPrefix("[") - ? try parseJSON(trimmed) - : try parseCSV(raw) + let rows = try sniffedRows(raw: raw, trimmed: trimmed) guard !rows.isEmpty else { throw IntentDataError.emptyPayload } guard rows.count <= maxRows else { throw IntentDataError.tooManyRows(maxRows) } return rows } + // A CSV header can open with `[` or `{`, as SQL Server's `[id],[name]` does, so only text that parses as JSON is + // JSON. Text that is neither keeps the JSON error, which names what broke in a payload meant as JSON. So does JSON + // that still splits into even CSV records, one object per line with a trailing comma: its headers hold `":`. + private static func sniffedRows(raw: String, trimmed: String) throws -> [PayloadRow] { + guard trimmed.hasPrefix("{") || trimmed.hasPrefix("[") else { return try parseCSV(raw) } + do { + return try parseJSON(trimmed) + } catch IntentDataError.malformedPayload(let reason) { + guard let rows = try? parseCSV(raw), !rows.isEmpty, + !rows.contains(where: { row in row.keys.contains { $0.contains("\":") } }) else { + throw IntentDataError.malformedPayload(reason) + } + return rows + } + } + static func parseSingle(data: String?, file: IntentFile?) async throws -> [PayloadRow] { let rows = try await parse(data: data, file: file) guard rows.count == 1 else { throw IntentDataError.expectedSingleRow } diff --git a/apple/TablePro/Core/Shortcuts/ShortcutsRowWriter.swift b/apple/TablePro/Core/Shortcuts/ShortcutsRowWriter.swift index 1baee5a518..4336e6c51f 100644 --- a/apple/TablePro/Core/Shortcuts/ShortcutsRowWriter.swift +++ b/apple/TablePro/Core/Shortcuts/ShortcutsRowWriter.swift @@ -157,10 +157,11 @@ internal enum ShortcutsRowWriter { let rowID = RowID.inserted(UUID()) let byColumn = Dictionary(values.map { ($0.column, $0.value) }, uniquingKeysWith: { first, _ in first }) insertedRowData[rowID] = names.map { name in - guard let value = byColumn[name] else { return .text(PluginCellValue.defaultMarkerText) } + guard let value = byColumn[name] else { return .columnDefault } return value.map(PluginCellValue.text) ?? .null } - changes.append(RowChange(rowID: rowID, type: .insert, sequence: index)) + let defaulted = Set(names.indices.filter { byColumn[names[$0]] == nil }) + changes.append(RowChange(rowID: rowID, type: .insert, expressionColumns: defaulted, sequence: index)) } let factory = RowChangeStatementFactory( diff --git a/apple/TablePro/Core/UsersRoles/PrincipalChangeManager.swift b/apple/TablePro/Core/UsersRoles/PrincipalChangeManager.swift index 1a7be6130b..578cfc531f 100644 --- a/apple/TablePro/Core/UsersRoles/PrincipalChangeManager.swift +++ b/apple/TablePro/Core/UsersRoles/PrincipalChangeManager.swift @@ -432,14 +432,20 @@ final class PrincipalChangeManager: ObservableObject { // MARK: - Output + // A staged drop keeps the principal's other edits so undoing the drop brings them back, but only the DROP runs. func pendingChanges() -> [PrincipalChange] { var changes: [PrincipalChange] = pendingCreates.map { .create($0) } changes += pendingAlters.compactMap { ref, definition in - guard let original = principals.first(where: { $0.ref == ref }) else { return nil } + guard pendingDrops[ref] == nil, + let original = principals.first(where: { $0.ref == ref }) + else { return nil } return .alter(old: Self.definition(from: original), new: definition) } - changes += pendingPasswords.map { .setPassword(ref: $0.key, password: $0.value) } + changes += pendingPasswords.compactMap { ref, password in + guard pendingDrops[ref] == nil else { return nil } + return .setPassword(ref: ref, password: password) + } changes += grantChangeSets().map { .modifyGrants($0) } changes += pendingDrops.map { .drop(ref: $0.key, options: $0.value) } return changes @@ -531,11 +537,12 @@ final class PrincipalChangeManager: ObservableObject { } private func recomputeChangeCount() { - let grantChanges = grantDeltas.values.reduce(0) { $0 + $1.count } + let isLive = { (ref: PluginPrincipalRef) in self.pendingDrops[ref] == nil } + let grantChanges = grantDeltas.reduce(0) { isLive($1.key) ? $0 + $1.value.count : $0 } changeCount = pendingCreates.count + pendingDrops.count - + pendingPasswords.count - + pendingAlters.count + + pendingPasswords.keys.count(where: isLive) + + pendingAlters.keys.count(where: isLive) + grantChanges } diff --git a/apple/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift b/apple/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift index 5161eb1e38..a805e5be70 100644 --- a/apple/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift +++ b/apple/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift @@ -189,11 +189,11 @@ enum QueryStatementScanner { } } - private static func located(_ command: RedisCommandSplitter.Command) -> SQLStatementScanner.LocatedStatement { + fileprivate static func located(_ command: RedisCommandSplitter.Command) -> SQLStatementScanner.LocatedStatement { SQLStatementScanner.LocatedStatement(wholeText: command.text, offset: command.range.location) } - private static func located( + fileprivate static func located( _ statement: JavaScriptStatementScanner.Statement ) -> SQLStatementScanner.LocatedStatement { SQLStatementScanner.LocatedStatement( @@ -203,3 +203,51 @@ enum QueryStatementScanner { ) } } + +// A JavaScript or Redis document's statements, read once per text, so a caret move finds its statement by search and +// answers what `QueryStatementScanner.locatedStatementAtCursor` would. A SQL document uses `SQLStatementCursorIndex`. +struct QueryStatementCaretTable { + private let model: QueryStatementModel + private let statements: [SQLStatementScanner.LocatedStatement] + // The last caret offset each statement owns, in document order. + private let reaches: [Int] + + init?(text: String, model: QueryStatementModel) { + self.model = model + switch model { + case .sql: + return nil + case .javascript: + let found = JavaScriptStatementScanner.executableStatements(in: text) + statements = found.map(QueryStatementScanner.located) + reaches = found.map { NSMaxRange($0.range) } + case .redis: + let found = RedisCommandSplitter.commandsWithCaretReach(in: text) + statements = found.map { QueryStatementScanner.located($0.command) } + reaches = found.map { $0.reach } + } + } + + func statement(atCursor cursor: Int) -> SQLStatementScanner.LocatedStatement { + var low = 0 + var high = reaches.count + while low < high { + let middle = (low + high) / 2 + if reaches[middle] < cursor { + low = middle + 1 + } else { + high = middle + } + } + let nothing = SQLStatementScanner.LocatedStatement(sql: "", offset: 0, hasContent: false) + switch model { + case .redis: + return low < statements.count ? statements[low] : nothing + case .javascript, .sql: + // Off every statement, the caret takes the one before it, or the first. + guard !statements.isEmpty else { return nothing } + if low < statements.count, statements[low].offset <= cursor { return statements[low] } + return statements[max(0, low - 1)] + } + } +} diff --git a/apple/TablePro/Core/Utilities/SQL/DisplayedResultReader.swift b/apple/TablePro/Core/Utilities/SQL/DisplayedResultReader.swift index 869817eb65..a9463a1cb4 100644 --- a/apple/TablePro/Core/Utilities/SQL/DisplayedResultReader.swift +++ b/apple/TablePro/Core/Utilities/SQL/DisplayedResultReader.swift @@ -15,12 +15,14 @@ internal enum DisplayedResultReader { internal let skippedDeletedCount: Int } - // Empty selectedDisplayIndices means every displayed row. + // Empty selectedDisplayIndices means every displayed row. A cell in defaultedCells has no value until the server + // gives it one, so it reads as NULL. internal static func read( tableRows: TableRows, displayIDs: [RowID]?, selectedDisplayIndices: Set, deletedRowIDs: Set = [], + defaultedCells: [RowID: Set] = [:], columns projection: VisibleColumnProjection ) -> Output { let positions: [Int] @@ -38,7 +40,7 @@ internal enum DisplayedResultReader { skippedDeleted += 1 return nil } - return projection.values(Self.values(of: row)) + return projection.values(Self.values(of: row, defaulted: defaultedCells[row.id] ?? [])) } return Output( @@ -50,12 +52,15 @@ internal enum DisplayedResultReader { } // A value the page left on the server says so, rather than reading as NULL. - private static func values(of row: Row) -> [PluginCellValue] { - guard row.hasUnloadedValues else { return Array(row.values) } + private static func values(of row: Row, defaulted: Set) -> [PluginCellValue] { + guard row.hasUnloadedValues || !defaulted.isEmpty else { return Array(row.values) } var values = Array(row.values) for (column, byteCount) in row.unloadedColumns where values.indices.contains(column) { values[column] = .text(UnloadedValueText.notLoaded(byteCount: byteCount)) } + for column in defaulted where values.indices.contains(column) { + values[column] = .null + } return values } } diff --git a/apple/TablePro/Core/Utilities/SQL/JsonRowConverter.swift b/apple/TablePro/Core/Utilities/SQL/JsonRowConverter.swift index efc41e9320..5cb6637e19 100644 --- a/apple/TablePro/Core/Utilities/SQL/JsonRowConverter.swift +++ b/apple/TablePro/Core/Utilities/SQL/JsonRowConverter.swift @@ -40,7 +40,7 @@ internal struct JsonRowConverter { } let cell = row[colIdx] - if cell.isNull || cell.isDefaultMarker { + if cell.isNull { result.append("null") appendPropertySuffix(to: &result, colIdx: colIdx) continue diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift index c326a505df..19d3dc08ff 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -102,16 +102,21 @@ enum QueryClassifier { isMultiStatement(sql, databaseType: databaseType, readings: databaseType.lexicalReadings) } - /// Whether `sql` runs more than one statement: it holds several, a `GO n` line runs its one statement n times, or it - /// ends inside a block it opened, which an engine that reads the opener as a name runs as several statements. A - /// caller that sends one statement once cannot honour the count, and dropping it runs the statement once. + /// Whether `sql` runs more than one statement: it holds several, a `GO n` line runs its one statement n times, it + /// ends inside a block it opened, which an engine that reads the opener as a name runs as several statements, or a + /// T-SQL block in it holds several. A caller that sends one statement once cannot honour the count, and dropping it + /// runs the statement once. static func isMultiStatement(_ sql: String, databaseType: DatabaseType, readings: SQLLexicalReadings) -> Bool { let model = QueryStatementModel.forDatabaseType(databaseType) guard let lexings = readings.lexings(of: sql) else { return true } return lexings.contains { lexing in let batches = QueryBatchPlanner.batches(in: lexing.text, model: model, grammar: lexing.grammar) - return batches.flatMap(\.statements).count > 1 || batches.contains { $0.repeatCount > 1 } + let statements = batches.flatMap(\.statements) + return statements.count > 1 || batches.contains { $0.repeatCount > 1 } || SQLStatementScanner.endsInsideOpenBlock(lexing.text, grammar: lexing.grammar) + || statements.contains { statement in + SQLUnterminatedStatements.blockHoldsSeveralStatements(statement.sql, grammar: lexing.grammar) + } } } diff --git a/apple/TablePro/Core/Utilities/SQL/ResultJsonSerializer.swift b/apple/TablePro/Core/Utilities/SQL/ResultJsonSerializer.swift index c2f9ead605..1bb897caff 100644 --- a/apple/TablePro/Core/Utilities/SQL/ResultJsonSerializer.swift +++ b/apple/TablePro/Core/Utilities/SQL/ResultJsonSerializer.swift @@ -36,6 +36,7 @@ internal enum ResultJsonSerializer { displayIDs: [RowID]?, selectedDisplayIndices: Set, deletedRowIDs: Set = [], + defaultedCells: [RowID: Set] = [:], columns projection: VisibleColumnProjection ) -> Output { let read = DisplayedResultReader.read( @@ -43,6 +44,7 @@ internal enum ResultJsonSerializer { displayIDs: displayIDs, selectedDisplayIndices: selectedDisplayIndices, deletedRowIDs: deletedRowIDs, + defaultedCells: defaultedCells, columns: projection ) diff --git a/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift b/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift index 4eac9054f4..3c763effc6 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift @@ -228,18 +228,6 @@ internal struct SQLRowToStatementConverter { } private func formatBinaryLiteral(_ data: Data) -> String { - var hex = "" - hex.reserveCapacity(data.count * 2) - for byte in data { - hex += String(format: "%02X", byte) - } - switch databaseType { - case .postgresql, .redshift, .cockroachdb, .pglite, .kingbasees: - return "'\\x\(hex)'::bytea" - case .mssql: - return "0x\(hex)" - default: - return "X'\(hex)'" - } + CompareSQLLiteral.binaryLiteral(for: data, databaseType: databaseType) ?? "X'\(data.uppercaseHex())'" } } diff --git a/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift b/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift index 4d012d8074..9d8bde2d77 100644 --- a/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift +++ b/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift @@ -51,65 +51,6 @@ final class VimTextBufferAdapter: VimTextBuffer { cachedLineCount = nil } - /// Incrementally update the cached line count based on text change delta. - /// Avoids a full O(n) recount on every keystroke. - func textDidChange(in range: NSRange, replacementLength: Int) { - guard let textView else { - cachedLineCount = nil - return - } - - guard let cached = cachedLineCount else { return } - - let nsString = textView.string as NSString - - // Pure insertion: count newlines in the new text and apply delta - if range.length == 0 { - var addedNewlines = 0 - let end = range.location + replacementLength - if replacementLength > 0 && end <= nsString.length { - for i in range.location.. 0 && range.location + range.length <= oldNs.length { - let end = range.location + range.length - for i in range.location.. 0 && replacementEnd <= nsString.length { - for i in range.location.. NSRange { guard let textView else { return NSRange(location: 0, length: 0) } let nsString = textView.string as NSString diff --git a/apple/TablePro/Extensions/PluginCellValue+DefaultMarker.swift b/apple/TablePro/Extensions/PluginCellValue+DefaultMarker.swift index 74b11c7a5b..63f9e24d11 100644 --- a/apple/TablePro/Extensions/PluginCellValue+DefaultMarker.swift +++ b/apple/TablePro/Extensions/PluginCellValue+DefaultMarker.swift @@ -7,16 +7,15 @@ import Foundation import TableProPluginKit internal extension PluginCellValue { - /// The sentinel a pending inserted row carries for a column the server fills in, written by - /// `RowOperationsManager` and stripped again by `SQLStatementGenerator`. - /// - /// It is an internal marker, never a value, so every view that renders a cell has to recognise - /// it. The grid shows it as a placeholder; anything that renders the same rows some other way - /// has to agree, or the marker leaks out as literal text. - static let defaultMarkerText = "__DEFAULT__" + // A cell left to the column's default is the SQL keyword, marked as SQL beside the value like any expression. The + // mark decides, never the text: a stored or typed value can read the same. + static let columnDefaultKeyword = "DEFAULT" + static let columnDefault = PluginCellValue.text(columnDefaultKeyword) - var isDefaultMarker: Bool { - guard case .text(let value) = self else { return false } - return value == Self.defaultMarkerText + // The spelling driver plugins read as the default in the rows the host hands them. + static let pluginDefaultMarker = PluginCellValue.text("__DEFAULT__") + + func isColumnDefault(isExpression: Bool) -> Bool { + isExpression && self == Self.columnDefault } } diff --git a/apple/TablePro/Models/ClickHouse/ClickHousePartInfo.swift b/apple/TablePro/Models/ClickHouse/ClickHousePartInfo.swift index baa347a335..afe2544e04 100644 --- a/apple/TablePro/Models/ClickHouse/ClickHousePartInfo.swift +++ b/apple/TablePro/Models/ClickHouse/ClickHousePartInfo.swift @@ -2,15 +2,13 @@ // ClickHousePartInfo.swift // TablePro // -// Model for ClickHouse partition/part information from system.parts. -// import Foundation -/// Represents a single part from the ClickHouse system.parts table struct ClickHousePartInfo: Identifiable { let id = UUID() let partition: String + let partitionId: String let name: String let rows: UInt64 let bytesOnDisk: UInt64 diff --git a/apple/TablePro/Models/Query/SelectionSummary.swift b/apple/TablePro/Models/Query/SelectionSummary.swift index af5da02cb1..923812aa44 100644 --- a/apple/TablePro/Models/Query/SelectionSummary.swift +++ b/apple/TablePro/Models/Query/SelectionSummary.swift @@ -32,8 +32,8 @@ struct SelectionSummaryInput: Sendable { let dataColumnsByDisplayPosition: [Int] let policy: SelectionSummaryColumnPolicy let deletedRowIDs: Set - let insertedRowIDs: Set - let modifiedCells: [RowID: Set] + // The cells a pending edit set to the column's default, which hold the keyword as text. + let defaultedCells: [RowID: Set] init( selection: GridSelection, @@ -42,8 +42,7 @@ struct SelectionSummaryInput: Sendable { dataColumnsByDisplayPosition: [Int], policy: SelectionSummaryColumnPolicy, deletedRowIDs: Set = [], - insertedRowIDs: Set = [], - modifiedCells: [RowID: Set] = [:] + defaultedCells: [RowID: Set] = [:] ) { self.selection = selection self.tableRows = tableRows @@ -51,8 +50,7 @@ struct SelectionSummaryInput: Sendable { self.dataColumnsByDisplayPosition = dataColumnsByDisplayPosition self.policy = policy self.deletedRowIDs = deletedRowIDs - self.insertedRowIDs = insertedRowIDs - self.modifiedCells = modifiedCells + self.defaultedCells = defaultedCells } } diff --git a/apple/TablePro/Models/Schema/ClickHousePartStatements.swift b/apple/TablePro/Models/Schema/ClickHousePartStatements.swift index 5436cfced7..c11617c1c6 100644 --- a/apple/TablePro/Models/Schema/ClickHousePartStatements.swift +++ b/apple/TablePro/Models/Schema/ClickHousePartStatements.swift @@ -2,15 +2,10 @@ // ClickHousePartStatements.swift // TablePro // -// The statements the Parts tab issues, built from the tab's own container. -// import Foundation -/// ClickHouse carries the database as a request parameter rather than in the session, and the app -/// moves that parameter whenever any tab runs somewhere else. An unqualified name therefore names -/// whichever database the connection was last pinned to, which for `DROP PARTITION` is data loss in -/// a database the user was not looking at. +// ClickHouse takes the database from a request parameter any tab can move, so every statement names its own. internal enum ClickHousePartStatements { internal static func qualifiedName( database: String, @@ -32,42 +27,49 @@ internal enum ClickHousePartStatements { internal static func dropPartition( database: String, table: String, - partition: String, - quote: (String) -> String, - escape: (String) -> String - ) -> String { + partitionId: String, + quote: (String) -> String + ) -> String? { + guard isValidPartitionId(partitionId) else { return nil } let name = qualifiedName(database: database, table: table, quote: quote) - return "ALTER TABLE \(name) DROP PARTITION '\(escape(partition))'" + return "ALTER TABLE \(name) DROP PARTITION ID '\(partitionId)'" } internal static func detachPartition( database: String, table: String, - partition: String, - quote: (String) -> String, - escape: (String) -> String - ) -> String { + partitionId: String, + quote: (String) -> String + ) -> String? { + guard isValidPartitionId(partitionId) else { return nil } let name = qualifiedName(database: database, table: table, quote: quote) - return "ALTER TABLE \(name) DETACH PARTITION '\(escape(partition))'" + return "ALTER TABLE \(name) DETACH PARTITION ID '\(partitionId)'" + } + + // The server's own rule (MergeTreePartInfo::validatePartitionID); it also keeps quotes out of the literal. + internal static func isValidPartitionId(_ id: String) -> Bool { + !id.isEmpty && id.unicodeScalars.allSatisfy { scalar in + switch scalar { + case "0"..."9", "a"..."z", "A"..."Z", "-": true + default: false + } + } } - /// A connection saved with no database of its own has none to name, and ClickHouse then resolves - /// an unqualified request against its own configured default, which `currentDatabase()` reports. - /// Comparing against an empty string instead matches nothing and empties the tab. + // With no database of its own, ClickHouse resolves a request against its configured default. private static func databasePredicate(_ database: String, escape: (String) -> String) -> String { guard !database.isEmpty else { return "currentDatabase()" } return "'\(escape(database))'" } - /// Filtered on the named database rather than `currentDatabase()`, which answers with the - /// request parameter and so listed another database's parts under this table's name. + // `partition` renders a tuple key as `(202401,'eu')` and no key as `tuple()`, so statements name `partition_id`. internal static func parts( database: String, table: String, escape: (String) -> String ) -> String { """ - SELECT partition, name, rows, bytes_on_disk, + SELECT partition, partition_id, name, rows, bytes_on_disk, toString(modification_time) AS mod_time, active FROM system.parts WHERE database = \(databasePredicate(database, escape: escape)) AND table = '\(escape(table))' diff --git a/apple/TablePro/Models/UI/MultiRowEditState.swift b/apple/TablePro/Models/UI/MultiRowEditState.swift index 26c98ce2f2..a9c0f4ae65 100644 --- a/apple/TablePro/Models/UI/MultiRowEditState.swift +++ b/apple/TablePro/Models/UI/MultiRowEditState.swift @@ -70,7 +70,7 @@ struct FieldEditState: Identifiable { var effectiveValue: String? { if isPendingDefault { - return "__DEFAULT__" + return PluginCellValue.columnDefaultKeyword } else if isPendingNull { return nil } else { @@ -244,6 +244,8 @@ final class MultiRowEditState: ObservableObject { hasCommittedEdit = true } else if originalValue == nil { isPendingNull = true + } else if expression.first == true, originalValue == PluginCellValue.columnDefaultKeyword { + isPendingDefault = true } else { pendingValue = originalValue isPendingExpression = expression.first == true @@ -508,7 +510,7 @@ final class MultiRowEditState: ObservableObject { guard index < fields.count else { return } resetPending(at: index) fields[index].isPendingDefault = true - onFieldChanged?(index, .text("__DEFAULT__"), .discrete) + onFieldExpressionChanged?(index, PluginCellValue.columnDefaultKeyword, .discrete) } func setFieldToExpression(at index: Int, expression: CellExpression) { @@ -619,7 +621,7 @@ final class MultiRowEditState: ObservableObject { columnName: field.columnName, newValue: field.effectiveValue, removesField: field.isPendingRemoval, - isExpression: field.isPendingExpression + isExpression: field.isPendingExpression || field.isPendingDefault ) } } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 4ba0fefbde..245084252f 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -55039,6 +55039,9 @@ } } } + }, + "Couldn't Detach Partition" : { + }, "Couldn't Discard Changes" : { "localizations" : { @@ -55052,6 +55055,9 @@ }, "Couldn't download cloud-sql-proxy. The server answered with HTTP status %lld." : { + }, + "Couldn't Drop Partition" : { + }, "Couldn't Duplicate the Theme" : { @@ -55119,6 +55125,9 @@ }, "Couldn't Move the Theme to the Trash" : { + }, + "Couldn't Optimize Table" : { + }, "Couldn't publish to the team catalog" : { "localizations" : { @@ -55254,6 +55263,9 @@ }, "Couldn't Read the DDL" : { + }, + "Couldn't Read the Parts" : { + }, "Couldn't read the referenced table" : { "localizations" : { @@ -120449,6 +120461,10 @@ } } }, + "No row has this primary key, so nothing was updated. Another client may have deleted it." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "No row holds the values it was loaded with. Another client may have changed or deleted it." : { }, @@ -145185,6 +145201,12 @@ "Reconnecting (%lld of %lld)" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Reconnecting could not reopen the database %1$@: %2$@" : { + + }, + "Reconnecting could not reopen the schema %1$@: %2$@" : { + }, "Reconnecting failed after %d attempts." : { "localizations" : { @@ -174217,6 +174239,9 @@ } } } + }, + "Startup commands failed, so this session runs without them:\n%@" : { + }, "Startup SQL" : { "comment" : "A section header for the startup SQL commands of a connection.", @@ -193166,6 +193191,9 @@ } } } + }, + "The server reported a partition ID that cannot be used in a statement, so nothing was run." : { + }, "The server reported an error." : { "localizations" : { @@ -194368,6 +194396,12 @@ } } } + }, + "The startup command %@ failed, so this session runs without it." : { + + }, + "The startup command %1$@ failed (%2$@), so this session runs without it." : { + }, "The statement before it stays applied." : { "localizations" : { @@ -198452,6 +198486,9 @@ }, "This database's driver cannot write SQL into %@. Update the driver." : { + }, + "This database's driver may read the text __DEFAULT__ in %@ as the column default, so it cannot be saved as text." : { + }, "This deletes only the queries listed here, across every connection. Queries the source, date, outcome and search filters are hiding stay. You cannot undo this." : { "comment" : "Message for the \"Delete this connection's query history\" button.", @@ -213159,6 +213196,10 @@ } } }, + "Values a spreadsheet would read as formulas, written with a leading single quote: %lld. Turn off Sanitize formula-like values to export them unchanged." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Values from %d loaded rows" : { "extractionState" : "stale", "localizations" : { diff --git a/apple/TablePro/ViewModels/SchemaEditorViewModel.swift b/apple/TablePro/ViewModels/SchemaEditorViewModel.swift index 3d86e3b1fe..5c20599902 100644 --- a/apple/TablePro/ViewModels/SchemaEditorViewModel.swift +++ b/apple/TablePro/ViewModels/SchemaEditorViewModel.swift @@ -330,7 +330,7 @@ final class SchemaEditorViewModel: ObservableObject { renameCommitted: plan.renamesFirst && committedFirst ) } catch { - failure = error.localizedDescription + failure = GatedOperationFailure(error).message return ApplyOutcome( succeeded: false, committedName: originalName ?? "", renameCommitted: false ) diff --git a/apple/TablePro/ViewModels/UserTypeEditorModel.swift b/apple/TablePro/ViewModels/UserTypeEditorModel.swift index 3d6bc764f8..ae428d77f5 100644 --- a/apple/TablePro/ViewModels/UserTypeEditorModel.swift +++ b/apple/TablePro/ViewModels/UserTypeEditorModel.swift @@ -302,7 +302,7 @@ final class UserTypeEditorModel: ObservableObject { do { outcome = try await UserTypeChangeRunner.run(plan, in: scope, connection: connection, gate: gate) } catch { - let failure = StructureApplyFailure(error) + let failure = GatedOperationFailure(error) if let message = failure.message { AlertHelper.showErrorSheet( title: String(localized: "Error Applying Changes"), diff --git a/apple/TablePro/ViewModels/UsersRolesViewModel+Changes.swift b/apple/TablePro/ViewModels/UsersRolesViewModel+Changes.swift index d54a6fc8bc..162da203eb 100644 --- a/apple/TablePro/ViewModels/UsersRolesViewModel+Changes.swift +++ b/apple/TablePro/ViewModels/UsersRolesViewModel+Changes.swift @@ -290,7 +290,7 @@ extension UsersRolesViewModel { .joined(separator: "\n") await load(forceReload: true) } catch { - applyFailure = error.localizedDescription + applyFailure = GatedOperationFailure(error).message } } diff --git a/apple/TablePro/Views/Compare/CompareRowDiffPane.swift b/apple/TablePro/Views/Compare/CompareRowDiffPane.swift index 542c2d52e9..f80cb6f1c3 100644 --- a/apple/TablePro/Views/Compare/CompareRowDiffPane.swift +++ b/apple/TablePro/Views/Compare/CompareRowDiffPane.swift @@ -208,12 +208,27 @@ internal struct CompareRowDiffPane: View { ) } + // Named by side, because the two tables usually share a name and only one database cut its stream. + private func rowCutNotices(for summary: DataDiffSummary, plan: DataComparePlan) -> [String] { + let sides: [(endpoint: DatabaseEndpoint?, cut: Int?)] = [ + (session.source, summary.sourceRowCut), (session.target, summary.targetRowCut) + ] + return sides.compactMap { side in + guard let cut = side.cut else { return nil } + let table = ([side.endpoint?.qualifiedDescription].compactMap { $0 } + [plan.table]).joined(separator: " / ") + return StreamRowCutWarning.text(table: table, limit: cut) + } + } + @ViewBuilder private func notices(for plan: DataComparePlan) -> some View { if let failure = plan.comparisonFailure { notice(failure, systemImage: "exclamationmark.octagon.fill") } if let summary = plan.summary { + ForEach(rowCutNotices(for: summary, plan: plan), id: \.self) { text in + notice(text, systemImage: "exclamationmark.triangle.fill") + } if summary.stoppedAtRowLimit { notice(rowLimitNotice(for: summary), systemImage: "info.circle.fill") } diff --git a/apple/TablePro/Views/DataGrid/Cells/DataGridCellContent.swift b/apple/TablePro/Views/DataGrid/Cells/DataGridCellContent.swift index b32be5c107..1f918e38b5 100644 --- a/apple/TablePro/Views/DataGrid/Cells/DataGridCellContent.swift +++ b/apple/TablePro/Views/DataGrid/Cells/DataGridCellContent.swift @@ -23,16 +23,17 @@ struct DataGridCellContent { static func placeholder( for rawValue: PluginCellValue, isAbsent: Bool = false, + isExpression: Bool = false, unloadedByteCount: Int? = nil, marksEmptyText: Bool = true ) -> DataGridCellPlaceholder? { if let unloadedByteCount { return .unloaded(byteCount: unloadedByteCount) } guard !isAbsent else { return .absent } + if rawValue.isColumnDefault(isExpression: isExpression) { return .defaultMarker } switch rawValue { case .null: return .null case .text(let value): - if value == PluginCellValue.defaultMarkerText { return .defaultMarker } return value.isEmpty && marksEmptyText ? .empty : nil case .bytes: return nil diff --git a/apple/TablePro/Views/DataGrid/Cells/DataGridCellPresenter.swift b/apple/TablePro/Views/DataGrid/Cells/DataGridCellPresenter.swift index 8f7745aac0..651da4002b 100644 --- a/apple/TablePro/Views/DataGrid/Cells/DataGridCellPresenter.swift +++ b/apple/TablePro/Views/DataGrid/Cells/DataGridCellPresenter.swift @@ -250,6 +250,7 @@ final class DataGridCellPresenter { placeholder: DataGridCellContent.placeholder( for: rawValue, isAbsent: row.isAbsent(dataIndex), + isExpression: visualState.isExpression(columnIndex: dataIndex), unloadedByteCount: row.unloadedByteCount(dataIndex), marksEmptyText: pass.marksEmptyText ) @@ -297,6 +298,7 @@ final class DataGridCellPresenter { switch DataGridCellContent.placeholder( for: rawValue, isAbsent: row.isAbsent(dataIndex), + isExpression: model.changeState?.isExpression(rowID: row.id, columnIndex: dataIndex) ?? false, unloadedByteCount: row.unloadedByteCount(dataIndex), marksEmptyText: model.configuration.marksEmptyText ) { diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift index e27584da1e..fd99df077b 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift @@ -133,6 +133,7 @@ extension DataGridCommands { tableRows: model.rows, displayIDs: model.order.filteredIDs, selectedDisplayIndices: displayIndices, + defaultedCells: model.changeState?.defaultedCells(in: model.rows) ?? [:], columns: projection ) guard output.rowCount > 0 else { return } diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+SetValue.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+SetValue.swift index cc2d571948..cdfa6e7fa8 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+SetValue.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+SetValue.swift @@ -78,7 +78,7 @@ extension DataGridCommands { private static func sampleText(ofColumn dataIndex: Int, in rows: TableRows) -> String? { for row in rows.rows where dataIndex < row.values.count { guard let text = row.values[dataIndex].asText, - !text.isEmpty, text != PluginCellValue.defaultMarkerText else { continue } + !text.isEmpty, row.values[dataIndex] != .columnDefault else { continue } return text } return nil @@ -91,7 +91,7 @@ extension DataGridCommands { case .null: return (value: .null, isExpression: false) case .defaultValue: - return (value: .text(PluginCellValue.defaultMarkerText), isExpression: false) + return (value: .columnDefault, isExpression: true) case .expression(let expression): return (value: .text(expression.sql), isExpression: true) case .uuid(let version): diff --git a/apple/TablePro/Views/DataGrid/DataGridViewController.swift b/apple/TablePro/Views/DataGrid/DataGridViewController.swift index dd7176f7af..a3b65c5666 100644 --- a/apple/TablePro/Views/DataGrid/DataGridViewController.swift +++ b/apple/TablePro/Views/DataGrid/DataGridViewController.swift @@ -388,8 +388,7 @@ final class DataGridViewController: NSViewController, DataGridMountedView { dataColumnsByDisplayPosition: model.columns.columns.map(\.dataIndex), policy: policy, deletedRowIDs: changeState?.deletedRowIDs ?? [], - insertedRowIDs: changeState?.insertedRowIDs ?? [], - modifiedCells: changeState?.modifiedCells ?? [:] + defaultedCells: changeState?.defaultedCells(in: rows) ?? [:] ) } } diff --git a/apple/TablePro/Views/DatabaseSwitcher/CreateDatabaseSheet.swift b/apple/TablePro/Views/DatabaseSwitcher/CreateDatabaseSheet.swift index 7cdc8a564d..1b6314b1ba 100644 --- a/apple/TablePro/Views/DatabaseSwitcher/CreateDatabaseSheet.swift +++ b/apple/TablePro/Views/DatabaseSwitcher/CreateDatabaseSheet.swift @@ -176,7 +176,7 @@ struct CreateDatabaseSheet: View { onCreated?(name) dismiss() } catch { - errorMessage = error.localizedDescription + errorMessage = GatedOperationFailure(error).message isCreating = false } } diff --git a/apple/TablePro/Views/Editor/StatementRunController.swift b/apple/TablePro/Views/Editor/StatementRunController.swift index 7909dcd81a..b7b0acef3c 100644 --- a/apple/TablePro/Views/Editor/StatementRunController.swift +++ b/apple/TablePro/Views/Editor/StatementRunController.swift @@ -17,7 +17,8 @@ enum StatementNavigationDirection { // The gutter's run controls and the caret's statement band both come from the scanner execution uses, so a control // never offers to run something other than what it is drawn beside. The band runs on every caret move, so a SQL -// document resolves it through `SQLStatementCursorIndex`, which rescans only from the last boundary before the caret. +// document resolves it through `SQLStatementCursorIndex`, which rescans only from the last boundary before the caret, +// and a JavaScript or Redis document through a `QueryStatementCaretTable` read once per text. // The controls need every statement, a full pass of 31ms at the size limit, so that refresh is debounced. @MainActor final class StatementRunController { @@ -44,11 +45,17 @@ final class StatementRunController { /// How the document divides into statements. JavaScript engines do not end a statement at every /// semicolon, so the gutter controls and the caret band have to ask the same splitter execution does. var statementModel: QueryStatementModel = .sql { - didSet { if statementModel != oldValue { caretIndex = nil } } + didSet { + if statementModel != oldValue { + caretIndex = nil + caretTable = nil + } + } } - // The band asks on every caret move, so a SQL document keeps its statement boundaries between edits. + // The band asks on every caret move, so a document keeps its statement boundaries between edits. private var caretIndex: SQLStatementCursorIndex? + private var caretTable: QueryStatementCaretTable? private weak var caretIndexStorage: NSTextStorage? private var caretIndexEdits: StorageEditObservation? @@ -56,6 +63,8 @@ final class StatementRunController { caretIndex?.unitsScanned ?? 0 } + private(set) var caretTableReads = 0 + /// Called with the SQL the pressed control stands for, read from the document at the moment of the press, and /// where that SQL starts in the document. /// @@ -213,6 +222,8 @@ final class StatementRunController { pendingRefresh?.cancel() pendingRefresh = nil caretIndex = nil + caretTable = nil + caretIndexStorage = nil caretIndexEdits = nil controller?.runnableStatements = [] controller?.highlightedStatementRange = nil @@ -267,7 +278,7 @@ final class StatementRunController { guard (text as NSString).length <= sizeLimit else { return nil } let statement: SQLStatementScanner.LocatedStatement - if statementModel == .sql, let storage = textView.textStorage { + if let storage = textView.textStorage { statement = indexedStatement(at: selection.range.location, in: text, storage: storage) } else { statement = QueryStatementScanner.locatedStatementAtCursor( @@ -286,19 +297,38 @@ final class StatementRunController { in text: String, storage: NSTextStorage ) -> SQLStatementScanner.LocatedStatement { - if caretIndex == nil || caretIndexStorage !== storage { - startCaretIndex(on: storage) + if caretIndexStorage !== storage { + observeEdits(on: storage) } // Inside an editing batch the text has changed but the storage has not posted its notification yet. if storage.editedMask.contains(.editedCharacters) { - caretIndex?.textDidChange(at: storage.editedRange.location) + textDidChange(at: storage.editedRange.location) + } + guard statementModel == .sql else { + if caretTable == nil { + caretTable = QueryStatementCaretTable(text: text, model: statementModel) + caretTableReads += 1 + } + return caretTable?.statement(atCursor: cursor) + ?? QueryStatementScanner.locatedStatementAtCursor( + in: text, cursorPosition: cursor, model: statementModel, grammar: grammar + ) + } + if caretIndex == nil { + caretIndex = SQLStatementCursorIndex(grammar: grammar) } return caretIndex?.statementAtCursor(cursor, in: text) ?? SQLStatementScanner.locatedStatementAtCursor(in: text, cursorPosition: cursor, grammar: grammar) } - private func startCaretIndex(on storage: NSTextStorage) { - caretIndex = SQLStatementCursorIndex(grammar: grammar) + private func textDidChange(at location: Int) { + caretIndex?.textDidChange(at: location) + caretTable = nil + } + + private func observeEdits(on storage: NSTextStorage) { + caretIndex = nil + caretTable = nil caretIndexStorage = storage // Every way the text changes passes through the storage, including the ones that skip the editor's delegate. let token = NotificationCenter.default.addObserver( @@ -313,7 +343,7 @@ final class StatementRunController { private func caretIndexStorageDidProcessEditing() { guard let storage = caretIndexStorage, storage.editedMask.contains(.editedCharacters) else { return } - caretIndex?.textDidChange(at: storage.editedRange.location) + textDidChange(at: storage.editedRange.location) } } diff --git a/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift b/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift index f9f0e6a53f..7e4e542b48 100644 --- a/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift +++ b/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift @@ -155,14 +155,10 @@ final class DataTabSQLExporter: DataGridSQLExporting { ) } - // Every marker in a row not saved yet, and in a saved row the cells a pending edit set to Default. + // The cells a pending insert or edit marked as the default. private func stagedDefaults(of row: Row, target: Target) -> Set { - let marker = PluginCellValue.text("__DEFAULT__") - if target.changeManager.insertedRowIDs.contains(row.id) { - return Set(row.values.indices.filter { row.values[$0] == marker }) - } - let pendingUpdate = target.changeManager.pendingUpdate(forRow: row.id) - return Set(pendingUpdate?.cellChanges.filter { $0.newValue == marker }.map(\.columnIndex) ?? []) + let marked = target.changeManager.expressionColumns(forRow: row.id) + return Set(marked.filter { row.values.indices.contains($0) && row.values[$0] == .columnDefault }) } private func nonEmpty(_ statements: String) -> String? { diff --git a/apple/TablePro/Views/Main/Child/MainEditorContentView.swift b/apple/TablePro/Views/Main/Child/MainEditorContentView.swift index 81b48e98af..a246fc2a70 100644 --- a/apple/TablePro/Views/Main/Child/MainEditorContentView.swift +++ b/apple/TablePro/Views/Main/Child/MainEditorContentView.swift @@ -734,6 +734,7 @@ struct MainEditorContentView: View { selectedRowIDs: Set(model.selectedRowIDs), displayIDs: model.displayedRowIDs, deletedRowIDs: changeManager.deletedRowIDs, + defaultedCells: changeManager.defaultedCells(in: rows), valueFilter: tab.valueFilter, dataRevision: coordinator.tabSessionRegistry.session(for: tab.id)?.dataRevision ?? 0, displayRevision: coordinator.gridDisplayRevision, diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift index d7eab33615..ecd1426d4c 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift @@ -677,10 +677,13 @@ extension MainContentCoordinator { try await dropContainer(target) services.catalogChangeService.record(.containerDropped(target, connectionId: connectionId)) } catch { - navigationLogger.error( - "Failed to drop \(target.id, privacy: .public): \(error.publicLogShape, privacy: .public)" - ) - failures.append((target.name, error.localizedDescription)) + let failure = GatedOperationFailure(error) + if failure.reportsFailure { + navigationLogger.error( + "Failed to drop \(target.id, privacy: .public): \(error.publicLogShape, privacy: .public)" + ) + } + if let message = failure.message { failures.append((target.name, message)) } } } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SafeModePromptRetry.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SafeModePromptRetry.swift new file mode 100644 index 0000000000..92846198f7 --- /dev/null +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SafeModePromptRetry.swift @@ -0,0 +1,28 @@ +// +// MainContentCoordinator+SafeModePromptRetry.swift +// TablePro +// + +import Foundation + +extension MainContentCoordinator { + // The prompt is one per window, so a first load that meets another run's prompt waits for it to close. + func turnAwayTableLoad(tabId: UUID, trigger: TableLoadTrigger) { + traceNavigationAbandoned(tabId: tabId, outcome: .safeModePromptAlreadyOpen) + tableLoadTurnedAwayByPrompt = ParkedTableLoad(tabId: tabId, trigger: trigger) + declineTableLoad(for: tabId) + } + + func safeModePromptDidChange(wasShowing: Bool) { + guard wasShowing, !isShowingSafeModePrompt, tableLoadTurnedAwayByPrompt != nil else { return } + // Once the run that lowered it has started, so a load it starts itself is not asked for twice. + Task { @MainActor [weak self] in self?.retryTableLoadTurnedAwayByPrompt() } + } + + private func retryTableLoadTurnedAwayByPrompt() { + guard !isShowingSafeModePrompt, !isTearingDown, let turnedAway = tableLoadTurnedAwayByPrompt else { return } + tableLoadTurnedAwayByPrompt = nil + guard turnedAway.tabId == tabManager.selectedTabId else { return } + lazyLoadCurrentTabIfNeeded(trigger: turnedAway.trigger) + } +} diff --git a/apple/TablePro/Views/Main/MainContentCommandActions.swift b/apple/TablePro/Views/Main/MainContentCommandActions.swift index ec02fa09d7..0decc6483f 100644 --- a/apple/TablePro/Views/Main/MainContentCommandActions.swift +++ b/apple/TablePro/Views/Main/MainContentCommandActions.swift @@ -66,6 +66,9 @@ final class MainContentCommandActions: ObservableObject { await RunningWorkApproval.confirm(prompt, window: window) } + // What a window close waits on while another connection's pane mounts. + var paneWaitClock: any Clock = ContinuousClock() + private var notificationTasks: [Task] = [] private var eventCancellables: Set = [] @@ -556,12 +559,7 @@ final class MainContentCommandActions: ObservableObject { // Every connection in the window: closing it closes them all, and skipping a background one is silent data loss. internal var hasUnsavedWorkInWindow: Bool { - guard let host = window?.contentViewController as? MainSplitViewController else { - return coordinator?.hasAnyUnsavedWork() ?? false - } - return host.workspaces.workspaces.contains { workspace in - workspace.sessionState?.coordinator.hasAnyUnsavedWork() == true - } + coordinatorsInWindow.contains { $0.hasAnyUnsavedWork() } } // This connection only: closing its tabs leaves another connection's pending work alone. @@ -650,13 +648,16 @@ final class MainContentCommandActions: ObservableObject { return .closed } // Work is stopped only once the close is certain: a cancelled or failed save keeps it running. - let result = await AlertHelper.confirmSaveChanges( - message: RunningWorkPrompt.saveMessage( - String(localized: "Your changes will be lost if you don't save them."), + let keepsTabsOpen = coordinatorsInWindow.contains { hosted in + hosted.tabManager.tabs.contains { hosted.savability(of: $0) == .mountedOnly } + } + let result = await confirmSaveChanges( + RunningWorkPrompt.saveMessage( + Self.unsavedWorkMessage(keepsTabsOpen: keepsTabsOpen), action: .close, alsoStopping: work ), - window: closeAnchorWindow + closeAnchorWindow ) switch result { @@ -775,13 +776,17 @@ final class MainContentCommandActions: ObservableObject { return true } + // The prompt asked about every connection in the window, so Save reaches every tab of each; one it cannot save keeps the window. private func saveAndClose(asBatchSurvivor: Bool?) async -> Bool { - guard let coordinator else { - finish(asBatchSurvivor: asBatchSurvivor) - return true + let host = window?.contentViewController as? MainSplitViewController + let shown = host?.workspaces.selectedConnectionId + for hosted in coordinatorsInWindow where hosted === coordinator || hosted.hasAnyUnsavedWork() { + // A save can raise a sheet, which has to sit over the connection it is about. + host?.workspaces.select(hosted.connectionId) + guard let actions = await shownActions(of: hosted) else { return false } + guard await actions.saveEveryTabsWork() else { return false } } - guard await applyStagedStructureEdits(in: coordinator.tabManager.tabs) else { return false } - guard await saveSelectedTabWork() else { return false } + if let shown { host?.workspaces.select(shown) } finish(asBatchSurvivor: asBatchSurvivor) return true } @@ -1407,6 +1412,38 @@ final class MainContentCommandActions: ObservableObject { } extension MainContentCommandActions { + // This connection's first. + private var coordinatorsInWindow: [MainContentCoordinator] { + guard let host = window?.contentViewController as? MainSplitViewController else { + return [coordinator].compactMap { $0 } + } + let hosted = host.workspaces.workspaces.compactMap { $0.sessionState?.coordinator } + guard let coordinator else { return hosted } + return [coordinator] + hosted.filter { $0 !== coordinator } + } + + // A connection never shown has no actions until its pane mounts, a few run loop turns after it is selected. + private func shownActions(of hosted: MainContentCoordinator) async -> MainContentCommandActions? { + if hosted === coordinator { return self } + return await Self.actions(of: hosted, waitingOn: paneWaitClock) + } + + // A cancelled close stops waiting at once rather than polling out the rest of the deadline. + private static func actions>( + of hosted: MainContentCoordinator, + waitingOn clock: C + ) async -> MainContentCommandActions? { + let deadline = clock.now.advanced(by: .seconds(2)) + while hosted.commandActions == nil, clock.now < deadline { + do { + try await clock.sleep(for: .milliseconds(20)) + } catch { + return nil + } + } + return hosted.commandActions + } + // Copy with the grid not focused takes the selected rows in the row copy format; a focused grid answers `copy:` itself. func copySelectedRowsInCopyFormat() { guard dataGridOwnsSelection, let model = coordinator?.activeGridModel else { diff --git a/apple/TablePro/Views/Main/MainContentCoordinator.swift b/apple/TablePro/Views/Main/MainContentCoordinator.swift index b338583e42..2c2c39815a 100644 --- a/apple/TablePro/Views/Main/MainContentCoordinator.swift +++ b/apple/TablePro/Views/Main/MainContentCoordinator.swift @@ -371,7 +371,12 @@ final class MainContentCoordinator: ObservableObject { // Guards re-entrant confirm dialogs (nested run loop during runModal). internal var isShowingConfirmAlert = false - internal var isShowingSafeModePrompt = false + // Lowered by whichever run raised it, so a table load it turned away is retried whatever path asked. + internal var isShowingSafeModePrompt = false { + didSet { safeModePromptDidChange(wasShowing: oldValue) } + } + + var tableLoadTurnedAwayByPrompt: ParkedTableLoad? @Published internal var rewindPlan: RewindPlan? @@ -1110,7 +1115,7 @@ final class MainContentCoordinator: ObservableObject { tab.execution.lastExecutedAt == nil { guard !isShowingSafeModePrompt else { - traceNavigationAbandoned(tabId: tab.id, outcome: .safeModePromptAlreadyOpen) + turnAwayTableLoad(tabId: tab.id, trigger: trigger) return nil } isShowingSafeModePrompt = true diff --git a/apple/TablePro/Views/ObjectCopy/CopyObjectsResultView.swift b/apple/TablePro/Views/ObjectCopy/CopyObjectsResultView.swift index 3bd997547b..38c7cf6698 100644 --- a/apple/TablePro/Views/ObjectCopy/CopyObjectsResultView.swift +++ b/apple/TablePro/Views/ObjectCopy/CopyObjectsResultView.swift @@ -41,6 +41,11 @@ internal struct CopyObjectsResultView: View { .font(.callout) .foregroundStyle(.secondary) } + ForEach(Array(result.warnings.enumerated()), id: \.offset) { _, warning in + NoticeLabel(warning, severity: .warning) + .font(.callout) + .textSelection(.enabled) + } outcomes(result) } .frame(maxWidth: .infinity, alignment: .leading) diff --git a/apple/TablePro/Views/ObjectSource/UserType/CreateUserTypeSheet.swift b/apple/TablePro/Views/ObjectSource/UserType/CreateUserTypeSheet.swift index 31526e51b4..8c0baa020c 100644 --- a/apple/TablePro/Views/ObjectSource/UserType/CreateUserTypeSheet.swift +++ b/apple/TablePro/Views/ObjectSource/UserType/CreateUserTypeSheet.swift @@ -245,11 +245,12 @@ struct CreateUserTypeSheet: View { try await UserTypeCreation.create(definition, in: scope, connection: connection) dismiss() onCreated(definition) - } catch ExecutionGateError.cancelledByUser { - return } catch { - Self.logger.error("Create Type failed: \(error.publicLogShape, privacy: .public)") - failure = error.localizedDescription + let outcome = GatedOperationFailure(error) + if outcome.reportsFailure { + Self.logger.error("Create Type failed: \(error.publicLogShape, privacy: .public)") + } + failure = outcome.message } } } diff --git a/apple/TablePro/Views/Results/ResultsJsonView.swift b/apple/TablePro/Views/Results/ResultsJsonView.swift index d8cc4d3395..6a04df3448 100644 --- a/apple/TablePro/Views/Results/ResultsJsonView.swift +++ b/apple/TablePro/Views/Results/ResultsJsonView.swift @@ -11,6 +11,7 @@ internal struct ResultsJsonView: View { let selectedRowIDs: Set let displayIDs: [RowID]? let deletedRowIDs: Set + let defaultedCells: [RowID: Set] let valueFilter: GridValueFilterState let dataRevision: Int let displayRevision: Int @@ -32,6 +33,7 @@ internal struct ResultsJsonView: View { selectedRowIDs: Set, displayIDs: [RowID]?, deletedRowIDs: Set, + defaultedCells: [RowID: Set] = [:], valueFilter: GridValueFilterState, dataRevision: Int, displayRevision: Int, @@ -41,6 +43,7 @@ internal struct ResultsJsonView: View { self.selectedRowIDs = selectedRowIDs self.displayIDs = displayIDs self.deletedRowIDs = deletedRowIDs + self.defaultedCells = defaultedCells self.valueFilter = valueFilter self.dataRevision = dataRevision self.displayRevision = displayRevision @@ -58,6 +61,7 @@ internal struct ResultsJsonView: View { let displayRevision: Int let selectedRowIDs: Set let deletedRowIDs: Set + let defaultedCells: [RowID: Set] let valueFilter: GridValueFilterState let hiddenColumns: Set let columnOrder: [String]? @@ -69,6 +73,7 @@ internal struct ResultsJsonView: View { displayRevision: displayRevision, selectedRowIDs: selectedRowIDs, deletedRowIDs: deletedRowIDs, + defaultedCells: defaultedCells, valueFilter: valueFilter, hiddenColumns: columnLayout.hiddenColumns, columnOrder: columnLayout.columnOrder @@ -202,6 +207,7 @@ internal struct ResultsJsonView: View { let ids = displayIDs let selectedIndices = Set(DataGridRowOrder(filteredIDs: ids, revision: 0).displayIndexes(of: selectedRowIDs, in: snapshot)) let deletedIDs = deletedRowIDs + let defaulted = defaultedCells let layout = columnLayout let result = await Task.detached(priority: .userInitiated) { @@ -210,6 +216,7 @@ internal struct ResultsJsonView: View { displayIDs: ids, selectedIndices: selectedIndices, deletedRowIDs: deletedIDs, + defaultedCells: defaulted, columnLayout: layout ) }.value @@ -245,6 +252,7 @@ internal struct ResultsJsonView: View { displayIDs: [RowID]?, selectedIndices: Set, deletedRowIDs: Set = [], + defaultedCells: [RowID: Set] = [:], columnLayout: ColumnLayoutState ) -> RenderedJson { let output = ResultJsonSerializer.serialize( @@ -252,6 +260,7 @@ internal struct ResultsJsonView: View { displayIDs: displayIDs, selectedDisplayIndices: selectedIndices, deletedRowIDs: deletedRowIDs, + defaultedCells: defaultedCells, columns: .fromColumnLayout(columnLayout, columns: tableRows.columns) ) return RenderedJson( diff --git a/apple/TablePro/Views/Sidebar/EditSequenceSheet.swift b/apple/TablePro/Views/Sidebar/EditSequenceSheet.swift index 11ae7629f4..c72f6f12ea 100644 --- a/apple/TablePro/Views/Sidebar/EditSequenceSheet.swift +++ b/apple/TablePro/Views/Sidebar/EditSequenceSheet.swift @@ -188,7 +188,7 @@ struct EditSequenceSheet: View { try await SequenceEditing.alter(details, to: settings, target: target, connection: connection) dismiss() } catch { - saveError = error.localizedDescription + saveError = GatedOperationFailure(error).message phase = .editing(details) } } diff --git a/apple/TablePro/Views/Sidebar/ObjectCommentSheet.swift b/apple/TablePro/Views/Sidebar/ObjectCommentSheet.swift index bdc4397a00..b73407bedf 100644 --- a/apple/TablePro/Views/Sidebar/ObjectCommentSheet.swift +++ b/apple/TablePro/Views/Sidebar/ObjectCommentSheet.swift @@ -144,10 +144,8 @@ struct ObjectCommentSheet: View { do { try await ObjectCommentEditing.setComment(draft.commentToSave, on: target, connection: connection) dismiss() - } catch ExecutionGateError.cancelledByUser { - phase = .editing } catch { - saveError = error.localizedDescription + saveError = GatedOperationFailure(error).message phase = .editing } } diff --git a/apple/TablePro/Views/Sidebar/SetSequenceValueSheet.swift b/apple/TablePro/Views/Sidebar/SetSequenceValueSheet.swift index e63e765d73..d970311881 100644 --- a/apple/TablePro/Views/Sidebar/SetSequenceValueSheet.swift +++ b/apple/TablePro/Views/Sidebar/SetSequenceValueSheet.swift @@ -176,7 +176,7 @@ struct SetSequenceValueSheet: View { try await SequenceEditing.setNextValue(nextValue, of: details, target: target, connection: connection) dismiss() } catch { - saveError = error.localizedDescription + saveError = GatedOperationFailure(error).message phase = .editing(details) } } diff --git a/apple/TablePro/Views/Structure/ClickHousePartsView.swift b/apple/TablePro/Views/Structure/ClickHousePartsView.swift index 6f37691102..f95d50739e 100644 --- a/apple/TablePro/Views/Structure/ClickHousePartsView.swift +++ b/apple/TablePro/Views/Structure/ClickHousePartsView.swift @@ -8,6 +8,56 @@ import SwiftUI import TableProPluginKit import TableProSQLGrammar +internal struct ClickHousePartsAlert: Equatable { + let title: String + let message: String +} + +// A failed or refused action never replaces the list it acted on; only a load with nothing shown does. +internal struct ClickHousePartsPaneState { + internal enum Display: Equatable { + case loading + case loadFailed(String) + case empty + case parts + } + + private(set) var parts: [ClickHousePartInfo] = [] + private(set) var isLoading = true + private(set) var loadError: String? + var alert: ClickHousePartsAlert? + + var display: Display { + guard parts.isEmpty else { return .parts } + if let loadError { return .loadFailed(loadError) } + return isLoading ? .loading : .empty + } + + mutating func beginLoad() { + isLoading = true + loadError = nil + } + + mutating func finishLoad(_ parts: [ClickHousePartInfo]) { + self.parts = parts + isLoading = false + } + + mutating func failLoad(_ message: String) { + isLoading = false + guard parts.isEmpty else { + alert = ClickHousePartsAlert(title: String(localized: "Couldn't Read the Parts"), message: message) + return + } + loadError = message + } + + mutating func actionEnded(_ failure: GatedOperationFailure, title: String) { + guard let message = failure.message else { return } + alert = ClickHousePartsAlert(title: title, message: message) + } +} + struct ClickHousePartsView: View { private static let logger = Logger(subsystem: "com.TablePro", category: "ClickHousePartsView") @@ -17,31 +67,31 @@ struct ClickHousePartsView: View { let connection: DatabaseConnection let reloadToken: Int - @State private var parts: [ClickHousePartInfo] = [] - @State private var isLoading = true - @State private var errorMessage: String? + @State private var pane = ClickHousePartsPaneState() @State private var loadedToken: Int? @StateObject private var grid = StructureListGridModel(accessibilityLabel: StructurePane.parts.displayName) @StateObject private var selection = StructureListSelection() var body: some View { VStack(spacing: 0) { - if isLoading { + switch pane.display { + case .loading: ProgressView() .frame(maxWidth: .infinity, maxHeight: .infinity) .gridStandInBackground() - } else if let error = errorMessage { - VStack(spacing: 8) { - Image(systemName: "exclamationmark.triangle") - .font(.largeTitle) - .foregroundStyle(.orange) - .accessibilityHidden(true) - RevealedTextView(error) - .foregroundStyle(.secondary) + case .loadFailed(let message): + UnavailableStateView { + Label(String(localized: "Couldn't Read the Parts"), systemImage: "exclamationmark.triangle") + } description: { + RevealedTextView(message) + } actions: { + Button(String(localized: "Try Again")) { + Task { await loadParts() } + } } .frame(maxWidth: .infinity, maxHeight: .infinity) .gridStandInBackground() - } else if parts.isEmpty { + case .empty: VStack(spacing: 8) { Image(systemName: "tray") .font(.largeTitle) @@ -52,11 +102,20 @@ struct ClickHousePartsView: View { } .frame(maxWidth: .infinity, maxHeight: .infinity) .gridStandInBackground() - } else { + case .parts: partsToolbar partsTable } } + .alert( + pane.alert?.title ?? "", + isPresented: Binding(get: { pane.alert != nil }, set: { if !$0 { pane.alert = nil } }), + presenting: pane.alert + ) { _ in + Button(String(localized: "OK"), role: .cancel) {} + } message: { alert in + Text(verbatim: RevealedText(alert.message).plainText) + } .task(id: reloadToken) { await loadPartsIfNeeded() } } @@ -92,7 +151,7 @@ struct ClickHousePartsView: View { columns: Self.columnTitles, defaultWidths: Self.columnWidths, minimumWidths: Self.columnMinimumWidths, - rows: parts.map(Self.gridRow(for:)), + rows: pane.parts.map(Self.gridRow(for:)), databaseType: connection.type, selectedIDs: selection.ids, select: { selection.ids = $0 }, @@ -130,6 +189,20 @@ struct ClickHousePartsView: View { ) } + // Column order is ClickHousePartStatements.parts. + internal static func part(from row: [PluginCellValue]) -> ClickHousePartInfo? { + guard let name = row[safe: 2]?.asText else { return nil } + return ClickHousePartInfo( + partition: row[safe: 0]?.asText ?? "", + partitionId: row[safe: 1]?.asText ?? "", + name: name, + rows: row[safe: 3]?.asText.flatMap { UInt64($0) } ?? 0, + bytesOnDisk: row[safe: 4]?.asText.flatMap { UInt64($0) } ?? 0, + modificationTime: row[safe: 5]?.asText ?? "", + active: row[safe: 6]?.asText == "1" + ) + } + // MARK: - Actions private func optimizeTable() { @@ -140,19 +213,20 @@ struct ClickHousePartsView: View { database: scope.database, table: tableName, quote: driver.quoteIdentifier ), description: String(localized: "Optimize Table"), + failureTitle: String(localized: "Couldn't Optimize Table"), kind: .maintenance ) } } private func dropSelectedPartition() { - guard let partitionValue = selectedPartitionValue() else { return } + guard let part = selectedPart() else { return } Task { @MainActor in let confirmed = await AlertHelper.confirmDestructive( title: String(localized: "Drop Partition?"), message: String( format: String(localized: "This will permanently delete all data in partition '%@'."), - partitionValue + part.partition ), confirmButton: String(localized: "Drop"), cancelButton: String(localized: "Cancel") @@ -160,27 +234,31 @@ struct ClickHousePartsView: View { guard confirmed else { return } guard let driver = DatabaseManager.shared.driver(for: scope.connectionId) else { return } - let sql = ClickHousePartStatements.dropPartition( + let failureTitle = String(localized: "Couldn't Drop Partition") + guard let sql = ClickHousePartStatements.dropPartition( database: scope.database, table: tableName, - partition: partitionValue, - quote: driver.quoteIdentifier, - escape: driver.escapeStringLiteral - ) - guard await run(sql, description: String(localized: "Drop Partition"), kind: .destructiveQuery) - else { return } + partitionId: part.partitionId, + quote: driver.quoteIdentifier + ) else { + pane.alert = ClickHousePartsAlert(title: failureTitle, message: Self.unusablePartitionIdMessage) + return + } + guard await run( + sql, description: String(localized: "Drop Partition"), failureTitle: failureTitle, kind: .destructiveQuery + ) else { return } selection.ids.removeAll() } } private func detachSelectedPartition() { - guard let partitionValue = selectedPartitionValue() else { return } + guard let part = selectedPart() else { return } Task { @MainActor in let confirmed = await AlertHelper.confirmDestructive( title: String(localized: "Detach Partition?"), message: String( format: String(localized: "This will detach partition '%@'. Data will be preserved but inaccessible until re-attached."), - partitionValue + part.partition ), confirmButton: String(localized: "Detach"), cancelButton: String(localized: "Cancel") @@ -188,22 +266,30 @@ struct ClickHousePartsView: View { guard confirmed else { return } guard let driver = DatabaseManager.shared.driver(for: scope.connectionId) else { return } - let sql = ClickHousePartStatements.detachPartition( + let failureTitle = String(localized: "Couldn't Detach Partition") + guard let sql = ClickHousePartStatements.detachPartition( database: scope.database, table: tableName, - partition: partitionValue, - quote: driver.quoteIdentifier, - escape: driver.escapeStringLiteral - ) - guard await run(sql, description: String(localized: "Detach Partition"), kind: .destructiveQuery) - else { return } + partitionId: part.partitionId, + quote: driver.quoteIdentifier + ) else { + pane.alert = ClickHousePartsAlert(title: failureTitle, message: Self.unusablePartitionIdMessage) + return + } + guard await run( + sql, description: String(localized: "Detach Partition"), failureTitle: failureTitle, kind: .destructiveQuery + ) else { return } selection.ids.removeAll() } } + private static var unusablePartitionIdMessage: String { + String(localized: "The server reported a partition ID that cannot be used in a statement, so nothing was run.") + } + @MainActor @discardableResult - private func run(_ sql: String, description: String, kind: OperationKind) async -> Bool { + private func run(_ sql: String, description: String, failureTitle: String, kind: OperationKind) async -> Bool { let scope = scope do { let decision = await ExecutionGateProvider.shared.authorize( @@ -217,10 +303,7 @@ struct ClickHousePartsView: View { operationDescription: description ) ) - guard case .authorized = decision else { - errorMessage = decision.deniedReason - return false - } + if let denial = decision.denialError { throw denial } _ = try await DatabaseManager.shared.withScopedDriver( scope: scope, route: DatabaseManager.shared.schemaChangeRoute(for: scope), @@ -228,30 +311,30 @@ struct ClickHousePartsView: View { ) { driver in try await driver.execute(query: sql) } - if kind != .maintenance { - CatalogChangeService.post( - .changed(CatalogChange(connectionId: scope.connectionId, database: scope.database, kinds: .tables)) - ) - } - await loadParts() - return true } catch { - Self.logger.error("\(description, privacy: .public) failed: \(error.publicLogShape, privacy: .public)") - errorMessage = error.localizedDescription + let failure = GatedOperationFailure(error) + if failure.reportsFailure { + Self.logger.error("\(description, privacy: .public) failed: \(error.publicLogShape, privacy: .public)") + } + pane.actionEnded(failure, title: failureTitle) return false } + if kind != .maintenance { + CatalogChangeService.post( + .changed(CatalogChange(connectionId: scope.connectionId, database: scope.database, kinds: .tables)) + ) + } + await loadParts() + return true } - private func selectedPartitionValue() -> String? { - Self.partition(of: selection.ids, in: parts) + private func selectedPart() -> ClickHousePartInfo? { + Self.selectedPart(of: selection.ids, in: pane.parts) } - internal static func partition(of selectedIDs: Set, in parts: [ClickHousePartInfo]) -> String? { - guard selectedIDs.count == 1, - let selectedId = selectedIDs.first, - let part = parts.first(where: { $0.id == selectedId }) - else { return nil } - return part.partition + internal static func selectedPart(of selectedIDs: Set, in parts: [ClickHousePartInfo]) -> ClickHousePartInfo? { + guard selectedIDs.count == 1, let selectedId = selectedIDs.first else { return nil } + return parts.first { $0.id == selectedId } } // MARK: - Data Loading @@ -263,8 +346,7 @@ struct ClickHousePartsView: View { private func loadParts() async { let token = reloadToken - isLoading = true - errorMessage = nil + pane.beginLoad() let scope = scope let tableName = tableName @@ -276,29 +358,12 @@ struct ClickHousePartsView: View { } guard !Task.isCancelled else { return } selection.ids = [] - parts = result.rows.compactMap { row -> ClickHousePartInfo? in - guard let name = row[safe: 1]?.asText else { return nil } - let partition = row[safe: 0]?.asText ?? "" - let rows = row[safe: 2]?.asText.flatMap { UInt64($0) } ?? 0 - let bytesOnDisk = row[safe: 3]?.asText.flatMap { UInt64($0) } ?? 0 - let modTime = row[safe: 4]?.asText ?? "" - let active = row[safe: 5]?.asText == "1" - return ClickHousePartInfo( - partition: partition, - name: name, - rows: rows, - bytesOnDisk: bytesOnDisk, - modificationTime: modTime, - active: active - ) - } + pane.finishLoad(result.rows.compactMap(Self.part(from:))) loadedToken = token } catch { guard !Task.isCancelled else { return } Self.logger.error("Failed to load parts: \(error.publicLogShape, privacy: .public)") - errorMessage = error.localizedDescription + pane.failLoad(error.localizedDescription) } - - isLoading = false } } diff --git a/apple/TablePro/Views/Structure/CreateTableDraft.swift b/apple/TablePro/Views/Structure/CreateTableDraft.swift index bffd4f2653..854b5fea41 100644 --- a/apple/TablePro/Views/Structure/CreateTableDraft.swift +++ b/apple/TablePro/Views/Structure/CreateTableDraft.swift @@ -469,8 +469,7 @@ internal final class CreateTableDraft: ObservableObject { coordinator: coordinator ) } catch { - Self.logger.error("Create table failed: \(error.publicLogShape, privacy: .public)") - reportFailure(error.localizedDescription, on: coordinator) + reportFailure(error: error, on: coordinator) } } } @@ -508,8 +507,7 @@ internal final class CreateTableDraft: ObservableObject { } catch let formError as PluginCreateTableFormError { form?.recordSubmissionError(formError) } catch { - Self.logger.error("Create table failed: \(error.publicLogShape, privacy: .public)") - reportFailure(error.localizedDescription, on: coordinator) + reportFailure(error: error, on: coordinator) } } } @@ -560,6 +558,15 @@ internal final class CreateTableDraft: ObservableObject { ) } + private func reportFailure(error: any Error, on coordinator: MainContentCoordinator) { + let failure = GatedOperationFailure(error) + if failure.reportsFailure { + Self.logger.error("Create table failed: \(error.publicLogShape, privacy: .public)") + } + guard let message = failure.message else { return } + reportFailure(message, on: coordinator) + } + private func reportFailure(_ message: String, on coordinator: MainContentCoordinator) { AlertHelper.showErrorSheet( title: String(localized: "Create Table Failed"), diff --git a/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift b/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift index b760819838..15048571f6 100644 --- a/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift +++ b/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift @@ -31,52 +31,13 @@ internal enum StructureSaveOutcome: Equatable { } } -/// How a save that did not apply ends, read from the error alone so it can be tested without -/// presenting anything. -/// -/// The gate's sheet is the one confirmation a save gets, so Cancel there is the ordinary way to -/// back out, and it ends quietly the way closing any confirmation does. Answering it with an -/// "Error Applying Changes" sheet put a second dialog in front of the user for the choice they had -/// just made. -internal enum StructureApplyFailure: Equatable { - case cancelledByUser - case refused(String) - case failed(String) - - internal init(_ error: any Error) { - guard let gateError = error as? ExecutionGateError else { - self = .failed(error.localizedDescription) - return - } - switch gateError { - case .cancelledByUser: - self = .cancelledByUser - case .denied(let reason): - self = .refused(reason) - } - } - - internal var outcome: StructureSaveOutcome { +internal extension GatedOperationFailure { + var outcome: StructureSaveOutcome { switch self { case .cancelledByUser, .refused: .refused case .failed(let message): .failed(message) } } - - /// What the error sheet says. Nil for a Cancel, which shows nothing. - internal var message: String? { - switch self { - case .cancelledByUser: nil - case .refused(let reason), .failed(let reason): reason - } - } - - /// A refusal and a Cancel both stop at the gate, before the first statement, so neither is a - /// failed operation to report or a reason to reload the catalog. - internal var reportsFailure: Bool { - if case .failed = self { return true } - return false - } } internal extension StructureEditingSession { @@ -159,7 +120,7 @@ internal extension StructureEditingSession { // A failed save that may have written; its error waits for the table to be read again. private struct AlterFailure { - let failure: StructureApplyFailure + let failure: GatedOperationFailure let startedAt: ContinuousClock.Instant } @@ -240,7 +201,7 @@ internal extension StructureEditingSession { report(.succeeded(OperationSummary()), startedAt: operationStart, coordinator: coordinator) return (.applied, nil) } catch { - let failure = StructureApplyFailure(error) + let failure = GatedOperationFailure(error) // A gate refusal and a driver's refusal both stop before the first statement. guard failure.reportsFailure, !(error is SchemaOperationRefusedError) else { present(failure, startedAt: operationStart, coordinator: coordinator) @@ -251,7 +212,7 @@ internal extension StructureEditingSession { } private func present( - _ failure: StructureApplyFailure, + _ failure: GatedOperationFailure, note: String? = nil, startedAt: ContinuousClock.Instant, coordinator: MainContentCoordinator? @@ -340,7 +301,7 @@ internal extension StructureEditingSession { } catch { changeManager.releaseHold(hold, written: false) isApplying = false - let failure = StructureApplyFailure(error) + let failure = GatedOperationFailure(error) if failure.reportsFailure { CatalogChangeService.post( .changed(CatalogChange(connectionId: connection.id, database: prepared.scope.database, kinds: .tables)) diff --git a/apple/TablePro/Views/Structure/StructureEditingSession+ColumnReorder.swift b/apple/TablePro/Views/Structure/StructureEditingSession+ColumnReorder.swift index 30c1ff93cc..47f269457e 100644 --- a/apple/TablePro/Views/Structure/StructureEditingSession+ColumnReorder.swift +++ b/apple/TablePro/Views/Structure/StructureEditingSession+ColumnReorder.swift @@ -65,7 +65,7 @@ internal extension StructureEditingSession { ) await finishColumnReorder(prepared, clearTarget: clearTarget) } catch { - if StructureApplyFailure(error).reportsFailure { + if GatedOperationFailure(error).reportsFailure { CatalogChangeService.post( .changed( CatalogChange( @@ -115,7 +115,7 @@ internal extension StructureEditingSession { } private func reportColumnReorderFailure(_ error: any Error) { - guard let message = StructureApplyFailure(error).message else { return } + guard let message = GatedOperationFailure(error).message else { return } AlertHelper.showErrorSheet( title: String(localized: "Column Reorder Failed"), message: message, diff --git a/apple/TablePro/Views/Structure/TriggerDetailView.swift b/apple/TablePro/Views/Structure/TriggerDetailView.swift index a0885aac43..34a9be327b 100644 --- a/apple/TablePro/Views/Structure/TriggerDetailView.swift +++ b/apple/TablePro/Views/Structure/TriggerDetailView.swift @@ -277,7 +277,7 @@ struct TriggerDetailView: View { name: trigger.name ) } catch { - actionError = error.localizedDescription + actionError = GatedOperationFailure(error).message } } } diff --git a/apple/TablePro/Views/Structure/TriggerEditorView.swift b/apple/TablePro/Views/Structure/TriggerEditorView.swift index 6dee7b596e..ef9b441168 100644 --- a/apple/TablePro/Views/Structure/TriggerEditorView.swift +++ b/apple/TablePro/Views/Structure/TriggerEditorView.swift @@ -29,7 +29,12 @@ struct TriggerEditorView: View { @State private var isApplying = false @State private var errorMessage: String? @State private var droppedDefinition: String? - @AppStorage("structureCodeFontSize", store: AppStorageEnvironment.shared.defaults) private var fontSize: Double = 13 + @AppStorage(TriggerEditorView.fontSizeKey, store: AppStorageEnvironment.shared.defaults) + private var fontSize = TriggerEditorView.defaultFontSize + + // The Structure tab's DDL pane writes its zoom here too. + static let fontSizeKey = "structureCodeFontSize" + static let defaultFontSize = 13.0 init( scope: DatabaseScope, @@ -45,7 +50,14 @@ struct TriggerEditorView: View { self.mode = mode self.onClose = onClose self._sql = State(wrappedValue: initialSQL) - self._editorConfiguration = State(wrappedValue: Self.makeConfiguration(fontSize: 13)) + self._editorConfiguration = State(wrappedValue: Self.initialConfiguration()) + } + + // The State is seeded once, here, so it reads the store; `fontSize` only follows later zooms. + static func initialConfiguration( + in defaults: UserDefaults = AppStorageEnvironment.shared.defaults + ) -> SourceEditorConfiguration { + makeConfiguration(fontSize: defaults.object(forKey: fontSizeKey) as? Double ?? defaultFontSize) } var body: some View { @@ -145,7 +157,7 @@ struct TriggerEditorView: View { if let dropped = (error as? TriggerEditingError)?.droppedDefinition { droppedDefinition = dropped } - errorMessage = error.localizedDescription + errorMessage = GatedOperationFailure(error).message } } } diff --git a/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift b/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift index e7c5b14de0..9b87a2bce5 100644 --- a/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift +++ b/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift @@ -21,22 +21,39 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { let columns: [ColumnInfo] } - private var tables: [TableInfo] + private var tables: [TableInfo] { + didSet { + cachedTableItems = nil + cachedTableColumnItems = nil + } + } + // Every table-slot request reads all of them, so they are built once per table list. + private var cachedTableItems: [SQLCompletionItem]? private let databaseType: DatabaseType private let schema: String? private let readColumns: QueryColumnReader private let readCatalogColumns: QueryColumnReader? private var columnReadsAllowed = true private var columnGeneration = 0 - private var columns: [ColumnKey: CachedColumns] = [:] + private var columns: [ColumnKey: CachedColumns] = [:] { + didSet { cachedTableColumnItems = nil } + } + // Every column request before a FROM reads all of them, so they are built once per column cache, table list and + // namespace list. + private var cachedTableColumnItems: [SQLCompletionItem]? + private(set) var tableColumnItemsBuildCount = 0 private var reads: [ColumnKey: Task<[ColumnInfo]?, any Error>] = [:] private var sessionReads: [ColumnKey: Task<[ColumnInfo]?, any Error>] = [:] private let readSchemaTables: QuerySchemaTableReader? private let readCatalogSchemaTables: QuerySchemaTableReader? private var sessionSchemaReads: [String: Task<[TableInfo]?, any Error>] = [:] private var sessionLocalTables: Set = [] - private var schemas: [String] - private var databases: [String] + private var schemas: [String] { + didSet { cachedTableColumnItems = nil } + } + private var databases: [String] { + didSet { cachedTableColumnItems = nil } + } private var schemaTables: [String: (generation: Int, tables: [TableInfo])] = [:] private var schemaTableReads: [String: Task<[TableInfo]?, any Error>] = [:] @@ -123,19 +140,21 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { } func tableCompletionItems() -> [SQLCompletionItem] { - tables.map { table in - let isView = table.type == .view || table.type == .materializedView - return SQLCompletionItem(label: table.name, kind: isView ? .view : .table, insertText: quoted(table.name)) - } + if let cachedTableItems { return cachedTableItems } + let items = tables.map { tableItem($0) } + cachedTableItems = items + return items + } + + private func tableItem(_ table: TableInfo) -> SQLCompletionItem { + let isView = table.type == .view || table.type == .materializedView + return SQLCompletionItem(label: table.name, kind: isView ? .view : .table, insertText: quoted(table.name)) } func tableCompletionItems(inSchema schema: String) async -> [SQLCompletionItem] { guard isKnownSchema(schema) else { return [] } guard !isActiveSchema(schema) else { return tableCompletionItems() } - return await tables(inSchema: schema).map { table in - let isView = table.type == .view || table.type == .materializedView - return SQLCompletionItem(label: table.name, kind: isView ? .view : .table, insertText: quoted(table.name)) - } + return await tables(inSchema: schema).map { tableItem($0) } } func schemaCompletionItems() -> [SQLCompletionItem] { @@ -317,6 +336,14 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { } func allColumnsFromCachedTables() -> [SQLCompletionItem] { + if let cachedTableColumnItems { return cachedTableColumnItems } + let items = buildTableColumnItems() + tableColumnItemsBuildCount += 1 + cachedTableColumnItems = items + return items + } + + private func buildTableColumnItems() -> [SQLCompletionItem] { // Sorted, because the ranker keeps emission order between equal scores and a dictionary's order is per process. let cached = columns.sorted { lhs, rhs in (lhs.key.schema ?? "", lhs.key.table) < (rhs.key.schema ?? "", rhs.key.table) diff --git a/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift b/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift index f5322a5f2e..5dd5104935 100644 --- a/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift @@ -400,6 +400,35 @@ struct QueryCompletionCatalogTests { #expect(await source.reads == ["orders", "customers"]) } + @Test("cached column suggestions are built once, and again only after a read, a table list or a namespace list") + func cachedColumnsAreBuiltOnce() async { + let source = CatalogColumnSource(["orders": [column("id")], "customers": [column("name")]]) + let catalog = catalog(source, tables: [table("orders"), table("customers")]) + _ = await catalog.columnCompletionItems(for: "orders", schema: nil) + + let first = await catalog.allColumnsFromCachedTables() + let second = await catalog.allColumnsFromCachedTables() + #expect(labels(second) == labels(first)) + #expect(await catalog.tableColumnItemsBuildCount == 1) + + _ = await catalog.columnCompletionItems(for: "customers", schema: nil) + let afterRead = await catalog.allColumnsFromCachedTables() + #expect(labels(afterRead) == ["name", "id"]) + #expect(await catalog.tableColumnItemsBuildCount == 2) + + let listedBefore = await catalog.tableCompletionItems() + #expect(labels(listedBefore) == ["orders", "customers"]) + await catalog.replaceTables([table("orders"), table("customers"), table("refunds")]) + _ = await catalog.allColumnsFromCachedTables() + #expect(await catalog.tableColumnItemsBuildCount == 3) + let listedAfter = await catalog.tableCompletionItems() + #expect(labels(listedAfter) == ["orders", "customers", "refunds"]) + + await catalog.replaceNamespaces(schemas: ["main"], databases: []) + _ = await catalog.allColumnsFromCachedTables() + #expect(await catalog.tableColumnItemsBuildCount == 4) + } + @Test("the catalog connection reads first and is not closed by a run; the session reads only without one") func catalogConnectionFirst() async { let catalogSource = CatalogColumnSource(["orders": [column("id")]]) diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift index d3d4c2d300..aed8b0f01c 100644 --- a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift @@ -92,8 +92,8 @@ struct SQLStatementGeneratorBitColumnTests { let rowID = RowID.inserted(UUID()) let statements = generator.generateStatements( - from: [RowChange(rowID: rowID, type: .insert)], - insertedRowData: [rowID: ["__DEFAULT__", "5", "Ann"]], + from: [RowChange(rowID: rowID, type: .insert, expressionColumns: [0])], + insertedRowData: [rowID: [.columnDefault, "5", "Ann"]], deletedRowIDs: [], insertedRowIDs: [rowID] ) diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorRowMatchTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorRowMatchTests.swift index 44d6a863ef..ea1bbb0f87 100644 --- a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorRowMatchTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorRowMatchTests.swift @@ -62,10 +62,12 @@ struct SQLStatementGeneratorRowMatchTests { @Test("A Databend row left entirely on defaults names one column with DEFAULT") func databendAllDefaultsInsert() throws { - let change = RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil) + let change = RowChange( + rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil, expressionColumns: [0, 1, 2, 3] + ) let statements = try generator(excluding: []).generateStatements( from: [change], - insertedRowData: [.existing(0): [.text("__DEFAULT__"), .text("__DEFAULT__"), .text("__DEFAULT__"), .text("__DEFAULT__")]], + insertedRowData: [.existing(0): [.columnDefault, .columnDefault, .columnDefault, .columnDefault]], deletedRowIDs: [], insertedRowIDs: [.existing(0)] ) diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorTests.swift index edeb219dea..731615a286 100644 --- a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorTests.swift @@ -89,14 +89,14 @@ struct SQLStatementGeneratorTests { #expect(statements[0].parameters[2] == nil) } - @Test("Insert skips __DEFAULT__ columns") + @Test("Insert leaves out a column marked as its default") func testInsertSkipsDefaultColumns() throws { let generator = try makeGenerator() let insertedRowData: [RowID: [PluginCellValue]] = [ - .existing(0): ["__DEFAULT__", "John", "john@example.com"] + .existing(0): [.columnDefault, "John", "john@example.com"] ] let changes: [RowChange] = [ - RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil) + RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil, expressionColumns: [0]) ] let statements = generator.generateStatements( @@ -114,11 +114,35 @@ struct SQLStatementGeneratorTests { #expect(stmt.parameters.count == 2) } + @Test("Insert binds text that only reads like the default, unmarked", arguments: ["__DEFAULT__", "DEFAULT"]) + func testInsertBindsUnmarkedDefaultText(text: String) throws { + let generator = try makeGenerator() + let insertedRowData: [RowID: [PluginCellValue]] = [ + .existing(0): [.columnDefault, .text(text), "john@example.com"] + ] + let changes: [RowChange] = [ + RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil, expressionColumns: [0]) + ] + + let statements = generator.generateStatements( + from: changes, + insertedRowData: insertedRowData, + deletedRowIDs: [], + insertedRowIDs: [.existing(0)] + ) + + #expect(statements.count == 1) + let stmt = statements[0] + #expect(!stmt.sql.contains("`id`")) + #expect(stmt.sql.contains("`name`")) + #expect(stmt.parameters.compactMap { $0 as? String } == [text, "john@example.com"]) + } + /// This used to expect no statement at all, which dropped the row from the batch while the rest /// of the save committed and reported success. A table of nothing but an identity column and /// defaults is exactly the shape that produces it. @Test( - "Insert with all __DEFAULT__ names no column", + "Insert with every column marked as its default names no column", arguments: [ (DatabaseType.mysql, "() VALUES ()"), (DatabaseType.postgresql, "DEFAULT VALUES"), @@ -128,10 +152,10 @@ struct SQLStatementGeneratorTests { func testInsertAllDefaultNamesNoColumn(databaseType: DatabaseType, expected: String) throws { let generator = try makeGenerator(databaseType: databaseType) let insertedRowData: [RowID: [PluginCellValue]] = [ - .existing(0): ["__DEFAULT__", "__DEFAULT__", "__DEFAULT__"] + .existing(0): [.columnDefault, .columnDefault, .columnDefault] ] let changes: [RowChange] = [ - RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil) + RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil, expressionColumns: [0, 1, 2]) ] let statements = generator.generateStatements( @@ -378,7 +402,7 @@ struct SQLStatementGeneratorTests { #expect(statements[0].parameters[0] == nil) } - @Test("Update with __DEFAULT__ value") + @Test("Update with a cell marked as its default") func testUpdateWithDefaultValue() throws { let generator = try makeGenerator() let changes: [RowChange] = [ @@ -386,7 +410,10 @@ struct SQLStatementGeneratorTests { rowID: .existing(0), type: .update, cellChanges: [ - CellChange(columnIndex: 1, columnName: "name", oldValue: "John", newValue: "__DEFAULT__") + CellChange( + columnIndex: 1, columnName: "name", oldValue: "John", newValue: .columnDefault, + newIsExpression: true + ) ], originalRow: ["1", "John", "john@example.com"] ) @@ -405,6 +432,32 @@ struct SQLStatementGeneratorTests { #expect(stmt.parameters.count == 1) } + @Test("Update binds typed text that only reads like the default", arguments: ["__DEFAULT__", "DEFAULT"]) + func testUpdateBindsUnmarkedDefaultText(text: String) throws { + let generator = try makeGenerator() + let changes: [RowChange] = [ + RowChange( + rowID: .existing(0), + type: .update, + cellChanges: [CellChange(columnIndex: 1, columnName: "name", oldValue: "John", newValue: .text(text))], + originalRow: ["1", "John", "john@example.com"] + ) + ] + + let statements = generator.generateStatements( + from: changes, + insertedRowData: [:], + deletedRowIDs: [], + insertedRowIDs: [] + ) + + #expect(statements.count == 1) + let stmt = statements[0] + #expect(stmt.sql.contains("`name` = ?")) + #expect(!stmt.sql.contains("DEFAULT")) + #expect(stmt.parameters.compactMap { $0 as? String } == [text, "1"]) + } + @Test("Update with a cell marked as SQL inlines it") func testUpdateWithSQLFunction() throws { let generator = try makeGenerator() diff --git a/apple/TableProTests/Core/Compare/CompareDriverRowCutTests.swift b/apple/TableProTests/Core/Compare/CompareDriverRowCutTests.swift new file mode 100644 index 0000000000..1d8668db4f --- /dev/null +++ b/apple/TableProTests/Core/Compare/CompareDriverRowCutTests.swift @@ -0,0 +1,90 @@ +// +// CompareDriverRowCutTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +struct CompareDriverRowCutTests { + private func stream(rowLimit: Int?, ids: [Int]) -> AsyncThrowingStream { + var header = PluginStreamHeader(columns: ["id", "name"], columnTypeNames: ["integer", "text"]) + header.rowLimit = rowLimit + let rows: [PluginRow] = ids.map { [.text(String($0)), .text("n")] } + return AsyncThrowingStream { continuation in + continuation.yield(.header(header)) + continuation.yield(.rows(rows)) + continuation.finish() + } + } + + private func targetRows(_ ids: [Int]) -> [DataRow] { + ids.map { DataRow(values: ["id": .text(String($0)), "name": .text("n")]) } + } + + private func engine() -> DataDiffEngine { + DataDiffEngine( + options: DataCompareOptions(), + shape: DataComparisonShape( + keyColumns: ["id"], + keyOrders: [.numeric], + comparedColumns: ["name"], + digestColumns: ["id", "name"] + ) + ) + } + + @Test("A stream that ends at the limit its driver declares reads as cut there") + func streamEndingAtTheDeclaredLimitIsCut() async throws { + let provider = StreamingRowProvider(stream: stream(rowLimit: 2, ids: [1, 2])) + while try await provider.nextRow() != nil {} + + #expect(provider.endedAtRowLimit) + #expect(provider.driverRowCut == 2) + } + + @Test("A stream that ends short of its declared limit read every row") + func streamShortOfTheDeclaredLimitIsWhole() async throws { + let provider = StreamingRowProvider(stream: stream(rowLimit: 5, ids: [1, 2])) + while try await provider.nextRow() != nil {} + + #expect(!provider.endedAtRowLimit) + #expect(provider.driverRowCut == nil) + } + + // A HANA source stops at 5,000,000 rows: every target row past them read as a delete, and a sync removed them. + @Test("A source the driver cut short reports no deletes past the cut") + func cutSourceProducesNoDeletesPastIt() async throws { + let summary = try await engine().compare( + source: StreamingRowProvider(stream: stream(rowLimit: 2, ids: [1, 2])), + target: ArrayRowProvider(rows: targetRows([1, 2, 3, 4])) + ) + + #expect(summary.deleteCount == 0) + #expect(summary.identicalCount == 2) + #expect(summary.stoppedAtRowLimit) + #expect(summary.sourceRowCut == 2) + #expect(summary.targetRowCut == nil) + #expect(summary.resumeKey == [.text("2")]) + } + + @Test("A target the driver cut short reports no inserts past the cut") + func cutTargetProducesNoInsertsPastIt() async throws { + let summary = try await engine().compare( + source: ArrayRowProvider(rows: targetRows([1, 2, 3, 4])), + target: StreamingRowProvider(stream: stream(rowLimit: 2, ids: [1, 2])) + ) + + #expect(summary.insertCount == 0) + #expect(summary.targetRowCut == 2) + #expect(summary.stoppedAtRowLimit) + } + + @Test("The cut is named in the wording the export uses") + func cutWordingMatchesTheExport() { + #expect(StreamRowCutWarning.text(table: "orders", limit: 5_000_000).hasPrefix("orders: ")) + } +} diff --git a/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift b/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift index d6ba71ac85..04baec1f63 100644 --- a/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift +++ b/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift @@ -99,14 +99,27 @@ struct CompareSQLLiteralTests { #expect(CompareSQLLiteral.prefixed("0x89504E47", databaseType: .mssql) == "0x89504E47") } + // `'\x…'::bytea` reads its backslash by `standard_conforming_strings`, and with that off a sync stored other bytes. @Test - func postgresFamilyUsesAByteaCast() throws { - for type in [DatabaseType.postgresql, .cockroachdb, .redshift, .pglite, .kingbasees] { + func postgresFamilyDecodesHex() throws { + for type in [DatabaseType.postgresql, .cockroachdb, .pglite] { let literal = try #require(CompareSQLLiteral.binaryLiteral(for: png, databaseType: type)) - #expect(literal == "'\\x89504e47'::bytea", "\(type.rawValue) uses a bytea literal") + #expect(literal == "decode('89504E47', 'hex')", "\(type.rawValue) decodes hex") } } + // KingbaseES documents `decode` only as Oracle's DECODE; its bytea decoder is not the PostgreSQL one. + @Test + func kingbaseKeepsTheByteaLiteral() { + #expect(CompareSQLLiteral.binaryLiteral(for: png, databaseType: .kingbasees) == "'\\x89504E47'::bytea") + } + + // Redshift has no bytea, and its DECODE is the CASE-like one; a binary column there is VARBYTE. + @Test + func redshiftReadsHexIntoVarbyte() { + #expect(CompareSQLLiteral.binaryLiteral(for: png, databaseType: .redshift) == "from_hex('89504E47')") + } + @Test func sQLServerUsesZeroX() throws { let literal = try #require(CompareSQLLiteral.binaryLiteral(for: png, databaseType: .mssql)) @@ -121,6 +134,12 @@ struct CompareSQLLiteralTests { #expect(literal == "HEXTORAW('89504E47')") } + // `HEXTORAW('')` stores NULL, and a `BLOB NOT NULL` refuses it. + @Test + func oracleWritesAnEmptyValueAsAnEmptyBlob() { + #expect(CompareSQLLiteral.binaryLiteral(for: Data(), databaseType: .oracle) == "EMPTY_BLOB()") + } + /// An engine this build does not name falls back to the driver's own spelling rather than /// guessing at one, which is why the helper returns nil instead of a default. @Test @@ -132,6 +151,6 @@ struct CompareSQLLiteralTests { func emptyDataStillProducesAValidLiteral() throws { let literal = try #require(CompareSQLLiteral.binaryLiteral(for: Data(), databaseType: .postgresql)) - #expect(literal == "'\\x'::bytea") + #expect(literal == "decode('', 'hex')") } } diff --git a/apple/TableProTests/Core/Compare/DataSyncScriptBuilderTests.swift b/apple/TableProTests/Core/Compare/DataSyncScriptBuilderTests.swift index 72c7c861e1..517a89829e 100644 --- a/apple/TableProTests/Core/Compare/DataSyncScriptBuilderTests.swift +++ b/apple/TableProTests/Core/Compare/DataSyncScriptBuilderTests.swift @@ -795,7 +795,7 @@ final class DataSyncScriptBuilderColumnTests: XCTestCase { XCTAssertEqual( statements.map(\.sql), - [#"INSERT INTO "public"."files" ("id", "blob") VALUES (1, '\x8950'::bytea)"#] + [#"INSERT INTO "public"."files" ("id", "blob") VALUES (1, decode('8950', 'hex'))"#] ) } diff --git a/apple/TableProTests/Core/Coordinators/RowEditingCoordinatorUUIDKeyTests.swift b/apple/TableProTests/Core/Coordinators/RowEditingCoordinatorUUIDKeyTests.swift index 71a31a2c8f..1386171d49 100644 --- a/apple/TableProTests/Core/Coordinators/RowEditingCoordinatorUUIDKeyTests.swift +++ b/apple/TableProTests/Core/Coordinators/RowEditingCoordinatorUUIDKeyTests.swift @@ -71,7 +71,7 @@ struct RowEditingCoordinatorUUIDKeyTests { coordinator.addNewRow() - #expect(try #require(lastRow(of: coordinator)).values[0] == .text("__DEFAULT__")) + #expect(try #require(lastRow(of: coordinator)).values[0] == .columnDefault) } @Test("Add Row on an integer key leaves it as before") diff --git a/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift b/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift index fb7a319753..10d189f39a 100644 --- a/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift +++ b/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift @@ -277,6 +277,87 @@ struct RewindPlannerTests { #expect(queries.first?.contains("SELECT *") == false) } + @Test("An edit that changed the key is read and restored under the key the save gave it") + func movedKeyIsFoundUnderItsNewKey() throws { + let moved = operation(kind: .update, preImage: ["k5", "Ada"], postImage: ["k6", "Ada"], writtenColumns: ["id"]) + let rewind = planner(operations: [moved]) + + let query = try #require(rewind.readQueries().first) + #expect(query.contains("k6")) + #expect(query.contains("k5")) + + let plan = try rewind.plan(currentRows: [RewindCurrentRow(values: ["k6", "Ada"])]) + #expect(plan.rows.map { $0.outcome } == [.willRestore]) + #expect(plan.statements.first?.sql.hasPrefix("UPDATE") == true) + #expect(plan.statements.first?.parameters.compactMap { $0 as? String } == ["k5", "k6"]) + } + + @Test("A moved row a rewind already put back reads as restored, and another row on its old key does not") + func movedKeyAlreadyRestored() throws { + let moved = operation( + kind: .update, preImage: ["k5", "Ada"], postImage: ["k6", "Grace"], writtenColumns: ["id", "name"] + ) + + let restored = try planner(operations: [moved]).plan(currentRows: [RewindCurrentRow(values: ["k5", "Ada"])]) + #expect(restored.rows.map { $0.outcome } == [.alreadyRestored]) + #expect(restored.statements.isEmpty) + + let taken = try planner(operations: [moved]).plan(currentRows: [RewindCurrentRow(values: ["k5", "Bo"])]) + #expect(taken.rows.map { $0.outcome } == [.rowMissing]) + #expect(taken.statements.isEmpty) + } + + @Test("A row that took the key a save moved to is left alone, though the save wrote only the key") + func rowOnTheMovedKeyIsNotRewritten() throws { + let moved = operation(kind: .update, preImage: ["k5", "Ada"], postImage: ["k6", "Ada"], writtenColumns: ["id"]) + + let plan = try planner(operations: [moved]).plan(currentRows: [RewindCurrentRow(values: ["k6", "Bo"])]) + + #expect(plan.rows.map { $0.outcome } == [.changedSinceSave]) + #expect(plan.statements.isEmpty) + } + + @Test("A moved row is not held to a column the server computes") + func movedKeyIgnoresGeneratedColumns() throws { + let columns = ["id", "name", "updated_at"] + let moved = RowWriteOperation( + kind: .update, target: target, columns: columns, primaryKeyColumns: ["id"], + preImage: ["k5", "Ada", "2026-01-01"], postImage: ["k6", "Ada", "2026-01-01"], + writtenColumns: ["id"], refusal: nil + ) + let record = RewindRecord( + id: UUID(), historyId: nil, connectionId: UUID(), databaseType: .sqlite, + target: target, capturedAt: Date(timeIntervalSince1970: 0), + generatedColumns: ["updated_at"], identityColumns: [], operations: [moved] + ) + let planner = RewindPlanner( + record: record, + factory: RowChangeStatementFactory( + tableName: target.table, schemaName: nil, columns: columns, primaryKeyColumns: ["id"], + generatedColumns: ["updated_at"], databaseType: .sqlite, pluginDriver: nil + ), + queryBuilder: sqliteQueryBuilder + ) + + let plan = try planner.plan(currentRows: [RewindCurrentRow(values: ["k6", "Ada", "2026-06-30"])]) + + #expect(plan.rows.map { $0.outcome } == [.willRestore]) + } + + @Test("Undoing a restore takes back what the restore did last first") + func inverseRecordRunsBackwards() throws { + let deleted = operation(kind: .delete, preImage: ["7", "Ada"], postImage: nil, writtenColumns: columns) + let inserted = operation(kind: .insert, preImage: nil, postImage: ["8", "Ada"], writtenColumns: columns) + let plan = try planner(operations: [deleted, inserted]) + .plan(currentRows: [RewindCurrentRow(values: ["8", "Ada"])]) + + let inverse = RewindExecutor.inverseOperations(of: plan) + #expect(inverse.map { $0.kind } == [.delete, .insert]) + + let undo = try planner(operations: inverse).plan(currentRows: [RewindCurrentRow(values: ["7", "Ada"])]) + #expect(undo.statements.map { String($0.sql.prefix(6)) } == ["DELETE", "INSERT"]) + } + @Test("The inverse runs newest first, so a value one row freed is not taken before the other gives it up") func inverseRunsInReverseOrder() throws { let first = operation(kind: .delete, preImage: ["7", "Ada"], postImage: nil, writtenColumns: columns) diff --git a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift index e688dff4ff..570e20367b 100644 --- a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift @@ -255,8 +255,8 @@ struct RowChangeStatementFactoryCoverageTests { let engine = DatabaseType(rawValue: "ProbeEngine") let hostOnly = LegacyStatementStubDriver { _, _, _, _ in nil } let inserted = RowID.inserted(UUID()) - let allDefaults: [PluginCellValue] = [.text("__DEFAULT__"), .text("__DEFAULT__")] - let changes = [nameEdit(), RowChange(rowID: inserted, type: .insert)] + let allDefaults: [PluginCellValue] = [.columnDefault, .columnDefault] + let changes = [nameEdit(), RowChange(rowID: inserted, type: .insert, expressionColumns: [0, 1])] let generator = try SQLStatementGenerator( tableName: "t", columns: columns, primaryKeyColumns: ["_id"], databaseType: engine, diff --git a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift index 93c24b8d32..0109aeb1ec 100644 --- a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift @@ -36,6 +36,21 @@ private final class ExpressionRecordingDriver: PluginDriverStub, @unchecked Send func quoteIdentifier(_ name: String) -> String { "[\(name)]" } } +private final class RestoringDriver: PluginDriverStub, @unchecked Sendable { + func generateIdentityPreservingInsert( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + rows: [[PluginCellValue]], + absentCells: [Int: Set] + ) -> [(statement: String, parameters: [PluginCellValue])]? { + rows.map { (statement: "INSERT INTO [events] VALUES (?, ?, ?)", parameters: $0) } + } + + func quoteIdentifier(_ name: String) -> String { "[\(name)]" } +} + @MainActor struct RowChangeStatementFactoryExpressionTests { private let columns = ["id", "name", "stamped_at"] @@ -192,7 +207,7 @@ struct RowChangeStatementFactoryExpressionTests { @Test("SQLite refuses DEFAULT on a saved row, naming the column") func sqliteDefaultUpdateIsRefused() { - let change = edit(to: .text(PluginCellValue.defaultMarkerText), isExpression: false, column: 1) + let change = edit(to: .columnDefault, isExpression: true, column: 1) let refused = refusal { _ = try factory(databaseType: .sqlite).statements(for: [change]) } #expect(refused?.kind == .update) #expect(refused?.reason.contains("name") == true) @@ -202,8 +217,8 @@ struct RowChangeStatementFactoryExpressionTests { func sqliteDefaultInsertIsWritten() throws { let rowID = RowID.inserted(UUID()) let statements = try factory(databaseType: .sqlite).statements( - for: [RowChange(rowID: rowID, type: .insert)], - insertedRowData: [rowID: ["1", .text(PluginCellValue.defaultMarkerText), "2026-01-01"]], + for: [RowChange(rowID: rowID, type: .insert, expressionColumns: [1])], + insertedRowData: [rowID: ["1", .columnDefault, "2026-01-01"]], insertedRowIDs: [rowID] ) #expect(statements.count == 1) @@ -212,8 +227,69 @@ struct RowChangeStatementFactoryExpressionTests { @Test("PostgreSQL takes DEFAULT on a saved row") func postgresDefaultUpdate() throws { - let change = edit(to: .text(PluginCellValue.defaultMarkerText), isExpression: false, column: 1) + let change = edit(to: .columnDefault, isExpression: true, column: 1) let statements = try factory().statements(for: [change]) #expect(statements.first?.sql.contains("\"name\" = DEFAULT") == true) } + + @Test("Text a row holds or a user typed that reads like the default is written as that text", arguments: [ + "__DEFAULT__", "DEFAULT", + ]) + func unmarkedDefaultTextIsAValue(text: String) throws { + let saved = try factory(databaseType: .sqlite).statements(for: [edit(to: .text(text), isExpression: false, column: 1)]) + #expect(saved.first?.sql.contains("DEFAULT") == false) + #expect(saved.first?.parameters.first as? String == text) + + let rowID = RowID.inserted(UUID()) + let inserted = try factory(databaseType: .sqlite).statements( + for: [RowChange(rowID: rowID, type: .insert)], + insertedRowData: [rowID: ["1", .text(text), "2026-01-01"]], + insertedRowIDs: [rowID] + ) + #expect(inserted.first?.parameters.compactMap { $0 as? String } == ["1", text, "2026-01-01"]) + } + + @Test("A driver is handed a default in the spelling it reads, with no SQL mark, whatever it declares") + func defaultReachesADriverInItsSpelling() throws { + let driver = ExpressionRecordingDriver(capabilities: []) + _ = try factory(databaseType: .mssql, driver: driver).statements( + for: [edit(to: .columnDefault, isExpression: true, column: 1)] + ) + #expect(driver.changes.first?.cellChanges.first?.newValue == .pluginDefaultMarker) + #expect(driver.changes.first?.expressionColumns == nil) + } + + // A driver may write DEFAULT for it, and none can say it will not, so every one that writes its own rows is refused. + @Test("Text spelled the way drivers read a default is refused by a driver that writes its own rows", arguments: [ + DatabaseType.mssql, DatabaseType.redis, + ]) + func textADriverMayReadAsDefaultIsRefused(databaseType: DatabaseType) { + let driver = ExpressionRecordingDriver(capabilities: []) + let refused = refusal { + _ = try factory(databaseType: databaseType, driver: driver).statements( + for: [edit(to: "__DEFAULT__", isExpression: false, column: 1)] + ) + } + #expect(refused?.kind == .update) + #expect(refused?.reason.contains("name") == true) + } + + @Test("A deleted row a driver puts back is refused when it holds that text", arguments: [ + DatabaseType.dameng, DatabaseType.redis, + ]) + func restoredTextADriverMayReadAsDefaultIsRefused(databaseType: DatabaseType) { + let refused = refusal { + _ = try factory(databaseType: databaseType, driver: RestoringDriver()) + .restoreStatements(rows: [["1", "__DEFAULT__", "2026-01-01"]]) + } + #expect(refused?.kind == .insert) + #expect(refused?.reason.contains("name") == true) + } + + @Test("A deleted row a driver puts back with no such text is restored") + func restoredRowWithoutThatTextIsWritten() throws { + let restored = try factory(databaseType: .dameng, driver: RestoringDriver()) + .restoreStatements(rows: [["1", "Ada", "2026-01-01"]]) + #expect(restored.statements.first?.parameters.compactMap { $0 as? String } == ["1", "Ada", "2026-01-01"]) + } } diff --git a/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift b/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift index d4c40a9899..fdf82b33af 100644 --- a/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift @@ -78,14 +78,14 @@ struct RowWriteOperationBuilderTests { #expect(now.first?.refusal == .serverComputedValue) let marker = operations( - changes: [ - cellEdit( - column: "updated_at", index: 2, from: "2026-01-01", - to: .text(PluginCellValue.defaultMarkerText) - ), - ] + changes: [cellEdit(column: "updated_at", index: 2, from: "2026-01-01", to: .columnDefault, isExpression: true)] ) #expect(marker.first?.refusal == .serverComputedValue) + + let text = operations( + changes: [cellEdit(column: "updated_at", index: 2, from: "2026-01-01", to: "__DEFAULT__")] + ) + #expect(text.first?.refusal == nil) } @Test("A generated column cannot be written back") @@ -136,8 +136,8 @@ struct RowWriteOperationBuilderTests { func serverAssignedKeyIsRefused() { let change = RowChange(rowID: .existing(0), type: .insert) let withMarker = operations( - changes: [change], - insertedRowData: [.existing(0): [.text(PluginCellValue.defaultMarkerText), "Ada", "2026-01-01"]], + changes: [RowChange(rowID: .existing(0), type: .insert, expressionColumns: [0])], + insertedRowData: [.existing(0): [.columnDefault, "Ada", "2026-01-01"]], insertedRowIDs: [.existing(0)] ) #expect(withMarker.first?.refusal == .serverAssignedKey) @@ -184,6 +184,31 @@ struct RowWriteOperationBuilderTests { #expect(literal.first?.refusal == nil) } + @Test("The record keeps the order the rows were changed in after an earlier change is cancelled") + func recordFollowsEditOrder() { + var pending = PendingChanges() + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Ada", newValue: "Grace", originalRow: ["7", "Ada", "2026-01-01"] + ) + pending.recordRowDeletion(rowID: .existing(1), originalRow: ["8", "Bo", "2026-01-01"]) + let inserted = RowID.inserted(UUID()) + pending.recordRowInsertion(rowID: inserted, values: ["9", "Bo", "2026-01-01"]) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "Grace", newValue: "Ada" + ) + #expect(pending.changes.map { $0.type } == [.insert, .delete]) + + let result = operations( + changes: pending.changes, + insertedRowData: pending.insertedRowData, + deletedRowIDs: pending.deletedRowIDs, + insertedRowIDs: pending.insertedRowIDs + ) + + #expect(result.map { $0.kind } == [.delete, .insert]) + } + @Test("An insert carrying its own key is reversible") func userSuppliedKeyIsReversible() { let result = operations( diff --git a/apple/TableProTests/Core/Database/DatabaseManagerTests.swift b/apple/TableProTests/Core/Database/DatabaseManagerTests.swift index 1d05998bfb..10989dcb28 100644 --- a/apple/TableProTests/Core/Database/DatabaseManagerTests.swift +++ b/apple/TableProTests/Core/Database/DatabaseManagerTests.swift @@ -412,8 +412,11 @@ struct DatabaseManagerDatabaseSwitchTests { let pluginDriver = DatabaseSwitchingDriver() let adapter = PluginDriverAdapter(connection: connection, pluginDriver: pluginDriver) - await DatabaseManager.shared.restoreSchemaAndDatabase(on: adapter, savedSchema: "idb", savedDatabase: "ice") + let warning = await DatabaseManager.shared.restoreSchemaAndDatabase( + on: adapter, savedSchema: "idb", savedDatabase: "ice" + ) #expect(pluginDriver.switches == ["database ice", "schema idb"]) + #expect(warning == nil) } } diff --git a/apple/TableProTests/Core/Database/SchemaWriteGateDenialTests.swift b/apple/TableProTests/Core/Database/SchemaWriteGateDenialTests.swift new file mode 100644 index 0000000000..5af1cd879d --- /dev/null +++ b/apple/TableProTests/Core/Database/SchemaWriteGateDenialTests.swift @@ -0,0 +1,86 @@ +// +// SchemaWriteGateDenialTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +// A denial came back as a failed statement, so Create Database, the schema editor and Create Table +// showed the user's own Cancel at the gate as an error. +@MainActor +struct SchemaWriteGateDenialTests { + private let scope = DatabaseScope(connectionId: UUID(), database: "app", schema: nil) + + private func gateFailure(of body: () async throws -> Void) async -> GatedOperationFailure? { + do { + try await body() + return nil + } catch { + return GatedOperationFailure(error) + } + } + + private func runCreateDatabase(gate: any ExecutionGate) async -> GatedOperationFailure? { + await gateFailure { + try await DatabaseManager.shared.runContainerOperation( + description: "Create database \"sales\"", + kind: .schemaMutation, + scope: scope, + databaseType: .postgresql, + event: nil, + gate: gate + ) { _ in + Issue.record("The driver was asked after the gate said no") + } + } + } + + @Test("A Cancel at the gate ends a database operation quietly, before the driver is asked") + func containerOperationCancelIsQuiet() async { + let failure = await runCreateDatabase(gate: AlwaysDenyGate.cancelling) + + #expect(failure == .cancelledByUser) + #expect(failure?.message == nil) + } + + @Test("A refused database operation says why and is not a failed statement") + func containerOperationRefusalExplainsItself() async { + let failure = await runCreateDatabase(gate: AlwaysDenyGate(reason: "Safe Mode is read-only")) + + #expect(failure == .refused("Safe Mode is read-only")) + #expect(failure?.reportsFailure == false) + } + + @Test("A Cancel at the gate ends a schema editor apply quietly") + func containerStatementsCancelIsQuiet() async { + let failure = await gateFailure { + try await DatabaseManager.shared.runContainerStatements( + .statements([SchemaStatement(sql: "CREATE SCHEMA sales", description: "Create", isDestructive: false)]), + description: "Create Schema", + scope: scope, + databaseType: .postgresql, + event: nil, + gate: AlwaysDenyGate.cancelling + ) + } + + #expect(failure == .cancelledByUser) + } + + @Test("A Cancel at the gate ends a Create Table quietly") + func createTableCancelIsQuiet() async { + let failure = await gateFailure { + try await DatabaseManager.shared.executeCreateTable( + statements: ["CREATE TABLE invoices (id INT)"], + databaseType: .postgresql, + scope: scope, + gate: AlwaysDenyGate.cancelling + ) + } + + #expect(failure == .cancelledByUser) + #expect(failure?.message == nil) + } +} diff --git a/apple/TableProTests/Core/Database/StartupCommandWarningTests.swift b/apple/TableProTests/Core/Database/StartupCommandWarningTests.swift new file mode 100644 index 0000000000..11fbac5867 --- /dev/null +++ b/apple/TableProTests/Core/Database/StartupCommandWarningTests.swift @@ -0,0 +1,135 @@ +// +// StartupCommandWarningTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct ServerRefusal: PluginDriverError { + let pluginErrorMessage: String + let pluginSqlState: String? +} + +// A startup command can be a safety rail, so a session that opened without one says so on the connection's warning. +@MainActor +struct StartupCommandWarningTests { + @Test("A startup command the server refused is named on the session's warning, the ones that ran are not") + func refusedStartupCommandIsNamed() async { + let driver = MockDatabaseDriver() + driver.failsStatementContaining = ("SET ROLE", DatabaseError.queryFailed("role \"app_ro\" does not exist")) + + let failures = await DatabaseManager.shared.applyTimeoutAndStartupCommands( + on: driver, + queryTimeoutSeconds: 0, + startupCommands: "SET ROLE app_ro\nSET search_path = app", + connectionName: "Reporting" + ) + let warning = DatabaseManager.startupWarning(for: failures) + + #expect(failures.map { $0.statement } == ["SET ROLE app_ro"]) + #expect(warning?.contains("SET ROLE app_ro") == true) + #expect(warning?.contains("SET search_path") == false) + } + + // PostgreSQL can quote the statement's values back in its message, so only the code reaches the screen. + @Test("The warning names the server's code, never its message") + func warningShowsTheCodeNotTheMessage() { + let refusal = ServerRefusal( + pluginErrorMessage: "invalid value for parameter \"app.api_key\": \"sk-live-123\"", pluginSqlState: "22023" + ) + let failure = StartupCommandFailure(statement: "SET app.api_key = 42", error: refusal) + let warning = DatabaseManager.startupWarning(for: [failure]) + + #expect(failure.code == "22023") + #expect(warning?.contains("SET app.api_key = 42") == true) + #expect(warning?.contains("22023") == true) + #expect(warning?.contains("sk-live-123") == false) + } + + @Test("Startup commands that all ran leave no warning") + func successfulStartupCommandsLeaveNoWarning() async { + let driver = MockDatabaseDriver() + + let failures = await DatabaseManager.shared.applyTimeoutAndStartupCommands( + on: driver, queryTimeoutSeconds: 0, startupCommands: "SET search_path = app", connectionName: "Reporting" + ) + + #expect(DatabaseManager.startupWarning(for: failures) == nil) + } + + @Test("Several failed startup commands are each named") + func severalFailuresAreEachNamed() { + let warning = DatabaseManager.startupWarning(for: [ + StartupCommandFailure(statement: "SET ROLE app_ro", code: "42704"), + StartupCommandFailure(statement: "SET default_transaction_read_only = on", code: nil), + ]) + + #expect(warning?.contains("SET ROLE app_ro (42704)") == true) + #expect(warning?.contains("SET default_transaction_read_only = on") == true) + } + + // A startup command can carry a password or a key, and the banner is on screen, screen sharing included. + @Test("A failed startup command is shown only up to its first literal or credential clause") + func shownStatementStopsBeforeSecrets() { + #expect(DatabaseManager.shownStatement("SELECT set_config('app.api_key', 'sk-live-123', false)") + == "SELECT set_config(…") + #expect(DatabaseManager.shownStatement("SET ROLE reporting IDENTIFIED BY hunter2") == "SET ROLE reporting…") + #expect(DatabaseManager.shownStatement("ALTER USER app WITH PASSWORD 'x'") == "ALTER USER app WITH…") + #expect(DatabaseManager.shownStatement("SET search_path = app") == "SET search_path = app") + + let warning = DatabaseManager.startupWarning(for: [ + StartupCommandFailure(statement: "SET ROLE reporting IDENTIFIED BY hunter2", code: "28000"), + ]) + #expect(warning?.contains("hunter2") == false) + } + + @Test("A reconnect whose startup commands all ran drops the banner about the session before it") + func cleanReconnectClearsTheStartupWarning() { + let connectionId = UUID() + let manager = DatabaseManager.shared + defer { manager.reportedStartupFailures.removeValue(forKey: connectionId) } + + let failed = manager.openingWarning( + driver: MockDatabaseDriver(), + startupFailures: [StartupCommandFailure(statement: "SET ROLE app_ro", code: "42704")], + for: connectionId + ) + let clean = manager.openingWarning(driver: MockDatabaseDriver(), startupFailures: [], for: connectionId) + + #expect(failed?.contains("SET ROLE app_ro") == true) + #expect(clean == nil) + #expect(manager.reportedStartupFailures[connectionId]?.isEmpty == true) + } + + @Test("The login's warning and the startup warning are both kept") + func warningsAreJoined() { + #expect(DatabaseManager.joinedWarning(["ORA-28002", nil, "startup"]) == "ORA-28002\n\nstartup") + #expect(DatabaseManager.joinedWarning([nil, ""]) == nil) + } + + // The pool opens entries for as long as the window is open, so a failure it was already told about stays dismissed. + @Test("A pool or lane session's failed startup command reaches the window once") + func poolFailureIsReportedOnce() { + let connection = TestFixtures.makeConnection(name: "Startup rail") + let manager = DatabaseManager.shared + manager.injectSession(ConnectionSession(connection: connection), for: connection.id) + defer { manager.removeSession(for: connection.id) } + let role = StartupCommandFailure(statement: "SET ROLE app_ro", code: "42704") + let path = StartupCommandFailure(statement: "SET search_path = app", code: "3F000") + + manager.reportStartupFailures([role], for: connection.id) + #expect(manager.session(for: connection.id)?.connectionWarning?.contains("SET ROLE app_ro") == true) + + manager.dismissConnectionWarning(for: connection.id) + manager.reportStartupFailures([role], for: connection.id) + #expect(manager.session(for: connection.id)?.connectionWarning == nil) + + manager.reportStartupFailures([role, path], for: connection.id) + let warning = manager.session(for: connection.id)?.connectionWarning + #expect(warning?.contains("SET search_path = app") == true) + #expect(warning?.contains("SET ROLE") == false) + } +} diff --git a/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift b/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift index e67a5599bd..0b19025ea9 100644 --- a/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift +++ b/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift @@ -437,3 +437,89 @@ struct TriggerEditSafetyTests { #expect(requests.isEmpty) } } + +// A Cancel in the gate's confirmation reached the trigger editor as an error row under the SQL. +@MainActor +struct TriggerGateDenialTests { + private func install() async throws -> (connection: DatabaseConnection, scope: DatabaseScope, stub: StubTriggerDriver) { + let connection = TestFixtures.makeConnection(database: "app", type: .postgresql) + let stub = StubTriggerDriver() + stub.dropToReturn = "DROP TRIGGER t" + let adapter = PluginDriverAdapter(connection: connection, pluginDriver: stub) + try await adapter.connect() + DatabaseManager.shared.injectSession(ConnectionSession(connection: connection, driver: adapter), for: connection.id) + let scope = DatabaseScope(connectionId: connection.id, database: "app", schema: nil) + MetadataConnectionPool.shared.injectEntry(adapter, scope: scope) + return (connection, scope, stub) + } + + private func uninstall(_ connection: DatabaseConnection) { + MetadataConnectionPool.shared.closeAll(connectionId: connection.id) + DatabaseManager.shared.removeSession(for: connection.id) + } + + private func failure(of body: () async throws -> Void) async -> GatedOperationFailure? { + do { + try await body() + return nil + } catch { + return GatedOperationFailure(error) + } + } + + @Test("A Cancel at the gate ends a trigger save and a trigger drop quietly, with nothing run") + func cancelIsQuiet() async throws { + let installed = try await install() + defer { uninstall(installed.connection) } + + let save = await failure { + try await TriggerEditing.apply( + scope: installed.scope, + connection: installed.connection, + tableName: "orders", + sql: "CREATE TRIGGER t", + isEdit: false, + originalName: nil, + originalDefinition: nil, + gate: AlwaysDenyGate.cancelling + ) + } + let drop = await failure { + try await TriggerEditing.drop( + scope: installed.scope, + connection: installed.connection, + tableName: "orders", + name: "t", + gate: AlwaysDenyGate.cancelling + ) + } + + #expect(save == .cancelledByUser) + #expect(drop == .cancelledByUser) + #expect(save?.message == nil) + #expect(installed.stub.executedQueries.isEmpty) + } + + @Test("A refused trigger save says why and is not a failed statement") + func refusalExplainsItself() async throws { + let installed = try await install() + defer { uninstall(installed.connection) } + + let save = await failure { + try await TriggerEditing.apply( + scope: installed.scope, + connection: installed.connection, + tableName: "orders", + sql: "CREATE TRIGGER t", + isEdit: false, + originalName: nil, + originalDefinition: nil, + gate: AlwaysDenyGate(reason: "Safe Mode is read-only") + ) + } + + #expect(save == .refused("Safe Mode is read-only")) + #expect(save?.reportsFailure == false) + #expect(installed.stub.executedQueries.isEmpty) + } +} diff --git a/apple/TableProTests/Core/LSP/LSPTransportBrokenPipeTests.swift b/apple/TableProTests/Core/LSP/LSPTransportBrokenPipeTests.swift new file mode 100644 index 0000000000..7af8c30567 --- /dev/null +++ b/apple/TableProTests/Core/LSP/LSPTransportBrokenPipeTests.swift @@ -0,0 +1,48 @@ +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +struct LSPTransportBrokenPipeTests { + static let perl = "/usr/bin/perl" + + static var hasPerl: Bool { + FileManager.default.isExecutableFile(atPath: perl) + } + + // A server on its way out: it stops reading, asks the client something, then says it has stopped. + private static let closingServer = #""" + close STDIN; $| = 1; + for my $body ('{"jsonrpc":"2.0","id":7,"method":"workspace/configuration","params":{}}', + '{"jsonrpc":"2.0","method":"test/inputClosed"}') { + print "Content-Length: " . length($body) . "\r\n\r\n" . $body; + } + sleep 30; + """# + + @Test( + "A server that stopped reading fails the request instead of ending the app", + .enabled(if: LSPTransportBrokenPipeTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func closedInputFailsTheRequest() async throws { + let transport = LSPTransport() + let inputClosed = AsyncGate() + await transport.onNotification(method: "test/inputClosed") { _ in + Task { await inputClosed.open() } + } + try await transport.start(executablePath: Self.perl, arguments: ["-e", Self.closingServer]) + await inputClosed.wait() + + let error = await #expect(throws: (any Error).self) { + _ = try await transport.sendRequest(method: "test/afterClose", params: EmptyLSPParams()) + } + await transport.stop() + + let failure = error.map { $0 as NSError } + let posix = failure?.domain == NSPOSIXErrorDomain + ? failure + : failure?.underlyingErrors.first.map { $0 as NSError } + #expect(posix?.code == Int(EPIPE)) + } +} diff --git a/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCutTests.swift b/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCutTests.swift new file mode 100644 index 0000000000..c9e9c4e532 --- /dev/null +++ b/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCutTests.swift @@ -0,0 +1,107 @@ +// +// ObjectCopyRowCutTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class CappedSourceDriver: PluginDriverStub, @unchecked Sendable { + let declaredLimit: Int? + let rowCount: Int + + init(declaredLimit: Int?, rowCount: Int) { + self.declaredLimit = declaredLimit + self.rowCount = rowCount + } + + func streamRows(query: String) -> AsyncThrowingStream { + var header = PluginStreamHeader(columns: ["id", "name"], columnTypeNames: []) + header.rowLimit = declaredLimit + let rows: [PluginRow] = (0 ..< rowCount).map { [.text(String($0)), .text("row\($0)")] } + return AsyncThrowingStream { continuation in + continuation.yield(.header(header)) + continuation.yield(.rows(rows)) + continuation.finish() + } + } +} + +private final class WritingTargetDriver: PluginDriverStub, @unchecked Sendable { + var parameterStyle: ParameterStyle { .questionMark } + + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ name: String) -> String { "`\(name)`" } +} + +struct ObjectCopyRowCutTests { + private func step() -> ObjectCopyTableStep { + ObjectCopyTableStep( + selection: ObjectCopySelection(kind: .table, name: "orders", schema: "public"), + dropStatements: [], + sequenceStatements: [], + createStatements: [], + truncateStatements: [], + columns: ["id", "name"], + targetColumnTypeNames: [:], + primaryKeyColumns: ["id"], + sourceQuery: "SELECT `id`, `name` FROM `public`.`orders`", + targetTable: "orders", + targetSchema: "public", + estimatedRows: nil, + copiesData: true, + copiesIdentityColumn: false, + coercer: nil, + serverSideInsert: nil, + note: nil + ) + } + + private func copy(from source: CappedSourceDriver) async throws -> ObjectCopyRowCopier.Outcome { + try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) + .copy(from: source, to: WritingTargetDriver()) { _ in } + } + + // A 6,000,000-row SAP HANA table arrived as 5,000,000 rows and the copy reported success. + @Test("A copy whose source stream stopped at the driver's limit says where") + func cutStreamIsReported() async throws { + let outcome = try await copy(from: CappedSourceDriver(declaredLimit: 3, rowCount: 3)) + + #expect(outcome.inserted == 3) + #expect(outcome.rowCut == 3) + } + + @Test("A copy that read fewer rows than the driver's limit is whole") + func shortStreamIsWhole() async throws { + let outcome = try await copy(from: CappedSourceDriver(declaredLimit: 10, rowCount: 3)) + + #expect(outcome.inserted == 3) + #expect(outcome.rowCut == nil) + } + + @Test("A cut table's outcome carries the export's wording") + func cutOutcomeCarriesTheWarning() { + let selection = ObjectCopySelection(kind: .table, name: "orders", schema: "public") + let result = ObjectCopyRunResult( + outcomes: [ + ObjectCopyObjectOutcome( + selection: selection, + rowsCopied: 3, + error: nil, + warning: StreamRowCutWarning.text(table: selection.qualifiedName, limit: 3) + ) + ], + rowsCopied: 3, + cancelled: false, + createdDatabase: nil + ) + + #expect(result.warnings == [StreamRowCutWarning.text(table: selection.qualifiedName, limit: 3)]) + #expect(result.failedCount == 0) + } +} diff --git a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolTests.swift b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolTests.swift index 0a184ad353..04b349fbe9 100644 --- a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolTests.swift +++ b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolTests.swift @@ -84,7 +84,7 @@ struct MetadataConnectionPoolTests { func prepareSessionPassesThrough() async throws { let driver = MockDatabaseDriver() - try await MetadataConnectionPool.prepareSession( + let failures = try await MetadataConnectionPool.prepareSession( driver, queryTimeoutSeconds: 30, startupCommands: nil, @@ -93,6 +93,25 @@ struct MetadataConnectionPoolTests { ) #expect(driver.applyQueryTimeoutValues == [30]) + #expect(failures.isEmpty) + } + + // A pool entry or a lane whose `SET ROLE` failed would otherwise run with the login's own rights unannounced. + @Test("session preparation hands back the startup commands that failed") + func prepareSessionReturnsFailedStartupCommands() async throws { + let driver = MockDatabaseDriver() + driver.failsStatementContaining = ("SET ROLE", DatabaseError.queryFailed("role \"app_ro\" does not exist")) + + let failures = try await MetadataConnectionPool.prepareSession( + driver, + queryTimeoutSeconds: 0, + startupCommands: "SET ROLE app_ro\nSET search_path = app", + connectionName: "test", + timeoutSeconds: 1 + ) + + #expect(failures.map { $0.statement } == ["SET ROLE app_ro"]) + #expect(driver.executedQueries == ["SET ROLE app_ro", "SET search_path = app"]) } @Test("database switch rejects a driver that cannot switch") diff --git a/apple/TableProTests/Core/Services/Query/SelectionSummaryCalculatorTests.swift b/apple/TableProTests/Core/Services/Query/SelectionSummaryCalculatorTests.swift index 2f7e00976f..953b41e5c4 100644 --- a/apple/TableProTests/Core/Services/Query/SelectionSummaryCalculatorTests.swift +++ b/apple/TableProTests/Core/Services/Query/SelectionSummaryCalculatorTests.swift @@ -25,8 +25,7 @@ struct SelectionSummaryCalculatorTests { dataColumns: [Int]? = nil, policy: SelectionSummaryColumnPolicy? = nil, deleted: Set = [], - inserted: Set = [], - modified: [RowID: Set] = [:] + defaulted: [RowID: Set] = [:] ) -> SelectionSummaryInput { SelectionSummaryInput( selection: selection(rects), @@ -39,8 +38,7 @@ struct SelectionSummaryCalculatorTests { columnCount: tableRows.columns.count ), deletedRowIDs: deleted, - insertedRowIDs: inserted, - modifiedCells: modified + defaultedCells: defaulted ) } @@ -113,25 +111,24 @@ struct SelectionSummaryCalculatorTests { #expect(summary.numbers?.sum.decimal == 1) } - @Test("NULL is empty, empty text is a value, and the default marker is empty only where an edit put it") + @Test("NULL is empty, empty text is a value, and DEFAULT is empty only where an edit marked it") func emptyCellRules() async throws { let insertedID = RowID.inserted(UUID()) var table = Self.tableRows( [ [.null, .text("")], - [.text(PluginCellValue.defaultMarkerText), .text(PluginCellValue.defaultMarkerText)], - [.text("7"), .text(PluginCellValue.defaultMarkerText)] + [.text("DEFAULT"), .text("DEFAULT")], + [.text("7"), .columnDefault] ], types: [Self.integer, Self.untyped] ) - table.appendInsertedRow(id: insertedID, values: [.text(PluginCellValue.defaultMarkerText), .text("5")]) + table.appendInsertedRow(id: insertedID, values: [.columnDefault, .text("5")]) let summary = try await Self.summarize( Self.input( table, selecting: [GridRect(rows: 0...3, columns: 0...1)], - inserted: [insertedID], - modified: [.existing(2): [1]] + defaulted: [.existing(2): [1], insertedID: [0]] ) ) diff --git a/apple/TableProTests/Core/Services/RowOperationsFieldAbsenceTests.swift b/apple/TableProTests/Core/Services/RowOperationsFieldAbsenceTests.swift index a0e0ecf3b2..2823bea961 100644 --- a/apple/TableProTests/Core/Services/RowOperationsFieldAbsenceTests.swift +++ b/apple/TableProTests/Core/Services/RowOperationsFieldAbsenceTests.swift @@ -82,7 +82,7 @@ extension GlobalStateSuites { #expect(tableRows.row(withID: nullKept.rowID)?.absentColumns.isEmpty == true) #expect(tableRows.row(withID: missingKept.rowID)?.absentColumns == [2]) #expect(changes.pending.insertedAbsentColumns(forRow: missingKept.rowID) == [2]) - #expect(missingKept.values[0] == .text("__DEFAULT__")) + #expect(missingKept.values[0] == .columnDefault) } @Test("Copying rows records which fields each lacked, in the copied column order") @@ -123,7 +123,7 @@ extension GlobalStateSuites { let (tableRows, result) = paste(clipboard, into: manager) - #expect(result.pastedRows.first?.values == [.text("__DEFAULT__"), "Ada", .null]) + #expect(result.pastedRows.first?.values == [.columnDefault, "Ada", .null]) #expect(tableRows.rows[0].absentColumns.isEmpty) } diff --git a/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift b/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift index 80819290f7..51e4d37d67 100644 --- a/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift +++ b/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift @@ -51,7 +51,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) #expect(values.count == 3) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("Smith, John")) #expect(values[2] == .text("a@test.com")) } @@ -120,7 +120,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("Jo")) #expect(values[2] == .text("e@x.com")) } @@ -136,7 +136,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) #expect(values.count == 3) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("Jo")) #expect(values[2] == .text("e@x.com")) } @@ -151,7 +151,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("2")) #expect(values[2] == .text("3")) } @@ -170,8 +170,8 @@ struct RowOperationsManagerPasteTests { let rows = pasteRows(clipboard, manager: makeManager()) #expect(rows.count == 2) - #expect(rows[0] == [.text("__DEFAULT__"), .text("Alice"), .text("a@test.com")]) - #expect(rows[1] == [.text("__DEFAULT__"), .text("Bob"), .null]) + #expect(rows[0] == [.columnDefault, .text("Alice"), .text("a@test.com")]) + #expect(rows[1] == [.columnDefault, .text("Bob"), .null]) } @Test("Plain-text TSV with commas inside a value parses as TSV, not CSV") @@ -182,7 +182,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) #expect(values.count == 3) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("Mozilla/5.0 (KHTML, like Gecko) Safari")) #expect(values[2] == .text("a@test.com")) } @@ -257,7 +257,7 @@ struct RowOperationsManagerPasteServerOwnedTests { func resetsNonKeyIdentity() { let values = paste(payload, columnIdentity: ["code": .always]) - #expect(values[1] == .text("__DEFAULT__")) + #expect(values[1] == .columnDefault) #expect(values[2] == .text("Ada")) } @@ -265,7 +265,7 @@ struct RowOperationsManagerPasteServerOwnedTests { func resetsGeneratedColumn() { let values = paste(payload, generatedColumns: ["code"]) - #expect(values[1] == .text("__DEFAULT__")) + #expect(values[1] == .columnDefault) #expect(values[2] == .text("Ada")) } diff --git a/apple/TableProTests/Core/Services/RowOperationsManagerTests.swift b/apple/TableProTests/Core/Services/RowOperationsManagerTests.swift index 7358b7b600..c56df2acad 100644 --- a/apple/TableProTests/Core/Services/RowOperationsManagerTests.swift +++ b/apple/TableProTests/Core/Services/RowOperationsManagerTests.swift @@ -118,8 +118,8 @@ struct RowOperationsManagerTests { let result = manager.addNewRow(tableRows: &tableRows) #expect(result != nil) - #expect(result?.values[0] == "__DEFAULT__") - #expect(result?.values[2] == "__DEFAULT__") + #expect(result?.values[0] == .columnDefault) + #expect(result?.values[2] == .columnDefault) } // PostgreSQL leaves `column_default` null for an identity column, so a new row sent an explicit NULL it refuses. @@ -133,7 +133,7 @@ struct RowOperationsManagerTests { let result = manager.addNewRow(tableRows: &tableRows) - #expect(result?.values[0] == "__DEFAULT__") + #expect(result?.values[0] == .columnDefault) #expect(result?.values[1] == .null) } @@ -146,7 +146,7 @@ struct RowOperationsManagerTests { let result = manager.addNewRow(tableRows: &tableRows) - #expect(result?.values[2] == "__DEFAULT__") + #expect(result?.values[2] == .columnDefault) #expect(result?.values[1] == .null) } @@ -159,7 +159,7 @@ struct RowOperationsManagerTests { let result = manager.addNewRow(tableRows: &tableRows) #expect(result?.values[1] == .null) - #expect(result?.values[2] == "__DEFAULT__") + #expect(result?.values[2] == .columnDefault) #expect(tableRows.serverAssignsValue(forColumn: "name") == false) } @@ -246,7 +246,7 @@ struct RowOperationsManagerTests { ) #expect(result != nil) - #expect(result?.values[0] == "__DEFAULT__") + #expect(result?.values[0] == .columnDefault) if case .rowsInserted(let indices) = result?.delta { #expect(indices == IndexSet(integer: 3)) } else { @@ -263,10 +263,40 @@ struct RowOperationsManagerTests { let result = manager.duplicateRow(sourceRowIndex: 1, tableRows: &tableRows) - #expect(result?.values[2] == "__DEFAULT__") + #expect(result?.values[2] == .columnDefault) #expect(result?.values[1] == sourceValues[1]) } + @Test("The change manager names the cells a new row leaves to their default, for the JSON view and Copy as JSON") + func defaultedCellsOfANewRow() throws { + let (manager, changeManager) = makeManager() + var tableRows = emptyTableRows(columnDefaults: ["id": "auto_increment", "email": "'x'"]) + + let added = manager.addNewRow(tableRows: &tableRows) + let result = try #require(added) + + #expect(changeManager.defaultedCells(in: tableRows) == [result.rowID: [0, 2]]) + } + + @Test("A duplicate writes stored text that reads like the default as text, and leaves only its key to the server") + func duplicateRowKeepsDefaultLookingText() throws { + let (manager, changeManager) = makeManager() + var tableRows = TableRows.from( + queryRows: [["7", "__DEFAULT__", "DEFAULT"]], + columns: Self.testColumns, + columnTypes: Self.testColumnTypes + ) + + let duplicated = manager.duplicateRow(sourceRowIndex: 0, tableRows: &tableRows) + let result = try #require(duplicated) + + #expect(changeManager.expressionColumns(forRow: result.rowID) == [0]) + let statements = try changeManager.generateSQL() + #expect(statements.count == 1) + #expect(statements.first?.sql.contains("`id`") == false) + #expect(statements.first?.parameters.compactMap { $0 as? String } == ["__DEFAULT__", "DEFAULT"]) + } + @Test("duplicateRow returns nil for invalid source index") func duplicateRowReturnsNilForInvalidIndex() { let (manager, _) = makeManager() diff --git a/apple/TableProTests/Core/Services/RowOperationsManagerUUIDKeyTests.swift b/apple/TableProTests/Core/Services/RowOperationsManagerUUIDKeyTests.swift index 5601c64f35..4122e13d78 100644 --- a/apple/TableProTests/Core/Services/RowOperationsManagerUUIDKeyTests.swift +++ b/apple/TableProTests/Core/Services/RowOperationsManagerUUIDKeyTests.swift @@ -80,7 +80,7 @@ struct RowOperationsManagerUUIDKeyTests { let result = try #require(manager.addNewRow(tableRows: &rows, uuidKeyForms: Self.uuidKey)) - #expect(result.values[0] == .text("__DEFAULT__")) + #expect(result.values[0] == .columnDefault) #expect(result.prefilledColumns.isEmpty) } @@ -99,7 +99,7 @@ struct RowOperationsManagerUUIDKeyTests { #expect(copyKey != sourceKey) #expect(copy.values[1] == .text("a")) #expect(copy.prefilledColumns == [0]) - #expect(plain.values[0] == .text("__DEFAULT__")) + #expect(plain.values[0] == .columnDefault) } @Test("Duplicating a pending row keeps the SQL staged in it, except in the columns the copy resets") @@ -118,14 +118,16 @@ struct RowOperationsManagerUUIDKeyTests { #expect(uuidText(copy.values[0]) != nil) } - @Test("Duplicating a saved row stages no SQL") + @Test("Duplicating a saved row stages no SQL but the default its reset key takes") func duplicateOfASavedRowHasNoMarks() throws { let (manager, changeManager) = makeManager() var rows = makeRows([[.text("0b1e7c9a-58f4-4b8e-9c55-1f1c86f0a001"), .text("now()"), .null]]) let copy = try #require(manager.duplicateRow(sourceRowIndex: 0, tableRows: &rows)) - #expect(changeManager.expressionColumns(forRow: copy.rowID).isEmpty) + #expect(changeManager.expressionColumns(forRow: copy.rowID) == [0]) + #expect(copy.values[0] == .columnDefault) + #expect(copy.values[1] == .text("now()")) } // MARK: - Paste rows @@ -188,7 +190,7 @@ struct RowOperationsManagerUUIDKeyTests { clipboard: clipboard ) - #expect(result.pastedRows.first?.values[0] == .text("__DEFAULT__")) + #expect(result.pastedRows.first?.values[0] == .columnDefault) } } @@ -249,8 +251,8 @@ extension GlobalStateSuites { let pasted = result.pastedRows.map(\.rowID) #expect(pasted.count == 2) - #expect(changeManager.expressionColumns(forRow: pasted[0]).isEmpty) - #expect(changeManager.expressionColumns(forRow: pasted[1]) == [2]) + #expect(changeManager.expressionColumns(forRow: pasted[0]) == [0]) + #expect(changeManager.expressionColumns(forRow: pasted[1]) == [0, 2]) #expect(result.pastedRows[0].values[2] == .text("CURRENT_TIMESTAMP")) #expect(result.pastedRows[1].values[2] == .text("now()")) } @@ -268,8 +270,9 @@ extension GlobalStateSuites { let result = try paste(manager, into: &rows) - let rowID = try #require(result.pastedRows.first?.rowID) - #expect(changeManager.expressionColumns(forRow: rowID) == [2]) + let pastedRow = try #require(result.pastedRows.first) + #expect(pastedRow.values[0] == .columnDefault) + #expect(changeManager.expressionColumns(forRow: pastedRow.rowID) == [0, 2]) } @Test("Cells the grid copied as wide as the table paste as rows with their marks") @@ -287,8 +290,8 @@ extension GlobalStateSuites { let pasted = result.pastedRows.map(\.rowID) #expect(pasted.count == 2) - #expect(changeManager.expressionColumns(forRow: pasted[0]) == [2]) - #expect(changeManager.expressionColumns(forRow: pasted[1]).isEmpty) + #expect(changeManager.expressionColumns(forRow: pasted[0]) == [0, 2]) + #expect(changeManager.expressionColumns(forRow: pasted[1]) == [0]) } } } diff --git a/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift b/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift index 9c1a9b1dd4..e886b4cf83 100644 --- a/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift +++ b/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift @@ -22,8 +22,8 @@ struct RowPasteForeignTextTests { let rows = try parse("\tAnn\tann@x.com\n\tBob\tbob@x.com") #expect(rows == [ - [.text("__DEFAULT__"), .text("Ann"), .text("ann@x.com")], - [.text("__DEFAULT__"), .text("Bob"), .text("bob@x.com")] + [.columnDefault, .text("Ann"), .text("ann@x.com")], + [.columnDefault, .text("Bob"), .text("bob@x.com")] ]) } @@ -74,7 +74,8 @@ struct RowPasteForeignTextTests { columns: Self.columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard ) - #expect(result.pastedRows.first?.values == [.text("__DEFAULT__"), .text("Ann"), .text("ann@x.com")]) + #expect(result.pastedRows.first?.values == [.columnDefault, .text("Ann"), .text("ann@x.com")]) + #expect(result.pastedRows.first.map { changeManager.expressionColumns(forRow: $0.rowID) } == [0]) } @Test("A blank pasted cell takes the column's default, an empty string where NULL is refused, else NULL") @@ -107,11 +108,12 @@ struct RowPasteForeignTextTests { columns: columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard ) - let unfilled: [PluginCellValue] = [.text("__DEFAULT__"), .text(""), .null, .null] + let unfilled: [PluginCellValue] = [.columnDefault, .text(""), .null, .null] #expect(result.pastedRows.map { $0.values } == [ - [.text("__DEFAULT__"), .text("Ann")] + unfilled, - [.text("__DEFAULT__"), .text("Bob")] + unfilled, + [.columnDefault, .text("Ann")] + unfilled, + [.columnDefault, .text("Bob")] + unfilled, ]) + #expect(result.pastedRows.map { changeManager.expressionColumns(forRow: $0.rowID) } == [[0, 2], [0, 2]]) } @Test("A pasted NULL stays NULL in a column with a default") @@ -136,7 +138,7 @@ struct RowPasteForeignTextTests { columns: columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard ) - #expect(result.pastedRows.first?.values == [.text("__DEFAULT__"), .null]) + #expect(result.pastedRows.first?.values == [.columnDefault, .null]) } } diff --git a/apple/TableProTests/Core/Shortcuts/PayloadRowPlannerHeaderMatchTests.swift b/apple/TableProTests/Core/Shortcuts/PayloadRowPlannerHeaderMatchTests.swift new file mode 100644 index 0000000000..a34eb3e553 --- /dev/null +++ b/apple/TableProTests/Core/Shortcuts/PayloadRowPlannerHeaderMatchTests.swift @@ -0,0 +1,59 @@ +// +// PayloadRowPlannerHeaderMatchTests.swift +// TableProTests +// + +import Foundation +import Testing + +@testable import TablePro + +struct PayloadRowPlannerHeaderMatchTests { + private func columns(_ names: [String]) -> [PayloadColumn] { + names.map { PayloadColumn(name: $0, isPrimaryKey: false, isServerOwned: false) } + } + + private func planned(_ rows: [PayloadRow], into names: [String]) throws -> [[String]] { + try PayloadRowPlanner.values(table: "people", columns: columns(names), rows: rows) + .map { row in row.map { "\($0.column)=\($0.value ?? "NULL")" } } + } + + // SQL Server exports its headers bracketed, and Shortcuts has no sheet to map them by hand. + @Test("A CSV with bracketed headers inserts into the columns they name") + func bracketedHeadersFindTheirColumns() async throws { + let rows = try await RowPayload.parse(data: "[id],[name]\n1,Bob\n", file: nil) + #expect(try planned(rows, into: ["id", "name"]) == [["id=1", "name=Bob"]]) + } + + @Test("A doubled closing bracket inside the brackets is one bracket of the name") + func doubledBracketIsUnescaped() throws { + let row = PayloadRow(values: ["[a]]b]": .text("x")]) + #expect(try planned([row], into: ["a]b"]) == [["a]b=x"]]) + } + + @Test("A lone closing bracket inside means the key is not a bracketed name") + func loneBracketIsNotUnwrapped() { + #expect(throws: IntentDataError.unknownColumns(["[a]b]"], "people")) { + _ = try planned([PayloadRow(values: ["[a]b]": .text("x")])], into: ["a]b"]) + } + } + + @Test("A key differing only in case finds its column, as the import sheet matches headers") + func caseInsensitiveMatch() throws { + #expect(try planned([PayloadRow(values: ["Name": .text("Ada")])], into: ["name"]) == [["name=Ada"]]) + } + + @Test("A case match two columns share is refused rather than guessed") + func ambiguousCaseMatchIsUnknown() { + #expect(throws: IntentDataError.unknownColumns(["NAME"], "people")) { + _ = try planned([PayloadRow(values: ["NAME": .text("Ada")])], into: ["Name", "name"]) + } + } + + @Test("A key spelled exactly claims its column before a bracketed one can") + func exactSpellingClaimsFirst() { + #expect(throws: IntentDataError.unknownColumns(["[id]"], "people")) { + _ = try planned([PayloadRow(values: ["id": .text("1"), "[id]": .text("2")])], into: ["id"]) + } + } +} diff --git a/apple/TableProTests/Core/Shortcuts/RowPayloadFormatTests.swift b/apple/TableProTests/Core/Shortcuts/RowPayloadFormatTests.swift new file mode 100644 index 0000000000..f5f9743a48 --- /dev/null +++ b/apple/TableProTests/Core/Shortcuts/RowPayloadFormatTests.swift @@ -0,0 +1,60 @@ +// +// RowPayloadFormatTests.swift +// TableProTests +// + +import Foundation +import Testing + +@testable import TablePro + +struct RowPayloadFormatTests { + // SQL Server exports its headers bracketed, so a valid CSV opens with `[`. + @Test("CSV with bracketed headers is read as CSV") + func bracketedHeaderIsCSV() async throws { + let rows = try await RowPayload.parse(data: "[id],[name]\n1,Bob\n", file: nil) + #expect(rows == [PayloadRow(values: ["[id]": .text("1"), "[name]": .text("Bob")])]) + } + + @Test("CSV whose first header opens with a brace is read as CSV") + func braceHeaderIsCSV() async throws { + let rows = try await RowPayload.parse(data: "{key},value\na,1\n", file: nil) + #expect(rows == [PayloadRow(values: ["{key}": .text("a"), "value": .text("1")])]) + } + + @Test("A JSON object and a JSON array of objects are still read as JSON") + func jsonStaysJSON() async throws { + let object = try await RowPayload.parse(data: #"{"name":"Ada"}"#, file: nil) + #expect(object == [PayloadRow(values: ["name": .text("Ada")])]) + let array = try await RowPayload.parse(data: #"[{"name":"Ada"},{"name":"Grace"}]"#, file: nil) + #expect(array.count == 2) + } + + @Test("Malformed JSON that is no CSV either reports the JSON error") + func malformedJSONKeepsItsError() async { + await #expect { + _ = try await RowPayload.parse(data: #"{"name": tru}"#, file: nil) + } throws: { error in + guard case .malformedPayload = error as? IntentDataError else { return false } + return true + } + } + + // Each line splits into the same number of CSV fields, so the text read as CSV with `{"id": 1` for a header. + @Test("JSON with a trailing comma after each line's object keeps the JSON error") + func jsonThatSplitsEvenlyKeepsItsError() async { + await #expect { + _ = try await RowPayload.parse(data: "{\"id\": 1, \"name\": \"a\"},\n{\"id\": 2, \"name\": \"b\"},\n", file: nil) + } throws: { error in + guard case .malformedPayload = error as? IntentDataError else { return false } + return true + } + } + + @Test("A JSON array of something other than objects keeps its own error") + func jsonArrayOfValuesIsNotRetriedAsCSV() async { + await #expect(throws: IntentDataError.jsonArrayContainsNonObject) { + _ = try await RowPayload.parse(data: "[1,2]", file: nil) + } + } +} diff --git a/apple/TableProTests/Core/UsersRoles/PrincipalApplyOpenTransactionTests.swift b/apple/TableProTests/Core/UsersRoles/PrincipalApplyOpenTransactionTests.swift index 4cb996f0e4..0ee582a754 100644 --- a/apple/TableProTests/Core/UsersRoles/PrincipalApplyOpenTransactionTests.swift +++ b/apple/TableProTests/Core/UsersRoles/PrincipalApplyOpenTransactionTests.swift @@ -62,3 +62,47 @@ struct PrincipalApplyOpenTransactionTests { #expect(driver.executedQueries.isEmpty) } } + +// The review sheet showed a Cancel at the gate as a failed apply. +@MainActor +struct PrincipalApplyGateDenialTests { + private func apply(gate: any ExecutionGate) async -> (failure: GatedOperationFailure?, executed: [String]) { + let driver = TransactionHoldingPrincipalDriver(rollsBack: true) + let connection = TestFixtures.makeConnection(type: .postgresql) + DatabaseManager.shared.injectSession( + ConnectionSession(connection: connection, driver: PluginDriverAdapter(connection: connection, pluginDriver: driver)), + for: connection.id + ) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let change = PrincipalChange.create(PluginPrincipalDefinition(ref: PluginPrincipalRef(name: "X"), password: "p")) + do { + try await DatabaseManager.shared.executePrincipalChanges( + changes: [change], + databaseType: .postgresql, + connectionId: connection.id, + gate: gate + ) + return (nil, driver.executedQueries) + } catch { + return (GatedOperationFailure(error), driver.executedQueries) + } + } + + @Test("A Cancel at the gate ends the apply quietly, with nothing run") + func cancelIsQuiet() async { + let result = await apply(gate: AlwaysDenyGate.cancelling) + + #expect(result.failure == .cancelledByUser) + #expect(result.failure?.message == nil) + #expect(result.executed.isEmpty) + } + + @Test("A refused apply says why and is not a failed statement") + func refusalExplainsItself() async { + let result = await apply(gate: AlwaysDenyGate(reason: "Safe Mode is read-only")) + + #expect(result.failure == .refused("Safe Mode is read-only")) + #expect(result.executed.isEmpty) + } +} diff --git a/apple/TableProTests/Core/UsersRoles/PrincipalChangeManagerTests.swift b/apple/TableProTests/Core/UsersRoles/PrincipalChangeManagerTests.swift index fcdd2c5053..d8c628c9eb 100644 --- a/apple/TableProTests/Core/UsersRoles/PrincipalChangeManagerTests.swift +++ b/apple/TableProTests/Core/UsersRoles/PrincipalChangeManagerTests.swift @@ -329,6 +329,46 @@ struct PrincipalChangeManagerTests { #expect(manager.changeCount == 1) } + @Test("A staged drop runs only the DROP and counts as one change") + func dropSupersedesOtherEdits() { + let manager = makeManager() + manager.stageAlter(PluginPrincipalDefinition(ref: alice, canLogin: false), for: alice) + manager.stageSetPassword("s3cret", for: alice) + manager.setGranted(true, privilege: "CREATE", scope: app, for: alice) + #expect(manager.changeCount == 3) + + manager.stageDrop(alice, options: PluginPrincipalDropOptions()) + + let changes = manager.pendingChanges() + #expect(changes.count == 1) + guard case .drop(let ref, _) = changes.first else { + Issue.record("Expected only the DROP, got \(changes)") + return + } + #expect(ref == alice) + #expect(manager.changeCount == 1) + } + + @Test("Undoing a staged drop brings the principal's other edits back") + func unstagingDropRestoresEdits() { + let manager = makeManager() + manager.stageAlter(PluginPrincipalDefinition(ref: alice, canLogin: false), for: alice) + manager.stageSetPassword("s3cret", for: alice) + manager.setGranted(true, privilege: "CREATE", scope: app, for: alice) + manager.stageDrop(alice, options: PluginPrincipalDropOptions()) + + manager.unstageDrop(alice) + + let changes = manager.pendingChanges() + let dropsAnything = changes.contains { change in + if case .drop = change { return true } + return false + } + #expect(changes.count == 3) + #expect(!dropsAnything) + #expect(manager.changeCount == 3) + } + @Test("Dropping the connected account is reported as self-impact") func reportsSelfImpact() { let manager = makeManager() diff --git a/apple/TableProTests/Core/Utilities/JsonRowConverterTests.swift b/apple/TableProTests/Core/Utilities/JsonRowConverterTests.swift index 759614a413..0e3f59c5d3 100644 --- a/apple/TableProTests/Core/Utilities/JsonRowConverterTests.swift +++ b/apple/TableProTests/Core/Utilities/JsonRowConverterTests.swift @@ -384,20 +384,16 @@ struct JsonRowConverterTests { // MARK: - Default-value marker - /// A pending inserted row carries the default marker for columns the server fills in. The grid - /// draws it as a placeholder, so JSON has to agree instead of printing the internal token. - @Test("The default-value marker renders as null, not as literal text") - func defaultMarkerRendersAsNull() { + // The default is marked beside the value, so text a table holds that spells the old marker is a value. + @Test("Stored text that spells the old default marker renders as that text") + func oldMarkerTextIsAValue() { let converter = makeConverter( - columns: ["id", "name"], - columnTypes: [.integer(rawType: nil), .text(rawType: nil)] - ) - let result = converter.generateJson( - rows: [[.text(PluginCellValue.defaultMarkerText), .text("Ada")]] + columns: ["status", "name"], + columnTypes: [.text(rawType: nil), .text(rawType: nil)] ) + let result = converter.generateJson(rows: [[.text("__DEFAULT__"), .text("Ada")]]) - #expect(!result.contains(PluginCellValue.defaultMarkerText)) - #expect(result.contains("\"id\": null")) + #expect(result.contains("\"status\": \"__DEFAULT__\"")) #expect(result.contains("\"Ada\"")) } diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierTSQLBlockTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierTSQLBlockTests.swift new file mode 100644 index 0000000000..86fc85825a --- /dev/null +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierTSQLBlockTests.swift @@ -0,0 +1,39 @@ +import Foundation +@testable import TablePro +import TableProSQLGrammar +import Testing + +// A T-SQL block runs as one statement, and a caller allowed one statement at a time is still refused one holding more. +struct QueryClassifierTSQLBlockTests { + @Test("A block holding more than one statement is multi-statement", arguments: [ + "IF @x = 1 BEGIN UPDATE a SET c = 1; UPDATE b SET c = 1; END", + "BEGIN TRY DELETE FROM a WHERE id = 1; DELETE FROM b WHERE id = 1; END TRY BEGIN CATCH SELECT 1; END CATCH", + "WHILE @i < 3 BEGIN SET @i += 1; PRINT @i; END", + "IF @x = 1 BEGIN SELECT 1; WITH c AS (SELECT id FROM t) DELETE FROM c; END", + "IF @x = 1 BEGIN SELECT 1; DISABLE TRIGGER trg ON t; END", + "BEGIN TRY SELECT 1; THROW 50000, 'x', 1; END TRY BEGIN CATCH SELECT 2; END CATCH", + "IF @x = 1 BEGIN SELECT 1; done: SELECT 2; END", + "IF @x = 1 BEGIN SELECT 1; (SELECT 2); END", + ]) + func blockOfSeveralIsMulti(sql: String) { + #expect(QueryClassifier.isMultiStatement(sql, databaseType: .mssql)) + } + + @Test("A conditional statement, or a block holding one, is a single statement", arguments: [ + "IF @x = 1 SELECT 1", + "IF @x = 1 BEGIN SELECT 1; END", + "BEGIN SELECT 1 END", + "BEGIN IF @x = 1 SELECT 1; ELSE SELECT 2; END", + "BEGIN TRY SELECT 1; END TRY BEGIN CATCH THROW; END CATCH", + ]) + func singleStatementIsNotMulti(sql: String) { + #expect(!QueryClassifier.isMultiStatement(sql, databaseType: .mssql)) + } + + @Test("A stored routine's body runs nothing, so it is one statement however many it holds") + func routineBodyIsNotMulti() { + #expect(!QueryClassifier.isMultiStatement( + "CREATE PROCEDURE dbo.p AS BEGIN UPDATE a SET c = 1; UPDATE b SET c = 1; END", databaseType: .mssql + )) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQLRowToStatementConverterTests.swift b/apple/TableProTests/Core/Utilities/SQLRowToStatementConverterTests.swift index b5df30bfd0..8fc75fa0d1 100644 --- a/apple/TableProTests/Core/Utilities/SQLRowToStatementConverterTests.swift +++ b/apple/TableProTests/Core/Utilities/SQLRowToStatementConverterTests.swift @@ -384,8 +384,9 @@ struct SQLRowToStatementConverterTests { #expect(lines.count == 50_000) } - @Test("PostgreSQL: binary cell renders as bytea hex literal in INSERT") - func postgresBinaryInsertEmitsByteaLiteral() throws { + // `'\x…'::bytea` reads its backslash by `standard_conforming_strings`; `decode` means the same bytes either way. + @Test("PostgreSQL: binary cell decodes hex in INSERT") + func postgresBinaryInsertDecodesHex() throws { let converter = try SQLRowToStatementConverter( tableName: "documents", columns: ["id", "payload"], @@ -396,10 +397,28 @@ struct SQLRowToStatementConverterTests { ) let bytes = Data([0xD3, 0x8C, 0xE5, 0x66]) let result = converter.generateInserts(rows: plain([[.text("1"), .bytes(bytes)]])) - #expect(result.contains("'\\xD38CE566'::bytea")) + #expect(result.contains("decode('D38CE566', 'hex')")) #expect(!result.contains("NULL")) } + // DuckDB reads `X'…'` as the text `x…` and Oracle has no such literal. + @Test("DuckDB and Oracle binary cells use their own hex functions in INSERT", arguments: [ + (DatabaseType.duckdb, "unhex('CAFE')"), (DatabaseType.oracle, "HEXTORAW('CAFE')"), + ]) + func binaryInsertUsesTheEnginesHexFunction(databaseType: DatabaseType, literal: String) throws { + let converter = try SQLRowToStatementConverter( + tableName: "documents", + columns: ["id", "payload"], + primaryKeyColumns: ["id"], + databaseType: databaseType, + quoteIdentifier: { "\"\($0)\"" }, + escapeStringLiteral: { $0.replacingOccurrences(of: "'", with: "''") } + ) + let result = converter.generateInserts(rows: plain([[.text("1"), .bytes(Data([0xCA, 0xFE]))]])) + #expect(result.contains(literal)) + #expect(!result.contains("X'CAFE'")) + } + @Test("MySQL: binary cell renders as X'...' literal in INSERT") func mysqlBinaryInsertEmitsXLiteral() throws { let converter = try makeConverter( @@ -441,7 +460,7 @@ struct SQLRowToStatementConverterTests { ) let bytes = Data([0xAB, 0xCD]) let result = converter.generateUpdates(rows: plain([[.text("42"), .bytes(bytes)]])) - #expect(result.contains("\"payload\" = '\\xABCD'::bytea")) + #expect(result.contains("\"payload\" = decode('ABCD', 'hex')")) #expect(result.contains("WHERE \"id\" = '42'")) } @@ -656,8 +675,8 @@ struct SQLRowToStatementConverterTests { databaseType: .pglite, dialect: Self.postgresDialect ) let rows = plain([[.text("1"), .bytes(Data([0xDE, 0xAD]))]]) - #expect(converter.generateInserts(rows: rows).contains("'\\xDEAD'::bytea")) - #expect(converter.generateUpdates(rows: rows).contains("\"payload\" = '\\xDEAD'::bytea")) + #expect(converter.generateInserts(rows: rows).contains("decode('DEAD', 'hex')")) + #expect(converter.generateUpdates(rows: rows).contains("\"payload\" = decode('DEAD', 'hex')")) #expect(!converter.generateInserts(rows: rows).contains("X'DEAD'")) } } diff --git a/apple/TableProTests/Core/Vim/VimTextBufferAdapterPerfTests.swift b/apple/TableProTests/Core/Vim/VimTextBufferAdapterPerfTests.swift index 6fadfaf773..48c97aac20 100644 --- a/apple/TableProTests/Core/Vim/VimTextBufferAdapterPerfTests.swift +++ b/apple/TableProTests/Core/Vim/VimTextBufferAdapterPerfTests.swift @@ -2,8 +2,7 @@ // VimTextBufferAdapterPerfTests.swift // TableProTests // -// Regression tests for VimTextBufferAdapter incremental lineCount -// and setSelectedRange guard +// Regression tests for VimTextBufferAdapter lineCount and setSelectedRange guard // import AppKit @@ -65,83 +64,29 @@ struct VimTextBufferAdapterPerfTests { #expect(adapter.lineCount == 2) } - // MARK: - textDidChange incremental (pure insertion) - - @Test("textDidChange with pure insertion updates line count incrementally") - func insertionUpdatesLineCount() { - let (adapter, textView) = makeAdapter(string: "hello") - - // Prime the cache - let initial = adapter.lineCount - #expect(initial == 1) - - // Simulate inserting "\nworld" at offset 5 - textView.replaceCharacters(in: NSRange(location: 5, length: 0), with: "\nworld") - adapter.textDidChange(in: NSRange(location: 5, length: 0), replacementLength: 6) - - #expect(adapter.lineCount == 2) - } - - @Test("textDidChange with multi-newline insertion") - func multiNewlineInsertion() { - let (adapter, textView) = makeAdapter(string: "hello") - - _ = adapter.lineCount // prime cache - - textView.replaceCharacters(in: NSRange(location: 5, length: 0), with: "\n\n\n") - adapter.textDidChange(in: NSRange(location: 5, length: 0), replacementLength: 3) - - #expect(adapter.lineCount == 4) - } - - // MARK: - textDidChange fallback on deletion - - @Test("textDidChange with deletion invalidates cache") - func deletionInvalidatesCache() { - let (adapter, textView) = makeAdapter(string: "a\nb\nc") - - let initial = adapter.lineCount - #expect(initial == 3) - - // Simulate deleting "\nb" (range.length > 0 means it's not pure insertion) - textView.replaceCharacters(in: NSRange(location: 1, length: 2), with: "") - adapter.textDidChange(in: NSRange(location: 1, length: 2), replacementLength: 0) - - // Cache should be invalidated; next access does a full recount - #expect(adapter.lineCount == 2) - } - - // MARK: - textDidChange(oldText:...) incremental replacement - - @Test("textDidChange with oldText correctly computes delta for replacement") - func oldTextReplacementDelta() { - let originalText = "a\nb\nc" - let (adapter, textView) = makeAdapter(string: originalText) - - let initial = adapter.lineCount - #expect(initial == 3) - - // Replace "b\nc" (range 2..4, contains 1 newline) with "x" (0 newlines) - textView.replaceCharacters(in: NSRange(location: 2, length: 3), with: "x") - adapter.textDidChange(oldText: originalText, in: NSRange(location: 2, length: 3), replacementLength: 1) - - // 3 original - 1 removed newline + 0 added = 2 - #expect(adapter.lineCount == 2) - } - - @Test("textDidChange with oldText adding newlines") - func oldTextAddingNewlines() { - let originalText = "abc" - let (adapter, textView) = makeAdapter(string: originalText) - - _ = adapter.lineCount // prime: 1 - - // Replace "b" with "x\ny\nz" (adding 2 newlines) - textView.replaceCharacters(in: NSRange(location: 1, length: 1), with: "x\ny\nz") - adapter.textDidChange(oldText: originalText, in: NSRange(location: 1, length: 1), replacementLength: 5) - - // 1 original - 0 removed + 2 added = 3 - #expect(adapter.lineCount == 3) + // MARK: - Line count after an edit + + // The editor drops the cache on every edit, so the next count reads the text as it is now. + @Test("After an edit the line count matches a fresh count of the new text", arguments: [ + (NSRange(location: 5, length: 0), "\nworld", "hello", 2), + (NSRange(location: 5, length: 0), "\n\n\n", "hello", 3), + (NSRange(location: 1, length: 2), "", "a\nb\nc", 2), + (NSRange(location: 2, length: 3), "x", "a\nb\nc", 2), + (NSRange(location: 1, length: 1), "x\ny\nz", "abc", 3), + (NSRange(location: 1, length: 0), "\r", "ab", 2), + ]) + func lineCountAfterAnEdit(range: NSRange, replacement: String, original: String, expected: Int) { + let (adapter, textView) = makeAdapter(string: original) + _ = adapter.lineCount + + textView.replaceCharacters(in: range, with: replacement) + adapter.invalidateLineCache() + + let counted = withExtendedLifetime(textView) { adapter.lineCount } + let (fresh, freshTextView) = makeAdapter(string: textView.string) + let freshCount = withExtendedLifetime(freshTextView) { fresh.lineCount } + #expect(counted == expected) + #expect(counted == freshCount) } // MARK: - setSelectedRange guard diff --git a/apple/TableProTests/Helpers/StubExecutionGates.swift b/apple/TableProTests/Helpers/StubExecutionGates.swift index 83dce282ed..afcfa85d48 100644 --- a/apple/TableProTests/Helpers/StubExecutionGates.swift +++ b/apple/TableProTests/Helpers/StubExecutionGates.swift @@ -27,13 +27,18 @@ internal struct AlwaysAllowGate: ExecutionGate { internal struct AlwaysDenyGate: ExecutionGate { internal let reason: String + internal let cause: OperationDenialCause - internal init(reason: String = "Read-Only connection") { + internal init(reason: String = "Read-Only connection", cause: OperationDenialCause = .policy) { self.reason = reason + self.cause = cause } + // The user answering Cancel in the gate's confirmation. + internal static let cancelling = AlwaysDenyGate(reason: "Operation cancelled by user", cause: .cancelledByUser) + internal func authorize(_ request: OperationRequest) async -> OperationDecision { - .denied(reason: reason) + .denied(reason: reason, cause: cause) } } diff --git a/apple/TableProTests/Models/MultiRowEditStateTests.swift b/apple/TableProTests/Models/MultiRowEditStateTests.swift index ce162e198a..1cdb06f345 100644 --- a/apple/TableProTests/Models/MultiRowEditStateTests.swift +++ b/apple/TableProTests/Models/MultiRowEditStateTests.swift @@ -95,14 +95,14 @@ struct MultiRowEditStateTests { #expect(field.effectiveValue == nil) } - @Test("effectiveValue returns __DEFAULT__ when isPendingDefault") + @Test("effectiveValue returns the DEFAULT keyword when isPendingDefault") func effectiveValueReturnsDefaultWhenPendingDefault() { let field = FieldEditState( columnIndex: 0, columnName: "id", columnTypeEnum: .text(rawType: nil), isLongText: false, isJson: false, originalValue: "1", hasMultipleValues: false, pendingValue: nil, isPendingNull: false, isPendingDefault: true ) - #expect(field.effectiveValue == "__DEFAULT__") + #expect(field.effectiveValue == "DEFAULT") } @Test("effectiveValue returns nil when no edit is pending") @@ -644,13 +644,14 @@ struct MultiRowEditStateTests { #expect(edited[0].newValue == nil) } - @Test("Returns __DEFAULT__ for default edit") + @Test("Returns the DEFAULT keyword marked as SQL for a default edit") func returnsDefaultForDefaultEdit() { let sut = makeSUT() sut.setFieldToDefault(at: 0) let edited = sut.getEditedFields() #expect(edited.count == 1) - #expect(edited[0].newValue == "__DEFAULT__") + #expect(edited[0].newValue == "DEFAULT") + #expect(edited[0].isExpression) } @Test("Returns empty array when no edits") @@ -774,18 +775,22 @@ struct MultiRowEditStateTests { #expect(callbackCalls[0].value == nil) } - @Test("setFieldToDefault fires callback with __DEFAULT__") + // Through `onFieldChanged` the keyword would be a literal, which a stored value can also be. + @Test("setFieldToDefault sends the DEFAULT keyword through the expression callback") func setFieldToDefaultFiresCallback() { let sut = makeSUT() - var callbackCalls: [(index: Int, value: String?)] = [] - sut.onFieldChanged = { index, value, _ in - callbackCalls.append((index, value.asText)) + var literals: [PluginCellValue] = [] + var callbackCalls: [(index: Int, sql: String)] = [] + sut.onFieldChanged = { _, value, _ in literals.append(value) } + sut.onFieldExpressionChanged = { index, sql, _ in + callbackCalls.append((index, sql)) } sut.setFieldToDefault(at: 0) + #expect(literals.isEmpty) #expect(callbackCalls.count == 1) #expect(callbackCalls[0].index == 0) - #expect(callbackCalls[0].value == "__DEFAULT__") + #expect(callbackCalls[0].sql == "DEFAULT") } // `onFieldChanged` carries a literal, so an expression sent through it would be saved as quoted text. diff --git a/apple/TableProTests/Models/Schema/ClickHousePartStatementsTests.swift b/apple/TableProTests/Models/Schema/ClickHousePartStatementsTests.swift index cc67c4e094..cdee26b95c 100644 --- a/apple/TableProTests/Models/Schema/ClickHousePartStatementsTests.swift +++ b/apple/TableProTests/Models/Schema/ClickHousePartStatementsTests.swift @@ -2,9 +2,6 @@ // ClickHousePartStatementsTests.swift // TableProTests // -// The Parts tab's statements name the tab's own database, because ClickHouse resolves an -// unqualified one from a request parameter the app moves whenever any tab runs elsewhere. -// import Foundation @testable import TablePro @@ -14,8 +11,7 @@ struct ClickHousePartStatementsTests { private let quote: (String) -> String = { "`\($0.replacingOccurrences(of: "`", with: "``"))`" } private let escape: (String) -> String = { $0.replacingOccurrences(of: "'", with: "\\'") } - /// The defect: an unqualified DROP PARTITION deleted data from whichever database the shared - /// driver was last pinned to, while the tab reported it was acting on its own. + // An unqualified DROP PARTITION deleted data from whichever database the shared driver was last pinned to. @Test("Every statement names the database it was given") func statementsCarryTheDatabase() { #expect( @@ -24,20 +20,59 @@ struct ClickHousePartStatementsTests { ) #expect( ClickHousePartStatements.dropPartition( - database: "analytics", table: "events", partition: "202401", - quote: quote, escape: escape - ) == "ALTER TABLE `analytics`.`events` DROP PARTITION '202401'" + database: "analytics", table: "events", partitionId: "202401", quote: quote + ) == "ALTER TABLE `analytics`.`events` DROP PARTITION ID '202401'" ) #expect( ClickHousePartStatements.detachPartition( - database: "analytics", table: "events", partition: "202401", - quote: quote, escape: escape - ) == "ALTER TABLE `analytics`.`events` DETACH PARTITION '202401'" + database: "analytics", table: "events", partitionId: "202401", quote: quote + ) == "ALTER TABLE `analytics`.`events` DETACH PARTITION ID '202401'" ) } - /// `currentDatabase()` answers with the request parameter, so the list showed another database's - /// parts under this table's name. + // A tuple key's rendered value `(202401,'eu')` and an unpartitioned table's `tuple()` were each + // wrapped as one string literal, which names no partition. + @Test("Drop and Detach name the partition by its ID for every key shape") + func everyKeyShapeUsesThePartitionId() { + let shapes = [ + "202401-7", + "all", + "4f8b2b7c9a1d3e5f60718293a4b5c6d7", + "1--5", + ] + for id in shapes { + #expect( + ClickHousePartStatements.dropPartition( + database: "analytics", table: "events", partitionId: id, quote: quote + ) == "ALTER TABLE `analytics`.`events` DROP PARTITION ID '\(id)'" + ) + #expect( + ClickHousePartStatements.detachPartition( + database: "analytics", table: "events", partitionId: id, quote: quote + ) == "ALTER TABLE `analytics`.`events` DETACH PARTITION ID '\(id)'" + ) + } + } + + @Test("An ID the server would refuse is never spliced into a statement") + func unusableIdBuildsNothing() { + let unusable = ["", "it's", "a\\'b", "(202401,'eu')", "tuple()", "2024 01", "202401_1", "Ć©"] + for id in unusable { + #expect(!ClickHousePartStatements.isValidPartitionId(id)) + #expect( + ClickHousePartStatements.dropPartition( + database: "analytics", table: "events", partitionId: id, quote: quote + ) == nil + ) + #expect( + ClickHousePartStatements.detachPartition( + database: "analytics", table: "events", partitionId: id, quote: quote + ) == nil + ) + } + } + + // `currentDatabase()` answers with the request parameter, so the list showed another database's parts. @Test("The parts read filters on the named database, never the connection's current one") func partsReadNamesTheDatabase() { let sql = ClickHousePartStatements.parts( @@ -48,9 +83,13 @@ struct ClickHousePartStatementsTests { #expect(!sql.contains("currentDatabase()")) } - /// A connection saved with no database of its own has nothing to qualify against, and an empty - /// qualifier is a statement the server refuses. The parts read has to fall back too: comparing - /// against '' matches nothing, where ClickHouse's own default database is what the user sees. + @Test("The parts read selects the partition ID beside the displayed partition value") + func partsReadSelectsThePartitionId() { + let sql = ClickHousePartStatements.parts(database: "analytics", table: "events", escape: escape) + #expect(sql.contains("SELECT partition, partition_id, name, rows, bytes_on_disk,")) + } + + // With no database the qualifier is refused by the server, and comparing against '' matches nothing. @Test("With no database the name stays unqualified and the read asks the server") func emptyDatabaseLeavesTheNameBare() { #expect( @@ -66,13 +105,11 @@ struct ClickHousePartStatementsTests { #expect(!sql.contains("database = ''")) } - @Test("A quote in a partition value is escaped, and a backtick in a name is quoted") - func valuesAreEscaped() { + @Test("A backtick in a database or table name is quoted") + func namesAreQuoted() { let sql = ClickHousePartStatements.dropPartition( - database: "we`ird", table: "ev`ents", partition: "it's", - quote: quote, escape: escape + database: "we`ird", table: "ev`ents", partitionId: "202401", quote: quote ) - #expect(sql.contains("`we``ird`.`ev``ents`")) - #expect(sql.contains("'it\\'s'")) + #expect(sql == "ALTER TABLE `we``ird`.`ev``ents` DROP PARTITION ID '202401'") } } diff --git a/apple/TableProTests/Views/Editor/StatementCaretTableTests.swift b/apple/TableProTests/Views/Editor/StatementCaretTableTests.swift new file mode 100644 index 0000000000..ba5ea78102 --- /dev/null +++ b/apple/TableProTests/Views/Editor/StatementCaretTableTests.swift @@ -0,0 +1,109 @@ +import AppKit +import Foundation +@testable import TablePro +import TableProEditorKit +import TableProGrammars +import TableProSQLGrammar +import TableProTextEngine +import Testing + +private let caretTableMongoScript = """ +db.users.find({ name: "a;b" }); +// note + +for (var i = 0; i < 3; i++) { + print(i); +} +db.orders.insertOne({ total: 1 }) +db.orders.find() +// trailing +""" + +private let caretTableRedisScript = "GET a; GET b\n\nSET k \"a\nb\"\nGET c \n " + +@MainActor +struct StatementCaretTableTests { + private struct Resolved: Equatable { + let offset: Int + let sql: String + let hasContent: Bool + let contentRange: NSRange + + init(_ statement: SQLStatementScanner.LocatedStatement) { + offset = statement.offset + sql = statement.sql + hasContent = statement.hasContent + contentRange = statement.contentRange + } + } + + @Test("Every caret resolves to what a scan of the whole document resolves to", arguments: [ + (QueryStatementModel.javascript, caretTableMongoScript), + (QueryStatementModel.redis, caretTableRedisScript), + (QueryStatementModel.javascript, ""), + (QueryStatementModel.redis, ""), + ]) + func everyCaretMatchesAFullScan(model: QueryStatementModel, text: String) throws { + let table = try #require(QueryStatementCaretTable(text: text, model: model)) + for caret in -1...((text as NSString).length + 1) { + let fresh = QueryStatementScanner.locatedStatementAtCursor( + in: text, cursorPosition: caret, model: model, grammar: .ansi + ) + #expect(Resolved(table.statement(atCursor: caret)) == Resolved(fresh), "caret \(caret)") + } + } + + @Test("A SQL document has no table, since it resumes through its own index") + func sqlHasNoTable() { + #expect(QueryStatementCaretTable(text: "SELECT 1;", model: .sql) == nil) + } + + private func makeController(text: String) -> TextViewController { + let configuration = SourceEditorConfiguration( + appearance: .init( + theme: ThemeEngine.shared.makeEditorTheme(), + font: .monospacedSystemFont(ofSize: 12, weight: .regular), + wrapLines: false + ), + peripherals: EditorPeripherals.editor(lineNumbers: true, folding: true, statementRunControls: true) + ) + let controller = TextViewController(string: text, language: .sql, configuration: configuration, cursorPositions: []) + controller.loadView() + return controller + } + + private func band(at caret: Int, in controller: TextViewController, subject: StatementRunController) -> NSRange? { + controller.setCursorPositions([CursorPosition(range: NSRange(location: caret, length: 0))]) + subject.refreshHighlight(in: controller) + return controller.highlightedStatementRange + } + + private func freshBand(at caret: Int, in controller: TextViewController) -> NSRange? { + let statement = QueryStatementScanner.locatedStatementAtCursor( + in: controller.textView.string, cursorPosition: caret, model: .javascript, grammar: .ansi + ) + guard statement.hasContent, statement.contentRange.length > 0 else { return nil } + return statement.contentRange + } + + @Test("Caret moves through a MongoDB script read it once, and an edit reads it again") + func caretMovesReadTheScriptOnce() { + let script = String(repeating: "db.orders.find({ total: 1 });\n", count: 2_000) + let controller = makeController(text: script) + let subject = StatementRunController() + subject.statementModel = .javascript + subject.install(on: controller) + let readsAtInstall = subject.caretTableReads + + for caret in stride(from: 59_000, to: 58_000, by: -7) { + #expect(band(at: caret, in: controller, subject: subject) == freshBand(at: caret, in: controller)) + } + #expect(subject.caretTableReads <= readsAtInstall + 1) + + controller.textView.replaceCharacters(in: NSRange(location: 0, length: 0), with: "db.a.find();\n") + let afterEdit = subject.caretTableReads + #expect(band(at: 100, in: controller, subject: subject) == freshBand(at: 100, in: controller)) + #expect(band(at: 58_013, in: controller, subject: subject) == freshBand(at: 58_013, in: controller)) + #expect(subject.caretTableReads == afterEdit + 1) + } +} diff --git a/apple/TableProTests/Views/Main/TableLoadAfterSafeModePromptTests.swift b/apple/TableProTests/Views/Main/TableLoadAfterSafeModePromptTests.swift new file mode 100644 index 0000000000..920544485f --- /dev/null +++ b/apple/TableProTests/Views/Main/TableLoadAfterSafeModePromptTests.swift @@ -0,0 +1,40 @@ +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +@MainActor +struct TableLoadAfterSafeModePromptTests { + @Test("A table load turned away while a query waits for approval runs once that approval closes") + func turnedAwayLoadRunsAfterThePrompt() async throws { + let connection = DatabaseConnection( + name: "Confirm", database: "app", type: .postgresql, safeModeLevel: .alertFull + ) + let fixture = CoordinatorFixture( + connection: connection, + driver: ScriptAnsweringDriver(connection: connection, sendsBatchesWhole: false) + ) + fixture.coordinator.toolbarState.safeModeLevel = .alertFull + let gate = HeldExecutionGate() + fixture.coordinator.executionGate = gate + defer { fixture.coordinator.cancelAllQueryTasks() } + + fixture.addTab(QueryTab(title: "Query", query: "SELECT 1", tabType: .query)) + #expect(fixture.coordinator.runStatement("SELECT 1")) + try await waitUntil { gate.askedCount == 1 } + + var table = QueryTab(title: "users", query: "SELECT * FROM users LIMIT 1000", tabType: .table, tableName: "users") + table.pagination.isLoading = true + fixture.addTab(table) + #expect(fixture.coordinator.executeTableTabQueryDirectly(viewport: .firstRow) == nil) + #expect(fixture.tabManager.tabs.first { $0.id == table.id }?.pagination.isLoading == false) + + await gate.answer(allowing: true) + try await waitUntil { gate.askedCount == 2 } + await gate.answer(allowing: false) + try await waitUntil { + fixture.tabManager.tabs.first { $0.id == table.id }?.execution.errorMessage == "Refused by the test" + } + } +} diff --git a/apple/TableProTests/Views/Main/WindowCloseSaveEveryConnectionTests.swift b/apple/TableProTests/Views/Main/WindowCloseSaveEveryConnectionTests.swift new file mode 100644 index 0000000000..bb0a30c859 --- /dev/null +++ b/apple/TableProTests/Views/Main/WindowCloseSaveEveryConnectionTests.swift @@ -0,0 +1,129 @@ +import AppKit +import Foundation +import SwiftUI +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +@MainActor +struct WindowCloseSaveEveryConnectionTests { + private func actions( + for coordinator: MainContentCoordinator, + in window: NSWindow, + attach: Bool = true + ) -> MainContentCommandActions { + var selectedTables: Set = [] + var pendingTruncates: Set = [] + var pendingDeletes: Set = [] + var tableOperationOptions: [DatabaseTreeTableRef: TableOperationOptions] = [:] + let actions = MainContentCommandActions( + coordinator: coordinator, + connection: coordinator.connection, + selectedTables: Binding(get: { selectedTables }, set: { selectedTables = $0 }), + pendingTruncates: Binding(get: { pendingTruncates }, set: { pendingTruncates = $0 }), + pendingDeletes: Binding(get: { pendingDeletes }, set: { pendingDeletes = $0 }), + tableOperationOptions: Binding( + get: { tableOperationOptions }, + set: { tableOperationOptions = $0 } + ), + trailingPaneState: TrailingPaneState() + ) + actions.window = window + if attach { coordinator.commandActions = actions } + return actions + } + + @Test("Save on a window close keeps the window while another connection holds edits only their tab can save") + func saveKeepsTheWindowForAnotherConnectionsEdits() async throws { + let harness = TwoConnectionWindow() + defer { harness.tearDown() } + let front = try harness.coordinator(of: harness.foreground) + let back = try harness.coordinator(of: harness.background) + let frontActions = actions(for: front, in: harness.window) + let backActions = actions(for: back, in: harness.window) + try harness.addSelectedTab(to: harness.foreground) + try harness.addSelectedTab(to: harness.background) + + var edited = QueryTab(title: "users", query: "SELECT * FROM users", tabType: .table, tableName: "users") + edited.pendingChanges.insertedRowIDs = [.inserted(UUID())] + edited.pendingChanges.columns = ["id", "name"] + edited.pendingChanges.primaryKeyColumns = ["id"] + back.tabManager.tabs.append(edited) + frontActions.confirmSaveChanges = { _, _ in .save } + + let outcome = await frontActions.closeWindowAwaiting() + + #expect(outcome == .cancelled) + #expect(back.tabManager.tabs.contains { $0.id == edited.id }) + withExtendedLifetime(backActions) {} + } + + // Its actions arrive only once its pane mounts, after Save has shown it. + @Test("Save waits for a connection that has not shown its pane yet, then saves it and closes") + func saveReachesAConnectionNotShownYet() async throws { + let harness = TwoConnectionWindow() + defer { harness.tearDown() } + let front = try harness.coordinator(of: harness.foreground) + let back = try harness.coordinator(of: harness.background) + front.contentWindow = harness.window + let frontActions = actions(for: front, in: harness.window) + let backActions = actions(for: back, in: harness.window, attach: false) + back.commandActions = nil + try harness.addSelectedTab(to: harness.foreground) + back.toolbarState.hasCreateTablePending = true + frontActions.confirmSaveChanges = { _, _ in + Task { @MainActor in back.commandActions = backActions } + return .save + } + + let outcome = await frontActions.closeWindowAwaiting() + + #expect(outcome == .closed) + } + + @Test("A close cancelled while it waits for a connection's pane stops after that one wait and keeps the window") + func cancelledCloseStopsWaiting() async throws { + let harness = TwoConnectionWindow() + defer { harness.tearDown() } + let front = try harness.coordinator(of: harness.foreground) + let back = try harness.coordinator(of: harness.background) + front.contentWindow = harness.window + let frontActions = actions(for: front, in: harness.window) + back.commandActions = nil + try harness.addSelectedTab(to: harness.foreground) + back.toolbarState.hasCreateTablePending = true + let clock = CountingClock() + frontActions.paneWaitClock = clock + frontActions.confirmSaveChanges = { _, _ in .save } + + let close = Task { await frontActions.closeWindowAwaiting() } + await clock.firstSleep.wait() + close.cancel() + let outcome = await close.value + + #expect(outcome == .cancelled) + #expect(clock.sleepCount == 1) + } +} + +// Time stands still, so a sleep ends only by cancellation; the first one opens a gate the test waits on. +private final class CountingClock: Clock, @unchecked Sendable { + let firstSleep = AsyncGate() + private let base = ManualClock() + private let lock = NSLock() + private var count = 0 + + var sleepCount: Int { lock.withLock { count } } + var now: ManualClock.Instant { base.now } + var minimumResolution: Duration { base.minimumResolution } + + func sleep(until deadline: ManualClock.Instant, tolerance: Duration?) async throws { + let isFirst = lock.withLock { + count += 1 + return count == 1 + } + if isFirst { await firstSleep.open() } + try await base.sleep(until: deadline, tolerance: tolerance) + } +} diff --git a/apple/TableProTests/Views/Results/ResultsJsonViewTests.swift b/apple/TableProTests/Views/Results/ResultsJsonViewTests.swift index d64d2975d0..167773b1fa 100644 --- a/apple/TableProTests/Views/Results/ResultsJsonViewTests.swift +++ b/apple/TableProTests/Views/Results/ResultsJsonViewTests.swift @@ -128,6 +128,30 @@ struct ResultsJsonViewTests { } } + @Test("A cell a pending edit set to the default renders as null, and stored text that reads DEFAULT stays text") + func pendingDefaultRendersAsNull() { + let inserted = RowID.inserted(UUID()) + let tableRows = TableRows( + rows: [ + Row(id: .existing(0), values: [.text("DEFAULT"), .text("a")]), + Row(id: inserted, values: [.columnDefault, .text("b")]), + ], + columns: ["status", "name"], + columnTypes: [.text(rawType: nil), .text(rawType: nil)] + ) + + let result = ResultsJsonView.computeJson( + tableRows: tableRows, + displayIDs: nil, + selectedIndices: [], + defaultedCells: [inserted: [0]], + columnLayout: ColumnLayoutState() + ) + + #expect(result.json.contains("\"status\": \"DEFAULT\"")) + #expect(result.json.contains("\"status\": null")) + } + @Test("an empty result renders an empty array") func emptyResultRendersEmptyArray() { let result = ResultsJsonView.computeJson( diff --git a/apple/TableProTests/Views/Results/SelectionSummaryTrackerTests.swift b/apple/TableProTests/Views/Results/SelectionSummaryTrackerTests.swift index 26110af586..926bf1f31f 100644 --- a/apple/TableProTests/Views/Results/SelectionSummaryTrackerTests.swift +++ b/apple/TableProTests/Views/Results/SelectionSummaryTrackerTests.swift @@ -176,7 +176,7 @@ struct SelectionSummaryTrackerTests { defer { grid.close() } grid.model.setSelection(Self.block(rows: 0...1, columns: 0...0)) - grid.edit(row: 1, to: .value(.text(PluginCellValue.defaultMarkerText), isExpression: false)) + grid.edit(row: 1, to: .value(.columnDefault, isExpression: true)) await grid.tracker.pendingTask?.value #expect(state.summary?.emptyCount == 1) diff --git a/apple/TableProTests/Views/Structure/ClickHousePartsGridTests.swift b/apple/TableProTests/Views/Structure/ClickHousePartsGridTests.swift index 41044fa125..fde04c7c82 100644 --- a/apple/TableProTests/Views/Structure/ClickHousePartsGridTests.swift +++ b/apple/TableProTests/Views/Structure/ClickHousePartsGridTests.swift @@ -5,12 +5,14 @@ import Foundation @testable import TablePro +import TableProPluginKit import Testing @MainActor struct ClickHousePartsGridTests { private let part = ClickHousePartInfo( partition: "202601", + partitionId: "202601", name: "202601_1_4_1", rows: 1_234_567, bytesOnDisk: 5_242_880, @@ -35,7 +37,7 @@ struct ClickHousePartsGridTests { @Test("An inactive part reads NO") func inactivePart() { let inactive = ClickHousePartInfo( - partition: "", name: "p", rows: 0, bytesOnDisk: 0, modificationTime: "", active: false + partition: "", partitionId: "", name: "p", rows: 0, bytesOnDisk: 0, modificationTime: "", active: false ) #expect(ClickHousePartsView.gridRow(for: inactive).values.last == "NO") } @@ -43,7 +45,7 @@ struct ClickHousePartsGridTests { @Test("Active reads aloud as Active or Inactive rather than YES or NO") func activeIsSpokenByName() { let inactive = ClickHousePartInfo( - partition: "", name: "p", rows: 0, bytesOnDisk: 0, modificationTime: "", active: false + partition: "", partitionId: "", name: "p", rows: 0, bytesOnDisk: 0, modificationTime: "", active: false ) let active = ClickHousePartsView.activeColumn @@ -59,23 +61,25 @@ struct ClickHousePartsGridTests { #expect(zip(ClickHousePartsView.columnMinimumWidths, ClickHousePartsView.columnWidths).allSatisfy { $0 <= $1 }) } - @Test("Drop and Detach name the one selected part's partition, and none while several are selected") + @Test("Drop and Detach act on the one selected part, and on none while several are selected") func selectedPartition() { let other = ClickHousePartInfo( - partition: "202602", name: "202602_1_1_0", rows: 1, bytesOnDisk: 1, modificationTime: "", active: true + partition: "202602", partitionId: "202602", name: "202602_1_1_0", rows: 1, bytesOnDisk: 1, + modificationTime: "", active: true ) let parts = [part, other] - #expect(ClickHousePartsView.partition(of: [other.id], in: parts) == "202602") - #expect(ClickHousePartsView.partition(of: [part.id, other.id], in: parts) == nil) - #expect(ClickHousePartsView.partition(of: [], in: parts) == nil) - #expect(ClickHousePartsView.partition(of: [UUID()], in: parts) == nil) + #expect(ClickHousePartsView.selectedPart(of: [other.id], in: parts)?.id == other.id) + #expect(ClickHousePartsView.selectedPart(of: [part.id, other.id], in: parts)?.id == nil) + #expect(ClickHousePartsView.selectedPart(of: [], in: parts)?.id == nil) + #expect(ClickHousePartsView.selectedPart(of: [UUID()], in: parts)?.id == nil) } @Test("A selected grid row names its own part") func selectedRowResolvesToItsPart() throws { let other = ClickHousePartInfo( - partition: "202602", name: "202602_1_1_0", rows: 1, bytesOnDisk: 1, modificationTime: "", active: true + partition: "202602", partitionId: "202602", name: "202602_1_1_0", rows: 1, bytesOnDisk: 1, + modificationTime: "", active: true ) let parts = [part, other] let grid = StructureListGridModel() @@ -95,7 +99,84 @@ struct ClickHousePartsGridTests { #expect(grid.id(for: second) == other.id) #expect(selected == [other.id]) - #expect(ClickHousePartsView.partition(of: selected, in: parts) == "202602") + #expect(ClickHousePartsView.selectedPart(of: selected, in: parts)?.partition == "202602") #expect(grid.model.rowID(at: 2) == nil) } + + @Test("A parts row keeps the partition ID a statement names beside the value the grid shows") + func rowCarriesThePartitionId() throws { + let row: [PluginCellValue] = [ + "(202401,'eu')", + "c1f8e6e2a7b04d4c9e1f0a2b3c4d5e6f", + "c1f8e6e2a7b04d4c9e1f0a2b3c4d5e6f_1_1_0", + "12", + "4096", + "2026-01-05 10:00:00", + "1", + ] + + let parsed = try #require(ClickHousePartsView.part(from: row)) + + #expect(parsed.partition == "(202401,'eu')") + #expect(parsed.partitionId == "c1f8e6e2a7b04d4c9e1f0a2b3c4d5e6f") + #expect(parsed.name == "c1f8e6e2a7b04d4c9e1f0a2b3c4d5e6f_1_1_0") + #expect(parsed.rows == 12) + #expect(parsed.bytesOnDisk == 4096) + #expect(parsed.modificationTime == "2026-01-05 10:00:00") + #expect(parsed.active) + #expect(ClickHousePartsView.gridRow(for: parsed).values[0] == "(202401,'eu')") + } + + // The pane used to swap the list and its toolbar for the error text until the tab was refreshed. + @Test("A refused or failed action keeps the parts list and its toolbar on screen") + func actionFailureKeepsTheList() { + var pane = ClickHousePartsPaneState() + pane.beginLoad() + pane.finishLoad([part]) + + pane.actionEnded(.refused("Safe Mode is read-only"), title: "Couldn't Drop Partition") + #expect(pane.display == .parts) + #expect(pane.alert == ClickHousePartsAlert(title: "Couldn't Drop Partition", message: "Safe Mode is read-only")) + + pane.alert = nil + pane.actionEnded(.failed("Wrong partition value"), title: "Couldn't Detach Partition") + #expect(pane.display == .parts) + #expect(pane.alert == ClickHousePartsAlert(title: "Couldn't Detach Partition", message: "Wrong partition value")) + } + + @Test("A Cancel at the gate's confirmation shows nothing") + func cancelIsQuiet() { + var pane = ClickHousePartsPaneState() + pane.finishLoad([part]) + + pane.actionEnded( + GatedOperationFailure(ExecutionGateError.cancelledByUser("Operation cancelled by user")), + title: "Couldn't Drop Partition" + ) + + #expect(pane.alert == nil) + #expect(pane.display == .parts) + } + + @Test("A load that fails with nothing shown offers Try Again, and a reload that fails keeps the list") + func loadFailure() { + var pane = ClickHousePartsPaneState() + #expect(pane.display == .loading) + + pane.failLoad("Connection refused") + #expect(pane.display == .loadFailed("Connection refused")) + #expect(pane.alert == nil) + + pane.beginLoad() + #expect(pane.display == .loading) + pane.finishLoad([]) + #expect(pane.display == .empty) + + pane.finishLoad([part]) + pane.beginLoad() + #expect(pane.display == .parts) + pane.failLoad("Timeout exceeded") + #expect(pane.display == .parts) + #expect(pane.alert?.message == "Timeout exceeded") + } } diff --git a/apple/TableProTests/Views/Structure/StructureListGridAccessibilityTests.swift b/apple/TableProTests/Views/Structure/StructureListGridAccessibilityTests.swift index b4e8440113..7910b8ae10 100644 --- a/apple/TableProTests/Views/Structure/StructureListGridAccessibilityTests.swift +++ b/apple/TableProTests/Views/Structure/StructureListGridAccessibilityTests.swift @@ -109,7 +109,8 @@ struct StructureListGridAccessibilityTests { defer { DataGridAccessibility.isActive = false } let parts = [true, false].map { active in ClickHousePartInfo( - partition: "202601", name: "p", rows: 1, bytesOnDisk: 1, modificationTime: "", active: active + partition: "202601", partitionId: "202601", name: "p", rows: 1, bytesOnDisk: 1, modificationTime: "", + active: active ) } let rows = parts.map { part in diff --git a/apple/TableProTests/Views/Structure/StructureSaveConfirmationTests.swift b/apple/TableProTests/Views/Structure/StructureSaveConfirmationTests.swift index 348093f635..fcb0310781 100644 --- a/apple/TableProTests/Views/Structure/StructureSaveConfirmationTests.swift +++ b/apple/TableProTests/Views/Structure/StructureSaveConfirmationTests.swift @@ -235,7 +235,7 @@ struct StructureSaveConfirmationTests { ).authorize(request) let error = try #require(decision.denialError) - #expect(StructureApplyFailure(error) == .cancelledByUser) + #expect(GatedOperationFailure(error) == .cancelledByUser) } @Test("A declined Touch ID is a refusal that says why, not a quiet Cancel") @@ -252,7 +252,7 @@ struct StructureSaveConfirmationTests { ).authorize(request) let error = try #require(decision.denialError) - let failure = StructureApplyFailure(error) + let failure = GatedOperationFailure(error) #expect(failure.outcome == .refused) #expect(failure.message != nil) #expect(!failure.reportsFailure) @@ -404,7 +404,7 @@ struct StructureSaveConfirmationTests { @Test("A Cancel leaves the edits staged and shows nothing") func cancelIsQuiet() { - let failure = StructureApplyFailure(ExecutionGateError.cancelledByUser("Operation cancelled by user")) + let failure = GatedOperationFailure(ExecutionGateError.cancelledByUser("Operation cancelled by user")) #expect(failure == .cancelledByUser) #expect(failure.outcome == .refused) @@ -414,7 +414,7 @@ struct StructureSaveConfirmationTests { @Test("A Safe Mode refusal leaves the edits staged and says why, and is not a failed operation") func refusalExplainsItself() { - let failure = StructureApplyFailure(ExecutionGateError.denied("Safe Mode is read-only")) + let failure = GatedOperationFailure(ExecutionGateError.denied("Safe Mode is read-only")) #expect(failure.outcome == .refused) #expect(failure.message == "Safe Mode is read-only") @@ -423,7 +423,7 @@ struct StructureSaveConfirmationTests { @Test("A statement the server rejected is a failed save") func serverRejectionIsAFailure() { - let failure = StructureApplyFailure(ServerRejection()) + let failure = GatedOperationFailure(ServerRejection()) #expect(failure.outcome == .failed("Duplicate column name 'notes'")) #expect(failure.message == "Duplicate column name 'notes'") diff --git a/apple/TableProTests/Views/Structure/TriggerEditorFontSizeTests.swift b/apple/TableProTests/Views/Structure/TriggerEditorFontSizeTests.swift new file mode 100644 index 0000000000..3745115a46 --- /dev/null +++ b/apple/TableProTests/Views/Structure/TriggerEditorFontSizeTests.swift @@ -0,0 +1,25 @@ +import AppKit +@testable import TablePro +import TableProEditorKit +import Testing + +@MainActor +struct TriggerEditorFontSizeTests { + @Test("New and Edit Trigger open at the size the DDL pane is zoomed to") + func opensAtTheStoredSize() throws { + let zoomed = try EphemeralDefaults() + zoomed.defaults.set(18.0, forKey: TriggerEditorView.fontSizeKey) + + #expect(TriggerEditorView.initialConfiguration(in: zoomed.defaults).appearance.font.pointSize == 18) + } + + @Test("Without a stored size the sheet opens at the default") + func opensAtTheDefaultSize() throws { + let fresh = try EphemeralDefaults() + + #expect( + TriggerEditorView.initialConfiguration(in: fresh.defaults).appearance.font.pointSize + == CGFloat(TriggerEditorView.defaultFontSize) + ) + } +} diff --git a/core/crates/core/src/export/csv.rs b/core/crates/core/src/export/csv.rs index f3ff38e741..55ad66485b 100644 --- a/core/crates/core/src/export/csv.rs +++ b/core/crates/core/src/export/csv.rs @@ -184,11 +184,30 @@ mod tests { } } + fn sanitized() -> CsvOptions { + CsvOptions { + sanitize_formulas: true, + ..Default::default() + } + } + + // An importer reads the quote a sanitized field gains as part of the value. #[test] - fn csv_defaults_quote_every_field_when_sanitising() { + fn csv_defaults_write_values_as_they_are() { + let columns = cols(&["id", "handle"]); + let rows = vec![ + vec![Value::Int(1), Value::Text("@handle".into())], + vec![Value::Int(2), Value::Text("-pending".into())], + ]; + let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + assert_eq!(out, "id,handle\n1,@handle\n2,-pending\n"); + } + + #[test] + fn csv_sanitising_quotes_every_field() { let columns = cols(&["id", "name"]); let rows = vec![vec![Value::Int(1), Value::Text("Alice".into())]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out, "\"id\",\"name\"\n\"1\",\"Alice\"\n"); } @@ -266,7 +285,7 @@ mod tests { let rows = vec![vec![Value::Text("=cmd".into())]]; let opts = CsvOptions { quote: CsvQuote::Never, - ..Default::default() + ..sanitized() }; let out = render_csv(&columns, &rows, &opts).unwrap(); assert_eq!(out, "\"a\"\n\"'=cmd\"\n"); @@ -277,7 +296,7 @@ mod tests { let columns = cols(&["a"]); for ch in ['=', '+', '-', '@'] { let rows = vec![vec![Value::Text(format!("{ch}cmd"))]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out, format!("\"a\"\n\"'{ch}cmd\"\n"), "prefix {ch} should be sanitized"); } } @@ -287,7 +306,7 @@ mod tests { let columns = cols(&["a"]); for lead in ['\t', '\r', '\n'] { let rows = vec![vec![Value::Text(format!("{lead}=cmd|'/C calc'!A0"))]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); let expected = format!("\"a\"\n\"'{lead}=cmd|'/C calc'!A0\"\n"); assert_eq!(out, expected, "lead {lead:?} should be sanitized and quoted"); } @@ -298,7 +317,7 @@ mod tests { let columns = cols(&["a"]); for lead in ['\u{FF1D}', '\u{FF0B}', '\u{FF0D}', '\u{FF20}'] { let rows = vec![vec![Value::Text(format!("{lead}SUM(A1)"))]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!( out, format!("\"a\"\n\"'{lead}SUM(A1)\"\n"), @@ -319,7 +338,7 @@ mod tests { fn csv_does_not_sanitize_non_formula_prefixes() { let columns = cols(&["a"]); let rows = vec![vec![Value::Text("plain text".into())]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out, "\"a\"\n\"plain text\"\n"); } @@ -347,7 +366,7 @@ mod tests { #[test] fn csv_keeps_negative_int() { let columns = cols(&["a"]); - let out = render_csv(&columns, &[vec![Value::Int(-5)]], &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &[vec![Value::Int(-5)]], &sanitized()).unwrap(); assert_eq!(out, "\"a\"\n\"-5\"\n"); } @@ -355,14 +374,14 @@ mod tests { fn csv_keeps_negative_decimal() { let columns = cols(&["a"]); let rows = vec![vec![Value::Decimal("-12.30".parse().expect("a decimal"))]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out, "\"a\"\n\"-12.30\"\n"); } #[test] fn csv_keeps_negative_float() { let columns = cols(&["a"]); - let out = render_csv(&columns, &[vec![Value::Float64(-1.5)]], &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &[vec![Value::Float64(-1.5)]], &sanitized()).unwrap(); assert_eq!(out, "\"a\"\n\"-1.5\"\n"); } @@ -371,7 +390,7 @@ mod tests { let columns = cols(&["a"]); for text in ["-12", "+1.5e3", "-.5", "1.", "-7E-3"] { let rows = vec![vec![Value::Text(text.into())]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out, format!("\"a\"\n\"{text}\"\n"), "{text} is a number, not a formula"); } } @@ -381,7 +400,7 @@ mod tests { let columns = cols(&["a"]); for text in ["-cmd", "=SUM(A1)", "+1+cmd", "-", "-.", "-1e"] { let rows = vec![vec![Value::Text(text.into())]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out, format!("\"a\"\n\"'{text}\"\n"), "{text} must be neutralised"); } } @@ -401,7 +420,7 @@ mod tests { other("-2+cmd"), ]; for value in neutralised { - let out = render_csv(&columns, &[vec![value.clone()]], &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &[vec![value.clone()]], &sanitized()).unwrap(); assert!(out.contains("\"'"), "{value:?} reached the sheet as a formula: {out}"); } let kept = [ @@ -412,7 +431,7 @@ mod tests { (other("+1.5e3"), "+1.5e3"), ]; for (value, text) in kept { - let out = render_csv(&columns, &[vec![value]], &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &[vec![value]], &sanitized()).unwrap(); assert_eq!(out, format!("\"a\"\n\"{text}\"\n"), "{text} is data, not a formula"); } } @@ -420,14 +439,14 @@ mod tests { #[test] fn csv_neutralises_formula_header() { let columns = cols(&["=HYPERLINK(\"x\")"]); - let out = render_csv(&columns, &[], &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &[], &sanitized()).unwrap(); assert_eq!(out, "\"'=HYPERLINK(\"\"x\"\")\"\n"); } #[test] fn text_csv_treats_every_field_as_free_text() { let records = vec![vec!["-cmd".to_string(), "12".to_string(), String::new()]]; - let out = render_text_csv(&["q", "n", "e"], &records, &CsvOptions::default()).unwrap(); + let out = render_text_csv(&["q", "n", "e"], &records, &sanitized()).unwrap(); assert_eq!(out, "\"q\",\"n\",\"e\"\n\"'-cmd\",\"12\",\"\"\n"); } @@ -449,11 +468,11 @@ mod tests { fn csv_null_modes() { let columns = cols(&["a"]); let rows = vec![vec![Value::Null]]; - let out_empty = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out_empty = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out_empty, "\"a\"\n\"\"\n"); let opts = CsvOptions { null_to_empty: false, - ..Default::default() + ..sanitized() }; let out_null = render_csv(&columns, &rows, &opts).unwrap(); assert_eq!(out_null, "\"a\"\n\"NULL\"\n"); diff --git a/core/crates/core/src/export/csv_options.rs b/core/crates/core/src/export/csv_options.rs index 3ff8f7adaf..409842ab0d 100644 --- a/core/crates/core/src/export/csv_options.rs +++ b/core/crates/core/src/export/csv_options.rs @@ -68,7 +68,9 @@ impl Default for CsvOptions { null_to_empty: true, line_break_to_space: false, header_row: true, - sanitize_formulas: true, + // RFC 4180 carries a field as written, and OWASP scopes the quote prefix to files meant + // for a spreadsheet: an importer keeps the quote as part of the value. + sanitize_formulas: false, delimiter: CsvDelimiter::Comma, quote: CsvQuote::IfNeeded, line_break: CsvLineBreak::Lf, diff --git a/core/crates/core/src/export/encoder/tests.rs b/core/crates/core/src/export/encoder/tests.rs index 1e65cb7aa8..6484071e73 100644 --- a/core/crates/core/src/export/encoder/tests.rs +++ b/core/crates/core/src/export/encoder/tests.rs @@ -377,7 +377,11 @@ fn csv_neutralises_text_not_negative_numbers_through_encoder() { vec![Value::Text("-12".into())], ]; - let out = streamed(CsvEncoder::new(columns, CsvOptions::default()), &rows); + let options = CsvOptions { + sanitize_formulas: true, + ..CsvOptions::default() + }; + let out = streamed(CsvEncoder::new(columns, options), &rows); assert_eq!( out, diff --git a/core/crates/mcp/src/export_tools.rs b/core/crates/mcp/src/export_tools.rs index 971b6b73a6..bdcf434f47 100644 --- a/core/crates/mcp/src/export_tools.rs +++ b/core/crates/mcp/src/export_tools.rs @@ -158,7 +158,15 @@ fn render( rows: &[Vec], ) -> Result { let encoded = match format { - ExportFormat::Csv => render_csv(columns, rows, &CsvOptions::default()), + // A tool result has no option to turn it on, so it keeps the neutralising the macOS tool result has. + ExportFormat::Csv => render_csv( + columns, + rows, + &CsvOptions { + sanitize_formulas: true, + ..CsvOptions::default() + }, + ), ExportFormat::Json => render_json(columns, rows), ExportFormat::Sql => { let dialect = session diff --git a/core/crates/storage/src/query_history/export.rs b/core/crates/storage/src/query_history/export.rs index b9d9bacdcc..a4753f5251 100644 --- a/core/crates/storage/src/query_history/export.rs +++ b/core/crates/storage/src/query_history/export.rs @@ -17,7 +17,11 @@ const HISTORY_CSV_HEADER: [&str; 10] = [ pub(super) fn history_csv(entries: &[Entry]) -> Result { let records: Vec> = entries.iter().map(history_record).collect(); - let options = tablepro_core::export::CsvOptions::default(); + // The history export has no option of its own, so it keeps neutralising rather than follow the export default. + let options = tablepro_core::export::CsvOptions { + sanitize_formulas: true, + ..tablepro_core::export::CsvOptions::default() + }; Ok(tablepro_core::export::render_text_csv( &HISTORY_CSV_HEADER, &records, diff --git a/core/crates/test-support/src/conformance/cases/large_table.rs b/core/crates/test-support/src/conformance/cases/large_table.rs index 4b77f07bfb..5dc295e763 100644 --- a/core/crates/test-support/src/conformance/cases/large_table.rs +++ b/core/crates/test-support/src/conformance/cases/large_table.rs @@ -6,7 +6,7 @@ use std::time::{Duration, Instant}; use async_trait::async_trait; use tablepro_core::browse::{ExportRequest, PageRead, PageRequest, PagingMode, build_export, read_page}; -use tablepro_core::export::{CsvEncoder, CsvOptions, RowEncoder}; +use tablepro_core::export::{CsvEncoder, CsvOptions, CsvQuote, RowEncoder}; use tablepro_core::meta::{PagePosition, RowIdentity, TableRef}; use tablepro_core::{ ColumnInfo, Connection, FilterSet, ReadStatement, ResultEvent, ResultSink, SinkFlow, UserSql, Value, WriteKind, @@ -538,7 +538,12 @@ impl ResultSink for CsvFileSink { match event { ResultEvent::ResultSetStarted { columns } => { lock(&self.tally).result_sets += 1; - let mut encoder = CsvEncoder::new(columns, CsvOptions::default()); + // `check_file` reads every field quoted. + let options = CsvOptions { + quote: CsvQuote::Always, + ..CsvOptions::default() + }; + let mut encoder = CsvEncoder::new(columns, options); encoder.begin(&mut self.buffer); self.encoder = Some(encoder); SinkFlow::Continue diff --git a/core/crates/test-support/src/conformance/cases/round_trip.rs b/core/crates/test-support/src/conformance/cases/round_trip.rs index 9e17db85c6..94f0a82d79 100644 --- a/core/crates/test-support/src/conformance/cases/round_trip.rs +++ b/core/crates/test-support/src/conformance/cases/round_trip.rs @@ -47,7 +47,7 @@ const IMPORT_DEADLINE: Duration = Duration::from_secs(120); /// that hangs. const EXPORT_DEADLINE: Duration = Duration::from_secs(120); -/// Under the export defaults these rows of 200-character labels make a +/// With every field quoted these rows of 200-character labels make a /// file just under `MAX_CSV_FILE_BYTES`. 200 characters is the widest /// label every engine's scratch table takes. const NEAR_LIMIT_ROWS: u64 = 315_000; @@ -428,7 +428,11 @@ pub async fn csv_export_near_the_import_limit_imports_back( FILL_DEADLINE, ) .await?; - let file = export_file(connection.as_ref(), &source, csv_encoder(CsvOptions::default())).await?; + let quoted = CsvOptions { + quote: CsvQuote::Always, + ..CsvOptions::default() + }; + let file = export_file(connection.as_ref(), &source, csv_encoder(quoted)).await?; let size = file.len(); expect( NEAR_LIMIT_CASE, diff --git a/linux/crates/app/src/ui/export/encoding_sink.rs b/linux/crates/app/src/ui/export/encoding_sink.rs index 947fcb8fa8..bb1b3bb509 100644 --- a/linux/crates/app/src/ui/export/encoding_sink.rs +++ b/linux/crates/app/src/ui/export/encoding_sink.rs @@ -239,7 +239,7 @@ mod tests { assert_eq!(finished, Some(1)); let text: Vec = chunks.into_iter().flat_map(|(bytes, _)| bytes).collect(); - assert_eq!(String::from_utf8(text).expect("utf-8"), "\"x\",\"y\"\n\"1\",\"2\"\n"); + assert_eq!(String::from_utf8(text).expect("utf-8"), "x,y\n1,2\n"); } #[tokio::test(start_paused = true)] diff --git a/linux/crates/app/src/ui/export/export_ledger/tests.rs b/linux/crates/app/src/ui/export/export_ledger/tests.rs index 97e830ca17..63fdf02bb8 100644 --- a/linux/crates/app/src/ui/export/export_ledger/tests.rs +++ b/linux/crates/app/src/ui/export/export_ledger/tests.rs @@ -250,8 +250,8 @@ fn entire_table_export_streams_every_row() { let text = std::fs::read_to_string(&path).expect("the export"); let lines: Vec<&str> = text.lines().collect(); assert_eq!(lines.len(), 25_001); - assert_eq!(lines[0], "\"id\",\"name\""); - assert_eq!(lines[25_000], "\"24999\",\"user 24999\""); + assert_eq!(lines[0], "id,name"); + assert_eq!(lines[25_000], "24999,user 24999"); assert_eq!( fake.connection.observed_deadlines("stream_read"), [None], @@ -298,7 +298,7 @@ fn entire_table_export_has_no_identity_columns() { "{sql}" ); let text = std::fs::read_to_string(&path).expect("the export"); - assert_eq!(text.lines().next(), Some("\"id\",\"name\"")); + assert_eq!(text.lines().next(), Some("id,name")); } #[gtk4::test] @@ -510,11 +510,7 @@ fn snapshot_export_of_10k_rows_keeps_main_loop_responsive() { assert!(ticks.get() > 0, "the main loop never ran while the export did"); let text = std::fs::read_to_string(&path).expect("the export"); assert_eq!(text.lines().count(), 10_001); - assert!( - text.starts_with("\"id\",\"name\",\"note\"\n\"0\",\"name 0\""), - "{}", - &text[..40] - ); + assert!(text.starts_with("id,name,note\n0,name 0"), "{}", &text[..40]); assert_eq!(harness.ledger.in_flight(), 0); assert_eq!(harness.hold.live(), 0, "the export kept the process held"); } diff --git a/linux/crates/platform/tests/settings.rs b/linux/crates/platform/tests/settings.rs index e95371763f..6939de58a3 100644 --- a/linux/crates/platform/tests/settings.rs +++ b/linux/crates/platform/tests/settings.rs @@ -213,7 +213,7 @@ async fn csv_options_round_trip() -> TestResult { null_to_empty: false, line_break_to_space: true, header_row: false, - sanitize_formulas: false, + sanitize_formulas: true, delimiter: CsvDelimiter::Pipe, quote: CsvQuote::Always, line_break: CsvLineBreak::CrLf, diff --git a/linux/data/app.tablepro.TablePro.gschema.xml.in b/linux/data/app.tablepro.TablePro.gschema.xml.in index 2044a4b947..af9c9f021a 100644 --- a/linux/data/app.tablepro.TablePro.gschema.xml.in +++ b/linux/data/app.tablepro.TablePro.gschema.xml.in @@ -324,7 +324,7 @@ - true + false Neutralise spreadsheet formulas Whether a CSV export prefixes values starting with =, +, - or @ so a spreadsheet does not run them. diff --git a/linux/docs/storage.md b/linux/docs/storage.md index f190f695be..ce39a839d1 100644 --- a/linux/docs/storage.md +++ b/linux/docs/storage.md @@ -378,7 +378,7 @@ The schema id is the application id and the path follows it: `app.tablepro.Table | `csv-null-to-empty` | `b` | `true` | | | `csv-line-break-to-space` | `b` | `false` | | | `csv-header-row` | `b` | `true` | | -| `csv-sanitize-formulas` | `b` | `true` | | +| `csv-sanitize-formulas` | `b` | `false` | | | `csv-delimiter` | enum | `comma` | comma, semicolon, tab, pipe | | `csv-quote` | enum | `if-needed` | always, if-needed, never | | `csv-line-break` | enum | `lf` | lf, crlf, cr | From 5b9d13a6df2e055f899b5c0d7d08d67602e7d308 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 11 Nov 2026 14:52:25 +0700 Subject: [PATCH 2/5] docs(macos): add the changelog fragments for #4185 --- apple/changelog.d/4185.fixed.1.md | 1 + apple/changelog.d/4185.fixed.10.md | 1 + apple/changelog.d/4185.fixed.11.md | 1 + apple/changelog.d/4185.fixed.12.md | 1 + apple/changelog.d/4185.fixed.13.md | 1 + apple/changelog.d/4185.fixed.14.md | 1 + apple/changelog.d/4185.fixed.15.md | 1 + apple/changelog.d/4185.fixed.16.md | 1 + apple/changelog.d/4185.fixed.17.md | 1 + apple/changelog.d/4185.fixed.18.md | 1 + apple/changelog.d/4185.fixed.19.md | 1 + apple/changelog.d/4185.fixed.2.md | 1 + apple/changelog.d/4185.fixed.20.md | 1 + apple/changelog.d/4185.fixed.21.md | 1 + apple/changelog.d/4185.fixed.22.md | 1 + apple/changelog.d/4185.fixed.23.md | 1 + apple/changelog.d/4185.fixed.24.md | 1 + apple/changelog.d/4185.fixed.25.md | 1 + apple/changelog.d/4185.fixed.26.md | 1 + apple/changelog.d/4185.fixed.27.md | 1 + apple/changelog.d/4185.fixed.28.md | 1 + apple/changelog.d/4185.fixed.29.md | 1 + apple/changelog.d/4185.fixed.3.md | 1 + apple/changelog.d/4185.fixed.30.md | 1 + apple/changelog.d/4185.fixed.31.md | 1 + apple/changelog.d/4185.fixed.32.md | 1 + apple/changelog.d/4185.fixed.4.md | 1 + apple/changelog.d/4185.fixed.5.md | 1 + apple/changelog.d/4185.fixed.6.md | 1 + apple/changelog.d/4185.fixed.7.md | 1 + apple/changelog.d/4185.fixed.8.md | 1 + apple/changelog.d/4185.fixed.9.md | 1 + apple/changelog.d/4185.fixed.md | 1 + linux/changelog.d/4185.fixed.md | 1 + 34 files changed, 34 insertions(+) create mode 100644 apple/changelog.d/4185.fixed.1.md create mode 100644 apple/changelog.d/4185.fixed.10.md create mode 100644 apple/changelog.d/4185.fixed.11.md create mode 100644 apple/changelog.d/4185.fixed.12.md create mode 100644 apple/changelog.d/4185.fixed.13.md create mode 100644 apple/changelog.d/4185.fixed.14.md create mode 100644 apple/changelog.d/4185.fixed.15.md create mode 100644 apple/changelog.d/4185.fixed.16.md create mode 100644 apple/changelog.d/4185.fixed.17.md create mode 100644 apple/changelog.d/4185.fixed.18.md create mode 100644 apple/changelog.d/4185.fixed.19.md create mode 100644 apple/changelog.d/4185.fixed.2.md create mode 100644 apple/changelog.d/4185.fixed.20.md create mode 100644 apple/changelog.d/4185.fixed.21.md create mode 100644 apple/changelog.d/4185.fixed.22.md create mode 100644 apple/changelog.d/4185.fixed.23.md create mode 100644 apple/changelog.d/4185.fixed.24.md create mode 100644 apple/changelog.d/4185.fixed.25.md create mode 100644 apple/changelog.d/4185.fixed.26.md create mode 100644 apple/changelog.d/4185.fixed.27.md create mode 100644 apple/changelog.d/4185.fixed.28.md create mode 100644 apple/changelog.d/4185.fixed.29.md create mode 100644 apple/changelog.d/4185.fixed.3.md create mode 100644 apple/changelog.d/4185.fixed.30.md create mode 100644 apple/changelog.d/4185.fixed.31.md create mode 100644 apple/changelog.d/4185.fixed.32.md create mode 100644 apple/changelog.d/4185.fixed.4.md create mode 100644 apple/changelog.d/4185.fixed.5.md create mode 100644 apple/changelog.d/4185.fixed.6.md create mode 100644 apple/changelog.d/4185.fixed.7.md create mode 100644 apple/changelog.d/4185.fixed.8.md create mode 100644 apple/changelog.d/4185.fixed.9.md create mode 100644 apple/changelog.d/4185.fixed.md create mode 100644 linux/changelog.d/4185.fixed.md diff --git a/apple/changelog.d/4185.fixed.1.md b/apple/changelog.d/4185.fixed.1.md new file mode 100644 index 0000000000..625f6be5b1 --- /dev/null +++ b/apple/changelog.d/4185.fixed.1.md @@ -0,0 +1 @@ +CSV export with a decimal comma rewrote text values that looked like decimals, such as a version 1.10. diff --git a/apple/changelog.d/4185.fixed.10.md b/apple/changelog.d/4185.fixed.10.md new file mode 100644 index 0000000000..c4812cbb1b --- /dev/null +++ b/apple/changelog.d/4185.fixed.10.md @@ -0,0 +1 @@ +Rewinding a save could fail on a unique key when an earlier edit in it had been undone before saving. diff --git a/apple/changelog.d/4185.fixed.11.md b/apple/changelog.d/4185.fixed.11.md new file mode 100644 index 0000000000..03913855f0 --- /dev/null +++ b/apple/changelog.d/4185.fixed.11.md @@ -0,0 +1 @@ +Rewind reported a row as missing when the save had changed its primary key. diff --git a/apple/changelog.d/4185.fixed.12.md b/apple/changelog.d/4185.fixed.12.md new file mode 100644 index 0000000000..427ed680e8 --- /dev/null +++ b/apple/changelog.d/4185.fixed.12.md @@ -0,0 +1 @@ +A value of __DEFAULT__ was saved as the column default when duplicating a row or typing it into a cell. diff --git a/apple/changelog.d/4185.fixed.13.md b/apple/changelog.d/4185.fixed.13.md new file mode 100644 index 0000000000..f3f1e23763 --- /dev/null +++ b/apple/changelog.d/4185.fixed.13.md @@ -0,0 +1 @@ +A table opened while another query waited for Safe Mode approval stayed empty with its loading indicator. diff --git a/apple/changelog.d/4185.fixed.14.md b/apple/changelog.d/4185.fixed.14.md new file mode 100644 index 0000000000..bfaa1727f7 --- /dev/null +++ b/apple/changelog.d/4185.fixed.14.md @@ -0,0 +1 @@ +Saving while closing a window could close it over unsaved edits in another connection or tab. diff --git a/apple/changelog.d/4185.fixed.15.md b/apple/changelog.d/4185.fixed.15.md new file mode 100644 index 0000000000..157fe467ec --- /dev/null +++ b/apple/changelog.d/4185.fixed.15.md @@ -0,0 +1 @@ +Saving a Cassandra row that another client deleted recreated it with every other column empty. diff --git a/apple/changelog.d/4185.fixed.16.md b/apple/changelog.d/4185.fixed.16.md new file mode 100644 index 0000000000..9b9cdee113 --- /dev/null +++ b/apple/changelog.d/4185.fixed.16.md @@ -0,0 +1 @@ +Running SET with NX or XX in the Redis console showed OK when the key was not written. diff --git a/apple/changelog.d/4185.fixed.17.md b/apple/changelog.d/4185.fixed.17.md new file mode 100644 index 0000000000..27e55c9db4 --- /dev/null +++ b/apple/changelog.d/4185.fixed.17.md @@ -0,0 +1 @@ +Pressing Stop as a MySQL tab closed could crash the app. diff --git a/apple/changelog.d/4185.fixed.18.md b/apple/changelog.d/4185.fixed.18.md new file mode 100644 index 0000000000..18a4d89177 --- /dev/null +++ b/apple/changelog.d/4185.fixed.18.md @@ -0,0 +1 @@ +A startup command that failed, such as SET ROLE, left the session running without it and nothing said so. diff --git a/apple/changelog.d/4185.fixed.19.md b/apple/changelog.d/4185.fixed.19.md new file mode 100644 index 0000000000..e052ba407c --- /dev/null +++ b/apple/changelog.d/4185.fixed.19.md @@ -0,0 +1 @@ +Exporting more than 5,000,000 MongoDB documents wrote the first 5,000,000 and reported success. diff --git a/apple/changelog.d/4185.fixed.2.md b/apple/changelog.d/4185.fixed.2.md new file mode 100644 index 0000000000..51fe78d8f7 --- /dev/null +++ b/apple/changelog.d/4185.fixed.2.md @@ -0,0 +1 @@ +Changing export or import options in another window changed a running export or import, or left foreign keys off. diff --git a/apple/changelog.d/4185.fixed.20.md b/apple/changelog.d/4185.fixed.20.md new file mode 100644 index 0000000000..a4aa8a48a5 --- /dev/null +++ b/apple/changelog.d/4185.fixed.20.md @@ -0,0 +1 @@ +Stopping a Teradata query ended the whole session, so volatile tables were lost and the next query failed. diff --git a/apple/changelog.d/4185.fixed.21.md b/apple/changelog.d/4185.fixed.21.md new file mode 100644 index 0000000000..21106f7c75 --- /dev/null +++ b/apple/changelog.d/4185.fixed.21.md @@ -0,0 +1 @@ +A query parameter such as zip code 00123 was sent to Trino as the number 123. diff --git a/apple/changelog.d/4185.fixed.22.md b/apple/changelog.d/4185.fixed.22.md new file mode 100644 index 0000000000..05b1cf291d --- /dev/null +++ b/apple/changelog.d/4185.fixed.22.md @@ -0,0 +1 @@ +TablePro could quit when the Copilot language server exited while TablePro was sending it a message. diff --git a/apple/changelog.d/4185.fixed.23.md b/apple/changelog.d/4185.fixed.23.md new file mode 100644 index 0000000000..244113e6d1 --- /dev/null +++ b/apple/changelog.d/4185.fixed.23.md @@ -0,0 +1 @@ +Run Current and the gutter cut a SQL Server IF, WHILE or TRY...CATCH block at each semicolon inside it. diff --git a/apple/changelog.d/4185.fixed.24.md b/apple/changelog.d/4185.fixed.24.md new file mode 100644 index 0000000000..d34f707ee7 --- /dev/null +++ b/apple/changelog.d/4185.fixed.24.md @@ -0,0 +1 @@ +Run Current split a PostgreSQL rule with several commands in parentheses at each semicolon. diff --git a/apple/changelog.d/4185.fixed.25.md b/apple/changelog.d/4185.fixed.25.md new file mode 100644 index 0000000000..4ff326bcbd --- /dev/null +++ b/apple/changelog.d/4185.fixed.25.md @@ -0,0 +1 @@ +New Trigger and Edit Trigger opened at 13 pt instead of the size the Structure tab's DDL pane was zoomed to. diff --git a/apple/changelog.d/4185.fixed.26.md b/apple/changelog.d/4185.fixed.26.md new file mode 100644 index 0000000000..65af010304 --- /dev/null +++ b/apple/changelog.d/4185.fixed.26.md @@ -0,0 +1 @@ +Moving the caret in a long MongoDB or Redis query re-read the whole document every time. diff --git a/apple/changelog.d/4185.fixed.27.md b/apple/changelog.d/4185.fixed.27.md new file mode 100644 index 0000000000..f0afd7c7e2 --- /dev/null +++ b/apple/changelog.d/4185.fixed.27.md @@ -0,0 +1 @@ +Column and table suggestions on iPhone and iPad were rebuilt on every keystroke. diff --git a/apple/changelog.d/4185.fixed.28.md b/apple/changelog.d/4185.fixed.28.md new file mode 100644 index 0000000000..27178d8fa0 --- /dev/null +++ b/apple/changelog.d/4185.fixed.28.md @@ -0,0 +1 @@ +The ClickHouse Parts tab hid its list and toolbar after a cancelled or failed partition action. diff --git a/apple/changelog.d/4185.fixed.29.md b/apple/changelog.d/4185.fixed.29.md new file mode 100644 index 0000000000..0ac47956a5 --- /dev/null +++ b/apple/changelog.d/4185.fixed.29.md @@ -0,0 +1 @@ +Dropping or detaching a ClickHouse partition failed on tables partitioned by several columns or not at all. diff --git a/apple/changelog.d/4185.fixed.3.md b/apple/changelog.d/4185.fixed.3.md new file mode 100644 index 0000000000..8fa5c94f07 --- /dev/null +++ b/apple/changelog.d/4185.fixed.3.md @@ -0,0 +1 @@ +Data sync and Copy as SQL wrote PostgreSQL binary values that stored other bytes with standard_conforming_strings off. diff --git a/apple/changelog.d/4185.fixed.30.md b/apple/changelog.d/4185.fixed.30.md new file mode 100644 index 0000000000..6be6044706 --- /dev/null +++ b/apple/changelog.d/4185.fixed.30.md @@ -0,0 +1 @@ +Cancelling the Safe Mode confirmation in an object editor or sheet showed an error. diff --git a/apple/changelog.d/4185.fixed.31.md b/apple/changelog.d/4185.fixed.31.md new file mode 100644 index 0000000000..c6d66b0771 --- /dev/null +++ b/apple/changelog.d/4185.fixed.31.md @@ -0,0 +1 @@ +Applying user changes ran ALTER and password statements for a user staged for drop, and overcounted pending changes. diff --git a/apple/changelog.d/4185.fixed.32.md b/apple/changelog.d/4185.fixed.32.md new file mode 100644 index 0000000000..98e47dea15 --- /dev/null +++ b/apple/changelog.d/4185.fixed.32.md @@ -0,0 +1 @@ +Moving the caret next to a bracket far below the visible text could crash the editor. diff --git a/apple/changelog.d/4185.fixed.4.md b/apple/changelog.d/4185.fixed.4.md new file mode 100644 index 0000000000..a1b1cd6aa7 --- /dev/null +++ b/apple/changelog.d/4185.fixed.4.md @@ -0,0 +1 @@ +Copy as SQL wrote DuckDB and Oracle binary values that DuckDB stored as text and Oracle rejected. diff --git a/apple/changelog.d/4185.fixed.5.md b/apple/changelog.d/4185.fixed.5.md new file mode 100644 index 0000000000..2aea41ab0b --- /dev/null +++ b/apple/changelog.d/4185.fixed.5.md @@ -0,0 +1 @@ +Data sync wrote an empty Oracle BLOB as NULL, which a NOT NULL column refused. diff --git a/apple/changelog.d/4185.fixed.6.md b/apple/changelog.d/4185.fixed.6.md new file mode 100644 index 0000000000..77fcfcfd1f --- /dev/null +++ b/apple/changelog.d/4185.fixed.6.md @@ -0,0 +1 @@ +Data sync and Copy as SQL wrote Redshift binary values in a form Redshift refused. diff --git a/apple/changelog.d/4185.fixed.7.md b/apple/changelog.d/4185.fixed.7.md new file mode 100644 index 0000000000..2b5c350f28 --- /dev/null +++ b/apple/changelog.d/4185.fixed.7.md @@ -0,0 +1 @@ +Add Rows in Shortcuts refused CSV with bracketed headers such as [id], as SQL Server exports them. diff --git a/apple/changelog.d/4185.fixed.8.md b/apple/changelog.d/4185.fixed.8.md new file mode 100644 index 0000000000..88dc60ec1c --- /dev/null +++ b/apple/changelog.d/4185.fixed.8.md @@ -0,0 +1 @@ +Add Rows in Shortcuts refused a header that differed from its column's name only in case. diff --git a/apple/changelog.d/4185.fixed.9.md b/apple/changelog.d/4185.fixed.9.md new file mode 100644 index 0000000000..081e98b9ba --- /dev/null +++ b/apple/changelog.d/4185.fixed.9.md @@ -0,0 +1 @@ +Copy to another database and Data Compare missed rows past a database's row limit without saying so. diff --git a/apple/changelog.d/4185.fixed.md b/apple/changelog.d/4185.fixed.md new file mode 100644 index 0000000000..a90c14e4ae --- /dev/null +++ b/apple/changelog.d/4185.fixed.md @@ -0,0 +1 @@ +CSV export put a single quote before values starting with =, +, - or @ by default, which then stayed in imports. diff --git a/linux/changelog.d/4185.fixed.md b/linux/changelog.d/4185.fixed.md new file mode 100644 index 0000000000..a90c14e4ae --- /dev/null +++ b/linux/changelog.d/4185.fixed.md @@ -0,0 +1 @@ +CSV export put a single quote before values starting with =, +, - or @ by default, which then stayed in imports. From 4bd9bc7dc60094b452cd050f80a2a5d907a6ae24 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 11 Nov 2026 17:26:04 +0700 Subject: [PATCH 3/5] build(pluginkit): raise the kit to 44 for the additive API --- apple/Configs/PluginKit.xcconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apple/Configs/PluginKit.xcconfig b/apple/Configs/PluginKit.xcconfig index fef1b5c3ce..207ba58516 100644 --- a/apple/Configs/PluginKit.xcconfig +++ b/apple/Configs/PluginKit.xcconfig @@ -1,3 +1,3 @@ // When to raise each, and how the build stamps them: .claude/rules/plugin-system.md, ABI. -TABLEPRO_PLUGINKIT_VERSION = 43 +TABLEPRO_PLUGINKIT_VERSION = 44 TABLEPRO_PLUGINKIT_MINIMUM_VERSION = 19 From 06db634b20c5b13e6c00ae5c03d172b9416b3479 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 11 Nov 2026 22:04:48 +0700 Subject: [PATCH 4/5] fix(macos): finish the audit wave 5 data items for find, Run All, pool, edits, sessions, secrets, TLS, JSON import and AI limits --- .claude/rules/data-sync-storage.md | 1 + .claude/rules/tests.md | 2 +- apple/Packages/TableProCore/Package.swift | 2 +- .../SQLSessionStateClassifier.swift | 708 ++++++++++++++++++ .../LibPQSystemTrustRule.swift | 32 + .../SystemTrustAnchors.swift | 284 +++++++ .../SQLSessionStateClassifierTests.swift | 409 ++++++++++ .../SystemTrustAnchorsTests.swift | 288 +++++++ .../JSONDocumentStreamingTests.swift | 124 ++- .../JSONDocumentOutline.swift | 13 +- .../JSONImportPlugin/JSONDocumentReader.swift | 115 ++- .../JSONImportPlugin/JSONElementBatches.swift | 18 +- .../JSONImportPlugin/JSONFieldSurvey.swift | 6 +- .../JSONImportPlugin/JSONImportParsing.swift | 54 +- .../JSONImportPlugin/JSONTextDecoder.swift | 109 +++ .../MSSQLStatementGenerator.swift | 6 +- .../LibPQConnectionString.swift | 59 +- .../LibPQPluginConnection.swift | 157 ++-- .../LibPQSSLMapping.swift | 17 + .../LibPQConnectionStringTests.swift | 102 ++- .../LibPQSSLMappingTests.swift | 29 + .../SurrealStatementGenerator.swift | 36 +- .../PluginDatabaseDriver.swift | 1 + .../TablePro/Core/AI/AnthropicProvider.swift | 9 +- .../TablePro/Core/AI/Chat/ChatTransport.swift | 7 + apple/TablePro/Core/AI/Chat/ChatTurn.swift | 21 +- .../AI/ClaudeAgent/ClaudeAgentProvider.swift | 16 + apple/TablePro/Core/AI/GeminiProvider.swift | 4 + .../Core/AI/OpenAICompatibleProvider.swift | 7 + .../Core/AI/OpenAIResponsesProvider.swift | 10 +- .../ChangeTracking/DataChangeManager.swift | 99 ++- .../ChangeTracking/DataChangeModels.swift | 117 ++- .../Core/ChangeTracking/PendingChanges.swift | 194 +++-- .../SQLStatementGenerator.swift | 28 +- .../Core/Coordinators/FindCoordinator.swift | 3 +- .../QueryExecutionCoordinator+Batches.swift | 125 +++- ...yExecutionCoordinator+MultiStatement.swift | 57 +- ...QueryExecutionCoordinator+Parameters.swift | 54 +- .../Coordinators/RowEditingCoordinator.swift | 112 +-- .../Core/Coordinators/RunResultPlan.swift | 103 +++ .../Core/DataGrid/DataGridModel+Find.swift | 156 ++-- .../Core/DataGrid/Engine/GridFindEngine.swift | 102 ++- .../Core/DataGrid/GridFindState.swift | 90 ++- .../Core/DataWrite/DataWritePlan.swift | 80 ++ .../DataWrite/Rewind/RewindExecutor.swift | 20 +- .../Core/DataWrite/Rewind/RewindPlanner.swift | 127 +++- .../DataWrite/RowChangeStatementFactory.swift | 62 +- .../Core/DataWrite/RowWriteCoverage.swift | 16 +- .../DataWrite/RowWriteOperationBuilder.swift | 46 +- .../Access/DatabaseAccessBridge+Scripts.swift | 18 +- .../Access/DatabaseAccessBridge.swift | 2 + .../DatabaseManager+CredentialLease.swift | 24 +- .../Database/DatabaseManager+Health.swift | 25 +- .../Database/DatabaseManager+Recovery.swift | 17 +- .../DatabaseManager+ScopedDriver.swift | 112 +-- .../Database/DatabaseManager+Sessions.swift | 24 +- .../Core/Database/DatabaseManager.swift | 10 + .../Core/Database/NativeDumpDescriptor.swift | 119 ++- .../Core/Database/NativeDumpRegistry.swift | 150 ++-- .../Core/Database/NativeDumpService.swift | 163 ++-- .../TablePro/Core/Database/SessionLanes.swift | 86 ++- .../Core/Database/SessionStateLedger.swift | 171 +++++ .../Core/Plugins/DatabaseType+Registry.swift | 29 +- ...uginMetadataRegistry+CuratedDefaults.swift | 32 +- ...luginMetadataRegistry+DuckDBDefaults.swift | 21 +- .../PluginMetadataRegistry+HanaDefaults.swift | 3 +- ...PluginMetadataRegistry+KafkaDefaults.swift | 17 +- ...luginMetadataRegistry+QdrantDefaults.swift | 1 + ...ginMetadataRegistry+RegistryDefaults.swift | 17 +- ...PluginMetadataRegistry+TrinoDefaults.swift | 4 +- .../Core/Plugins/PluginMetadataRegistry.swift | 227 +++--- .../Formatting/BlobFormattingService.swift | 9 +- .../Formatting/CellDisplayFormatter.swift | 31 +- .../Formatting/DateDisplayRenderer.swift | 118 +++ .../Formatting/DateFormattingService.swift | 170 +---- .../Formatting/ValueDisplayFormatter.swift | 21 +- .../Query/DatabaseTreeMetadataService.swift | 81 +- .../Query/MetadataConnectionPool.swift | 532 ++++++++----- .../Core/Storage/ConnectionStorage.swift | 57 +- .../Core/Storage/KeychainHelper.swift | 24 + .../Query/GridDisplayOrderResolver.swift | 42 +- .../TablePro/Resources/Localizable.xcstrings | 69 ++ .../AIChatViewModel+MessageEditing.swift | 3 +- .../AIChatViewModel+Streaming.swift | 23 +- .../ViewModels/WelcomeViewModel+Library.swift | 21 +- .../ViewModels/WelcomeViewModel.swift | 2 + .../Views/AIChat/AIChatMessageView.swift | 18 + .../ConnectionFormCoordinator.swift | 5 + .../Panes/OptionsPaneView.swift | 11 +- .../DataGridViewController+ValueFilter.swift | 122 ++- .../Views/DataGrid/DataGridFindBar.swift | 17 +- .../Views/Main/MainContentCoordinator.swift | 4 +- .../Views/Welcome/WelcomePresentations.swift | 13 + .../TableProMobile/Localizable.xcstrings | 4 + .../Core/AI/OutputLimitSignalTests.swift | 177 +++++ .../Autocomplete/SQLSchemaProviderTests.swift | 4 + .../EditedRowChangeTrackingTests.swift | 355 +++++++++ .../Core/Coordinators/FindMatcherTests.swift | 4 +- .../Coordinators/RunResultPlanTests.swift | 110 +++ .../DataGrid/Engine/GridFindEngineTests.swift | 386 +++++++++- .../DataWrite/PluginKeyedChangesTests.swift | 29 +- .../Core/DataWrite/RewindPlannerTests.swift | 133 ++++ .../RewindRecordIdentityDecodingTests.swift | 57 ++ ...wChangeStatementFactoryCoverageTests.swift | 34 + .../RowWriteOperationBuilderTests.swift | 31 + .../CredentialLeaseRoutingTests.swift | 39 +- .../Core/Database/SessionLanesTests.swift | 130 +++- .../Database/SessionRecoveryLoopTests.swift | 204 ++++- .../SwitchDatabasePooledConnectionTests.swift | 4 +- ...tadataRegistryCuratedCapabilityTests.swift | 72 +- .../Services/DateFormattingServiceTests.swift | 59 ++ .../DatabaseTreeMetadataServiceTests.swift | 62 ++ .../MetadataConnectionPoolCapacityTests.swift | 34 - .../MetadataConnectionPoolSchemaTests.swift | 399 ++++++++++ .../Storage/ConnectionSecretScopeTests.swift | 81 +- .../Core/Storage/KeychainItemScopeTests.swift | 20 + .../Database/NativeDumpRegistryTests.swift | 181 +++-- .../Helpers/CountingOpener.swift | 42 ++ .../Helpers/ScopeRecordingKeychain.swift | 23 + .../Models/DatabaseTypeTests.swift | 2 +- .../ConnectionFormChildObservationTests.swift | 8 + .../ViewModels/SSLPaneViewModelTests.swift | 23 +- .../ViewModels/WelcomeViewModelTests.swift | 36 + .../Views/DataGrid/DataGridCommandTests.swift | 61 ++ .../DataGrid/DataGridControllerTests.swift | 3 +- .../Main/ResultRowsBuildOffMainTests.swift | 13 + .../InvisibleCharacterSettingTests.swift | 13 +- apple/project.yml | 4 +- .../probes/check-system-trust-anchors.sh | 126 ++++ 129 files changed, 8215 insertions(+), 1650 deletions(-) create mode 100644 apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLSessionStateClassifier.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/LibPQSystemTrustRule.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/SystemTrustAnchors.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLSessionStateClassifierTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/SystemTrustAnchorsTests.swift create mode 100644 apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift create mode 100644 apple/TablePro/Core/Coordinators/RunResultPlan.swift create mode 100644 apple/TablePro/Core/Database/SessionStateLedger.swift create mode 100644 apple/TablePro/Core/Services/Formatting/DateDisplayRenderer.swift create mode 100644 apple/TableProTests/Core/AI/OutputLimitSignalTests.swift create mode 100644 apple/TableProTests/Core/Coordinators/RunResultPlanTests.swift create mode 100644 apple/TableProTests/Core/Services/Query/MetadataConnectionPoolSchemaTests.swift create mode 100644 apple/TableProTests/Helpers/CountingOpener.swift create mode 100755 apple/scripts/probes/check-system-trust-anchors.sh diff --git a/.claude/rules/data-sync-storage.md b/.claude/rules/data-sync-storage.md index 49b33f7ff5..9a07bfc6b1 100644 --- a/.claude/rules/data-sync-storage.md +++ b/.claude/rules/data-sync-storage.md @@ -181,6 +181,7 @@ This path holds user data. A silent wrong answer here looks exactly like an empt | Snowflake SSO and MFA tokens, per host and user. A bare account's `NAME.USER` item stays for 0.83.1, which reads only that key | Keychain, device-local | `SnowflakeTokenCache` | | Whether `connections.json` was ever given an integrity tag, so a deleted tag reads as modified | Keychain `com.TablePro.connectionStoreStamped.`, device-local | `ConnectionStoreIntegrity` | | Whether Local only connections' secrets were copied to device-only items | UserDefaults `com.TablePro.localOnlySecretsKeptOnDevice`, device-local, never synced | `ConnectionStorage` | +| iCloud Keychain copies of Local only secrets the keychain refused to remove, retried at launch | UserDefaults `com.TablePro.localOnlySyncedCopiesPendingRemoval`, device-local, never synced | `ConnectionStorage` | | Whether plugin secure fields moved out of `connections.json` into the Keychain, set only when every field moved and the save succeeded | UserDefaults `com.TablePro.pluginSecureFieldsMovedToKeychain`, device-local, never synced | `ConnectionStorage` | | Whether a saved-queries store replaced a damaged file and waits for a full fetch, so push treats it as unreadable | Table `sync_refill` in the saved-queries SQLite store, dropped once a full fetch applied | `SQLFavoriteStorage` | | Deletions the server reported for a sync category that is off, applied when the category is next read | UserDefaults, device-local, never synced | `SyncMetadataStorage.deferredDeletions` | diff --git a/.claude/rules/tests.md b/.claude/rules/tests.md index 6b366b0dee..cd1648a222 100644 --- a/.claude/rules/tests.md +++ b/.claude/rules/tests.md @@ -73,7 +73,7 @@ UI flows: `.claude/rules/ui-tests.md`. Running tests and reading verdicts: `.cla Use one before writing a local helper. - `TableProTestSupport` (TableProCore): `waitUntil`, `AsyncGate`, `ManualClock` (production code takes `clock: any Clock = ContinuousClock()`), `BoundedCall`, `HeldOpenWriter`, `BackgroundPipeWriter`, `BackgroundPipeReader`. -- `apple/TableProTests/Helpers/`: `GlobalStateSuites`, `EphemeralDefaults`, `.appSettings { }`, `.recordingClipboard`, `CoordinatorFixture`, `OfflineRegistry`, `InMemoryColumnLayoutStore`, `InMemoryKeychain`, `PluginDriverStub`, `TestFixtures`, `AlwaysAllowGate`, `AlwaysDenyGate`, `MCPInMemoryApprovalStore`, `MCPFixedApprovalReader`, `ArrayRowProvider`, `ScopeRecordingKeychain`. +- `apple/TableProTests/Helpers/`: `GlobalStateSuites`, `EphemeralDefaults`, `.appSettings { }`, `.recordingClipboard`, `CoordinatorFixture`, `OfflineRegistry`, `InMemoryColumnLayoutStore`, `InMemoryKeychain`, `PluginDriverStub`, `TestFixtures`, `AlwaysAllowGate`, `AlwaysDenyGate`, `CountingOpener`, `MCPInMemoryApprovalStore`, `MCPFixedApprovalReader`, `ArrayRowProvider`, `ScopeRecordingKeychain`. - `TableProMCPTestSupport` (TableProMacCore): `MCPTestClock`, `MockHttpServer`, `RecordingBridgeLogger`. `TableProTLSTestFixtures` (TableProCore), also for the iOS tests. ## Known failures and flakes diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index 7b85a5049e..a59c9a69de 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -541,7 +541,7 @@ let package = Package( ), .testTarget( name: "TableProTLSClientIdentityTests", - dependencies: ["TableProTLSClientIdentity", "TableProTLSTestFixtures"], + dependencies: ["TableProTLSClientIdentity", "TableProTLSTestFixtures", "TableProTestSupport"], path: "Tests/TableProTLSClientIdentityTests" ), .testTarget( diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLSessionStateClassifier.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLSessionStateClassifier.swift new file mode 100644 index 0000000000..9b4c09ba0d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLSessionStateClassifier.swift @@ -0,0 +1,708 @@ +import Foundation + +public struct SQLSessionStateKinds: OptionSet, Sendable, Hashable { + public let rawValue: Int + + public init(rawValue: Int) { + self.rawValue = rawValue + } + + public static let temporaryObjects = SQLSessionStateKinds(rawValue: 1 << 0) + public static let sessionSettings = SQLSessionStateKinds(rawValue: 1 << 1) + public static let userVariables = SQLSessionStateKinds(rawValue: 1 << 2) + public static let preparedStatements = SQLSessionStateKinds(rawValue: 1 << 3) + public static let heldCursors = SQLSessionStateKinds(rawValue: 1 << 4) + public static let sessionLocks = SQLSessionStateKinds(rawValue: 1 << 5) + public static let listeners = SQLSessionStateKinds(rawValue: 1 << 6) + + public static let all: SQLSessionStateKinds = [ + .temporaryObjects, .sessionSettings, .userVariables, .preparedStatements, .heldCursors, .sessionLocks, + .listeners, + ] +} + +public struct SQLSessionStateEffect: Sendable, Equatable { + public let creates: SQLSessionStateKinds + public let clears: SQLSessionStateKinds + + public init(creates: SQLSessionStateKinds = [], clears: SQLSessionStateKinds = []) { + self.creates = creates + self.clears = clears + } + + public static let none = SQLSessionStateEffect() + + public var isEmpty: Bool { + creates.isEmpty && clears.isEmpty + } + + public func followed(by next: SQLSessionStateEffect) -> SQLSessionStateEffect { + SQLSessionStateEffect( + creates: creates.subtracting(next.clears).union(next.creates), + clears: clears.union(next.clears) + ) + } + + public func applied(to kinds: SQLSessionStateKinds) -> SQLSessionStateKinds { + kinds.subtracting(clears).union(creates) + } +} + +public enum SQLSessionDialect: Sendable, Hashable { + case postgres + case mySQL + case sqlServer + case oracle + case sqlite + case duckDB +} + +// No engine reports a session's temporary tables, settings or variables, and a dead connection answers nothing, so +// the text decides. A `CALL`, `DO`, `EXEC proc` or PL/SQL block is not read: what it sets up is out of sight. +public enum SQLSessionStateClassifier: Sendable { + // Past this a text is data, an INSERT's VALUES, and reading it all would cost a pass per run. Each text is cut before + // it is split, so a setup past this point in one long text, such as a T-SQL batch with no `;`, is not seen. + private static let classifiedPrefixLength = 65_536 + private static let modifierLookahead = 5 + + private static let period = UInt16(UnicodeScalar(".").value) + private static let atSign = UInt16(UnicodeScalar("@").value) + private static let colon = UInt16(UnicodeScalar(":").value) + private static let equals = UInt16(UnicodeScalar("=").value) + + // What `skipping` names runs again on every new connection, such as a startup command, so it is never lost. + public static func effect( + ofStatements statements: [String], + dialect: SQLSessionDialect, + grammar: SQLLexicalGrammar, + skipping skipped: [String] = [] + ) -> SQLSessionStateEffect { + let skippedTexts = Set(skipped.map(comparableText)) + var effect = SQLSessionStateEffect.none + for text in statements { + let head = classifiedHead(of: text) + guard SQLSessionStateQuickCheck.mayChangeSession(head, dialect: dialect) else { continue } + for statement in SQLStatementClassifier.statements(of: head, grammar: grammar) + where SQLSessionStateQuickCheck.mayChangeSession(statement, dialect: dialect) + && !skippedTexts.contains(comparableText(statement)) { + effect = effect.followed(by: statementEffect(of: statement, dialect: dialect, grammar: grammar)) + } + } + return effect + } + + public static func effect(of sql: String, dialect: SQLSessionDialect, grammar: SQLLexicalGrammar) -> SQLSessionStateEffect { + effect(ofStatements: [sql], dialect: dialect, grammar: grammar) + } + + static func comparableText(_ statement: String) -> String { + var text = statement.trimmingCharacters(in: .whitespacesAndNewlines) + while text.hasSuffix(";") { + text = String(text.dropLast()).trimmingCharacters(in: .whitespacesAndNewlines) + } + return text + } + + // Every server reading of a `/*!` comment counts what it sets up, and only what all of them clear is cleared. + private static func statementEffect( + of statement: String, + dialect: SQLSessionDialect, + grammar: SQLLexicalGrammar + ) -> SQLSessionStateEffect { + // More versions than a reader takes one by one leaves the text as written, its executable comments unread. + let texts = SQLExecutableCommentReadings.texts(of: statement, grammar: grammar).map { Array($0) } ?? [statement] + var creates: SQLSessionStateKinds = [] + var clears: SQLSessionStateKinds? + for text in texts { + let reading = SQLStatementClassifier.runnableStatements(of: text, grammar: grammar) + .reduce(SQLSessionStateEffect.none) { partial, piece in + partial.followed(by: pieceEffect(of: piece, dialect: dialect, grammar: grammar)) + } + creates.formUnion(reading.creates) + clears = clears.map { $0.intersection(reading.clears) } ?? reading.clears + } + return SQLSessionStateEffect(creates: creates, clears: clears ?? []) + } + + private static func classifiedHead(of statement: String) -> String { + let text = statement as NSString + guard text.length > classifiedPrefixLength else { return statement } + let cut = text.rangeOfComposedCharacterSequence(at: classifiedPrefixLength).location + return text.substring(to: cut) + } + + private static func pieceEffect( + of piece: String, + dialect: SQLSessionDialect, + grammar: SQLLexicalGrammar + ) -> SQLSessionStateEffect { + guard !SQLProceduralBlock.runs(piece, grammar: grammar) else { return .none } + let code = StatementCode(piece, grammar: grammar) + guard let start = code.tokens.firstIndex(where: { $0.kind != .open }), code.tokens[start].word != nil else { + return .none + } + switch dialect { + case .postgres: + return postgresEffect(code, start: start) + case .mySQL: + return SQLSessionStateEffect(creates: mySQLKinds(code, start: start)) + case .sqlServer: + return SQLSessionStateEffect(creates: sqlServerKinds(code, start: start)) + case .oracle: + return SQLSessionStateEffect(creates: oracleKinds(code, start: start)) + case .sqlite: + return SQLSessionStateEffect(creates: sqliteKinds(code, start: start)) + case .duckDB: + return SQLSessionStateEffect(creates: duckDBKinds(code, start: start)) + } + } +} + +// MARK: - PostgreSQL + +extension SQLSessionStateClassifier { + private static let postgresObjectKinds: Set = ["TABLE", "VIEW", "SEQUENCE", "FUNCTION", "PROCEDURE", "TYPE"] + + // `SET LOCAL` lasts until the transaction ends, `SET TRANSACTION` covers one, and `SET CONSTRAINTS` the current one. + private static let postgresTransactionScopedSets: Set = ["LOCAL", "TRANSACTION", "CONSTRAINTS"] + + // KingbaseES ships each `pg_` function under a `sys_` name too. The `_xact_` forms end with the transaction. + private static let postgresSessionLockFunctions: Set = [ + "PG_ADVISORY_LOCK", "PG_ADVISORY_LOCK_SHARED", "PG_TRY_ADVISORY_LOCK", "PG_TRY_ADVISORY_LOCK_SHARED", + "SYS_ADVISORY_LOCK", "SYS_ADVISORY_LOCK_SHARED", "SYS_TRY_ADVISORY_LOCK", "SYS_TRY_ADVISORY_LOCK_SHARED", + ] + + private static func postgresEffect(_ code: StatementCode, start: Int) -> SQLSessionStateEffect { + let keyword = code.word(at: start) ?? "" + let next = code.word(at: start + 1) + // A prepared statement runs what it holds only when executed. + var kinds: SQLSessionStateKinds = keyword == "PREPARE" ? [] : postgresCallKinds(code) + switch keyword { + case "CREATE": + let temporary = createsTemporary(code, after: start, kinds: postgresObjectKinds) + || createsInSchema(code, after: start, kinds: postgresObjectKinds) { $0.hasPrefix("PG_TEMP") } + // `ON COMMIT DROP` ends the table with the transaction. + if temporary, !code.tokens.indices.contains(where: { code.spells(["ON", "COMMIT", "DROP"], at: $0) }) { + kinds.insert(.temporaryObjects) + } + case "SELECT", "WITH": + if selectsIntoPostgresTemporary(code) { + kinds.insert(.temporaryObjects) + } + case "SET": + if let next, !postgresTransactionScopedSets.contains(next) { + kinds.insert(.sessionSettings) + } + case "PREPARE": + if next != nil, next != "TRANSACTION" { + kinds.insert(.preparedStatements) + } + case "DECLARE": + if declaresHeldCursor(code, after: start) { + kinds.insert(.heldCursors) + } + case "LISTEN": + kinds.insert(.listeners) + case "DISCARD": + switch next { + case "ALL": + // `RESET ALL` is no clear: `role` and `session_authorization` are `GUC_NO_RESET_ALL`, and only this + // resets them too. + return SQLSessionStateEffect(creates: kinds, clears: .all) + case "TEMP", "TEMPORARY": + return SQLSessionStateEffect(creates: kinds, clears: .temporaryObjects) + default: + break + } + case "DEALLOCATE": + let target = next == "PREPARE" ? code.word(at: start + 2) : next + if target == "ALL" { + return SQLSessionStateEffect(creates: kinds, clears: .preparedStatements) + } + case "UNLISTEN": + if code.isSymbol(SqlLexer.star, at: start + 1) { + return SQLSessionStateEffect(creates: kinds, clears: .listeners) + } + default: + break + } + return SQLSessionStateEffect(creates: kinds) + } + + private static func postgresCallKinds(_ code: StatementCode) -> SQLSessionStateKinds { + var kinds: SQLSessionStateKinds = [] + for index in code.tokens.indices where code.opensCall(at: index) { + guard let name = code.word(at: index) else { continue } + if postgresSessionLockFunctions.contains(name) { + kinds.insert(.sessionLocks) + } else if name == "SET_CONFIG", !isTransactionLocalSetConfig(code, opening: index + 1) { + kinds.insert(.sessionSettings) + } + } + return kinds + } + + // `set_config(name, value, true)` lasts until the transaction ends. + private static func isTransactionLocalSetConfig(_ code: StatementCode, opening open: Int) -> Bool { + let outside = code.tokens[open].depth + var argument = 0 + var third: [TSQLTokens.Token] = [] + var index = open + 1 + while index < code.tokens.count { + let token = code.tokens[index] + if token.kind == .close, token.depth == outside { break } + if token.kind == .comma, token.depth == outside + 1 { + argument += 1 + } else if argument == 2 { + third.append(token) + } + index += 1 + } + return third.count == 1 && third[0].word == "TRUE" + } + + private static func selectsIntoPostgresTemporary(_ code: StatementCode) -> Bool { + code.tokens.indices.contains { index in + guard code.word(at: index) == "INTO", let target = code.word(at: index + 1) else { return false } + return target == "TEMP" || target == "TEMPORARY" + || (target.hasPrefix("PG_TEMP") && code.isSymbol(period, at: index + 2)) + } + } + + // A cursor outlives its transaction only `WITH HOLD`. + private static func declaresHeldCursor(_ code: StatementCode, after start: Int) -> Bool { + let end = code.tokens[(start + 1)...].firstIndex { $0.word == "FOR" } ?? code.tokens.count + return ((start + 1).. = ["GLOBAL", "PERSIST", "PERSIST_ONLY"] + private static let mySQLSessionScopes: Set = ["SESSION", "LOCAL"] + + private static func mySQLKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + let keyword = code.word(at: start) ?? "" + let next = code.word(at: start + 1) + var kinds: SQLSessionStateKinds = [] + switch keyword { + case "CREATE": + if createsTemporary(code, after: start, kinds: ["TABLE"]) { + kinds.insert(.temporaryObjects) + } + case "SET": + kinds.formUnion(mySQLSetKinds(code, from: start + 1)) + case "PREPARE": + return next == nil ? [] : [.preparedStatements] + case "LOCK": + if next == "TABLE" || next == "TABLES" { + kinds.insert(.sessionLocks) + } + default: + break + } + if code.tokens.indices.contains(where: { code.opensCall(at: $0) && code.word(at: $0) == "GET_LOCK" }) { + kinds.insert(.sessionLocks) + } + if assignsMySQLUserVariable(code) { + kinds.insert(.userVariables) + } + return kinds + } + + // Plain `SET TRANSACTION` covers the next transaction only, `SET PASSWORD` and `SET DEFAULT ROLE` change the + // account, and MariaDB's `SET STATEMENT ... FOR` one statement. + private static func mySQLSetKinds(_ code: StatementCode, from first: Int) -> SQLSessionStateKinds { + guard code.tokens.indices.contains(first) else { return [] } + let next = code.word(at: first) + let after = code.word(at: first + 1) + if after == "TRANSACTION", let next { + return mySQLSessionScopes.contains(next) ? [.sessionSettings] : [] + } + switch next { + case "PASSWORD", "STATEMENT", "TRANSACTION", "RESOURCE": + return [] + case "DEFAULT" where after == "ROLE": + return [] + case "NAMES", "CHARACTER", "CHARSET", "ROLE": + return [.sessionSettings] + default: + break + } + return mySQLAssignmentKinds(code, from: first) + } + + // A scope word sets the scope of the assignments after it that name none, as the manual says. + private static func mySQLAssignmentKinds(_ code: StatementCode, from first: Int) -> SQLSessionStateKinds { + let depth = code.tokens[first].depth + var kinds: SQLSessionStateKinds = [] + var sessionScope = true + var startsAssignment = true + for index in first.. Bool { + code.tokens.indices.contains { index in + guard code.isSymbol(atSign, at: index), !code.isSymbol(atSign, at: index - 1), + !code.isSymbol(atSign, at: index + 1) + else { return false } + if code.word(at: index - 1) == "INTO" { return true } + return code.isSymbol(colon, at: index + 2) && code.isSymbol(equals, at: index + 3) + } + } +} + +// MARK: - SQL Server + +extension SQLSessionStateClassifier { + private static let sqlServerTemporaryKinds: Set = ["TABLE", "PROC", "PROCEDURE"] + + private static func sqlServerKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + let keyword = code.word(at: start) ?? "" + let next = code.word(at: start + 1) + switch keyword { + case "CREATE": + return createsSQLServerTemporary(code, after: start) ? [.temporaryObjects] : [] + case "SELECT", "WITH": + let intoTemporary = code.tokens.indices.contains { index in + code.word(at: index) == "INTO" && namesSQLServerTemporary(code, at: index + 1) + } + return intoTemporary ? [.temporaryObjects] : [] + case "SET": + // A `@variable` dies with its batch. + guard let next, !next.hasPrefix("@") else { return [] } + return [.sessionSettings] + case "EXEC", "EXECUTE": + // `EXECUTE AS` impersonates until a `REVERT`. + return next == "AS" ? [.sessionSettings] : sqlServerProcedureKinds(code, nameAt: start + 1) + case "DECLARE": + return declaresGlobalCursor(code, after: start) ? [.heldCursors] : [] + default: + // A batch that opens with a procedure's name runs it. + return sqlServerProcedureKinds(code, nameAt: start) + } + } + + private static func createsSQLServerTemporary(_ code: StatementCode, after start: Int) -> Bool { + let window = (start + 1).. Bool { + guard code.tokens.indices.contains(index) else { return false } + let token = code.tokens[index] + let first: Int + switch token.kind { + case .word, .name: + first = token.location + case .quoted: + first = token.location + 1 + default: + return false + } + return first < code.text.length && code.text.character(at: first) == SqlLexer.hash + } + + private static func sqlServerProcedureKinds(_ code: StatementCode, nameAt start: Int) -> SQLSessionStateKinds { + var index = start + if code.word(at: index)?.hasPrefix("@") == true, code.isSymbol(equals, at: index + 1) { + index += 2 + } + while code.isSymbol(period, at: index + 1), code.word(at: index + 2) != nil { + index += 2 + } + switch code.word(at: index) { + // An application role holds the session's security context until `sp_unsetapprole` or the end of the session. + case "SP_SET_SESSION_CONTEXT", "SP_SETAPPROLE": + return [.sessionSettings] + case "SP_GETAPPLOCK": + return [.sessionLocks] + default: + return [] + } + } + + // A cursor is global unless declared `LOCAL`; a cursor variable `DECLARE @c CURSOR` dies with its batch. + private static func declaresGlobalCursor(_ code: StatementCode, after start: Int) -> Bool { + guard let name = code.word(at: start + 1), !name.hasPrefix("@") else { return false } + let end = code.tokens[(start + 1)...].firstIndex { $0.word == "FOR" } ?? code.tokens.count + let words = code.tokens[(start + 1).. = ["SET", "ENABLE", "DISABLE", "FORCE"] + private static let sqliteObjectKinds: Set = ["TABLE", "VIEW", "TRIGGER", "INDEX"] + private static let duckDBObjectKinds: Set = ["TABLE", "VIEW", "SEQUENCE", "MACRO", "FUNCTION", "TYPE"] + + // Rows in a global temporary table are not counted: the table itself is permanent. + private static func oracleKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + let next = code.word(at: start + 1) + switch code.word(at: start) { + case "ALTER": + let verb = code.word(at: start + 2) + return next == "SESSION" && verb.map(oracleSessionVerbs.contains) == true ? [.sessionSettings] : [] + case "SET": + return next == "ROLE" ? [.sessionSettings] : [] + case "CREATE": + // A private temporary table goes at commit unless it keeps its definition. + let isPrivate = code.spells(["PRIVATE", "TEMPORARY", "TABLE"], at: start + 1) + let preserved = code.tokens.indices.contains { code.spells(["PRESERVE", "DEFINITION"], at: $0) } + return isPrivate && preserved ? [.temporaryObjects] : [] + default: + return [] + } + } + + private static func sqliteKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + guard code.word(at: start) == "CREATE" else { return [] } + let temporary = createsTemporary(code, after: start, kinds: sqliteObjectKinds) + || createsInSchema(code, after: start, kinds: sqliteObjectKinds) { $0 == "TEMP" } + return temporary ? [.temporaryObjects] : [] + } + + private static func duckDBKinds(_ code: StatementCode, start: Int) -> SQLSessionStateKinds { + let next = code.word(at: start + 1) + switch code.word(at: start) { + case "CREATE": + let temporary = createsTemporary(code, after: start, kinds: duckDBObjectKinds) + || createsInSchema(code, after: start, kinds: duckDBObjectKinds) { $0 == "TEMP" } + return temporary ? [.temporaryObjects] : [] + case "SET": + switch next { + case nil, "GLOBAL", "LOCAL": + return [] + case "VARIABLE": + return [.userVariables] + default: + return [.sessionSettings] + } + default: + return [] + } + } +} + +// MARK: - Shared readings + +extension SQLSessionStateClassifier { + private static let temporaryMarkers: Set = ["TEMP", "TEMPORARY"] + + // `CREATE [OR REPLACE] [GLOBAL | LOCAL] TEMP TABLE`: the marker stands before the kind, a few words in. + private static func createsTemporary(_ code: StatementCode, after start: Int, kinds: Set) -> Bool { + for index in (start + 1).., + schema isTemporarySchema: (String) -> Bool + ) -> Bool { + let window = (start + 1).. String? { + tokens.indices.contains(index) ? tokens[index].word : nil + } + + func opensCall(at index: Int) -> Bool { + tokens.indices.contains(index + 1) && tokens[index].word != nil && tokens[index + 1].kind == .open + } +} + +// Whole words, read without the grammar: a hit inside a comment or a literal only costs the full read, and a data word +// such as `offset`, `block` or `template`, or an email address, is no hit. Every rule above needs one of these. +enum SQLSessionStateQuickCheck: Sendable { + private static let atSign = UInt16(UnicodeScalar("@").value) + private static let colon = UInt16(UnicodeScalar(":").value) + private static let equals = UInt16(UnicodeScalar("=").value) + private static let shortestWord = 3 + private static let longestWord = 32 + + private static let postgresWords: Set = [ + "TEMP", "TEMPORARY", "SET", "SET_CONFIG", "PREPARE", "DECLARE", "LISTEN", "UNLISTEN", "DISCARD", "DEALLOCATE", + "PG_ADVISORY_LOCK", "PG_ADVISORY_LOCK_SHARED", "PG_TRY_ADVISORY_LOCK", "PG_TRY_ADVISORY_LOCK_SHARED", + "SYS_ADVISORY_LOCK", "SYS_ADVISORY_LOCK_SHARED", "SYS_TRY_ADVISORY_LOCK", "SYS_TRY_ADVISORY_LOCK_SHARED", + ] + private static let mySQLWords: Set = ["TEMP", "TEMPORARY", "SET", "PREPARE", "LOCK", "GET_LOCK"] + private static let sqlServerWords: Set = [ + "SET", "EXEC", "EXECUTE", "DECLARE", "SP_SET_SESSION_CONTEXT", "SP_GETAPPLOCK", "SP_SETAPPROLE", + ] + private static let oracleWords: Set = ["SESSION", "ROLE", "PRIVATE"] + private static let sqliteWords: Set = ["TEMP", "TEMPORARY"] + private static let duckDBWords: Set = ["TEMP", "TEMPORARY", "SET"] + + static func mayChangeSession(_ text: String, dialect: SQLSessionDialect) -> Bool { + let units = text as NSString + let length = units.length + let triggers = words(for: dialect) + var lastWord = "" + var lastWordEnd = -1 + var index = 0 + while index < length { + let unit = units.character(at: index) + if isWordUnit(unit) { + let start = index + while index < length, isWordUnit(units.character(at: index)) { + index += 1 + } + let count = index - start + lastWord = count >= shortestWord && count <= longestWord + ? units.substring(with: NSRange(location: start, length: count)).uppercased() + : "" + lastWordEnd = index + if triggers.contains(lastWord) || (dialect == .postgres && lastWord.hasPrefix("PG_TEMP")) { + return true + } + continue + } + if dialect == .mySQL, unit == atSign, + mayAssignMySQLVariable(units, at: index, gluedToInto: lastWord == "INTO" && lastWordEnd == index) { + return true + } + if dialect == .sqlServer, unit == SqlLexer.hash, startsSQLServerName(units, at: index) { + return true + } + index += 1 + } + return false + } + + private static func words(for dialect: SQLSessionDialect) -> Set { + switch dialect { + case .postgres: return postgresWords + case .mySQL: return mySQLWords + case .sqlServer: return sqlServerWords + case .oracle: return oracleWords + case .sqlite: return sqliteWords + case .duckDB: return duckDBWords + } + } + + // ASCII letters and `_`: a digit or `$` ends the run, so `1SET` still holds the `SET` the server reads. + private static func isWordUnit(_ unit: UInt16) -> Bool { + (0x41...0x5A).contains(unit) || (0x61...0x7A).contains(unit) || unit == 0x5F + } + + private static func isNameUnit(_ unit: UInt16) -> Bool { + isWordUnit(unit) || (0x30...0x39).contains(unit) || unit == 0x24 || unit >= 0x80 + } + + // An address glues its `@` to the name before it, as in `ana@example.com`. A variable stands apart, follows + // `INTO`, or is assigned with `:=`. A second `@` makes a system variable, which a statement only reads. + private static func mayAssignMySQLVariable(_ text: NSString, at index: Int, gluedToInto: Bool) -> Bool { + let length = text.length + if index > 0, text.character(at: index - 1) == atSign { return false } + guard index + 1 < length, text.character(at: index + 1) != atSign else { return false } + if gluedToInto { return true } + let previous = index > 0 ? text.character(at: index - 1) : SqlLexer.space + if !isNameUnit(previous), !"._%+-".utf16.contains(previous) { return true } + return assignedWithColonEquals(text, after: index) + } + + private static func assignedWithColonEquals(_ text: NSString, after index: Int) -> Bool { + let length = text.length + var cursor = index + 1 + let opener = text.character(at: cursor) + if opener == SqlLexer.singleQuote || opener == SqlLexer.doubleQuote || opener == SqlLexer.backtick { + cursor += 1 + while cursor < length, text.character(at: cursor) != opener { + cursor += 1 + } + cursor += 1 + } else { + while cursor < length, isNameUnit(text.character(at: cursor)) { + cursor += 1 + } + } + cursor = pastBlanksAndBlockComments(text, from: cursor) + return cursor + 1 < length && text.character(at: cursor) == colon && text.character(at: cursor + 1) == equals + } + + private static func pastBlanksAndBlockComments(_ text: NSString, from start: Int) -> Int { + let length = text.length + var cursor = start + while cursor < length { + let blank = StatementBlank.blankLength(in: text, at: cursor) + if blank > 0 { + cursor += blank + continue + } + guard cursor + 1 < length, text.character(at: cursor) == SqlLexer.slash, + text.character(at: cursor + 1) == SqlLexer.star + else { break } + let close = text.range( + of: "*/", options: [], range: NSRange(location: cursor + 2, length: length - cursor - 2) + ) + guard close.location != NSNotFound else { return length } + cursor = NSMaxRange(close) + } + return cursor + } + + // `#t`, `##t`, `[#t]` and `"#t"`: a `#` that begins a name. One inside a name, `a#b`, is part of it. + private static func startsSQLServerName(_ text: NSString, at index: Int) -> Bool { + let length = text.length + guard index + 1 < length else { return false } + if index > 0 { + let previous = text.character(at: index - 1) + if isNameUnit(previous) || previous == SqlLexer.hash || previous == atSign { return false } + } + let next = text.character(at: index + 1) + return isNameUnit(next) || next == SqlLexer.hash + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/LibPQSystemTrustRule.swift b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/LibPQSystemTrustRule.swift new file mode 100644 index 0000000000..d9b8f9c169 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/LibPQSystemTrustRule.swift @@ -0,0 +1,32 @@ +// +// LibPQSystemTrustRule.swift +// TableProTLSClientIdentity +// + +import Foundation + +// One rule for the driver and for pg_dump and pg_restore, so a session and its backup trust the same roots. +public enum LibPQSystemTrustRule { + // Verify Identity with no CA file, over TCP, with no root file the user named for libpq. + public static func usesSystemTrustAnchors( + verifiesHostname: Bool, + caCertificatePath: String, + host: String, + environment: [String: String], + fileExists: (String) -> Bool + ) -> Bool { + // libpq opens no TLS on a socket, and an empty host is its default socket. + guard verifiesHostname, caCertificatePath.isEmpty, !host.isEmpty, !host.hasPrefix("/") else { return false } + if let named = environment["PGSSLROOTCERT"], !named.isEmpty { return false } + return !fileExists(defaultRootCertificatePath(environment: environment)) + } + + // libpq reads HOME first and the password database only when it is unset or empty. + public static func defaultRootCertificatePath(environment: [String: String]) -> String { + let home = environment["HOME"].flatMap { $0.isEmpty ? nil : $0 } ?? NSHomeDirectory() + return URL(fileURLWithPath: home, isDirectory: true) + .appendingPathComponent(".postgresql", isDirectory: true) + .appendingPathComponent("root.crt") + .path + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/SystemTrustAnchors.swift b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/SystemTrustAnchors.swift new file mode 100644 index 0000000000..ec8d75984d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProTLSClientIdentity/SystemTrustAnchors.swift @@ -0,0 +1,284 @@ +#if os(macOS) +import Foundation +import os +import Security + +public enum SystemTrustDomain: Sendable { + case user + case admin + case system + + var securityDomain: SecTrustSettingsDomain { + switch self { + case .user: .user + case .admin: .admin + case .system: .system + } + } +} + +public enum SystemTrustResult: Sendable, Equatable { + case trustRoot + case trustAsRoot + case deny + case unspecified +} + +public struct SystemTrustSetting: Sendable, Equatable { + public var appliesToSSL: Bool + public var namesApplication: Bool + public var hostname: String? + // Nil when the entry carries no result, which Apple reads as TrustRoot. + public var result: SystemTrustResult? + + public init( + appliesToSSL: Bool = true, + namesApplication: Bool = false, + hostname: String? = nil, + result: SystemTrustResult? = nil + ) { + self.appliesToSSL = appliesToSSL + self.namesApplication = namesApplication + self.hostname = hostname + self.result = result + } +} + +public protocol SystemTrustSource { + // Empty when the domain has no trust settings. + func certificates(in domain: SystemTrustDomain) throws -> [SecCertificate] + // Nil when the domain holds no settings for the certificate. + func settings(for certificate: SecCertificate, in domain: SystemTrustDomain) throws -> [SystemTrustSetting]? + func anchorCertificates() throws -> [SecCertificate] +} + +public enum SystemTrustAnchorsError: Error, Sendable, Equatable { + case settingsUnreadable(OSStatus) + case noTrustedRoots + case exportFailed(OSStatus) + case fileUnwritable +} + +extension SystemTrustAnchorsError: LocalizedError { + public var message: String { + String( + localized: """ + Could not read this Mac's trusted certificates to verify the server. Choose the server's CA \ + certificate under SSL, then connect again. + """ + ) + } + + public var errorDescription: String? { + message + } +} + +// For a TLS library with no trust store of its own, such as the bundled OpenSSL. The roots come from the trust +// settings, never from evaluating each root alone: a SHA-1 self-signed root fails as a leaf and still anchors chains. +public enum SystemTrustAnchors { + public static let shared = SystemTrustAnchorCache(source: SecuritySystemTrustSource(), clock: ContinuousClock()) + + static let logger = Logger(subsystem: "com.TablePro", category: "SystemTrustAnchors") + + // Per-user settings override admin ones, which override the system's. The first domain with a result decides. + public static func trustedRoots(from source: any SystemTrustSource) throws -> [SecCertificate] { + var results: [Data: SystemTrustResult] = [:] + var decided: [(der: Data, certificate: SecCertificate)] = [] + for domain in [SystemTrustDomain.user, .admin, .system] { + let (certificates, fromAnchorList) = try certificates(in: domain, from: source) + for certificate in certificates { + let der = SecCertificateCopyData(certificate) as Data + guard results[der] == nil else { continue } + let result: SystemTrustResult? = fromAnchorList ? .trustRoot : decision(for: certificate, in: domain, from: source) + guard let result else { continue } + results[der] = result + decided.append((der, certificate)) + } + } + return decided.filter { results[$0.der] == .trustRoot || results[$0.der] == .trustAsRoot }.map(\.certificate) + } + + public static func pem(from source: any SystemTrustSource) throws -> Data { + let roots = try trustedRoots(from: source) + guard !roots.isEmpty else { throw SystemTrustAnchorsError.noTrustedRoots } + var exported: CFData? + let status = SecItemExport(roots as CFArray, .formatPEMSequence, [], nil, &exported) + guard status == errSecSuccess, let exported, CFDataGetLength(exported) > 0 else { + logger.error("Exporting the trusted roots failed: \(status, privacy: .public)") + throw SystemTrustAnchorsError.exportFailed(status) + } + return exported as Data + } + + static func effectiveResult(of settings: [SystemTrustSetting]) -> SystemTrustResult? { + guard !settings.isEmpty else { return .trustRoot } + for setting in settings where setting.appliesToSSL && !setting.namesApplication { + let result = setting.result ?? .trustRoot + // A PEM cannot carry a hostname limit, so a scoped trust is left out and a scoped deny covers every host. + if setting.hostname != nil { + if result == .deny { return .deny } + continue + } + if result != .unspecified { return result } + } + return nil + } + + // The system domain is the one that must answer. If it cannot, the anchors macOS ships stand in for it. + private static func certificates( + in domain: SystemTrustDomain, + from source: any SystemTrustSource + ) throws -> (certificates: [SecCertificate], fromAnchorList: Bool) { + do { + let certificates = try source.certificates(in: domain) + if domain == .system, certificates.isEmpty { + logger.error("The system trust settings list no certificates; using the system anchors") + return (try source.anchorCertificates(), true) + } + return (certificates, false) + } catch { + logger.error( + "Reading the \(String(describing: domain), privacy: .public) trust settings failed: \(String(describing: error), privacy: .private)" + ) + guard domain == .system else { throw error } + return (try source.anchorCertificates(), true) + } + } + + // A setting that cannot be read could be a deny, so it counts as one. + private static func decision( + for certificate: SecCertificate, + in domain: SystemTrustDomain, + from source: any SystemTrustSource + ) -> SystemTrustResult? { + do { + guard let settings = try source.settings(for: certificate, in: domain) else { return nil } + return effectiveResult(of: settings) + } catch { + logger.error("A certificate's \(String(describing: domain), privacy: .public) trust settings could not be read") + return .deny + } + } +} + +// The bytes are kept for a minute, so a root just trusted or distrusted in Keychain Access applies on a later connect +// without a relaunch. A failed build is never kept. +public final class SystemTrustAnchorCache: @unchecked Sendable where C.Duration == Duration { + private let source: any SystemTrustSource + private let clock: C + private let lifetime: Duration + private let lock = NSLock() + private var cached: (pem: Data, builtAt: C.Instant)? + + public init(source: any SystemTrustSource, clock: C, lifetime: Duration = .seconds(60)) { + self.source = source + self.clock = clock + self.lifetime = lifetime + } + + public func pem() throws -> Data { + try lock.withLock { + if let cached, cached.builtAt.duration(to: clock.now) < lifetime { + return cached.pem + } + let built = try SystemTrustAnchors.pem(from: source) + cached = (built, clock.now) + return built + } + } + + public func writeFile(in parent: URL = FileManager.default.temporaryDirectory) throws -> SystemTrustAnchorFile { + try SystemTrustAnchorFile.write(pem(), in: parent) + } +} + +// One file per use, in a folder only this user can open, removed when the use ends. +public struct SystemTrustAnchorFile: Sendable { + public let url: URL + let directory: URL + + static func write(_ pem: Data, in parent: URL) throws -> SystemTrustAnchorFile { + let directory = parent.appendingPathComponent("tablepro-trust-\(UUID().uuidString)", isDirectory: true) + let url = directory.appendingPathComponent("roots.pem") + do { + try FileManager.default.createDirectory( + at: directory, + withIntermediateDirectories: false, + attributes: [.posixPermissions: 0o700] + ) + guard FileManager.default.createFile( + atPath: url.path, + contents: pem, + attributes: [.posixPermissions: 0o600] + ) else { + throw SystemTrustAnchorsError.fileUnwritable + } + } catch { + try? FileManager.default.removeItem(at: directory) + SystemTrustAnchors.logger.error("Writing the trusted roots file failed") + throw SystemTrustAnchorsError.fileUnwritable + } + return SystemTrustAnchorFile(url: url, directory: directory) + } + + public func remove() { + try? FileManager.default.removeItem(at: directory) + } +} + +public struct SecuritySystemTrustSource: SystemTrustSource, Sendable { + public init() {} + + public func certificates(in domain: SystemTrustDomain) throws -> [SecCertificate] { + var certificates: CFArray? + let status = SecTrustSettingsCopyCertificates(domain.securityDomain, &certificates) + if status == errSecNoTrustSettings { return [] } + guard status == errSecSuccess else { throw SystemTrustAnchorsError.settingsUnreadable(status) } + return (certificates as? [SecCertificate]) ?? [] + } + + public func settings(for certificate: SecCertificate, in domain: SystemTrustDomain) throws -> [SystemTrustSetting]? { + var settings: CFArray? + let status = SecTrustSettingsCopyTrustSettings(certificate, domain.securityDomain, &settings) + if status == errSecItemNotFound || status == errSecNoTrustSettings { return nil } + guard status == errSecSuccess else { throw SystemTrustAnchorsError.settingsUnreadable(status) } + let entries = (settings as? [[String: Any]]) ?? [] + return entries.map(Self.setting(from:)) + } + + public func anchorCertificates() throws -> [SecCertificate] { + var anchors: CFArray? + let status = SecTrustCopyAnchorCertificates(&anchors) + guard status == errSecSuccess else { throw SystemTrustAnchorsError.settingsUnreadable(status) } + return (anchors as? [SecCertificate]) ?? [] + } + + public static func setting(from entry: [String: Any]) -> SystemTrustSetting { + let hostname = entry[kSecTrustSettingsPolicyString as String] as? String + return SystemTrustSetting( + appliesToSSL: entry[kSecTrustSettingsPolicy as String].map(isSSLPolicy) ?? true, + namesApplication: entry[kSecTrustSettingsApplication as String] != nil, + hostname: hostname?.isEmpty == false ? hostname : nil, + result: (entry[kSecTrustSettingsResult as String] as? NSNumber).map { result(of: $0.uint32Value) } + ) + } + + private static func isSSLPolicy(_ value: Any) -> Bool { + guard CFGetTypeID(value as CFTypeRef) == SecPolicyGetTypeID() else { return false } + let policy = unsafeDowncast(value as AnyObject, to: SecPolicy.self) + let properties = SecPolicyCopyProperties(policy) as? [String: Any] + return properties?[kSecPolicyOid as String] as? String == kSecPolicyAppleSSL as String + } + + // An unknown value could mean distrust, so it counts as a deny. + private static func result(of rawValue: UInt32) -> SystemTrustResult { + switch SecTrustSettingsResult(rawValue: rawValue) { + case .trustRoot?: .trustRoot + case .trustAsRoot?: .trustAsRoot + case .unspecified?: .unspecified + default: .deny + } + } +} +#endif diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLSessionStateClassifierTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLSessionStateClassifierTests.swift new file mode 100644 index 0000000000..20c3f6bb9d --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLSessionStateClassifierTests.swift @@ -0,0 +1,409 @@ +import Foundation +@testable import TableProSQLGrammar +import Testing + +struct SQLSessionStateClassifierTests { + private static func grammar(_ engine: String) -> SQLLexicalGrammar { + SQLLexicalReadings.resolve(databaseTypeId: engine, declared: nil, session: nil).execution + } + + private static let engines: [SQLSessionDialect: String] = [ + .postgres: "PostgreSQL", .mySQL: "MySQL", .sqlServer: "SQL Server", .oracle: "Oracle", .sqlite: "SQLite", + .duckDB: "DuckDB", + ] + + private func effect(_ sql: String, _ dialect: SQLSessionDialect) -> SQLSessionStateEffect { + SQLSessionStateClassifier.effect(of: sql, dialect: dialect, grammar: Self.grammar(Self.engines[dialect] ?? "")) + } + + private func creates(_ sql: String, _ dialect: SQLSessionDialect) -> SQLSessionStateKinds { + effect(sql, dialect).creates + } + + // MARK: - PostgreSQL + + @Test("PostgreSQL temporary objects, by marker, by pg_temp and by SELECT INTO", arguments: [ + "CREATE TEMP TABLE t (id int)", + "create temporary table t (id int)", + "CREATE GLOBAL TEMPORARY TABLE t (id int)", + "CREATE OR REPLACE TEMP VIEW v AS SELECT 1", + "CREATE TEMP SEQUENCE s", + "CREATE TEMP TABLE t (id int) ON COMMIT DELETE ROWS", + "CREATE TABLE pg_temp.t (id int)", + "CREATE TABLE IF NOT EXISTS pg_temp.t (id int)", + "CREATE FUNCTION pg_temp.f() RETURNS int AS $$ SELECT 1 $$ LANGUAGE sql", + "SELECT * INTO TEMP t FROM orders", + "SELECT * INTO TEMPORARY TABLE t FROM orders", + ]) + func postgresTemporaryObjects(sql: String) { + #expect(creates(sql, .postgres) == .temporaryObjects) + } + + @Test("PostgreSQL tables that outlive the session or end with the transaction are not session state", arguments: [ + "CREATE TABLE t (id int)", + "CREATE TABLE temp (id int)", + "CREATE UNLOGGED TABLE t (id int)", + "CREATE TEMP TABLE t (id int) ON COMMIT DROP", + "SELECT * INTO archive FROM orders", + ]) + func postgresLastingTables(sql: String) { + #expect(creates(sql, .postgres).isEmpty) + } + + @Test("PostgreSQL SET, SET SESSION and a session set_config are session settings", arguments: [ + "SET search_path TO app", + "SET SESSION statement_timeout = 0", + "SET ROLE reporting", + "SET SESSION AUTHORIZATION alice", + "SET SESSION CHARACTERISTICS AS TRANSACTION READ ONLY", + "SET app.key = 'secret'", + "SELECT set_config('app.user', '42', false)", + "SELECT pg_catalog.set_config('app.user', '42', false)", + ]) + func postgresSettings(sql: String) { + #expect(creates(sql, .postgres) == .sessionSettings) + } + + @Test("PostgreSQL settings that last one transaction are not session state", arguments: [ + "SET LOCAL search_path TO app", + "SET TRANSACTION ISOLATION LEVEL SERIALIZABLE", + "SET CONSTRAINTS ALL DEFERRED", + "SELECT set_config('app.user', '42', true)", + ]) + func postgresTransactionSettings(sql: String) { + #expect(creates(sql, .postgres).isEmpty) + } + + @Test("PostgreSQL prepared statements, held cursors, session advisory locks and LISTEN") + func postgresOtherKinds() { + #expect(creates("PREPARE q (int) AS SELECT $1", .postgres) == .preparedStatements) + #expect(creates("PREPARE TRANSACTION 'tx1'", .postgres).isEmpty) + #expect(creates("DECLARE c CURSOR WITH HOLD FOR SELECT 1", .postgres) == .heldCursors) + #expect(creates("DECLARE c CURSOR FOR SELECT 1", .postgres).isEmpty) + #expect(creates("DECLARE c CURSOR WITHOUT HOLD FOR SELECT 1", .postgres).isEmpty) + #expect(creates("SELECT pg_advisory_lock(1)", .postgres) == .sessionLocks) + #expect(creates("SELECT pg_try_advisory_lock_shared(1, 2)", .postgres) == .sessionLocks) + #expect(creates("SELECT pg_advisory_xact_lock(1)", .postgres).isEmpty) + #expect(creates("LISTEN jobs", .postgres) == .listeners) + } + + @Test("DISCARD ALL clears every kind; DISCARD TEMP, DEALLOCATE ALL and UNLISTEN * clear their own") + func postgresClears() { + #expect(effect("DISCARD ALL", .postgres).clears == .all) + #expect(effect("DISCARD TEMP", .postgres).clears == .temporaryObjects) + #expect(effect("discard temporary", .postgres).clears == .temporaryObjects) + #expect(effect("DEALLOCATE ALL", .postgres).clears == .preparedStatements) + #expect(effect("DEALLOCATE PREPARE ALL", .postgres).clears == .preparedStatements) + #expect(effect("UNLISTEN *", .postgres).clears == .listeners) + #expect(effect("UNLISTEN jobs", .postgres) == .none) + #expect(effect("RESET search_path", .postgres) == .none) + #expect(effect("RESET ALL", .postgres) == .none) + #expect(effect("DISCARD PLANS", .postgres) == .none) + } + + @Test("RESET ALL keeps the role and the session authorization, so it clears no settings") + func resetAllKeepsTheRole() { + let grammar = Self.grammar("PostgreSQL") + let effect = SQLSessionStateClassifier.effect( + ofStatements: ["SET ROLE app_admin", "RESET ALL"], dialect: .postgres, grammar: grammar + ) + #expect(effect.applied(to: []) == .sessionSettings) + let discarded = SQLSessionStateClassifier.effect( + ofStatements: ["SET ROLE app_admin", "DISCARD ALL"], dialect: .postgres, grammar: grammar + ) + #expect(discarded.applied(to: []).isEmpty) + } + + @Test("Statements apply in order, so what a later one clears goes and what it sets up stays") + func statementsApplyInOrder() { + let grammar = Self.grammar("PostgreSQL") + let cleared = SQLSessionStateClassifier.effect( + ofStatements: ["CREATE TEMP TABLE t (id int)", "DISCARD ALL"], dialect: .postgres, grammar: grammar + ) + #expect(cleared.applied(to: []).isEmpty) + let reset = SQLSessionStateClassifier.effect( + ofStatements: ["DISCARD ALL", "SET search_path TO app"], dialect: .postgres, grammar: grammar + ) + #expect(reset.applied(to: [.temporaryObjects, .listeners]) == .sessionSettings) + let partly = SQLSessionStateClassifier.effect( + ofStatements: ["CREATE TEMP TABLE t (id int); SET search_path TO app", "DISCARD TEMP"], + dialect: .postgres, + grammar: grammar + ) + #expect(partly.applied(to: []) == .sessionSettings) + } + + @Test("Text inside a comment, a literal, a DO body or a procedure call never counts", arguments: [ + "SELECT 'SET search_path TO x'", + "-- SET search_path TO x\nSELECT 1", + "SELECT 1 /* CREATE TEMP TABLE t (id int) */", + "DO $$ BEGIN CREATE TEMP TABLE t (id int); END $$", + "CALL set_up_session()", + "SELECT $$pg_advisory_lock(1)$$", + ]) + func postgresUnseenText(sql: String) { + #expect(effect(sql, .postgres) == .none) + } + + @Test("The effect carries kinds only, never a value the statement held") + func effectHoldsNoText() { + let noted = effect("SET app.key = 'secret'", .postgres) + #expect(noted.creates == .sessionSettings) + #expect(!String(describing: noted).contains("secret")) + #expect(!String(describing: noted).contains("app.key")) + } + + @Test("A statement equal to a startup command is skipped, one that differs still counts") + func startupStatementsAreSkipped() { + let grammar = Self.grammar("PostgreSQL") + let skipped = SQLSessionStateClassifier.effect( + ofStatements: ["SET search_path TO app;", "CREATE TEMP TABLE t (id int)"], + dialect: .postgres, + grammar: grammar, + skipping: [" SET search_path TO app ;\n"] + ) + #expect(skipped.creates == .temporaryObjects) + let differs = SQLSessionStateClassifier.effect( + ofStatements: ["SET search_path TO other"], + dialect: .postgres, + grammar: grammar, + skipping: ["SET search_path TO app"] + ) + #expect(differs.creates == .sessionSettings) + } + + // MARK: - MySQL and MariaDB + + @Test("MySQL temporary tables", arguments: [ + "CREATE TEMPORARY TABLE t (id int)", + "CREATE OR REPLACE TEMPORARY TABLE t (id int)", + "CREATE TEMPORARY TABLE IF NOT EXISTS t SELECT 1", + ]) + func mySQLTemporaryTables(sql: String) { + #expect(creates(sql, .mySQL) == .temporaryObjects) + } + + @Test("MySQL session system variables, NAMES, CHARACTER SET, SESSION TRANSACTION and ROLE", arguments: [ + "SET sql_mode = ''", + "SET autocommit = 0", + "SET SESSION sql_mode = ''", + "SET LOCAL sql_mode = ''", + "SET @@sql_mode = ''", + "SET @@SESSION.sql_mode = ''", + "SET NAMES utf8mb4", + "SET CHARACTER SET utf8mb4", + "SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED", + "SET ROLE reporting", + "SET GLOBAL max_connections = 500, SESSION sql_mode = ''", + "SET GLOBAL max_connections = 500, @@SESSION.sql_mode = ''", + "/*!40101 SET NAMES utf8mb4 */", + ]) + func mySQLSettings(sql: String) { + #expect(creates(sql, .mySQL) == .sessionSettings) + } + + @Test("MySQL global settings, account changes and next-transaction settings are not session state", arguments: [ + "SET GLOBAL max_connections = 500", + "SET GLOBAL max_connections = 500, wait_timeout = 60", + "SET PERSIST max_connections = 500", + "SET PERSIST_ONLY max_connections = 500", + "SET @@GLOBAL.max_connections = 500", + "SET PASSWORD = 'x'", + "SET PASSWORD FOR app = 'x'", + "SET DEFAULT ROLE ALL TO app", + "SET TRANSACTION ISOLATION LEVEL READ COMMITTED", + "SET GLOBAL TRANSACTION READ ONLY", + "SET STATEMENT max_statement_time = 1 FOR SELECT 1", + "CREATE TABLE t (id int)", + ]) + func mySQLLastingChanges(sql: String) { + #expect(creates(sql, .mySQL).isEmpty) + } + + @Test("MySQL user variables are set by SET, := and INTO", arguments: [ + "SET @id = 42", + "SELECT @id := 42", + "SELECT id INTO @id FROM t LIMIT 1", + "SELECT id FROM t LIMIT 1 INTO @id", + "SELECT 1 INTO@id", + "SELECT@id:=1", + "SELECT @id /* note */ := 1", + "/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE */", + ]) + func mySQLUserVariables(sql: String) { + #expect(creates(sql, .mySQL) == .userVariables) + } + + @Test("MySQL prepared statements and session locks, and what only reads") + func mySQLOtherKinds() { + #expect(creates("PREPARE s FROM 'SELECT 1'", .mySQL) == .preparedStatements) + #expect(creates("SELECT GET_LOCK('job', 10)", .mySQL) == .sessionLocks) + #expect(creates("LOCK TABLES t WRITE", .mySQL) == .sessionLocks) + #expect(creates("SET @got = GET_LOCK('job', 10)", .mySQL) == [.userVariables, .sessionLocks]) + #expect(creates("SELECT @id", .mySQL).isEmpty) + #expect(creates("SELECT @@version", .mySQL).isEmpty) + #expect(creates("SELECT 'SET NAMES utf8mb4'", .mySQL).isEmpty) + #expect(creates("# SET NAMES latin1\nSELECT 1", .mySQL).isEmpty) + #expect(creates("CALL set_up_session()", .mySQL).isEmpty) + } + + // MARK: - SQL Server + + @Test("SQL Server # and ## tables, SELECT INTO a # table and a temporary procedure", arguments: [ + "CREATE TABLE #orders (id int)", + "CREATE TABLE ##orders (id int)", + "SELECT * INTO #orders FROM dbo.orders", + "CREATE PROCEDURE #p AS SELECT 1", + "SELECT 1\nCREATE TABLE #t (id int)", + "CREATE TABLE [#orders] (id int)", + "CREATE TABLE \"#orders\" (id int)", + "SELECT * INTO [#orders] FROM dbo.orders", + "CREATE TABLE #tĆ«mp (id int)", + ]) + func sqlServerTemporaryObjects(sql: String) { + #expect(creates(sql, .sqlServer) == .temporaryObjects) + } + + @Test("SQL Server SET options, EXECUTE AS and session context", arguments: [ + "SET NOCOUNT ON", + "SET XACT_ABORT ON", + "SET TRANSACTION ISOLATION LEVEL SNAPSHOT", + "SELECT 1 SET NOCOUNT ON", + "EXECUTE AS USER = 'auditor'", + "EXEC sp_set_session_context 'tenant', 42", + "EXEC sys.sp_set_session_context @key = N'tenant', @value = 42", + "sp_set_session_context 'tenant', 42", + "EXEC sp_setapprole 'reporting', 'secret'", + "EXEC sys.sp_setapprole @rolename = 'reporting', @password = 'secret'", + "sp_setapprole 'reporting', 'secret'", + ]) + func sqlServerSettings(sql: String) { + #expect(creates(sql, .sqlServer) == .sessionSettings) + } + + @Test("SQL Server variables, procedure calls, dynamic SQL and permanent tables are not session state", arguments: [ + "SET @n = 1", + "DECLARE @n int; SET @n = 1", + "UPDATE t SET a = 1", + "EXEC dbo.rebuild", + "EXEC('SET NOCOUNT ON')", + "CREATE TABLE orders (id int)", + "SELECT * INTO archive FROM orders", + "INSERT INTO #orders VALUES (1)", + ]) + func sqlServerNotSessionState(sql: String) { + #expect(creates(sql, .sqlServer).isEmpty) + } + + @Test("SQL Server global cursors and application locks") + func sqlServerCursorsAndLocks() { + #expect(creates("DECLARE c CURSOR FOR SELECT 1", .sqlServer) == .heldCursors) + #expect(creates("DECLARE c CURSOR GLOBAL FOR SELECT 1", .sqlServer) == .heldCursors) + #expect(creates("DECLARE c INSENSITIVE CURSOR FOR SELECT 1", .sqlServer) == .heldCursors) + #expect(creates("DECLARE c CURSOR LOCAL FOR SELECT 1", .sqlServer).isEmpty) + #expect(creates("DECLARE @c CURSOR", .sqlServer).isEmpty) + #expect(creates("EXEC sp_getapplock @Resource = 'job', @LockMode = 'Exclusive', @LockOwner = 'Session'", .sqlServer) + == .sessionLocks) + #expect(creates("EXEC @rc = sp_getapplock 'job', 'Exclusive', 'Session'", .sqlServer) == .sessionLocks) + } + + // MARK: - Oracle, SQLite and DuckDB + + @Test("Oracle ALTER SESSION, SET ROLE and a private temporary table that keeps its definition") + func oracleKinds() { + #expect(creates("ALTER SESSION SET NLS_DATE_FORMAT = 'YYYY-MM-DD'", .oracle) == .sessionSettings) + #expect(creates("ALTER SESSION SET CURRENT_SCHEMA = app", .oracle) == .sessionSettings) + #expect(creates("ALTER SESSION ENABLE PARALLEL DML", .oracle) == .sessionSettings) + #expect(creates("SET ROLE reporting", .oracle) == .sessionSettings) + #expect(creates("SET TRANSACTION READ ONLY", .oracle).isEmpty) + #expect( + creates("CREATE PRIVATE TEMPORARY TABLE ora$ptt_t (id NUMBER) ON COMMIT PRESERVE DEFINITION", .oracle) + == .temporaryObjects + ) + #expect(creates("CREATE PRIVATE TEMPORARY TABLE ora$ptt_t (id NUMBER) ON COMMIT DROP DEFINITION", .oracle).isEmpty) + #expect(creates("CREATE GLOBAL TEMPORARY TABLE t (id NUMBER) ON COMMIT PRESERVE ROWS", .oracle).isEmpty) + #expect(creates("BEGIN DBMS_SESSION.SET_ROLE('reporting'); END;", .oracle).isEmpty) + } + + @Test("SQLite and DuckDB temporary objects, by marker and by the temp schema") + func embeddedTemporaryObjects() { + #expect(creates("CREATE TEMP TABLE t (id INTEGER)", .sqlite) == .temporaryObjects) + #expect(creates("CREATE TEMPORARY VIEW v AS SELECT 1", .sqlite) == .temporaryObjects) + #expect(creates("CREATE TABLE temp.t (id INTEGER)", .sqlite) == .temporaryObjects) + #expect(creates("CREATE TABLE t (id INTEGER)", .sqlite).isEmpty) + #expect(creates("CREATE TABLE temp (id INTEGER)", .sqlite).isEmpty) + #expect(creates("CREATE TEMP TABLE t (id INTEGER)", .duckDB) == .temporaryObjects) + #expect(creates("CREATE OR REPLACE TEMP MACRO m(a) AS a + 1", .duckDB) == .temporaryObjects) + #expect(creates("CREATE TABLE temp.t (id INTEGER)", .duckDB) == .temporaryObjects) + } + + @Test("DuckDB SET without GLOBAL is a session setting, SET VARIABLE a variable") + func duckDBSettings() { + #expect(creates("SET threads = 4", .duckDB) == .sessionSettings) + #expect(creates("SET SESSION threads = 4", .duckDB) == .sessionSettings) + #expect(creates("SET VARIABLE x = 42", .duckDB) == .userVariables) + #expect(creates("SET GLOBAL threads = 4", .duckDB).isEmpty) + } + + // MARK: - Effects + + @Test("A later clear removes what an earlier statement set up, a later setup survives an earlier clear") + func effectComposition() { + let setUp = SQLSessionStateEffect(creates: .temporaryObjects) + let discard = SQLSessionStateEffect(clears: .all) + #expect(setUp.followed(by: discard) == SQLSessionStateEffect(creates: [], clears: .all)) + let settings = SQLSessionStateEffect(creates: .sessionSettings) + #expect(discard.followed(by: settings).applied(to: .userVariables) == .sessionSettings) + } + + @Test("A statement longer than the classified head is read only to there") + func longStatementIsReadToItsHead() { + let values = String(repeating: "a", count: 100_000) + #expect(creates("INSERT INTO t VALUES ('\(values)')", .mySQL).isEmpty) + #expect(creates("SET @id = 42, @blob = '\(values)'", .mySQL) == .userVariables) + } + + @Test("A text is cut to its head before it is split, so a setup past it in the same text is not read") + func textIsCutBeforeItIsSplit() { + let values = String(repeating: "a", count: 70_000) + #expect(creates("SET NOCOUNT ON\nINSERT INTO t VALUES ('\(values)')", .sqlServer) == .sessionSettings) + #expect(creates("INSERT INTO t VALUES ('\(values)'); SET NOCOUNT ON", .sqlServer).isEmpty) + let grammar = Self.grammar("SQL Server") + let separate = SQLSessionStateClassifier.effect( + ofStatements: ["INSERT INTO t VALUES ('\(values)')", "SET NOCOUNT ON"], dialect: .sqlServer, grammar: grammar + ) + #expect(separate.creates == .sessionSettings) + } + + @Test("The quick check passes over data words and addresses", arguments: [ + ("SELECT * FROM t ORDER BY id LIMIT 10 OFFSET 20", SQLSessionDialect.postgres), + ("INSERT INTO t VALUES ('template', 'asset', 'block', 'clock')", .postgres), + ("INSERT INTO users VALUES ('ana@example.com', 'bo.li@example.org')", .mySQL), + ("INSERT INTO t VALUES ('reset', 'offset', 'blocked')", .mySQL), + ("INSERT INTO t VALUES ('Order#5', 'a#b')", .sqlServer), + ("SELECT * FROM sessions WHERE role_id = 1", .oracle), + ("SELECT pg_advisory_xact_lock(1)", .postgres), + ]) + func quickCheckSkipsData(sql: String, dialect: SQLSessionDialect) { + #expect(!SQLSessionStateQuickCheck.mayChangeSession(sql, dialect: dialect)) + } + + @Test("The quick check keeps every form a rule reads, however it is spelled", arguments: [ + ("set search_path to app", SQLSessionDialect.postgres), + ("SELECT * INTO pg_temp_3.t FROM orders", .postgres), + ("SELECT 1SET", .postgres), + ("SELECT 1 INTO@v", .mySQL), + ("SELECT 1 INTO /* v */ @v", .mySQL), + ("SELECT@v/* note */:=1", .mySQL), + ("SELECT @`v` := 1", .mySQL), + ("/*!40101 SET NAMES utf8mb4 */", .mySQL), + ("CREATE TABLE [#t] (id int)", .sqlServer), + ("SELECT * INTO ##t FROM orders", .sqlServer), + ("sp_setapprole 'reporting', 'secret'", .sqlServer), + ("ALTER SESSION SET CURRENT_SCHEMA = app", .oracle), + ("CREATE TABLE temp.t (id INTEGER)", .sqlite), + ]) + func quickCheckKeepsEveryRule(sql: String, dialect: SQLSessionDialect) { + #expect(SQLSessionStateQuickCheck.mayChangeSession(sql, dialect: dialect)) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/SystemTrustAnchorsTests.swift b/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/SystemTrustAnchorsTests.swift new file mode 100644 index 0000000000..47418aa408 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTLSClientIdentityTests/SystemTrustAnchorsTests.swift @@ -0,0 +1,288 @@ +#if os(macOS) +import Foundation +import Security +import TableProTestSupport +import TableProTLSClientIdentity +import TableProTLSTestFixtures +import Testing + +private final class FixtureTrustSource: SystemTrustSource { + struct Entry { + let certificate: SecCertificate + let settings: [SystemTrustSetting]? + var unreadable = false + } + + var domains: [SystemTrustDomain: [Entry]] = [:] + var unreadableDomains: Set = [] + var anchors: [SecCertificate] = [] + private(set) var userDomainReads = 0 + + func add(_ certificate: SecCertificate, to domain: SystemTrustDomain, _ settings: [SystemTrustSetting]?) { + domains[domain, default: []].append(Entry(certificate: certificate, settings: settings)) + } + + func certificates(in domain: SystemTrustDomain) throws -> [SecCertificate] { + if domain == .user { userDomainReads += 1 } + guard !unreadableDomains.contains(domain) else { throw SystemTrustAnchorsError.settingsUnreadable(-1) } + return domains[domain, default: []].map(\.certificate) + } + + func settings(for certificate: SecCertificate, in domain: SystemTrustDomain) throws -> [SystemTrustSetting]? { + let der = SecCertificateCopyData(certificate) as Data + guard let entry = domains[domain, default: []].first(where: { SecCertificateCopyData($0.certificate) as Data == der }) + else { return nil } + guard !entry.unreadable else { throw SystemTrustAnchorsError.settingsUnreadable(-1) } + return entry.settings + } + + func anchorCertificates() throws -> [SecCertificate] { + anchors + } +} + +struct SystemTrustAnchorsTests { + private let rootA: SecCertificate + private let rootB: SecCertificate + + init() throws { + rootA = try #require(SecCertificateCreateWithData(nil, TLSTestFixtures.der(TLSTestFixtures.caCertificate) as CFData)) + rootB = try #require( + SecCertificateCreateWithData(nil, TLSTestFixtures.der(TLSTestFixtures.serverNameCACertificate) as CFData) + ) + } + + private func ders(_ certificates: [SecCertificate]) -> Set { + Set(certificates.map { SecCertificateCopyData($0) as Data }) + } + + private func exported(_ source: FixtureTrustSource) throws -> Set { + ders(try SystemTrustAnchors.trustedRoots(from: source)) + } + + @Test("A certificate whose trust settings are empty is trusted as a root") + func emptySettingsTrustTheRoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + + #expect(try exported(source) == ders([rootA])) + } + + @Test("A user Deny removes a root the system trusts") + func aUserDenyRemovesASystemRoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootB, to: .system, []) + source.add(rootA, to: .user, [SystemTrustSetting(result: .deny)]) + + #expect(try exported(source) == ders([rootB])) + } + + @Test("A root trusted in the admin domain joins the system roots") + func anAdminTrustAddsARoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootB, to: .admin, [SystemTrustSetting(result: .trustRoot)]) + + #expect(try exported(source) == ders([rootA, rootB])) + } + + @Test("User settings override admin settings in both directions") + func userSettingsOverrideAdminSettings() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .admin, [SystemTrustSetting(result: .deny)]) + source.add(rootA, to: .user, [SystemTrustSetting(result: .trustRoot)]) + source.add(rootB, to: .admin, [SystemTrustSetting(result: .trustRoot)]) + source.add(rootB, to: .user, [SystemTrustSetting(result: .deny)]) + + #expect(try exported(source) == ders([rootA])) + } + + @Test("A setting for another policy or for one application does not apply") + func anEntryForAnotherPolicyDoesNotApply() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootA, to: .user, [SystemTrustSetting(appliesToSSL: false, result: .deny)]) + source.add(rootB, to: .user, [SystemTrustSetting(namesApplication: true, result: .trustRoot)]) + + #expect(try exported(source) == ders([rootA])) + } + + @Test("A trust limited to one hostname is left out, because a PEM file cannot carry the limit") + func aHostnameScopedTrustIsNotExported() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootB, to: .user, [SystemTrustSetting(hostname: "db.example.com", result: .trustRoot)]) + + #expect(try exported(source) == ders([rootA])) + } + + @Test("A Deny limited to one hostname removes the root for every host") + func aHostnameScopedDenyRemovesTheRoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootA, to: .user, [ + SystemTrustSetting(hostname: "db.example.com", result: .deny), + SystemTrustSetting(result: .trustRoot) + ]) + + #expect(try exported(source).isEmpty) + } + + @Test("Unspecified moves on to the next entry, then to the next domain") + func unspecifiedFallsThroughToTheNextDomain() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootA, to: .user, [SystemTrustSetting(result: .unspecified)]) + source.add(rootB, to: .user, [SystemTrustSetting(result: .unspecified)]) + + #expect(try exported(source) == ders([rootA])) + + source.add(rootB, to: .admin, [SystemTrustSetting(result: .unspecified), SystemTrustSetting()]) + #expect(try exported(source) == ders([rootA, rootB])) + } + + @Test("A settings array the domain cannot read counts as a Deny") + func unreadableSettingsFailClosed() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.domains[.user] = [FixtureTrustSource.Entry(certificate: rootA, settings: [], unreadable: true)] + + #expect(try exported(source).isEmpty) + } + + @Test("A user or admin domain that cannot be read fails the build") + func anUnreadableUserDomainIsAnError() { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.unreadableDomains = [.admin] + + #expect(throws: SystemTrustAnchorsError.settingsUnreadable(-1)) { + try SystemTrustAnchors.trustedRoots(from: source) + } + } + + @Test("The system anchors stand in when the system domain cannot be read, and a user Deny still wins") + func theSystemAnchorsAreTheFallback() throws { + let source = FixtureTrustSource() + source.unreadableDomains = [.system] + source.anchors = [rootA, rootB] + source.add(rootB, to: .user, [SystemTrustSetting(result: .deny)]) + + #expect(try exported(source) == ders([rootA])) + + source.unreadableDomains = [] + #expect(try exported(source) == ders([rootA])) + } + + @Test("No trusted root is an error, never an empty file") + func noAnchorsIsAnError() { + let source = FixtureTrustSource() + source.add(rootA, to: .system, [SystemTrustSetting(result: .deny)]) + + #expect(throws: SystemTrustAnchorsError.noTrustedRoots) { + try SystemTrustAnchors.pem(from: source) + } + } + + @Test("The PEM holds one block per trusted root") + func thePEMHoldsEveryRoot() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + source.add(rootB, to: .system, []) + + let pem = try #require(String(bytes: SystemTrustAnchors.pem(from: source), encoding: .utf8)) + + #expect(pem.components(separatedBy: "-----BEGIN CERTIFICATE-----").count == 3) + } + + @Test("This Mac's own trust settings export as PEM") + func theRealStoreExportsPEM() throws { + let source = SecuritySystemTrustSource() + + let roots = try SystemTrustAnchors.trustedRoots(from: source) + let pem = try #require(String(bytes: SystemTrustAnchors.pem(from: source), encoding: .utf8)) + + #expect(!roots.isEmpty) + #expect(pem.hasPrefix("-----BEGIN CERTIFICATE-----")) + } + + @Test("A trust settings entry is read for its policy, hostname, application and result") + func readsATrustSettingsEntry() { + let scoped = SecuritySystemTrustSource.setting(from: [ + kSecTrustSettingsPolicy as String: SecPolicyCreateSSL(true, nil), + kSecTrustSettingsPolicyString as String: "db.example.com", + kSecTrustSettingsResult as String: NSNumber(value: 3) + ]) + #expect(scoped == SystemTrustSetting(hostname: "db.example.com", result: .deny)) + + let otherPolicy = SecuritySystemTrustSource.setting(from: [kSecTrustSettingsPolicy as String: SecPolicyCreateBasicX509()]) + #expect(otherPolicy == SystemTrustSetting(appliesToSSL: false)) + + let application = SecuritySystemTrustSource.setting(from: [kSecTrustSettingsApplication as String: "app"]) + #expect(application == SystemTrustSetting(namesApplication: true)) + } + + @Test("A result value Apple does not define counts as a Deny", arguments: [0, 9]) + func anUnknownResultCountsAsADeny(value: Int) { + let setting = SecuritySystemTrustSource.setting(from: [kSecTrustSettingsResult as String: NSNumber(value: value)]) + #expect(setting.result == .deny) + } + + @Test("The bytes are reused for a minute, then rebuilt, and a failed build is not kept") + func theCacheRebuildsAfterAMinute() throws { + let source = FixtureTrustSource() + let clock = ManualClock() + let cache = SystemTrustAnchorCache(source: source, clock: clock) + + #expect(throws: SystemTrustAnchorsError.noTrustedRoots) { try cache.pem() } + source.add(rootA, to: .system, []) + let first = try cache.pem() + #expect(source.userDomainReads == 2) + + clock.advance(by: .seconds(59)) + #expect(try cache.pem() == first) + #expect(source.userDomainReads == 2) + + clock.advance(by: .seconds(1)) + _ = try cache.pem() + #expect(source.userDomainReads == 3) + } + + @Test("Each use gets its own owner-only file in an owner-only folder, removed with its folder") + func aWrittenFileIsOwnerOnly() throws { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + let cache = SystemTrustAnchorCache(source: source, clock: ManualClock()) + let parent = FileManager.default.temporaryDirectory.appendingPathComponent("trust-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: parent, withIntermediateDirectories: false) + defer { try? FileManager.default.removeItem(at: parent) } + + let file = try cache.writeFile(in: parent) + let other = try cache.writeFile(in: parent) + let folder = file.url.deletingLastPathComponent() + + #expect(file.url != other.url) + #expect(try Data(contentsOf: file.url) == cache.pem()) + let folderMode = try FileManager.default.attributesOfItem(atPath: folder.path)[.posixPermissions] as? NSNumber + let fileMode = try FileManager.default.attributesOfItem(atPath: file.url.path)[.posixPermissions] as? NSNumber + #expect(folderMode?.int16Value == 0o700) + #expect(fileMode?.int16Value == 0o600) + + file.remove() + #expect(!FileManager.default.fileExists(atPath: folder.path)) + #expect(FileManager.default.fileExists(atPath: other.url.path)) + } + + @Test("A file that cannot be written is an error and leaves nothing behind") + func anUnwritableFolderIsAnError() { + let source = FixtureTrustSource() + source.add(rootA, to: .system, []) + let cache = SystemTrustAnchorCache(source: source, clock: ManualClock()) + let missing = FileManager.default.temporaryDirectory.appendingPathComponent("missing-\(UUID().uuidString)") + + #expect(throws: SystemTrustAnchorsError.fileUnwritable) { try cache.writeFile(in: missing) } + #expect(!FileManager.default.fileExists(atPath: missing.path)) + } +} +#endif diff --git a/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift b/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift index 42d318e2c1..7a5792e4a7 100644 --- a/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift +++ b/apple/Plugins/FormatPluginTests/JSONDocumentStreamingTests.swift @@ -63,6 +63,39 @@ struct JSONDocumentStreamingTests { return try JSONImportParsing.parseElements(at: url, targetTable: targetTable) } + // The rows each element becomes, nil for one that is not an object. + private func rows(_ elements: [Any]) -> [[String: PluginCellValue]?] { + elements.map { try? JSONImportParsing.row(from: $0) } + } + + private func importedRows(_ data: Data, chunkSize: Int = JSONDocumentReader.defaultChunkSize) throws -> [[String: PluginCellValue]] { + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + var reader = try JSONDocumentReader(url: url, chunkSize: chunkSize, checkCancellation: {}) + let layout = try JSONDocumentOutline.rows(in: &reader, targetTable: nil) { reader, _ in try reader.skipValue() } + switch layout { + case .array(let offset, _, _): + return try drain(JSONStreamedElements(reader: reader, arrayAt: offset)).map { try JSONImportParsing.row(from: $0) } + case .singleObject: + reader.close() + return try drain(JSONImportParsing.rowElements(at: url, targetTable: nil, checkCancellation: {}).elements) + .map { try JSONImportParsing.row(from: $0) } + } + } + + private static let encodings: [String.Encoding] = [.utf16BigEndian, .utf16LittleEndian, .utf32BigEndian, .utf32LittleEndian] + + // Foundation writes no byte order mark for an encoding that names its byte order. + private static func encoded(_ text: String, _ encoding: String.Encoding, withMark: Bool) -> Data { + let mark: [UInt8] = switch encoding { + case .utf16BigEndian: [0xFE, 0xFF] + case .utf16LittleEndian: [0xFF, 0xFE] + case .utf32BigEndian: [0x00, 0x00, 0xFE, 0xFF] + default: [0xFF, 0xFE, 0x00, 0x00] + } + return (withMark ? Data(mark) : Data()) + (text.data(using: encoding) ?? Data()) + } + @Test("Elements read one at a time equal a whole-file parse, whatever the chunk size", arguments: [1, 3, 64, 1 << 20]) func streamedElementsMatchAWholeFileParse(chunkSize: Int) throws { let array = "[\n " + Self.trickyRows.joined(separator: ",\n ") + "\n]" @@ -122,9 +155,15 @@ struct JSONDocumentStreamingTests { (#"[{"a":1},]"#, nil), (#"{"t":[{"a":1} , ],}"#, nil), ] as [(String, String?)]) + // A lone object streams as a built row, so it is compared as rows; an array's elements are compared as values. func shapesMatchAWholeFileParse(json: String, targetTable: String?) throws { let imported = try importedElements(json, targetTable: targetTable) - #expect((imported as NSArray).isEqual(to: try parsedElements(json, targetTable: targetTable))) + let parsed = try parsedElements(json, targetTable: targetTable) + if imported.contains(where: { $0 is [String: PluginCellValue] }) { + #expect(rows(imported) == rows(parsed)) + } else { + #expect((imported as NSArray).isEqual(to: parsed)) + } } // A whole-file parse hands the tables over as a dictionary, whose order changes from launch to launch. @@ -157,7 +196,7 @@ struct JSONDocumentStreamingTests { } } - @Test("A UTF-16 file still reads, through JSONSerialization's own decoding") + @Test("A UTF-16 file with a byte order mark streams its rows and fields") func utf16FileStillReads() throws { let url = try writeJSON(try #require(#"{"t":[{"name":"Ada"},{"name":"Grace"}]}"#.data(using: .utf16))) defer { try? FileManager.default.removeItem(at: url) } @@ -167,6 +206,87 @@ struct JSONDocumentStreamingTests { #expect(try JSONImportParsing.detectFields(at: url, targetTable: nil).map(\.name) == ["name"]) } + // `JSONSerialization` reads UTF-32LE with a byte order mark as UTF-16LE and fails, so the oracle + // is the parse of the same text without its mark, which it reads in every encoding. + @Test("UTF-16 and UTF-32 text streams to the rows a whole-file parse gives, with or without a mark", arguments: [1, 3, 7, 1 << 20]) + func decodedTextMatchesAWholeFileParse(chunkSize: Int) throws { + let json = "[\n " + Self.trickyRows.joined(separator: ",\n ") + #",{"emoji":"šŸ˜€ š„ž a"}]"# + for encoding in Self.encodings { + let url = try writeJSON(Self.encoded(json, encoding, withMark: false)) + defer { try? FileManager.default.removeItem(at: url) } + let expected = rows(try JSONImportParsing.parseElements(at: url, targetTable: nil)) + #expect(expected.count == Self.trickyRows.count + 1) + for withMark in [true, false] { + let data = Self.encoded(json, encoding, withMark: withMark) + #expect(try importedRows(data, chunkSize: chunkSize).map(Optional.some) == expected) + } + } + } + + @Test("A decoded file holds one chunk and one element, not the file") + func decodedReaderHoldsOneChunkAndOneElement() throws { + let rows = (1...2_000).map { #"{"id":\#($0),"name":"HĆ  Nį»™i \#($0)"}"# } + let data = Self.encoded("[" + rows.joined(separator: ",") + "]", .utf16LittleEndian, withMark: true) + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + let chunkSize = 128 + var reader = try JSONDocumentReader(url: url, chunkSize: chunkSize) + let layout = try JSONDocumentOutline.rows(in: &reader, targetTable: nil) { reader, _ in try reader.skipValue() } + guard case .array(let offset, let count, _) = layout else { + Issue.record("Expected the rows of an array, got \(layout)") + return + } + var elements = try JSONStreamedElements(reader: reader, arrayAt: offset) + defer { elements.close() } + var mostHeld = 0 + while try elements.next() != nil { + mostHeld = max(mostHeld, elements.heldByteCount) + } + let longestElement = rows.map(\.utf8.count).max() ?? 0 + #expect(count == 2_000) + #expect(mostHeld <= chunkSize * 2 + longestElement) + #expect(data.count > 100 * (chunkSize * 2 + longestElement)) + } + + @Test("An error in decoded text points at the byte of the file", arguments: [String.Encoding.utf16LittleEndian, .utf32BigEndian]) + func decodedErrorNamesTheFileByte(encoding: String.Encoding) throws { + let text = #"[{"a":"Ć©šŸ˜€"} x]"# + let data = Self.encoded(text, encoding, withMark: true) + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + let mark = encoding == .utf16LittleEndian ? 2 : 4 + let unitsBefore = Array(#"[{"a":"Ć©šŸ˜€"} "#.utf16).count + let expected = encoding == .utf16LittleEndian ? mark + unitsBefore * 2 : mark + (#"[{"a":"Ć©šŸ˜€"} "#.unicodeScalars.count) * 4 + #expect(throws: JSONDocumentError.unexpectedCharacter(byteOffset: expected)) { + _ = try JSONImportParsing.rowElements(at: url, targetTable: nil, checkCancellation: {}) + } + } + + @Test("Text that is not valid UTF-16 fails instead of importing", arguments: [ + Data([0xFF, 0xFE, 0x5B, 0x00, 0x00, 0xD8, 0x5D, 0x00]), + Data([0xFF, 0xFE, 0x5B, 0x00, 0x5D]), + ]) + func invalidDecodedTextFails(data: Data) throws { + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + #expect(throws: (any Error).self) { + _ = try JSONImportParsing.rowElements(at: url, targetTable: nil, checkCancellation: {}) + } + } + + @Test("A lone object is one row, read member by member, keeping the first of two members with one name") + func singleObjectStreamsItsMembers() throws { + let tags = "[" + (1...3_000).map { #""tag \#($0)""# }.joined(separator: ",") + "]" + let json = #"{"id":7,"tags":"# + tags + #","id":8,"meta":{"k":null},"ok":true}"# + let data = Data(json.utf8) + let streamed = try importedRows(data, chunkSize: 64) + let url = try writeJSON(data) + defer { try? FileManager.default.removeItem(at: url) } + #expect(streamed.map(Optional.some) == rows(try JSONImportParsing.parseElements(at: url, targetTable: nil))) + #expect(streamed.first?["id"] == .text("7")) + #expect(try JSONImportParsing.detectFields(at: url, targetTable: nil).map(\.name) == ["id", "meta", "ok", "tags"]) + } + @Test("A file that starts with a UTF-8 byte order mark streams") func byteOrderMarkIsSkipped() throws { let url = try writeJSON(Data([0xEF, 0xBB, 0xBF]) + Data(#"[{"a":1}]"#.utf8)) diff --git a/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift b/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift index 6d562e9811..4aa0e25e01 100644 --- a/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift +++ b/apple/Plugins/JSONImportPlugin/JSONDocumentOutline.swift @@ -9,8 +9,7 @@ import TableProPluginKit enum JSONDocumentRows: Equatable { // The rows are the elements of the array that opens at `offset`. `slot` is the one `visit` was given for them. case array(offset: Int, elementCount: Int, slot: Int) - // A single object, which is one row, or text in UTF-16 or UTF-32: `JSONSerialization` reads the file whole. - case wholeDocument + case singleObject(offset: Int) } // Finds which array of a `.json` file holds the rows, by the rules `JSONImportParsing.extractElements` @@ -32,7 +31,7 @@ enum JSONDocumentOutline { targetTable: String?, visit: (inout JSONDocumentReader, _ slot: Int?) throws -> Void ) throws -> JSONDocumentRows { - guard try reader.startDocument() else { return .wholeDocument } + try reader.startDocument() guard let first = try reader.nextSignificantByte() else { throw JSONDocumentError.endsEarly } switch first { case JSONDocumentReader.openBracket: @@ -41,11 +40,12 @@ enum JSONDocumentOutline { try reader.expectEnd() return .array(offset: offset, elementCount: array.count, slot: 0) case JSONDocumentReader.openBrace: - return try memberRows(in: &reader, targetTable: targetTable, visit: visit) + let offset = reader.offset + return try memberRows(in: &reader, at: offset, targetTable: targetTable, visit: visit) case 0x22, 0x2D, 0x30...0x39, 0x66, 0x6E, 0x74: throw JSONImportParsing.unexpectedShapeError default: - throw JSONDocumentError.unexpectedCharacter(byteOffset: reader.offset) + throw JSONDocumentError.unexpectedCharacter(byteOffset: reader.fileOffset) } } @@ -69,6 +69,7 @@ enum JSONDocumentOutline { private static func memberRows( in reader: inout JSONDocumentReader, + at objectOffset: Int, targetTable: String?, visit: (inout JSONDocumentReader, _ slot: Int?) throws -> Void ) throws -> JSONDocumentRows { @@ -96,7 +97,7 @@ enum JSONDocumentOutline { } try reader.expectEnd() - guard !members.isEmpty, members.allSatisfy({ $0.holdsRows }) else { return .wholeDocument } + guard !members.isEmpty, members.allSatisfy({ $0.holdsRows }) else { return .singleObject(offset: objectOffset) } let match = targetTable.flatMap { target in members.first { $0.name.caseInsensitiveCompare(target) == .orderedSame } ?? members.first { $0.name.split(separator: ".").last?.caseInsensitiveCompare(target) == .orderedSame } diff --git a/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift b/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift index ea927d568d..8bace6ac1c 100644 --- a/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift +++ b/apple/Plugins/JSONImportPlugin/JSONDocumentReader.swift @@ -8,6 +8,7 @@ import Foundation enum JSONDocumentError: LocalizedError, Equatable { case endsEarly case unexpectedCharacter(byteOffset: Int) + case invalidText var errorDescription: String? { switch self { @@ -15,13 +16,17 @@ enum JSONDocumentError: LocalizedError, Equatable { return String(localized: "The JSON file ends before its last value is complete.") case .unexpectedCharacter(let byteOffset): return String(format: String(localized: "The JSON file is not valid at byte %@."), (byteOffset + 1).formatted()) + case .invalidText: + return String(localized: "The JSON file has text that is not valid in its encoding.") } } } -// Walks a UTF-8 JSON file one chunk at a time. It finds where each value starts and ends without +// Walks a JSON file one chunk at a time. It finds where each value starts and ends without // building it, so the file is held one chunk and one value at a time. `JSONSerialization` parses // each value a caller asks for, so every value reads exactly as it would in a whole-file parse. +// UTF-16 and UTF-32 text is turned into UTF-8 as it is read, so `offset` counts UTF-8 bytes and +// `fileOffset` the file's own. struct JSONDocumentReader { static let defaultChunkSize = 1 << 20 @@ -42,6 +47,11 @@ struct JSONDocumentReader { private var bufferStart = 0 private var valueStart: Int? private var reachedEnd = false + private var decoder: JSONTextDecoder? + // Where the text starts in the file, past any byte order mark, and how many of its code units + // the buffer has already dropped. Only a decoded file uses them. + private var textStart = 0 + private var droppedUnits = 0 init( url: URL, @@ -55,34 +65,54 @@ struct JSONDocumentReader { var offset: Int { bufferStart + position } + var fileOffset: Int { fileOffset(atBufferIndex: position) } + var heldByteCount: Int { buffer.count } func close() { try? handle.close() } - // Steps over a UTF-8 byte order mark. False for UTF-16 or UTF-32 text, which `JSONSerialization` - // also reads: it has a byte order mark of its own or a zero byte in its first two. - mutating func startDocument() throws -> Bool { + // Steps over a byte order mark and, for UTF-16 or UTF-32 text, decodes what was read so far. + mutating func startDocument() throws { while buffer.count - position < 4, try refill() {} - let head = buffer[position...].prefix(4) - if head.starts(with: [0xEF, 0xBB, 0xBF]) { - position += 3 - return true - } - if head.starts(with: [0xFE, 0xFF]) || head.starts(with: [0xFF, 0xFE]) { - return false + let detected = JSONTextEncoding.detect(buffer[position...]) + guard detected.encoding != .utf8 else { + position += detected.markLength + return } - return !head.prefix(2).contains(0) + var decoder = JSONTextDecoder(encoding: detected.encoding) + textStart = bufferStart + position + detected.markLength + let raw = Data(buffer[(position + detected.markLength)...]) + buffer = try decoder.utf8(from: raw) + if reachedEnd { try decoder.finish() } + self.decoder = decoder + bufferStart = 0 + position = 0 + droppedUnits = 0 } + // `offset` is one `self.offset` gave. Decoded text has no file position for it, so the text is + // decoded again from its start up to there. mutating func seek(to offset: Int) throws { - try handle.seek(toOffset: UInt64(offset)) buffer.removeAll(keepingCapacity: true) - bufferStart = offset position = 0 valueStart = nil reachedEnd = false + guard decoder != nil else { + try handle.seek(toOffset: UInt64(offset)) + bufferStart = offset + return + } + try handle.seek(toOffset: UInt64(textStart)) + decoder?.reset() + bufferStart = 0 + droppedUnits = 0 + while bufferStart + buffer.count < offset { + position = buffer.count + guard try refill() else { throw JSONDocumentError.endsEarly } + } + position = offset - bufferStart } // The next byte that is not JSON whitespace, left unread, or nil at the end of the file. @@ -120,11 +150,10 @@ struct JSONDocumentReader { mutating func memberName() throws -> String { guard let byte = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } - guard byte == Self.quote else { throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) } - let start = offset + guard byte == Self.quote else { throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) } let name = try readValue() guard let decoded = try JSONSerialization.jsonObject(with: name, options: .fragmentsAllowed) as? String else { - throw JSONDocumentError.unexpectedCharacter(byteOffset: start) + throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset(atBufferIndex: position - name.count)) } try consume(Self.colon) return decoded @@ -132,7 +161,7 @@ struct JSONDocumentReader { mutating func expectEnd() throws { guard try nextSignificantByte() == nil else { - throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) + throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) } } @@ -150,7 +179,7 @@ struct JSONDocumentReader { private mutating func continueAfterValue(closing: UInt8) throws -> Bool { guard let byte = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } guard byte == Self.comma || byte == closing else { - throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) + throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) } position += 1 if byte == closing { return false } @@ -161,7 +190,7 @@ struct JSONDocumentReader { private mutating func consume(_ expected: UInt8) throws { guard let byte = try nextSignificantByte() else { throw JSONDocumentError.endsEarly } - guard byte == expected else { throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) } + guard byte == expected else { throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) } position += 1 } @@ -191,7 +220,7 @@ struct JSONDocumentReader { case 0x2D, 0x30...0x39, 0x66, 0x6E, 0x74: try passLiteral() default: - throw JSONDocumentError.unexpectedCharacter(byteOffset: offset) + throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset) } } @@ -262,7 +291,7 @@ struct JSONDocumentReader { return false } position = index - if mismatched { throw JSONDocumentError.unexpectedCharacter(byteOffset: offset - 1) } + if mismatched { throw JSONDocumentError.unexpectedCharacter(byteOffset: fileOffset(atBufferIndex: position - 1)) } if closed { return } guard try refill() else { throw JSONDocumentError.endsEarly } } @@ -292,17 +321,47 @@ struct JSONDocumentReader { try checkCancellation() let keep = valueStart ?? position if keep > 0 { + if decoder != nil { + droppedUnits += codeUnits(in: buffer[0.. Int { + guard let decoder else { return bufferStart + index } + let units = droppedUnits + codeUnits(in: buffer[0..) -> Int { + let countsPairs = decoder?.encoding.unitWidth == 2 + var units = 0 + for byte in bytes where byte & 0xC0 != 0x80 { + units += countsPairs && byte >= 0xF0 ? 2 : 1 } - buffer.append(contentsOf: chunk) - return true + return units } } diff --git a/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift b/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift index 9835b41ddf..d11b26c8b3 100644 --- a/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift +++ b/apple/Plugins/JSONImportPlugin/JSONElementBatches.swift @@ -13,7 +13,7 @@ protocol JSONElementSequence { func close() } -// The elements of one array of a UTF-8 file, each read and parsed as it is reached. +// The elements of one array of the file, each read and parsed as it is reached. struct JSONStreamedElements: JSONElementSequence { private var reader: JSONDocumentReader private var hasMore: Bool @@ -39,19 +39,17 @@ struct JSONStreamedElements: JSONElementSequence { } } -// The elements of a file `JSONSerialization` has already read whole. -struct JSONParsedElements: JSONElementSequence { - private let elements: [Any] - private var cursor = 0 +// A file whose top-level object is the one row, built member by member. +struct JSONSingleRow: JSONElementSequence { + private var row: [String: PluginCellValue]? - init(_ elements: [Any]) { - self.elements = elements + init(_ row: [String: PluginCellValue]) { + self.row = row } mutating func next() -> Any? { - guard cursor < elements.count else { return nil } - defer { cursor += 1 } - return elements[cursor] + defer { row = nil } + return row } func close() {} diff --git a/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift b/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift index 636afd94f9..f4c5acf4a8 100644 --- a/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift +++ b/apple/Plugins/JSONImportPlugin/JSONFieldSurvey.swift @@ -15,10 +15,14 @@ struct JSONFieldSurvey { mutating func add(_ row: NSDictionary) { row.enumerateKeysAndObjects { key, value, _ in guard let name = key as? String else { return } - profiles[name, default: JSONFieldProfile()].add(value) + add(value, named: name) } } + mutating func add(_ value: Any, named name: String) { + profiles[name, default: JSONFieldProfile()].add(value) + } + var fields: [PluginImportField] { profiles.keys.sorted().map { name in let profile = profiles[name] ?? JSONFieldProfile() diff --git a/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift b/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift index 453890598d..e92faf3392 100644 --- a/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift +++ b/apple/Plugins/JSONImportPlugin/JSONImportParsing.swift @@ -37,9 +37,9 @@ enum JSONImportParsing { .importFailed("The file contains multiple tables and none matches the target table") } - // The values of a `.json` file that become rows, and how many there are. A UTF-8 file is read - // one element at a time after a pass that finds its rows and checks its structure, so a broken - // file fails before the first row is imported. + // The values of a `.json` file that become rows, and how many there are. The file is read one + // element at a time after a pass that finds its rows and checks its structure, so a broken file + // fails before the first row is imported. static func rowElements( at url: URL, targetTable: String?, @@ -50,29 +50,53 @@ enum JSONImportParsing { let rows = try JSONDocumentOutline.rows(in: &reader, targetTable: targetTable) { reader, _ in try reader.skipValue() } - if case .array(let offset, let count, _) = rows { + switch rows { + case .array(let offset, let count, _): return (try JSONStreamedElements(reader: reader, arrayAt: offset), count) + case .singleObject(let offset): + defer { reader.close() } + var row: [String: PluginCellValue] = [:] + try members(ofObjectAt: offset, in: &reader) { name, value in + row[name] = cellValue(from: value) + } + return (JSONSingleRow(row), 1) } } catch { reader.close() throw error } - reader.close() - let elements = try parseElements(at: url, targetTable: targetTable) - return (JSONParsedElements(elements), elements.count) + } + + // Each member of the object at `offset`, parsed one at a time. A parsed object keeps the first + // of two members that share a name, so the later one is passed over. + static func members( + ofObjectAt offset: Int, + in reader: inout JSONDocumentReader, + _ body: (String, Any) throws -> Void + ) throws { + try reader.seek(to: offset) + guard try reader.enterObject() else { return } + var names: Set = [] + repeat { + let name = try reader.memberName() + let value = try reader.readValue() + guard names.insert(name).inserted else { continue } + try autoreleasepool { + try body(name, try JSONSerialization.jsonObject(with: value, options: .fragmentsAllowed)) + } + } while try reader.continueObject() } static func row(from element: Any) throws -> [String: PluginCellValue] { + if let row = element as? [String: PluginCellValue] { + return row + } guard let object = element as? NSDictionary else { throw PluginImportError.importFailed(String(localized: "Each array element must be a JSON object")) } return convertRow(object) } - static func parseRows(at url: URL, targetTable: String?) throws -> [NSDictionary] { - try parseElements(at: url, targetTable: targetTable).compactMap { $0 as? NSDictionary } - } - static func parseElements(at url: URL, targetTable: String?) throws -> [Any] { let data = try Data(contentsOf: url) let object = try JSONSerialization.jsonObject(with: data) @@ -181,8 +205,12 @@ enum JSONImportParsing { switch rows { case .array(_, _, let slot): return surveys[slot]?.fields ?? [] - case .wholeDocument: - return try detectFields(in: try parseRows(at: url, targetTable: targetTable)) + case .singleObject(let offset): + var survey = JSONFieldSurvey() + try members(ofObjectAt: offset, in: &reader) { name, value in + survey.add(value, named: name) + } + return survey.fields } } diff --git a/apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift b/apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift new file mode 100644 index 0000000000..6d937f2826 --- /dev/null +++ b/apple/Plugins/JSONImportPlugin/JSONTextDecoder.swift @@ -0,0 +1,109 @@ +// +// JSONTextDecoder.swift +// JSONImportPlugin +// + +import Foundation + +// The five encodings `JSONSerialization` reads, with or without a byte order mark. +enum JSONTextEncoding: Equatable { + case utf8 + case utf16BigEndian + case utf16LittleEndian + case utf32BigEndian + case utf32LittleEndian + + var unitWidth: Int { + switch self { + case .utf8: return 1 + case .utf16BigEndian, .utf16LittleEndian: return 2 + case .utf32BigEndian, .utf32LittleEndian: return 4 + } + } + + // RFC 4627, section 3: the first two characters are ASCII, so the zero bytes in the first four + // name the encoding when no byte order mark does. + static func detect(_ head: some Collection) -> (encoding: JSONTextEncoding, markLength: Int) { + let bytes = Array(head.prefix(4)) + if bytes.starts(with: [0xEF, 0xBB, 0xBF]) { return (.utf8, 3) } + if bytes.starts(with: [0x00, 0x00, 0xFE, 0xFF]) { return (.utf32BigEndian, 4) } + if bytes.starts(with: [0xFF, 0xFE, 0x00, 0x00]) { return (.utf32LittleEndian, 4) } + if bytes.starts(with: [0xFE, 0xFF]) { return (.utf16BigEndian, 2) } + if bytes.starts(with: [0xFF, 0xFE]) { return (.utf16LittleEndian, 2) } + guard bytes.count >= 2 else { return (.utf8, 0) } + if bytes.count >= 4 { + if bytes[0] == 0, bytes[1] == 0, bytes[2] == 0 { return (.utf32BigEndian, 0) } + if bytes[1] == 0, bytes[2] == 0, bytes[3] == 0 { return (.utf32LittleEndian, 0) } + } + if bytes[0] == 0 { return (.utf16BigEndian, 0) } + if bytes[1] == 0 { return (.utf16LittleEndian, 0) } + return (.utf8, 0) + } +} + +// Turns UTF-16 or UTF-32 text into UTF-8 a chunk at a time. A code unit, or a surrogate pair, that a +// chunk cuts in two waits for the next chunk. +struct JSONTextDecoder { + let encoding: JSONTextEncoding + private var pending: [UInt8] = [] + + init(encoding: JSONTextEncoding) { + self.encoding = encoding + } + + mutating func reset() { + pending.removeAll(keepingCapacity: true) + } + + mutating func utf8(from chunk: Data) throws -> [UInt8] { + guard encoding != .utf8 else { return Array(chunk) } + pending.append(contentsOf: chunk) + var usable = pending.count - pending.count % encoding.unitWidth + var output: [UInt8] = [] + output.reserveCapacity(usable) + let failed: Bool + switch encoding { + case .utf8: + return Array(chunk) + case .utf16BigEndian, .utf16LittleEndian: + var units: [UInt16] = [] + units.reserveCapacity(usable / 2) + var index = 0 + while index < usable { + let first = UInt16(pending[index]) + let second = UInt16(pending[index + 1]) + units.append(encoding == .utf16BigEndian ? first << 8 | second : second << 8 | first) + index += 2 + } + if let last = units.last, (0xD800...0xDBFF).contains(last) { + units.removeLast() + usable -= 2 + } + failed = transcode(units.makeIterator(), from: UTF16.self, to: UTF8.self, stoppingOnError: true) { + output.append($0) + } + case .utf32BigEndian, .utf32LittleEndian: + var units: [UInt32] = [] + units.reserveCapacity(usable / 4) + var index = 0 + while index < usable { + let bytes = pending[index..<(index + 4)].map(UInt32.init) + units.append(encoding == .utf32BigEndian + ? bytes[0] << 24 | bytes[1] << 16 | bytes[2] << 8 | bytes[3] + : bytes[3] << 24 | bytes[2] << 16 | bytes[1] << 8 | bytes[0]) + index += 4 + } + failed = transcode(units.makeIterator(), from: UTF32.self, to: UTF8.self, stoppingOnError: true) { + output.append($0) + } + } + if failed { throw JSONDocumentError.invalidText } + pending.removeSubrange(0.., insertedRowIndices: Set ) throws -> [PluginRowWrite] { + // In the order received, which is edit order: a row deleted to free a unique value goes before the row taking it. var writes: [PluginRowWrite] = [] - var deletes: [PluginRowWrite] = [] for change in changes { switch change.type { case .insert: @@ -47,10 +47,10 @@ struct MSSQLStatementGenerator { case .delete: guard deletedRowIndices.contains(change.rowIndex), let statement = try deleteStatement(for: change) else { continue } - deletes.append(PluginRowWrite(statement: statement.statement, parameters: statement.parameters, rowIndices: [change.rowIndex])) + writes.append(PluginRowWrite(statement: statement.statement, parameters: statement.parameters, rowIndices: [change.rowIndex])) } } - return writes + deletes + return writes } /// The statements alone, for a host that asks without a context. With no context there is nothing to refuse. diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQConnectionString.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQConnectionString.swift index 04bb56d2ea..7cd8bc1d45 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQConnectionString.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQConnectionString.swift @@ -6,6 +6,7 @@ import Foundation import TableProPluginKit import TableProSQLQuoting +import TableProTLSClientIdentity internal struct LibPQConnectTimeout: Equatable, Sendable { static let defaultMilliseconds = 10_000 @@ -46,9 +47,8 @@ internal struct LibPQConnectTimeout: Equatable, Sendable { } } -/// One monotonic wall-clock budget for libpq's TCP/TLS/authentication and the bootstrap queries -/// that make the session usable. Keeping the absolute instant here prevents every catalog probe -/// from receiving the full timeout again. +// One absolute instant for the TCP, TLS and sign-in steps and the bootstrap queries, so no catalog probe gets the full +// timeout again. internal struct LibPQConnectDeadline: Equatable, Sendable { private let expiresAtMicroseconds: Int64 @@ -67,6 +67,13 @@ internal struct LibPQConnectDeadline: Equatable, Sendable { } } +// Where libpq takes the roots it checks the server's certificate against. +internal enum LibPQRootCertificateSource: Equatable, Sendable { + case caFile(String) + case libpqDefault + case systemTrustAnchors +} + internal enum LibPQConnectionString { static let clientEncoding = "UTF8" @@ -75,11 +82,10 @@ internal enum LibPQConnectionString { static let sessionApplicationName = "TablePro" static let metadataApplicationName = "TablePro Metadata" - /// Left to the kernel, macOS sends the first keepalive after 7,200 s of idle, while an AWS - /// Network Load Balancer drops an idle flow after 350 s and an Azure Load Balancer silently after - /// 4 minutes. These keep a parked session's flow open and let the kernel declare a dead peer - /// about 90 s after the last traffic. `tcp_user_timeout` is left out: macOS has no - /// `TCP_USER_TIMEOUT`, so libpq accepts it and changes nothing. + // Left to the kernel, macOS sends the first keepalive after 7,200 s of idle, while an AWS Network Load Balancer drops + // an idle flow after 350 s and an Azure Load Balancer silently after 4 minutes. These keep a parked session's flow + // open and find a dead peer about 90 s after the last traffic. No `tcp_user_timeout`: macOS has no + // `TCP_USER_TIMEOUT`, so libpq accepts it and changes nothing. static let keepaliveParameters: [(String, String)] = [ ("keepalives", "1"), ("keepalives_idle", "60"), @@ -98,7 +104,8 @@ internal enum LibPQConnectionString { applicationName: String? = nil, connectTimeoutSeconds: Int? = nil, kerberos: LibPQKerberosSignIn? = nil, - serverName: String? = nil + serverName: String? = nil, + systemTrustAnchorsPath: String? = nil ) throws -> String { var parameters: [(String, String)] = [] if let serverName, !serverName.isEmpty, serverName != host, isAddressLiteral(host) { @@ -142,6 +149,8 @@ internal enum LibPQConnectionString { } if sslConfig.verifiesCertificate, !sslConfig.caCertificatePath.isEmpty { parameters.append(("sslrootcert", sslConfig.caCertificatePath)) + } else if sslConfig.verifiesHostname, let systemTrustAnchorsPath { + parameters.append(("sslrootcert", systemTrustAnchorsPath)) } if !sslConfig.clientCertificatePath.isEmpty { parameters.append(("sslcert", sslConfig.clientCertificatePath)) @@ -181,6 +190,29 @@ internal enum LibPQConnectionString { // Nothing can exist under a device file, so libpq finds no file there and says nothing. static let unreadablePassfile = "/dev/null/pgpass" + // Nil when the mode checks no certificate. Verify CA never takes this Mac's roots: it would accept any server with a + // public certificate, which libpq refuses for sslrootcert=system. A `~/.postgresql/root.crt` keeps libpq's default. + static func rootCertificateSource( + for sslConfig: SSLConfiguration, + host: String, + environment: [String: String] = ProcessInfo.processInfo.environment, + fileExists: (String) -> Bool = { FileManager.default.fileExists(atPath: $0) } + ) -> LibPQRootCertificateSource? { + guard sslConfig.verifiesCertificate else { return nil } + guard sslConfig.caCertificatePath.isEmpty else { return .caFile(sslConfig.caCertificatePath) } + return LibPQSystemTrustRule.usesSystemTrustAnchors( + verifiesHostname: sslConfig.verifiesHostname, + caCertificatePath: sslConfig.caCertificatePath, + host: host, + environment: environment, + fileExists: fileExists + ) ? .systemTrustAnchors : .libpqDefault + } + + static func defaultRootCertificatePath(environment: [String: String]) -> String { + LibPQSystemTrustRule.defaultRootCertificatePath(environment: environment) + } + private static func isAddressLiteral(_ host: String) -> Bool { var ipv4 = in_addr() var ipv6 = in6_addr() @@ -191,16 +223,15 @@ internal enum LibPQConnectionString { purpose == "metadata" ? metadataApplicationName : sessionApplicationName } - /// libpq applies `fallback_application_name` over an `application_name` set in `options`, so a - /// name the user already gave the session there has to keep the fallback out entirely. + // libpq applies `fallback_application_name` over an `application_name` set in `options`, so a name the user gave the + // session there keeps the fallback out. static func namesApplication(_ options: String?) -> Bool { guard let options else { return false } return settingNames(in: options).contains("application_name") } - /// The setting each `-c name=value`, `-cname=value` and `--name=value` in `options` assigns, spelled - /// the way the server reads a name: in any case, with `-` and `_` as the same character. Only the - /// names, because a value such as `search_path=application_name` assigns something else. + // Spelled the way the server reads a name: any case, `-` and `_` alike. Only the names, because a value such as + // `search_path=application_name` assigns something else. static func settingNames(in options: String) -> [String] { let tokens = options.split(whereSeparator: \.isWhitespace).map(String.init) var names: [String] = [] diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift index 20b22f439c..a57fb8a048 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift @@ -15,6 +15,7 @@ import TableProCoreTypes import TableProKerberos import TableProPluginKit import TableProPostgreSQLCore +import TableProTLSClientIdentity private final class LibPQCancelRequest: @unchecked Sendable { private let cancelObject: OpaquePointer @@ -32,11 +33,11 @@ private final class LibPQCancelRequest: @unchecked Sendable { // MARK: - Connection Class -/// libpq allows one thread at a time on a `PGconn`, so every libpq call on `conn` runs on `queue`, -/// and a member that other files can reach and that makes such a call opens with -/// `preconditionOnQueue()`. `stateLock` guards `conn`, `serverMessages` and the underscored state, -/// is never held across a libpq call, and is taken in this file only. Cancelling a running query -/// is the one path that calls libpq off the queue. +// libpq allows one thread at a time on a `PGconn`, so every libpq call on `conn` runs on `queue`, +// and a member that other files can reach and that makes such a call opens with +// `preconditionOnQueue()`. `stateLock` guards `conn`, `serverMessages` and the underscored state, +// is never held across a libpq call, and is taken in this file only. Cancelling a running query +// is the one path that calls libpq off the queue. final class LibPQPluginConnection: @unchecked Sendable { static let logger = Logger(subsystem: "com.TablePro.PostgreSQLDriver", category: "LibPQPluginConnection") @@ -77,7 +78,7 @@ final class LibPQPluginConnection: @unchecked Sendable { private var _interruptsBlockedCallOnDisconnect = false private var serverMessages: Unmanaged? private var _standardConformingStrings = true - /// Queue-confined. Non-nil until every schema/flavor/catalog bootstrap probe has completed. + // Queue-confined. Non-nil until every schema/flavor/catalog bootstrap probe has completed. private var activeConnectDeadline: LibPQConnectDeadline? private var bootstrapTransportFailed = false // Queue-confined. Nil until a probe, sent while no block was open, showed whether the server takes savepoints: @@ -204,7 +205,10 @@ final class LibPQPluginConnection: @unchecked Sendable { KerberosTicket.discard(cachePath) } } - let conninfo = try connectionString() + // libpq reads the roots file while it polls, and every reconnect writes a new one. + let trustAnchors = try systemTrustAnchorsFile() + defer { trustAnchors?.remove() } + let conninfo = try connectionString(systemTrustAnchorsPath: trustAnchors?.url.path) let connection = try signingIn(until: deadline) { () throws -> OpaquePointer in guard let connection = conninfo.withCString({ PQconnectStart($0) }) else { throw LibPQPluginError.connectionFailed @@ -242,6 +246,18 @@ final class LibPQPluginConnection: @unchecked Sendable { adopted = true } + // A failure fails the connect: without sslrootcert libpq would look for a file that is not there. + private func systemTrustAnchorsFile() throws -> SystemTrustAnchorFile? { + guard LibPQConnectionString.rootCertificateSource(for: sslConfig, host: host) == .systemTrustAnchors else { + return nil + } + do { + return try SystemTrustAnchors.shared.writeFile() + } catch { + throw LibPQPluginError(message: error.localizedDescription, sqlState: nil, detail: nil) + } + } + // PQconnectStart can already acquire the GSS credential, so the whole start sits in the scope. private func signingIn(until deadline: LibPQConnectDeadline, _ body: () throws -> T) throws -> T { guard let kerberos else { return try body() } @@ -257,8 +273,8 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// The core calls this only after schema and flavor-specific catalog probes. Until then every - /// statement uses libpq's nonblocking API and the original absolute connect deadline. + // The core calls this only after schema and flavor-specific catalog probes. Until then every + // statement uses libpq's nonblocking API and the original absolute connect deadline. func finishConnecting() async throws { try await pluginDispatchAsync(on: queue) { [self] in try checkConnectCancellation() @@ -325,9 +341,9 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// `PGRES_POLLING_*` only says whether the socket wants a read or a write, so it cannot tell - /// a TLS handshake from an authentication exchange. `PQstatus` can, and reading it costs one - /// pointer dereference per poll slice. + // `PGRES_POLLING_*` only says whether the socket wants a read or a write, so it cannot tell + // a TLS handshake from an authentication exchange. `PQstatus` can, and reading it costs one + // pointer dereference per poll slice. private func reportHandshakeStage( of connection: OpaquePointer, last: inout ConnStatusType?, @@ -354,6 +370,9 @@ final class LibPQPluginConnection: @unchecked Sendable { private func connectionError(from connection: OpaquePointer) -> Error { let error = getError(from: connection) + if let missingRoots = LibPQSSLMapping.missingRootCertificateError(error.message, sslConfig: sslConfig) { + return missingRoots + } if let sslError = LibPQSSLClassifier.classifySSLError(error.message) { return sslError } @@ -449,9 +468,9 @@ final class LibPQPluginConnection: @unchecked Sendable { return LibPQStringConformance.isOn(String(cString: value)) } - /// Executes a bootstrap statement without ever entering libpq's blocking `PQexec` path. A - /// statement can produce more than one result, so this mirrors `PQexec` by returning the last - /// one while still polling every byte under the same connect deadline. + // Executes a bootstrap statement without ever entering libpq's blocking `PQexec` path. A + // statement can produce more than one result, so this mirrors `PQexec` by returning the last + // one while still polling every byte under the same connect deadline. private func bootstrapResult( for statement: String, on connection: OpaquePointer, @@ -582,7 +601,7 @@ final class LibPQPluginConnection: @unchecked Sendable { Self.logger.warning("standard_conforming_strings is off; string literals escape backslashes") } - private func connectionString() throws -> String { + private func connectionString(systemTrustAnchorsPath: String?) throws -> String { try LibPQConnectionString.build( host: host, port: port, @@ -594,7 +613,8 @@ final class LibPQPluginConnection: @unchecked Sendable { applicationName: applicationName, connectTimeoutSeconds: connectTimeout.nativeSeconds, kerberos: kerberos, - serverName: serverName + serverName: serverName, + systemTrustAnchorsPath: systemTrustAnchorsPath ) } @@ -700,20 +720,18 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// Runs a read whose statement changes session settings with `SET LOCAL`, and confines those - /// settings to the read whether or not the session has a transaction block open. - /// - /// With no block open, the settings and the read are one implicit transaction that ends with it. - /// Inside a block they would outlast the read, so a savepoint scopes them and is rolled back - /// straight after. The status check and every statement run in one block on this connection's - /// queue. Split into separate dispatches, another caller sharing the session could run between - /// the savepoint and its rollback, and the rollback silently undid that caller's work. PGlite has - /// no pooled connection, so a query tab and the metadata reads share one session. - /// - /// The `ROLLBACK TO` is its own `PQexec`, because `PQexec` returns only a string's last result - /// and the rows are the read's. A rollback that fails after a read that succeeded is reported - /// rather than the rows: the session would otherwise keep the read's settings for the rest of - /// the user's transaction. + // Runs a read whose statement changes session settings with `SET LOCAL`, and confines those + // settings to the read whether or not the session has a transaction block open. + // With no block open, the settings and the read are one implicit transaction that ends with it. + // Inside a block they would outlast the read, so a savepoint scopes them and is rolled back + // straight after. The status check and every statement run in one block on this connection's + // queue. Split into separate dispatches, another caller sharing the session could run between + // the savepoint and its rollback, and the rollback silently undid that caller's work. PGlite has + // no pooled connection, so a query tab and the metadata reads share one session. + // The `ROLLBACK TO` is its own `PQexec`, because `PQexec` returns only a string's last result + // and the rows are the read's. A rollback that fails after a read that succeeded is reported + // rather than the rows: the session would otherwise keep the read's settings for the rest of + // the user's transaction. func executeTransactionScopedRead(_ statement: String) async throws -> LibPQPluginQueryResult { let statementToRun = String(statement) @@ -739,9 +757,9 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - /// A catalog query the driver sends on its own while reading a result, on the connection's queue, scoped the way - /// `executeTransactionScopedRead` scopes one. The caller clears the result. Inside an aborted block it hands back - /// the savepoint's own `25P02` failure. + // A catalog query the driver sends on its own while reading a result, on the connection's queue, scoped the way + // `executeTransactionScopedRead` scopes one. The caller clears the result. Inside an aborted block it hands back + // the savepoint's own `25P02` failure. func pinnedCatalogSideQueryOnQueue(_ statement: String, conn: OpaquePointer) -> OpaquePointer? { preconditionOnQueue() guard pinsCatalogSearchPath else { return statement.withCString { PQexec(conn, $0) } } @@ -791,11 +809,11 @@ final class LibPQPluginConnection: @unchecked Sendable { private static let scopedReadSavepoint = "SAVEPOINT tablepro_scoped_read" private static let scopedReadRollback = "ROLLBACK TO SAVEPOINT tablepro_scoped_read; RELEASE SAVEPOINT tablepro_scoped_read" - /// Runs one of the driver's own session-setting statements so a failure cannot abort a transaction block the - /// user has open. Inside one, a savepoint holds the statement: rolled back when it fails, released when it - /// succeeds, which keeps what it set in the block. One dispatch, for the reason `executeTransactionScopedRead` - /// gives. In an aborted block the statement goes as it is and the server answers `25P02`. `requiresIdleSession` - /// refuses in the same dispatch, so no transaction can open between the check and the statement. + // Runs one of the driver's own session-setting statements so a failure cannot abort a transaction block the + // user has open. Inside one, a savepoint holds the statement: rolled back when it fails, released when it + // succeeds, which keeps what it set in the block. One dispatch, for the reason `executeTransactionScopedRead` + // gives. In an aborted block the statement goes as it is and the server answers `25P02`. `requiresIdleSession` + // refuses in the same dispatch, so no transaction can open between the check and the statement. func executeSessionSetting( _ statement: String, parameters: [PluginCellValue], @@ -829,19 +847,18 @@ final class LibPQPluginConnection: @unchecked Sendable { private static let sessionSettingRollback = "ROLLBACK TO SAVEPOINT tablepro_session_setting; RELEASE SAVEPOINT tablepro_session_setting" - /// Whether the session is inside a transaction block, including one a failed statement has - /// aborted, so a statement sent now joins it. Called on the connection's queue only: one `PGconn` - /// may not be used from two threads at once, and the lock guards the pointer rather than the call. + // Whether the session is inside a transaction block, including one a failed statement has + // aborted, so a statement sent now joins it. Called on the connection's queue only: one `PGconn` + // may not be used from two threads at once, and the lock guards the pointer rather than the call. private func isInsideTransactionBlockOnQueue() -> Bool { let state = transactionStateOnQueue() return state == .inTransaction || state == .inError } - /// What the session has open, from the `ReadyForQuery` status libpq keeps from the last reply. - /// - /// No round trip and no server work: measured against PostgreSQL 17.11, a million calls took - /// 1.5ms, and the answer is local enough to survive a backend another session terminated - /// (`PQtransactionStatus` still reported `INTRANS` with `PQstatus` OK). + // What the session has open, from the `ReadyForQuery` status libpq keeps from the last reply. + // No round trip and no server work: measured against PostgreSQL 17.11, a million calls took + // 1.5ms, and the answer is local enough to survive a backend another session terminated + // (`PQtransactionStatus` still reported `INTRANS` with `PQstatus` OK). func transactionState() async -> LibPQTransactionState { do { return try await pluginDispatchAsync(on: queue) { [self] in @@ -858,10 +875,10 @@ final class LibPQPluginConnection: @unchecked Sendable { return Self.transactionState(PQtransactionStatus(conn)) } - /// libpq has no round trip that sends no statement, so the socket is read first: that sees a - /// server that closed the session, and inside a transaction block it is the whole check - /// (`LibPQSessionCheck.sendsStatement`). A statement the server refuses still proves the - /// backend answered, so it does not fail the ping. + // libpq has no round trip that sends no statement, so the socket is read first: that sees a + // server that closed the session, and inside a transaction block it is the whole check + // (`LibPQSessionCheck.sendsStatement`). A statement the server refuses still proves the + // backend answered, so it does not fail the ping. func ping() async throws { try await pluginDispatchAsync(on: queue) { [self] in guard !isShuttingDown, let conn = connectionHandle else { throw LibPQPluginError.notConnected } @@ -1444,9 +1461,9 @@ final class LibPQPluginConnection: @unchecked Sendable { // MARK: - Pending Results - /// A statement the user did not get to see is worse than a slow one, so a COPY ended by any - /// drain is reported rather than swallowed: `INSERT INTO t VALUES (1); COPY t FROM STDIN` used - /// to come back as "INSERT 0 1" with the COPY discarded. + // A statement the user did not get to see is worse than a slow one, so a COPY ended by any + // drain is reported rather than swallowed: `INSERT INTO t VALUES (1); COPY t FROM STDIN` used + // to come back as "INSERT 0 1" with the COPY discarded. private func abandonedCopyError(_ outcome: LibPQDrainOutcome, generation: Int) -> Error? { guard let copy = outcome.abandonedCopy else { return nil } if cancellationGate.isCancelled(generation) { return CancellationError() } @@ -1468,9 +1485,9 @@ final class LibPQPluginConnection: @unchecked Sendable { LibPQPluginError(message: copy.direction.unsupportedMessage, sqlState: nil, detail: nil) } - /// Reading `COPY TO STDOUT` to its end defeats the row cap the bounded read exists for, so the - /// statement is cancelled first wherever a cancel is safe. Inside a transaction block it is not, - /// because a cancel aborts the transaction the user opened. + // Reading `COPY TO STDOUT` to its end defeats the row cap the bounded read exists for, so the + // statement is cancelled first wherever a cancel is safe. Inside a transaction block it is not, + // because a cancel aborts the transaction the user opened. private func cancelsAbandonedOutput(_ conn: OpaquePointer) -> Bool { !suppressServerSideCancel && PQtransactionStatus(conn) != PQTRANS_INTRANS } @@ -1500,9 +1517,9 @@ final class LibPQPluginConnection: @unchecked Sendable { return finishPendingResults(conn, cancellingOutput: false) } - /// The abort is polled by the producer rather than acted on from `onTermination`, because both - /// run on one serial queue: a drain enqueued from the handler sits behind the producer and runs - /// only once the whole result has been read, which is no abort at all. + // The abort is polled by the producer rather than acted on from `onTermination`, because both + // run on one serial queue: a drain enqueued from the handler sits behind the producer and runs + // only once the whole result has been read, which is no abort at all. func streamQuery(_ query: String, readOnly: Bool = false) -> AsyncThrowingStream { let queryToRun = String(query) @@ -1519,15 +1536,15 @@ final class LibPQPluginConnection: @unchecked Sendable { defer { cancellationGate.endQuery(generation) } defer { refreshStandardConformingStrings(from: conn) } - /// The consumer can go away before this block is scheduled, in which case the - /// query is never sent at all. + // The consumer can go away before this block is scheduled, in which case the + // query is never sent at all. guard !abort.isAborted else { continuation.finish() return } - /// Read before the query goes out: once it is in flight the status is - /// PQTRANS_ACTIVE, and the transaction this guard exists for is invisible. + // Read before the query goes out: once it is in flight the status is + // PQTRANS_ACTIVE, and the transaction this guard exists for is invisible. let cancelsOutput = cancelsAbandonedOutput(conn) finishAbandonedResults(conn, cancellingOutput: cancelsOutput) if let ended = sessionEndedBeforeSending(conn) { @@ -1777,12 +1794,12 @@ final class LibPQPluginConnection: @unchecked Sendable { private func sessionEndedBeforeSending(_ conn: OpaquePointer) -> LibPQConnectionLostError? { recordTransactionState(of: conn) - /// All three calls earn their place, measured against a terminated backend on 9.1.24 and - /// 17.11. One `PQconsumeInput` leaves the status `CONNECTION_OK`, so a single read never - /// sees the loss; the second one turns it `CONNECTION_BAD`. `PQisBusy` never moves the - /// status, but it is what parses the buffered message: without it every closed-session - /// case loses the server's FATAL and its SQLSTATE and reports libpq's own "server closed - /// the connection unexpectedly" instead. + // All three calls earn their place, measured against a terminated backend on 9.1.24 and + // 17.11. One `PQconsumeInput` leaves the status `CONNECTION_OK`, so a single read never + // sees the loss; the second one turns it `CONNECTION_BAD`. `PQisBusy` never moves the + // status, but it is what parses the buffered message: without it every closed-session + // case loses the server's FATAL and its SQLSTATE and reports libpq's own "server closed + // the connection unexpectedly" instead. if PQstatus(conn) == CONNECTION_OK { _ = PQconsumeInput(conn) _ = PQisBusy(conn) diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQSSLMapping.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQSSLMapping.swift index 366ba4f142..b263b12cce 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQSSLMapping.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQSSLMapping.swift @@ -11,4 +11,21 @@ enum LibPQSSLMapping { case .verifyIdentity: return "verify-full" } } + + // libpq's own text suggests sslrootcert=system, which TablePro does not offer and libpq refuses under verify-ca. + static func missingRootCertificateError(_ message: String, sslConfig: SSLConfiguration) -> LibPQPluginError? { + guard sslConfig.mode == .verifyCa, sslConfig.caCertificatePath.isEmpty else { return nil } + let lowered = message.lowercased() + guard lowered.contains("root certificate file"), lowered.contains("does not exist") else { return nil } + return LibPQPluginError( + message: String( + localized: """ + Verify CA needs the server's CA certificate. Choose it under SSL, or use Verify Identity to check \ + the server against this Mac's trusted certificates. + """ + ), + sqlState: nil, + detail: nil + ) + } } diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQConnectionStringTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQConnectionStringTests.swift index ff3a6b976b..a38c092d50 100644 --- a/apple/Plugins/PostgreSQLDriverTests/LibPQConnectionStringTests.swift +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQConnectionStringTests.swift @@ -14,7 +14,8 @@ struct LibPQConnectionStringTests { sslConfig: SSLConfiguration = SSLConfiguration(), options: String? = nil, applicationName: String? = nil, - connectTimeoutSeconds: Int? = nil + connectTimeoutSeconds: Int? = nil, + systemTrustAnchorsPath: String? = nil ) throws -> String { try LibPQConnectionString.build( host: "db.example.com", @@ -25,7 +26,8 @@ struct LibPQConnectionStringTests { sslConfig: sslConfig, options: options, applicationName: applicationName, - connectTimeoutSeconds: connectTimeoutSeconds + connectTimeoutSeconds: connectTimeoutSeconds, + systemTrustAnchorsPath: systemTrustAnchorsPath ) } @@ -260,6 +262,98 @@ struct LibPQConnectionStringTests { #expect(verifying.contains("sslrootcert='/ca.pem'")) } + @Test("Verify Identity with no CA file checks the server against this Mac's trusted roots") + func verifyIdentityWithoutACAUsesTheSystemAnchors() throws { + let conninfo = try build( + sslConfig: SSLConfiguration(mode: .verifyIdentity), + systemTrustAnchorsPath: "/tmp/roots.pem" + ) + + #expect(conninfo.contains("sslmode='verify-full'")) + #expect(conninfo.contains("sslrootcert='/tmp/roots.pem'")) + } + + @Test("A chosen CA file wins over this Mac's trusted roots") + func aCAFileWinsOverTheSystemAnchors() throws { + let conninfo = try build( + sslConfig: SSLConfiguration(mode: .verifyIdentity, caCertificatePath: "/ca.pem"), + systemTrustAnchorsPath: "/tmp/roots.pem" + ) + + #expect(conninfo.contains("sslrootcert='/ca.pem'")) + #expect(!conninfo.contains("/tmp/roots.pem")) + } + + @Test("Verify CA never checks the server against this Mac's trusted roots") + func verifyCAWithoutACAGetsNoSystemAnchors() throws { + let conninfo = try build(sslConfig: SSLConfiguration(mode: .verifyCa), systemTrustAnchorsPath: "/tmp/roots.pem") + + #expect(conninfo.contains("sslmode='verify-ca'")) + #expect(!conninfo.contains("sslrootcert")) + } + + @Test("A mode that checks no certificate gets no root file", arguments: [SSLMode.disabled, .preferred, .required]) + func requiredAndPreferredNeverGetSystemAnchors(mode: SSLMode) throws { + let conninfo = try build(sslConfig: SSLConfiguration(mode: mode), systemTrustAnchorsPath: "/tmp/roots.pem") + + #expect(!conninfo.contains("sslrootcert")) + } + + @Test("The roots come from the CA file, then libpq's own root.crt, then this Mac's trusted roots") + func rootCertificateSourceOrder() { + let home = ["HOME": "/Users/test"] + let rootCrt = "/Users/test/.postgresql/root.crt" + func source( + _ mode: SSLMode, + ca: String = "", + host: String = "db.example.com", + rootCrtExists: Bool = false + ) -> LibPQRootCertificateSource? { + LibPQConnectionString.rootCertificateSource( + for: SSLConfiguration(mode: mode, caCertificatePath: ca), + host: host, + environment: home, + fileExists: { rootCrtExists && $0 == rootCrt } + ) + } + + #expect(source(.verifyIdentity, ca: "/ca.pem", rootCrtExists: true) == .caFile("/ca.pem")) + #expect(source(.verifyCa, ca: "/ca.pem") == .caFile("/ca.pem")) + #expect(source(.verifyIdentity, rootCrtExists: true) == .libpqDefault) + #expect(source(.verifyIdentity) == .systemTrustAnchors) + #expect(source(.verifyIdentity, host: "/tmp") == .libpqDefault) + #expect(source(.verifyCa) == .libpqDefault) + #expect(source(.required) == nil) + #expect(source(.preferred) == nil) + #expect(source(.disabled) == nil) + } + + @Test("A PGSSLROOTCERT the user set, or an empty host, leaves the roots to libpq") + func namedRootFileAndDefaultSocketStayWithLibpq() { + let config = SSLConfiguration(mode: .verifyIdentity) + #expect(LibPQConnectionString.rootCertificateSource( + for: config, host: "db.example.com", environment: ["HOME": "/Users/test", "PGSSLROOTCERT": "/etc/corp-ca.pem"], + fileExists: { _ in false } + ) == .libpqDefault) + #expect(LibPQConnectionString.rootCertificateSource( + for: config, host: "", environment: ["HOME": "/Users/test"], fileExists: { _ in false } + ) == .libpqDefault) + #expect(LibPQConnectionString.rootCertificateSource( + for: config, host: "db.example.com", environment: ["HOME": "/Users/test", "PGSSLROOTCERT": ""], + fileExists: { _ in false } + ) == .systemTrustAnchors) + } + + @Test("libpq's own root file is found under HOME, and under the account's home when HOME is unset or empty") + func defaultRootCertificateFollowsHome() { + let fallback = NSHomeDirectory() + "/.postgresql/root.crt" + + #expect(LibPQConnectionString.defaultRootCertificatePath(environment: ["HOME": "/Users/test"]) + == "/Users/test/.postgresql/root.crt") + #expect(LibPQConnectionString.defaultRootCertificatePath(environment: ["HOME": ""]) == fallback) + #expect(LibPQConnectionString.defaultRootCertificatePath(environment: [:]) == fallback) + } + @Test("A client certificate and key reach libpq as sslcert and sslkey") func clientCertificateIsSent() throws { let conninfo = try build(sslConfig: SSLConfiguration( @@ -330,8 +424,8 @@ struct LibPQConnectionStringTests { #expect(try !build(applicationName: "").contains("application_name")) } - /// Measured on libpq 17: a fallback name replaces an `application_name` set in `options`, so it - /// has to stay out whenever the user named the session there, in any of the server's spellings. + // Measured on libpq 17: a fallback name replaces an `application_name` set in `options`, so it + // has to stay out whenever the user named the session there, in any of the server's spellings. @Test( "A name the user gives the session in the connection options keeps the fallback out", arguments: [ diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQSSLMappingTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQSSLMappingTests.swift index c20d11494d..4c5a1a3633 100644 --- a/apple/Plugins/PostgreSQLDriverTests/LibPQSSLMappingTests.swift +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQSSLMappingTests.swift @@ -32,4 +32,33 @@ struct LibPQSSLMappingTests { func verifyIdentity() { #expect(LibPQSSLMapping.sslmode(for: .verifyIdentity) == "verify-full") } + + private static let missingRootMessage = """ + root certificate file "/Users/test/.postgresql/root.crt" does not exist + Either provide the file, use the system's trusted roots with sslrootcert=system, or change sslmode to disable \ + server certificate verification. + """ + + @Test("Verify CA with no CA file names the missing CA and the mode that needs none, never sslrootcert=system") + func verifyCAWithoutACAExplainsTheMissingCA() throws { + let error = try #require( + LibPQSSLMapping.missingRootCertificateError(Self.missingRootMessage, sslConfig: SSLConfiguration(mode: .verifyCa)) + ) + + #expect(error.message.contains("Verify CA needs the server's CA certificate")) + #expect(error.message.contains("Verify Identity")) + #expect(!error.message.contains("sslrootcert")) + } + + @Test("libpq's text is kept for any other mode, for a chosen CA file, and for any other failure") + func otherFailuresKeepLibpqText() { + let otherModes = [SSLMode.verifyIdentity, .required, .preferred].map { SSLConfiguration(mode: $0) } + for sslConfig in otherModes { + #expect(LibPQSSLMapping.missingRootCertificateError(Self.missingRootMessage, sslConfig: sslConfig) == nil) + } + let withCA = SSLConfiguration(mode: .verifyCa, caCertificatePath: "/ca.pem") + #expect(LibPQSSLMapping.missingRootCertificateError(Self.missingRootMessage, sslConfig: withCA) == nil) + let refused = "connection to server at \"db\" failed: FATAL: password authentication failed for user \"app\"" + #expect(LibPQSSLMapping.missingRootCertificateError(refused, sslConfig: SSLConfiguration(mode: .verifyCa)) == nil) + } } diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift index c98626d2bd..528271ce9a 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift @@ -29,25 +29,27 @@ public enum SurrealStatementGenerator { insertedRowIndices: Set ) throws(PluginRowWriteRefusal) -> [PluginRowWrite] { var writes: [PluginRowWrite] = [] + var insertedWritten: Set = [] let typing = FieldTyping(kinds: kinds, held: heldKinds, records: recordKinds, declared: declaredKinds) - for change in changes where change.type == .update && !insertedRowIndices.contains(change.rowIndex) { - let write = try update(table: table, scope: scope, columns: columns, typing: typing, change: change) - guard let write else { continue } - writes.append(write) - } - - for index in insertedRowIndices.sorted() { - guard let values = insertedRowData[index] else { continue } - writes.append(try insert( - table: table, scope: scope, columns: columns, typing: typing, values: values, rowIndex: index - )) - } - - for change in changes where change.type == .delete || deletedRowIndices.contains(change.rowIndex) { - guard !insertedRowIndices.contains(change.rowIndex) else { continue } - guard let write = delete(table: table, scope: scope, columns: columns, change: change) else { continue } - writes.append(write) + // In the order received, which is edit order: a record deleted to free a unique value goes before the one taking it. + for change in changes { + let index = change.rowIndex + if insertedRowIndices.contains(index) { + guard insertedWritten.insert(index).inserted, let values = insertedRowData[index] else { continue } + writes.append(try insert( + table: table, scope: scope, columns: columns, typing: typing, values: values, rowIndex: index + )) + continue + } + if change.type == .update, + let write = try update(table: table, scope: scope, columns: columns, typing: typing, change: change) { + writes.append(write) + } + if change.type == .delete || deletedRowIndices.contains(index), + let write = delete(table: table, scope: scope, columns: columns, change: change) { + writes.append(write) + } } return writes diff --git a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index 53d2af6929..ed53c66e35 100644 --- a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -55,6 +55,7 @@ public struct PluginRowChange: Sendable { case delete } + // One row can arrive as several changes, each with its own key and the row as it stands before that change. public let rowIndex: Int public let type: ChangeType public let cellChanges: [(columnIndex: Int, columnName: String, oldValue: PluginCellValue, newValue: PluginCellValue)] diff --git a/apple/TablePro/Core/AI/AnthropicProvider.swift b/apple/TablePro/Core/AI/AnthropicProvider.swift index 0019e43fda..7106932e2f 100644 --- a/apple/TablePro/Core/AI/AnthropicProvider.swift +++ b/apple/TablePro/Core/AI/AnthropicProvider.swift @@ -405,7 +405,14 @@ final class AnthropicProvider: ChatTransport { let tokens = usage["output_tokens"] as? Int { state.outputTokens = tokens } - return [] + switch (json["delta"] as? [String: Any])?["stop_reason"] as? String { + case "max_tokens": + return [.outputLimitReached(.outputTokens)] + case "model_context_window_exceeded": + return [.outputLimitReached(.contextWindow)] + default: + return [] + } case "error": if let errorObj = json["error"] as? [String: Any], let message = errorObj["message"] as? String { diff --git a/apple/TablePro/Core/AI/Chat/ChatTransport.swift b/apple/TablePro/Core/AI/Chat/ChatTransport.swift index 5ef3b2230a..03f685965a 100644 --- a/apple/TablePro/Core/AI/Chat/ChatTransport.swift +++ b/apple/TablePro/Core/AI/Chat/ChatTransport.swift @@ -83,6 +83,12 @@ struct ChatToolSpec: Codable, Equatable, Sendable { } } +// Why a provider stopped the response before the model finished it. +enum ChatOutputLimit: String, Codable, Sendable { + case outputTokens + case contextWindow +} + enum ChatStreamEvent: Sendable { case textDelta(String) case toolUseStart(id: String, name: String, providerMetadata: [String: String]? = nil) @@ -93,6 +99,7 @@ enum ChatStreamEvent: Sendable { case reasoningStart(id: String) case reasoningDelta(id: String, text: String) case reasoningEnd(id: String, opaque: ReasoningOpaque?) + case outputLimitReached(ChatOutputLimit) } final class ToolReplyToken: Sendable { diff --git a/apple/TablePro/Core/AI/Chat/ChatTurn.swift b/apple/TablePro/Core/AI/Chat/ChatTurn.swift index b7bb4c1d5f..429a06f5b5 100644 --- a/apple/TablePro/Core/AI/Chat/ChatTurn.swift +++ b/apple/TablePro/Core/AI/Chat/ChatTurn.swift @@ -105,6 +105,7 @@ final class ChatTurn: ObservableObject, Identifiable { @Published var usage: AITokenUsage? @Published var modelId: String? @Published var providerId: String? + @Published var outputLimit: ChatOutputLimit? init( id: UUID = UUID(), @@ -113,7 +114,8 @@ final class ChatTurn: ObservableObject, Identifiable { timestamp: Date = Date(), usage: AITokenUsage? = nil, modelId: String? = nil, - providerId: String? = nil + providerId: String? = nil, + outputLimit: ChatOutputLimit? = nil ) { self.id = id self.role = role @@ -122,6 +124,7 @@ final class ChatTurn: ObservableObject, Identifiable { self.usage = usage self.modelId = modelId self.providerId = providerId + self.outputLimit = outputLimit } init(wire: ChatTurnWire) { @@ -132,6 +135,7 @@ final class ChatTurn: ObservableObject, Identifiable { self.usage = wire.usage self.modelId = wire.modelId self.providerId = wire.providerId + self.outputLimit = wire.outputLimit } var plainText: String { @@ -152,7 +156,8 @@ final class ChatTurn: ObservableObject, Identifiable { timestamp: timestamp, usage: usage, modelId: modelId, - providerId: providerId + providerId: providerId, + outputLimit: outputLimit ) } @@ -337,6 +342,7 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { var usage: AITokenUsage? var modelId: String? var providerId: String? + var outputLimit: ChatOutputLimit? init( id: UUID = UUID(), @@ -345,7 +351,8 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { timestamp: Date = Date(), usage: AITokenUsage? = nil, modelId: String? = nil, - providerId: String? = nil + providerId: String? = nil, + outputLimit: ChatOutputLimit? = nil ) { self.id = id self.role = role @@ -354,6 +361,7 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { self.usage = usage self.modelId = modelId self.providerId = providerId + self.outputLimit = outputLimit } init(from decoder: Decoder) throws { @@ -364,6 +372,10 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { usage = try container.decodeIfPresent(AITokenUsage.self, forKey: .usage) modelId = try container.decodeIfPresent(String.self, forKey: .modelId) providerId = try container.decodeIfPresent(String.self, forKey: .providerId) + // A value a later version wrote reads as no note rather than failing the conversation. + outputLimit = (try? container.decodeIfPresent(String.self, forKey: .outputLimit)).flatMap { + $0.flatMap(ChatOutputLimit.init(rawValue:)) + } if let decodedBlocks = try container.decodeIfPresent([LossyContentBlockWire].self, forKey: .blocks) { blocks = decodedBlocks.compactMap(\.block) @@ -386,6 +398,7 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { try container.encodeIfPresent(usage, forKey: .usage) try container.encodeIfPresent(modelId, forKey: .modelId) try container.encodeIfPresent(providerId, forKey: .providerId) + try container.encodeIfPresent(outputLimit, forKey: .outputLimit) } var plainText: String { @@ -399,7 +412,7 @@ struct ChatTurnWire: Codable, Equatable, Sendable, Identifiable { } private enum CodingKeys: String, CodingKey { - case id, role, blocks, timestamp, usage, modelId, providerId + case id, role, blocks, timestamp, usage, modelId, providerId, outputLimit } private enum LegacyKeys: String, CodingKey { diff --git a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift index e21593ca8f..cdf25b831a 100644 --- a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift +++ b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift @@ -51,6 +51,8 @@ final class ClaudeAgentProvider: ChatTransport { continuation.yield(.textDelta(text)) case .usage(let usage): continuation.yield(.usage(usage)) + case .outputLimit(let limit): + continuation.yield(.outputLimitReached(limit)) case .failure(let message): await toolBridge.release(grant) grant = nil @@ -90,6 +92,7 @@ final class ClaudeAgentProvider: ChatTransport { enum DecodedEvent: Equatable { case text(String) case usage(AITokenUsage) + case outputLimit(ChatOutputLimit) case failure(String) } @@ -129,6 +132,7 @@ final class ClaudeAgentProvider: ChatTransport { guard let json = decodeJSON(line), let type = json["type"] as? String else { return nil } switch type { case "stream_event": + if let limit = outputLimit(json) { return .outputLimit(limit) } guard let text = partialText(json) else { return nil } return .text(text) case "result": @@ -149,6 +153,18 @@ final class ClaudeAgentProvider: ChatTransport { return text } + // The CLI passes the Messages API stream through, so a cut response ends with its message_delta stop reason. + static func outputLimit(_ json: [String: Any]) -> ChatOutputLimit? { + guard let event = json["event"] as? [String: Any], + event["type"] as? String == "message_delta", + let delta = event["delta"] as? [String: Any] else { return nil } + switch delta["stop_reason"] as? String { + case "max_tokens": return .outputTokens + case "model_context_window_exceeded": return .contextWindow + default: return nil + } + } + static func resultEvent(_ json: [String: Any]) -> DecodedEvent? { if json["is_error"] as? Bool == true { let message = (json["result"] as? String).flatMap { $0.isEmpty ? nil : $0 } diff --git a/apple/TablePro/Core/AI/GeminiProvider.swift b/apple/TablePro/Core/AI/GeminiProvider.swift index b59e1a7663..b2197bd9cd 100644 --- a/apple/TablePro/Core/AI/GeminiProvider.swift +++ b/apple/TablePro/Core/AI/GeminiProvider.swift @@ -321,6 +321,10 @@ final class GeminiProvider: ChatTransport { } } } + if let candidates = json["candidates"] as? [[String: Any]], + candidates.first?["finishReason"] as? String == "MAX_TOKENS" { + events.append(.outputLimitReached(.outputTokens)) + } if let usageMetadata = json["usageMetadata"] as? [String: Any] { if let prompt = usageMetadata["promptTokenCount"] as? Int { state.inputTokens = prompt diff --git a/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift b/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift index 87cba934b7..59d67841f6 100644 --- a/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift +++ b/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift @@ -171,6 +171,13 @@ final class OpenAICompatibleProvider: ChatTransport { if finishReason == "tool_calls" { events.append(contentsOf: state.flushToolUseEnds()) } + if finishReason == "length" { + events.append(.outputLimitReached(.outputTokens)) + } + } + // Ollama's own API names the cut in `done_reason`. + if (json["done"] as? Bool) == true, (json["done_reason"] as? String) == "length" { + events.append(.outputLimitReached(.outputTokens)) } if let usage = json["usage"] as? [String: Any], diff --git a/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift b/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift index e246479c13..62ac71204d 100644 --- a/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift +++ b/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift @@ -371,15 +371,17 @@ final class OpenAIResponsesProvider: ChatTransport { return [.textDelta(delta)] } return [] - case "response.completed": - if let responseObj = json["response"] as? [String: Any], - let usage = responseObj["usage"] as? [String: Any] { + case "response.completed", "response.incomplete": + let responseObj = json["response"] as? [String: Any] + if let usage = responseObj?["usage"] as? [String: Any] { let input = usage["input_tokens"] as? Int ?? 0 let output = usage["output_tokens"] as? Int ?? 0 state.inputTokens = input state.outputTokens = output } - return [] + let details = responseObj?["incomplete_details"] as? [String: Any] + guard type == "response.incomplete", details?["reason"] as? String == "max_output_tokens" else { return [] } + return [.outputLimitReached(.outputTokens)] case "response.failed": if let responseObj = json["response"] as? [String: Any], let errorObj = responseObj["error"] as? [String: Any], diff --git a/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift b/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift index 1c6d8f757c..4ceae454c5 100644 --- a/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift +++ b/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift @@ -109,6 +109,34 @@ final class DataChangeManager: ObservableObject, ChangeManaging { private var loadedValues: [RowID: [Int: PluginCellValue]] = [:] private var coalescedOrder: [CoalescedCellKey] = [] + // Process-wide, so a run restored from another tab never shares an id with a new command. + private static var lastActionID = 0 + private var commandAction: Int? + private var typingAction: Int? + + private static func newActionID() -> Int { + lastActionID += 1 + return lastActionID + } + + // One undo step over many cells, so a row it writes more than once stays one write in the save. + func recordAsOneAction(_ body: () -> Void) { + guard commandAction == nil else { + body() + return + } + commandAction = Self.newActionID() + defer { commandAction = nil } + body() + } + + private func typingRunAction() -> Int { + if let typingAction { return typingAction } + let action = Self.newActionID() + typingAction = action + return action + } + /// Whether a typed edit is waiting to become an undo step. The Edit menu reads it, because the /// undo manager has nothing registered yet while the run is open. var hasCoalescedUndoRun: Bool { !coalescedOrder.isEmpty } @@ -323,6 +351,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { wasExpression: pending.isExpression(rowID: rowID, columnIndex: columnIndex), isExpression: isExpression ) + let action = coalescesWithPrevious ? typingRunAction() : commandAction ?? Self.newActionID() objectWillChange.send() let recorded = pending.recordCellChange( rowID: rowID, @@ -332,7 +361,8 @@ final class DataChangeManager: ObservableObject, ChangeManaging { newValue: newValue, originalRow: originalRow, absence: absence, - isExpression: isExpression + isExpression: isExpression, + action: action ) guard recorded else { hasChanges = !pending.isEmpty @@ -346,11 +376,14 @@ final class DataChangeManager: ObservableObject, ChangeManaging { ) } else { registerUndo(actionName: String(localized: "Edit Cell")) { target in - target.applyDataUndo(.cellEdit( - rowID: rowID, columnIndex: columnIndex, columnName: columnName, - previousValue: oldValue, newValue: newValue, originalRow: originalRow, - absence: absence, expression: expression - )) + target.applyDataUndo( + .cellEdit( + rowID: rowID, columnIndex: columnIndex, columnName: columnName, + previousValue: oldValue, newValue: newValue, originalRow: originalRow, + absence: absence, expression: expression + ), + actionID: action + ) } } hasChanges = !pending.isEmpty @@ -382,25 +415,28 @@ final class DataChangeManager: ObservableObject, ChangeManaging { coalescedOrder.append(key) } - /// Registers the run's cells as one undo step, in the order they were first written. - /// - /// A cell typed back to where it started contributes nothing: registering it would leave a step - /// that restores a value the cell already holds, and undoing it would record a pending change - /// over a value that already matches the server. + // A cell typed back to where it started is left out: undoing it would stage a value the server already holds. func endCoalescedUndoRun() { - guard !coalescedOrder.isEmpty else { return } + guard !coalescedOrder.isEmpty else { + typingAction = nil + return + } let edits = coalescedOrder.compactMap { coalescedEdits[$0] }.filter { !$0.isNoOp } + let action = typingAction discardCoalescedUndoRun() guard !edits.isEmpty, let undoManager = undoManagerProvider?() else { return } undoManager.beginUndoGrouping() for edit in edits { undoManager.registerUndo(withTarget: self) { target in - target.applyDataUndo(.cellEdit( - rowID: edit.rowID, columnIndex: edit.columnIndex, columnName: edit.columnName, - previousValue: edit.previousValue, newValue: edit.newValue, originalRow: edit.originalRow, - absence: edit.absence, expression: edit.expression - )) + target.applyDataUndo( + .cellEdit( + rowID: edit.rowID, columnIndex: edit.columnIndex, columnName: edit.columnName, + previousValue: edit.previousValue, newValue: edit.newValue, originalRow: edit.originalRow, + absence: edit.absence, expression: edit.expression + ), + actionID: action + ) } } undoManager.setActionName(String(localized: "Edit Cell")) @@ -410,6 +446,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { private func discardCoalescedUndoRun() { coalescedEdits.removeAll() coalescedOrder.removeAll() + typingAction = nil } private func discardUndoHistory() { @@ -560,7 +597,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { // MARK: - Core Undo Application - private func applyDataUndo(_ action: UndoAction) { + private func applyDataUndo(_ action: UndoAction, actionID: Int? = nil) { objectWillChange.send() switch action { case .cellEdit(let rowID, let columnIndex, let columnName, let previousValue, let newValue, let originalRow, @@ -568,7 +605,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { applyCellEditUndo( rowID: rowID, columnIndex: columnIndex, columnName: columnName, previousValue: previousValue, newValue: newValue, originalRow: originalRow, - absence: absence, expression: expression, action: action + absence: absence, expression: expression, action: action, actionID: actionID ) case .rowInsertion(let rowID, let image): @@ -597,20 +634,25 @@ final class DataChangeManager: ObservableObject, ChangeManaging { private func applyCellEditUndo( rowID: RowID, columnIndex: Int, columnName: String, previousValue: PluginCellValue, newValue: PluginCellValue, originalRow: [PluginCellValue]?, - absence: FieldAbsence, expression: ExpressionTransition, action: UndoAction + absence: FieldAbsence, expression: ExpressionTransition, action: UndoAction, actionID: Int? ) { + let isUndoing = undoManagerProvider?()?.isUndoing ?? false registerUndo(actionName: String(localized: "Edit Cell")) { target in - target.applyDataUndo(.cellEdit( - rowID: rowID, columnIndex: columnIndex, columnName: columnName, - previousValue: newValue, newValue: previousValue, originalRow: originalRow, - absence: absence.reversed, expression: expression.reversed - )) + target.applyDataUndo( + .cellEdit( + rowID: rowID, columnIndex: columnIndex, columnName: columnName, + previousValue: newValue, newValue: previousValue, originalRow: originalRow, + absence: absence.reversed, expression: expression.reversed + ), + actionID: actionID + ) } + // A redo is a new write, so it goes back after the other rows' changes still pending, where it was. let updateHoldsCell = pending.change(forRow: rowID, type: .update)? .cellChanges.contains { $0.columnIndex == columnIndex } ?? false - if updateHoldsCell { - pending.revertUpdateCell( + if updateHoldsCell, isUndoing { + pending.revertCellWrite( rowID: rowID, columnIndex: columnIndex, columnName: columnName, previousValue: previousValue, previousIsAbsent: absence.wasAbsent, @@ -627,7 +669,8 @@ final class DataChangeManager: ObservableObject, ChangeManaging { rowID: rowID, columnIndex: columnIndex, columnName: columnName, originalDBValue: newValue, newValue: previousValue, originalRow: originalRow, - absence: absence.reversed, expression: expression.reversed + absence: absence.reversed, expression: expression.reversed, + action: actionID ) } lastUndoResult = UndoResult( diff --git a/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift b/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift index f86e46a25c..a3910c6c8f 100644 --- a/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift +++ b/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift @@ -79,6 +79,47 @@ struct CellChange: Identifiable, Equatable { var restoresOriginal: Bool { oldValue == newValue && oldIsAbsent == newIsAbsent && oldIsExpression == newIsExpression } + + func followed(by later: CellChange) -> CellChange { + CellChange( + columnIndex: columnIndex, + columnName: later.columnName, + oldValue: oldValue, + newValue: later.newValue, + oldIsAbsent: oldIsAbsent, + newIsAbsent: later.newIsAbsent, + oldIsExpression: oldIsExpression, + newIsExpression: later.newIsExpression + ) + } +} + +// A cell's old value is the cell as this run found it, not as the row was read. +struct UpdateRun: Equatable { + let sequence: Int + let action: Int + var cellChanges: [CellChange] + + func holds(_ columnIndex: Int) -> Bool { + cellChanges.contains { $0.columnIndex == columnIndex } + } + + func writesExpression(into columns: Set) -> Bool { + cellChanges.contains { $0.newIsExpression && columns.contains($0.columnIndex) } + } + + mutating func write(_ cellChange: CellChange) { + guard let index = cellChanges.firstIndex(where: { $0.columnIndex == cellChange.columnIndex }) else { + cellChanges.append(cellChange) + return + } + let written = cellChanges[index].followed(by: cellChange) + if written.restoresOriginal { + cellChanges.remove(at: index) + } else { + cellChanges[index] = written + } + } } struct RowChange: Identifiable, Equatable { @@ -104,6 +145,9 @@ struct RowChange: Identifiable, Equatable { /// and reusing its unique value in a new one only works in that order. var sequence: Int + // An update's edits in the order they were made. Empty reads as one run at `sequence`. + var runs: [UpdateRun] + init( rowID: RowID, type: ChangeType, @@ -111,7 +155,8 @@ struct RowChange: Identifiable, Equatable { originalRow: [PluginCellValue]? = nil, absentColumns: Set = [], expressionColumns: Set = [], - sequence: Int = 0 + sequence: Int = 0, + runs: [UpdateRun] = [] ) { self.id = UUID() self.rowID = rowID @@ -121,6 +166,76 @@ struct RowChange: Identifiable, Equatable { self.absentColumns = absentColumns self.expressionColumns = expressionColumns self.sequence = sequence + self.runs = runs + } + + // A row keeps one UPDATE at its first run when a later step could not find it: with no key, a keyless match fails + // once the server rewrites a column on the first UPDATE, and an earlier run's SQL in a key is not the key it wrote. + static func saveSteps(_ changes: [RowChange], keyColumns: Set) -> [RowChange] { + var steps: [(step: RowChange, source: Int)] = [] + for (source, change) in changes.enumerated() { + guard change.type == .update, let first = change.runs.first else { + steps.append((change, source)) + continue + } + guard !keyColumns.isEmpty, change.runs.count > 1, var row = change.originalRow, + !change.runs.dropLast().contains(where: { $0.writesExpression(into: keyColumns) }) + else { + var placed = change + placed.sequence = first.sequence + steps.append((placed, source)) + continue + } + var absent = change.absentColumns + for run in change.runs { + steps.append(( + RowChange( + rowID: change.rowID, type: .update, cellChanges: run.cellChanges, originalRow: row, + absentColumns: absent, expressionColumns: change.expressionColumns, sequence: run.sequence + ), + source + )) + for cellChange in run.cellChanges { + if row.indices.contains(cellChange.columnIndex) { + row[cellChange.columnIndex] = cellChange.newValue + } + if cellChange.newIsAbsent { + absent.insert(cellChange.columnIndex) + } else { + absent.remove(cellChange.columnIndex) + } + } + } + } + + let ordered = steps.enumerated() + .sorted { ($0.element.step.sequence, $0.offset) < ($1.element.step.sequence, $1.offset) } + .map(\.element) + // Two runs with nothing left between them, once a change recorded there was cancelled, are one UPDATE. + var merged: [(step: RowChange, source: Int)] = [] + for entry in ordered { + if let last = merged.last, last.source == entry.source, entry.step.type == .update { + merged[merged.count - 1].step = last.step.followed(by: entry.step) + } else { + merged.append(entry) + } + } + return merged.map(\.step) + } + + private func followed(by later: RowChange) -> RowChange { + var cells = cellChanges + for cellChange in later.cellChanges { + if let index = cells.firstIndex(where: { $0.columnIndex == cellChange.columnIndex }) { + cells[index] = cells[index].followed(by: cellChange) + } else { + cells.append(cellChange) + } + } + return RowChange( + rowID: rowID, type: type, cellChanges: cells, originalRow: originalRow, + absentColumns: absentColumns, expressionColumns: expressionColumns, sequence: sequence + ) } // A value loaded after the change was recorded replaces the placeholder its pre-image captured. diff --git a/apple/TablePro/Core/ChangeTracking/PendingChanges.swift b/apple/TablePro/Core/ChangeTracking/PendingChanges.swift index c4882458bd..e9d21fe2bc 100644 --- a/apple/TablePro/Core/ChangeTracking/PendingChanges.swift +++ b/apple/TablePro/Core/ChangeTracking/PendingChanges.swift @@ -2,12 +2,6 @@ // PendingChanges.swift // TablePro // -// Value type holding all uncommitted edits to a result set. -// Owns the consistency invariants between `changes`, `changeIndex`, -// `deletedRowIDs`, `insertedRowIDs`, `modifiedCells`, and -// `insertedRowData`. Callers mutate through methods that maintain -// the cross-collection state. -// import Foundation import TableProPluginKit @@ -16,9 +10,7 @@ struct PendingChanges: Equatable { private(set) var changes: [RowChange] = [] private(set) var deletedRowIDs: Set = [] - /// Stamped onto every change so statement generation can recover the order the user worked in. - /// `changes` cannot carry that order itself: a cancelled change is removed by swapping the last - /// element into its slot. + // `changes` cannot carry the edit order: a cancelled change is swap-removed from it. private var nextSequence = 0 private(set) var insertedRowIDs: Set = [] private(set) var modifiedCells: [RowID: Set] = [:] @@ -108,11 +100,7 @@ struct PendingChanges: Equatable { // MARK: - Mutate (recording user edits) - /// Whether the recorded edit is a no-op (oldValue == newValue with no prior modification). - /// Returns the result so the caller can decide whether to register undo. - /// - /// A field going missing, or coming back, is a change even when the value reads `.null` on - /// both sides. So is an expression put in place of text that reads the same, or text in place of one. + // False for a no-op, so no undo is registered. A nil `action` is an undo step of its own. @discardableResult mutating func recordCellChange( rowID: RowID, @@ -122,7 +110,8 @@ struct PendingChanges: Equatable { newValue: PluginCellValue, originalRow: [PluginCellValue]? = nil, absence: FieldAbsence = FieldAbsence(), - isExpression: Bool = false + isExpression: Bool = false, + action: Int? = nil ) -> Bool { let wasExpression = self.isExpression(rowID: rowID, columnIndex: columnIndex) if oldValue == newValue && absence.wasAbsent == absence.isAbsent && wasExpression == isExpression { @@ -152,14 +141,12 @@ struct PendingChanges: Equatable { } if let updateIdx = changeIndex[RowChangeKey(rowID: rowID, type: .update)] { - mergeUpdateCell(at: updateIdx, cellChange: cellChange) + mergeUpdateCell(at: updateIdx, cellChange: cellChange, action: action) } else { - appendChange(RowChange( - rowID: rowID, type: .update, - cellChanges: [cellChange], originalRow: originalRow, - absentColumns: absence.originalRow - )) - modifiedCells[rowID, default: []].insert(columnIndex) + appendUpdate( + rowID: rowID, cellChange: cellChange, originalRow: originalRow, + absentColumns: absence.originalRow, action: action + ) } return true } @@ -246,9 +233,7 @@ struct PendingChanges: Equatable { recordRowDeletion(rowID: rowID, originalRow: originalRow, absentColumns: absentColumns) } - /// Re-apply a cell edit during undo replay (skips undo registration). - /// `originalDBValue` is the cell's value in the unmodified database row. - /// It must be preserved so that a later collapse compares correctly. + // `originalDBValue` is the cell as it stands before this write. mutating func reapplyCellChange( rowID: RowID, columnIndex: Int, @@ -257,7 +242,8 @@ struct PendingChanges: Equatable { newValue: PluginCellValue, originalRow: [PluginCellValue]?, absence: FieldAbsence = FieldAbsence(), - expression: ExpressionTransition = ExpressionTransition() + expression: ExpressionTransition = ExpressionTransition(), + action: Int? = nil ) { let cellChange = CellChange( columnIndex: columnIndex, @@ -279,14 +265,12 @@ struct PendingChanges: Equatable { } if let updateIdx = changeIndex[RowChangeKey(rowID: rowID, type: .update)] { - mergeUpdateCell(at: updateIdx, cellChange: cellChange) + mergeUpdateCell(at: updateIdx, cellChange: cellChange, action: action) } else { - appendChange(RowChange( - rowID: rowID, type: .update, - cellChanges: [cellChange], originalRow: originalRow, - absentColumns: absence.originalRow - )) - modifiedCells[rowID, default: []].insert(columnIndex) + appendUpdate( + rowID: rowID, cellChange: cellChange, originalRow: originalRow, + absentColumns: absence.originalRow, action: action + ) } } @@ -306,8 +290,8 @@ struct PendingChanges: Equatable { ) } - /// Restore a cell's value during undo replay when an existing change matches. - mutating func revertUpdateCell( + // Undo takes back the cell's latest write, which the newest run holding the cell made. + mutating func revertCellWrite( rowID: RowID, columnIndex: Int, columnName: String, @@ -316,29 +300,24 @@ struct PendingChanges: Equatable { previousIsExpression: Bool = false ) { guard let updateIdx = changeIndex[RowChangeKey(rowID: rowID, type: .update)], - let cellIdx = changes[updateIdx].cellChanges.firstIndex(where: { $0.columnIndex == columnIndex }) + let current = changes[updateIdx].cellChanges.first(where: { $0.columnIndex == columnIndex }) else { return } - let original = changes[updateIdx].cellChanges[cellIdx] - if previousValue == original.oldValue && previousIsAbsent == original.oldIsAbsent - && previousIsExpression == original.oldIsExpression { - changes[updateIdx].cellChanges.remove(at: cellIdx) - removeModifiedCell(rowID: rowID, columnIndex: columnIndex) - if changes[updateIdx].cellChanges.isEmpty { - removeChangeAt(updateIdx) - } - } else { - changes[updateIdx].cellChanges[cellIdx] = CellChange( - columnIndex: columnIndex, - columnName: columnName, - oldValue: original.oldValue, - newValue: previousValue, - oldIsAbsent: original.oldIsAbsent, - newIsAbsent: previousIsAbsent, - oldIsExpression: original.oldIsExpression, - newIsExpression: previousIsExpression - ) - } + materializeRuns(at: updateIdx) + let reverted = CellChange( + columnIndex: columnIndex, + columnName: columnName, + oldValue: current.newValue, + newValue: previousValue, + oldIsAbsent: current.newIsAbsent, + newIsAbsent: previousIsAbsent, + oldIsExpression: current.newIsExpression, + newIsExpression: previousIsExpression + ) + guard writeNetCell(reverted, at: updateIdx), + let runIdx = changes[updateIdx].runs.lastIndex(where: { $0.holds(columnIndex) }) + else { return } + write(reverted, intoRun: runIdx, at: updateIdx) } /// Insert a synthetic .insert RowChange for undo replay (e.g., after redoing a deletion's undo). @@ -430,7 +409,7 @@ struct PendingChanges: Equatable { modifiedCells = snapshot.modifiedCells insertedRowData = snapshot.insertedRowData supersededUpdates = snapshot.supersededUpdates - let sequences = changes.map(\.sequence) + supersededUpdates.values.map(\.sequence) + let sequences = (changes + supersededUpdates.values).flatMap { [$0.sequence] + $0.runs.map(\.sequence) } nextSequence = (sequences.max() ?? -1) + 1 rebuildChangeIndex() } @@ -458,6 +437,29 @@ struct PendingChanges: Equatable { changeIndex[RowChangeKey(rowID: stamped.rowID, type: stamped.type)] = changes.count - 1 } + private mutating func appendUpdate( + rowID: RowID, + cellChange: CellChange, + originalRow: [PluginCellValue]?, + absentColumns: Set, + action: Int? + ) { + let run = UpdateRun( + sequence: nextSequence, action: action ?? Self.ownAction(at: nextSequence), cellChanges: [cellChange] + ) + appendChange(RowChange( + rowID: rowID, type: .update, + cellChanges: [cellChange], originalRow: originalRow, + absentColumns: absentColumns, runs: [run] + )) + modifiedCells[rowID, default: []].insert(cellChange.columnIndex) + } + + // Below every id DataChangeManager hands out, so no other write ever joins it. + private static func ownAction(at sequence: Int) -> Int { + -(sequence + 1) + } + @discardableResult private mutating func removeChange(rowID: RowID, type: ChangeType) -> Bool { let key = RowChangeKey(rowID: rowID, type: type) @@ -528,32 +530,72 @@ struct PendingChanges: Equatable { } } - private mutating func mergeUpdateCell(at updateIdx: Int, cellChange: CellChange) { + private mutating func mergeUpdateCell(at updateIdx: Int, cellChange: CellChange, action: Int?) { + materializeRuns(at: updateIdx) + guard writeNetCell(cellChange, at: updateIdx) else { return } + + // The newest run takes the write when the same step opened it or nothing was recorded since. + if let newest = changes[updateIdx].runs.indices.last { + let run = changes[updateIdx].runs[newest] + if (action != nil && run.action == action) || run.sequence == nextSequence - 1 { + write(cellChange, intoRun: newest, at: updateIdx) + return + } + } + changes[updateIdx].runs.append(UpdateRun( + sequence: nextSequence, action: action ?? Self.ownAction(at: nextSequence), cellChanges: [cellChange] + )) + nextSequence += 1 + } + + // An update built outside this type reads as one run at its own place. + private mutating func materializeRuns(at updateIdx: Int) { + let update = changes[updateIdx] + guard update.runs.isEmpty, !update.cellChanges.isEmpty else { return } + changes[updateIdx].runs = [UpdateRun( + sequence: update.sequence, action: Self.ownAction(at: update.sequence), cellChanges: update.cellChanges + )] + } + + // False when the write left the cell as it was read, so no run writes it. + private mutating func writeNetCell(_ cellChange: CellChange, at updateIdx: Int) -> Bool { let rowID = changes[updateIdx].rowID guard let cellIdx = changes[updateIdx].cellChanges.firstIndex(where: { $0.columnIndex == cellChange.columnIndex }) else { changes[updateIdx].cellChanges.append(cellChange) modifiedCells[rowID, default: []].insert(cellChange.columnIndex) - return + return true } - let original = changes[updateIdx].cellChanges[cellIdx] - let merged = CellChange( - columnIndex: cellChange.columnIndex, - columnName: cellChange.columnName, - oldValue: original.oldValue, - newValue: cellChange.newValue, - oldIsAbsent: original.oldIsAbsent, - newIsAbsent: cellChange.newIsAbsent, - oldIsExpression: original.oldIsExpression, - newIsExpression: cellChange.newIsExpression - ) - changes[updateIdx].cellChanges[cellIdx] = merged + let merged = changes[updateIdx].cellChanges[cellIdx].followed(by: cellChange) + guard merged.restoresOriginal else { + changes[updateIdx].cellChanges[cellIdx] = merged + return true + } + removeUpdateCell(at: updateIdx, cellIdx: cellIdx) + return false + } + + private mutating func write(_ cellChange: CellChange, intoRun runIdx: Int, at updateIdx: Int) { + changes[updateIdx].runs[runIdx].write(cellChange) + if changes[updateIdx].runs[runIdx].cellChanges.isEmpty { + changes[updateIdx].runs.remove(at: runIdx) + } + } - guard merged.restoresOriginal else { return } + // A cell the grid shows unchanged leaves every run too, so no trigger fires for a value nobody changed. + private mutating func removeUpdateCell(at updateIdx: Int, cellIdx: Int) { + let rowID = changes[updateIdx].rowID + let columnIndex = changes[updateIdx].cellChanges[cellIdx].columnIndex changes[updateIdx].cellChanges.remove(at: cellIdx) - removeModifiedCell(rowID: rowID, columnIndex: cellChange.columnIndex) + removeModifiedCell(rowID: rowID, columnIndex: columnIndex) + for runIdx in changes[updateIdx].runs.indices.reversed() { + changes[updateIdx].runs[runIdx].cellChanges.removeAll { $0.columnIndex == columnIndex } + if changes[updateIdx].runs[runIdx].cellChanges.isEmpty { + changes[updateIdx].runs.remove(at: runIdx) + } + } if changes[updateIdx].cellChanges.isEmpty { removeChangeAt(updateIdx) } @@ -575,11 +617,7 @@ struct PendingChanges: Equatable { changes[updateIdx].cellChanges[cellIdx].oldIsExpression == restoredIsExpression else { return false } - changes[updateIdx].cellChanges.remove(at: cellIdx) - removeModifiedCell(rowID: rowID, columnIndex: columnIndex) - if changes[updateIdx].cellChanges.isEmpty { - removeChangeAt(updateIdx) - } + removeUpdateCell(at: updateIdx, cellIdx: cellIdx) return true } } diff --git a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift index 21df0ea117..5583026125 100644 --- a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift +++ b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift @@ -17,12 +17,12 @@ struct ParameterizedStatement: @unchecked Sendable { let parameters: [Any?] } -/// A statement plus the rows it is meant to touch, so a caller can hold the server to that count -/// and tell a change that became a statement from one that did not. +// The rows it should touch hold the server to a count; the changes it writes tell a written change from a dropped one. struct AttributedStatement: @unchecked Sendable { let statement: ParameterizedStatement let kind: RowWriteKind let rowIDs: [RowID] + let changeIDs: [UUID] var refusesSharedKey = false var rowCount: Int { rowIDs.count } @@ -125,16 +125,8 @@ struct SQLStatementGenerator { ).map(\.statement) } - /// The same statements, each carrying how many rows it is meant to touch. - /// - /// Emitted in the order the user made the changes, because that order can be load-bearing: a - /// row deleted to free a unique value, and a new row taking that value, only work if the - /// DELETE runs first. Grouping every INSERT ahead of every DELETE, which is what this used to - /// do, turned that save into a constraint violation and rolled the whole thing back. - /// - /// Deletes still batch, but only across a consecutive run of them. That keeps the case that - /// matters, selecting many rows and pressing Delete, on one statement, while a delete - /// separated from another delete by an insert stays on its own side of it. + // In edit order, since a row deleted to free a unique value must go before the row that takes it. Only a + // consecutive run of deletes batches. func generateAttributedStatements( from changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]], @@ -156,7 +148,8 @@ struct SQLStatementGenerator { flushDeleteRun() if let stmt = generateUpdateSQL(for: change) { statements.append(AttributedStatement( - statement: stmt, kind: .update, rowIDs: [change.rowID], refusesSharedKey: matchesEitherWay(change) + statement: stmt, kind: .update, rowIDs: [change.rowID], changeIDs: [change.id], + refusesSharedKey: matchesEitherWay(change) )) } case .insert: @@ -164,7 +157,9 @@ struct SQLStatementGenerator { guard insertedRowIDs.contains(change.rowID) else { continue } flushDeleteRun() if let stmt = generateInsertSQL(for: change, insertedRowData: insertedRowData) { - statements.append(AttributedStatement(statement: stmt, kind: .insert, rowIDs: [change.rowID])) + statements.append(AttributedStatement( + statement: stmt, kind: .insert, rowIDs: [change.rowID], changeIDs: [change.id] + )) } case .delete: // SAFETY: Verify the row is still marked as deleted @@ -553,12 +548,12 @@ struct SQLStatementGenerator { private func generateDeleteStatements(for changes: [RowChange]) -> [AttributedStatement] { let rowMatches = changes.compactMap { change in - deleteRowMatches(for: change).map { (rowID: change.rowID, matches: $0) } + deleteRowMatches(for: change).map { (rowID: change.rowID, changeID: change.id, matches: $0) } } guard !rowMatches.isEmpty else { return [] } var statements: [AttributedStatement] = [] - var chunk: [(rowID: RowID, matches: [DeleteColumnMatch])] = [] + var chunk: [(rowID: RowID, changeID: UUID, matches: [DeleteColumnMatch])] = [] var chunkParameterCount = 0 func flush() { @@ -567,6 +562,7 @@ struct SQLStatementGenerator { statement: deleteStatement(for: chunk.map(\.matches)), kind: .delete, rowIDs: chunk.map(\.rowID), + changeIDs: chunk.map(\.changeID), refusesSharedKey: chunk.contains { $0.matches.contains { matchesEitherWay($0.boundValue) } } ) ) diff --git a/apple/TablePro/Core/Coordinators/FindCoordinator.swift b/apple/TablePro/Core/Coordinators/FindCoordinator.swift index 146c4ac2af..0d6b7f247a 100644 --- a/apple/TablePro/Core/Coordinators/FindCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/FindCoordinator.swift @@ -19,7 +19,8 @@ final class FindCoordinator { var hasActiveFind: Bool { guard let find = model?.find else { return false } - return find.isVisible && !find.matches.isEmpty + // A typed term's search has no matches yet; Find Next waits for it rather than going to the editor. + return find.isVisible && (!find.matches.isEmpty || find.isAwaitingLoadedRows) } func showFind(replace: Bool = false) { diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift index 12236dcb7f..3072505f7c 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift @@ -71,6 +71,8 @@ private struct PrintedBatchAnswer: CarriesServerOutput { private struct BatchResultPlan { let queries: [[SQLStatementScanner.ExecutableStatement]?] let built: StatementRowsBuild + // Over every result the batches produce, in order: each result set, and one count result per batch with none. + let results: RunResultPlan } private struct BatchRun { @@ -271,6 +273,11 @@ extension QueryExecutionCoordinator { ) { batch in try await Self.runBatch(batch, driver: driver, failureOutput: failureOutput) } + await DatabaseManager.shared.noteSessionStatements( + prepared.prefix(outcome.ranCount(of: prepared.count)).flatMap { $0.batch.statements.map(\.sql) }, + ranOn: driver, + scope: scope + ) let sessionState = await driver.heldSessionTransactionState() return BatchRun(outcome: outcome, plan: plan, sessionState: sessionState, startState: startState) } @@ -522,47 +529,83 @@ extension QueryExecutionCoordinator { let grammar = parent.lexicalGrammar let firstNumber = parent.tabManager.tabs.first { $0.id == tabId }?.display.pinnedResultCount ?? 0 let reading = parent.tableNameReading + let runPlan = plan.results var builtIndex = 0 // Taken in the order `batchResultPlan` built them: batch by batch, each batch's result sets in order. func nextBuilt() -> StatementRows? { defer { builtIndex += 1 } return plan.built.statement(at: builtIndex, readBy: reading) } - var resultSets: [ResultSet] = [] + var own: [Int: ResultSet] = [:] + var parts: [Int: RunSummaryPart] = [:] + var candidate = 0 + // A result left out by the plan is never built; one the summary covers keeps only its counts and output. + func place(_ result: QueryResult?, count output: BatchOutput? = nil, build: (Int) -> ResultSet) { + defer { candidate += 1 } + if let position = runPlan.position(ofStatement: candidate) { + own[candidate] = build(firstNumber + position) + } else if runPlan.summarizes(statement: candidate) { + if let result { + parts[candidate] = RunSummaryPart(result) + } else if let output { + parts[candidate] = RunSummaryPart( + rowsAffected: output.rowsAffected, + executionTime: output.executionTime, + serverOutput: .none, + statusMessage: nil + ) + } + } + } for ((entry, output), queries) in zip(zip(prepared, outputs), plan.queries) { - let produced: [ResultSet] + let firstCandidate = candidate if let queries { - produced = zip(queries, output.resultSets).map { statement, result in - let replayable = entry.parameterValues == nil - && !BatchResultMapping.referencesLocalVariable(statement.sql, grammar: grammar) - return makeStatementResultSet( - result: result, - sql: statement.sql, - index: firstNumber + resultSets.count, - baseQuery: replayable ? statement.sql : nil, - tabId: tabId, - anchor: StatementAnchor(statement), - built: nextBuilt() - ) + for (statement, result) in zip(queries, output.resultSets) { + let built = nextBuilt() + place(result) { index in + let replayable = entry.parameterValues == nil + && !BatchResultMapping.referencesLocalVariable(statement.sql, grammar: grammar) + return makeStatementResultSet( + result: result, + sql: statement.sql, + index: index, + baseQuery: replayable ? statement.sql : nil, + tabId: tabId, + anchor: StatementAnchor(statement), + built: built + ) + } } } else if output.resultSets.isEmpty { - produced = output.errors.isEmpty - ? [countResultSet(output, anchor: entry.batch.anchor, index: firstNumber + resultSets.count)] - : [] + if output.errors.isEmpty { + place(nil, count: output) { index in countResultSet(output, anchor: entry.batch.anchor, index: index) } + } } else { - produced = output.resultSets.enumerated().map { offset, result in - numberedResultSet( - result, - anchor: entry.batch.anchor, - index: firstNumber + resultSets.count + offset, - tabId: tabId, - built: nextBuilt() - ) + for result in output.resultSets { + let built = nextBuilt() + place(result) { index in + numberedResultSet(result, anchor: entry.batch.anchor, index: index, tabId: tabId, built: built) + } } } - produced.last?.serverOutput = output.serverOutput - resultSets += produced + // The batch's printed output belongs to its last result, wherever that result went. + guard candidate > firstCandidate else { continue } + let last = candidate - 1 + if let resultSet = own[last] { + resultSet.serverOutput = output.serverOutput + } else if let part = parts[last] { + parts[last] = part.withServerOutput(output.serverOutput) + } + } + let resultSets = runPlan.entries.compactMap { entry -> ResultSet? in + switch entry { + case .statement(let index): + return own[index] + case .summary(let indices): + return makeRunSummaryResultSet(indices.compactMap { parts[$0] }) + } } + appendDroppedNotice(of: runPlan, to: resultSets) return resultSets } @@ -573,11 +616,31 @@ extension QueryExecutionCoordinator { let queries = zip(prepared, outputs).map { entry, output in Self.mappedQueries(of: entry, output: output, databaseType: databaseType, grammar: grammar) } - let sources = zip(queries, outputs).flatMap { mapped, output -> [(sql: String?, result: QueryResult)] in - guard let mapped else { return output.resultSets.map { (sql: nil, result: $0) } } - return zip(mapped, output.resultSets).map { (sql: $0.sql as String?, result: $1) } + // One entry per result in the order `batchResultSets` places them; a count result has no rows to build. + var shapes: [Bool] = [] + var sources: [(sql: String?, result: QueryResult)] = [] + var sourceCandidates: [Int] = [] + for (mapped, output) in zip(queries, outputs) { + if mapped == nil, output.resultSets.isEmpty { + if output.errors.isEmpty { + shapes.append(false) + } + continue + } + let pairs: [(sql: String?, result: QueryResult)] = mapped.map { mapped in + zip(mapped, output.resultSets).map { (sql: $0.sql as String?, result: $1) } + } ?? output.resultSets.map { (sql: nil, result: $0) } + for pair in pairs { + sourceCandidates.append(shapes.count) + shapes.append(!pair.result.columns.isEmpty) + sources.append(pair) + } + } + let results = RunResultPlan(producesRows: shapes) + let shown = zip(sources, sourceCandidates).map { source, candidate in + results.showsRows(ofStatement: candidate) ? source : nil } - return BatchResultPlan(queries: queries, built: await buildStatementRows(sources, tabId: tabId)) + return BatchResultPlan(queries: queries, built: await buildStatementRows(shown, tabId: tabId), results: results) } // The query each of a batch's result sets belongs to, or nil when the batch's shape does not pin them. diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift index 9c9c7f27ba..defbffd1ea 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift @@ -14,9 +14,10 @@ struct StatementRows: Sendable { } // The names hold only while the session still reads them the same way, so the build keeps the reading it used. +// A statement whose rows no result shows has nothing built. struct StatementRowsBuild: Sendable { let reading: TableNameReading - let statements: [StatementRows] + let statements: [StatementRows?] func statement(at index: Int, readBy current: TableNameReading) -> StatementRows? { guard current == reading, statements.indices.contains(index) else { return nil } @@ -125,8 +126,8 @@ extension QueryExecutionCoordinator { return TableRows.from(prepared: prepared, columnTypes: result.columnTypes, columnOrigins: result.columnOrigins) } - // A nil statement is a result no statement names, which reads no table. - func buildStatementRows(_ statements: [(sql: String?, result: QueryResult)], tabId: UUID) async -> StatementRowsBuild { + // A nil sql is a result no statement names, which reads no table. A nil entry builds nothing. + func buildStatementRows(_ statements: [(sql: String?, result: QueryResult)?], tabId: UUID) async -> StatementRowsBuild { let reveals = parent.tabManager.tabs.first { $0.id == tabId }?.revealsMaskedValues == true return await Self.statementRows( statements, @@ -139,11 +140,13 @@ extension QueryExecutionCoordinator { // this build runs to the end. @concurrent nonisolated static func statementRows( - _ statements: [(sql: String?, result: QueryResult)], + _ statements: [(sql: String?, result: QueryResult)?], reading: TableNameReading, policy: MaskingPolicy ) async -> StatementRowsBuild { - StatementRowsBuild(reading: reading, statements: statements.map { sql, result in + StatementRowsBuild(reading: reading, statements: statements.map { entry -> StatementRows? in + guard let entry else { return nil } + let (sql, result) = entry let tableName = sql.flatMap(reading.tableName(in:)) let shape = TableRows.RowShape( columns: result.columns, @@ -160,6 +163,50 @@ extension QueryExecutionCoordinator { }) } + static let summaryMessageLimit = 5 + + // The statements of one run that returned no rows, as one result: rows affected and time added + // up, server output in the order it was printed. + func makeRunSummaryResultSet(_ parts: [RunSummaryPart]) -> ResultSet { + let count = parts.count.formatted() + let resultSet = ResultSet(label: String(format: String(localized: "%@ Statements"), count)) + resultSet.rowsAffected = parts.reduce(0) { $0 + $1.rowsAffected } + resultSet.executionTime = parts.reduce(0) { $0 + $1.executionTime } + // A driver's own message (a warning, a server reply) stays, each once, after the count; a long script's + // varied replies are cut to the first few. + var seen: Set = [] + var messages: [String] = [] + var unshown = 0 + for message in parts.compactMap(\.statusMessage) where !message.isEmpty && seen.insert(message).inserted { + if messages.count < Self.summaryMessageLimit { + messages.append(message) + } else { + unshown += 1 + } + } + if unshown == 1 { + messages.append(String(localized: "1 more message.")) + } else if unshown > 1 { + messages.append(String(format: String(localized: "%@ more messages."), unshown.formatted())) + } + resultSet.statusMessage = ([String(format: String(localized: "Ran %@ statements that returned no rows."), count)] + messages) + .joined(separator: " ") + resultSet.serverOutput = PluginServerOutput( + lines: parts.flatMap(\.serverOutput.lines), + isTruncated: parts.contains { $0.serverOutput.isTruncated } + ) + return resultSet + } + + func appendDroppedNotice(of plan: RunResultPlan, to resultSets: [ResultSet]) { + guard let notice = plan.droppedNotice else { return } + for resultSet in resultSets { + resultSet.statusMessage = [resultSet.statusMessage, notice] + .compactMap { $0?.isEmpty == false ? $0 : nil } + .joined(separator: " ") + } + } + /// Each statement in a multi-statement run targets its own table, so each result carries its /// own identity. Nothing fetches key columns for these statements, so the origin says so and /// the result is read-only until something does. Inheriting the previous run's keys made an diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift index c0bcf998c7..6c21ed18a1 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift @@ -215,6 +215,8 @@ extension QueryExecutionCoordinator { route: DatabaseManager.shared.executionRoute(for: scope), cancellation: .cancellableRead(lease) ) { [queryExecutor = parent.queryExecutor, boundValues] driver in + // Before it runs, so a statement that fails or a connection lost under it is still noted. + await DatabaseManager.shared.noteSessionStatements([statement.sql], ranOn: driver, scope: scope) let fetched = try await queryExecutor.executeQuery( driver: driver, sql: statement.sql, @@ -433,7 +435,13 @@ extension QueryExecutionCoordinator { case .completed(let ran), .cancelled(let ran), .failed(let ran, _, _): results = ran } - let built = await buildStatementRows(zip(sent, results).map { (sql: $0.originalSQL as String?, result: $1) }, tabId: tabId) + let resultPlan = RunResultPlan(producesRows: results.map { !$0.columns.isEmpty }) + let built = await buildStatementRows( + zip(sent, results).enumerated().map { index, pair in + resultPlan.showsRows(ofStatement: index) ? (sql: pair.0.originalSQL as String?, result: pair.1) : nil + }, + tabId: tabId + ) switch outcome { case .cancelled(let results): @@ -563,6 +571,10 @@ extension QueryExecutionCoordinator { ) } } + let sent = await running.prepared + await DatabaseManager.shared.noteSessionStatements( + sent.prefix(outcome.ranCount(of: sent.count)).map(\.sentSQL), ranOn: driver, scope: scope + ) guard sessionPlan == .sessionTransaction else { let endState: PluginSessionTransactionState = asksEndState ? await driver.heldSessionTransactionState() @@ -671,25 +683,35 @@ extension QueryExecutionCoordinator { ) } + // Only the statements the plan keeps become results, so a long script builds a few, not one per statement. private func statementResultSets(_ history: ExecutedStatementHistory, tabId: UUID, built: StatementRowsBuild) -> [ResultSet] { let firstNumber = parent.tabManager.tabs.first { $0.id == tabId }?.display.pinnedResultCount ?? 0 let reading = parent.tableNameReading - return zip(history.prepared, history.results).enumerated().map { index, pair in - let (statement, result) = pair - return makeStatementResultSet( - result: result, - sql: statement.originalSQL, - index: firstNumber + index, - baseQuery: statement.executableSQL, - baseQueryParameterValues: statement.parameterValues?.map { $0 as? String }, - namedParameterStatement: statement.parameterValues == nil - ? nil - : NamedParameterStatement(sql: statement.originalSQL, parameters: history.parameters), - tabId: tabId, - anchor: statement.anchor, - built: built.statement(at: index, readBy: reading) - ) + let ran = Array(zip(history.prepared, history.results)) + let plan = RunResultPlan(producesRows: ran.map { !$0.1.columns.isEmpty }) + let resultSets = plan.entries.enumerated().map { position, entry -> ResultSet in + switch entry { + case .summary(let indices): + return makeRunSummaryResultSet(indices.map { RunSummaryPart(ran[$0].1) }) + case .statement(let index): + let (statement, result) = ran[index] + return makeStatementResultSet( + result: result, + sql: statement.originalSQL, + index: firstNumber + position, + baseQuery: statement.executableSQL, + baseQueryParameterValues: statement.parameterValues?.map { $0 as? String }, + namedParameterStatement: statement.parameterValues == nil + ? nil + : NamedParameterStatement(sql: statement.originalSQL, parameters: history.parameters), + tabId: tabId, + anchor: statement.anchor, + built: built.statement(at: index, readBy: reading) + ) + } } + appendDroppedNotice(of: plan, to: resultSets) + return resultSets } /// A commit whose connection died before the answer leaves every statement of the batch in a diff --git a/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift b/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift index 9cf2f91330..f43a74ee09 100644 --- a/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift @@ -296,46 +296,48 @@ final class RowEditingCoordinator: ObservableObject { var edits: [(row: Int, column: Int, value: PluginCellValue)] = [] var removedCells: Set = [] - for (rowID, value) in valuesByRow { - guard let storageRow = tableRows.index(of: rowID) else { continue } - let row = tableRows.rows[storageRow] - guard row.unloadedByteCount(columnIndex) == nil else { continue } - let values = Array(row.values) - let removesField = removedRowIDs.contains(rowID) && parent.changeManager.supportsFieldRemoval - let absence = FieldAbsence( - wasAbsent: row.isAbsent(columnIndex), isAbsent: removesField, originalRow: row.absentColumns - ) - let isExpression = expressionRowIDs.contains(rowID) && !removesField - let wasExpression = parent.changeManager.isExpression(rowID: rowID, columnIndex: columnIndex) - guard values.indices.contains(columnIndex), - values[columnIndex] != value || absence.wasAbsent != absence.isAbsent - || wasExpression != isExpression else { continue } - if continuity == .typing { - parent.changeManager.recordTypedCellChange( - rowID: rowID, - columnIndex: columnIndex, - columnName: columnName, - oldValue: values[columnIndex], - newValue: value, - originalRow: values, - absence: absence, - isExpression: isExpression - ) - } else { - parent.changeManager.recordCellChange( - rowID: rowID, - columnIndex: columnIndex, - columnName: columnName, - oldValue: values[columnIndex], - newValue: value, - originalRow: values, - absence: absence, - isExpression: isExpression + parent.changeManager.recordAsOneAction { + for (rowID, value) in valuesByRow { + guard let storageRow = tableRows.index(of: rowID) else { continue } + let row = tableRows.rows[storageRow] + guard row.unloadedByteCount(columnIndex) == nil else { continue } + let values = Array(row.values) + let removesField = removedRowIDs.contains(rowID) && parent.changeManager.supportsFieldRemoval + let absence = FieldAbsence( + wasAbsent: row.isAbsent(columnIndex), isAbsent: removesField, originalRow: row.absentColumns ) - } - edits.append((row: storageRow, column: columnIndex, value: value)) - if removesField { - removedCells.insert(CellPosition(row: storageRow, column: columnIndex)) + let isExpression = expressionRowIDs.contains(rowID) && !removesField + let wasExpression = parent.changeManager.isExpression(rowID: rowID, columnIndex: columnIndex) + guard values.indices.contains(columnIndex), + values[columnIndex] != value || absence.wasAbsent != absence.isAbsent + || wasExpression != isExpression else { continue } + if continuity == .typing { + parent.changeManager.recordTypedCellChange( + rowID: rowID, + columnIndex: columnIndex, + columnName: columnName, + oldValue: values[columnIndex], + newValue: value, + originalRow: values, + absence: absence, + isExpression: isExpression + ) + } else { + parent.changeManager.recordCellChange( + rowID: rowID, + columnIndex: columnIndex, + columnName: columnName, + oldValue: values[columnIndex], + newValue: value, + originalRow: values, + absence: absence, + isExpression: isExpression + ) + } + edits.append((row: storageRow, column: columnIndex, value: value)) + if removesField { + removedCells.insert(CellPosition(row: storageRow, column: columnIndex)) + } } } guard !edits.isEmpty else { return } @@ -351,21 +353,23 @@ final class RowEditingCoordinator: ObservableObject { let tableRows = parent.tabSessionRegistry.tableRows(for: tabId) var edits: [(row: Int, column: Int, value: PluginCellValue)] = [] var removedCells: Set = [] - for edit in resolved { - guard let storageRow = tableRows.index(of: edit.rowID) else { continue } - changeManager.recordCellChange( - rowID: edit.rowID, - columnIndex: edit.dataIndex, - columnName: edit.columnName, - oldValue: edit.oldValue, - newValue: edit.newValue, - originalRow: edit.originalRow, - absence: edit.absence, - isExpression: edit.isExpression - ) - edits.append((row: storageRow, column: edit.dataIndex, value: edit.newValue)) - if edit.absence.isAbsent { - removedCells.insert(CellPosition(row: storageRow, column: edit.dataIndex)) + changeManager.recordAsOneAction { + for edit in resolved { + guard let storageRow = tableRows.index(of: edit.rowID) else { continue } + changeManager.recordCellChange( + rowID: edit.rowID, + columnIndex: edit.dataIndex, + columnName: edit.columnName, + oldValue: edit.oldValue, + newValue: edit.newValue, + originalRow: edit.originalRow, + absence: edit.absence, + isExpression: edit.isExpression + ) + edits.append((row: storageRow, column: edit.dataIndex, value: edit.newValue)) + if edit.absence.isAbsent { + removedCells.insert(CellPosition(row: storageRow, column: edit.dataIndex)) + } } } guard !edits.isEmpty else { return false } diff --git a/apple/TablePro/Core/Coordinators/RunResultPlan.swift b/apple/TablePro/Core/Coordinators/RunResultPlan.swift new file mode 100644 index 0000000000..45ec117b36 --- /dev/null +++ b/apple/TablePro/Core/Coordinators/RunResultPlan.swift @@ -0,0 +1,103 @@ +// +// RunResultPlan.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +// Which statements of a run become results of their own. Two or more statements that return no +// rows share one summary, placed where the last of them ran, so the result of the last statement +// stays the last result. Only the last `maxRowResults` results that return rows are kept. +struct RunResultPlan: Equatable, Sendable { + static let maxRowResults = 100 + + enum Entry: Equatable, Sendable { + case statement(Int) + case summary([Int]) + } + + let entries: [Entry] + let rowResultCount: Int + let keptRowResultCount: Int + private let ownPositions: [Int: Int] + private let summarized: Set + + var droppedRowResultCount: Int { rowResultCount - keptRowResultCount } + + init(producesRows: [Bool], maxRowResults: Int = Self.maxRowResults) { + let rowIndices = producesRows.indices.filter { producesRows[$0] } + let kept = Set(rowIndices.suffix(max(0, maxRowResults))) + let summarized = producesRows.indices.filter { !producesRows[$0] } + let collapses = summarized.count >= 2 + + var entries: [Entry] = [] + for index in producesRows.indices { + if producesRows[index] { + if kept.contains(index) { entries.append(.statement(index)) } + } else if !collapses { + entries.append(.statement(index)) + } else if index == summarized.last { + entries.append(.summary(summarized)) + } + } + self.entries = entries + var ownPositions: [Int: Int] = [:] + var covered: Set = [] + for (position, entry) in entries.enumerated() { + switch entry { + case .statement(let index): ownPositions[index] = position + case .summary(let indices): covered.formUnion(indices) + } + } + self.ownPositions = ownPositions + self.summarized = covered + rowResultCount = rowIndices.count + keptRowResultCount = kept.count + } + + // True when the statement's rows are shown on a result of its own, so they are worth building. + func showsRows(ofStatement index: Int) -> Bool { + ownPositions[index] != nil + } + + // Where the statement's own result sits among the run's results. + func position(ofStatement index: Int) -> Int? { + ownPositions[index] + } + + func summarizes(statement index: Int) -> Bool { + summarized.contains(index) + } + + var droppedNotice: String? { + guard droppedRowResultCount > 0 else { return nil } + return String( + format: String(localized: "Showing the last %1$@ of %2$@ results that returned rows."), + keptRowResultCount.formatted(), + rowResultCount.formatted() + ) + } +} + +struct RunSummaryPart { + let rowsAffected: Int + let executionTime: TimeInterval + let serverOutput: PluginServerOutput + let statusMessage: String? + + func withServerOutput(_ output: PluginServerOutput) -> RunSummaryPart { + RunSummaryPart(rowsAffected: rowsAffected, executionTime: executionTime, serverOutput: output, statusMessage: statusMessage) + } +} + +extension RunSummaryPart { + init(_ result: QueryResult) { + self.init( + rowsAffected: result.rowsAffected, + executionTime: result.executionTime, + serverOutput: result.serverOutput, + statusMessage: result.statusMessage + ) + } +} diff --git a/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift b/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift index d718cdff74..af13e95a3d 100644 --- a/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift +++ b/apple/TablePro/Core/DataGrid/DataGridModel+Find.swift @@ -95,14 +95,7 @@ extension DataGridModel { searchDataset(revealsFirstMatch: true) return } - let source = findSource - let result = GridFindEngine.search(find.term, options: find.options, in: source) - let current = GridFindEngine.index( - atOrAfter: selection.activeCell, - in: GridFindEngine.positions(of: result.matches, in: source) - ) - find.setResults(result.matches, current: current, problem: result.problem, basis: findBasis(of: source)) - revealCurrentFindMatch() + searchLoadedRows(GridFindState.Landing(revealsFirstMatch: true, cursorAtStart: activeCellIdentity)) } // The rows changed under the bar: keep the current match when it survives and leave the selection alone. @@ -123,33 +116,24 @@ extension DataGridModel { searchDataset(revealsFirstMatch: false) return } - let source = findSource - let basis = findBasis(of: source) - let previous = find.currentMatch - let result: GridFindEngine.Result - if let changes, find.loadedRowsBasis == basis, - let refreshed = GridFindEngine.refreshed( - find.matches, - writtenCells: changes.writtenCells, - addedRows: changes.addedRows, - term: find.term, - options: find.options, - in: source - ) { - result = refreshed - } else { - result = GridFindEngine.search(find.term, options: find.options, in: source) - } - let current: Int? - if let previous, let kept = result.matches.firstIndex(of: previous) { - current = kept - } else { - current = GridFindEngine.index( - atOrAfter: selection.activeCell, - in: GridFindEngine.positions(of: result.matches, in: source) - ) + // A running search leaves no basis, so a change meanwhile searches again. + if let changes { + let source = findSource + let basis = findBasis(of: source) + if find.loadedRowsBasis == basis, + let refreshed = GridFindEngine.refreshed( + find.matches, + writtenCells: changes.writtenCells, + addedRows: changes.addedRows, + term: find.term, + options: find.options, + in: source + ) { + landLoadedRowsResult(refreshed, landing: GridFindState.Landing(), basis: basis, source: source) + return + } } - find.setResults(result.matches, current: current, problem: result.problem, basis: basis) + searchLoadedRows(GridFindState.Landing()) } // The owner's rows changed (an edit, an undo, a filter): search them again, keeping the current match. @@ -170,11 +154,11 @@ extension DataGridModel { find.stopDatasetSearch() } - // Returns once no whole-dataset search is waiting or running. + // Returns once no search of the loaded rows or the whole dataset is waiting or running. func findSearchSettled() async { - while let task = find.datasetSearch { + while let task = find.datasetSearch ?? find.loadedRowsSearch { await task.value - if find.datasetSearch == task { return } + if find.datasetSearch == task || find.loadedRowsSearch == task { return } } } @@ -215,7 +199,12 @@ extension DataGridModel { // MARK: - Replacing func replaceCurrentMatch() { - guard canReplaceInFind, find.problem == nil, let match = find.currentMatch, commitEditing() else { return } + guard canReplaceInFind else { return } + // Replace acts on the match the bar is about to show. + if find.isAwaitingLoadedRows { + searchLoadedRowsNow() + } + guard find.problem == nil, let match = find.currentMatch, commitEditing() else { return } if let finder = datasetFinder { replaceDatasetMatch(match, finder: finder) return @@ -252,10 +241,12 @@ extension DataGridModel { return nil } let source = findSource - let result = GridFindEngine.search(find.term, options: find.options, in: source) - guard result.problem == nil else { return nil } + if find.isAwaitingLoadedRows || find.loadedRowsBasis != findBasis(of: source) { + searchLoadedRowsNow() + } + guard find.problem == nil else { return nil } let replacement = GridFindEngine.replacements( - for: result.matches, + for: find.matches, term: find.term, options: find.options, template: find.replacement, @@ -308,9 +299,16 @@ extension DataGridModel { stepDatasetFind(finder, forward: forward) return } - if find.matches.isEmpty { - refreshFind() + if find.isAwaitingLoadedRows { + find.deferStep(forward: forward) + } else if find.matches.isEmpty { + searchLoadedRows(GridFindState.Landing(steps: forward ? 1 : -1)) + } else { + stepThroughFindMatches(forward: forward) } + } + + private func stepThroughFindMatches(forward: Bool) { guard !find.matches.isEmpty else { return } let index = GridFindEngine.step( from: find.currentIndex, @@ -322,6 +320,78 @@ extension DataGridModel { revealCurrentFindMatch() } + // MARK: - Loaded rows + + // Off the main actor. A term with nothing to scan is answered here, as is an invalid pattern. + private func searchLoadedRows(_ landing: GridFindState.Landing) { + let source = findSource + let basis = findBasis(of: source) + let term = find.term + let options = find.options + let generation = find.beginLoadedRowsSearch(landing) + if let answered = GridFindEngine.resultWithoutScanning(term, options: options, in: source) { + if let merged = find.endLoadedRowsSearch(generation: generation) { + landLoadedRowsResult(answered, landing: merged, basis: basis, source: source) + } + return + } + let dateFormat = DateFormattingService.shared.currentFormat + find.attachLoadedRowsSearch(Task { [weak self] in + let result = await GridFindEngine.searchOffTheMainActor(term, options: options, in: source, dateFormat: dateFormat) + self?.loadedRowsSearchDidFinish(result, basis: basis, generation: generation) + }, generation: generation) + } + + private func loadedRowsSearchDidFinish(_ result: GridFindEngine.Result?, basis: GridFindEngine.Basis, generation: Int) { + guard let landing = find.endLoadedRowsSearch(generation: generation), let result, find.isVisible else { return } + let source = findSource + // A setting the cell text follows changed while it ran, and nothing refreshed the bar. + guard findBasis(of: source) == basis else { + searchLoadedRows(landing) + return + } + landLoadedRowsResult(result, landing: landing, basis: basis, source: source) + } + + // On the main actor, landing as a running search would have. + private func searchLoadedRowsNow() { + let source = findSource + let basis = findBasis(of: source) + let generation = find.beginLoadedRowsSearch(GridFindState.Landing()) + guard let landing = find.endLoadedRowsSearch(generation: generation) else { return } + let result = GridFindEngine.search(find.term, options: find.options, in: source) + landLoadedRowsResult(result, landing: landing, basis: basis, source: source) + } + + private func landLoadedRowsResult( + _ result: GridFindEngine.Result, + landing: GridFindState.Landing, + basis: GridFindEngine.Basis, + source: GridFindEngine.Source + ) { + let current: Int? + if !landing.revealsFirstMatch, let previous = find.currentMatch, let kept = result.matches.firstIndex(of: previous) { + current = kept + } else { + current = GridFindEngine.index( + atOrAfter: selection.activeCell, + in: GridFindEngine.positions(of: result.matches, in: source) + ) + } + find.setResults(result.matches, current: current, problem: result.problem, basis: basis) + if landing.revealsFirstMatch, landing.cursorAtStart == activeCellIdentity { + revealCurrentFindMatch() + } + for _ in 0.. 0) + } + } + + // By identity, so a row change that moves the cursor's position is not a move by the user. + private var activeCellIdentity: GridFindState.Match? { + activeCell.map { GridFindState.Match(rowID: $0.rowID, columnID: $0.column.id) } + } + // The match becomes the active cell, so Copy, Return and the arrow keys act on the found column. private func revealCurrentFindMatch() { guard let match = find.currentMatch else { return } diff --git a/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift b/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift index 730159e873..0890bd9e78 100644 --- a/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift +++ b/apple/TablePro/Core/DataGrid/Engine/GridFindEngine.swift @@ -120,7 +120,7 @@ struct GridFindMatcher { @MainActor enum GridFindEngine { - struct Source { + struct Source: Sendable { let rows: TableRows let order: DataGridRowOrder let columns: DataGridColumnSet @@ -129,11 +129,18 @@ enum GridFindEngine { var revealsInvisibleCharacters = false } - struct Result: Equatable { + struct Result: Equatable, Sendable { var matches: [GridFindState.Match] = [] var problem: String? } + private enum Plan { + case answered(Result) + case scan(GridFindMatcher, columns: [DataGridColumn]) + } + + nonisolated static let cancellationInterval = 4_096 + // Everything a loaded-rows search read besides the rows themselves, the settings the cell text follows included. struct Basis: Equatable { let term: String @@ -173,7 +180,7 @@ enum GridFindEngine { var cells: Int { edits.count } } - static var invalidPatternProblem: String { + nonisolated static var invalidPatternProblem: String { String(localized: "Invalid regular expression") } @@ -187,22 +194,39 @@ enum GridFindEngine { } static func search(_ term: String, options: GridFindState.Options, in source: Source) -> Result { - let matcher: GridFindMatcher? - do { - matcher = try GridFindMatcher.make(term: term, options: options) - } catch { - return Result(problem: invalidPatternProblem) + search(term, options: options, in: source, dates: DateFormattingService.shared.renderer, isCancelled: { false }) ?? Result() + } + + // Nil once cancelled, which is asked every 4,096 cells. + nonisolated static func search( + _ term: String, + options: GridFindState.Options, + in source: Source, + dates: DateDisplayRenderer, + isCancelled: () -> Bool + ) -> Result? { + let matcher: GridFindMatcher + let searched: [DataGridColumn] + switch plan(term, options: options, in: source) { + case .answered(let result): + return result + case .scan(let planned, let columns): + matcher = planned + searched = columns } - guard let matcher else { return Result() } - let searched = searchedColumns(options: options, in: source) - guard !searched.isEmpty else { return Result() } var found: [GridFindState.Match] = [] + var budget = 0 let rowCount = source.order.count(in: source.rows) for displayRow in 0.. Result? { + search( + term, + options: options, + in: source, + dates: DateDisplayRenderer(format: dateFormat), + isCancelled: { Task.isCancelled } + ) + } + + // What a search answers without reading a cell: an invalid pattern, a term that finds nothing, no column to search. + nonisolated static func resultWithoutScanning(_ term: String, options: GridFindState.Options, in source: Source) -> Result? { + guard case .answered(let result) = plan(term, options: options, in: source) else { return nil } + return result + } + + nonisolated private static func plan(_ term: String, options: GridFindState.Options, in source: Source) -> Plan { + let matcher: GridFindMatcher? + do { + matcher = try GridFindMatcher.make(term: term, options: options) + } catch { + return .answered(Result(problem: invalidPatternProblem)) + } + guard let matcher else { return .answered(Result()) } + let searched = searchedColumns(options: options, in: source) + guard !searched.isEmpty else { return .answered(Result()) } + return .scan(matcher, columns: searched) + } + // What `search` finds after only these cells (by data index) were written and these rows added, others having // left, on the same basis. Nil when the term cannot match, which `search` answers as cheaply. static func refreshed( @@ -273,7 +333,7 @@ enum GridFindEngine { } // A scope naming a column that is no longer presented searches every presented column. - static func searchedColumns(options: GridFindState.Options, in source: Source) -> [DataGridColumn] { + nonisolated static func searchedColumns(options: GridFindState.Options, in source: Source) -> [DataGridColumn] { let candidates: [DataGridColumn] if let scope = options.columnScope, let scoped = source.columns.column(id: scope) { candidates = [scoped] @@ -285,8 +345,17 @@ enum GridFindEngine { } } - // Nil for NULL, an empty cell, and bytes the column's format could not decode, which show as hex. static func searchableText(in row: Row, column: DataGridColumn, source: Source) -> String? { + searchableText(in: row, column: column, source: source, dates: DateFormattingService.shared.renderer) + } + + // Nil for NULL, an empty cell, and bytes the column's format could not decode, which show as hex. + nonisolated static func searchableText( + in row: Row, + column: DataGridColumn, + source: Source, + dates: DateDisplayRenderer + ) -> String? { let dataIndex = column.dataIndex guard dataIndex < row.values.count else { return nil } let rawValue = row.values[dataIndex] @@ -305,7 +374,8 @@ enum GridFindEngine { rawValue, columnType: column.type, displayFormat: format, - databaseType: source.databaseType + databaseType: source.databaseType, + dates: dates ) guard !shown.isEmpty else { return nil } return source.revealsInvisibleCharacters ? shown : shown.shownWithoutMarks @@ -439,7 +509,7 @@ enum GridFindEngine { return String(format: String(localized: "Replaced %1$@ in %2$@."), matchPhrase, cellPhrase) } - private static func displayFormat(for column: DataGridColumn, in source: Source) -> ValueDisplayFormat? { + nonisolated private static func displayFormat(for column: DataGridColumn, in source: Source) -> ValueDisplayFormat? { column.dataIndex < source.displayFormats.count ? source.displayFormats[column.dataIndex] : nil } diff --git a/apple/TablePro/Core/DataGrid/GridFindState.swift b/apple/TablePro/Core/DataGrid/GridFindState.swift index ba2c2f68b0..e297c1740d 100644 --- a/apple/TablePro/Core/DataGrid/GridFindState.swift +++ b/apple/TablePro/Core/DataGrid/GridFindState.swift @@ -3,18 +3,28 @@ import Foundation @MainActor final class GridFindState: ObservableObject { - struct Options: Equatable { + struct Options: Equatable, Sendable { var matchesCase = false var matchesWholeWord = false var usesRegularExpression = false var columnScope: DataGridColumnID? } - struct Match: Hashable { + struct Match: Hashable, Sendable { let rowID: RowID let columnID: DataGridColumnID } + // What a search of the loaded rows does once its matches land. + struct Landing: Equatable { + // A typed term selects the first match from the cursor; a refresh keeps the current match. + var revealsFirstMatch = false + // The active cell when the typed term's search began. A user who moved it since keeps their place. + var cursorAtStart: Match? + // Find Next presses minus Find Previous presses while the search ran. + var steps = 0 + } + @Published var isVisible = false @Published var showsReplace = false @Published var term = "" @@ -26,6 +36,8 @@ final class GridFindState: ObservableObject { @Published private(set) var isSearching = false // The user stopped a whole-dataset search before it finished. @Published private(set) var searchStopped = false + // An empty match list means nothing yet; unlike isSearching it shows no progress. + @Published private(set) var isAwaitingLoadedRows = false weak var datasetFinder: (any DataGridDatasetFinding)? private(set) var datasetQuery: GridDatasetFindQuery? @@ -33,6 +45,16 @@ final class GridFindState: ObservableObject { private var datasetGeneration = 0 // What the matches were searched on, while they came from the loaded rows; nil once anything else set them. private(set) var loadedRowsBasis: GridFindEngine.Basis? + // Runs off the main actor and sets no isSearching: a refresh keeps the matches shown until the new ones land. + private(set) var loadedRowsSearch: Task? + private var loadedRowsGeneration = 0 + private var loadedRowsLanding = Landing() + + // A closed grid's scan would otherwise run to the end and hold its rows until then. + deinit { + loadedRowsSearch?.cancel() + datasetSearch?.cancel() + } var currentMatch: Match? { guard let currentIndex, matches.indices.contains(currentIndex) else { return nil } @@ -60,6 +82,7 @@ final class GridFindState: ObservableObject { func reset() { cancelDatasetSearch() + cancelLoadedRowsSearch() datasetQuery = nil searchStopped = false term = "" @@ -70,11 +93,74 @@ final class GridFindState: ObservableObject { problem = nil } + // MARK: - Loaded-rows search + + // The search hands this generation back when it lands. Any other search stops, and a running one this + // replaces hands on what it was to do on landing. + func beginLoadedRowsSearch(_ landing: Landing) -> Int { + var merged = landing + if loadedRowsSearch != nil { + if !landing.revealsFirstMatch, loadedRowsLanding.revealsFirstMatch { + merged.revealsFirstMatch = true + merged.cursorAtStart = loadedRowsLanding.cursorAtStart + } + merged.steps += loadedRowsLanding.steps + } + cancelLoadedRowsSearch() + if datasetSearch != nil { + cancelDatasetSearch() + } + loadedRowsLanding = merged + loadedRowsBasis = nil + // The old term's count and current match would read as the new term's until the scan lands. + if merged.revealsFirstMatch { + matches = [] + currentIndex = nil + problem = nil + } + return loadedRowsGeneration + } + + func attachLoadedRowsSearch(_ task: Task, generation: Int) { + guard generation == loadedRowsGeneration else { + task.cancel() + return + } + loadedRowsSearch = task + // Set here, not in begin, so a term answered without scanning never publishes it. + isAwaitingLoadedRows = true + } + + // Nil when a later search, a reset or a dataset search took this one's place. + func endLoadedRowsSearch(generation: Int) -> Landing? { + guard generation == loadedRowsGeneration else { return nil } + let landing = loadedRowsLanding + loadedRowsSearch = nil + if isAwaitingLoadedRows { isAwaitingLoadedRows = false } + loadedRowsLanding = Landing() + loadedRowsGeneration &+= 1 + return landing + } + + func deferStep(forward: Bool) { + loadedRowsLanding.steps += forward ? 1 : -1 + } + + func cancelLoadedRowsSearch() { + loadedRowsSearch?.cancel() + loadedRowsSearch = nil + // @Published fires on every write, so skip the redraw when nothing changes. + if isAwaitingLoadedRows { isAwaitingLoadedRows = false } + loadedRowsGeneration &+= 1 + loadedRowsLanding = Landing() + } + // MARK: - Whole-dataset search // The search hands this generation back when it finishes; any later start or cancel makes it stale. func beginDatasetSearch(_ query: GridDatasetFindQuery?, clearsResults: Bool) -> Int { cancelDatasetSearch() + cancelLoadedRowsSearch() datasetQuery = query searchStopped = false problem = nil diff --git a/apple/TablePro/Core/DataWrite/DataWritePlan.swift b/apple/TablePro/Core/DataWrite/DataWritePlan.swift index 0c0e29ccdd..f67c4c5bf5 100644 --- a/apple/TablePro/Core/DataWrite/DataWritePlan.swift +++ b/apple/TablePro/Core/DataWrite/DataWritePlan.swift @@ -66,6 +66,10 @@ struct RowWriteOperation: Codable, Sendable, Equatable { // What the driver found the row by when it was read, for an update or a delete. A rewind writes to that row and // no other. Nil in a record saved before it was kept, and on an engine that keeps none. var rowLocator: String? + // Where the row's first write ran in the save. Nil in a record saved before it was kept, which runs by list order. + var order: Int? + // Only for an update written in two or more steps. An older build ignores it and rewinds the net images. + var steps: [RowWriteStep]? var isReversible: Bool { refusal == nil } @@ -85,6 +89,82 @@ struct RowWriteOperation: Codable, Sendable, Equatable { guard let image else { return [] } return Set(image.indices.filter { image[$0].isNull }) } + + // The row as each step left it, applying the steps to the pre-image in turn. + func stepImages() -> [RowWriteStepImage]? { + guard let steps, var row = preImage else { return nil } + var absent = absentColumnsBeforeWrite + return steps.map { step in + for (offset, column) in step.columns.enumerated() { + if row.indices.contains(column), step.after.indices.contains(offset) { + row[column] = step.after[offset] + } + if step.afterAbsent.contains(column) { + absent.insert(column) + } else { + absent.remove(column) + } + } + return RowWriteStepImage(step: step, after: row, absentAfter: absent) + } + } +} + +// `before` and `after` line up with `columns`, which are indices into the operation's columns. +struct RowWriteStep: Codable, Sendable, Equatable { + let order: Int + let columns: [Int] + let before: [PluginCellValue] + let after: [PluginCellValue] + let beforeAbsent: Set + let afterAbsent: Set + + init( + order: Int, + columns: [Int], + before: [PluginCellValue], + after: [PluginCellValue], + beforeAbsent: Set = [], + afterAbsent: Set = [] + ) { + self.order = order + self.columns = columns + self.before = before + self.after = after + self.beforeAbsent = beforeAbsent + self.afterAbsent = afterAbsent + } + + init(_ step: RowChange) { + self.init( + order: step.sequence, + columns: step.cellChanges.map(\.columnIndex), + before: step.cellChanges.map(\.oldValue), + after: step.cellChanges.map(\.newValue), + beforeAbsent: Set(step.cellChanges.filter(\.oldIsAbsent).map(\.columnIndex)), + afterAbsent: Set(step.cellChanges.filter(\.newIsAbsent).map(\.columnIndex)) + ) + } + + var inverted: RowWriteStep { + RowWriteStep( + order: order, columns: columns, before: after, after: before, + beforeAbsent: afterAbsent, afterAbsent: beforeAbsent + ) + } + + func reordered(_ order: Int) -> RowWriteStep { + RowWriteStep( + order: order, columns: columns, before: before, after: after, + beforeAbsent: beforeAbsent, afterAbsent: afterAbsent + ) + } +} + +struct RowWriteStepImage { + let step: RowWriteStep + let after: [PluginCellValue] + let absentAfter: Set } // A driver's `PluginRowLookup`, bound for the executor. diff --git a/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift b/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift index 3686d2ca50..dfca08ec8c 100644 --- a/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift +++ b/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift @@ -134,13 +134,23 @@ struct RewindExecutor { ) } - // In the order the restore ran them, since undoing the restore runs its record backwards. + // In the order the restore ran them, each step ordered by when the restore ran it, since undoing the restore + // runs its record backwards. static func inverseOperations(of plan: RewindPlan) -> [RowWriteOperation] { - plan.rows.filter { $0.outcome.restores }.reversed().map { row -> RowWriteOperation in - let source = row.operation + let restored = plan.rows.filter { $0.outcome.restores }.map(\.operation) + var rank: [RewindPlanner.RunEntry: Int] = [:] + for (position, entry) in RewindPlanner.runOrder(of: restored).enumerated() { + rank[entry] = position + } + return restored.enumerated().reversed().map { index, source -> RowWriteOperation in let kind = source.kind.inverted // The same stored refusal a save through a view writes, so a build without the view flag refuses too. let throughView = plan.record.writtenThroughView == true && kind == .delete + let steps = source.steps.map { sourceSteps in + sourceSteps.indices.reversed().map { stepIndex in + sourceSteps[stepIndex].inverted.reordered(rank[.init(operation: index, step: stepIndex)] ?? 0) + } + } return RowWriteOperation( kind: kind, target: source.target, @@ -152,7 +162,9 @@ struct RewindExecutor { refusal: throughView ? .identityNotPreservable : nil, preImageAbsentColumns: source.postImageAbsentColumns, postImageAbsentColumns: source.preImageAbsentColumns, - rowLocator: source.rowLocator + rowLocator: source.rowLocator, + order: steps?.first?.order ?? rank[.init(operation: index, step: nil)], + steps: steps ) } } diff --git a/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift b/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift index 361fb6bc40..7b6b7116d3 100644 --- a/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift +++ b/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift @@ -56,26 +56,12 @@ struct RewindPlanner { let byLocator = indexByLocator(currentRows) let matchesByLocator = locatorQuery() != nil || !byLocator.isEmpty - var rows: [RewindRowPlan] = [] - var restorable: [RowWriteOperation] = [] - for operation in record.operations { + var found: [FoundRow] = record.operations.map { operation in if let refusal = record.refusal(of: operation) { - rows.append( - RewindRowPlan( - id: UUID(), operation: operation, - outcome: .notReversible(refusal), currentImage: nil - ) - ) - continue + return FoundRow(outcome: .notReversible(refusal), current: nil) } guard keyIsComparable(operation) else { - rows.append( - RewindRowPlan( - id: UUID(), operation: operation, - outcome: .notReversible(.keyNotComparable), currentImage: nil - ) - ) - continue + return FoundRow(outcome: .notReversible(.keyNotComparable), current: nil) } let currentRow: RewindCurrentRow? if let locator = operation.rowLocator, matchesByLocator { @@ -84,11 +70,17 @@ struct RewindPlanner { currentRow = primaryKey(of: operation).flatMap { current[$0] } ?? rowRestoredUnderKeyBefore(operation, in: current) } - let outcome = classify(operation, current: currentRow) + return FoundRow(outcome: classify(operation, current: currentRow), current: currentRow) + } + restoreDeletesWhoseKeyAnInsertFrees(&found) + + var rows: [RewindRowPlan] = [] + var restorable: [RowWriteOperation] = [] + for (operation, row) in zip(record.operations, found) { rows.append( - RewindRowPlan(id: UUID(), operation: operation, outcome: outcome, currentImage: currentRow?.values) + RewindRowPlan(id: UUID(), operation: operation, outcome: row.outcome, currentImage: row.current?.values) ) - if outcome.restores { + if row.outcome.restores { restorable.append(operation) } } @@ -106,6 +98,36 @@ struct RewindPlanner { // MARK: - Classification + private struct FoundRow { + var outcome: RewindRowOutcome + var current: RewindCurrentRow? + } + + // A new row that took a deleted row's key is removed before the deleted row is put back, so that key is free. + private func restoreDeletesWhoseKeyAnInsertFrees(_ found: inout [FoundRow]) { + var rank: [Int: Int] = [:] + for (position, entry) in Self.runOrder(of: record.operations).enumerated() where rank[entry.operation] == nil { + rank[entry.operation] = position + } + let operations = record.operations + for (index, delete) in operations.enumerated() + where delete.kind == .delete && found[index].outcome == .rowAlreadyPresent { + guard let key = primaryKey(of: delete), + let occupant = found[index].current, + primaryKey(of: occupant.values, columns: delete.columns, keyColumns: delete.primaryKeyColumns) == key, + let deleteRank = rank[index] + else { continue } + let freed = operations.indices.contains { other in + operations[other].kind == .insert && found[other].outcome == .willRestore + && primaryKey(of: operations[other]) == key + && (rank[other] ?? .max) < deleteRank + } + if freed { + found[index] = FoundRow(outcome: .willRestore, current: nil) + } + } + } + private var readsThroughView: Bool { record.writtenThroughView == true } private func classify(_ operation: RowWriteOperation, current: RewindCurrentRow?) -> RewindRowOutcome { @@ -244,19 +266,43 @@ struct RewindPlanner { // MARK: - Statements - /// The inverse operations, newest first. - /// - /// Reverse order matters when one row in the save freed a value another row took: putting the - /// second row's change back before the first row's is the only order that does not collide. - /// The forward generator groups by verb, which loses that order, so the inverse is generated - /// one operation at a time. + struct RunEntry: Hashable { + let operation: Int + let step: Int? + } + + // Newest first, step by step, since a value one write freed may be one a later write took. Ties, and a record + // saved before orders were kept, run by list position, the later first. + static func runOrder(of operations: [RowWriteOperation]) -> [RunEntry] { + var entries: [(order: Int?, entry: RunEntry)] = [] + for (index, operation) in operations.enumerated() { + if let steps = operation.steps, !steps.isEmpty { + for (stepIndex, step) in steps.enumerated() { + entries.append((step.order, RunEntry(operation: index, step: stepIndex))) + } + } else { + entries.append((operation.order, RunEntry(operation: index, step: nil))) + } + } + return entries.sorted { lhs, rhs in + (lhs.order ?? .min, lhs.entry.operation, lhs.entry.step ?? 0) + > (rhs.order ?? .min, rhs.entry.operation, rhs.entry.step ?? 0) + }.map(\.entry) + } + private func statements(for operations: [RowWriteOperation]) throws -> RestoreStatements { var statements: [ParameterizedStatement] = [] var lookups: [RowLookup?] = [] var prologue: [String] = [] var epilogue: [String] = [] - for operation in operations.reversed() { - let inverse = try inverseStatements(for: operation) + for entry in Self.runOrder(of: operations) { + let operation = operations[entry.operation] + let inverse: RestoreStatements + if let step = entry.step { + inverse = try inverseStatements(for: operation, step: step) + } else { + inverse = try inverseStatements(for: operation) + } statements.append(contentsOf: inverse.statements) lookups.append(contentsOf: inverse.lookups) prologue.append(contentsOf: inverse.prologue.filter { !prologue.contains($0) }) @@ -265,6 +311,31 @@ struct RewindPlanner { return RestoreStatements(statements: statements, lookups: lookups, prologue: prologue, epilogue: epilogue) } + // Sets the step's columns back, keyed on the row as that step left it. + private func inverseStatements(for operation: RowWriteOperation, step index: Int) throws -> RestoreStatements { + guard let image = operation.stepImages()?[safe: index] else { return RestoreStatements(statements: []) } + var factory = factory + factory.rowLocators = operation.rowLocator.map { [.existing(0): $0] } ?? [:] + let step = image.step + let cellChanges = step.columns.enumerated().compactMap { offset, column -> CellChange? in + guard operation.columns.indices.contains(column), image.after.indices.contains(column), + step.before.indices.contains(offset) + else { return nil } + return CellChange( + columnIndex: column, columnName: operation.columns[column], + oldValue: image.after[column], newValue: step.before[offset], + oldIsAbsent: image.absentAfter.contains(column), newIsAbsent: step.beforeAbsent.contains(column) + ) + } + guard !cellChanges.isEmpty else { return RestoreStatements(statements: []) } + return try factory.lookedUpStatements( + for: [RowChange( + rowID: .existing(0), type: .update, cellChanges: cellChanges, + originalRow: image.after, absentColumns: image.absentAfter + )] + ) + } + private func inverseStatements(for operation: RowWriteOperation) throws -> RestoreStatements { var factory = factory factory.rowLocators = operation.rowLocator.map { [.existing(0): $0] } ?? [:] diff --git a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift index 19517845a3..1feffebad3 100644 --- a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift +++ b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift @@ -131,22 +131,22 @@ struct RowChangeStatementFactory { } } - /// Every statement the changes need, or a throw naming the changes that would be left out. - /// - /// The host's statements carry the rows they touch, so the executor can hold the server to that - /// count. A driver's carry one only for a keyless update or delete of one row. + // Split into save steps first, so the refusals, every writer, the preview and the coverage see what runs. func rowWriteStatements( for changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]] = [:], deletedRowIDs: Set = [], insertedRowIDs: Set = [] ) throws -> RowWriteStatements { - try refuseServerOwnedEdits(in: changes) - try refuseDefaultOnUpdate(in: changes) - try refuseUnwritableExpressions(in: changes, insertedRowData: insertedRowData) - try refuseMisdimensionedVectors(in: changes, insertedRowData: insertedRowData) + let steps = RowChange.saveSteps( + changes, keyColumns: Set(primaryKeyColumns.compactMap { columns.firstIndex(of: $0) }) + ) + try refuseServerOwnedEdits(in: steps) + try refuseDefaultOnUpdate(in: steps) + try refuseUnwritableExpressions(in: steps, insertedRowData: insertedRowData) + try refuseMisdimensionedVectors(in: steps, insertedRowData: insertedRowData) let insertedRowData = insertedRowData.mapValues(leavingServerOwnedColumnsToTheServer) - let changes = changes.map { markingServerOwnedDefaults($0, in: insertedRowData) } + let changes = steps.map { markingServerOwnedDefaults($0, in: insertedRowData) } if let driverStatements = try pluginRowWrites( for: changes, insertedRowData: insertedRowData, @@ -392,7 +392,7 @@ struct RowChangeStatementFactory { changes, deletedRowIDs: deletedRowIDs, insertedRowIDs: insertedRowIDs, - writtenRowIDs: Set(statements.flatMap(\.rowIDs)) + writtenChangeIDs: Set(statements.flatMap(\.changeIDs)) ) if let kind = UnwrittenRowCounts(unwritten).leadingKind { throw DataWriteError.rowsNotIdentifiable(tableName, kind) @@ -572,7 +572,7 @@ struct RowChangeStatementFactory { changes, deletedRowIDs: deletedRowIDs, insertedRowIDs: insertedRowIDs, - writtenRowIDs: Set(writes.flatMap(\.rowIndices).compactMap(keyed.rowID(forIndex:))) + writtenChangeIDs: Set(writes.flatMap(\.rowIndices).compactMap { keyed.change(forIndex: $0)?.id }) ) guard unwritten.isEmpty else { throw DataWriteError.changesNotWritable(table: tableName, unwritten: UnwrittenRowCounts(unwritten)) @@ -697,13 +697,15 @@ struct RowChangeStatementFactory { } } +// One key per change, so a row written in two steps reaches the driver as two changes. struct PluginKeyedChanges { let changes: [PluginRowChange] let insertedRowData: [Int: [PluginCellValue]] let deletedRowIndices: Set let insertedRowIndices: Set - /// The row each key stands for, indexed by key. + // Indexed by key. let rowIDs: [RowID] + private let steps: [RowChange] init( changes: [RowChange], @@ -712,23 +714,25 @@ struct PluginKeyedChanges { insertedRowIDs: Set ) { let ordered = changes.sorted { $0.sequence < $1.sequence } - var keys: [RowID: Int] = [:] - var rowIDs: [RowID] = [] - for change in ordered where keys[change.rowID] == nil { - keys[change.rowID] = rowIDs.count - rowIDs.append(change.rowID) - } - self.rowIDs = rowIDs - self.changes = ordered.compactMap { change in - keys[change.rowID].map { PluginRowChange(change, key: $0) } + var insertKeys: [RowID: Int] = [:] + var deleteKeys: [RowID: Int] = [:] + for (key, change) in ordered.enumerated() { + switch change.type { + case .insert: insertKeys[change.rowID] = key + case .delete: deleteKeys[change.rowID] = key + case .update: break + } } + self.steps = ordered + self.rowIDs = ordered.map(\.rowID) + self.changes = ordered.enumerated().map { key, change in PluginRowChange(change, key: key) } self.insertedRowData = Dictionary( uniqueKeysWithValues: insertedRowData.compactMap { rowID, values in - keys[rowID].map { ($0, values) } + insertKeys[rowID].map { ($0, values) } } ) - self.deletedRowIndices = Set(deletedRowIDs.compactMap { keys[$0] }) - self.insertedRowIndices = Set(insertedRowIDs.compactMap { keys[$0] }) + self.deletedRowIndices = Set(deletedRowIDs.compactMap { deleteKeys[$0] }) + self.insertedRowIndices = Set(insertedRowIDs.compactMap { insertKeys[$0] }) } func locators(_ byRow: [RowID: String]) -> [Int: String] { @@ -739,14 +743,14 @@ struct PluginKeyedChanges { return keyed } - /// The row a driver's `rowIndex` names, or nil when it names none of these changes. - func rowID(forIndex index: Int) -> RowID? { - rowIDs.indices.contains(index) ? rowIDs[index] : nil + // Nil when a driver's `rowIndex` names none of these changes. + func change(forIndex index: Int) -> RowChange? { + steps.indices.contains(index) ? steps[index] : nil } func writeKind(ofRowIndex index: Int) -> RowWriteKind? { - guard let change = changes.first(where: { $0.rowIndex == index }) else { return nil } - switch change.type { + guard let type = change(forIndex: index)?.type else { return nil } + switch type { case .insert: return .insert case .update: return .update case .delete: return .delete diff --git a/apple/TablePro/Core/DataWrite/RowWriteCoverage.swift b/apple/TablePro/Core/DataWrite/RowWriteCoverage.swift index fef6c92260..188f5e7e30 100644 --- a/apple/TablePro/Core/DataWrite/RowWriteCoverage.swift +++ b/apple/TablePro/Core/DataWrite/RowWriteCoverage.swift @@ -23,20 +23,21 @@ enum RowWriteCoverage { } } + // By change, not by row: a row edited around another change is written in steps, and each one has to be. static func unwrittenChanges( _ changes: [RowChange], deletedRowIDs: Set, insertedRowIDs: Set, - writtenRowIDs: Set + writtenChangeIDs: Set ) -> [RowChange] { changes.filter { change in isPending(change, deletedRowIDs: deletedRowIDs, insertedRowIDs: insertedRowIDs) - && !writtenRowIDs.contains(change.rowID) + && !writtenChangeIDs.contains(change.id) } } } -/// How many changes of each kind a save could not write, which is what the refusal names. +// How many rows of each kind a save could not write, which is what the refusal names. struct UnwrittenRowCounts: Equatable, Sendable { let updates: Int let inserts: Int @@ -49,11 +50,10 @@ struct UnwrittenRowCounts: Equatable, Sendable { } init(_ changes: [RowChange]) { - self.init( - updates: changes.count { $0.type == .update }, - inserts: changes.count { $0.type == .insert }, - deletes: changes.count { $0.type == .delete } - ) + func rows(_ type: ChangeType) -> Int { + Set(changes.lazy.filter { $0.type == type }.map(\.rowID)).count + } + self.init(updates: rows(.update), inserts: rows(.insert), deletes: rows(.delete)) } var isEmpty: Bool { updates == 0 && inserts == 0 && deletes == 0 } diff --git a/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift b/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift index 11cafebc8f..77503f51b6 100644 --- a/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift +++ b/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift @@ -34,20 +34,36 @@ enum RowWriteOperationBuilder { writesThroughView: Bool = false, rowLocators: [RowID: String] = [:] ) -> [RowWriteOperation] { + // The same steps the save runs, so the record matches what ran. + let keyColumns = Set(primaryKeyColumns.compactMap { columns.firstIndex(of: $0) }) + var stepsByRow: [RowID: [RowChange]] = [:] + for step in RowChange.saveSteps(changes, keyColumns: keyColumns) where step.type == .update { + stepsByRow[step.rowID, default: []].append(step) + } + func firstOrder(of change: RowChange) -> Int { + guard change.type == .update else { return change.sequence } + return stepsByRow[change.rowID]?.first?.sequence ?? change.sequence + } + // A rewind undoes them newest first, and `changes` stops being in edit order once a change is cancelled. - changes.sorted { $0.sequence < $1.sequence }.compactMap { change in + let ordered = changes.enumerated() + .sorted { (firstOrder(of: $0.element), $0.offset) < (firstOrder(of: $1.element), $1.offset) } + .map(\.element) + return ordered.compactMap { change -> RowWriteOperation? in + var operation: RowWriteOperation switch change.type { case .update: - var operation = update( - change, target: target, columns: columns, + let steps = stepsByRow[change.rowID] ?? [change] + operation = update( + change, steps: steps, target: target, columns: columns, primaryKeyColumns: primaryKeyColumns, generatedColumns: generatedColumns, containsTableOperation: containsTableOperation ) operation.rowLocator = rowLocators[change.rowID] - return operation + operation.steps = steps.count > 1 ? steps.map { RowWriteStep($0) } : nil case .delete: guard deletedRowIDs.contains(change.rowID) else { return nil } - var operation = delete( + operation = delete( change, target: target, columns: columns, primaryKeyColumns: primaryKeyColumns, containsTableOperation: containsTableOperation, @@ -55,21 +71,23 @@ enum RowWriteOperationBuilder { throughView: writesThroughView ) operation.rowLocator = rowLocators[change.rowID] - return operation case .insert: guard insertedRowIDs.contains(change.rowID) else { return nil } - return insert( + operation = insert( change, values: insertedRowData[change.rowID], target: target, columns: columns, primaryKeyColumns: primaryKeyColumns, containsTableOperation: containsTableOperation ) } + operation.order = firstOrder(of: change) + return operation } } private static func update( _ change: RowChange, + steps: [RowChange], target: DataWriteTarget, columns: [String], primaryKeyColumns: [String], @@ -95,16 +113,18 @@ enum RowWriteOperationBuilder { postImage: postImage, writtenColumns: writtenColumns, refusal: updateRefusal( - change, preImage: preImage, primaryKeyColumns: primaryKeyColumns, - generatedColumns: generatedColumns, containsTableOperation: containsTableOperation + writing: change.cellChanges + steps.flatMap(\.cellChanges), preImage: preImage, + primaryKeyColumns: primaryKeyColumns, generatedColumns: generatedColumns, + containsTableOperation: containsTableOperation ), preImageAbsentColumns: change.absentColumns, postImageAbsentColumns: change.absentColumnsAfterUpdate ) } + // Every step's cells count: the inverse of a later step would write an earlier step's SQL back as a literal. private static func updateRefusal( - _ change: RowChange, + writing cellChanges: [CellChange], preImage: [PluginCellValue]?, primaryKeyColumns: [String], generatedColumns: Set, @@ -112,14 +132,14 @@ enum RowWriteOperationBuilder { ) -> RewindRefusal? { if containsTableOperation { return .destructiveTableOperation } if primaryKeyColumns.isEmpty || preImage == nil { return .noPrimaryKey } - if change.cellChanges.contains(where: \.newIsExpression) { + if cellChanges.contains(where: \.newIsExpression) { return .serverComputedValue } - if change.cellChanges.contains(where: { generatedColumns.contains($0.columnName) }) { + if cellChanges.contains(where: { generatedColumns.contains($0.columnName) }) { return .serverComputedValue } if let preImage, preImage.contains(where: exceedsCapture) { return .valueTooLarge } - if change.cellChanges.contains(where: { exceedsCapture($0.newValue) }) { return .valueTooLarge } + if cellChanges.contains(where: { exceedsCapture($0.newValue) }) { return .valueTooLarge } return nil } diff --git a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift index fb726e0bd5..dc9e221372 100644 --- a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift +++ b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift @@ -134,9 +134,16 @@ extension DatabaseAccessBridge { onLateCompletion: { CatalogChangeService.post(statementsRan) } ) { driver in reach.mark() - return try await ScriptBatchRun.run( - batches, startLines: startLines, rowCap: rowCap, driver: driver, progress: progress, context: context - ) + do { + let answered = try await ScriptBatchRun.run( + batches, startLines: startLines, rowCap: rowCap, driver: driver, progress: progress, context: context + ) + await manager.noteSessionStatements(Self.ranStatements(of: batches, progress), ranOn: driver, scope: scope) + return answered + } catch { + await manager.noteSessionStatements(Self.ranStatements(of: batches, progress), ranOn: driver, scope: scope) + throw error + } } } catch { if classification.tier != .safe { @@ -151,6 +158,11 @@ extension DatabaseAccessBridge { return run.outcome(executionTimeMs: (CFAbsoluteTimeGetCurrent() - startTime) * 1_000) } + // The batch a script stopped at ran too, and can have created a temporary table before its error. + private static func ranStatements(of batches: [ExecutableBatch], _ progress: ScriptBatchProgress) -> [String] { + batches.prefix(progress.completedBatchCount + 1).flatMap { $0.statements.map(\.sql) } + } + private static func postSucceededBatches( of batches: [ExecutableBatch], progress: ScriptBatchProgress, diff --git a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift index d3577f51c1..58b00eb116 100644 --- a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift +++ b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift @@ -415,6 +415,8 @@ internal actor DatabaseAccessBridge { onLateCompletion: { CatalogChangeService.post(statementRan) } ) { driver in reach.mark() + // Before it runs, so a statement that fails or a connection lost under it is still noted. + await manager.noteSessionStatements([sql], ranOn: driver, scope: scope) let answer = try await driver.executeUserQuery( query: sql, rowCap: fetchCap, diff --git a/apple/TablePro/Core/Database/DatabaseManager+CredentialLease.swift b/apple/TablePro/Core/Database/DatabaseManager+CredentialLease.swift index 164eba3a84..a0d45461fe 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+CredentialLease.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+CredentialLease.swift @@ -48,15 +48,19 @@ extension DatabaseManager { } } - // The lane reopens on its next use, signing in with a new lease. A transaction it held is reported as lost. + // The lane reopens on its next use, signing in with a new lease. A transaction or session state it held is + // reported as lost. private func closeLane(_ database: String, for ending: CredentialLeaseEnding) async { let connectionId = ending.connectionId guard let driver = sessionLanes.parkedDriver(for: connectionId, database: database) else { return } let held = await driver.heldSessionTransactionState() guard credentialLeaseAction(for: ending) == .closeLane(database: database) else { return } - if !held.permitsAppTransaction { - sessionLanes.markTransactionLost(database: database, for: connectionId) - } + sessionLanes.markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: sessionStateLedger.state(heldBy: driver, for: connectionId), + for: connectionId + ) Self.logger.info("Closing a lane whose Vault lease is ending for \(connectionId, privacy: .public)") sessionLanes.close(database: database, for: connectionId, supersedingOpens: false) } @@ -73,11 +77,13 @@ extension DatabaseManager { guard credentialLeaseAction(for: ending) == .reconnectSession else { return } try? await sessionDriverGate.withExclusiveAccess(browsedGateKey(for: connectionId)) { guard credentialLeaseAction(for: ending) == .reconnectSession else { return } - if holdsTransaction(connectionId, driver: ending.driver) { - // Reported to the next statement on this database, as a lane's lost transaction is. - let database = activeSessions[connectionId]?.resolvedBrowseDatabase ?? "" - sessionLanes.markTransactionLost(database: database, for: connectionId) - } + // Reported to the next statement on this database, as a lane's lost transaction is. What the session set up + // is noted by the reconnect, just before it closes this driver. + sessionLanes.markLost( + database: activeSessions[connectionId]?.resolvedBrowseDatabase ?? "", + transaction: holdsTransaction(connectionId, driver: ending.driver), + for: connectionId + ) Self.logger.info("Vault lease ending, reconnecting \(connectionId, privacy: .public) with a new one") let outcome = await performHealthMonitorReconnect(connectionId: connectionId, failedDriver: ending.driver) guard case .retry = outcome, credentialLeaseAction(for: ending) == .reconnectSession else { return } diff --git a/apple/TablePro/Core/Database/DatabaseManager+Health.swift b/apple/TablePro/Core/Database/DatabaseManager+Health.swift index 25bde8326d..a82634ef22 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Health.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Health.swift @@ -186,17 +186,25 @@ extension DatabaseManager { } } - // Read before the driver is closed and replaced, which prunes the hold. The user's next statement there - // is refused once, as on a lane, instead of running outside the transaction it was written for. - internal func noteTransactionLost(with driver: DatabaseDriver?, for connectionId: UUID) { + // Read before the driver is closed and replaced, which prunes the hold and the ledger. The user's next statement + // there is refused once, as on a lane, instead of running without the transaction or session it was written for. + // A switch the user asked for passes `includingState: false`: they chose to leave that session. + internal func noteSessionLost(with driver: DatabaseDriver?, for connectionId: UUID, includingState: Bool = true) { guard let driver, let session = activeSessions[connectionId] else { return } let lost = sessionTransactions.holds(for: connectionId).filter { sessionTransactions.driver(for: $0) === driver } for hold in lost { - sessionLanes.markTransactionLost( + sessionLanes.markLost( database: hold.key.lane ?? session.resolvedBrowseDatabase, + transaction: true, for: connectionId ) } + guard includingState else { return } + sessionLanes.markLost( + database: session.resolvedBrowseDatabase, + state: sessionStateLedger.state(heldBy: driver, for: connectionId), + for: connectionId + ) } internal func reconnectVerdict(for error: Error) -> ConnectionFailureClassifier.ReconnectVerdict { @@ -259,7 +267,7 @@ extension DatabaseManager { generation: Int ) async throws -> ReconnectResult? { let connectionId = session.connection.id - noteTransactionLost(with: session.driver, for: connectionId) + noteSessionLost(with: session.driver, for: connectionId) session.driver?.disconnect() // Resolved as the connect resolved it; a reference that lost its approval needs a person. let resolved = try await envVarReferenceConsent.resolve(session.connection, trigger: .background) @@ -415,7 +423,8 @@ extension DatabaseManager { // A debug target dialed through this transport would be left holding its locks once it goes. await RoutineDebugSessions.shared.stopAll(connectionId: sessionId) MetadataConnectionPool.shared.beginTransportReplacement(connectionId: sessionId) - await closeTransportLanes(for: sessionId) { self.isCurrentAttempt(generation, for: sessionId) } + // The user asked for this reconnect, so what the lanes' sessions set up is not reported as lost. + await closeTransportLanes(for: sessionId, includingState: false) { self.isCurrentAttempt(generation, for: sessionId) } } defer { if replacesPooledTransport { @@ -467,7 +476,9 @@ extension DatabaseManager { if movesPooledEndpoint { await RoutineDebugSessions.shared.stopAll(connectionId: sessionId) MetadataConnectionPool.shared.beginTransportReplacement(connectionId: sessionId) - await closeTransportLanes(for: sessionId) { self.isCurrentAttempt(generation, for: sessionId) } + await closeTransportLanes(for: sessionId, includingState: false) { + self.isCurrentAttempt(generation, for: sessionId) + } } defer { if movesPooledEndpoint { diff --git a/apple/TablePro/Core/Database/DatabaseManager+Recovery.swift b/apple/TablePro/Core/Database/DatabaseManager+Recovery.swift index 59fab36098..3d657c0891 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Recovery.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Recovery.swift @@ -1,5 +1,6 @@ import Foundation import os +import TableProSQLGrammar import TableProUtilities extension DatabaseManager { @@ -239,7 +240,12 @@ extension DatabaseManager { // Closes only the lanes parked when it starts, each while still parked as the same driver and only while // `owns` holds: a connect that took the session over parks lanes, and transactions, that are not this teardown's. - internal func closeTransportLanes(for connectionId: UUID, while owns: () -> Bool) async { + // A reconnect the user asked for passes `includingState: false`, as their own database switch does. + internal func closeTransportLanes( + for connectionId: UUID, + includingState: Bool = true, + while owns: () -> Bool + ) async { guard owns() else { return } let lanes = sessionLanes.parkedDatabases(for: connectionId).compactMap { database in sessionLanes.parkedDriver(for: connectionId, database: database).map { (database, $0) } @@ -250,9 +256,12 @@ extension DatabaseManager { let held = await driver.heldSessionTransactionState() guard owns() else { return } guard sessionLanes.parkedDriver(for: connectionId, database: database) === driver else { continue } - if !held.permitsAppTransaction { - sessionLanes.markTransactionLost(database: database, for: connectionId) - } + sessionLanes.markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: includingState ? sessionStateLedger.state(heldBy: driver, for: connectionId) : [], + for: connectionId + ) sessionLanes.close(database: database, for: connectionId, supersedingOpens: false) } } diff --git a/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift b/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift index ad1ba9c387..2189070fbf 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift @@ -5,6 +5,7 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar /// Where an operation bound to a `DatabaseScope` runs. enum ScopedDriverRoute: Equatable { @@ -76,16 +77,15 @@ extension DatabaseManager { else { return .sessionDriver } - /// A database the user has browsed keeps its own session connection, so its tabs carry on in - /// the transaction, temp tables and settings they left there. - /// A database whose connection ended with a transaction still open takes the session path - /// once more, where that loss is reported before anything runs on a replacement. + // A database the user has browsed keeps its own session connection, so its tabs carry on in the transaction, + // temp tables and settings they left there. A database whose connection ended holding a transaction or session + // state takes the session path once more, where that loss is reported before anything runs on a replacement. if usesDatabaseLanes(session) { if sessionLanes.parkedDriver(for: scope.connectionId, database: scope.database) != nil { return .sessionDriver } - // A table read depends on no transaction, so a lost one is not its to hear about. - if opensLane, sessionLanes.hasTransactionLoss(database: scope.database, for: scope.connectionId) { + // A table read depends on no transaction or session state, so a lost one is not its to hear about. + if opensLane, sessionLanes.hasLoss(database: scope.database, for: scope.connectionId) { return .sessionDriver } // A temporary table lives in the session that made it, which a new connection is not. @@ -106,10 +106,9 @@ extension DatabaseManager { return .pooled } - /// `cancellation` decides whether Stop, or a navigation that supersedes a tab, can reach the - /// leased driver. The parameter has no default on purpose: a new call site must state which it - /// is, because getting it wrong silently either makes a query uncancellable or makes a commit - /// killable. + // `cancellation` decides whether Stop, or a navigation that supersedes a tab, can reach the leased driver. The + // parameter has no default on purpose: a new call site must state which it is, because getting it wrong silently + // either makes a query uncancellable or makes a commit killable. func withScopedDriver( scope: DatabaseScope, route: ScopedDriverRoute, @@ -395,9 +394,9 @@ extension DatabaseManager { } } - /// `ownsSessionState` is false for a table read or a metadata read, which depend on nothing the - /// session holds. Such a turn neither opens a session connection nor takes the report of a lost - /// transaction, which is kept for the user's next statement. + // `ownsSessionState` is false for a table read or a metadata read, which depend on nothing the session holds. Such + // a turn neither opens a session connection nor takes the report of a lost session, which is kept for the next + // turn that owns session state. That turn is refused once with the whole loss, whatever work it is. private func withSessionDriverTurn( scope: DatabaseScope, ownsSessionState: Bool, @@ -406,20 +405,20 @@ extension DatabaseManager { let connectionId = scope.connectionId let lane = activeSessions[connectionId].flatMap { laneDatabase(for: scope, in: $0) } let startsOnBrowsed = lane == nil || lane == activeSessions[connectionId]?.resolvedBrowseDatabase - /// Decided before the turn is queued. Work written against a connection that was there when - /// it queued and is gone when its turn comes is refused, never moved onto a new one: the - /// transaction it meant to continue ended with the connection it was on. + // Decided before the turn is queued. Work written against a connection that was there when it queued and is + // gone when its turn comes is refused, never moved onto a new one: the transaction it meant to continue ended + // with the connection it was on. let startsWithoutLane = !startsOnBrowsed && !scope.isSessionBound && lane.flatMap { sessionLanes.parkedDriver(for: connectionId, database: $0) } == nil if startsOnBrowsed { - /// Outside the gate on purpose. A verification that has to reconnect runs the whole - /// reconnect, which restores the schema and the database on the new driver, and doing that - /// while holding the gate would deadlock the very thing waiting to be pinned. + // Outside the gate on purpose. A verification that has to reconnect runs the whole reconnect, which + // restores the schema and the database on the new driver, and doing that while holding the gate would + // deadlock the very thing waiting to be pinned. await verifyBeforeUse(connectionId) } - /// A check that failed and could not recover left the driver installed and disconnected, - /// so the presence of a driver below is not enough. Refusing here is the point of checking - /// at all: without it the user's own work runs on a handle the app already knows is dead. + // A check that failed and could not recover left the driver installed and disconnected, so the presence of a + // driver below is not enough. Refusing here is the point of checking at all: without it the user's own work + // runs on a handle the app already knows is dead. guard isUsable(connectionId) else { throw DatabaseError.notConnected } @@ -428,19 +427,19 @@ extension DatabaseManager { ) { try await trackOperation(sessionId: connectionId) { try Task.checkCancellation() - /// Asked again once the lease has its turn, because a database switch that held the - /// gate can have left the driver the same way while this lease waited, or moved the - /// database this turn belongs to from the browsed connection to a parked one. - /// A database whose own connection is gone, its entry closed, falls back to the browsed - /// driver only to be turned away: a table read re-routes to the pool and anything else - /// is refused by `pin` before it runs a statement there. + // Asked again once the lease has its turn, because a database switch that held the gate can have left + // the driver the same way while this lease waited, or moved the database this turn belongs to from the + // browsed connection to a parked one. A database whose own connection is gone, its entry closed, falls + // back to the browsed driver only to be turned away: a table read re-routes to the pool and anything + // else is refused by `pin` before it runs a statement there. guard isUsable(connectionId), let browsedDriver = activeSessions[connectionId]?.driver else { throw DatabaseError.notConnected } guard let lane else { - // A driver replaced while it held a transaction, by a reconnect or a Vault lease rotation. - if ownsSessionState, let lost = sessionLanes.takeAnyTransactionLoss(for: connectionId) { - throw DatabaseError.connectionFailed(Self.parkedTransactionLostMessage(database: lost)) + // A driver replaced while it held a transaction or session state, by a reconnect or a Vault lease + // rotation. + if ownsSessionState, let lost = sessionLanes.takeAnyLoss(for: connectionId) { + throw DatabaseError.connectionFailed(Self.sessionLostMessage(database: lost.database, loss: lost.loss)) } return try await recordingTransaction(on: browsedDriver, for: .init(connectionId: connectionId, lane: nil)) { try await turn(browsedDriver) @@ -451,7 +450,7 @@ extension DatabaseManager { driver = held } else if ownsSessionState, !scope.isSessionBound, startsWithoutLane || sessionLanes.wasClosedIdle(database: lane, for: connectionId), - !sessionLanes.hasTransactionLoss(database: lane, for: connectionId) { + !sessionLanes.hasLoss(database: lane, for: connectionId) { driver = try await openedLaneDriver(database: lane, for: connectionId) } else { driver = browsedDriver @@ -459,8 +458,8 @@ extension DatabaseManager { let laneDriver = sessionLanes.isParked(driver, for: connectionId) ? try await verifiedParkedDriver(driver, database: lane, for: connectionId) : driver - if ownsSessionState, sessionLanes.takeTransactionLoss(database: lane, for: connectionId) { - throw DatabaseError.connectionFailed(Self.parkedTransactionLostMessage(database: lane)) + if ownsSessionState, let loss = sessionLanes.takeLoss(database: lane, for: connectionId) { + throw DatabaseError.connectionFailed(Self.sessionLostMessage(database: lane, loss: loss)) } sessionLanes.noteOpenedUse(database: lane, for: connectionId) defer { sessionLanes.noteOpenedUse(database: lane, for: connectionId) } @@ -514,6 +513,12 @@ extension DatabaseManager { return scope.isServerScoped ? session.resolvedBrowseDatabase : scope.database } + // The key a scope's turn records its transaction under, which the session state ledger shares. + internal func sessionKey(for scope: DatabaseScope) -> SessionTransactionTracker.Key? { + guard let session = activeSessions[scope.connectionId] else { return nil } + return SessionTransactionTracker.Key(connectionId: scope.connectionId, lane: laneDatabase(for: scope, in: session)) + } + internal func sessionDriver(holding scope: DatabaseScope) -> DatabaseDriver? { guard let session = activeSessions[scope.connectionId] else { return nil } return sessionDriver(for: scope.connectionId, laneDatabase: laneDatabase(for: scope, in: session)) @@ -528,18 +533,17 @@ extension DatabaseManager { return sessionLanes.parkedDriver(for: connectionId, database: laneDatabase) } - /// A parked connection nobody watched may have died while it sat idle. One that no longer - /// answers is replaced before the work runs, and if it was holding a transaction the work is - /// refused once, because running it on the new connection would carry on as if that transaction - /// had not just been rolled back by the server. + // A parked connection nobody watched may have died while it sat idle. One that no longer answers is replaced + // before the work runs, and if it held a transaction or session state the work is refused once, because running it + // on the new connection would carry on as if the server had not just dropped them. private func verifiedParkedDriver( _ driver: DatabaseDriver, database: String, for connectionId: UUID ) async throws -> DatabaseDriver { if sessionLanes.isFresh(driver) { return driver } - /// Read before the ping: a ping that finds the socket closed leaves libpq reporting no - /// transaction state at all, which would read as nothing to lose. + // Read before the ping: a ping that finds the socket closed leaves libpq reporting no transaction state at + // all, which would read as nothing to lose. let held = await driver.heldSessionTransactionState() let wasOpened = sessionLanes.isOpened(database: database, for: connectionId) sessionLanes.beginVerifying(driver) @@ -549,12 +553,15 @@ extension DatabaseManager { sessionLanes.markVerified(driver) return driver } catch { + // Recorded before the close, which prunes the ledger, and before the reopen, which can fail or be + // superseded: the session is gone either way, and whatever runs on this database next has to hear of it. + sessionLanes.markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: sessionStateLedger.state(heldBy: driver, for: connectionId), + for: connectionId + ) sessionLanes.close(database: database, for: connectionId, supersedingOpens: false) - /// Recorded before the reopen, which can fail or be superseded: the transaction is gone - /// either way, and whatever runs on this database next has to hear about it. - if !held.permitsAppTransaction { - sessionLanes.markTransactionLost(database: database, for: connectionId) - } let generation = sessionLanes.generation(for: connectionId) let reopened = try await sessionLanes.open( DatabaseScope(connectionId: connectionId, database: database, schema: nil) @@ -617,10 +624,9 @@ extension DatabaseManager { throw DatabaseError.notConnected } - /// Closes the connections opened for SQL on databases the user never browsed that nobody has - /// used for `idleFor`. One that is running work or holds a transaction stays. Whatever else the - /// session held goes with it unreported: on PostgreSQL its temp tables, `SET` values and - /// advisory locks. + // Closes the connections opened for SQL on databases the user never browsed that nobody has used for `idleFor`. + // One that is running work, holds a transaction or holds session state the user's statements set up stays until + // disconnect, as a browsed one does. What a routine set up is out of sight and goes with the close. internal func closeIdleOpenedLanes( for connectionId: UUID, idleFor: TimeInterval = SessionLanes.openedIdleTimeout, @@ -631,8 +637,8 @@ extension DatabaseManager { } } - /// Asked inside the lane's own turn, so no statement runs between the answer and the close. Only - /// a driver that says it is idle is closed: one that cannot say may be holding a transaction. + // Asked inside the lane's own turn, so no statement runs between the answer and the close. Only a driver that says + // it is idle is closed: one that cannot say may be holding a transaction. private func closeOpenedLaneIfIdle( _ database: String, driver: DatabaseDriver, @@ -640,10 +646,12 @@ extension DatabaseManager { ) async -> Bool { let key = SessionDriverGate.Key(connectionId: connectionId, database: database) guard !sessionDriverGate.isHeld(key) else { return false } + let ledgerKey = SessionTransactionTracker.Key(connectionId: connectionId, lane: database) let closed = try? await sessionDriverGate.withExclusiveAccess(key) { () async -> Bool in guard sessionLanes.isOpened(database: database, for: connectionId), sessionLanes.parkedDriver(for: connectionId, database: database) === driver, - !sessionTransactions.holdsTransaction(connectionId: connectionId, container: database) + !sessionTransactions.holdsTransaction(connectionId: connectionId, container: database), + sessionStateLedger.state(for: ledgerKey, heldBy: driver).isEmpty else { return false } let state = await Self.boundedTransactionState(of: driver, clock: ContinuousClock()) guard state == .idle, diff --git a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift index 66dcd2fad0..36c19de2ac 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift @@ -574,21 +574,21 @@ extension DatabaseManager { ) } - /// The connection parked for `database`, if it still answers. One that died is closed, and if it - /// was holding a transaction, the next work on that database is told the server rolled it back. - /// A turn running on it right now is answer enough, and asking would wait for that turn. + // The connection parked for `database`, if it still answers. One that died is closed, and if it held a transaction + // or session state, the next work on that database is told what the server dropped. A turn running on it right now + // is answer enough, and asking would wait for that turn. private func answeringParkedDriver(database: String, for connectionId: UUID) async -> DatabaseDriver? { let key = SessionDriverGate.Key(connectionId: connectionId, database: database) guard var parked = sessionLanes.parkedDriver(for: connectionId, database: database) else { return nil } - /// A check already running decides for this switch too: once it ends it has either confirmed - /// the driver or parked a replacement, and that is what gets promoted. + // A check already running decides for this switch too: once it ends it has either confirmed the driver or + // parked a replacement, and that is what gets promoted. if sessionLanes.isVerifying(parked) { await sessionLanes.waitForVerification(of: parked) guard let settled = sessionLanes.parkedDriver(for: connectionId, database: database) else { return nil } parked = settled } if sessionLanes.isFresh(parked) || sessionDriverGate.isHeld(key) { return parked } - /// Read before the ping, which on a closed socket leaves libpq reporting no state at all. + // Read before the ping, which on a closed socket leaves libpq reporting no state at all. let held = await parked.heldSessionTransactionState() do { try await sessionDriverGate.withExclusiveAccess(key) { @@ -598,10 +598,13 @@ extension DatabaseManager { return parked } catch { guard sessionLanes.parkedDriver(for: connectionId, database: database) === parked else { return nil } + sessionLanes.markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: sessionStateLedger.state(heldBy: parked, for: connectionId), + for: connectionId + ) sessionLanes.close(database: database, for: connectionId, supersedingOpens: false) - if !held.permitsAppTransaction { - sessionLanes.markTransactionLost(database: database, for: connectionId) - } return nil } } @@ -657,7 +660,7 @@ extension DatabaseManager { let previousSavedSchema = appSettingsStorage.loadLastSchema(for: connectionId) // Noted under the database being left, before the session names the target. - noteTransactionLost(with: previous.driver, for: connectionId) + noteSessionLost(with: previous.driver, for: connectionId, includingState: false) updateSession(connectionId) { session in session.connection.database = database session.browseDatabase = database @@ -1014,6 +1017,7 @@ extension DatabaseManager { activeSessions[connectionId] = session connectionStatusVersions[connectionId, default: 0] &+= 1 sessionTransactions.prune(connectionId) + sessionStateLedger.prune(connectionId) AppEvents.shared.connectionStatusChanged.send( ConnectionStatusChange(connectionId: connectionId, status: session.reportedStatus) ) diff --git a/apple/TablePro/Core/Database/DatabaseManager.swift b/apple/TablePro/Core/Database/DatabaseManager.swift index 5c4db15f3c..275998de5a 100644 --- a/apple/TablePro/Core/Database/DatabaseManager.swift +++ b/apple/TablePro/Core/Database/DatabaseManager.swift @@ -9,6 +9,7 @@ import Combine import Foundation import os import TableProPluginKit +import TableProSQLGrammar /// Manages database connections and active drivers @MainActor @@ -143,6 +144,11 @@ final class DatabaseManager: ObservableObject { self?.isLiveSessionDriver(driver, for: key) ?? false } + // What the user's statements set up on each session connection, read before a reconnect closes one. + internal lazy var sessionStateLedger = SessionStateLedger { [weak self] key, driver in + self?.isLiveSessionDriver(driver, for: key) ?? false + } + /// The driver each connection's health monitor last pinged, so the reconnect that follows a /// failed ping acts only on that driver and never on one a database switch put in its place. /// Held, not just identified: a closed driver that was freed would hand its address, and with it @@ -212,6 +218,10 @@ final class DatabaseManager: ObservableObject { self.recoveryClock = recoveryClock sessionLanes.onClose = { [weak self] connectionId in self?.sessionTransactions.prune(connectionId) + self?.sessionStateLedger.prune(connectionId) + } + sessionLanes.recordedState = { [weak self] driver, connectionId in + self?.sessionStateLedger.state(heldBy: driver, for: connectionId) ?? [] } observeConnectionUpdates() observeHealthCheckSetting() diff --git a/apple/TablePro/Core/Database/NativeDumpDescriptor.swift b/apple/TablePro/Core/Database/NativeDumpDescriptor.swift index 2d93bf3683..c48f7f9e47 100644 --- a/apple/TablePro/Core/Database/NativeDumpDescriptor.swift +++ b/apple/TablePro/Core/Database/NativeDumpDescriptor.swift @@ -6,34 +6,26 @@ import Foundation import TableProPluginKit -/// Everything one engine's own dump and restore need, as data. -/// -/// The three engines that had this hard-coded were all PostgreSQL. Generalizing it is not a matter -/// of parameterizing a binary name: `mysqldump` writes SQL to standard output where `pg_dump -Fc` -/// writes an archive to a path, `mongodump` takes a URI rather than host and port flags, `sqlite3` -/// needs no network arguments at all, and DuckDB runs no binary because its dump is a statement its -/// own in-process engine executes. So a descriptor names its mechanism and supplies its own -/// arguments or statements, rather than filling slots in one shared command shape. +// The three engines that had this hard-coded were all PostgreSQL. Generalizing it is not a matter +// of parameterizing a binary name: `mysqldump` writes SQL to standard output where `pg_dump -Fc` +// writes an archive to a path, `mongodump` takes a URI rather than host and port flags, `sqlite3` +// needs no network arguments at all, and DuckDB runs no binary because its dump is a statement its +// own in-process engine executes. So a descriptor names its mechanism and supplies its own +// arguments or statements, rather than filling slots in one shared command shape. struct NativeDumpDescriptor: Sendable { - /// How the tool hands back what it produced. This is the difference that no shared argument - /// list can paper over. enum OutputDelivery: Sendable, Equatable { - /// The tool is told where to write, and writes there itself. case toolWritesFile - /// The tool writes to standard output and the caller redirects it to the file. case standardOutput } - /// What the caller offers as a file name, and what a restore will accept back. - /// - /// `id` exists because one engine can offer two: DuckDB writes either a single `.duckdb` file - /// or a directory of Parquet, and the sheet has to name which one it is about to write. + // `id` exists because one engine can offer two: DuckDB writes either a single `.duckdb` file + // or a directory of Parquet, and the sheet has to name which one it is about to write. struct ArchiveFormat: Sendable, Equatable, Identifiable { let id: String let fileExtension: String let contentDescription: String - /// A destination the user chooses with an open panel rather than a save panel, because the - /// engine writes a folder of files rather than one file. + // A destination the user chooses with an open panel rather than a save panel, because the + // engine writes a folder of files rather than one file. let producesDirectory: Bool init( @@ -56,25 +48,19 @@ struct NativeDumpDescriptor: Sendable { let password: String? let scope: NativeDumpScope - /// The engine's own name for the database in front of it, read from the live connection. - /// - /// Only DuckDB needs one, and it needs it badly: `COPY FROM DATABASE TO ` names - /// a catalog, and DuckDB derives that catalog from the file's basename. Measured, a file - /// `my-weird.name.db` attaches as `my-weird`, and `COPY FROM DATABASE main` fails with - /// `Catalog "main" does not exist`. So neither the file name nor a constant will do. + // Only DuckDB needs one, and it needs it badly: `COPY FROM DATABASE TO ` names + // a catalog, and DuckDB derives that catalog from the file's basename. Measured, a file + // `my-weird.name.db` attaches as `my-weird`, and `COPY FROM DATABASE main` fails with + // `Catalog "main" does not exist`. So neither the file name nor a constant will do. let currentCatalog: String? - /// The path a file-backed driver actually opens. - /// - /// SQLite keeps it in `database`, while DuckDB and libSQL keep it in a plugin-declared - /// additional field and leave `database` empty. Resolved by the caller through - /// `LocalFilePathField` rather than guessed here, because reading `connection.database` for - /// libSQL handed `sqlite3` an empty path, which exits 0 and writes a 52-byte file that the - /// result sheet then reports as a successful backup. + // SQLite keeps the file's path in `database`; DuckDB and libSQL keep it in a plugin-declared field and leave + // `database` empty. Resolved by the caller through `LocalFilePathField`: reading `connection.database` for libSQL + // handed `sqlite3` an empty path, which exits 0 and writes a 52-byte file the result sheet reports as a backup. let localFilePath: String? - /// What the live session's driver reports the server to be, so a tool can be given the - /// flags that server needs. `mysqldump` 8.0 reads a table no MariaDB server has. + // What the live session's driver reports the server to be, so a tool can be given the + // flags that server needs. `mysqldump` 8.0 reads a table no MariaDB server has. let serverVersion: String? // Whether the server's greeting offered TLS, for a tool whose SSL options depend on it. Nil when not read. @@ -83,6 +69,9 @@ struct NativeDumpDescriptor: Sendable { // Set by `NativeDumpService.buildCommand` before the arguments are built. var clientPluginFolder: MariaDBClientPluginFolder? + // Set by `NativeDumpService` when the tool's `needsSystemTrustAnchors` says so, and removed when the tool exits. + var systemTrustAnchorsFile: URL? + init( connection: DatabaseConnection, database: String, @@ -120,25 +109,24 @@ struct NativeDumpDescriptor: Sendable { let value: String } - /// A tool on the user's Mac that the app spawns. struct CommandLineTool: Sendable { - /// The candidate names, in the order they are tried. More than one because a tool can ship - /// under two names: MariaDB renamed `mysqldump` to `mariadb-dump` in 11.0 and keeps the old - /// name only as a symlink that some builds omit. + // The candidate names, in the order they are tried. More than one because a tool can ship + // under two names: MariaDB renamed `mysqldump` to `mariadb-dump` in 11.0 and keeps the old + // name only as a symlink that some builds omit. let backupBinaries: [String] let restoreBinaries: [String] - /// What to tell the user to install when neither name resolves. There is no portable - /// answer, so each engine names its own package. + // What to tell the user to install when neither name resolves. There is no portable + // answer, so each engine names its own package. let installHint: String let backupDelivery: OutputDelivery let restoreDelivery: OutputDelivery - /// True when the tool has no channel for a password but its own argument list, which every - /// process on the machine can read through `ps`. Only `sqlpackage` is in that position; - /// every other engine here takes one from the environment or a file. The flows that touch - /// such a tool say so rather than leaving the user to find out. + // True when the tool has no channel for a password but its own argument list, which every + // process on the machine can read through `ps`. Only `sqlpackage` is in that position; + // every other engine here takes one from the environment or a file. The flows that touch + // such a tool say so rather than leaving the user to find out. let exposesPasswordInArguments: Bool let configurationFileEntries: @Sendable (Request) -> [ConfigurationEntry] @@ -147,14 +135,14 @@ struct NativeDumpDescriptor: Sendable { internal let requiresUntranslatedMessages: Bool - /// Picks the binary to run for a known server version, when the engine's tools refuse - /// some servers. Nil leaves the plain PATH lookup in place. + // Picks the binary to run for a known server version, when the engine's tools refuse + // some servers. Nil leaves the plain PATH lookup in place. let toolForServer: (@Sendable (_ binary: String, _ serverVersion: String?) -> NativeDumpToolSelection)? - /// Says what the resolved binary actually is, for an engine whose tools forked their - /// options. It runs a process, so it runs once per resolution rather than once per - /// argument list. Nil leaves the tool unidentified, which is what every engine but MySQL - /// and MariaDB wants. + // Says what the resolved binary actually is, for an engine whose tools forked their + // options. It runs a process, so it runs once per resolution rather than once per + // argument list. Nil leaves the tool unidentified, which is what every engine but MySQL + // and MariaDB wants. let identifyExecutable: (@Sendable (_ name: String, _ path: String) -> NativeDumpResolvedTool)? let backupArguments: @Sendable (Request, NativeDumpResolvedTool) throws -> [String] @@ -166,6 +154,9 @@ struct NativeDumpDescriptor: Sendable { let clientPluginFolder: @Sendable (Request, NativeDumpResolvedTool) throws -> MariaDBClientPluginFolder? + // True when the tool has no trust store of its own to check the server with, so it gets a file of this Mac's. + let needsSystemTrustAnchors: @Sendable (Request) -> Bool + init( backupBinaries: [String], restoreBinaries: [String], @@ -182,7 +173,8 @@ struct NativeDumpDescriptor: Sendable { environment: @escaping @Sendable (Request) -> [String: String] = { _ in [:] }, configurationFileEntries: @escaping @Sendable (Request) -> [ConfigurationEntry] = { _ in [] }, prepare: @escaping @Sendable (Request, NativeDumpResolvedTool) async throws -> Request = { request, _ in request }, - clientPluginFolder: @escaping @Sendable (Request, NativeDumpResolvedTool) throws -> MariaDBClientPluginFolder? = { _, _ in nil } + clientPluginFolder: @escaping @Sendable (Request, NativeDumpResolvedTool) throws -> MariaDBClientPluginFolder? = { _, _ in nil }, + needsSystemTrustAnchors: @escaping @Sendable (Request) -> Bool = { _ in false } ) { self.backupBinaries = backupBinaries self.restoreBinaries = restoreBinaries @@ -200,6 +192,7 @@ struct NativeDumpDescriptor: Sendable { self.configurationFileEntries = configurationFileEntries self.prepare = prepare self.clientPluginFolder = clientPluginFolder + self.needsSystemTrustAnchors = needsSystemTrustAnchors } func binaries(for kind: NativeDumpKind) -> [String] { @@ -214,8 +207,8 @@ struct NativeDumpDescriptor: Sendable { try kind == .backup ? backupArguments(request, resolved) : restoreArguments(request, resolved) } - /// What the app knows about the binary it resolved. An engine that declares no - /// `identifyExecutable` gets the plain answer, which is what its arguments already assume. + // What the app knows about the binary it resolved. An engine that declares no + // `identifyExecutable` gets the plain answer, which is what its arguments already assume. func identify(name: String, path: String) -> NativeDumpResolvedTool { identifyExecutable?(name, path) ?? NativeDumpResolvedTool(name: name, path: path) } @@ -229,19 +222,17 @@ struct NativeDumpDescriptor: Sendable { } } - /// Statements the engine already in front of the app runs for itself. - /// - /// There is no binary to find, no password to hand over and nothing to reap, which is why this - /// cannot be a `CommandLineTool` with an empty argument list. An embedded engine also has no - /// server to write on the user's behalf, so the file lands where the panel picked. + // There is no binary to find, no password to hand over and nothing to reap, which is why this + // cannot be a `CommandLineTool` with an empty argument list. An embedded engine also has no + // server to write on the user's behalf, so the file lands where the panel picked. struct EngineStatements: Sendable { - /// The statements for one direction, in order. `alias` is a name for the attached backup, - /// unique per run so a `DETACH` that could not run leaves nothing blocking the next one. + // The statements for one direction, in order. `alias` is a name for the attached backup, + // unique per run so a `DETACH` that could not run leaves nothing blocking the next one. let backupStatements: @Sendable (Request, String) -> [String] let restoreStatements: @Sendable (Request, String) -> [String] - /// Run after a failure and ignored. A `DETACH` inside an aborted transaction fails too, and - /// the caller has the engine's real error to report rather than this one. + // Run after a failure and ignored. A `DETACH` inside an aborted transaction fails too, and + // the caller has the engine's real error to report rather than this one. let cleanupStatements: @Sendable (String) -> [String] // Set when the archive's own SQL must not run on the session: the driver imports it into a new file of this @@ -271,9 +262,9 @@ struct NativeDumpDescriptor: Sendable { let objectScope: NativeDumpObjectScope let restoreSemantics: RestoreSemantics - /// True when the tool opens the database file itself and so cannot reach a connection that has - /// no local file. libSQL claims the SQLite descriptor and reaches either a file or a Turso URL, - /// and only the file can be handed to `sqlite3`. + // True when the tool opens the database file itself and so cannot reach a connection that has + // no local file. libSQL claims the SQLite descriptor and reaches either a file or a Turso URL, + // and only the file can be handed to `sqlite3`. let requiresLocalFile: Bool init( diff --git a/apple/TablePro/Core/Database/NativeDumpRegistry.swift b/apple/TablePro/Core/Database/NativeDumpRegistry.swift index d1ec365aa5..88421a5b11 100644 --- a/apple/TablePro/Core/Database/NativeDumpRegistry.swift +++ b/apple/TablePro/Core/Database/NativeDumpRegistry.swift @@ -6,21 +6,19 @@ import Foundation import TableProPluginKit import TableProSQLQuoting - -/// Which engines can dump and restore, and how to drive them. -/// -/// Curated here rather than declared by each driver plugin, because the facts are about tools on -/// the user's Mac rather than about the driver's own behaviour, and adding a `DriverPlugin` static -/// for them would mean an ABI change and a re-release of every registry plugin for data no plugin -/// produces. -/// -/// A password never reaches the argument list. Anything in `argv` is readable by every process on -/// the machine through `ps`, so each engine's own out-of-band channel is used instead: `PGPASSWORD` -/// and `MYSQL_PWD` for the two that read the environment, and a `0600` config file for MongoDB, -/// whose tools read neither. +import TableProTLSClientIdentity + +// Curated here rather than declared by each driver plugin, because the facts are about tools on +// the user's Mac rather than about the driver's own behaviour, and adding a `DriverPlugin` static +// for them would mean an ABI change and a re-release of every registry plugin for data no plugin +// produces. +// A password never reaches the argument list. Anything in `argv` is readable by every process on +// the machine through `ps`, so each engine's own out-of-band channel is used instead: `PGPASSWORD` +// and `MYSQL_PWD` for the two that read the environment, and a `0600` config file for MongoDB, +// whose tools read neither. enum NativeDumpRegistry { - /// The formats one engine can write, in the order the sheet offers them. The first is the - /// default. Only DuckDB has more than one. + // The formats one engine can write, in the order the sheet offers them. The first is the + // default. Only DuckDB has more than one. static func formats(for type: DatabaseType) -> [NativeDumpDescriptor.ArchiveFormat] { switch type { case .duckdb: @@ -58,12 +56,11 @@ enum NativeDumpRegistry { descriptor(for: type) != nil } - /// Whether this particular connection can be dumped, which is not always the same question as - /// whether its engine can. - /// - /// libSQL reaches either a local file or a Turso URL, and only the file can be handed to - /// `sqlite3`. Answering by type alone is how a remote libSQL connection produced a 52-byte - /// backup the result sheet reported as a success. + // Whether this particular connection can be dumped, which is not always the same question as + // whether its engine can. + // libSQL reaches either a local file or a Turso URL, and only the file can be handed to + // `sqlite3`. Answering by type alone is how a remote libSQL connection produced a 52-byte + // backup the result sheet reported as a success. static func supports(_ connection: DatabaseConnection, localFilePath: String?) -> Bool { guard let descriptor = descriptor(for: connection.type) else { return false } guard descriptor.requiresLocalFile else { return true } @@ -73,8 +70,8 @@ enum NativeDumpRegistry { // MARK: - PostgreSQL - /// Redshift reports 8.0.2, which no `pg_dump` release claims to support, so only PostgreSQL - /// itself is matched to the server. + // Redshift reports 8.0.2, which no `pg_dump` release claims to support, so only PostgreSQL + // itself is matched to the server. static let postgresToolForServer: @Sendable (String, String?) -> NativeDumpToolSelection = { binary, serverVersion in PostgreSQLDumpToolLocator.select(binary: binary, serverVersion: serverVersion) } @@ -121,10 +118,14 @@ enum NativeDumpRegistry { environment["PGHOSTADDR"] = hostaddr } environment.merge( - postgresSSLEnvironment(request.connection.transportSSLConfiguration) + postgresSSLEnvironment( + request.connection.transportSSLConfiguration, + systemTrustAnchorsFile: request.systemTrustAnchorsFile + ) ) { _, new in new } return environment - } + }, + needsSystemTrustAnchors: { request in postgresNeedsSystemTrustAnchors(request) } ) ), archiveFormat: NativeDumpDescriptor.ArchiveFormat( @@ -186,18 +187,23 @@ enum NativeDumpRegistry { } } - /// The whole of the connection's SSL configuration, not just its mode. - /// - /// libpq falls back to `~/.postgresql/root.crt` when no root certificate is named, and measured - /// with pg_dump 17.11 a `verify-ca` connection with no such file fails before connecting with - /// `root certificate file "..." does not exist`. So a Verify CA connection that opens in the - /// app could never be dumped. The rule matches what `LibPQConnectionString` already sends on - /// the live connection, down to the CA being tied to the modes that verify. - static func postgresSSLEnvironment(_ ssl: SSLConfiguration) -> [String: String] { + // What `LibPQConnectionString` sends on the live connection. Measured with pg_dump 17.11, a verifying mode with no + // root file named and no `~/.postgresql/root.crt` fails before connecting with `root certificate file "..." does not + // exist`. + static func postgresSSLEnvironment( + _ ssl: SSLConfiguration, + systemTrustAnchorsFile: URL? = nil, + inherited: [String: String] = ProcessInfo.processInfo.environment + ) -> [String: String] { guard ssl.isEnabled, let mode = postgresSSLMode(ssl.mode) else { return [:] } var environment = ["PGSSLMODE": mode] if ssl.verifiesCertificate, !ssl.caCertificatePath.isEmpty { environment["PGSSLROOTCERT"] = ssl.caCertificatePath + } else if ssl.verifiesHostname, let systemTrustAnchorsFile { + environment["PGSSLROOTCERT"] = systemTrustAnchorsFile.path + } else if ssl.verifiesCertificate, let named = inherited["PGSSLROOTCERT"], !named.isEmpty { + // The tool starts from a minimal environment, so a root file the user named for libpq is carried to it. + environment["PGSSLROOTCERT"] = named } if !ssl.clientCertificatePath.isEmpty { environment["PGSSLCERT"] = ssl.clientCertificatePath @@ -208,10 +214,32 @@ enum NativeDumpRegistry { return environment } + // The driver's rule, `LibPQConnectionString.rootCertificateSource`: Verify Identity with no CA file checks this Mac's + // trusted roots, unless the server is a socket or libpq's own `~/.postgresql/root.crt` is there. The tool inherits + // HOME, so it looks for that file where this does. + static func postgresNeedsSystemTrustAnchors( + _ request: NativeDumpDescriptor.Request, + environment: [String: String] = ProcessInfo.processInfo.environment, + fileExists: (String) -> Bool = { FileManager.default.fileExists(atPath: $0) } + ) -> Bool { + let ssl = request.connection.transportSSLConfiguration + return LibPQSystemTrustRule.usesSystemTrustAnchors( + verifiesHostname: ssl.verifiesHostname, + caCertificatePath: ssl.caCertificatePath, + host: postgresEndpoint(request).host, + environment: environment, + fileExists: fileExists + ) + } + + static func postgresDefaultRootCertificatePath(environment: [String: String]) -> String { + LibPQSystemTrustRule.defaultRootCertificatePath(environment: environment) + } + // MARK: - MySQL and MariaDB - /// MariaDB 11.0 renamed every client, keeping the `mysql`-prefixed names as symlinks that some - /// builds leave out, so both spellings are tried. + // MariaDB 11.0 renamed every client, keeping the `mysql`-prefixed names as symlinks that some + // builds leave out, so both spellings are tried. private static var mysql: NativeDumpDescriptor { NativeDumpDescriptor( mechanism: .commandLineTool( @@ -269,12 +297,12 @@ enum NativeDumpRegistry { ) } - /// `--` before the database, because `my_getopt` does not stop parsing options at the first - /// positional argument and every name after it is the user's. Measured with mysqldump 8.4.11 - /// and 12.3.2, a database named `--no-data` with the terminator behind it dumped a *different* - /// database with no rows and exited 0, which the result sheet reports as a successful backup; - /// a table named `--ssl-mode=DISABLED` in the same slot sends the whole dump in cleartext. With - /// `--` in front of the database, both are names again. + // `--` before the database, because `my_getopt` does not stop parsing options at the first + // positional argument and every name after it is the user's. Measured with mysqldump 8.4.11 + // and 12.3.2, a database named `--no-data` with the terminator behind it dumped a *different* + // database with no rows and exited 0, which the result sheet reports as a successful backup; + // a table named `--ssl-mode=DISABLED` in the same slot sends the whole dump in cleartext. With + // `--` in front of the database, both are names again. private static func mysqlObjectArguments(_ request: NativeDumpDescriptor.Request) -> [String] { ["--", request.database] + request.scope.objects.map(\.name) } @@ -390,9 +418,9 @@ enum NativeDumpRegistry { // MARK: - MongoDB - /// `mongodump` reads no password from the environment, and a password in `argv` is readable by - /// every process on the machine. Its `--config` file is the only channel left, so the caller - /// writes one at mode `0600` and passes its path; `NativeDumpService` removes it afterwards. + // `mongodump` reads no password from the environment, and a password in `argv` is readable by + // every process on the machine. Its `--config` file is the only channel left, so the caller + // writes one at mode `0600` and passes its path; `NativeDumpService` removes it afterwards. private static var mongodb: NativeDumpDescriptor { NativeDumpDescriptor( mechanism: .commandLineTool( @@ -432,8 +460,8 @@ enum NativeDumpRegistry { // The tools check the name in the URI, 127.0.0.1 through a tunnel, and no option names another. Verify CA skips it. private static let mongoModesThroughTunnel: [SSLMode] = [.disabled, .preferred, .required, .verifyCa] - /// A whole-database dump names the database; a narrowed one names each collection, because - /// `--db` plus `--collection` takes only one collection at a time while `--nsInclude` repeats. + // A whole-database dump names the database; a narrowed one names each collection, because + // `--db` plus `--collection` takes only one collection at a time while `--nsInclude` repeats. private static func mongoNamespaceFlags(_ request: NativeDumpDescriptor.Request) -> [String] { guard !request.scope.isWholeDatabase else { return ["--db=\(request.database)"] } return request.scope.objects.map { object in @@ -458,13 +486,12 @@ enum NativeDumpRegistry { // MARK: - SQL Server - /// `sqlpackage` is Microsoft's own client-side export, and the only one that writes a whole - /// database to a local file. `bcp` moves one table at a time and produces no schema, so it - /// cannot stand in for a dump. - /// - /// Not verified against a live server here: the arguments come from Microsoft's documented CLI - /// surface rather than from a run, so a mistake in them surfaces as the tool's own usage error - /// rather than a wrong file. + // `sqlpackage` is Microsoft's own client-side export, and the only one that writes a whole + // database to a local file. `bcp` moves one table at a time and produces no schema, so it + // cannot stand in for a dump. + // Not verified against a live server here: the arguments come from Microsoft's documented CLI + // surface rather than from a run, so a mistake in them surfaces as the tool's own usage error + // rather than a wrong file. private static var sqlServer: NativeDumpDescriptor { NativeDumpDescriptor( mechanism: .commandLineTool( @@ -522,10 +549,10 @@ enum NativeDumpRegistry { return "\(schema).\(object.name)" } - /// `sqlpackage` takes no password flag and no password environment variable: the connection - /// string is the only channel it has. That puts the password in `argv`, where `ps` can read it, - /// so the string is built here and `NativeDumpService` keeps SQL Server out of the flows that - /// promise otherwise. Anything better needs Entra or integrated auth, which have no password. + // `sqlpackage` takes no password flag and no password environment variable: the connection + // string is the only channel it has. That puts the password in `argv`, where `ps` can read it, + // so the string is built here and `NativeDumpService` keeps SQL Server out of the flows that + // promise otherwise. Anything better needs Entra or integrated auth, which have no password. private static func sqlServerConnectionString(_ request: NativeDumpDescriptor.Request) throws -> String { let quoted = NativeDumpArgumentQuoting.sqlServerConnectionStringValue var parts = try [ @@ -559,12 +586,11 @@ enum NativeDumpRegistry { // MARK: - SQLite - /// The database is a file the tool opens directly, so there is no host, port or password, and - /// `.dump` writes SQL to standard output. - /// - /// The path comes from `localFilePath` rather than from `connection.database`. libSQL claims - /// this descriptor too and keeps its path in a plugin-declared additional field, so reading - /// `database` handed `sqlite3` an empty string, which exits 0 after writing a 52-byte file. + // The database is a file the tool opens directly, so there is no host, port or password, and + // `.dump` writes SQL to standard output. + // The path comes from `localFilePath` rather than from `connection.database`. libSQL claims + // this descriptor too and keeps its path in a plugin-declared additional field, so reading + // `database` handed `sqlite3` an empty string, which exits 0 after writing a 52-byte file. private static var sqlite: NativeDumpDescriptor { NativeDumpDescriptor( mechanism: .commandLineTool( diff --git a/apple/TablePro/Core/Database/NativeDumpService.swift b/apple/TablePro/Core/Database/NativeDumpService.swift index 33b825cf0a..6e8bee1bfa 100644 --- a/apple/TablePro/Core/Database/NativeDumpService.swift +++ b/apple/TablePro/Core/Database/NativeDumpService.swift @@ -14,25 +14,24 @@ import os import TableProConnectionLibrary import TableProPluginKit import TableProProcess +import TableProTLSClientIdentity import Yams // MARK: - Public Types -/// What the service is doing: dump (back up) a database or restore a dump file. enum NativeDumpKind: Equatable, Sendable { case backup case restore } -/// Observable state of a backup or restore. enum NativeDumpState: Equatable { case idle case running(database: String, fileURL: URL, bytesProcessed: Int64, totalBytes: Int64?) case cancelling case finished(database: String, fileURL: URL, bytesProcessed: Int64, skippedSettings: [String] = []) - /// A restore that fails part way through has already replayed some of the dump, and the - /// target is left in whatever state that reached. A backup writes only to its own file, which - /// is removed, so nothing of the user's is touched. + // A restore that fails part way through has already replayed some of the dump, and the + // target is left in whatever state that reached. A backup writes only to its own file, which + // is removed, so nothing of the user's is touched. case failed(message: String, targetMayBeModified: Bool) case cancelled } @@ -48,6 +47,7 @@ enum NativeDumpError: LocalizedError, Equatable { case unreachableServer(message: String) case nulInArgument case uncheckedThroughTunnel(tool: String, mode: SSLMode, supportedModes: [SSLMode]) + case systemTrustUnavailable(message: String) var errorDescription: String? { switch self { @@ -67,7 +67,7 @@ enum NativeDumpError: LocalizedError, Equatable { return String(localized: "The selected backup file is not readable.") case .engineStatementUnavailable: return String(localized: "TablePro could not read the database's own name from the connection.") - case .incompatibleTool(let message), .unreachableServer(let message): + case .incompatibleTool(let message), .unreachableServer(let message), .systemTrustUnavailable(let message): return message case .nulInArgument: return String(localized: "The database name or a sign-in value holds a NUL character, which a command line cannot carry.") @@ -98,29 +98,28 @@ internal enum NativeDumpExitPolicy: Equatable, Sendable { } } -/// Parameters for a single backup or restore subprocess. struct NativeDumpCommand: Equatable { let executable: URL let arguments: [String] let environment: [String: String] let stderrByteCap: Int - /// Where the tool's output goes. `pg_dump -Fc` is told a path and writes it itself, while - /// `mysqldump` and `sqlite3 .dump` write SQL to standard output for the caller to redirect. + // Where the tool's output goes. `pg_dump -Fc` is told a path and writes it itself, while + // `mysqldump` and `sqlite3 .dump` write SQL to standard output for the caller to redirect. let delivery: NativeDumpDescriptor.OutputDelivery - /// The file standard output is redirected to, or read from on a restore. Nil when the tool - /// handles the file itself. + // The file standard output is redirected to, or read from on a restore. Nil when the tool + // handles the file itself. let redirectedFileURL: URL? - /// A credentials file written for tools that read a password from neither the environment nor - /// standard input. Removed once the process exits, however it exits. + // A credentials file written for tools that read a password from neither the environment nor + // standard input. Removed once the process exits, however it exits. let temporaryCredentialsFileURL: URL? let clientPluginFolder: MariaDBClientPluginFolder? - /// Which direction a redirected file goes: a restore feeds it in as standard input, a backup - /// takes standard output and writes it out. + // Which direction a redirected file goes: a restore feeds it in as standard input, a backup + // takes standard output and writes it out. let isRestore: Bool internal let exitPolicy: NativeDumpExitPolicy @@ -150,11 +149,10 @@ struct NativeDumpCommand: Equatable { } } -/// Parameters for a dump the engine runs for itself. struct NativeDumpStatementJob: Sendable, Equatable { let statements: [String] - /// Run after a failure and ignored. A `DETACH` inside a transaction the engine already aborted - /// fails too, and the caller has the real error to report instead of this one. + // Run after a failure and ignored. A `DETACH` inside a transaction the engine already aborted + // fails too, and the caller has the real error to report instead of this one. let cleanupStatements: [String] let scope: DatabaseScope var isolatedImport: NativeDumpIsolatedImport? @@ -168,8 +166,8 @@ struct NativeDumpIsolatedImport: Sendable, Equatable { let file: URL } -/// What a run is, so the service can pick a runner without either runner accepting a shape it -/// cannot execute. +// What a run is, so the service can pick a runner without either runner accepting a shape it +// cannot execute. enum NativeDumpJob { case process(NativeDumpCommand) case statements(NativeDumpStatementJob) @@ -201,7 +199,6 @@ internal extension NativeDumpJob { } } -/// Captured terminal state of a finished/cancelled run. struct NativeDumpRunResult: Equatable { let exitCode: Int32 let stderr: String @@ -209,14 +206,12 @@ struct NativeDumpRunResult: Equatable { var targetUntouched = false } -/// Runs one backup or restore and reports how it ended. Abstracted so the state machine can be -/// tested without launching a process or reaching a database. +// Runs one backup or restore and reports how it ended. Abstracted so the state machine can be +// tested without launching a process or reaching a database. protocol NativeDumpRunner: AnyObject { - /// Begins the work. Throws synchronously if it cannot be started at all. func start() throws - /// Asks for cancellation. Safe to call more than once. + // Safe to call more than once. func cancel() - /// Resolves once the work has ended, however it ended. var result: NativeDumpRunResult { get async } // How far a restore has read its file, or nil when the runner cannot tell. var bytesRead: Int64? { get } @@ -244,6 +239,8 @@ final class NativeDumpService: ObservableObject { private var preparation: Task? // The Vault lease the tool signs in with, held until the tool exits. private var credentialLease: VaultLease? + // The roots file the tool checks the server with, removed when the tool exits. + private var systemTrustAnchors: SystemTrustAnchorFile? private let runningWork: RunningWorkRegistry private let reportCompletion: @MainActor (OperationCompletion) -> Void private var trackedRestore: TrackedRestore? @@ -273,13 +270,13 @@ final class NativeDumpService: ObservableObject { state = newState switch newState { case .finished: - releaseCredentialLease() + releaseRunResources() finishTrackedRestore(.succeeded(OperationSummary())) case .failed(let message, _): - releaseCredentialLease() + releaseRunResources() finishTrackedRestore(.failed(reason: message)) case .cancelled: - releaseCredentialLease() + releaseRunResources() finishTrackedRestore(.cancelled) case .idle, .running, .cancelling: break @@ -417,12 +414,12 @@ final class NativeDumpService: ObservableObject { effective.username = username } let password = signIn.password - releaseCredentialLease() + releaseRunResources() credentialLease = signIn.lease signIn.lease?.start { _ in } var jobStarted = false defer { - if !jobStarted { releaseCredentialLease() } + if !jobStarted { releaseRunResources() } } let localFilePath = Self.localFilePath(for: effective) let catalog = descriptor.engineStatements == nil @@ -453,6 +450,7 @@ final class NativeDumpService: ObservableObject { return } request = prepared + request.systemTrustAnchorsFile = try await holdSystemTrustAnchors(for: request, tool: tool) let command = try Self.buildCommand( kind: kind, tool: tool, @@ -480,9 +478,29 @@ final class NativeDumpService: ObservableObject { Self.logger.info("\(self.toolName, privacy: .public) started db=\(database, privacy: .private(mask: .hash))") } - private func releaseCredentialLease() { + private func releaseRunResources() { credentialLease?.release() credentialLease = nil + systemTrustAnchors?.remove() + systemTrustAnchors = nil + } + + // Off the main actor: the first build in a minute reads every trust setting, 150 to 275 ms measured. + private func holdSystemTrustAnchors( + for request: NativeDumpDescriptor.Request, + tool: NativeDumpDescriptor.CommandLineTool + ) async throws -> URL? { + let file = try await Task.detached { () throws -> SystemTrustAnchorFile? in + guard tool.needsSystemTrustAnchors(request) else { return nil } + do { + return try SystemTrustAnchors.shared.writeFile() + } catch { + throw NativeDumpError.systemTrustUnavailable(message: error.localizedDescription) + } + }.value + systemTrustAnchors?.remove() + systemTrustAnchors = file + return file?.url } // Nil when a cancel arrived while it ran: the checks it was making stop, and the run reads as cancelled rather than @@ -561,12 +579,11 @@ final class NativeDumpService: ObservableObject { } } - /// A cancel that arrives before the tool is running is latched rather than dropped. - /// - /// `start` finds the binary and asks it what it is before there is anything to cancel, and both - /// steps spawn a process, so the window is real: a Cancel taken during it used to be discarded - /// and the tool launched afterwards, which on a restore means writing to the target database - /// the user just said to leave alone. + // A cancel that arrives before the tool is running is latched rather than dropped. + // `start` finds the binary and asks it what it is before there is anything to cancel, and both + // steps spawn a process, so the window is real: a Cancel taken during it used to be discarded + // and the tool launched afterwards, which on a restore means writing to the target database + // the user just said to leave alone. func cancel() { cancelRequested = true preparation?.cancel() @@ -634,21 +651,18 @@ final class NativeDumpService: ObservableObject { return refusal.map { .incompatible($0) } ?? .missing } - /// The path a file-backed driver actually opens, read from wherever that driver keeps it. - /// SQLite uses `database`; DuckDB and libSQL use a plugin-declared additional field. + // SQLite uses `database`; DuckDB and libSQL use a plugin-declared additional field. static func localFilePath(for connection: DatabaseConnection) -> String? { guard let field = PluginManager.shared.localFilePathField(for: connection.type) else { return nil } let path = connection.localFilePath(in: field) return path.isEmpty ? nil : path } - /// The engine's own name for the database being dumped. - /// - /// `COPY FROM DATABASE` names a catalog, and DuckDB derives a catalog name from the file's - /// basename: measured, `my-weird.name.db` attaches as `my-weird`, and `COPY FROM DATABASE main` - /// fails with `Catalog "main" does not exist`. DuckDB also reaches more than one catalog at once - /// through `ATTACH`, so the answer is the picked database when the engine lists it, and only - /// otherwise the one the session is on. + // `COPY FROM DATABASE` names a catalog, and DuckDB derives a catalog name from the file's + // basename: measured, `my-weird.name.db` attaches as `my-weird`, and `COPY FROM DATABASE main` + // fails with `Catalog "main" does not exist`. DuckDB also reaches more than one catalog at once + // through `ATTACH`, so the answer is the picked database when the engine lists it, and only + // otherwise the one the session is on. static func resolvedCatalog(connectionId: UUID, database: String) async -> String? { guard let scope = DatabaseManager.shared.resolvedScope( database: database, schema: nil, for: connectionId @@ -664,9 +678,9 @@ final class NativeDumpService: ObservableObject { } } - /// Pure so the choice is testable without an engine. The picked name wins when the engine says - /// it exists, because a connection with two catalogs attached has to dump the one that was - /// ticked rather than whichever the session is parked on. + // Pure so the choice is testable without an engine. The picked name wins when the engine says + // it exists, because a connection with two catalogs attached has to dump the one that was + // ticked rather than whichever the session is parked on. nonisolated static func catalogName( picked: String, reported: [String], @@ -716,18 +730,18 @@ final class NativeDumpService: ObservableObject { ) } - /// Unique per run, so a `DETACH` that could not execute leaves nothing behind that blocks the - /// next attempt with "database with name already exists". + // Unique per run, so a `DETACH` that could not execute leaves nothing behind that blocks the + // next attempt with "database with name already exists". static func backupAlias() -> String { "tablepro_backup_\(UUID().uuidString.replacingOccurrences(of: "-", with: "").prefix(12))" } // MARK: - Command Construction - /// The descriptor supplies the arguments and the environment; this adds the parts every tool - /// shares. A password never joins the argument list, because `argv` is world readable through - /// `ps`: it goes in the environment where the tool reads one, and in a `0600` file where it - /// does not. + // The descriptor supplies the arguments and the environment; this adds the parts every tool + // shares. A password never joins the argument list, because `argv` is world readable through + // `ps`: it goes in the environment where the tool reads one, and in a `0600` file where it + // does not. nonisolated static func buildCommand( kind: NativeDumpKind, tool: NativeDumpDescriptor.CommandLineTool, @@ -773,9 +787,9 @@ final class NativeDumpService: ObservableObject { ) } - /// `mongodump` and `mongorestore` read a password from neither the environment nor standard - /// input, and one in `argv` is readable by every process on the machine. Their `--config` file - /// is the remaining channel, so it is written owner-only and removed when the process exits. + // `mongodump` and `mongorestore` read a password from neither the environment nor standard + // input, and one in `argv` is readable by every process on the machine. Their `--config` file + // is the remaining channel, so it is written owner-only and removed when the process exits. nonisolated static func writeMongoConfigurationFile( _ entries: [NativeDumpDescriptor.ConfigurationEntry] ) throws -> URL { @@ -799,10 +813,10 @@ final class NativeDumpService: ObservableObject { return try Yams.serialize(node: .mapping(Node.Mapping(pairs)), width: -1, allowUnicode: true) } - /// The locale variables a spawned tool reads, named rather than spelled at each use site. The - /// preference-key guard scans this target's sources for a quoted string passed to a `forKey:` - /// label and reports anything outside the `com.TablePro` namespace, so a locale variable - /// written that way reads as a stray `UserDefaults` key. Comments are scanned too. + // The locale variables a spawned tool reads, named rather than spelled at each use site. The + // preference-key guard scans this target's sources for a quoted string passed to a `forKey:` + // label and reports anything outside the `com.TablePro` namespace, so a locale variable + // written that way reads as a stray `UserDefaults` key. Comments are scanned too. nonisolated internal enum LocaleEnvironmentKey { static let everyCategory = "LC_ALL" static let characterHandling = "LC_CTYPE" @@ -875,9 +889,9 @@ final class NativeDumpService: ObservableObject { Self.logger.error("\(self.toolName, privacy: .public) failed code=\(result.exitCode) db=\(database, privacy: .private(mask: .hash)) stderr=\(result.stderr)") } - /// A DuckDB Parquet backup is a folder, so its size is the sum of what is inside it and its - /// removal is recursive. `removeItem` already handles a directory; `attributesOfItem` does not - /// sum one. + // A DuckDB Parquet backup is a folder, so its size is the sum of what is inside it and its + // removal is recursive. `removeItem` already handles a directory; `attributesOfItem` does not + // sum one. nonisolated static func sizeOfDestination(_ url: URL) -> Int64 { let manager = FileManager.default var isDirectory: ObjCBool = false @@ -937,13 +951,12 @@ final class NativeDumpService: ObservableObject { // MARK: - Database Size Helper extension NativeDumpService { - /// Best-effort estimate of the database's on-disk size. Used as an upper bound for the backup - /// progress bar; the dump file is typically much smaller because of compression, so the bar - /// tops out at the size and then jumps when the tool exits. Returns nil if the query fails. - /// - /// Routed through `withMetadataDriver` rather than taken straight off `driver(for:)`. On the - /// session driver this query queues behind whatever the user is running and joins whatever - /// transaction a query tab left open, which is the hazard `metadataRoute` exists to avoid. + // Best-effort estimate of the database's on-disk size. Used as an upper bound for the backup + // progress bar; the dump file is typically much smaller because of compression, so the bar + // tops out at the size and then jumps when the tool exits. + // Routed through `withMetadataDriver` rather than taken straight off `driver(for:)`. On the + // session driver this query queues behind whatever the user is running and joins whatever + // transaction a query tab left open, which is the hazard `metadataRoute` exists to avoid. static func estimatedDatabaseSize( connection: DatabaseConnection, database: String @@ -959,8 +972,8 @@ extension NativeDumpService { return Int64(text) } - /// An engine with no cheap size answer returns nil, which leaves the progress bar - /// indeterminate rather than showing a percentage of a number nobody measured. + // An engine with no cheap size answer returns nil, which leaves the progress bar + // indeterminate rather than showing a percentage of a number nobody measured. nonisolated static func sizeQuery(for type: DatabaseType) -> String? { switch type { case .postgresql, .redshift: diff --git a/apple/TablePro/Core/Database/SessionLanes.swift b/apple/TablePro/Core/Database/SessionLanes.swift index ebcd95ee09..b1c291d62b 100644 --- a/apple/TablePro/Core/Database/SessionLanes.swift +++ b/apple/TablePro/Core/Database/SessionLanes.swift @@ -5,6 +5,18 @@ import Foundation import os +import TableProSQLGrammar + +// What a connection replaced or closed without the user asking took with it: an open transaction, what the user's +// own statements set up, or both. +internal struct SessionLoss: Equatable, Sendable { + var transaction: Bool + var state: SQLSessionStateKinds + + func merging(_ other: SessionLoss) -> SessionLoss { + SessionLoss(transaction: transaction || other.transaction, state: state.union(other.state)) + } +} /// The user connections a session keeps open on databases other than the one it is browsing. /// @@ -33,9 +45,9 @@ internal final class SessionLanes { /// Drivers being checked before use. A turn held on one of these is not a query proving the /// driver answers, so a switch back waits for the check instead of promoting it on trust. private var verifying: [ObjectIdentifier: [CheckedContinuation]] = [:] - /// Databases whose connection died holding a transaction and was replaced. The next turn there - /// is refused once, so work meant to continue that transaction does not carry on without it. - private var lostTransactions: [UUID: Set] = [:] + // Databases whose connection died holding a transaction or session state and was replaced. The next turn there is + // refused once, so work meant for that session does not carry on without it. + private var losses: [UUID: [String: SessionLoss]] = [:] private struct Opened { weak var driver: AnyObject? @@ -46,8 +58,8 @@ internal final class SessionLanes { /// switched away from, nothing on screen closes one, so it is closed once idle. Browsing its /// database takes it off this list. private var opened: [UUID: [String: Opened]] = [:] - /// Databases whose opened connection was closed for being idle. Work queued for one reopens it: - /// no transaction was lost. The session's temp tables, `SET` values and advisory locks were. + // Databases whose opened connection was closed for being idle. Work queued for one reopens it. One holding a + // transaction or session state the user's statements set up is never closed idle. private var closedIdle: [UUID: Set] = [:] internal static let openedIdleTimeout: TimeInterval = MetadataConnectionPool.idleTimeout @@ -59,6 +71,9 @@ internal final class SessionLanes { /// Told after a parked connection closes, which ends any transaction it held. internal var onClose: (@MainActor (UUID) -> Void)? + // What the user's statements set up on a driver, from the ledger the session owns. + internal var recordedState: (@MainActor (DatabaseDriver, UUID) -> SQLSessionStateKinds)? + internal init(opener: Opener? = nil) { self.opener = opener ?? { scope in try await MetadataConnectionPool.openServerDriver(for: scope, purpose: .session) @@ -122,13 +137,13 @@ internal final class SessionLanes { /// Closes every parked connection of a session. A disconnect ends them, and so does a rebuilt /// transport: they were dialed through the tunnel port that is going away. Opens still in flight /// lose their generation, so one that lands afterwards closes its own driver. - internal func closeAll(for connectionId: UUID, keepingLostTransactions: Bool = false) { + internal func closeAll(for connectionId: UUID, keepingLosses: Bool = false) { generations[connectionId, default: 0] &+= 1 let drivers = parked.removeValue(forKey: connectionId) ?? [:] opened.removeValue(forKey: connectionId) closedIdle.removeValue(forKey: connectionId) - if !keepingLostTransactions { - lostTransactions.removeValue(forKey: connectionId) + if !keepingLosses { + losses.removeValue(forKey: connectionId) } for driver in drivers.values { verifiedAt.removeValue(forKey: ObjectIdentifier(driver)) @@ -196,35 +211,54 @@ internal final class SessionLanes { } } - internal func markTransactionLost(database: String, for connectionId: UUID) { - lostTransactions[connectionId, default: []].insert(database) + internal func markLost( + database: String, + transaction: Bool = false, + state: SQLSessionStateKinds = [], + for connectionId: UUID + ) { + guard transaction || !state.isEmpty else { return } + let loss = SessionLoss(transaction: transaction, state: state) + losses[connectionId, default: [:]][database] = losses[connectionId]?[database]?.merging(loss) ?? loss + Self.logger.info( + "noted a lost session connId=\(connectionId, privacy: .public) transaction=\(transaction) kinds=\(state.rawValue)" + ) } - internal func hasTransactionLoss(database: String, for connectionId: UUID) -> Bool { - lostTransactions[connectionId]?.contains(database) ?? false + internal func hasLoss(database: String, for connectionId: UUID) -> Bool { + losses[connectionId]?[database] != nil } - /// Closes every parked connection when the transport under them is rebuilt, first noting each - /// one that was holding a transaction, so the next work on that database is told the server - /// rolled it back instead of quietly starting over on a new connection. - internal func closeAllNotingLostTransactions(for connectionId: UUID) async { + // Closes every parked connection when the transport under them is rebuilt, first noting each one that held a + // transaction or session state, so the next work on that database is told what the server dropped instead of + // quietly starting over on a new connection. + internal func closeAllNotingLosses(for connectionId: UUID) async { for (database, driver) in parked[connectionId] ?? [:] { let held = await driver.heldSessionTransactionState() - if !held.permitsAppTransaction { - lostTransactions[connectionId, default: []].insert(database) - } + markLost( + database: database, + transaction: !held.permitsAppTransaction, + state: recordedState?(driver, connectionId) ?? [], + for: connectionId + ) } - closeAll(for: connectionId, keepingLostTransactions: true) + closeAll(for: connectionId, keepingLosses: true) } - /// Reports a lost transaction once, clearing it. - internal func takeTransactionLoss(database: String, for connectionId: UUID) -> Bool { - lostTransactions[connectionId]?.remove(database) != nil + // Reported once, whole, to the first turn that owns session state, as a lost transaction always was. + internal func takeLoss(database: String, for connectionId: UUID) -> SessionLoss? { + let loss = losses[connectionId]?.removeValue(forKey: database) + if losses[connectionId]?.isEmpty == true { + losses.removeValue(forKey: connectionId) + } + return loss } - // A connection with one driver for every database lost the transaction whichever name it was noted under. - internal func takeAnyTransactionLoss(for connectionId: UUID) -> String? { - lostTransactions.removeValue(forKey: connectionId)?.min() + // A connection with one driver for every database lost the session whichever name it was noted under. + internal func takeAnyLoss(for connectionId: UUID) -> (database: String, loss: SessionLoss)? { + guard let noted = losses.removeValue(forKey: connectionId), let database = noted.keys.min() else { return nil } + let loss = noted.values.reduce(SessionLoss(transaction: false, state: [])) { $0.merging($1) } + return (database: database, loss: loss) } internal func markOpened(_ driver: DatabaseDriver, database: String, for connectionId: UUID, at date: Date = Date()) { diff --git a/apple/TablePro/Core/Database/SessionStateLedger.swift b/apple/TablePro/Core/Database/SessionStateLedger.swift new file mode 100644 index 0000000000..f248ee458c --- /dev/null +++ b/apple/TablePro/Core/Database/SessionStateLedger.swift @@ -0,0 +1,171 @@ +// +// SessionStateLedger.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar + +// What the user's own statements set up on each session connection, as kinds only: a value a `SET` carried is kept +// nowhere. A reconnect reads it for the driver it is about to close, since the server cannot be asked once it is gone. +@MainActor +internal final class SessionStateLedger { + private struct Entry { + // Weak, and compared by identity: a closed driver that was freed hands its address to the next one allocated. + weak var driver: AnyObject? + var state: SQLSessionStateKinds + } + + private var entries: [SessionTransactionTracker.Key: Entry] = [:] + private let isLive: @MainActor (SessionTransactionTracker.Key, DatabaseDriver) -> Bool + + internal init(isLive: @escaping @MainActor (SessionTransactionTracker.Key, DatabaseDriver) -> Bool) { + self.isLive = isLive + } + + internal func record(_ effect: SQLSessionStateEffect, on driver: DatabaseDriver, for key: SessionTransactionTracker.Key) { + let current = entries[key].flatMap { $0.driver === driver ? $0.state : nil } ?? [] + // A turn can outlive its session's teardown; its late answer must not land on the replacement. + guard isLive(key, driver) else { + if entries[key]?.driver === driver { entries.removeValue(forKey: key) } + return + } + let state = effect.applied(to: current) + entries[key] = state.isEmpty ? nil : Entry(driver: driver, state: state) + } + + internal func state(for key: SessionTransactionTracker.Key, heldBy driver: DatabaseDriver) -> SQLSessionStateKinds { + guard let entry = entries[key], entry.driver === driver else { return [] } + return entry.state + } + + internal func state(heldBy driver: DatabaseDriver, for connectionId: UUID) -> SQLSessionStateKinds { + entries.reduce(into: SQLSessionStateKinds()) { state, item in + if item.key.connectionId == connectionId, item.value.driver === driver { + state.formUnion(item.value.state) + } + } + } + + // Drops what a closed, replaced or disconnected driver held, as the transaction tracker drops its holds. + internal func prune(_ connectionId: UUID) { + for (key, entry) in entries where key.connectionId == connectionId { + if let driver = entry.driver as? DatabaseDriver, isLive(key, driver) { continue } + entries.removeValue(forKey: key) + } + } +} + +private struct SessionStateReading: Sendable { + let key: SessionTransactionTracker.Key + let dialect: SQLSessionDialect + let grammar: SQLLexicalGrammar + let startup: [String] +} + +extension DatabaseManager { + // Called inside the user's own lease, where the driver is the one the text ran on. A statement that failed counts: + // it can have created a temporary table before its error. + internal func noteSessionStatements(_ statements: [String], ranOn driver: DatabaseDriver, scope: DatabaseScope) async { + guard !statements.isEmpty, + let session = activeSessions[scope.connectionId], + let dialect = SQLSessionDialect(TransactionEngineFamily.of(session.connection.type)), + let key = sessionKey(for: scope), + isLiveSessionDriver(driver, for: key) + else { return } + let databaseType = session.connection.type + let grammar = SQLLexicalResolver.readings(for: databaseType, session: driver.sessionLexicalState).execution + // A reconnect runs the startup commands again, so what they set up is never lost. + let startup = Self.startupStatements( + from: session.connection.startupCommands, databaseType: databaseType, grammar: grammar + ) + let reading = SessionStateReading(key: key, dialect: dialect, grammar: grammar, startup: startup) + let effect = await Self.sessionStateEffect(of: statements, reading: reading) + guard !effect.isEmpty else { return } + sessionStateLedger.record(effect, on: driver, for: key) + } + + // A Run All of a long script reads every statement, which would hold the main actor. + @concurrent + nonisolated private static func sessionStateEffect( + of statements: [String], + reading: SessionStateReading + ) async -> SQLSessionStateEffect { + SQLSessionStateClassifier.effect( + ofStatements: statements, dialect: reading.dialect, grammar: reading.grammar, skipping: reading.startup + ) + } + + // Kinds only, never a statement, a name or a value: a `SET` can carry a secret. + static func sessionLostMessage(database: String, loss: SessionLoss) -> String { + guard !loss.state.isEmpty else { return parkedTransactionLostMessage(database: database) } + let kinds = ListFormatter.localizedString(byJoining: loss.state.localizedNames) + guard loss.transaction else { + return String( + format: String( + // swiftlint:disable:next line_length + localized: "The connection to %1$@ was lost while it waited and has been reconnected. The new session does not have what the old one set up: %2$@. Set it up again, then run the statements." + ), + database, kinds + ) + } + return String( + format: String( + // swiftlint:disable:next line_length + localized: "The connection to %1$@ was lost while it waited, and the server rolled back its open transaction. The new session does not have what the old one set up either: %2$@. Set it up again, then run the statements." + ), + database, kinds + ) + } +} + +extension SQLSessionDialect { + // Every engine left out records nothing, as before the ledger existed. + init?(_ family: TransactionEngineFamily) { + switch family { + case .postgres, .redshift, .cockroach: + self = .postgres + case .mysql: + self = .mySQL + case .sqlServer: + self = .sqlServer + case .oracle: + self = .oracle + case .sqlite: + self = .sqlite + case .duckdb: + self = .duckDB + case .redis, .other: + return nil + } + } +} + +extension SQLSessionStateKinds { + // In bit order, so a message names the kinds the same way every time. + var localizedNames: [String] { + var names: [String] = [] + if contains(.temporaryObjects) { names.append(String(localized: "temporary tables")) } + if contains(.sessionSettings) { names.append(String(localized: "session settings")) } + if contains(.userVariables) { names.append(String(localized: "user variables")) } + if contains(.preparedStatements) { names.append(String(localized: "prepared statements")) } + if contains(.heldCursors) { names.append(String(localized: "cursors")) } + if contains(.sessionLocks) { names.append(String(localized: "session locks")) } + if contains(.listeners) { names.append(String(localized: "LISTEN registrations")) } + return names + } +} + +extension BatchStatementOutcome { + // The statement that failed ran too, as `CatalogEvent.statementsRan` counts it. A Stop ran only what it returned. + func ranCount(of totalCount: Int) -> Int { + switch self { + case .completed: + return totalCount + case .cancelled(let results): + return results.count + case .failed(let results, let failure, _): + return failure.ranStatementCount(executedCount: results.count, totalCount: totalCount) + } + } +} diff --git a/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift b/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift index 549259fa11..ac55d9e63a 100644 --- a/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift +++ b/apple/TablePro/Core/Plugins/DatabaseType+Registry.swift @@ -9,23 +9,16 @@ import SwiftUI import TableProAutocomplete import TableProPluginKit -/// Everything a `DatabaseType` knows by asking the plugin registry. -/// -/// It lives apart from the type itself because these are the members that make naming a database -/// type cost the whole plugin system at compile time. `DatabaseType` is a string a connection -/// carries and a value the display layer compares against a constant; keeping the lookups here -/// means a file that only does that does not compile against the registry, the snapshot table and -/// the theme engine behind them. +// Apart from `DatabaseType`, so a file that only names or compares a type does not compile against the registry, the +// snapshot table and the theme engine behind them. extension DatabaseType { - /// All registered database types, derived dynamically from the plugin metadata registry. static var allKnownTypes: [DatabaseType] { PluginMetadataRegistry.shared.allRegisteredTypeIds().map { DatabaseType(rawValue: $0) } } } extension DatabaseType { - /// Returns nil if rawValue doesn't match any registered type. init?(validating rawValue: String) { guard PluginMetadataRegistry.shared.hasType(rawValue) else { return nil } self.rawValue = rawValue @@ -33,14 +26,11 @@ extension DatabaseType { } extension DatabaseType { - /// Plugin type ID used for PluginManager lookup, resolved via the registry. var pluginTypeId: String { PluginMetadataRegistry.shared.pluginTypeId(for: rawValue) } - /// Genuinely a fact about the plugin binary rather than the database, so it asks by - /// `pluginTypeId`: Redshift is served by the bundled PostgreSQL plugin and there is nothing - /// of its own to download. + // Asks by `pluginTypeId`: Redshift is served by the bundled PostgreSQL plugin and has nothing of its own to download. var isDownloadablePlugin: Bool { PluginMetadataRegistry.shared.snapshot(forRegisteredTypeId: pluginTypeId)?.isDownloadable ?? false } @@ -49,8 +39,7 @@ extension DatabaseType { PluginMetadataRegistry.shared.snapshot(for: self)?.iconName ?? "cylinder" } - /// Returns the correct SwiftUI Image for this database type, handling both - /// SF Symbol names (e.g. "cylinder.fill") and asset catalog names (e.g. "mysql-icon"). + // `iconName` is an SF Symbol (`cylinder.fill`) or an asset catalog name (`mysql-icon`). var iconImage: Image { let name = iconName if NSImage(systemSymbolName: name, accessibilityDescription: nil) != nil { @@ -88,6 +77,10 @@ extension DatabaseType { PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.verifiesServerWithSystemTrust ?? false } + var requestedTLSVerifiesIdentity: Bool { + PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.requestedTLSVerifiesIdentity ?? false + } + var supportsPerConnectionCertificatePaths: Bool { PluginMetadataRegistry.shared.snapshot(for: self)?.capabilities.supportsPerConnectionCertificatePaths ?? true } @@ -98,7 +91,7 @@ extension DatabaseType { } var sslModeWhenTLSEnabled: SSLMode { - verifiesServerWithSystemTrust ? .verifyIdentity : .required + requestedTLSVerifiesIdentity ? .verifyIdentity : .required } func impliedSSLMode(forPort port: Int) -> SSLMode? { @@ -261,8 +254,8 @@ extension DatabaseType { .capabilities.supportsUserDefinedTypeBrowse ?? false } - /// The object kinds this engine fetches. The sidebar waits on these fetches before calling a - /// container empty, and keeps a section for one that failed; it lists a section only for rows. + // The object kinds this engine fetches. The sidebar waits on these fetches before calling a + // container empty, and keeps a section for one that failed; it lists a section only for rows. var declaredObjectKinds: Set { var kinds: Set = [] if supportsRoutines { diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift index 298a4c1a62..410a02df9b 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+CuratedDefaults.swift @@ -8,17 +8,16 @@ import TableProPluginKit import TableProSQLVocabulary import TableProUtilities -/// What the app knows about a database type before its plugin loads. -/// -/// The primary type ids here are overwritten by `buildMetadataSnapshot` the moment the plugin -/// registers, so these are the pre-load answer for those, apart from an explain list the plugin -/// leaves empty, which keeps the one here. For a variant id they are the whole -/// answer: `registerVariant` keeps the curated entry and ignores the plugin's own statics, which -/// is the only reason MariaDB, TiDB, Databend, OceanBase, Redshift, CockroachDB and PGlite can differ from -/// the plugin that drives them. +// What the app knows about a database type before its plugin loads. +// The primary type ids here are overwritten by `buildMetadataSnapshot` the moment the plugin +// registers, so these are the pre-load answer for those, apart from an explain list the plugin +// leaves empty, which keeps the one here. For a variant id they are the whole +// answer: `registerVariant` keeps the curated entry and ignores the plugin's own statics, which +// is the only reason MariaDB, TiDB, Databend, OceanBase, Redshift, CockroachDB and PGlite can differ from +// the plugin that drives them. extension PluginMetadataRegistry { - /// SQLite's own extensions. libSQL and Turso name a new local file with them too, since the file - /// their driver creates is a SQLite database. + // SQLite's own extensions. libSQL and Turso name a new local file with them too, since the file + // their driver creates is a SQLite database. static let sqliteFileExtensions = ["db", "db3", "s3db", "sl3", "sqlite", "sqlite3", "sqlitedb"] // swiftlint:disable:next function_body_length @@ -192,8 +191,8 @@ extension PluginMetadataRegistry { ) ] - /// Cloud SQL signs IAM principals in to MySQL and PostgreSQL only, so SQL Server and the - /// MySQL and PostgreSQL forks keep the AWS-only picker. + // Cloud SQL signs IAM principals in to MySQL and PostgreSQL only, so SQL Server and the + // MySQL and PostgreSQL forks keep the AWS-only picker. let cloudIAMFields = GoogleCloudSQLAuthFields.standardWithAWS() + [AWSAuthFields.rdsEndpointField()] let postgresqlAuthFields = KerberosAuthFields.standardWithAWSAndGoogleCloud() + [ @@ -201,8 +200,8 @@ extension PluginMetadataRegistry { AWSAuthFields.rdsEndpointField() ] - /// The MySQL plugin's types only. PostgreSQL shares the IAM fields and must not pick this up: - /// its driver holds no releasable resource, so the setting would do nothing. + // The MySQL plugin's types only. PostgreSQL shares the IAM fields and must not pick this up: + // its driver holds no releasable resource, so the setting would do nothing. let mysqlIdleReleaseField = ConnectionField( id: "mysqlIdleReleaseMinutes", label: String(localized: "Release the Server Connection After (minutes, 0 to keep it)"), @@ -409,6 +408,7 @@ extension PluginMetadataRegistry { supportsDatabaseTriggerBrowse: true, supportsUserDefinedTypeBrowse: true, defaultSSLMode: .preferred, + verifiesServerWithSystemTrust: true, readOnlyEnforcement: .sessionDefault ), schema: PluginMetadataSnapshot.SchemaInfo( @@ -470,7 +470,8 @@ extension PluginMetadataRegistry { supportsDropSchema: true, supportsCreateSchema: true, supportsSchemaOwner: true, - defaultSSLMode: .preferred + defaultSSLMode: .preferred, + verifiesServerWithSystemTrust: true ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "public", @@ -548,6 +549,7 @@ extension PluginMetadataRegistry { supportsCheckConstraintEditing: false, supportsGeneratedColumns: true, defaultSSLMode: .preferred, + verifiesServerWithSystemTrust: true, readOnlyEnforcement: .sessionDefault ), schema: PluginMetadataSnapshot.SchemaInfo( diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+DuckDBDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+DuckDBDefaults.swift index d8106a1092..6e29479517 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+DuckDBDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+DuckDBDefaults.swift @@ -8,7 +8,6 @@ import TableProPluginKit import TableProUtilities extension PluginMetadataRegistry { - /// The curated snapshot for DuckDB, alongside its connection fields in the sibling file. func duckdbPluginDefaults( dialect: SQLDialectDescriptor, columnTypes: [String: [String]] @@ -47,6 +46,7 @@ extension PluginMetadataRegistry { // httpfs checks a server against the system CA bundle when no CA is given, so a URL asking // for TLS with no mode means a verified connection, never Required's unchecked one. verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, supportsConnectionPooling: false, localFilePathField: .additionalField("duckdbFilePath"), serverEndpointFields: ServerEndpointFields(hostFieldId: "duckdbHost", portFieldId: "duckdbPort"), @@ -64,16 +64,15 @@ extension PluginMetadataRegistry { systemDatabaseNames: ["system", "temp"], systemSchemaNames: [], fileExtensions: ["duckdb", "ddb", "parquet", "csv", "tsv", "json", "ndjson"], - /// Only DuckDB's own storage. The data formats above are recognised by name - /// alone, because `duckdb_open` picks their reader from the extension and - /// refuses a Parquet file called anything else. - /// - /// `DUCK` sits behind the header's eight-byte checksum, followed by the storage - /// version as a little-endian `uint64`. Four bytes on their own are not enough - /// to name a format, and `SELECT 'DUCK';` spells them at exactly that offset, - /// so the version's high six bytes have to be zero as well. Storage versions - /// are still in the sixties, and a version past 65535 would cost recognition by - /// content rather than break it. + // Only DuckDB's own storage. The data formats above are recognised by name + // alone, because `duckdb_open` picks their reader from the extension and + // refuses a Parquet file called anything else. + // `DUCK` sits behind the header's eight-byte checksum, followed by the storage + // version as a little-endian `uint64`. Four bytes on their own are not enough + // to name a format, and `SELECT 'DUCK';` spells them at exactly that offset, + // so the version's high six bytes have to be zero as well. Storage versions + // are still in the sixties, and a version past 65535 would cost recognition by + // content rather than break it. fileSignatures: [.magic("DUCK", at: 8).andZeroes(at: 14, count: 6)], databaseGroupingStrategy: .bySchema, structureColumnFields: [.name, .type, .nullable, .defaultValue, .autoIncrement, .comment] diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+HanaDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+HanaDefaults.swift index 16e6804ba0..4bbd5fd4e0 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+HanaDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+HanaDefaults.swift @@ -41,7 +41,8 @@ extension PluginMetadataRegistry { defaultSSLMode: .verifyIdentity, supportsOpportunisticTLS: false, tlsImpliedPorts: [443], - verifiesServerWithSystemTrust: true + verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true ), schema: PluginMetadataSnapshot.SchemaInfo( defaultSchemaName: "", diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift index 2905785999..aded48a792 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+KafkaDefaults.swift @@ -7,13 +7,11 @@ import Foundation import TableProPluginKit extension PluginMetadataRegistry { - /// The curated snapshot for Kafka. - /// - /// A registry-only driver is not installed until someone connects with it, so without an - /// entry here the type is absent from the New Connection picker and there is no way to - /// create the connection that would trigger the install. The connection fields are - /// duplicated from `KafkaConnectionField` for the same reason: the plugin that owns them - /// is not loaded yet. + // A registry-only driver is not installed until someone connects with it, so without an + // entry here the type is absent from the New Connection picker and there is no way to + // create the connection that would trigger the install. The connection fields are + // duplicated from `KafkaConnectionField` for the same reason: the plugin that owns them + // is not loaded yet. func kafkaPluginDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { [ ("Kafka", PluginMetadataSnapshot( @@ -43,6 +41,7 @@ extension PluginMetadataRegistry { defaultSSLMode: .disabled, supportsOpportunisticTLS: false, verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, supportsColumnSort: false, supportsRawFilter: false, statementPolicyInDriver: true @@ -92,8 +91,8 @@ private let kafkaColumnTypes: [String: [String]] = [ "Coordinates": ["INTEGER", "BIGINT", "TIMESTAMP"] ] -/// Kept identical to `KafkaConnectionField.fields()` in the plugin. The two cannot share code: -/// this list has to exist before the bundle that owns it is downloaded. +// Kept identical to `KafkaConnectionField.fields()` in the plugin. The two cannot share code: +// this list has to exist before the bundle that owns it is downloaded. private func kafkaConnectionFields() -> [ConnectionField] { [ ConnectionField( diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift index 95c510298b..6286e8fee2 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+QdrantDefaults.swift @@ -33,6 +33,7 @@ extension PluginMetadataRegistry { supportsModifyPrimaryKey: false, supportsOpportunisticTLS: false, verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, columnsAreSampled: true, supportsPrincipalConnectionLimit: false ), diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift index 6adb6fe2d9..dd3765b193 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift @@ -7,9 +7,9 @@ import Foundation import TableProPluginKit extension PluginMetadataRegistry { - /// SQL Server refuses `=` on these (Msg 402 for `ntext`, `text`, `image` and `xml`, 403 for the spatial types), so - /// a keyless match compares them through a cast, which is exact for each. A `hierarchyid` reads as its bytes, which - /// the cast to `varbinary(max)` compares without an implicit conversion back to the type. + // SQL Server refuses `=` on these (Msg 402 for `ntext`, `text`, `image` and `xml`, 403 for the spatial types), so + // a keyless match compares them through a cast, which is exact for each. A `hierarchyid` reads as its bytes, which + // the cast to `varbinary(max)` compares without an implicit conversion back to the type. static let mssqlRowMatchTextTypePrefixes = ["NTEXT", "TEXT", "XML", "IMAGE", "GEOGRAPHY", "GEOMETRY", "HIERARCHYID"] // Measured on ClickHouse 25.8: a JSON value or a NaN never equals the text it was read as, and concat() of it does. @@ -17,13 +17,13 @@ extension PluginMetadataRegistry { "JSON", "OBJECT(", "DYNAMIC", "FLOAT", "BFLOAT16", "NULLABLE(FLOAT", "NULLABLE(BFLOAT16", ] - /// `sql_variant` refuses `=` too (Msg 206), and no cast keeps its base type: the integer `1` and the string `N'1'` - /// cast to the same text, so a keyless match on it could pick either row. + // `sql_variant` refuses `=` too (Msg 206), and no cast keeps its base type: the integer `1` and the string `N'1'` + // cast to the same text, so a keyless match on it could pick either row. static let mssqlRowMatchExcludedTypePrefixes = ["SQL_VARIANT"] - /// Oracle cannot compare a LOB, `LONG`, `LONG RAW`, `BFILE`, `XMLTYPE`, `JSON`, `VECTOR` or spatial value with `=` - /// (ORA-22848 on a CLOB), so a keyless match leaves them out. Matched against the bare type name, which an - /// owner-qualified column such as `"MDSYS"."SDO_GEOMETRY"` classifies by. + // Oracle cannot compare a LOB, `LONG`, `LONG RAW`, `BFILE`, `XMLTYPE`, `JSON`, `VECTOR` or spatial value with `=` + // (ORA-22848 on a CLOB), so a keyless match leaves them out. Matched against the bare type name, which an + // owner-qualified column such as `"MDSYS"."SDO_GEOMETRY"` classifies by. static let oracleRowMatchExcludedTypePrefixes = [ "CLOB", "NCLOB", "BLOB", "LONG", "BFILE", "XMLTYPE", "JSON", "VECTOR", "SDO_" ] @@ -602,6 +602,7 @@ extension PluginMetadataRegistry { supportsOpportunisticTLS: false, tlsImpliedPorts: [8_443, 443], verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, readOnlyEnforcement: .engineEnforced ), schema: PluginMetadataSnapshot.SchemaInfo( diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TrinoDefaults.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TrinoDefaults.swift index 974471c1f2..d147b1d00d 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TrinoDefaults.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry+TrinoDefaults.swift @@ -37,7 +37,8 @@ extension PluginMetadataRegistry { defaultSSLMode: .disabled, supportsOpportunisticTLS: false, tlsImpliedPorts: [443], - verifiesServerWithSystemTrust: true + verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true ), schema: Self.trinoSchemaInfo, editor: PluginMetadataSnapshot.EditorConfig( @@ -88,6 +89,7 @@ extension PluginMetadataRegistry { supportsOpportunisticTLS: false, tlsImpliedPorts: [443], verifiesServerWithSystemTrust: true, + requestedTLSVerifiesIdentity: true, // OFFSET is off by default (offset_clause_enabled), and the grammar takes neither // FETCH NEXT nor LIMIT before OFFSET. pagination: .leadingRowsOnly(maximumRows: nil) diff --git a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift index c50ece4e4d..196da8f824 100644 --- a/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift +++ b/apple/TablePro/Core/Plugins/PluginMetadataRegistry.swift @@ -73,93 +73,96 @@ struct PluginMetadataSnapshot: Sendable { var defaultSSLMode: SSLMode = .disabled var supportsOpportunisticTLS: Bool = true var tlsImpliedPorts: [Int] = [] + // Verify Identity with no CA file checks the server against the system trust store. var verifiesServerWithSystemTrust: Bool = false + // A URL or port that asks for TLS without naming a mode gets Verify Identity rather than Required. Kept apart from + // `verifiesServerWithSystemTrust`: a libpq server on a private CA that connects with `ssl=true` today would stop. + var requestedTLSVerifiesIdentity: Bool = false var supportsPerConnectionCertificatePaths: Bool = true var supportsClientKeyPassphrase: Bool = false var supportsConnectionPooling: Bool = true - /// Whether two pooled drivers for the same connection sit on one physical session. Snowflake - /// keys its session on the account and role and deliberately not on the database, so every - /// pooled scope shares one mutable `currentDatabase`: a statement that selects a database on - /// behalf of one entry selects it for all of them. + // Whether two pooled drivers for the same connection sit on one physical session. Snowflake + // keys its session on the account and role and deliberately not on the database, so every + // pooled scope shares one mutable `currentDatabase`: a statement that selects a database on + // behalf of one entry selects it for all of them. var pooledDriversShareOneSession: Bool = false var authenticationIsDatabaseScoped: Bool = false - /// Whether a connection that names no database has nothing to browse until one is chosen. A - /// MySQL session opened without one has no current database: `SHOW TABLES` there is - /// `ERROR 1046 No database selected`. An engine whose session falls back to a default - /// database, such as ClickHouse's `default`, leaves this false. + // Whether a connection that names no database has nothing to browse until one is chosen. A + // MySQL session opened without one has no current database: `SHOW TABLES` there is + // `ERROR 1046 No database selected`. An engine whose session falls back to a default + // database, such as ClickHouse's `default`, leaves this false. var browsingRequiresSelectedDatabase: Bool = false var pagination: PaginationCapability = .offset - /// Whether an exact count reads the whole table with no estimate to stand in for it, and only the driver - /// knows how to count it. DynamoDB bills a `Scan` of every item and has no `COUNT(*)`; Cassandra reads - /// every partition, because `LIMIT` does not bound `COUNT(*)`, and a filtered count needs CQL the host - /// cannot write. Such an engine is counted only when the user asks, and only by its driver. + // Whether an exact count reads the whole table with no estimate to stand in for it, and only the driver + // knows how to count it. DynamoDB bills a `Scan` of every item and has no `COUNT(*)`; Cassandra reads + // every partition, because `LIMIT` does not bound `COUNT(*)`, and a filtered count needs CQL the host + // cannot write. Such an engine is counted only when the user asks, and only by its driver. var exactRowCountIsFullScan: Bool = false - /// Whether a table can be ordered by any of its columns. CQL orders rows only by clustering columns inside - /// one partition, so a Cassandra table offers no column sort and a query result sorts the rows it holds. + // Whether a table can be ordered by any of its columns. CQL orders rows only by clustering columns inside + // one partition, so a Cassandra table offers no column sort and a query result sorts the rows it holds. var supportsColumnSort: Bool = true - /// Whether filters can ask for rows matching any one of several conditions. CQL has no `OR`. + // Whether filters can ask for rows matching any one of several conditions. CQL has no `OR`. var supportsMatchAnyFilters: Bool = true - /// Whether a filter row can hold a condition written in the engine's own query language. A - /// driver that answers one with an error, such as etcd or DynamoDB, or reads it as a column - /// name, is not offered the row at all. + // Whether a filter row can hold a condition written in the engine's own query language. A + // driver that answers one with an error, such as etcd or DynamoDB, or reads it as a column + // name, is not offered the row at all. var supportsRawFilter: Bool = true - /// Whether a table's columns are a sample of its rows rather than a declared schema. A - /// MongoDB collection lists the fields found in its first documents, so a field missing from - /// one side's list says nothing about whether that side holds it. + // Whether a table's columns are a sample of its rows rather than a declared schema. A + // MongoDB collection lists the fields found in its first documents, so a field missing from + // one side's list says nothing about whether that side holds it. var columnsAreSampled: Bool = false - /// Whether a contains, starts with or ends with filter matches a scalar column of any type by the text it shows, - /// so a search can send number and date columns too. MongoDB's builder converts the value before it matches. + // Whether a contains, starts with or ends with filter matches a scalar column of any type by the text it shows, + // so a search can send number and date columns too. MongoDB's builder converts the value before it matches. var patternFiltersReadAnyColumn: Bool = false - /// Whether the driver types a filter's values by the kinds it learned from rows it read, so a count of a - /// filtered table runs on the driver that read the page. A pooled driver read nothing and would type the same - /// filter another way: a date compared as a string counts no rows. + // Whether the driver types a filter's values by the kinds it learned from rows it read, so a count of a + // filtered table runs on the driver that read the page. A pooled driver read nothing and would type the same + // filter another way: a date compared as a string counts no rows. var filtersTypedByReadRows: Bool = false var isEngineReadOnly: Bool = false - /// Which connection field carries the path of the local database file this driver opens, - /// for the types that open one. Nil for every driver that reaches its database over the - /// network, which is what makes it the test for "can this connection name a remote file". + // Which connection field carries the path of the local database file this driver opens, + // for the types that open one. Nil for every driver that reaches its database over the + // network, which is what makes it the test for "can this connection name a remote file". var localFilePathField: LocalFilePathField? - /// The plugin fields that name the server, for a type the built-in Host and Port do not - /// reach. Nil for every type whose driver dials `config.host`, which is what keeps a tunnel - /// forwarding and rewriting the built-in endpoint there. + // The plugin fields that name the server, for a type the built-in Host and Port do not + // reach. Nil for every type whose driver dials `config.host`, which is what keeps a tunnel + // forwarding and rewriting the built-in endpoint there. var serverEndpointFields: ServerEndpointFields? - /// Fields a driver older than this app ignores to the user's cost. A connection that sets one - /// away from its default, while it is in use, is refused when the loaded driver does not - /// declare it: DuckDB's SSL Mode read by a driver from before it existed would connect as - /// Automatic, which is plain HTTP on a loopback host, however the user set it. + // Fields a driver older than this app ignores to the user's cost. A connection that sets one + // away from its default, while it is in use, is refused when the loaded driver does not + // declare it: DuckDB's SSL Mode read by a driver from before it existed would connect as + // Automatic, which is plain HTTP on a loopback host, however the user set it. var fieldsRequiringDriverSupport: [String] = [] - /// The extensions a new database file can be named with, the first one preferred. Empty - /// when the driver refuses a path with nothing at it, as Beancount does, so the form offers - /// no way to name one. DuckDB lists only its own format: a missing Parquet or CSV path is - /// refused rather than created. + // The extensions a new database file can be named with, the first one preferred. Empty + // when the driver refuses a path with nothing at it, as Beancount does, so the form offers + // no way to name one. DuckDB lists only its own format: a missing Parquet or CSV path is + // refused rather than created. var newDatabaseFileExtensions: [String] = [] var supportsSOCKSProxy: Bool { supportsSSH } - /// `cloudflared access tcp` forwards a loopback port just as an SSH tunnel does. A stored flag - /// drifted: it defaulted to true in the built-in snapshots, so a registry type the plugin - /// declares without SSH offered Cloudflare Tunnel until that plugin was installed. + // `cloudflared access tcp` forwards a loopback port just as an SSH tunnel does. A stored flag + // drifted: it defaulted to true in the built-in snapshots, so a registry type the plugin + // declares without SSH offered Cloudflare Tunnel until that plugin was installed. var supportsCloudflareTunnel: Bool { supportsSSH } - /// A tunnel command forwards a loopback port to the server the connection names, so it - /// applies wherever an SSH tunnel would. Computed for the same reason `supportsSOCKSProxy` - /// is: a stored flag would need an opt-out line in every hand-written snapshot. + // A tunnel command forwards a loopback port to the server the connection names, so it + // applies wherever an SSH tunnel would. Computed for the same reason `supportsSOCKSProxy` + // is: a stored flag would need an opt-out line in every hand-written snapshot. var supportsTunnelCommand: Bool { supportsSSH } - /// Whether this type may point at a file on an SSH server instead of a local one. - /// - /// Deliberately not derived from `localFilePathField`. Beancount opens a local file and must - /// still be excluded: a ledger is a graph of files reached through `include`, so one file - /// out of it either fails to load or presents incomplete accounts, which is worse. + // Whether this type may point at a file on an SSH server instead of a local one. + // Deliberately not derived from `localFilePathField`. Beancount opens a local file and must + // still be excluded: a ledger is a graph of files reached through `include`, so one file + // out of it either fails to load or presents incomplete accounts, which is worse. var supportsRemoteDatabaseFile: Bool = false - /// Whether this type can run its statements on an SSH server against a live database file, - /// rather than only fetching a read-only copy. True for SQLite alone today, because the - /// remote agent opens the file with the server's `libsqlite3`. + // Whether this type can run its statements on an SSH server against a live database file, + // rather than only fetching a read-only copy. True for SQLite alone today, because the + // remote agent opens the file with the server's `libsqlite3`. var supportsRemoteDatabaseSession: Bool = false var supportsPrincipalConnectionLimit: Bool = true @@ -211,24 +214,24 @@ struct PluginMetadataSnapshot: Sendable { let systemDatabaseNames: [String] let systemSchemaNames: [String] let fileExtensions: [String] - /// Curated in the app rather than declared by the plugin: claiming a format from the - /// system also needs a `CFBundleDocumentTypes` entry only the app bundle can make. + // Curated in the app rather than declared by the plugin: claiming a format from the + // system also needs a `CFBundleDocumentTypes` entry only the app bundle can make. let fileSignatures: [DatabaseFileSignature] let databaseGroupingStrategy: GroupingStrategy let structureColumnFields: [StructureColumnField] let implicitSchemaName: String? let rowMatchExcludedTypePrefixes: [String] - /// Column types whose keyless row match has to compare the server's own text rendering - /// rather than the value itself. A grid reads every cell back as text, and on these types - /// that text does not compare equal to what it was read from: measured on MySQL 8.4.11 and - /// OceanBase 4.4.2.1, a `FLOAT` holding 1.1 reads back `1.1` and `WHERE f = '1.1'` matches - /// no row, because the server widens the stored single to a different double than it parses - /// the literal into. `JSON` never compares equal to a string at all. Listing a type here - /// wraps only the keyless comparison, and only for the types measured to need it: `CONCAT` - /// over a `BIT` column returns its raw bytes and breaks a match that works today. + // Column types whose keyless row match has to compare the server's own text rendering + // rather than the value itself. A grid reads every cell back as text, and on these types + // that text does not compare equal to what it was read from: measured on MySQL 8.4.11 and + // OceanBase 4.4.2.1, a `FLOAT` holding 1.1 reads back `1.1` and `WHERE f = '1.1'` matches + // no row, because the server widens the stored single to a different double than it parses + // the literal into. `JSON` never compares equal to a string at all. Listing a type here + // wraps only the keyless comparison, and only for the types measured to need it: `CONCAT` + // over a `BIT` column returns its raw bytes and breaks a match that works today. let rowMatchTextTypePrefixes: [String] - /// Teradata and Snowflake compare database names without regard to case, so `dbc` is the - /// system database `DBC`. Every other engine keeps case: MySQL can hold a user database `MySQL`. + // Teradata and Snowflake compare database names without regard to case, so `dbc` is the + // system database `DBC`. Every other engine keeps case: MySQL can hold a user database `MySQL`. let systemDatabaseNamesIgnoreCase: Bool // Nil for an engine that keeps the case of an unquoted name, or whose rule is not curated yet. let identifierRules: SQLIdentifierRules? @@ -313,10 +316,10 @@ struct PluginMetadataSnapshot: Sendable { let category: DatabaseCategory let tagline: String let hidesBuiltInPassword: Bool - /// The driver takes no container name on the connection, so the built-in field would - /// be a second, meaningless place to type one: an embedded engine reads it from the - /// file it opens, Redis numbers its databases through its own field, and a key-value - /// store may have no container at all. + // The driver takes no container name on the connection, so the built-in field would + // be a second, meaningless place to type one: an embedded engine reads it from the + // file it opens, Redis numbers its databases through its own field, and a key-value + // store may have no container at all. let hidesBuiltInDatabase: Bool let defaultUnixSocketPath: String? // Non-nil means the form offers Connect Using > Socket. @@ -505,11 +508,10 @@ final class PluginMetadataRegistry: @unchecked Sendable { registerBuiltInDefaults() } - /// The curated table the app ships, before any plugin or test registers over it. - /// - /// The live snapshots are not that table: a loaded plugin replaces its entry, and a test can - /// register a synthetic engine into the same shared registry. Anything that means to check the - /// table itself reads this instead of iterating `allRegisteredTypeIds()`. + // The curated table the app ships, before any plugin or test registers over it. + // The live snapshots are not that table: a loaded plugin replaces its entry, and a test can + // register a synthetic engine into the same shared registry. Anything that means to check the + // table itself reads this instead of iterating `allRegisteredTypeIds()`. func builtInDefaults() -> [(typeId: String, snapshot: PluginMetadataSnapshot)] { Self.curatedDefaults() + registryPluginDefaults() } @@ -542,12 +544,12 @@ final class PluginMetadataRegistry: @unchecked Sendable { registerLocked(snapshot: snapshot, forTypeId: typeId, preserveIcon: preserveIcon) } - /// Registers an additional database type served by a multi-type plugin (Redshift, - /// CockroachDB, PGlite on the PostgreSQL plugin). A plugin's statics are per-class, so - /// they cannot express per-type facts like PGlite's disabled SSL or single-connection limit. - /// The curated built-in entry is therefore authoritative for a variant; the plugin fills the - /// EXPLAIN variants the curated entry leaves open, and supplies the editor config. A variant - /// with no curated entry falls back to deriving its snapshot from the plugin. + // Registers an additional database type served by a multi-type plugin (Redshift, + // CockroachDB, PGlite on the PostgreSQL plugin). A plugin's statics are per-class, so + // they cannot express per-type facts like PGlite's disabled SSL or single-connection limit. + // The curated built-in entry is therefore authoritative for a variant; the plugin fills the + // EXPLAIN variants the curated entry leaves open, and supplies the editor config. A variant + // with no curated entry falls back to deriving its snapshot from the plugin. func registerVariant( pluginSnapshot: PluginMetadataSnapshot, forTypeId typeId: String, @@ -611,33 +613,30 @@ final class PluginMetadataRegistry: @unchecked Sendable { } } - /// A raw lookup by the exact id a snapshot was registered under, for the callers that hold an - /// id rather than a type: registration itself, and iteration over `allRegisteredTypeIds()`. - /// - /// It is deliberately not named `snapshot(forTypeId:)` any more. That spelling read as the - /// way to ask about a `DatabaseType`, so 62 call sites passed it `databaseType.pluginTypeId` - /// and every variant was answered with its primary's facts. Asking about a type is - /// `snapshot(for:)`; this overload cannot be reached from a `DatabaseType` without first - /// choosing which id you mean, which is the point. + // A raw lookup by the exact id a snapshot was registered under, for the callers that hold an + // id rather than a type: registration itself, and iteration over `allRegisteredTypeIds()`. + // It is deliberately not named `snapshot(forTypeId:)` any more. That spelling read as the + // way to ask about a `DatabaseType`, so 62 call sites passed it `databaseType.pluginTypeId` + // and every variant was answered with its primary's facts. Asking about a type is + // `snapshot(for:)`; this overload cannot be reached from a `DatabaseType` without first + // choosing which id you mean, which is the point. func snapshot(forRegisteredTypeId typeId: String) -> PluginMetadataSnapshot? { lock.lock() defer { lock.unlock() } return snapshots[typeId] } - /// The snapshot describing a database type, which for a variant is its own curated entry - /// rather than the entry of the plugin that serves it. Reaching a snapshot through - /// `pluginTypeId` asks the primary instead, which is how a Redshift tab came to be told - /// PostgreSQL's `ILIKE` folds non-ASCII, and how PGlite came to be offered the SSH, SSL, - /// Cloudflare Tunnel and SOCKS panes its own entry declares it does not support. - /// - /// The fallback covers a type registered by a plugin with no curated entry of its own, where - /// the primary's snapshot is the only one there is. - /// - /// This is the only way to read a snapshot for a `DatabaseType`. `pluginTypeId` answers a - /// different question, "which plugin serves this type", and belongs to driver lookup alone. - /// The entry the app ships for a type, kept after a plugin registers over it: what a driver - /// built with this app declares, against which an older loaded driver can be told apart. + // The snapshot describing a database type, which for a variant is its own curated entry + // rather than the entry of the plugin that serves it. Reaching a snapshot through + // `pluginTypeId` asks the primary instead, which is how a Redshift tab came to be told + // PostgreSQL's `ILIKE` folds non-ASCII, and how PGlite came to be offered the SSH, SSL, + // Cloudflare Tunnel and SOCKS panes its own entry declares it does not support. + // The fallback covers a type registered by a plugin with no curated entry of its own, where + // the primary's snapshot is the only one there is. + // This is the only way to read a snapshot for a `DatabaseType`. `pluginTypeId` answers a + // different question, "which plugin serves this type", and belongs to driver lookup alone. + // The entry the app ships for a type, kept after a plugin registers over it: what a driver + // built with this app declares, against which an older loaded driver can be told apart. func curatedSnapshot(for databaseType: DatabaseType) -> PluginMetadataSnapshot? { lock.lock() defer { lock.unlock() } @@ -665,42 +664,37 @@ final class PluginMetadataRegistry: @unchecked Sendable { // MARK: - Dynamic Type Registration - /// Registers an alias type ID that maps to a primary type ID. - /// Used for multi-type plugins (e.g., MariaDB → MySQL, Redshift → PostgreSQL). + // For multi-type plugins: MariaDB to MySQL, Redshift to PostgreSQL. func registerTypeAlias(_ aliasTypeId: String, primaryTypeId: String) { lock.lock() defer { lock.unlock() } reverseTypeIndex[aliasTypeId] = primaryTypeId } - /// The inverse of `registerTypeAlias`. `unregister(typeId:)` drops a snapshot and leaves the - /// alias behind, so a test that registers one has nothing to undo it with. + // The inverse of `registerTypeAlias`. `unregister(typeId:)` drops a snapshot and leaves the + // alias behind, so a test that registers one has nothing to undo it with. func removeTypeAlias(_ aliasTypeId: String) { lock.lock() defer { lock.unlock() } reverseTypeIndex.removeValue(forKey: aliasTypeId) } - /// Returns all registered type IDs (sorted for deterministic UI ordering). func allRegisteredTypeIds() -> [String] { lock.lock() defer { lock.unlock() } return Array(snapshots.keys).sorted() } - /// Resolves a database type raw value to its plugin type ID for driver lookup. - /// For multi-type plugins (MySQL serves MariaDB), maps the alias to the primary. - /// - /// This is the answer to "which plugin serves this type", never "what is this type like". - /// A snapshot read for a `DatabaseType` belongs in `snapshot(for:)`, which asks the variant - /// first; `snapshot(forTypeId:)` is for a bare id that is already the one you want. + // For multi-type plugins (MySQL serves MariaDB), maps the alias to the primary. + // This is the answer to "which plugin serves this type", never "what is this type like". + // A snapshot read for a `DatabaseType` belongs in `snapshot(for:)`, which asks the variant + // first; `snapshot(forTypeId:)` is for a bare id that is already the one you want. func pluginTypeId(for rawValue: String) -> String { lock.lock() defer { lock.unlock() } return reverseTypeIndex[rawValue] ?? rawValue } - /// Checks if a type ID is registered (has a snapshot). func hasType(_ typeId: String) -> Bool { lock.lock() defer { lock.unlock() } @@ -709,8 +703,7 @@ final class PluginMetadataRegistry: @unchecked Sendable { // MARK: - Snapshot Builder - /// Builds a PluginMetadataSnapshot from a DriverPlugin's protocol properties. - /// Used by PluginManager to self-register plugins at load time. + // PluginManager registers each plugin with this at load time. func buildMetadataSnapshot( from driverType: any DriverPlugin.Type, isDownloadable: Bool = false @@ -802,6 +795,8 @@ final class PluginMetadataRegistry: @unchecked Sendable { tlsImpliedPorts: existingSnapshot?.capabilities.tlsImpliedPorts ?? [], verifiesServerWithSystemTrust: existingSnapshot?.capabilities .verifiesServerWithSystemTrust ?? false, + requestedTLSVerifiesIdentity: existingSnapshot?.capabilities + .requestedTLSVerifiesIdentity ?? false, supportsPerConnectionCertificatePaths: existingSnapshot?.capabilities .supportsPerConnectionCertificatePaths ?? true, supportsClientKeyPassphrase: existingSnapshot?.capabilities.supportsClientKeyPassphrase ?? false, diff --git a/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift b/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift index 41f93a634b..5b2e70e303 100644 --- a/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift +++ b/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift @@ -9,18 +9,13 @@ import Foundation import TableProPluginKit enum BlobDisplayContext { - /// Data grid cell: compact single-line "0x48656C6C6F..." case grid - /// Sidebar detail view: full multi-line hex dump case detail - /// Copy to clipboard: compact hex case copy - /// Editable hex in sidebar: space-separated hex bytes "48 65 6C 6C 6F" case edit } -@MainActor -final class BlobFormattingService { +final class BlobFormattingService: Sendable { static let shared = BlobFormattingService() private init() {} @@ -68,12 +63,10 @@ final class BlobFormattingService { return String(data: data, encoding: .isoLatin1) } - /// Whether the given column type requires BLOB formatting. func requiresFormatting(columnType: ColumnType) -> Bool { columnType.isBlobType } - /// Format a value if the column type is a BLOB type; otherwise return the original value. func formatIfNeeded(_ value: String, columnType: ColumnType?, for context: BlobDisplayContext) -> String { guard let columnType, requiresFormatting(columnType: columnType) else { return value diff --git a/apple/TablePro/Core/Services/Formatting/CellDisplayFormatter.swift b/apple/TablePro/Core/Services/Formatting/CellDisplayFormatter.swift index b1794b9b94..9c9590f766 100644 --- a/apple/TablePro/Core/Services/Formatting/CellDisplayFormatter.swift +++ b/apple/TablePro/Core/Services/Formatting/CellDisplayFormatter.swift @@ -9,15 +9,31 @@ import Foundation import TableProPluginKit -@MainActor enum CellDisplayFormatter { static let maxDisplayLength = 10_000 + @MainActor static func format( _ rawValue: PluginCellValue, columnType: ColumnType?, displayFormat: ValueDisplayFormat? = nil, databaseType: DatabaseType? = nil + ) -> String? { + format( + rawValue, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: DateFormattingService.shared.renderer + ) + } + + static func format( + _ rawValue: PluginCellValue, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dates: DateDisplayRenderer ) -> String? { switch rawValue { case .null: @@ -34,14 +50,11 @@ enum CellDisplayFormatter { var displayValue = value if let displayFormat, displayFormat != .raw, displayFormat.isApplicable(to: columnType, databaseType: databaseType), - let formatted = ValueDisplayFormatter.apply(value, format: displayFormat, columnType: columnType) { + let formatted = ValueDisplayFormatter.apply(value, format: displayFormat, columnType: columnType, dates: dates) { displayValue = formatted } else if let columnType { if columnType.isDateType { - if let formatted = DateFormattingService.shared.format( - dateString: displayValue, - columnType: columnType - ) { + if let formatted = dates.format(dateString: displayValue, columnType: columnType) { displayValue = formatted } } else if columnType.isVectorType { @@ -56,9 +69,9 @@ enum CellDisplayFormatter { } } - /// A row is one line, so anything a format produces is capped and stripped of line breaks - /// before it reaches a cell. Binary read as text is the case that needs it: a blob carries - /// newlines and runs to megabytes where hex never did. A lone CR stays for the cell to mark. + // A row is one line, so anything a format produces is capped and stripped of line breaks before it + // reaches a cell. Binary read as text needs it: a blob carries newlines and runs to megabytes where + // hex never did. A lone CR stays for the cell to mark. private static func fitToCell(_ value: String) -> String { var displayValue = value let nsDisplay = displayValue as NSString diff --git a/apple/TablePro/Core/Services/Formatting/DateDisplayRenderer.swift b/apple/TablePro/Core/Services/Formatting/DateDisplayRenderer.swift new file mode 100644 index 0000000000..6b37d06584 --- /dev/null +++ b/apple/TablePro/Core/Services/Formatting/DateDisplayRenderer.swift @@ -0,0 +1,118 @@ +import Foundation + +// One column shape's formatters: the pattern as the user chose it, and the same pattern with a slot +// for the sub-second digits a value may carry, used only for a value that arrived with a fraction. +private struct TemporalFormatters { + let plain: DateFormatter + let withFractionalSeconds: DateFormatter? +} + +// Not Sendable: each value sets its formatter's zone, so one renderer serves one thread. Work off the +// main actor builds its own from the format option. +final class DateDisplayRenderer { + let format: DateFormatOption + + private let dateTimeFormatters: TemporalFormatters + private let dateOnlyFormatters: TemporalFormatters + private let timeOnlyFormatters: TemporalFormatters + + // The only formatter that renders an instant rather than a wall clock the value carries, so its zone + // is the reader's. `.autoupdatingCurrent` follows a system zone change; a captured `.current` is + // frozen for the life of the process, measured. + private let instantFormatter: DateFormatter + + private let formatCache = NSCache() + + init(format: DateFormatOption) { + self.format = format + dateTimeFormatters = Self.createFormatters(for: format, components: .dateAndTime) + dateOnlyFormatters = Self.createFormatters(for: format, components: .dateOnly) + timeOnlyFormatters = Self.createFormatters(for: format, components: .timeOnly) + instantFormatter = Self.createFormatter(pattern: format.formatString(for: .dateAndTime)) + instantFormatter.timeZone = .autoupdatingCurrent + formatCache.countLimit = 100_000 + } + + func format(_ date: Date) -> String { + instantFormatter.string(from: date) + } + + // Nil when the text does not parse. The value renders in the zone it was written in, not the + // reader's, so the grid and the cell editor agree on the day. A value with an offset prints that + // offset as the literal text it arrived with: a pattern token prints the formatter's zone, which + // would give a naive `DATETIME` an offset the database never held. The cache key needs no zone, + // since the zone is read off the value. + func format(dateString: String, columnType: ColumnType?) -> String? { + let components = temporalComponents(for: columnType) + let cacheKey = "\(components.rawValue)|\(dateString)" as NSString + if let cached = formatCache.object(forKey: cacheKey) { + return cached.length == 0 ? nil : cached as String + } + + guard let parsed = DatabaseDateParser.parse(dateString) else { + formatCache.setObject("" as NSString, forKey: cacheKey) + return nil + } + var result = render(parsed, with: formatters(for: components)) + if self.format.rendersTime(for: components), let suffix = parsed.layout.timeZoneSuffix { + result += suffix + } + formatCache.setObject(result as NSString, forKey: cacheKey) + return result + } + + // A column with no type reads as a full timestamp, the grid's fallback before the editor and the + // formatter shared this vocabulary. + private func temporalComponents(for columnType: ColumnType?) -> TemporalComponents { + guard let columnType else { return .dateAndTime } + return DateEditingService.components(for: columnType) + } + + private func formatters(for components: TemporalComponents) -> TemporalFormatters { + switch components { + case .dateOnly: return dateOnlyFormatters + case .timeOnly: return timeOnlyFormatters + case .dateAndTime: return dateTimeFormatters + } + } + + // The value's own sub-second digits are spliced in, not rendered, so `.789012` stays `.789012` and two + // rows a microsecond apart never share a value filter entry. A fraction of only zeros is padding + // (MySQL writes one on every `DATETIME(6)` row) and is dropped. + private func render(_ parsed: ParsedTemporalValue, with formatters: TemporalFormatters) -> String { + guard let fractionalSeconds = parsed.layout.fractionalSeconds, + fractionalSeconds.contains(where: { $0 != "0" && $0 != "." }), + let marked = formatters.withFractionalSeconds + else { + formatters.plain.timeZone = parsed.timeZone + return formatters.plain.string(from: parsed.wholeSecond) + } + marked.timeZone = parsed.timeZone + return marked.string(from: parsed.wholeSecond) + .replacingOccurrences(of: String(DateFormatOption.fractionalSecondsMarker), with: fractionalSeconds) + } + + // A fixed pattern needs a fixed locale and calendar: the user's region would render timestamps in + // the Buddhist or Japanese calendar and change the value the grid writes back to SQL. + private static let fixedPatternLocale = Locale(identifier: "en_US_POSIX") + + private static func createFormatters( + for format: DateFormatOption, + components: TemporalComponents + ) -> TemporalFormatters { + TemporalFormatters( + plain: createFormatter(pattern: format.formatString(for: components)), + withFractionalSeconds: format.fractionalSecondsFormatString(for: components) + .map(createFormatter(pattern:)) + ) + } + + private static func createFormatter(pattern: String) -> DateFormatter { + let formatter = DateFormatter() + formatter.dateFormat = pattern + formatter.locale = fixedPatternLocale + formatter.calendar = Calendar(identifier: .gregorian) + formatter.timeZone = TimeZone.current + return formatter + } +} diff --git a/apple/TablePro/Core/Services/Formatting/DateFormattingService.swift b/apple/TablePro/Core/Services/Formatting/DateFormattingService.swift index 25e5751407..67449a233a 100644 --- a/apple/TablePro/Core/Services/Formatting/DateFormattingService.swift +++ b/apple/TablePro/Core/Services/Formatting/DateFormattingService.swift @@ -2,73 +2,33 @@ // DateFormattingService.swift // TablePro // -// Centralized date formatting service that respects user settings. -// Thread-safe singleton that formats dates according to DataGridSettings.dateFormat. -// import Combine import Foundation -/// The pair of formatters one column shape needs: the pattern as the user chose it, and the same -/// pattern with a slot for the sub-second digits a value may carry. -private struct TemporalFormatters { - let plain: DateFormatter - let withFractionalSeconds: DateFormatter? -} - -/// Centralized date formatting service that respects user settings @MainActor final class DateFormattingService { static let shared = DateFormattingService() - // MARK: - Properties - - /// Cached formatters for the current user-selected format. - /// - /// Two per column shape, not one: the marked variant carries a quoted slot after the seconds - /// and is used only for a value that arrived with a fraction, so a value without one pays - /// neither the second format nor the splice. - private var dateTimeFormatters: TemporalFormatters - private var dateOnlyFormatters: TemporalFormatters - private var timeOnlyFormatters: TemporalFormatters - - /// The one formatter that renders an instant rather than a wall clock the value already - /// carries, so it is the only one whose zone is the reader's. - /// - /// It holds `.autoupdatingCurrent`, which follows a system time zone change with no - /// reassignment; a captured `.current` is frozen for the life of the process, measured. Its - /// pattern still tracks the user's chosen format, so `updateFormat` rewrites that and never - /// the zone. - private let instantFormatter: DateFormatter + // Replaced on a format change rather than reconfigured, so no text cached under the old format survives. + private(set) var renderer: DateDisplayRenderer private var systemTimeZoneObserver: NSObjectProtocol? - /// Rises each time the Mac's zone moves, so a display cache built in the old one compares - /// unequal and is replaced rather than repaired. + // Rises each time the Mac's zone moves, so a display cache built in the old one compares unequal and + // is replaced rather than repaired. private(set) var systemTimeZoneGeneration = 0 - /// Current date format option - private(set) var currentFormat: DateFormatOption - - /// Cache for formatted date strings to avoid repeated parsing - private let formatCache = NSCache() - - // MARK: - Initialization + var currentFormat: DateFormatOption { renderer.format } + // AppSettingsManager sets the user's format once it has loaded. private init() { - // Will be updated by AppSettingsManager after it completes initialization - self.currentFormat = .iso8601 - self.dateTimeFormatters = Self.createFormatters(for: .iso8601, components: .dateAndTime) - self.dateOnlyFormatters = Self.createFormatters(for: .iso8601, components: .dateOnly) - self.timeOnlyFormatters = Self.createFormatters(for: .iso8601, components: .timeOnly) - self.instantFormatter = Self.createFormatter(pattern: DateFormatOption.iso8601.formatString(for: .dateAndTime)) - self.instantFormatter.timeZone = .autoupdatingCurrent - formatCache.countLimit = 100_000 + renderer = DateDisplayRenderer(format: .iso8601) observeSystemTimeZone() } - /// `NSSystemTimeZoneDidChange` documents no posting thread, and this app has already shipped a - /// crash from assuming one, so the observer names the main queue and hops explicitly. + // `NSSystemTimeZoneDidChange` documents no posting thread, and this app has already shipped a crash + // from assuming one, so the observer names the main queue and hops explicitly. private func observeSystemTimeZone() { systemTimeZoneObserver = NotificationCenter.default.addObserver( forName: .NSSystemTimeZoneDidChange, @@ -82,122 +42,16 @@ final class DateFormattingService { } } - // MARK: - Public Methods - - /// Update the date format (called by AppSettingsManager when settings change) func updateFormat(_ format: DateFormatOption) { guard format != currentFormat else { return } - currentFormat = format - dateTimeFormatters = Self.createFormatters(for: format, components: .dateAndTime) - dateOnlyFormatters = Self.createFormatters(for: format, components: .dateOnly) - timeOnlyFormatters = Self.createFormatters(for: format, components: .timeOnly) - instantFormatter.dateFormat = format.formatString(for: .dateAndTime) - // Clear cache when format changes since all cached values are now stale - formatCache.removeAllObjects() + renderer = DateDisplayRenderer(format: format) } - /// Format a date using current user settings - /// - Parameter date: The date to format - /// - Returns: Formatted date string func format(_ date: Date) -> String { - instantFormatter.string(from: date) + renderer.format(date) } - /// Format a string date value (parse then format). - /// - /// The value is rendered in the zone it was written in, not the reader's. A value carrying an - /// offset names an instant in that offset, and the cell editor opens it there, so formatting it - /// somewhere else made the grid and the picker disagree about which day the row held. A value - /// with no offset is naive and parses in GMT, so it prints the wall clock it was written with - /// whatever zone the reader is in. - /// - /// The offset itself is printed with it, because a wall clock alone does not name an instant. - /// It is the literal text the value arrived with, off the same layout the write-back reads. A - /// pattern token cannot serve: it prints the formatter's own zone, which for a value carrying - /// no offset is the reader's, so a `DATETIME` would gain an offset the database never held. - /// - /// The cache key needs no zone: the zone is read off the value, so the string determines it. - /// - Parameter dateString: Date string from database (ISO 8601, MySQL timestamp, etc.) - /// - Parameter columnType: Column type, used to pick date-only / time-only / datetime variant - /// - Returns: Formatted date string, or nil if unparseable func format(dateString: String, columnType: ColumnType? = nil) -> String? { - let components = temporalComponents(for: columnType) - let cacheKey = "\(components.rawValue)|\(dateString)" as NSString - if let cached = formatCache.object(forKey: cacheKey) { - return cached.length == 0 ? nil : cached as String - } - - guard let parsed = DatabaseDateParser.parse(dateString) else { - formatCache.setObject("" as NSString, forKey: cacheKey) - return nil - } - var result = render(parsed, with: formatters(for: components)) - if currentFormat.rendersTime(for: components), let suffix = parsed.layout.timeZoneSuffix { - result += suffix - } - formatCache.setObject(result as NSString, forKey: cacheKey) - return result - } - - /// A column with no type at all is read as a full timestamp, which is what the grid's own - /// fallback did before the editor and the formatter shared this vocabulary. - private func temporalComponents(for columnType: ColumnType?) -> TemporalComponents { - guard let columnType else { return .dateAndTime } - return DateEditingService.components(for: columnType) - } - - private func formatters(for components: TemporalComponents) -> TemporalFormatters { - switch components { - case .dateOnly: return dateOnlyFormatters - case .timeOnly: return timeOnlyFormatters - case .dateAndTime: return dateTimeFormatters - } - } - - /// The value's own sub-second digits are spliced in, not rendered, so `.5` stays `.5` and - /// `.789012` stays `.789012`. Two rows one microsecond apart used to draw the same text, which - /// also collapsed them into one entry in the column's value filter. - /// - /// A fraction of nothing but zeros is padding rather than precision, and MySQL writes one on - /// every row of a `DATETIME(6)` where PostgreSQL trims it at the server. It is dropped, so such - /// a column reads exactly as it did before and only a value that carries real digits grows. - private func render(_ parsed: ParsedTemporalValue, with formatters: TemporalFormatters) -> String { - guard let fractionalSeconds = parsed.layout.fractionalSeconds, - fractionalSeconds.contains(where: { $0 != "0" && $0 != "." }), - let marked = formatters.withFractionalSeconds - else { - formatters.plain.timeZone = parsed.timeZone - return formatters.plain.string(from: parsed.wholeSecond) - } - marked.timeZone = parsed.timeZone - return marked.string(from: parsed.wholeSecond) - .replacingOccurrences(of: String(DateFormatOption.fractionalSecondsMarker), with: fractionalSeconds) - } - - // MARK: - Private Helper Methods - - /// `dateFormat` is a fixed pattern, so it must be evaluated against a fixed locale and - /// calendar. Following the user's region renders database timestamps in the Buddhist or - /// Japanese calendar and reformats the very value the grid round-trips back to SQL. - private static let fixedPatternLocale = Locale(identifier: "en_US_POSIX") - - private static func createFormatters( - for format: DateFormatOption, - components: TemporalComponents - ) -> TemporalFormatters { - TemporalFormatters( - plain: createFormatter(pattern: format.formatString(for: components)), - withFractionalSeconds: format.fractionalSecondsFormatString(for: components) - .map(createFormatter(pattern:)) - ) - } - - private static func createFormatter(pattern: String) -> DateFormatter { - let formatter = DateFormatter() - formatter.dateFormat = pattern - formatter.locale = fixedPatternLocale - formatter.calendar = Calendar(identifier: .gregorian) - formatter.timeZone = TimeZone.current - return formatter + renderer.format(dateString: dateString, columnType: columnType) } } diff --git a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift index 3732e7d5b0..96a4ee5e54 100644 --- a/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift +++ b/apple/TablePro/Core/Services/Formatting/ValueDisplayFormatter.swift @@ -12,14 +12,23 @@ import TableProUtilities // per cell, and it has to stay reachable without dragging the preference layer behind it. // Resolving *which* format a column carries is a separate job with separate dependencies, and it // lives in `ValueDisplayFormatService`. -@MainActor enum ValueDisplayFormatter { + @MainActor + static func apply( + _ rawValue: String, + format: ValueDisplayFormat, + columnType: ColumnType? = nil + ) -> String? { + apply(rawValue, format: format, columnType: columnType, dates: DateFormattingService.shared.renderer) + } + // nil where the format cannot render this value, so the caller falls back to how the column // would have read without it rather than showing a value the format did not actually produce. static func apply( _ rawValue: String, format: ValueDisplayFormat, - columnType: ColumnType? = nil + columnType: ColumnType?, + dates: DateDisplayRenderer ) -> String? { switch format { case .raw: @@ -31,9 +40,9 @@ enum ValueDisplayFormatter { case .uuidSwappedTime: return formatAsUuid(rawValue, order: .swappedTime) case .unixTimestamp: - return formatAsTimestamp(rawValue, divideBy: 1) + return formatAsTimestamp(rawValue, divideBy: 1, dates: dates) case .unixTimestampMillis: - return formatAsTimestamp(rawValue, divideBy: 1_000) + return formatAsTimestamp(rawValue, divideBy: 1_000, dates: dates) case .json, .phpSerialized: return rawValue case .messagePack, .protobuf, .igbinary, .avro, .protobufSchema, .schemaRegistry: @@ -105,11 +114,11 @@ enum ValueDisplayFormatter { return "\(p1)-\(p2)-\(p3)-\(p4)-\(p5)" } - private static func formatAsTimestamp(_ rawValue: String, divideBy divisor: Double) -> String { + private static func formatAsTimestamp(_ rawValue: String, divideBy divisor: Double, dates: DateDisplayRenderer) -> String { guard let numericValue = Double(rawValue) else { return rawValue } let seconds = numericValue / divisor let date = Date(timeIntervalSince1970: seconds) - return DateFormattingService.shared.format(date) + return dates.format(date) } } diff --git a/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService.swift b/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService.swift index f7e894e09b..dd23d2eac5 100644 --- a/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService.swift +++ b/apple/TablePro/Core/Services/Query/DatabaseTreeMetadataService.swift @@ -58,8 +58,12 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { let commit: ReferenceWritableKeyPath> let dedup: OnceTask, Value> let label: String + // What a read uses while nothing is on screen; a reread of rows already shown always goes bulk. + let workload: MetadataConnectionPool.Workload } + private typealias ListFetch = @Sendable (MetadataConnectionPool.Workload) async throws -> Value + @Published private(set) var databaseList: [UUID: MetadataLoadState<[DatabaseMetadata]>] = [:] @Published private(set) var schemaList: [DatabaseKey: MetadataLoadState<[String]>] = [:] @Published private(set) var tablesState: [ObjectsKey: MetadataLoadState<[TableInfo]>] = [:] @@ -98,27 +102,27 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { private init() {} private var schemaLists: TreeList { - TreeList(state: \.schemaList, commit: \.schemaListCommit, dedup: schemaDedup, label: "schemas") + TreeList(state: \.schemaList, commit: \.schemaListCommit, dedup: schemaDedup, label: "schemas", workload: .interactive) } private var tableLists: TreeList { - TreeList(state: \.tablesState, commit: \.tablesCommit, dedup: tablesDedup, label: "tables") + TreeList(state: \.tablesState, commit: \.tablesCommit, dedup: tablesDedup, label: "tables", workload: .interactive) } private var routineLists: TreeList { - TreeList(state: \.routinesState, commit: \.routinesCommit, dedup: routinesDedup, label: "routines") + TreeList(state: \.routinesState, commit: \.routinesCommit, dedup: routinesDedup, label: "routines", workload: .bulk) } private var triggerLists: TreeList { - TreeList(state: \.triggersState, commit: \.triggersCommit, dedup: triggersDedup, label: "triggers") + TreeList(state: \.triggersState, commit: \.triggersCommit, dedup: triggersDedup, label: "triggers", workload: .bulk) } private var typeLists: TreeList { - TreeList(state: \.typesState, commit: \.typesCommit, dedup: typesDedup, label: "types") + TreeList(state: \.typesState, commit: \.typesCommit, dedup: typesDedup, label: "types", workload: .bulk) } private var partitionLists: TreeList { - TreeList(state: \.partitionsState, commit: \.partitionsCommit, dedup: partitionsDedup, label: "partitions") + TreeList(state: \.partitionsState, commit: \.partitionsCommit, dedup: partitionsDedup, label: "partitions", workload: .interactive) } // MARK: - Reads @@ -467,10 +471,14 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { } } - private func fetchDatabaseList(connectionId: UUID, databaseType: DatabaseType) async throws -> [DatabaseMetadata] { + private func fetchDatabaseList( + connectionId: UUID, + databaseType: DatabaseType, + workload: MetadataConnectionPool.Workload = .interactive + ) async throws -> [DatabaseMetadata] { let systemNames = Set(PluginManager.shared.systemDatabaseNames(for: databaseType)) return try await databaseDedup.execute(key: connectionId) { [self] in - try await withDriver(connectionId: connectionId, database: nil) { driver in + try await withDriver(connectionId: connectionId, database: nil, workload: workload) { driver in try await driver.fetchDatabases().sorted().map { DatabaseMetadata.minimal(name: $0, isSystem: systemNames.contains($0)) } @@ -550,7 +558,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { } guard isConnected(connectionId) else { return } do { - let list = try await fetchDatabaseList(connectionId: connectionId, databaseType: databaseType) + let list = try await fetchDatabaseList(connectionId: connectionId, databaseType: databaseType, workload: .bulk) guard databaseListFence.isCurrent(token, for: connectionId) else { return } databaseList[connectionId] = .loaded(list) } catch is CancellationError { @@ -648,7 +656,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { connectionId: UUID, in list: TreeList, token explicitToken: Int? = nil, - fetch: @Sendable @escaping () async throws -> Value + fetch: @escaping ListFetch ) async { guard isConnected(connectionId) else { return } let token = explicitToken ?? self[keyPath: list.commit].fence.token(for: key) @@ -656,15 +664,24 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { !self[keyPath: list.commit].freshness.isCurrent(key) else { return } let revision = self[keyPath: list.commit].freshness.revision(for: key) self[keyPath: list.commit].freshness.noteFetchStarted(revision, for: key) + // Rows on screen mean nobody waits on this read, so completion goes ahead of it. + let workload: MetadataConnectionPool.Workload switch self[keyPath: list.state][key] ?? .idle { - case .loaded, .loading: break - case .idle, .failed: self[keyPath: list.state][key] = .loading + case .loaded: + workload = .bulk + case .loading: + workload = list.workload + case .idle, .failed: + workload = list.workload + self[keyPath: list.state][key] = .loading } let outcome: MetadataFetchOutcome var failure: Error? do { - outcome = .fetched(try await list.dedup.execute(key: ListFetchKey(key: key, revision: revision), work: fetch)) + outcome = .fetched(try await list.dedup.execute(key: ListFetchKey(key: key, revision: revision)) { + try await fetch(workload) + }) } catch is CancellationError { outcome = .cancelled } catch { @@ -713,7 +730,7 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { _ key: Key, connectionId: UUID, in list: TreeList, - fetch: @Sendable @escaping () async throws -> Value + fetch: @escaping ListFetch ) async { let token = self[keyPath: list.commit].fence.supersede(key) self[keyPath: list.commit].freshness.markChanged(key) @@ -722,55 +739,55 @@ final class DatabaseTreeMetadataService: ObservableObject, CatalogChangeTarget { await read(key, connectionId: connectionId, in: list, token: token, fetch: fetch) } - private func schemaListFetch(_ key: DatabaseKey) -> @Sendable () async throws -> [String] { - { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database) { driver in + private func schemaListFetch(_ key: DatabaseKey) -> ListFetch<[String]> { + { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchSchemas() } } } - private func tableListFetch(_ key: ObjectsKey) -> @Sendable () async throws -> [TableInfo] { + private func tableListFetch(_ key: ObjectsKey) -> ListFetch<[TableInfo]> { let schema = key.schema - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchTables(schema: schema) } } } - private func routineListFetch(_ key: ObjectsKey) -> @Sendable () async throws -> [RoutineInfo] { + private func routineListFetch(_ key: ObjectsKey) -> ListFetch<[RoutineInfo]> { let schema = key.schema - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database, workload: .bulk) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchRoutines(schema: schema) } } } - private func triggerListFetch(_ key: ObjectsKey) -> @Sendable () async throws -> [TriggerInfo] { + private func triggerListFetch(_ key: ObjectsKey) -> ListFetch<[TriggerInfo]> { let schema = key.schema - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database, workload: .bulk) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchAllTriggers(schema: schema) } } } - private func typeListFetch(_ key: ObjectsKey) -> @Sendable () async throws -> [UserDefinedTypeInfo] { + private func typeListFetch(_ key: ObjectsKey) -> ListFetch<[UserDefinedTypeInfo]> { let schema = key.schema - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database, workload: .bulk) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchUserDefinedTypes(schema: schema) } } } - private func partitionListFetch(_ key: PartitionsKey) -> @Sendable () async throws -> [PartitionInfo] { + private func partitionListFetch(_ key: PartitionsKey) -> ListFetch<[PartitionInfo]> { let schema = key.schema let table = key.table - return { [self] in - try await withDriver(connectionId: key.connectionId, database: key.database) { driver in + return { [self] workload in + try await withDriver(connectionId: key.connectionId, database: key.database, workload: workload) { driver in try await driver.fetchPartitionDetails(table: table, schema: schema) } } diff --git a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift index c7a274f88f..9de54b604c 100644 --- a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift +++ b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift @@ -4,6 +4,7 @@ // import Foundation +import os import TableProPluginKit // The lease a task's work runs under. That work keeps the lease until a lease it asks for returns, so such a @@ -11,10 +12,13 @@ import TableProPluginKit @MainActor private final class PooledLease { let connectionId: UUID + // A lease nested in this one never takes this entry: it would queue behind the work that waits on it. + let entryId: ObjectIdentifier var isOpen = true - init(connectionId: UUID) { + init(connectionId: UUID, entryId: ObjectIdentifier) { self.connectionId = connectionId + self.entryId = entryId } } @@ -31,43 +35,81 @@ final class MetadataConnectionPool { case bulk } + // A scope with no schema reads wherever the connection's own settings put it, so its entries never switch. + private enum Slot: Hashable, Sendable { + case home + case followsSchema + } + + // No schema in the key: an entry moves to each lease's schema, so the schemas of one database share connections. private struct Key: Hashable, Sendable { - let scope: DatabaseScope + let connectionId: UUID + let database: String let workload: Workload + let slot: Slot + + init(scope: DatabaseScope, workload: Workload) { + connectionId = scope.connectionId + database = scope.database + self.workload = workload + slot = scope.schema == nil ? .home : .followsSchema + } + } + + private struct OpenKey: Hashable, Sendable { + let key: Key + let schema: String? + // Set for a lease inside another lease's work, so its open is its own and nobody queues on it. + let owner: UUID? + } + + private enum LeaseOutcome: Sendable { + case ran(T) + // The entry was left in a transaction and is closing, so the lease takes another. + case retired } @MainActor private final class Entry { let driver: DatabaseDriver - var lastUsed: Date - var inFlightCount: Int - var closeWhenIdle: Bool + var lastUsed = Date() + var inFlightCount = 0 + var closeWhenIdle = false + // The schema the last lease moved it to, used only to pick an entry; nil until a lease has moved it. + var schema: String? + // Only leases on one schema queue on a busy entry, so this is the schema its queue is on. + var queuedSchema: String? + // A startup command can move the search path after the driver read it, so the driver's cache is not trusted. + var forcesNextSwitch: Bool + var hasServedALease = false private var tail: Task = Task {} - init(driver: DatabaseDriver) { + init(driver: DatabaseDriver, schema: String?, forcesNextSwitch: Bool) { self.driver = driver - self.lastUsed = Date() - self.inFlightCount = 0 - self.closeWhenIdle = false + self.schema = schema + self.queuedSchema = schema + self.forcesNextSwitch = forcesNextSwitch } - /// The work runs in its own task so the next caller can queue behind it, so - /// cancelling the caller has to be forwarded explicitly or a stopped query - /// would keep running with nobody waiting on it. + // The work runs in its own task so the next caller can queue behind it, so a cancel is forwarded by hand. func runSerially( connectionId: UUID, + prepare: @escaping @MainActor @Sendable (Entry) async throws -> Bool, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T - ) async throws -> T { + ) async throws -> LeaseOutcome { let previous = tail let driver = self.driver - let work = Task { @MainActor () async throws -> T in + let work = Task { @MainActor [self] () async throws -> LeaseOutcome in await previous.value try Task.checkCancellation() - let lease = PooledLease(connectionId: connectionId) + guard try await prepare(self) else { return .retired } + hasServedALease = true + let lease = PooledLease(connectionId: connectionId, entryId: ObjectIdentifier(self)) defer { lease.isOpen = false } - return try await PooledLeaseContext.$open.withValue(PooledLeaseContext.open + [lease]) { + let value = try await PooledLeaseContext.$open.withValue(PooledLeaseContext.open + [lease]) { try await body(driver) } + return .ran(value) } tail = Task { @MainActor in _ = try? await work.value } return try await withTaskCancellationHandler( @@ -79,27 +121,22 @@ final class MetadataConnectionPool { typealias DriverOpener = @MainActor (DatabaseScope) async throws -> DatabaseDriver - /// Why an open still in progress was taken away from the callers waiting on it. + // Why an open still in progress was taken away from the callers waiting on it. private enum Withdrawal { - /// The transport it was dialing is being rebuilt, so its callers take a connection again - /// once the replacement is in place. + // The transport it was dialing is being rebuilt, so its callers take a connection again after the replacement. case transportReplaced - /// What it was opened for is going away, so nothing may open it again. + // What it was opened for is going away, so nothing may open it again. case closed } - /// One open in progress, shared by every caller that asks for its key while it runs. It stays - /// listed until the last of those callers has taken its entry, because an entry nobody has - /// taken yet looks idle and would otherwise be the first one closed. + // Shared by every caller that asks for its key and schema while it runs. It stays listed until the last of them + // has taken its entry, because an entry nobody has taken yet looks idle and would be the first one closed. @MainActor private final class PendingOpen { - let task: Task + var task: Task? + var entry: Entry? var withdrawal: Withdrawal? var waiterCount = 0 - - init(task: Task) { - self.task = task - } } private struct TransportWaiter { @@ -111,22 +148,27 @@ final class MetadataConnectionPool { @MainActor private final class RoomTicket { let key: Key + let schema: String? + let held: Set var continuation: CheckedContinuation? var isWithdrawn = false - init(key: Key) { + init(key: Key, schema: String?, held: Set) { self.key = key + self.schema = schema + self.held = held } } - private var entries: [Key: Entry] = [:] - private var pending: [Key: PendingOpen] = [:] + private var entries: [Key: [Entry]] = [:] + private var pending: [OpenKey: PendingOpen] = [:] private var transportReplacements: [UUID: Int] = [:] private var transportWaiters: [UUID: [TransportWaiter]] = [:] private var roomQueues: [UUID: [RoomTicket]] = [:] private let openDriver: DriverOpener static let maxPerConnection = 6 private var sweeper: Task? + private static let logger = Logger(subsystem: "com.TablePro", category: "MetadataConnectionPool") /// How long a pooled connection may sit unused before it is handed back. /// @@ -156,38 +198,47 @@ final class MetadataConnectionPool { workload: Workload = .interactive, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T { - let entry = try await acquireEntry(scope: scope, workload: workload) - entry.inFlightCount += 1 - entry.lastUsed = Date() - defer { releaseEntry(entry, connectionId: scope.connectionId) } - return try await entry.runSerially(connectionId: scope.connectionId, body) + let key = Key(scope: scope, workload: workload) + while true { + let entry = try await acquireEntry(forKey: key, schema: scope.schema, scope: scope) + if case .ran(let value) = try await run(body, on: entry, inKey: key, schema: scope.schema) { + return value + } + } + } + + private func run( + _ body: @Sendable @escaping (DatabaseDriver) async throws -> T, + on entry: Entry, + inKey key: Key, + schema: String? + ) async throws -> LeaseOutcome { + defer { releaseEntry(entry, connectionId: key.connectionId) } + return try await entry.runSerially( + connectionId: key.connectionId, + prepare: { [self] entry in try await moveEntry(entry, inKey: key, to: schema) }, + body + ) } /// Closes only the leases attached to one database, which is what a rename of that database /// needs: PostgreSQL refuses `ALTER DATABASE ... RENAME` while any backend is connected to it, /// and an expanded row or a tab that ran a query there leaves one here. func closeAll(connectionId: UUID, database: String) { - closeEntries(withdrawingOpensAs: .closed) { scope in - scope.connectionId == connectionId && scope.database == database - } + closeEntries(withdrawingOpensAs: .closed) { $0 == connectionId && $1 == database } } func closeAll(connectionId: UUID) { - closeEntries(withdrawingOpensAs: .closed) { $0.connectionId == connectionId } + closeEntries(withdrawingOpensAs: .closed) { id, _ in id == connectionId } } - /// Holds a connection's pooled work while its transport is rebuilt. - /// - /// A pooled entry stands on the session's effective connection plus its own database, so it only - /// has to go when that endpoint does: a tunnel rebuilt on a new local port, or a reconnect - /// recovering a connection that stopped answering. An open already dialing is withdrawn, and once - /// it has returned the callers waiting on it take a connection again after `endTransportReplacement` - /// rather than failing a load nobody cancelled. A new caller waits for the same moment instead of - /// dialing an endpoint about to close. Replacements nest, and pooled work resumes when the last one - /// ends. Closing the connection, or one of its databases, fails the callers waiting for it. + // Holds a connection's pooled work while its transport is rebuilt: a tunnel on a new local port, or a reconnect of + // a connection that stopped answering. An open already dialing is withdrawn and its callers take a connection again + // after `endTransportReplacement`; a new caller waits for that too. Replacements nest. Closing the connection, or + // one of its databases, fails the callers waiting for it. func beginTransportReplacement(connectionId: UUID) { transportReplacements[connectionId, default: 0] += 1 - closeEntries(withdrawingOpensAs: .transportReplaced) { $0.connectionId == connectionId } + closeEntries(withdrawingOpensAs: .transportReplaced) { id, _ in id == connectionId } } /// Has to follow every `beginTransportReplacement` on every exit, cancellation included, or the @@ -205,29 +256,35 @@ final class MetadataConnectionPool { } #if DEBUG - /// Seeds a connected driver as the pooled connection for `scope`, so a test can observe - /// what runs on the pool without a plugin to open a real connection. + // Seeds a connected driver as a pooled connection already on `scope`'s schema, for a test with no plugin to open one. internal func injectEntry( _ driver: DatabaseDriver, scope: DatabaseScope, workload: Workload = .interactive, lastUsed: Date = Date() ) { - let entry = Entry(driver: driver) + let entry = Entry(driver: driver, schema: scope.schema, forcesNextSwitch: false) entry.lastUsed = lastUsed - entries[Key(scope: scope, workload: workload)] = entry + entry.hasServedALease = true + entries[Key(scope: scope, workload: workload), default: []].append(entry) } internal func pooledDriverCount(for connectionId: UUID) -> Int { - entries.keys.filter { $0.scope.connectionId == connectionId }.count + pooled(on: connectionId).count } internal func heldConnectionCount(for connectionId: UUID) -> Int { heldCount(for: connectionId) } + internal func leaseCount(for connectionId: UUID) -> Int { + pooled(on: connectionId).reduce(0) { $0 + $1.entry.inFlightCount } + } + internal func markInFlight(scope: DatabaseScope, workload: Workload = .interactive) { - entries[Key(scope: scope, workload: workload)]?.inFlightCount += 1 + let list = entries[Key(scope: scope, workload: workload)] ?? [] + let entry = list.first(where: { $0.schema == scope.schema }) ?? list.first + entry?.inFlightCount += 1 } internal var hasSweeper: Bool { @@ -275,33 +332,43 @@ final class MetadataConnectionPool { handRoom(on: connectionId) } - private func closeOrDeferEntry(forKey key: Key) { - guard let entry = entries.removeValue(forKey: key) else { return } + @discardableResult + private func detach(_ entry: Entry, fromKey key: Key) -> Bool { + guard var list = entries[key], let index = list.firstIndex(where: { $0 === entry }) else { return false } + list.remove(at: index) + entries[key] = list.isEmpty ? nil : list + return true + } + + private func removeEntry(_ entry: Entry, fromKey key: Key) { + guard detach(entry, fromKey: key) else { return } if entry.inFlightCount == 0 { entry.driver.disconnect() } else { entry.closeWhenIdle = true } - handRoom(on: key.scope.connectionId) + handRoom(on: key.connectionId) } private func closeEntries( withdrawingOpensAs withdrawal: Withdrawal, - where matches: (DatabaseScope) -> Bool + where matches: (UUID, String) -> Bool ) { var withdrawnFrom: Set = [] - for key in pending.keys where matches(key.scope) { - guard let open = pending.removeValue(forKey: key) else { continue } + for openKey in pending.keys where matches(openKey.key.connectionId, openKey.key.database) { + guard let open = pending.removeValue(forKey: openKey) else { continue } open.withdrawal = withdrawal - open.task.cancel() - withdrawnFrom.insert(key.scope.connectionId) + open.task?.cancel() + withdrawnFrom.insert(openKey.key.connectionId) } if withdrawal == .closed { failTransportWaiters(where: matches) withdrawRoomTickets(where: matches) } - for key in entries.keys where matches(key.scope) { - closeOrDeferEntry(forKey: key) + for (key, list) in entries where matches(key.connectionId, key.database) { + for entry in list { + removeEntry(entry, fromKey: key) + } } for connectionId in withdrawnFrom { handRoom(on: connectionId) @@ -309,14 +376,12 @@ final class MetadataConnectionPool { stopSweeperIfEmpty() } - /// A caller parked for a replacement is waiting on its connection as much as one waiting on a - /// pending open, so closing what it waits for fails it too. Left parked, it would wake when the - /// replacement ends and open a connection for a session, or a database, that has gone. - private func failTransportWaiters(where matches: (DatabaseScope) -> Bool) { + // A parked caller would otherwise wake when the replacement ends and open a connection for what has gone. + private func failTransportWaiters(where matches: (UUID, String) -> Bool) { for (connectionId, waiters) in transportWaiters { - let failing = waiters.filter { matches($0.scope) } + let failing = waiters.filter { matches($0.scope.connectionId, $0.scope.database) } guard !failing.isEmpty else { continue } - let remaining = waiters.filter { !matches($0.scope) } + let remaining = waiters.filter { !matches($0.scope.connectionId, $0.scope.database) } transportWaiters[connectionId] = remaining.isEmpty ? nil : remaining for waiter in failing { waiter.continuation.resume(throwing: CancellationError()) @@ -324,24 +389,29 @@ final class MetadataConnectionPool { } } - private func acquireEntry(scope: DatabaseScope, workload: Workload) async throws -> Entry { - let key = Key(scope: scope, workload: workload) + // In order: an idle entry on the schema, any idle entry, the busy entry on the schema, an open for the schema + // already dialing, a new entry if there is room, else a place in the room line. A lease inside another lease's + // work skips the busy entry and the open: two such leases nesting in opposite directions would wait on each other. + private func acquireEntry(forKey key: Key, schema: String?, scope: DatabaseScope) async throws -> Entry { + let held = entriesHeldByThisTask() + let openKey = OpenKey(key: key, schema: schema, owner: held.isEmpty ? nil : UUID()) while true { try await waitForTransport(for: scope) - if let entry = reusableEntry(forKey: key) { + if let entry = claimEntry(forKey: key, schema: schema, excluding: held) { return entry } - if pending[key] == nil { - try await waitForRoom(forKey: key) - // A wait can outlast its reason: the key opened meanwhile, or the transport is being rebuilt. - if pending[key] != nil || entries[key] != nil || isReplacingTransport(for: scope.connectionId) { + if pending[openKey] == nil { + try await waitForRoom(forKey: key, schema: schema, held: held) + // A wait can outlast its reason: an entry came free, the schema's open started, or the transport is being rebuilt. + if pending[openKey] != nil || placeWithoutOpening(forKey: key, schema: schema, excluding: held) != nil + || isReplacingTransport(for: key.connectionId) { continue } } - let open = try pending[key] ?? startOpen(forKey: key) - let failure = await completion(of: open, forKey: key) - /// A withdrawn open can end either way: a driver whose connect honours cancellation - /// throws, and one that finished first returns without keeping its entry. + let open = try pending[openKey] ?? startOpen(openKey) + let failure = await completion(of: open, for: openKey) + // A withdrawn open can end either way: a driver whose connect honours cancellation throws, and one that + // finished first returns without keeping its entry. if open.withdrawal == .transportReplaced { try Task.checkCancellation() continue @@ -349,72 +419,170 @@ final class MetadataConnectionPool { if let failure { throw failure } - if let entry = entries[key] { + if let entry = open.entry, isListed(entry, inKey: key) { + claim(entry, for: schema) return entry } if open.withdrawal == .closed { throw DatabaseError.notConnected } - forgetSpentOpen(open, forKey: key) + forgetSpentOpen(open, for: openKey) } } - /// An open whose entry went before this caller could take it has nothing left to hand out. - /// Joining it again returns at once, without a suspension, so a caller that did would spin on - /// the main actor and starve the other callers still waiting to resume from it. - private func forgetSpentOpen(_ open: PendingOpen, forKey key: Key) { - guard pending[key] === open else { return } - pending.removeValue(forKey: key) - handRoom(on: key.scope.connectionId) + private func entriesHeldByThisTask() -> Set { + Set(PooledLeaseContext.open.filter { $0.isOpen }.map { $0.entryId }) } - /// A cached entry is only worth reusing while it is both connected and recent. The - /// staleness half matters even with the sweeper running, because a Mac that slept comes - /// back with entries the sweeper never got to and sockets the server has long since - /// closed. The old entry is closed rather than left behind: overwriting `entries[key]` - /// with a fresh one used to leak the driver it replaced. - private func reusableEntry(forKey key: Key) -> Entry? { - guard let entry = entries[key] else { return nil } - if entry.driver.status == .connected, !entry.driver.hasLostConnection, !Self.isStale(entry.lastUsed) { - return entry + private func claimEntry(forKey key: Key, schema: String?, excluding held: Set) -> Entry? { + // A Mac that slept comes back with entries the sweeper never reached and sockets the server already closed. + for entry in entries[key] ?? [] where !isUsable(entry) { + removeEntry(entry, fromKey: key) } - closeOrDeferEntry(forKey: key) - return nil + guard let entry = placeWithoutOpening(forKey: key, schema: schema, excluding: held) else { return nil } + claim(entry, for: schema) + return entry } - private func startOpen(forKey key: Key) throws -> PendingOpen { - guard DatabaseManager.shared.session(for: key.scope.connectionId) != nil else { + // At most one busy entry per schema, so two schemas share a connection only while one of them is idle. + private func placeWithoutOpening( + forKey key: Key, + schema: String?, + excluding held: Set + ) -> Entry? { + let candidates = (entries[key] ?? []).filter { !held.contains(ObjectIdentifier($0)) && isUsable($0) } + let idle = candidates.filter { isIdle($0) } + if let onSchema = idle.first(where: { $0.schema == schema }) { + return onSchema + } + if let any = idle.min(by: { $0.lastUsed < $1.lastUsed }) { + return any + } + guard held.isEmpty else { return nil } + return candidates.first { $0.inFlightCount > 0 && $0.queuedSchema == schema } + } + + private func claim(_ entry: Entry, for schema: String?) { + entry.inFlightCount += 1 + entry.lastUsed = Date() + entry.queuedSchema = schema + } + + // Runs on the entry's queue before the lease's work, so the switch lands where the work reads. + private func moveEntry(_ entry: Entry, inKey key: Key, to schema: String?) async throws -> Bool { + guard key.slot == .followsSchema, let schema, let switchable = entry.driver as? SchemaSwitchable else { + return true + } + let unconditionally = entry.forcesNextSwitch + // Asked on every lease: Snowflake's drivers for one connection share a session, so another one may have moved it. + if !unconditionally, switchable.currentSchema == schema { + entry.schema = schema + return true + } + // A SET sent inside a transaction is undone by its rollback while the driver keeps the new name. A fresh + // entry's transaction came from the startup commands, and every replacement would carry one too. + if entry.hasServedALease { + let state = await entry.driver.sessionTransactionState() + if state == .inTransaction || state == .abortedTransaction { + Self.logger.info("Closing a pooled connection left in a transaction instead of moving its schema") + removeEntry(entry, fromKey: key) + return false + } + } + do { + try await Self.switchSchema( + entry.driver, + to: schema, + timeoutSeconds: Self.stepTimeoutSeconds(for: key.connectionId), + unconditionally: unconditionally + ) + } catch { + // The server may have run a statement the client gave up on, so the next lease sends its own. + entry.schema = nil + entry.forcesNextSwitch = true + if !Self.isConnected(entry.driver) { + removeEntry(entry, fromKey: key) + } + throw error + } + entry.schema = schema + entry.forcesNextSwitch = false + return true + } + + private static func stepTimeoutSeconds(for connectionId: UUID) -> Double? { + guard let seconds = DatabaseManager.shared.session(for: connectionId)?.effectiveQueryTimeoutSeconds, + seconds > 0 + else { return nil } + return Double(seconds) + } + + // Joining an open that has already finished returns without a suspension, so a caller that rejoined one whose entry + // went would spin on the main actor and starve the callers still waiting to resume from it. + private func forgetSpentOpen(_ open: PendingOpen, for openKey: OpenKey) { + guard pending[openKey] === open else { return } + pending.removeValue(forKey: openKey) + handRoom(on: openKey.key.connectionId) + } + + private static func isConnected(_ driver: DatabaseDriver) -> Bool { + driver.status == .connected && !driver.hasLostConnection + } + + private func isUsable(_ entry: Entry) -> Bool { + Self.isConnected(entry.driver) && !Self.isStale(entry.lastUsed) + } + + private func isListed(_ entry: Entry, inKey key: Key) -> Bool { + entries[key]?.contains { $0 === entry } == true + } + + private func isAwaitingItsOpeners(_ entry: Entry) -> Bool { + pending.values.contains { $0.entry === entry } + } + + private func isIdle(_ entry: Entry) -> Bool { + entry.inFlightCount == 0 && !isAwaitingItsOpeners(entry) + } + + // A new entry opens on the connection's own default and each lease moves it, so it can serve every schema. + private func startOpen(_ openKey: OpenKey) throws -> PendingOpen { + let key = openKey.key + guard let session = DatabaseManager.shared.session(for: key.connectionId) else { throw DatabaseError.notConnected } - evictIdleIfNeeded(for: key.scope.connectionId) - let task = Task { [self] in - let entry = Entry(driver: try await openDriver(key.scope)) + evictIdleIfNeeded(for: key.connectionId) + let scope = DatabaseScope(connectionId: key.connectionId, database: key.database, schema: nil) + let forcesFirstSwitch = key.slot == .followsSchema + && DatabaseManager.hasStartupCommands(session.connection.startupCommands) + let open = PendingOpen() + open.task = Task { [self] in + let entry = Entry(driver: try await openDriver(scope), schema: nil, forcesNextSwitch: forcesFirstSwitch) if Task.isCancelled { entry.driver.disconnect() return } - entries[key] = entry + entries[key, default: []].append(entry) + open.entry = entry startSweeperIfNeeded() } - let open = PendingOpen(task: task) - pending[key] = open + pending[openKey] = open return open } - /// Waits for an open to finish, takes it off the pending list once the last caller waiting on it - /// has, and returns how it failed. - private func completion(of open: PendingOpen, forKey key: Key) async -> Error? { + // Waits for an open to finish, takes it off the pending list once its last caller has, and returns how it failed. + private func completion(of open: PendingOpen, for openKey: OpenKey) async -> Error? { open.waiterCount += 1 defer { open.waiterCount -= 1 - if open.waiterCount == 0, pending[key] === open { - pending.removeValue(forKey: key) + if open.waiterCount == 0, pending[openKey] === open { + pending.removeValue(forKey: openKey) // A failed open frees its place, and a taken one leaves an entry that is now evictable once idle. - handRoom(on: key.scope.connectionId) + handRoom(on: openKey.key.connectionId) } } do { - try await open.task.value + _ = try await open.task?.value return nil } catch { return error @@ -464,10 +632,10 @@ final class MetadataConnectionPool { } // First come first served, interactive ahead of bulk; a caller arriving while others wait goes behind them. - private func waitForRoom(forKey key: Key) async throws { - let connectionId = key.scope.connectionId + private func waitForRoom(forKey key: Key, schema: String?, held: Set) async throws { + let connectionId = key.connectionId guard !PooledLeaseContext.open.contains(where: { $0.isOpen && $0.connectionId == connectionId }) else { return } - let ticket = RoomTicket(key: key) + let ticket = RoomTicket(key: key, schema: schema, held: held) roomQueues[connectionId, default: []].append(ticket) defer { leaveRoomQueue(ticket) } while !mayOpen(ticket) { @@ -477,11 +645,18 @@ final class MetadataConnectionPool { } } - private func mayOpen(_ ticket: RoomTicket) -> Bool { + private func proceedsWithoutAPlace(_ ticket: RoomTicket) -> Bool { let key = ticket.key - let connectionId = key.scope.connectionId - if pending[key] != nil || entries[key] != nil || isReplacingTransport(for: connectionId) { return true } - return nextInLine(on: connectionId) === ticket && hasRoom(for: key.workload, on: connectionId) + let joinsAnOpen = ticket.held.isEmpty && pending[OpenKey(key: key, schema: ticket.schema, owner: nil)] != nil + return joinsAnOpen + || placeWithoutOpening(forKey: key, schema: ticket.schema, excluding: ticket.held) != nil + || isReplacingTransport(for: key.connectionId) + } + + private func mayOpen(_ ticket: RoomTicket) -> Bool { + if proceedsWithoutAPlace(ticket) { return true } + let connectionId = ticket.key.connectionId + return nextInLine(on: connectionId) === ticket && hasRoom(for: ticket.key.workload, on: connectionId) } private func nextInLine(on connectionId: UUID) -> RoomTicket? { @@ -489,18 +664,24 @@ final class MetadataConnectionPool { return line.first { $0.key.workload == .interactive } ?? line.first } - // Wakes only the caller next in line, so a freed place goes to it rather than to whoever resumes first. + // A freed place goes only to the caller next in line. Callers that need no new place go too, and first, so an + // idle entry is reused before the next in line can evict it. private func handRoom(on connectionId: UUID) { - guard let next = nextInLine(on: connectionId), - let continuation = next.continuation, - mayOpen(next) - else { return } - next.continuation = nil - continuation.resume() + let parked = (roomQueues[connectionId] ?? []).filter { $0.continuation != nil } + var woken = parked.filter { proceedsWithoutAPlace($0) } + if let next = nextInLine(on: connectionId), next.continuation != nil, + !woken.contains(where: { $0 === next }), mayOpen(next) { + woken.append(next) + } + for ticket in woken { + guard let continuation = ticket.continuation else { continue } + ticket.continuation = nil + continuation.resume() + } } private func leaveRoomQueue(_ ticket: RoomTicket) { - let connectionId = ticket.key.scope.connectionId + let connectionId = ticket.key.connectionId roomQueues[connectionId]?.removeAll { $0 === ticket } if roomQueues[connectionId]?.isEmpty == true { roomQueues.removeValue(forKey: connectionId) @@ -536,9 +717,9 @@ final class MetadataConnectionPool { } // Same reason as `failTransportWaiters`: a parked caller would otherwise open what was just closed. - private func withdrawRoomTickets(where matches: (DatabaseScope) -> Bool) { + private func withdrawRoomTickets(where matches: (UUID, String) -> Bool) { for line in roomQueues.values { - for ticket in line where matches(ticket.key.scope) { + for ticket in line where matches(ticket.key.connectionId, ticket.key.database) { ticket.isWithdrawn = true failRoomTicket(ticket) } @@ -660,14 +841,23 @@ final class MetadataConnectionPool { } } - static func switchSchema(_ driver: DatabaseDriver, to schema: String, timeoutSeconds: Double?) async throws { + static func switchSchema( + _ driver: DatabaseDriver, + to schema: String, + timeoutSeconds: Double?, + unconditionally: Bool = false + ) async throws { guard let switchable = driver as? SchemaSwitchable else { return } try await bounded( driver: driver, timeoutSeconds: timeoutSeconds, timeoutMessage: String(format: String(localized: "Switching to schema '%@' timed out."), schema) ) { - try await switchable.switchSchemaIfNeeded(to: schema) + if unconditionally { + try await switchable.switchSchema(to: schema) + } else { + try await switchable.switchSchemaIfNeeded(to: schema) + } } } @@ -740,15 +930,16 @@ final class MetadataConnectionPool { sweeper = nil } - /// Hands back every connection nobody has used for `idleTimeout`. An entry with work on it is - /// left alone whatever its age: `lastUsed` is stamped when the work starts, so a long export - /// would otherwise have its own connection closed underneath it. + // An entry with work on it stays whatever its age: `lastUsed` is stamped when the work starts, so a long export + // would otherwise have its own connection closed underneath it. internal func sweepIdleEntries(now: Date = Date()) { var sweptFrom: Set = [] - for (key, entry) in entries where entry.inFlightCount == 0 && Self.isStale(entry.lastUsed, now: now) { - entries.removeValue(forKey: key) - entry.driver.disconnect() - sweptFrom.insert(key.scope.connectionId) + for (key, list) in entries { + for entry in list where entry.inFlightCount == 0 && Self.isStale(entry.lastUsed, now: now) { + detach(entry, fromKey: key) + entry.driver.disconnect() + sweptFrom.insert(key.connectionId) + } } for connectionId in sweptFrom { handRoom(on: connectionId) @@ -765,22 +956,34 @@ final class MetadataConnectionPool { // Bulk never takes the last place, so a completion read or a sidebar expand never waits behind a burst of bulk work. private func hasRoom(for workload: Workload, on connectionId: UUID) -> Bool { let placeIsFree = heldCount(for: connectionId) < Self.maxPerConnection - || leastRecentlyUsedIdleKey(for: connectionId) != nil + || leastRecentlyUsedIdleEntry(for: connectionId) != nil guard workload == .bulk else { return placeIsFree } return placeIsFree && busyCount(for: connectionId) < Self.maxPerConnection - 1 } - private func busyCount(for connectionId: UUID) -> Int { - var keys = Set(pending.keys.filter { $0.scope.connectionId == connectionId }) - for (key, entry) in entries where key.scope.connectionId == connectionId && entry.inFlightCount > 0 { - keys.insert(key) + private func pooled(on connectionId: UUID) -> [(key: Key, entry: Entry)] { + var found: [(key: Key, entry: Entry)] = [] + for (key, list) in entries where key.connectionId == connectionId { + found += list.map { (key: key, entry: $0) } } - return keys.count + return found + } + + // An open still dialing holds a place. One that landed holds it through its entry, and counts once. + private func openingCount(for connectionId: UUID) -> Int { + pending.filter { openKey, open in + guard openKey.key.connectionId == connectionId else { return false } + guard let landed = open.entry else { return true } + return !isListed(landed, inKey: openKey.key) + }.count } - /// Keeps a connection at the limit once its work ends. Closing only one idle entry per open, as - /// the pool used to, left a burst's high-water mark standing until the idle sweep: a tree Refresh - /// across many databases held every connection it opened for ten minutes (#3103). + private func busyCount(for connectionId: UUID) -> Int { + pooled(on: connectionId).filter { !isIdle($0.entry) }.count + openingCount(for: connectionId) + } + + // Closing one idle entry per open, as the pool used to, left a burst's high-water mark standing until the idle + // sweep: a tree Refresh across many databases held every connection it opened for ten minutes (#3103). private func trimIdleEntries(for connectionId: UUID) { while heldCount(for: connectionId) > Self.maxPerConnection { guard evictLeastRecentlyUsedIdleEntry(for: connectionId) else { break } @@ -788,29 +991,22 @@ final class MetadataConnectionPool { stopSweeperIfEmpty() } - /// Every key the connection holds a connection for or is opening one for. Counted by key, so an - /// open that has finished but is still listed until its callers take the entry counts once. private func heldCount(for connectionId: UUID) -> Int { - var keys = Set(entries.keys.filter { $0.scope.connectionId == connectionId }) - keys.formUnion(pending.keys.filter { $0.scope.connectionId == connectionId }) - return keys.count + pooled(on: connectionId).count + openingCount(for: connectionId) } @discardableResult private func evictLeastRecentlyUsedIdleEntry(for connectionId: UUID) -> Bool { - guard let key = leastRecentlyUsedIdleKey(for: connectionId), - let entry = entries.removeValue(forKey: key) + guard let victim = leastRecentlyUsedIdleEntry(for: connectionId), + detach(victim.entry, fromKey: victim.key) else { return false } - entry.driver.disconnect() + victim.entry.driver.disconnect() return true } - /// An entry whose open is still listed has callers that have not taken it yet, so it is not idle - /// however it looks. - private func leastRecentlyUsedIdleKey(for connectionId: UUID) -> Key? { - let idle = entries.filter { key, entry in - key.scope.connectionId == connectionId && entry.inFlightCount == 0 && pending[key] == nil - } - return idle.min(by: { $0.value.lastUsed < $1.value.lastUsed })?.key + private func leastRecentlyUsedIdleEntry(for connectionId: UUID) -> (key: Key, entry: Entry)? { + pooled(on: connectionId) + .filter { isIdle($0.entry) } + .min { $0.entry.lastUsed < $1.entry.lastUsed } } } diff --git a/apple/TablePro/Core/Storage/ConnectionStorage.swift b/apple/TablePro/Core/Storage/ConnectionStorage.swift index 8fadc3d6d4..80ef70f90b 100644 --- a/apple/TablePro/Core/Storage/ConnectionStorage.swift +++ b/apple/TablePro/Core/Storage/ConnectionStorage.swift @@ -789,6 +789,7 @@ final class ConnectionStorage { "cloudflaretokenid", "cloudflaretokensecret", "cloudsqlproxyserviceaccountkey", "socksproxypassword" ] private static let localOnlySecretsMigratedKey = "com.TablePro.localOnlySecretsKeptOnDevice" + private static let pendingSyncedCopyRemovalsKey = "com.TablePro.localOnlySyncedCopiesPendingRemoval" private func secretKeys(for connection: DatabaseConnection) -> [String] { let id = connection.id.uuidString @@ -854,16 +855,27 @@ final class ConnectionStorage { staging?.edits.deletedKeys.append(key) } - // Rewrites each stored secret into the scope the connection now has. Going device-only keeps the copy iCloud - // Keychain holds: removing it would remove it from every device. False when a secret could not be read or written. + // Rewrites each stored secret into the scope the connection now has. removingSyncedCopy then removes the iCloud + // Keychain copy, from every device, but only of a secret this Mac now holds itself. False when a secret could not + // be read or written. @discardableResult - private func rewriteSecrets(of connection: DatabaseConnection, readingOwnItemFirst: Bool, deviceOnly: Bool) -> Bool { + private func rewriteSecrets( + of connection: DatabaseConnection, + readingOwnItemFirst: Bool, + deviceOnly: Bool, + removingSyncedCopy: Bool = false + ) -> Bool { var complete = true for key in secretKeys(for: connection) { switch keychain.readStringResult(forKey: key, deviceOnly: readingOwnItemFirst) { case .found(let value): - if !keychain.writeString(value, forKey: key, deviceOnly: deviceOnly) { + guard keychain.writeString(value, forKey: key, deviceOnly: deviceOnly) else { complete = false + continue + } + if removingSyncedCopy, !keychain.removeSyncedCopy(forKey: key) { + Self.logger.warning("A Local only secret kept its iCloud Keychain copy: \(key, privacy: .public)") + notePendingSyncedCopyRemoval(key, connectionId: connection.id) } case .notFound: break @@ -874,7 +886,8 @@ final class ConnectionStorage { return complete } - // Each direction reads the value the connection was using, so the toggle never changes a secret. + // Each direction reads the value the connection was using, so the toggle never changes a secret. Becoming Local + // only also takes the secrets off iCloud Keychain, so the other devices ask for them again. private func moveSecretsWhoseScopeChanged(from previous: [DatabaseConnection], to current: [DatabaseConnection]) { let wasSynced = Dictionary( previous.map { ($0.id, $0.participatesInSync) }, @@ -882,12 +895,42 @@ final class ConnectionStorage { ) for connection in current { guard let synced = wasSynced[connection.id], synced != connection.participatesInSync else { continue } - rewriteSecrets(of: connection, readingOwnItemFirst: !synced, deviceOnly: synced) + rewriteSecrets(of: connection, readingOwnItemFirst: !synced, deviceOnly: synced, removingSyncedCopy: synced) + } + } + + private func notePendingSyncedCopyRemoval(_ key: String, connectionId: UUID) { + var pending = defaults.dictionary(forKey: Self.pendingSyncedCopyRemovalsKey) as? [String: String] ?? [:] + pending[key] = connectionId.uuidString + defaults.set(pending, forKey: Self.pendingSyncedCopyRemovalsKey) + } + + // A removal the user was promised and the keychain refused is tried again at launch, while the connection is + // still Local only. One that is back in sync, or gone, wants its synced copy kept or already removed it. + func removePendingSyncedCopies() { + guard let pending = defaults.dictionary(forKey: Self.pendingSyncedCopyRemovalsKey) as? [String: String], + !pending.isEmpty else { return } + let connections = loadConnections() + guard !lastLoadFailed else { return } + let byId = Dictionary(connections.map { ($0.id.uuidString, $0) }, uniquingKeysWith: { first, _ in first }) + var remaining: [String: String] = [:] + for (key, idString) in pending { + guard let connection = byId[idString], !connection.participatesInSync else { continue } + if !keychain.removeSyncedCopy(forKey: key) { + remaining[key] = idString + } + } + if remaining.isEmpty { + defaults.removeObject(forKey: Self.pendingSyncedCopyRemovalsKey) + } else { + defaults.set(remaining, forKey: Self.pendingSyncedCopyRemovalsKey) } } - // Builds before this one wrote a Local only connection's secrets wherever the password sync setting pointed. + // Builds before this one wrote a Local only connection's secrets wherever the password sync setting pointed. Only + // copied: the user saw no note saying the other devices lose them. func keepLocalOnlySecretsOnDeviceIfNeeded() { + removePendingSyncedCopies() guard !defaults.bool(forKey: Self.localOnlySecretsMigratedKey) else { return } let connections = loadConnections() guard !lastLoadFailed else { return } diff --git a/apple/TablePro/Core/Storage/KeychainHelper.swift b/apple/TablePro/Core/Storage/KeychainHelper.swift index de76462df9..f62a632250 100644 --- a/apple/TablePro/Core/Storage/KeychainHelper.swift +++ b/apple/TablePro/Core/Storage/KeychainHelper.swift @@ -37,6 +37,9 @@ protocol KeychainStoring: Sendable { func delete(forKey key: String) // deviceOnly removes this Mac's own item and leaves the one iCloud Keychain holds, which every device shares. func delete(forKey key: String, deviceOnly: Bool) + // Removes the iCloud Keychain copy, which removes it from every device. Refused while this Mac has no item of its own. + @discardableResult + func removeSyncedCopy(forKey key: String) -> Bool } extension KeychainStoring { @@ -52,6 +55,12 @@ extension KeychainStoring { func writeString(_ value: String, forKey key: String, deviceOnly: Bool) -> Bool { writeString(value, forKey: key) } + + // A store with one item per key has no synced copy apart from the one it holds. + @discardableResult + func removeSyncedCopy(forKey key: String) -> Bool { + true + } } struct KeychainCopiedItem: Sendable { @@ -251,6 +260,21 @@ final class KeychainHelper: KeychainStoring { } } + @discardableResult + func removeSyncedCopy(forKey key: String) -> Bool { + let own = items.copyItem(itemQuery(forKey: key, synchronizable: false)) + guard own.status == errSecSuccess, own.data != nil else { + log(status: own.status, operation: "find the local copy before removing the synced one", key: key) + return false + } + let status = items.delete(itemQuery(forKey: key, synchronizable: true)) + guard status == errSecSuccess || status == errSecItemNotFound else { + log(status: status, operation: "remove the synced copy", key: key) + return false + } + return true + } + // MARK: - String Convenience @discardableResult diff --git a/apple/TablePro/Models/Query/GridDisplayOrderResolver.swift b/apple/TablePro/Models/Query/GridDisplayOrderResolver.swift index e87b1d2e84..25d3fe0501 100644 --- a/apple/TablePro/Models/Query/GridDisplayOrderResolver.swift +++ b/apple/TablePro/Models/Query/GridDisplayOrderResolver.swift @@ -6,15 +6,12 @@ import Foundation import TableProPluginKit -/// Turns a per-column value filter into the order the grid displays. -/// -/// Pure, so both the grid and the coordinator that has to answer while the grid is unmounted can -/// call it and cannot disagree. Matching is done on the displayed string rather than the raw value, -/// because that is what the filter popover offered the user to pick from. +// Turns a per-column value filter into the order the grid displays. Pure, so the grid and the +// coordinator that answers while the grid is unmounted cannot disagree. Matching is on the displayed +// string rather than the raw value, because that is what the filter popover offered. @MainActor internal enum GridDisplayOrderResolver { - /// - Returns: the displayed row ids, or nil when no filter is active and the display order is - /// the storage order. + // Nil when no filter is active and the display order is the storage order. internal static func resolve( tableRows: TableRows, valueFilter: GridValueFilterState, @@ -38,27 +35,42 @@ internal enum GridDisplayOrderResolver { return result } - /// The displayed form of a cell, which is what a value filter stores and matches against. + // The displayed form of a cell, which is what a value filter stores and matches against. internal static func displayValue( _ rawValue: PluginCellValue, columnType: ColumnType?, displayFormat: ValueDisplayFormat?, databaseType: DatabaseType? + ) -> String { + displayValue( + rawValue, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: DateFormattingService.shared.renderer + ) + } + + nonisolated internal static func displayValue( + _ rawValue: PluginCellValue, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dates: DateDisplayRenderer ) -> String { CellDisplayFormatter.format( rawValue, columnType: columnType, displayFormat: displayFormat, - databaseType: databaseType + databaseType: databaseType, + dates: dates ) ?? rawValue.asText ?? "" } - /// Drops filters whose column has moved or disappeared since they were set. - /// - /// A filter is keyed by column index and remembers the name it was set on, so a result set of a - /// different shape would otherwise apply one column's chosen values to another column's cells. - /// Reading it out rather than mutating keeps this callable from a getter; the mutating - /// `GridValueFilterState.prune` still tidies the stored state at the points that replace rows. + // Drops filters whose column has moved or disappeared since they were set. A filter is keyed by + // column index and remembers the name it was set on, so a result of a different shape would apply + // one column's chosen values to another's cells. Reading it out rather than mutating keeps this + // callable from a getter; `GridValueFilterState.prune` tidies the stored state where rows are replaced. private static func activeFilters( _ valueFilter: GridValueFilterState, columns: [String] diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index ebbd35aef7..cfd857cba9 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -4339,6 +4339,9 @@ }, "%@ May Still Be Running" : { + }, + "%@ more messages." : { + }, "%@ more, in the script beside this" : { "localizations" : { @@ -5419,6 +5422,9 @@ }, "%@ statements" : { + }, + "%@ Statements" : { + }, "%@ stops being a computed column" : { "localizations" : { @@ -13583,6 +13589,9 @@ } } } + }, + "1 more message." : { + }, "1 new column" : { @@ -54543,6 +54552,10 @@ }, "Could not read the table." : { + }, + "Could not read this Mac's trusted certificates to verify the server. Choose the server's CA certificate under SSL, then connect again." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Could not read which rows are already in the table" : { @@ -57647,6 +57660,9 @@ } } } + }, + "cursors" : { + }, "CURTIME()" : { "extractionState" : "stale", @@ -75718,6 +75734,9 @@ } } } + }, + "Exclude from iCloud Sync?" : { + }, "Exclude the AUTO_INCREMENT counter" : { "extractionState" : "stale", @@ -105371,6 +105390,9 @@ }, "Listen for Notifications" : { + }, + "LISTEN registrations" : { + }, "Listens on all interfaces (0.0.0.0), reachable from your local network." : { "localizations" : { @@ -137000,6 +137022,9 @@ } } } + }, + "prepared statements" : { + }, "Preparing Import…" : { "localizations" : { @@ -142962,6 +142987,9 @@ } } } + }, + "Ran %@ statements that returned no rows." : { + }, "ran %@Ɨ" : { "comment" : "Run count in a detail line, %@ is a count", @@ -158199,6 +158227,9 @@ } } } + }, + "Saved passwords are removed from iCloud Keychain, and your other devices will need them entered again." : { + }, "Saved passwords can only be exported to an encrypted file." : { "extractionState" : "manual", @@ -158395,6 +158426,9 @@ } } } + }, + "Saving removes its passwords from iCloud Keychain. Your other devices will need them entered again." : { + }, "Saving your choice" : { @@ -164391,6 +164425,12 @@ } } } + }, + "session locks" : { + + }, + "session settings" : { + }, "Session SID %1$d, SERIAL# %2$d. %3$@ A user with ALTER SYSTEM can end it now with %4$@." : { "extractionState" : "manual", @@ -168042,6 +168082,9 @@ } } } + }, + "Showing the last %1$@ of %2$@ results that returned rows." : { + }, "Shown beside each key" : { "localizations" : { @@ -183835,6 +183878,9 @@ }, "Temporary Table" : { + }, + "temporary tables" : { + }, "Teradata uses TLS on its HTTPS port, which the tunnel does not forward. Set SSL Mode to Disabled or Preferred to connect through the tunnel, or turn the tunnel off." : { "extractionState" : "manual", @@ -187320,6 +187366,12 @@ } } } + }, + "The connection to %1$@ was lost while it waited and has been reconnected. The new session does not have what the old one set up: %2$@. Set it up again, then run the statements." : { + + }, + "The connection to %1$@ was lost while it waited, and the server rolled back its open transaction. The new session does not have what the old one set up either: %2$@. Set it up again, then run the statements." : { + }, "The connection to Redis dropped after EXEC was sent, so whether the block ran is unknown." : { "extractionState" : "manual", @@ -189511,6 +189563,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The JSON file has text that is not valid in its encoding." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The JSON file is not valid at byte %@." : { "extractionState" : "manual", "generatesSymbol" : false @@ -192074,6 +192130,12 @@ } } } + }, + "The response stopped at the model's output token limit." : { + + }, + "The response stopped because the conversation filled the model's context window." : { + }, "The REST endpoint is not a valid URL." : { "extractionState" : "manual", @@ -211906,6 +211968,9 @@ } } } + }, + "user variables" : { + }, "User-Defined" : { "localizations" : { @@ -213698,6 +213763,10 @@ } } }, + "Verify CA needs the server's CA certificate. Choose it under SSL, or use Verify Identity to check the server against this Mac's trusted certificates." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Verify certificate and hostname" : { "localizations" : { "fr" : { diff --git a/apple/TablePro/ViewModels/AIChatViewModel+MessageEditing.swift b/apple/TablePro/ViewModels/AIChatViewModel+MessageEditing.swift index 414d5a6e4c..0eac80c09b 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel+MessageEditing.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel+MessageEditing.swift @@ -51,7 +51,8 @@ extension AIChatViewModel { timestamp: snapshot.timestamp, usage: snapshot.usage, modelId: snapshot.modelId, - providerId: snapshot.providerId + providerId: snapshot.providerId, + outputLimit: snapshot.outputLimit ) } diff --git a/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift b/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift index 9bbc597baa..29ca4d0a62 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift @@ -23,6 +23,7 @@ extension AIChatViewModel { let toolUseNames: [String: String] let toolUseInputs: [String: String] let toolUseMetadata: [String: [String: String]] + let outputLimit: ChatOutputLimit? let cancelled: Bool } @@ -221,7 +222,8 @@ extension AIChatViewModel { order: round.toolUseOrder, names: round.toolUseNames, inputs: round.toolUseInputs, - metadata: round.toolUseMetadata + metadata: round.toolUseMetadata, + outputLimit: round.outputLimit ) let context = await MainActor.run { ChatToolContext( @@ -466,6 +468,7 @@ extension AIChatViewModel { var toolUseNames: [String: String] = [:] var toolUseInputs: [String: String] = [:] var toolUseMetadata: [String: [String: String]] = [:] + var outputLimit: ChatOutputLimit? var hasRenderedFirstText = false for try await event in stream { @@ -515,6 +518,9 @@ extension AIChatViewModel { opaque: opaque, idMap: &reasoningIDMap.value ) + case .outputLimitReached(let limit): + outputLimit = limit + turn(withID: assistantID)?.outputLimit = limit } } @@ -523,6 +529,7 @@ extension AIChatViewModel { toolUseNames: toolUseNames, toolUseInputs: toolUseInputs, toolUseMetadata: toolUseMetadata, + outputLimit: outputLimit, cancelled: Task.isCancelled ) } @@ -668,21 +675,29 @@ extension AIChatViewModel { response hit the output token limit. Call the tool again with complete arguments. """ + nonisolated static let cutToolArgumentsContent = """ + This tool call did not run: the response hit the output limit before its arguments were complete. \ + Call the tool again with complete arguments, and keep them shorter if you can. + """ + // A call whose arguments do not parse comes back already denied, so it never reaches approval or execution. + // A response cut at its limit was cut in its last call, whatever arguments a provider that parses them hands over. nonisolated static func assembleToolUseBlocks( order: [String], names: [String: String], inputs: [String: String], - metadata: [String: [String: String]] = [:] + metadata: [String: [String: String]] = [:], + outputLimit: ChatOutputLimit? = nil ) -> [ToolUseBlock] { order.compactMap { id -> ToolUseBlock? in guard let name = names[id] else { return nil } - guard let arguments = toolArguments(from: inputs[id] ?? "") else { + let isCut = outputLimit != nil && id == order.last + guard !isCut, let arguments = toolArguments(from: inputs[id] ?? "") else { return ToolUseBlock( id: id, name: name, input: .object([:]), - approvalState: .denied(reason: unparsableToolArgumentsContent), + approvalState: .denied(reason: isCut ? cutToolArgumentsContent : unparsableToolArgumentsContent), providerMetadata: metadata[id] ) } diff --git a/apple/TablePro/ViewModels/WelcomeViewModel+Library.swift b/apple/TablePro/ViewModels/WelcomeViewModel+Library.swift index 9ac471d344..676d4aa587 100644 --- a/apple/TablePro/ViewModels/WelcomeViewModel+Library.swift +++ b/apple/TablePro/ViewModels/WelcomeViewModel+Library.swift @@ -282,6 +282,21 @@ extension WelcomeViewModel { loadConnections() } + // Leaving sync takes the secrets off iCloud Keychain on every device, so it asks first. + func requestExcludeFromSync(_ ids: [UUID]) { + let targets = ids.compactMap { connectionsById[$0] }.filter(\.participatesInSync) + guard !targets.isEmpty else { return } + connectionsToExcludeFromSync = targets + showExcludeFromSyncConfirmation = true + } + + func confirmExcludeFromSync() { + let ids = connectionsToExcludeFromSync.map(\.id) + connectionsToExcludeFromSync = [] + guard !ids.isEmpty else { return } + setIncludedInSync(ids, included: false) + } + func duplicateConnection(_ id: UUID) { guard let connection = connectionsById[id] else { return } guard let duplicate = storage.duplicateConnection(connection) else { @@ -460,7 +475,11 @@ extension WelcomeViewModel { case .moveToNewGroup(let ids): requestNewGroup(parentId: nil, movingConnectionIds: ids) case .setIncludedInSync(let ids, let included): - setIncludedInSync(ids, included: included) + if included { + setIncludedInSync(ids, included: true) + } else { + requestExcludeFromSync(ids) + } case .deleteConnections(let ids): requestDeleteConnections(ids) case .removeFromRecent(let ids): diff --git a/apple/TablePro/ViewModels/WelcomeViewModel.swift b/apple/TablePro/ViewModels/WelcomeViewModel.swift index ebaefbff7e..1153d64200 100644 --- a/apple/TablePro/ViewModels/WelcomeViewModel.swift +++ b/apple/TablePro/ViewModels/WelcomeViewModel.swift @@ -119,6 +119,8 @@ final class WelcomeViewModel: ObservableObject { @Published var presentsWelcomeSheet = false @Published var connectionsToDelete: [DatabaseConnection] = [] @Published var showDeleteConfirmation = false + @Published var connectionsToExcludeFromSync: [DatabaseConnection] = [] + @Published var showExcludeFromSyncConfirmation = false @Published var pendingDeleteHasFavorites = false private var deleteRequestToken = UUID() @Published var showDeleteGroupConfirmation = false diff --git a/apple/TablePro/Views/AIChat/AIChatMessageView.swift b/apple/TablePro/Views/AIChat/AIChatMessageView.swift index 82960082ce..6622c98621 100644 --- a/apple/TablePro/Views/AIChat/AIChatMessageView.swift +++ b/apple/TablePro/Views/AIChat/AIChatMessageView.swift @@ -105,6 +105,13 @@ struct AIChatMessageView: View, Equatable { } } .padding(.horizontal, 8) + + if let limit = message.outputLimit { + Label(limit.note, systemImage: "exclamationmark.triangle") + .font(.caption) + .foregroundStyle(.secondary) + .padding(.horizontal, 8) + } } if let onRetry { @@ -245,3 +252,14 @@ struct ChatTypingIndicatorView: View { .accessibilityLabel(String(localized: "Responding")) } } + +private extension ChatOutputLimit { + var note: String { + switch self { + case .outputTokens: + return String(localized: "The response stopped at the model's output token limit.") + case .contextWindow: + return String(localized: "The response stopped because the conversation filled the model's context window.") + } + } +} diff --git a/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift b/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift index 3be561720c..963a6e019b 100644 --- a/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift +++ b/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift @@ -57,6 +57,11 @@ final class ConnectionFormCoordinator: ObservableObject { @Published var pluginInstallConnection: DatabaseConnection? @Published var pluginDiagnostic: PluginDiagnosticItem? + // A connection that syncs today and is saved Local only has its secrets taken off iCloud Keychain. + var removesSyncedSecretsOnSave: Bool { + advanced.localOnly && originalConnection?.participatesInSync == true + } + @Published var saveError: String? @Published var isChoosingType: Bool = false diff --git a/apple/TablePro/Views/ConnectionForm/Panes/OptionsPaneView.swift b/apple/TablePro/Views/ConnectionForm/Panes/OptionsPaneView.swift index 84c5296869..c188a6dffb 100644 --- a/apple/TablePro/Views/ConnectionForm/Panes/OptionsPaneView.swift +++ b/apple/TablePro/Views/ConnectionForm/Panes/OptionsPaneView.swift @@ -441,9 +441,14 @@ struct OptionsPaneView: View { } header: { Text(String(localized: "iCloud Sync")) } footer: { - Text("This connection won't sync to other devices via iCloud.") - .font(.caption) - .foregroundStyle(.secondary) + VStack(alignment: .leading, spacing: 4) { + Text("This connection won't sync to other devices via iCloud.") + if coordinator.removesSyncedSecretsOnSave { + Text("Saving removes its passwords from iCloud Keychain. Your other devices will need them entered again.") + } + } + .font(.caption) + .foregroundStyle(.secondary) } } } diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift index e4a9b39fb1..e56a98cd32 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridViewController+ValueFilter.swift @@ -9,10 +9,69 @@ import TableProPluginKit // Each loaded value as the cell shows it, counted, in the order first seen. struct ColumnValueTally: Sendable { + static let cancellationInterval = 4_096 + var counts: [String: Int] = [:] var order: [String] = [] var nullCount = 0 + // Nil once cancelled, which is asked every 4,096 rows. + static func tally( + _ rows: TableRows, + dataIndex: Int, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dates: DateDisplayRenderer, + isCancelled: () -> Bool + ) -> ColumnValueTally? { + var counted = ColumnValueTally() + guard dataIndex >= 0, dataIndex < rows.columns.count else { return counted } + for (offset, row) in rows.rows.enumerated() { + if offset % cancellationInterval == 0, isCancelled() { return nil } + guard dataIndex < row.values.count else { continue } + let raw = row.values[dataIndex] + if case .null = raw { + counted.nullCount += 1 + continue + } + let display = GridDisplayOrderResolver.displayValue( + raw, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: dates + ) + if counted.counts[display] == nil { + counted.order.append(display) + } + counted.counts[display, default: 0] += 1 + } + return counted + } + + // A column of unique values costs a format per row and n log n locale-aware comparisons, too slow for the + // main actor. A renderer is never shared across threads, so this one builds its own from the format. + @concurrent + static func distinctValuesOffTheMainActor( + _ rows: TableRows, + dataIndex: Int, + columnType: ColumnType?, + displayFormat: ValueDisplayFormat?, + databaseType: DatabaseType?, + dateFormat: DateFormatOption + ) async -> [ColumnDistinctValue]? { + tally( + rows, + dataIndex: dataIndex, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dates: DateDisplayRenderer(format: dateFormat), + isCancelled: { Task.isCancelled } + )?.sorted() + } + func sorted() -> [ColumnDistinctValue] { var values = order .sorted { $0.localizedStandardCompare($1) == .orderedAscending } @@ -22,47 +81,20 @@ struct ColumnValueTally: Sendable { } return values } - - // A column of unique values makes n log n locale-aware comparisons, too slow for the main actor. - @concurrent - func sortedOffTheMainActor() async -> [ColumnDistinctValue] { - sorted() - } } extension DataGridCommands { // Every loaded value as the cell shows it, which is what a value filter stores and matches; NULL first. func distinctValues(for column: DataGridColumn) -> [ColumnDistinctValue] { - valueTally(for: column).sorted() - } - - // Formatting reads main-actor state, so the tally is taken here and only the sort leaves the main actor. - func valueTally(for column: DataGridColumn) -> ColumnValueTally { - let rows = model.rows - let dataIndex = column.dataIndex - guard dataIndex >= 0, dataIndex < rows.columns.count else { return ColumnValueTally() } - let format = model.displayFormat(for: column) - let databaseType = configuration.traits.databaseType - var tally = ColumnValueTally() - for row in rows.rows { - guard dataIndex < row.values.count else { continue } - let raw = row.values[dataIndex] - if case .null = raw { - tally.nullCount += 1 - continue - } - let display = GridDisplayOrderResolver.displayValue( - raw, - columnType: column.type, - displayFormat: format, - databaseType: databaseType - ) - if tally.counts[display] == nil { - tally.order.append(display) - } - tally.counts[display, default: 0] += 1 - } - return tally + ColumnValueTally.tally( + model.rows, + dataIndex: column.dataIndex, + columnType: column.type, + displayFormat: model.displayFormat(for: column), + databaseType: configuration.traits.databaseType, + dates: DateFormattingService.shared.renderer, + isCancelled: { false } + )?.sorted() ?? [] } } @@ -80,15 +112,25 @@ extension DataGridViewController { }) else { return } guard model.anchor(for: column) != nil else { return } - let tally = gridCommands.valueTally(for: column) + let columnType = column.type + let displayFormat = model.displayFormat(for: column) + let databaseType = model.configuration.traits.databaseType + let dateFormat = DateFormattingService.shared.currentFormat let initialFilter = model.store.valueFilter.filter(forColumn: dataIndex) let loadedRowCount = rows.count let listedModel = model dismissValueFilterPopover() - // A reload, insert or removal while the list sorts cancels it, as it would close the open popover. + // A reload, insert or removal while the list builds cancels it, as it would close the open popover. menuState.valueFilterListing = Task { [weak self] in - let values = await tally.sortedOffTheMainActor() - guard !Task.isCancelled, let self else { return } + let listed = await ColumnValueTally.distinctValuesOffTheMainActor( + rows, + dataIndex: dataIndex, + columnType: columnType, + displayFormat: displayFormat, + databaseType: databaseType, + dateFormat: dateFormat + ) + guard let values = listed, !Task.isCancelled, let self else { return } self.menuState.valueFilterListing = nil // A grid that took another tab's model meanwhile shows other rows. guard self.model === listedModel, diff --git a/apple/TablePro/Views/DataGrid/DataGridFindBar.swift b/apple/TablePro/Views/DataGrid/DataGridFindBar.swift index 2879242aea..2e4752a173 100644 --- a/apple/TablePro/Views/DataGrid/DataGridFindBar.swift +++ b/apple/TablePro/Views/DataGrid/DataGridFindBar.swift @@ -172,11 +172,21 @@ struct DataGridFindBar: View { // MARK: - State private var counterText: String { + Self.counterText(status: status, model: model) + } + + private var showsSearchAllRows: Bool { + Self.showsSearchAllRows(offered: capabilities.offersSearchAllRows, model: model) + } + + static func counterText(status: String?, model: DataGridModel) -> String { + let find = model.find if let status { return status } if let problem = find.problem { return problem } guard model.hasFindTerm else { return "" } // A search that refreshes keeps showing the count it is about to replace. if find.matches.isEmpty { + if find.isAwaitingLoadedRows { return "" } if find.isSearching { return String(localized: "Searching…") } if find.searchStopped { return String(localized: "Search stopped") } } @@ -188,9 +198,10 @@ struct DataGridFindBar: View { } // Only when this page has no match, so the bar stays short while matches are on screen. - private var showsSearchAllRows: Bool { - capabilities.offersSearchAllRows && model.searchAllRowsKeepsTheOptions && model.hasFindTerm - && find.problem == nil && find.matches.isEmpty + static func showsSearchAllRows(offered: Bool, model: DataGridModel) -> Bool { + let find = model.find + return offered && model.searchAllRowsKeepsTheOptions && model.hasFindTerm + && find.problem == nil && find.matches.isEmpty && !find.isAwaitingLoadedRows } // Use Selection for Find searches on its own, so only typing goes through the delay. diff --git a/apple/TablePro/Views/Main/MainContentCoordinator.swift b/apple/TablePro/Views/Main/MainContentCoordinator.swift index 2c2c39815a..d603b2f471 100644 --- a/apple/TablePro/Views/Main/MainContentCoordinator.swift +++ b/apple/TablePro/Views/Main/MainContentCoordinator.swift @@ -1292,7 +1292,9 @@ final class MainContentCoordinator: ObservableObject { scope: scope, isTableTab: isTableTab, lease: lease - ) { [queryExecutor] driver in + ) { [queryExecutor, manager = services.databaseManager] driver in + // Before it runs, so a statement that fails or a connection lost under it is still noted. + if !isAutoLoad { await manager.noteSessionStatements([statement.sql], ranOn: driver, scope: scope) } let fetched = try await queryExecutor.executeQuery( driver: driver, sql: statement.sql, diff --git a/apple/TablePro/Views/Welcome/WelcomePresentations.swift b/apple/TablePro/Views/Welcome/WelcomePresentations.swift index 4a1379516b..3913fea692 100644 --- a/apple/TablePro/Views/Welcome/WelcomePresentations.swift +++ b/apple/TablePro/Views/Welcome/WelcomePresentations.swift @@ -137,6 +137,19 @@ private struct WelcomeDeletionAlerts: ViewModifier { } message: { deletionMessage } + .alert( + String(localized: "Exclude from iCloud Sync?"), + isPresented: $vm.showExcludeFromSyncConfirmation + ) { + Button(String(localized: "Exclude"), role: .destructive) { + vm.confirmExcludeFromSync() + } + Button(String(localized: "Cancel"), role: .cancel) { + vm.connectionsToExcludeFromSync = [] + } + } message: { + Text("Saved passwords are removed from iCloud Keychain, and your other devices will need them entered again.") + } } @ViewBuilder diff --git a/apple/TableProMobile/TableProMobile/Localizable.xcstrings b/apple/TableProMobile/TableProMobile/Localizable.xcstrings index 6c3e6f49ca..5d5d58a6bf 100644 --- a/apple/TableProMobile/TableProMobile/Localizable.xcstrings +++ b/apple/TableProMobile/TableProMobile/Localizable.xcstrings @@ -7217,6 +7217,10 @@ } } }, + "Could not read this Mac's trusted certificates to verify the server. Choose the server's CA certificate under SSL, then connect again." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Could not run the credential_process command for profile \"%@\": %@" : { "extractionState" : "manual", "generatesSymbol" : false, diff --git a/apple/TableProTests/Core/AI/OutputLimitSignalTests.swift b/apple/TableProTests/Core/AI/OutputLimitSignalTests.swift new file mode 100644 index 0000000000..cf91ae70ea --- /dev/null +++ b/apple/TableProTests/Core/AI/OutputLimitSignalTests.swift @@ -0,0 +1,177 @@ +// +// OutputLimitSignalTests.swift +// TableProTests +// + +import Foundation +import TableProMCP +@testable import TablePro +import Testing + +private func limits(_ events: [ChatStreamEvent]) -> [ChatOutputLimit] { + events.compactMap { event in + if case .outputLimitReached(let limit) = event { return limit } + return nil + } +} + +struct OutputLimitSignalTests { + @Test("Anthropic message_delta stop reasons that cut the reply are reported", arguments: [ + ("max_tokens", [ChatOutputLimit.outputTokens]), + ("model_context_window_exceeded", [.contextWindow]), + ("end_turn", []), + ("tool_use", []), + ] as [(String, [ChatOutputLimit])]) + func anthropicStopReason(reason: String, expected: [ChatOutputLimit]) throws { + var state = AnthropicStreamState() + let events = try AnthropicProvider.parseChunk([ + "type": "message_delta", + "delta": ["stop_reason": reason], + "usage": ["output_tokens": 12], + ], state: &state) + #expect(limits(events) == expected) + #expect(state.outputTokens == 12) + } + + @Test("Chat Completions finish_reason length is reported, stop is not", arguments: [ + ("length", [ChatOutputLimit.outputTokens]), + ("stop", []), + ("tool_calls", []), + ] as [(String, [ChatOutputLimit])]) + func chatCompletionsFinishReason(reason: String, expected: [ChatOutputLimit]) { + var state = OpenAIStreamState() + let result = OpenAICompatibleProvider.parseChunk([ + "choices": [["delta": [:] as [String: Any], "finish_reason": reason]], + ], state: &state) + #expect(limits(result.events) == expected) + } + + @Test("Ollama's done_reason length is reported") + func ollamaDoneReason() { + var state = OpenAIStreamState() + let result = OpenAICompatibleProvider.parseChunk([ + "message": ["content": ""], + "done": true, + "done_reason": "length", + ], state: &state) + #expect(limits(result.events) == [.outputTokens]) + } + + @Test("A Responses stream that ends incomplete at max_output_tokens is reported, with its usage") + func responsesIncomplete() throws { + var state = ResponsesStreamState() + let events = try OpenAIResponsesProvider.parseEvent([ + "type": "response.incomplete", + "response": [ + "status": "incomplete", + "incomplete_details": ["reason": "max_output_tokens"], + "usage": ["input_tokens": 30, "output_tokens": 64], + ], + ], state: &state) + #expect(limits(events) == [.outputTokens]) + #expect(state.inputTokens == 30) + #expect(state.outputTokens == 64) + } + + @Test("A Responses stream cut by its content filter is not an output limit") + func responsesContentFilter() throws { + var state = ResponsesStreamState() + let events = try OpenAIResponsesProvider.parseEvent([ + "type": "response.incomplete", + "response": ["incomplete_details": ["reason": "content_filter"]], + ], state: &state) + #expect(limits(events).isEmpty) + } + + @Test("Gemini finishReason MAX_TOKENS is reported, STOP is not", arguments: [ + ("MAX_TOKENS", [ChatOutputLimit.outputTokens]), + ("STOP", []), + ] as [(String, [ChatOutputLimit])]) + func geminiFinishReason(reason: String, expected: [ChatOutputLimit]) { + var state = GeminiStreamState() + let events = GeminiProvider.parseChunk([ + "candidates": [["content": ["parts": [["text": "partial"]]], "finishReason": reason]], + ], state: &state, idGenerator: { "id" }) + #expect(limits(events) == expected) + } + + @Test("Claude Code's passed-through message_delta reports the cut") + func claudeAgentStopReason() { + let line = #"{"type":"stream_event","event":{"type":"message_delta","delta":{"stop_reason":"max_tokens"}}}"# + #expect(ClaudeAgentProvider.decodeEvent(line) == .outputLimit(.outputTokens)) + let ended = #"{"type":"stream_event","event":{"type":"message_delta","delta":{"stop_reason":"end_turn"}}}"# + #expect(ClaudeAgentProvider.decodeEvent(ended) == nil) + } + + @Test("At the output limit the last call is denied even with no arguments yet, and complete calls before it stand") + func cutLastCallIsDenied() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1", "call_2"], + names: ["call_1": "list_tables", "call_2": "execute_query"], + inputs: ["call_1": #"{"schema":"public"}"#, "call_2": ""], + outputLimit: .outputTokens + ) + #expect(blocks.count == 2) + #expect(blocks[0].approvalState == .approved) + #expect(blocks[0].input == .object(["schema": .string("public")])) + #expect(blocks[1].approvalState == .denied(reason: AIChatViewModel.cutToolArgumentsContent)) + } + + @Test("At the output limit cut arguments name the limit in the error the model gets") + func cutArgumentsNameTheLimit() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1"], + names: ["call_1": "execute_query"], + inputs: ["call_1": #"{"query":"SELECT * FR"#], + outputLimit: .outputTokens + ) + #expect(blocks.first?.approvalState == .denied(reason: AIChatViewModel.cutToolArgumentsContent)) + } + + @Test("At the output limit the last call is denied even when its arguments parse, as Gemini and Ollama hand them over") + func cutLastCallIsDeniedWithParsedArguments() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1"], + names: ["call_1": "execute_query"], + inputs: ["call_1": "{}"], + outputLimit: .outputTokens + ) + #expect(blocks.first?.approvalState == .denied(reason: AIChatViewModel.cutToolArgumentsContent)) + } + + @Test("At the output limit a malformed call before the last keeps the plain reason") + func earlierMalformedCallKeepsThePlainReason() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1", "call_2"], + names: ["call_1": "describe_table", "call_2": "list_tables"], + inputs: ["call_1": #"{"table":"#, "call_2": "{}"], + outputLimit: .outputTokens + ) + #expect(blocks[0].approvalState == .denied(reason: AIChatViewModel.unparsableToolArgumentsContent)) + #expect(blocks[1].approvalState == .denied(reason: AIChatViewModel.cutToolArgumentsContent)) + } + + @Test("Without the limit a call with no arguments still runs with none") + func noLimitKeepsEmptyArguments() { + let blocks = AIChatViewModel.assembleToolUseBlocks( + order: ["call_1"], + names: ["call_1": "list_tables"], + inputs: ["call_1": ""] + ) + #expect(blocks.first?.approvalState == .approved) + } + + @Test("A turn keeps its output limit through the saved conversation, and an unknown value reads as none") + func outputLimitPersists() throws { + let wire = ChatTurnWire(role: .assistant, blocks: [.text("cut")], outputLimit: .contextWindow) + let data = try JSONEncoder().encode(wire) + #expect(try JSONDecoder().decode(ChatTurnWire.self, from: data).outputLimit == .contextWindow) + + var object = try #require(try JSONSerialization.jsonObject(with: data) as? [String: Any]) + object["outputLimit"] = "somethingNew" + let future = try JSONSerialization.data(withJSONObject: object) + let decoded = try JSONDecoder().decode(ChatTurnWire.self, from: future) + #expect(decoded.outputLimit == nil) + #expect(decoded.plainText == "cut") + } +} diff --git a/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift b/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift index ad9c23b09f..f7bd3cfbb5 100644 --- a/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift +++ b/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift @@ -387,11 +387,15 @@ final class MockDatabaseDriver: DatabaseDriver, SchemaSwitchable, @unchecked Sen func commitTransaction() async throws { commitTransactionCallCount += 1 } func rollbackTransaction() async throws {} + // A switch to this schema fails, the way a server refuses a schema it does not have. + var switchSchemaError: (schema: String, error: any Error)? + func switchSchema(to schema: String) async throws { await onSwitchSchema?() if switchSchemaDelaySeconds > 0 { try await Task.sleep(nanoseconds: UInt64(switchSchemaDelaySeconds * 1_000_000_000)) } + if let failure = switchSchemaError, failure.schema == schema { throw failure.error } switchSchemaCallCount += 1 currentSchema = schema } diff --git a/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift b/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift index 5abc551188..fff4ca6e57 100644 --- a/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift @@ -249,4 +249,359 @@ struct UpdateSaveOrderTests { #expect(pending.change(forRow: .existing(0), type: .update)?.sequence == sequence) #expect(pending.isCellModified(rowID: .existing(0), columnIndex: 1)) } + + @Test("A row edited again after another row's delete is written after that delete") + func laterEditRunsAfterTheDeleteBeforeIt() throws { + let (manager, _) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + + let statements = try manager.generateSQL() + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE", "UPDATE"]) + try #require(statements.count == 3) + #expect(EditAroundADelete.strings(statements[0]) == ["Cyd", "3"]) + #expect(EditAroundADelete.strings(statements[1]) == ["2"]) + #expect(EditAroundADelete.strings(statements[2]) == ["b@test.com", "3"]) + } + + @Test("A swap through a temporary value saves each step in the order it was made") + func swapThroughATemporaryValueSavesInOrder() throws { + let (manager, _) = EditAroundADelete.manager() + let rowA: [PluginCellValue] = ["1", "Al", "a@test.com"] + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "email", + oldValue: "a@test.com", newValue: "tmp", originalRow: rowA + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 2, columnName: "email", + oldValue: "b@test.com", newValue: "a@test.com", originalRow: EditAroundADelete.rowB + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "email", + oldValue: "tmp", newValue: "b@test.com", originalRow: ["1", "Al", "tmp"] + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "UPDATE", "UPDATE"]) + #expect(bound == [["tmp", "1"], ["a@test.com", "2"], ["b@test.com", "1"]]) + } + + @Test("A later edit of a row keys on the key an earlier edit of it wrote") + func laterRunKeysOnTheKeyTheEarlierRunWrote() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 0, columnName: "id", + oldValue: "3", newValue: "6", originalRow: EditAroundADelete.rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: ["6", "Cy", "c@test.com"] + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE", "UPDATE"]) + #expect(bound == [["6", "3"], ["2"], ["Cyd", "6"]]) + } + + @Test("A tab switch keeps the place of every edit of a row, so a later change sorts after them") + func aTabSwitchKeepsEveryRunsPlace() throws { + let (manager, _) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + let state = manager.saveState() + + manager.restoreState(from: state, tableName: "users", databaseType: .mysql, generatedColumns: []) + manager.recordRowInsertion(rowID: .inserted(UUID()), values: ["9", "Di", "d@test.com"]) + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["UPDATE", "DELETE", "UPDATE", "INSERT"]) + } + + @Test("Undoing the delete of a row edited around another delete brings back both of its edits in place") + func undoingADeleteBringsBackEveryRun() throws { + let (manager, _) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + manager.recordRowDeletion(rowID: .existing(2), originalRow: ["3", "Cyd", "b@test.com"]) + + manager.undoRowDeletion(rowID: .existing(2)) + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["UPDATE", "DELETE", "UPDATE"]) + } + + @Test("Two edits of a row with nothing between them stay one UPDATE") + func editsWithNothingBetweenStayOneUpdate() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: EditAroundADelete.rowC + ) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "c@test.com", newValue: "x@test.com", originalRow: ["3", "Cyd", "c@test.com"] + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE"]) + #expect(bound.first == ["Cyd", "x@test.com", "3"]) + } + + @Test("A cell typed back to its value as read is not written, whatever came between") + func aCellTypedBackToItsValueAsReadIsNotWritten() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "c@test.com", newValue: "x@test.com", originalRow: EditAroundADelete.rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "x@test.com", newValue: "c@test.com", originalRow: ["3", "Cy", "x@test.com"] + ) + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["DELETE"]) + } + + @Test("Edits of a row around a change that was then cancelled are one UPDATE") + func aChangeCancelledBetweenRunsLeavesOneUpdate() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: EditAroundADelete.rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.undoRowDeletion(rowID: .existing(1)) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "c@test.com", newValue: "x@test.com", originalRow: ["3", "Cyd", "c@test.com"] + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE"]) + #expect(bound == [["Cyd", "x@test.com", "3"]]) + } + + @Test("A row whose key an earlier edit set to SQL stays one UPDATE keyed on the key as read") + func aKeySetToSQLBeforeAnotherChangeKeepsTheRowWhole() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 0, columnName: "id", oldValue: "3", newValue: "NOW()", + originalRow: EditAroundADelete.rowC, absence: FieldAbsence(), isExpression: true + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: ["NOW()", "Cy", "c@test.com"] + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE"]) + #expect(statements.first?.sql.contains("= NOW()") == true) + #expect(bound.first == ["Cyd", "3"]) + } + + @Test("A key set to SQL by a row's last edit still saves that edit in its own place") + func aKeySetToSQLInTheLastEditStillSplits() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: EditAroundADelete.rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: EditAroundADelete.rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 0, columnName: "id", oldValue: "3", newValue: "NOW()", + originalRow: ["3", "Cyd", "c@test.com"], absence: FieldAbsence(), isExpression: true + ) + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE", "UPDATE"]) + #expect(bound == [["Cyd", "3"], ["2"], ["3"]]) + } + + @Test("A row of a table with no primary key stays one UPDATE at its first edit's place") + func aKeylessRowStaysOneUpdate() throws { + let (manager, _) = EditAroundADelete.manager(primaryKeyColumns: []) + EditAroundADelete.record(on: manager) + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["UPDATE", "DELETE"]) + } +} + +@MainActor +struct UpdateRunActionTests { + @Test("One command over several rows writes each row once, whatever its other rows' changes") + func oneCommandOverSeveralRowsIsOneUpdatePerRow() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordRowDeletion(rowID: .existing(3), originalRow: ["4", "Di", "d@test.com"]) + manager.recordAsOneAction { + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Al", newValue: "Alf", originalRow: ["1", "Al", "a@test.com"] + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 1, columnName: "name", + oldValue: "Bo", newValue: "Bob", originalRow: EditAroundADelete.rowB + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "email", + oldValue: "a@test.com", newValue: "alf@test.com", originalRow: ["1", "Alf", "a@test.com"] + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 2, columnName: "email", + oldValue: "b@test.com", newValue: "bob@test.com", originalRow: ["2", "Bob", "b@test.com"] + ) + } + manager.recordRowDeletion(rowID: .existing(4), originalRow: ["5", "Ed", "e@test.com"]) + + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["DELETE", "UPDATE", "UPDATE", "DELETE"]) + } + + @Test("Typing into several rows at once writes each row once") + func typingIntoSeveralRowsIsOneUpdatePerRow() throws { + let (manager, _) = EditAroundADelete.manager() + manager.recordTypedCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Al", newValue: "A", originalRow: ["1", "Al", "a@test.com"] + ) + manager.recordTypedCellChange( + rowID: .existing(1), columnIndex: 1, columnName: "name", + oldValue: "Bo", newValue: "A", originalRow: EditAroundADelete.rowB + ) + manager.recordTypedCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "A", newValue: "Ab", originalRow: ["1", "A", "a@test.com"] + ) + manager.endCoalescedUndoRun() + let verbs = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(verbs == ["UPDATE", "UPDATE"]) + } +} + +@MainActor +struct UpdateRunUndoTests { + @Test("Redoing a row's later edit puts it back after the delete it followed") + func redoPutsTheLaterEditBackAfterTheDelete() throws { + let (manager, undoManager) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + + undoManager.undo() + undoManager.redo() + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE", "UPDATE"]) + #expect(bound.last == ["b@test.com", "3"]) + } + + @Test("Undoing a row's later edit leaves its earlier one where it was") + func undoingTheLaterEditLeavesTheEarlierUpdate() throws { + let (manager, undoManager) = EditAroundADelete.manager() + EditAroundADelete.record(on: manager) + + undoManager.undo() + + let statements = try manager.generateSQL() + let bound = statements.map { EditAroundADelete.strings($0) } + #expect(EditAroundADelete.verbs(statements) == ["UPDATE", "DELETE"]) + #expect(bound.first == ["Cyd", "3"]) + } + + @Test("Undoing and redoing a command over two rows still writes each row once") + func undoAndRedoOfACommandKeepOneUpdatePerRow() throws { + let (manager, undoManager) = EditAroundADelete.manager() + manager.recordRowDeletion(rowID: .existing(3), originalRow: ["4", "Di", "d@test.com"]) + undoManager.beginUndoGrouping() + manager.recordAsOneAction { + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Al", newValue: "Alf", originalRow: ["1", "Al", "a@test.com"] + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 1, columnName: "name", + oldValue: "Bo", newValue: "Bob", originalRow: EditAroundADelete.rowB + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "email", + oldValue: "a@test.com", newValue: "alf@test.com", originalRow: ["1", "Alf", "a@test.com"] + ) + manager.recordCellChange( + rowID: .existing(1), columnIndex: 2, columnName: "email", + oldValue: "b@test.com", newValue: "bob@test.com", originalRow: ["2", "Bob", "b@test.com"] + ) + } + undoManager.endUndoGrouping() + + undoManager.undo() + let undone = try EditAroundADelete.verbs(manager.generateSQL()) + undoManager.redo() + let redone = try EditAroundADelete.verbs(manager.generateSQL()) + + #expect(undone == ["DELETE"]) + #expect(redone == ["DELETE", "UPDATE", "UPDATE"]) + } +} + +// Edit C, delete B, then give C the email B freed: each step is valid on a unique `email`. +@MainActor +enum EditAroundADelete { + static let columns = ["id", "name", "email"] + static let rowB: [PluginCellValue] = ["2", "Bo", "b@test.com"] + static let rowC: [PluginCellValue] = ["3", "Cy", "c@test.com"] + + static func manager( + table: String = "users", + primaryKeyColumns: [String] = ["id"], + databaseType: DatabaseType = .mysql + ) -> (DataChangeManager, UndoManager) { + let manager = DataChangeManager() + let undoManager = UndoManager() + undoManager.groupsByEvent = false + manager.undoManagerProvider = { undoManager } + manager.configureForTable( + tableName: table, + columns: columns, + primaryKeyColumns: primaryKeyColumns, + databaseType: databaseType, + generatedColumns: [] + ) + return (manager, undoManager) + } + + static func record(on manager: DataChangeManager) { + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "name", + oldValue: "Cy", newValue: "Cyd", originalRow: rowC + ) + manager.recordRowDeletion(rowID: .existing(1), originalRow: rowB) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 2, columnName: "email", + oldValue: "c@test.com", newValue: "b@test.com", originalRow: ["3", "Cyd", "c@test.com"] + ) + } + + nonisolated static func verbs(_ statements: [ParameterizedStatement]) -> [String] { + statements.map { String($0.sql.prefix(while: { $0 != " " })) } + } + + nonisolated static func strings(_ statement: ParameterizedStatement) -> [String] { + statement.parameters.compactMap { $0 as? String } + } } diff --git a/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift b/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift index 32b9aea657..a965934eae 100644 --- a/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift +++ b/apple/TableProTests/Core/Coordinators/FindMatcherTests.swift @@ -95,12 +95,14 @@ struct FindCoordinatorRoutingTests { } @Test("Find Next and Find Previous step the selected tab's matches") - func stepsTheSelectedTabsMatches() throws { + func stepsTheSelectedTabsMatches() async throws { let (coordinator, first, _) = makeTabs() let model = coordinator.gridModel(forTab: first) coordinator.findCoordinator.showFind() model.find.term = "ali" model.findQueryDidChange() + #expect(coordinator.findCoordinator.hasActiveFind, "Find Next waits for the running search instead of going to the editor") + await model.findSearchSettled() #expect(model.find.matches.count == 2) #expect(coordinator.findCoordinator.hasActiveFind) let start = try #require(model.find.currentIndex) diff --git a/apple/TableProTests/Core/Coordinators/RunResultPlanTests.swift b/apple/TableProTests/Core/Coordinators/RunResultPlanTests.swift new file mode 100644 index 0000000000..89855fefbd --- /dev/null +++ b/apple/TableProTests/Core/Coordinators/RunResultPlanTests.swift @@ -0,0 +1,110 @@ +// +// RunResultPlanTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import TableProTestSupport +@testable import TablePro +import Testing + +struct RunResultPlanTests { + @Test("Statements that return no rows share one summary placed where the last of them ran") + func columnlessStatementsCollapse() { + let plan = RunResultPlan(producesRows: [false, false, true, false, true]) + #expect(plan.entries == [.statement(2), .summary([0, 1, 3]), .statement(4)]) + #expect(plan.droppedNotice == nil) + } + + @Test("A single statement without rows keeps its own result") + func singleColumnlessStatementStays() { + let plan = RunResultPlan(producesRows: [false, true]) + #expect(plan.entries == [.statement(0), .statement(1)]) + } + + @Test("A run that ends on a statement without rows ends on the summary") + func summaryIsLastWhenTheRunEndsWithoutRows() { + let plan = RunResultPlan(producesRows: [true, false, false]) + #expect(plan.entries.last == .summary([1, 2])) + } + + @Test("Only the last results that return rows are kept, and only their rows are built") + func rowResultsAreCapped() { + let plan = RunResultPlan(producesRows: Array(repeating: true, count: 5), maxRowResults: 2) + #expect(plan.entries == [.statement(3), .statement(4)]) + #expect(plan.droppedRowResultCount == 3) + #expect(plan.showsRows(ofStatement: 4)) + #expect(!plan.showsRows(ofStatement: 0)) + #expect(plan.droppedNotice != nil) + } + + @Test("A long script of writes becomes one result") + func longWriteScriptIsOneResult() { + let plan = RunResultPlan(producesRows: Array(repeating: false, count: 10_000)) + #expect(plan.entries.count == 1) + #expect(!plan.showsRows(ofStatement: 0)) + } +} + +@MainActor +struct RunAllSummaryResultTests { + @Test("Run All of statements that return no rows shows one summary result") + func runAllCollapsesWrites() async throws { + let connection = TestFixtures.makeConnection() + let driver = MockDatabaseDriver(connection: connection) + var session = ConnectionSession(connection: connection, driver: driver) + session.status = .connected + session.browseDatabase = connection.database + DatabaseManager.shared.injectSession(session, for: connection.id) + let tabManager = QueryTabManager() + let coordinator = MainContentCoordinator( + connection: connection, + tabManager: tabManager, + changeManager: DataChangeManager(), + toolbarState: ConnectionToolbarState() + ) + defer { + coordinator.cancelAllQueryTasks() + coordinator.teardown() + DatabaseManager.shared.removeSession(for: connection.id) + } + let script = "CREATE TABLE a (x int);\nINSERT INTO a VALUES (1);\nUPDATE a SET x = 2;" + let tab = QueryTab(title: "Query", query: script, tabType: .query) + tabManager.tabs.append(tab) + tabManager.selectedTabId = tab.id + + coordinator.runAllStatements() + try await waitUntil { + !coordinator.tabExecution.isExecuting(tab.id) && tabManager.tabs.first?.display.resultSets.isEmpty == false + } + + let results = try #require(tabManager.tabs.first?.display.resultSets) + #expect(results.count == 1) + let summary = try #require(results.first) + #expect(summary.resultColumns.isEmpty) + #expect(summary.statusMessage?.contains("3") == true) + #expect(summary.statementAnchor == nil) + } + + @Test("The summary adds up rows affected and time, keeps each driver message once and the output in order") + func summaryCarriesWhatTheStatementsReported() { + let fixture = CoordinatorFixture() + let parts = [ + RunSummaryPart(rowsAffected: 2, executionTime: 0.5, serverOutput: PluginServerOutput(lines: ["a"], isTruncated: false), statusMessage: "INSERT: 2 rows"), + RunSummaryPart(rowsAffected: 3, executionTime: 0.25, serverOutput: PluginServerOutput(lines: ["b"], isTruncated: true), statusMessage: "Still creating"), + RunSummaryPart(rowsAffected: 0, executionTime: 0.25, serverOutput: .none, statusMessage: "Still creating"), + ] + + let summary = fixture.coordinator.queryExecutionCoordinator.makeRunSummaryResultSet(parts) + + #expect(summary.rowsAffected == 5) + #expect(summary.executionTime == 1) + #expect(summary.serverOutput == PluginServerOutput(lines: ["a", "b"], isTruncated: true)) + let status = summary.statusMessage ?? "" + #expect(status.contains("INSERT: 2 rows")) + #expect(status.components(separatedBy: "Still creating").count == 2) + #expect(summary.resultColumns.isEmpty) + #expect(summary.origin == nil) + } +} diff --git a/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift b/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift index 35cc1d7d7f..1017472355 100644 --- a/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift +++ b/apple/TableProTests/Core/DataGrid/Engine/GridFindEngineTests.swift @@ -294,6 +294,125 @@ struct GridFindEngineTests { #expect(search("8950", in: source).matches.isEmpty) } + // MARK: - Off the main actor + + private static let shownValues = TableRows( + rows: [ + Row(id: .existing(0), values: [ + .text("2024-03-15 09:30:00.123456"), + .text("2024-12-31 23:59:59+07"), + .text("2024-03-15"), + .bytes(Data("signup".utf8)), + .bytes(Data((0..<16).map { UInt8($0) })), + .text("1700000000"), + .text("1700000000000"), + .text("a\rb") + ]), + Row(id: .existing(1), values: [ + .text("2023-11-14 22:13:20"), + .null, + .text("1999-01-01"), + .bytes(Data([0x89, 0x50, 0x4E, 0x47])), + .bytes(Data(repeating: 0xAB, count: 16)), + .text("1700003600"), + .text("1700003600000"), + .text("plain") + ]) + ], + columns: ["created", "updated", "born", "payload", "token", "stamp", "stampMillis", "note"], + columnTypes: [ + .datetime(rawType: "DATETIME(6)"), + .timestamp(rawType: "TIMESTAMPTZ"), + .date(rawType: "DATE"), + .blob(rawType: "VARBINARY(255)"), + .blob(rawType: "BINARY(16)"), + .integer(rawType: "BIGINT"), + .integer(rawType: "BIGINT"), + .text(rawType: "TEXT") + ] + ) + + private static let shownFormats: [ValueDisplayFormat?] = [nil, nil, nil, .text, .uuid, .unixTimestamp, .unixTimestampMillis, nil] + + private static let shownTerms = [ + "2024", "2023", "09:30:00.123456", "+07", "/", "PM", "signup", "00010203-0405", "abab", "a b", "b", "11" + ] + + // Both sides use the same renderer code, so this pins the same cells on and off the main actor, not the shown text. + @Test("The same search finds the same cells on and off the main actor, under every date format") + func searchOffTheMainActorMatchesTheMainActor() async { + for revealsInvisibleCharacters in [false, true] { + let source = self.source(Self.shownValues, formats: Self.shownFormats, revealsInvisibleCharacters: revealsInvisibleCharacters) + for format in DateFormatOption.allCases { + for term in Self.shownTerms { + let onMain = GridFindEngine.search( + term, + options: GridFindState.Options(), + in: source, + dates: DateDisplayRenderer(format: format), + isCancelled: { false } + ) + let offMain = await GridFindEngine.searchOffTheMainActor( + term, + options: GridFindState.Options(), + in: source, + dateFormat: format + ) + #expect(offMain == onMain, "same cells on and off the main actor: \(format) \(term) reveals: \(revealsInvisibleCharacters)") + } + } + for term in Self.shownTerms { + let shared = await GridFindEngine.searchOffTheMainActor( + term, + options: GridFindState.Options(), + in: source, + dateFormat: DateFormattingService.shared.currentFormat + ) + #expect(shared == search(term, in: source), "\(term)") + } + } + } + + @Test("The off-main search renders dates, bytes and timestamps the way the cells show them") + func searchOffTheMainActorReadsTheShownText() async { + let source = self.source(Self.shownValues, formats: Self.shownFormats) + func column(_ name: String) -> DataGridColumnID { DataGridColumnID(name: name, occurrence: 0) } + func found(_ term: String, _ format: DateFormatOption = .iso8601) async -> [GridFindState.Match]? { + await GridFindEngine.searchOffTheMainActor(term, options: GridFindState.Options(), in: source, dateFormat: format)?.matches + } + + #expect(await found("15/03/2024", .euLong) == [match(0, column("created")), match(0, column("born"))]) + #expect(await found("09:30:00.123456 AM", .usLong) == [match(0, column("created"))]) + #expect(await found("23:59:59+07") == [match(0, column("updated"))]) + #expect(await found("signup") == [match(0, column("payload"))]) + #expect(await found("00010203-0405-0607-0809-0a0b0c0d0e0f") == [match(0, column("token"))]) + #expect(await found("a b") == [match(0, column("note"))]) + #expect(await found("8950")?.isEmpty == true) + } + + @Test("A search asks whether to stop every 4,096 cells and gives up when told to") + func searchChecksForCancellation() { + let rows = TableRows.from( + queryRows: (0..<10_000).map { [PluginCellValue.text("row \($0)")] }, + columns: ["v"], + columnTypes: [.text(rawType: "TEXT")] + ) + let dates = DateDisplayRenderer(format: .iso8601) + var checks = 0 + + let finished = GridFindEngine.search("row", options: GridFindState.Options(), in: source(rows), dates: dates) { + checks += 1 + return false + } + let stopped = GridFindEngine.search( + "row", options: GridFindState.Options(), in: source(rows), dates: dates, isCancelled: { true } + ) + + #expect(finished?.matches.count == 10_000) + #expect(checks == 3) + #expect(stopped == nil) + } + // MARK: - Replacing private static let contacts = TableRows.from( @@ -474,18 +593,19 @@ struct DataGridModelFindTests { GridSelectionReducer.single(GridCoord(row: row, displayColumn: column)) } - private func search(_ model: DataGridModel, _ term: String) { + private func search(_ model: DataGridModel, _ term: String) async { model.showFind() model.find.term = term model.findQueryDidChange() + await model.findSearchSettled() } @Test("A match becomes the active cell, column included") - func matchSetsActiveCell() { + func matchSetsActiveCell() async { let (model, owner) = mount() model.setSelection(cell(2, 0)) - search(model, "acme") + await search(model, "acme") #expect(model.find.matches.count == 4) #expect(model.find.currentIndex == 1) @@ -495,9 +615,9 @@ struct DataGridModelFindTests { } @Test("Find Next walks the matches and wraps") - func findNextWraps() { + func findNextWraps() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") #expect(model.selection.activeCell == GridCoord(row: 1, displayColumn: 2)) model.findNext() @@ -512,9 +632,9 @@ struct DataGridModelFindTests { } @Test("Find Previous starts from where the user moved the cursor") - func findPreviousFromCursor() { + func findPreviousFromCursor() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") model.setSelection(cell(3, 0)) model.findPrevious() @@ -524,14 +644,15 @@ struct DataGridModelFindTests { } @Test("A refresh keeps the current match by identity and leaves the selection alone") - func refreshKeepsCurrentMatch() { + func refreshKeepsCurrentMatch() async { let (model, owner) = mount() model.setSelection(cell(2, 0)) - search(model, "acme") + await search(model, "acme") owner.tableRows.rows[0].values[2] = .text("Acme Town") model.reloadRows() model.refreshFind() + await model.findSearchSettled() #expect(model.find.matches.count == 5) #expect(model.find.currentMatch == GridFindState.Match(rowID: .existing(2), columnID: Self.company)) @@ -540,9 +661,9 @@ struct DataGridModelFindTests { } @Test("Replace commits the current match as one edit and moves to the next match") - func replaceCurrent() { + func replaceCurrent() async { let (model, owner) = mount { $0.isEditable = true } - search(model, "acme") + await search(model, "acme") model.find.replacement = "Umbrella" model.replaceCurrentMatch() @@ -556,12 +677,12 @@ struct DataGridModelFindTests { } @Test("Replace All is one commit and leaves a column it may not write") - func replaceAllOneCommit() throws { + func replaceAllOneCommit() async throws { let (model, owner) = mount { $0.isEditable = true $0.lockedColumns = ["city"] } - search(model, "acme") + await search(model, "acme") model.find.replacement = "Umbrella" let outcome = try #require(model.replaceAllMatches()) @@ -588,11 +709,12 @@ struct DataGridModelFindTests { } @Test("Hiding the bar clears the term, the matches, the scope and the replace row") - func hideClears() { + func hideClears() async { let (model, owner) = mount { $0.isEditable = true } model.findInColumn(Self.company, replace: true) model.find.term = "acme" model.findQueryDidChange() + await model.findSearchSettled() #expect(model.find.matches.count == 2) model.hideFind() @@ -606,24 +728,24 @@ struct DataGridModelFindTests { } @Test("Search All Rows hands the trimmed term to the owner only where the grid offers it") - func searchAllRows() { + func searchAllRows() async { let (plain, plainOwner) = mount() - search(plain, " zzz ") + await search(plain, " zzz ") plain.searchAllRows() #expect(plainOwner.searchedTerms.isEmpty) #expect(plain.find.isVisible) let (offered, owner) = mount { $0.commands = [.searchAllRows] } - search(offered, " zzz ") + await search(offered, " zzz ") offered.searchAllRows() #expect(owner.searchedTerms == ["zzz"]) #expect(offered.find.isVisible == false) } @Test("Search All Rows waits for plain options, since the server matches the term as typed in any column") - func searchAllRowsNeedsPlainOptions() { + func searchAllRowsNeedsPlainOptions() async { let (model, owner) = mount { $0.commands = [.searchAllRows] } - search(model, "^zzz$") + await search(model, "^zzz$") model.find.options.usesRegularExpression = true model.searchAllRows() @@ -636,9 +758,9 @@ struct DataGridModelFindTests { } @Test("New rows under an open bar are searched again") - func reloadSearchesTheNewRows() { + func reloadSearchesTheNewRows() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") #expect(model.find.matches.count == 4) owner.tableRows = TableRows.from( @@ -647,6 +769,7 @@ struct DataGridModelFindTests { columnTypes: [.integer(rawType: "INT"), .text(rawType: "TEXT"), .text(rawType: "TEXT")] ) model.reloadRows() + await model.findSearchSettled() #expect(model.find.matches.count == 1) } @@ -657,9 +780,9 @@ struct DataGridModelFindTests { } @Test("Edits under an open bar keep the matches a full search finds") - func editsKeepTheMatchesInStep() { + func editsKeepTheMatchesInStep() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") owner.tableRows.rows[2].values[1] = .text("Umbrella") model.apply(.cellChanged(row: 2, column: 1)) @@ -672,9 +795,9 @@ struct DataGridModelFindTests { } @Test("Rows added or removed under an open bar keep the matches a full search finds") - func insertsAndRemovalsKeepTheMatchesInStep() { + func insertsAndRemovalsKeepTheMatchesInStep() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") let inserted = owner.tableRows.appendInsertedRow(values: [.text("5"), .text("Acme East"), .text("Oslo")]) model.apply(inserted) @@ -696,7 +819,7 @@ struct DataGridModelFindTests { // An edit keeps its row on screen (#2667); an insert hides it, and once undone and inserted again it shows again. @Test("A row a value filter shows again after an insert is searched again") - func rowBackInTheFilteredOrderIsSearched() { + func rowBackInTheFilteredOrderIsSearched() async { let (model, owner) = mount() var filter = GridValueFilterState() filter.set( @@ -705,7 +828,7 @@ struct DataGridModelFindTests { forColumn: 2 ) model.setValueFilter(filter) - search(model, "acme") + await search(model, "acme") #expect(model.find.matches == [match(1, Self.city), match(2, Self.company), match(3, Self.company), match(3, Self.city)]) owner.tableRows.rows[2].values[2] = .text("Lyon") @@ -726,13 +849,14 @@ struct DataGridModelFindTests { } @Test("A term typed but not yet searched makes the next refresh search every row") - func changedTermSearchesEveryRow() { + func changedTermSearchesEveryRow() async { let (model, owner) = mount() - search(model, "acme") + await search(model, "acme") model.find.term = "paris" owner.tableRows.rows[0].values[2] = .text("Paris") model.apply(.cellChanged(row: 3, column: 1)) + await model.findSearchSettled() #expect(model.find.matches == [match(0, Self.city), match(2, Self.city)]) } @@ -748,16 +872,218 @@ struct DataGridModelFindTests { } @Test("Use Selection for Find takes the cell's text, escaped for a regular expression") - func useSelectionForFind() { + func useSelectionForFind() async { let (model, owner) = mount() model.setSelection(cell(0, 2)) model.find.options.usesRegularExpression = true model.useSelectionForFind() + await model.findSearchSettled() #expect(model.find.isVisible) #expect(model.find.term == NSRegularExpression.escapedPattern(for: "St. Louis")) #expect(model.find.matches == [GridFindState.Match(rowID: .existing(0), columnID: DataGridColumnID(name: "city", occurrence: 0))]) #expect(owner.commits.isEmpty) } + + // MARK: - A search off the main actor + + // Each test ends on `owner`: the model holds its data source weakly, and without it the grid has no rows. + + @Test("A typed term drops the old term's matches at once, and a refresh keeps its matches until its search lands") + func typedTermDropsTheOldMatches() async { + let (model, owner) = mount() + await search(model, "paris") + + model.find.term = "acme" + model.findQueryDidChange() + + #expect(model.find.loadedRowsSearch != nil) + #expect(model.find.isSearching == false) + #expect(model.find.matches.isEmpty) + #expect(model.find.currentIndex == nil) + + await model.findSearchSettled() + + #expect(model.find.loadedRowsSearch == nil) + #expect(model.find.matches.count == 4) + #expect(model.selection.activeCell == GridCoord(row: 3, displayColumn: 1)) + + model.refreshFind() + + #expect(model.find.loadedRowsSearch != nil) + #expect(model.find.matches.count == 4) + #expect(model.find.currentIndex == 2) + + await model.findSearchSettled() + #expect(model.find.currentIndex == 2) + #expect(owner.commits.isEmpty) + } + + @Test("A typed term drops the old term's invalid-pattern message at once") + func typedTermDropsTheOldProblem() async { + let (model, owner) = mount() + model.showFind() + model.find.options.usesRegularExpression = true + model.find.term = "[" + model.findQueryDidChange() + #expect(model.find.problem != nil) + + model.find.term = "acme" + model.findQueryDidChange() + + #expect(model.find.loadedRowsSearch != nil) + #expect(model.find.problem == nil) + + await model.findSearchSettled() + #expect(model.find.problem == nil) + #expect(owner.commits.isEmpty) + } + + @Test("Of two terms typed in quick succession only the second one's matches land") + func onlyTheLastTermLands() async throws { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + let first = try #require(model.find.loadedRowsSearch) + + model.find.term = "paris" + model.findQueryDidChange() + await model.findSearchSettled() + + #expect(first.isCancelled) + #expect(model.find.matches == [match(2, Self.city)]) + #expect(model.selection.activeCell == GridCoord(row: 2, displayColumn: 2)) + #expect(owner.commits.isEmpty) + } + + @Test("A row added while a search runs is among the matches that land") + func rowAddedDuringASearchIsFound() async throws { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + let running = try #require(model.find.loadedRowsSearch) + + model.apply(owner.tableRows.appendInsertedRow(values: [.text("5"), .text("Acme East"), .text("Oslo")])) + await model.findSearchSettled() + + let insertedID = owner.tableRows.rows[4].id + #expect(running.isCancelled) + #expect(model.find.matches.last == GridFindState.Match(rowID: insertedID, columnID: Self.company)) + #expect(model.find.matches == fullSearch(model, owner)) + #expect(model.selection.activeCell == GridCoord(row: 1, displayColumn: 2), "the typed term still lands on its first match") + } + + @Test("A typed term's matches land without moving a cursor the user moved during the search") + func landingKeepsAMovedCursor() async { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + + model.setSelection(cell(0, 0)) + await model.findSearchSettled() + + #expect(model.find.matches.count == 4) + #expect(model.find.currentIndex == 0) + #expect(model.selection.activeCell == GridCoord(row: 0, displayColumn: 0)) + #expect(owner.commits.isEmpty) + } + + @Test("Find Next pressed while a search runs steps once its matches land") + func findNextWaitsForTheSearch() async { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + + model.findNext() + #expect(model.selection.activeCell == nil) + + await model.findSearchSettled() + + #expect(model.find.currentIndex == 1) + #expect(model.selection.activeCell == GridCoord(row: 2, displayColumn: 1)) + #expect(owner.commits.isEmpty) + } + + @Test("Every Find Next and Find Previous pressed while a search runs counts once its matches land") + func deferredStepsAreCounted() async { + let (model, owner) = mount() + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + + model.findNext() + model.findNext() + model.findNext() + model.findPrevious() + await model.findSearchSettled() + + #expect(model.find.currentIndex == 2) + #expect(model.selection.activeCell == GridCoord(row: 3, displayColumn: 1)) + #expect(owner.commits.isEmpty) + } + + @Test("Replace pressed while a search runs replaces the match that search lands on") + func replaceWaitsForTheRunningSearch() async { + let (model, owner) = mount { $0.isEditable = true } + await search(model, "paris") + model.find.term = "acme" + model.findQueryDidChange() + model.find.replacement = "Umbrella" + + model.replaceCurrentMatch() + + #expect(model.find.loadedRowsSearch == nil) + #expect(owner.commits.first?.edits.map { $0.rowID } == [.existing(3)]) + #expect(owner.commits.first?.edits.map { $0.newValue } == [.text("Umbrella Corp")]) + } + + @Test("The bar says No matches and offers Search All Rows only once the search lands") + func noMatchesWaitsForTheSearch() async { + let (model, owner) = mount { $0.commands = [.searchAllRows] } + model.showFind() + model.find.term = "zzz" + model.findQueryDidChange() + + #expect(model.find.isAwaitingLoadedRows) + #expect(DataGridFindBar.counterText(status: nil, model: model).isEmpty) + #expect(!DataGridFindBar.showsSearchAllRows(offered: model.offersSearchAllRows, model: model)) + + await model.findSearchSettled() + + #expect(!model.find.isAwaitingLoadedRows) + #expect(DataGridFindBar.counterText(status: nil, model: model) == String(localized: "No matches")) + #expect(DataGridFindBar.showsSearchAllRows(offered: model.offersSearchAllRows, model: model)) + + model.find.term = "" + model.findQueryDidChange() + + #expect(!model.find.isAwaitingLoadedRows, "an empty term is answered without a search") + #expect(model.find.loadedRowsSearch == nil) + #expect(owner.commits.isEmpty) + } + + @Test("A grid that goes away cancels the search still running") + func releasedModelCancelsTheSearch() throws { + let owner: FindGridOwner + let running: Task + weak var released: DataGridModel? + do { + let (model, mounted) = mount() + owner = mounted + model.showFind() + model.find.term = "acme" + model.findQueryDidChange() + running = try #require(model.find.loadedRowsSearch) + released = model + } + + #expect(released == nil) + #expect(running.isCancelled) + #expect(owner.commits.isEmpty) + } } diff --git a/apple/TableProTests/Core/DataWrite/PluginKeyedChangesTests.swift b/apple/TableProTests/Core/DataWrite/PluginKeyedChangesTests.swift index e09362a589..697ed53c68 100644 --- a/apple/TableProTests/Core/DataWrite/PluginKeyedChangesTests.swift +++ b/apple/TableProTests/Core/DataWrite/PluginKeyedChangesTests.swift @@ -10,7 +10,7 @@ import Testing @testable import TablePro struct PluginKeyedChangesTests { - @Test("Every row gets its own key, and the key agrees across the changes and the sets") + @Test("Every change gets its own key, and the key agrees across the changes and the sets") func keysAgreeAcrossCollections() { let insertedA = RowID.inserted(UUID()) let insertedB = RowID.inserted(UUID()) @@ -93,4 +93,31 @@ struct PluginKeyedChangesTests { #expect(crossed?.cellChanges.first?.newValue == "b") #expect(crossed?.originalRow == ["3", "a"]) } + + @MainActor + @Test("A row edited around another change reaches the driver once per edit, each with the row as it then stands") + func aRowEditedAroundAnotherChangeReachesTheDriverTwice() throws { + let handed = HandedChanges() + let driver = RowWriteStubDriver { changes, _, _, _ in + handed.changes = changes + return changes.map { PluginRowWrite(statement: "write(\($0.rowIndex))", rowIndices: [$0.rowIndex]) } + } + let (manager, _) = EditAroundADelete.manager(table: "items", databaseType: DatabaseType(rawValue: "MongoDB")) + manager.pluginDriver = driver + EditAroundADelete.record(on: manager) + + _ = try manager.generateSQL() + let types = handed.changes.map { $0.type } + let keys = Set(handed.changes.map { $0.rowIndex }) + let laterColumns = handed.changes.last?.cellChanges.map { $0.columnName } + + #expect(types == [.update, .delete, .update]) + #expect(keys.count == 3) + #expect(handed.changes.last?.originalRow == ["3", "Cyd", "c@test.com"]) + #expect(laterColumns == ["email"]) + } +} + +private final class HandedChanges { + var changes: [PluginRowChange] = [] } diff --git a/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift b/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift index 10d189f39a..b27435f4c6 100644 --- a/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift +++ b/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift @@ -358,6 +358,139 @@ struct RewindPlannerTests { #expect(undo.statements.map { String($0.sql.prefix(6)) } == ["DELETE", "INSERT"]) } + private func recordedOperations(_ edits: (inout PendingChanges) -> Void) -> [RowWriteOperation] { + var pending = PendingChanges() + edits(&pending) + return RowWriteOperationBuilder.operations( + from: pending.changes, + insertedRowData: pending.insertedRowData, + deletedRowIDs: pending.deletedRowIDs, + insertedRowIDs: pending.insertedRowIDs, + target: target, + columns: columns, + primaryKeyColumns: ["id"], + generatedColumns: [], + containsTableOperation: false + ) + } + + // A.name a to tmp, B.name b to a, A.name tmp to b: valid step by step on a unique `name`. + private func swapOperations() -> [RowWriteOperation] { + recordedOperations { pending in + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "a", newValue: "tmp", + originalRow: ["k1", "a"] + ) + pending.recordCellChange( + rowID: .existing(1), columnIndex: 1, columnName: "name", oldValue: "b", newValue: "a", + originalRow: ["k2", "b"] + ) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "tmp", newValue: "b", + originalRow: ["k1", "tmp"] + ) + } + } + + private func bound(_ statements: [ParameterizedStatement]) -> [[String]] { + statements.map { statement in statement.parameters.compactMap { $0 as? String } } + } + + @Test("A swap rewinds its newest step first, so no step takes a value another still holds") + func aSwapRewindsNewestStepFirst() throws { + let plan = try planner(operations: swapOperations()).plan(currentRows: [ + RewindCurrentRow(values: ["k1", "b"]), RewindCurrentRow(values: ["k2", "a"]) + ]) + + #expect(plan.rows.map { $0.outcome } == [.willRestore, .willRestore]) + #expect(bound(plan.statements) == [["tmp", "k1"], ["b", "k2"], ["a", "k1"]]) + } + + @Test("Undoing the rewind of a swap replays the save in its own order") + func undoingASwapRewindReplaysTheSave() throws { + let rewind = try planner(operations: swapOperations()).plan(currentRows: [ + RewindCurrentRow(values: ["k1", "b"]), RewindCurrentRow(values: ["k2", "a"]) + ]) + + let undo = try planner(operations: RewindExecutor.inverseOperations(of: rewind)).plan(currentRows: [ + RewindCurrentRow(values: ["k1", "a"]), RewindCurrentRow(values: ["k2", "b"]) + ]) + + #expect(bound(undo.statements) == [["tmp", "k1"], ["a", "k2"], ["b", "k1"]]) + } + + @Test("A key moved in an earlier step is rewound under the key each step left") + func aKeyMovedInTheFirstStepRewindsUnderEachStepsKey() throws { + let operations = recordedOperations { pending in + pending.recordCellChange( + rowID: .existing(0), columnIndex: 0, columnName: "id", oldValue: "k5", newValue: "k6", + originalRow: ["k5", "Ada"] + ) + pending.recordRowDeletion(rowID: .existing(1), originalRow: ["k7", "Bo"]) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "Ada", newValue: "Grace", + originalRow: ["k6", "Ada"] + ) + } + + let plan = try planner(operations: operations).plan(currentRows: [RewindCurrentRow(values: ["k6", "Grace"])]) + let verbs = plan.statements.map { String($0.sql.prefix(6)) } + + #expect(plan.rows.map { $0.outcome } == [.willRestore, .willRestore]) + #expect(verbs == ["UPDATE", "INSERT", "UPDATE"]) + #expect(bound(plan.statements).first == ["Ada", "k6"]) + #expect(bound(plan.statements).last == ["k5", "k6"]) + } + + @Test("A step that wrote SQL refuses the row, though a later step replaced it with a value") + func aStepThatWroteSQLRefusesTheRow() throws { + let operations = recordedOperations { pending in + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "Ada", newValue: "now()", + originalRow: ["k1", "Ada"], isExpression: true + ) + pending.recordRowDeletion(rowID: .existing(1), originalRow: ["k2", "Bo"]) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "now()", newValue: "Grace", + originalRow: ["k1", "now()"] + ) + } + + let update = try #require(operations.first { $0.kind == .update }) + #expect(update.refusal == .serverComputedValue) + let plan = try planner(operations: operations).plan(currentRows: [RewindCurrentRow(values: ["k1", "Grace"])]) + #expect(plan.rows.first?.outcome == .notReversible(.serverComputedValue)) + } + + @Test("A record saved before steps were kept rewinds newest first by list position") + func aRecordWithoutStepsRewindsAsBefore() throws { + let first = operation(kind: .update, preImage: ["k1", "Ada"], postImage: ["k1", "Grace"], writtenColumns: ["name"]) + let second = operation(kind: .update, preImage: ["k2", "Bo"], postImage: ["k2", "Cy"], writtenColumns: ["name"]) + + let plan = try planner(operations: [first, second]).plan(currentRows: [ + RewindCurrentRow(values: ["k1", "Grace"]), RewindCurrentRow(values: ["k2", "Cy"]) + ]) + + #expect(bound(plan.statements) == [["Bo", "k2"], ["Ada", "k1"]]) + } + + @Test("A deleted row comes back once the new row that took its key is removed, and not while that row was changed") + func aDeletedRowComesBackAfterTheNewRowThatTookItsKey() throws { + let operations = recordedOperations { pending in + pending.recordRowDeletion(rowID: .existing(0), originalRow: ["k5", "Ada"]) + pending.recordRowInsertion(rowID: .inserted(UUID()), values: ["k5", "Bo"]) + } + + let plan = try planner(operations: operations).plan(currentRows: [RewindCurrentRow(values: ["k5", "Bo"])]) + let verbs = plan.statements.map { String($0.sql.prefix(6)) } + #expect(plan.rows.map { $0.outcome } == [.willRestore, .willRestore]) + #expect(verbs == ["DELETE", "INSERT"]) + + let changed = try planner(operations: operations).plan(currentRows: [RewindCurrentRow(values: ["k5", "Cy"])]) + #expect(changed.rows.map { $0.outcome } == [.rowAlreadyPresent, .changedSinceSave]) + #expect(changed.statements.isEmpty) + } + @Test("The inverse runs newest first, so a value one row freed is not taken before the other gives it up") func inverseRunsInReverseOrder() throws { let first = operation(kind: .delete, preImage: ["7", "Ada"], postImage: nil, writtenColumns: columns) diff --git a/apple/TableProTests/Core/DataWrite/RewindRecordIdentityDecodingTests.swift b/apple/TableProTests/Core/DataWrite/RewindRecordIdentityDecodingTests.swift index 5480185201..87d1d1b8d0 100644 --- a/apple/TableProTests/Core/DataWrite/RewindRecordIdentityDecodingTests.swift +++ b/apple/TableProTests/Core/DataWrite/RewindRecordIdentityDecodingTests.swift @@ -5,6 +5,7 @@ import Foundation @testable import TablePro +import TableProPluginKit import Testing struct RewindRecordIdentityDecodingTests { @@ -42,6 +43,62 @@ struct RewindRecordIdentityDecodingTests { } } +struct RewindRecordStepDecodingTests { + private func record() -> RewindRecord { + let target = DataWriteTarget(database: "shop", schema: nil, table: "users") + let operation = RowWriteOperation( + kind: .update, + target: target, + columns: ["id", "name"], + primaryKeyColumns: ["id"], + preImage: ["1", "a"], + postImage: ["1", "b"], + writtenColumns: ["name"], + refusal: nil, + order: 3, + steps: [ + RowWriteStep(order: 3, columns: [1], before: ["a"], after: ["tmp"]), + RowWriteStep(order: 5, columns: [1], before: ["tmp"], after: ["b"], afterAbsent: [1]) + ] + ) + return RewindRecord( + id: UUID(), + historyId: nil, + connectionId: UUID(), + databaseType: .postgresql, + target: target, + capturedAt: Date(timeIntervalSince1970: 1_700_000_000), + generatedColumns: [], + identityColumns: [], + operations: [operation] + ) + } + + @Test + func aRecordSavedBeforeStepsDecodes() throws { + let encoded = try JSONEncoder().encode(record()) + var object = try #require(try JSONSerialization.jsonObject(with: encoded) as? [String: Any]) + var operations = try #require(object["operations"] as? [[String: Any]]) + operations[0].removeValue(forKey: "order") + operations[0].removeValue(forKey: "steps") + object["operations"] = operations + let legacy = try JSONSerialization.data(withJSONObject: object) + + let decoded = try JSONDecoder().decode(RewindRecord.self, from: legacy) + let operation = try #require(decoded.operations.first) + #expect(operation.order == nil) + #expect(operation.steps == nil) + #expect(operation.postImage == ["1", "b"]) + } + + @Test + func stepsRoundTrip() throws { + let original = record() + let decoded = try JSONDecoder().decode(RewindRecord.self, from: JSONEncoder().encode(original)) + #expect(decoded.operations == original.operations) + } +} + struct RewindRecordColumnTypeDecodingTests { private func record(columnTypeNames: [String: String]?) -> RewindRecord { RewindRecord( diff --git a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift index 570e20367b..c6c74fa0fc 100644 --- a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift @@ -371,6 +371,40 @@ struct RowChangeStatementFactoryCoverageTests { #expect(statements.allSatisfy { $0.keylessRowCount == nil }) } + @Test("A driver that writes only the first edit of a row edited around a delete refuses the save") + func aDriverThatWritesOneRunOfARowRefusesTheSave() { + let driver = RowWriteStubDriver { changes, _, _, _ in + let firstUpdate = changes.first { $0.type == .update }?.rowIndex + return changes.compactMap { change in + guard change.type != .update || change.rowIndex == firstUpdate else { return nil } + return PluginRowWrite(statement: "write(\(change.rowIndex))", rowIndices: [change.rowIndex]) + } + } + let (manager, _) = EditAroundADelete.manager(table: "items", databaseType: DatabaseType(rawValue: "MongoDB")) + manager.pluginDriver = driver + EditAroundADelete.record(on: manager) + + #expect(throws: DataWriteError.changesNotWritable(table: "items", unwritten: UnwrittenRowCounts(updates: 1))) { + _ = try manager.generateSQL() + } + } + + @Test("A refusal counts the rows it cannot write, not the edits of each") + func aRefusalCountsRowsNotRuns() { + let driver = RowWriteStubDriver { changes, _, _, _ in + changes.filter { $0.type == .delete }.map { + PluginRowWrite(statement: "deleteOne(\($0.rowIndex))", rowIndices: [$0.rowIndex]) + } + } + let (manager, _) = EditAroundADelete.manager(table: "items", databaseType: DatabaseType(rawValue: "MongoDB")) + manager.pluginDriver = driver + EditAroundADelete.record(on: manager) + + #expect(throws: DataWriteError.changesNotWritable(table: "items", unwritten: UnwrittenRowCounts(updates: 1))) { + _ = try manager.generateSQL() + } + } + @Test("Preview SQL and Save both refuse the same mixed set") func buildRowWritesRefusesWhatGenerateSQLRefuses() { let manager = DataChangeManager() diff --git a/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift b/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift index fdf82b33af..c98576f909 100644 --- a/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift @@ -209,6 +209,37 @@ struct RowWriteOperationBuilderTests { #expect(result.map { $0.kind } == [.delete, .insert]) } + @Test("A row edited around another change is one operation with its net images and both steps") + func aRowEditedInTwoRunsIsOneOperationWithBothSteps() throws { + var pending = PendingChanges() + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Ada", newValue: "Grace", originalRow: ["7", "Ada", "2026-01-01"] + ) + pending.recordRowDeletion(rowID: .existing(1), originalRow: ["8", "Bo", "2026-01-01"]) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "updated_at", + oldValue: "2026-01-01", newValue: "2026-02-02", originalRow: ["7", "Grace", "2026-01-01"] + ) + + let result = operations( + changes: pending.changes, + deletedRowIDs: pending.deletedRowIDs + ) + + #expect(result.map { $0.kind } == [.update, .delete]) + let update = try #require(result.first) + #expect(update.preImage == ["7", "Ada", "2026-01-01"]) + #expect(update.postImage == ["7", "Grace", "2026-02-02"]) + #expect(update.order == 0) + let steps = try #require(update.steps) + #expect(steps.map { $0.order } == [0, 2]) + #expect(steps.map { $0.columns } == [[1], [2]]) + #expect(steps.map { $0.after } == [["Grace"], ["2026-02-02"]]) + #expect(result.last?.order == 1) + #expect(result.last?.steps == nil) + } + @Test("An insert carrying its own key is reversible") func userSuppliedKeyIsReversible() { let result = operations( diff --git a/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift b/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift index 81ec4f95e9..6eb56754f3 100644 --- a/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift +++ b/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift @@ -5,6 +5,8 @@ import Foundation @testable import TablePro +import TableProPluginKit +import TableProSQLGrammar import Testing @Suite("Credential lease routing", .serialized) @@ -60,6 +62,35 @@ struct CredentialLeaseRoutingTests { #expect(manager.activeSessions[connection.id]?.driver === home) } + @Test("A lane closed for its ending lease reports the session state the user set up on it") + func vaultRotationReportsSessionState() async { + let connection = TestFixtures.makeConnection(database: "app") + let home = MockDatabaseDriver(connection: connection) + let lane = MockDatabaseDriver(connection: connection) + lane.sessionTransactionStateToReturn = .idle + var session = ConnectionSession(connection: connection) + session.driver = home + session.status = .connected + let manager = DatabaseManager.shared + manager.injectSession(session, for: connection.id) + manager.sessionLanes.park(lane, database: "logs", for: connection.id) + defer { + manager.sessionLanes.closeAll(for: connection.id) + manager.removeSession(for: connection.id) + } + let key = SessionTransactionTracker.Key(connectionId: connection.id, lane: "logs") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .preparedStatements), on: lane, for: key) + + await manager.credentialLeaseEnding(ending(connection.id, lane)) + + #expect(lane.disconnectCallCount == 1) + #expect(manager.sessionStateLedger.state(heldBy: lane, for: connection.id).isEmpty) + #expect( + manager.sessionLanes.takeLoss(database: "logs", for: connection.id) + == SessionLoss(transaction: false, state: .preparedStatements) + ) + } + @Test("A browsed driver replaced while it held a transaction reports the loss to the next statement, once") func reportsLostBrowsedTransaction() async throws { let connection = TestFixtures.makeConnection(database: "app", type: .mysql) @@ -69,7 +100,7 @@ struct CredentialLeaseRoutingTests { let manager = DatabaseManager.shared manager.injectSession(session, for: connection.id) defer { manager.removeSession(for: connection.id) } - manager.sessionLanes.markTransactionLost(database: "app", for: connection.id) + manager.sessionLanes.markLost(database: "app", transaction: true, for: connection.id) let scope = DatabaseScope(connectionId: connection.id, database: "app", schema: nil) let error = await #expect(throws: DatabaseError.self) { @@ -89,14 +120,14 @@ struct CredentialLeaseRoutingTests { defer { manager.removeSession(for: connection.id) } let scope = DatabaseScope(connectionId: connection.id, database: "", schema: nil) #expect(manager.metadataRoute(for: scope) == .sessionDriver) - manager.sessionLanes.markTransactionLost(database: "", for: connection.id) + manager.sessionLanes.markLost(database: "", transaction: true, for: connection.id) try await manager.withMetadataDriver(scope: scope) { _ in } - #expect(manager.sessionLanes.hasTransactionLoss(database: "", for: connection.id)) + #expect(manager.sessionLanes.hasLoss(database: "", for: connection.id)) await #expect(throws: DatabaseError.self) { try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .protectedWrite) { _ in } } - #expect(!manager.sessionLanes.hasTransactionLoss(database: "", for: connection.id)) + #expect(!manager.sessionLanes.hasLoss(database: "", for: connection.id)) } } diff --git a/apple/TableProTests/Core/Database/SessionLanesTests.swift b/apple/TableProTests/Core/Database/SessionLanesTests.swift index 4b4e31a158..0baef7ec88 100644 --- a/apple/TableProTests/Core/Database/SessionLanesTests.swift +++ b/apple/TableProTests/Core/Database/SessionLanesTests.swift @@ -10,6 +10,7 @@ import Foundation @testable import TablePro import TableProPluginKit +import TableProSQLGrammar import TableProTestSupport import Testing @@ -299,6 +300,24 @@ struct SessionLanesTests { #expect(manager.sessionLanes.parkedDatabases(for: harness.connection.id).sorted() == ["audit", "logs"]) } + @Test("An opened connection holding session state the user set up is not closed for being idle") + func openedLaneHoldingSessionStateIsNotClosedIdle() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + _ = try await run(harness, on: "logs", leaving: .idle) + let logs = try #require(harness.opener.opened["logs"]) + let lane = SessionTransactionTracker.Key(connectionId: connectionId, lane: "logs") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .sessionSettings), on: logs, for: lane) + + await manager.closeIdleOpenedLanes(for: connectionId, idleFor: 0) + + #expect(logs.disconnectCallCount == 0) + #expect(manager.sessionLanes.parkedDriver(for: connectionId, database: "logs") === logs) + #expect(manager.sessionLanes.isOpened(database: "logs", for: connectionId)) + } + @Test("The connection of a database the user browsed and left is not closed for being idle") func browsedConnectionIsNotClosedWhenIdle() async throws { let harness = makeHarness() @@ -634,7 +653,7 @@ struct SessionLanesTests { harness.home.sessionTransactionStateToReturn = .inTransaction let scope = DatabaseScope(connectionId: harness.connection.id, database: "app", schema: nil) - await DatabaseManager.shared.sessionLanes.closeAllNotingLostTransactions(for: harness.connection.id) + await DatabaseManager.shared.sessionLanes.closeAllNotingLosses(for: harness.connection.id) #expect(harness.home.disconnectCallCount == 1) #expect(DatabaseManager.shared.executionRoute(for: scope) == .sessionDriver) @@ -645,7 +664,110 @@ struct SessionLanesTests { cancellation: .untracked ) { _ in () } } - #expect(!DatabaseManager.shared.sessionLanes.hasTransactionLoss(database: "app", for: harness.connection.id)) + #expect(!DatabaseManager.shared.sessionLanes.hasLoss(database: "app", for: harness.connection.id)) + } + + @Test("A rebuilt transport reports what the user's statements set up on a parked connection, once") + func transportRebuildReportsALanesSessionState() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + try await manager.switchDatabase(to: "logs", for: connectionId, persist: false) + harness.home.sessionTransactionStateToReturn = .idle + let lane = SessionTransactionTracker.Key(connectionId: connectionId, lane: "app") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .temporaryObjects), on: harness.home, for: lane) + let scope = DatabaseScope(connectionId: connectionId, database: "app", schema: nil) + + await manager.closeTransportLanes(for: connectionId) { true } + + #expect(harness.home.disconnectCallCount == 1) + #expect(manager.executionRoute(for: scope) == .sessionDriver) + let error = await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver( + scope: scope, route: manager.executionRoute(for: scope), cancellation: .untracked + ) { _ in () } + } + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .temporaryObjects) + ) + #expect(error?.localizedDescription.contains(expected) == true) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: connectionId)) + } + + @Test("A parked connection that died holding session state the user set up is replaced and the work refused once") + func deadParkedLaneReportsItsSessionState() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + try await manager.switchDatabase(to: "logs", for: connectionId, persist: false) + harness.home.pingError = DatabaseError.connectionFailed("server closed the connection") + harness.home.sessionTransactionStateToReturn = .idle + let lane = SessionTransactionTracker.Key(connectionId: connectionId, lane: "app") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .userVariables), on: harness.home, for: lane) + let scope = DatabaseScope(connectionId: connectionId, database: "app", schema: nil) + + let error = await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver( + scope: scope, route: .sessionDriver, cancellation: .untracked + ) { _ in () } + } + + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .userVariables) + ) + #expect(error?.localizedDescription.contains(expected) == true) + let replacement = try #require(harness.opener.opened["app"]) + #expect(harness.home.disconnectCallCount == 1) + let ranOn = try await manager.withScopedDriver( + scope: scope, route: .sessionDriver, cancellation: .untracked + ) { driver in ObjectIdentifier(driver) } + #expect(ranOn == ObjectIdentifier(replacement)) + } + + @Test("A grid save on a database whose lane died with session state is refused once, and the next one runs") + func firstOwningTurnOnALostLaneTakesTheReport() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + try await manager.switchDatabase(to: "logs", for: connectionId, persist: false) + harness.home.sessionTransactionStateToReturn = .idle + let lane = SessionTransactionTracker.Key(connectionId: connectionId, lane: "app") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .temporaryObjects), on: harness.home, for: lane) + await manager.closeTransportLanes(for: connectionId) { true } + let scope = DatabaseScope(connectionId: connectionId, database: "app", schema: nil) + + let error = await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver( + scope: scope, route: manager.executionRoute(for: scope), cancellation: .protectedWrite + ) { _ in () } + } + + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .temporaryObjects) + ) + #expect(error?.localizedDescription.contains(expected) == true) + #expect(harness.opener.opened["app"] == nil) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: connectionId)) + let savedOn = try await manager.withScopedDriver( + scope: scope, route: manager.executionRoute(for: scope), cancellation: .protectedWrite + ) { driver in ObjectIdentifier(driver) } + let reopened = try #require(harness.opener.opened["app"]) + #expect(savedOn == ObjectIdentifier(reopened)) + } + + @Test("A run notes what ran: every statement, through the one that failed, or only those a Stop let run") + func ranCountFollowsTheOutcome() { + let completed = BatchStatementOutcome.completed(results: [1, 2, 3]) + let stopped = BatchStatementOutcome.cancelled(results: [1]) + let failed = BatchStatementOutcome.failed( + results: [1], failure: .statement(sql: "SET x = 1"), errorDescription: "failed" + ) + #expect(completed.ranCount(of: 3) == 3) + #expect(stopped.ranCount(of: 3) == 1) + #expect(failed.ranCount(of: 3) == 2) } @Test("A table read leaves a lost transaction's report for the user's next statement") @@ -653,7 +775,7 @@ struct SessionLanesTests { let harness = makeHarness() defer { cleanUp(harness) } let manager = DatabaseManager.shared - manager.sessionLanes.markTransactionLost(database: "app", for: harness.connection.id) + manager.sessionLanes.markLost(database: "app", transaction: true, for: harness.connection.id) let scope = DatabaseScope(connectionId: harness.connection.id, database: "app", schema: nil) let ranOn = try await manager.withTableReadDriver(scope: scope, cancellation: .untracked) { driver in @@ -661,7 +783,7 @@ struct SessionLanesTests { } #expect(ranOn == ObjectIdentifier(harness.home)) - #expect(manager.sessionLanes.hasTransactionLoss(database: "app", for: harness.connection.id)) + #expect(manager.sessionLanes.hasLoss(database: "app", for: harness.connection.id)) await #expect(throws: DatabaseError.self) { try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in () } } diff --git a/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift b/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift index d076adf949..3ce71ed1b5 100644 --- a/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift +++ b/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift @@ -2,6 +2,7 @@ import Foundation @testable import TablePro import TableProMCP import TableProPluginKit +import TableProSQLGrammar import TableProSSHConfig import TableProTestSupport import Testing @@ -314,12 +315,12 @@ struct SessionRecoveryLoopTests { #expect(fixture.session?.driver !== fixture.driver) #expect(!manager.sessionTransactions.holdsTransaction(connectionId: fixture.id, container: "app")) - #expect(manager.sessionLanes.hasTransactionLoss(database: "app", for: fixture.id)) + #expect(manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) let error = await #expect(throws: DatabaseError.self) { try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in () } } #expect(error?.localizedDescription.contains(DatabaseManager.parkedTransactionLostMessage(database: "app")) == true) - #expect(!manager.sessionLanes.hasTransactionLoss(database: "app", for: fixture.id)) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) await cleanUp(fixture) } @@ -337,7 +338,200 @@ struct SessionRecoveryLoopTests { await loop.value #expect(fixture.session?.driver !== fixture.driver) - #expect(!manager.sessionLanes.hasTransactionLoss(database: "app", for: fixture.id)) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + await cleanUp(fixture) + } + + // Inside the user's own lease, as a query tab notes what it runs. + private func runUserStatements(_ statements: [String], on fixture: Fixture) async throws { + let manager = fixture.manager + let scope = DatabaseScope(connectionId: fixture.id, database: "app", schema: nil) + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { driver in + await manager.noteSessionStatements(statements, ranOn: driver, scope: scope) + } + } + + private func recover(_ fixture: Fixture) async throws { + fixture.manager.beginRecovery(for: fixture.id, cause: .stoppedResponding) + let loop = try loopTask(fixture) + await runNextAttempt(fixture) + await loop.value + } + + private func runNextStatement(_ fixture: Fixture, ownsSessionState: Bool = true) async throws { + let scope = DatabaseScope(connectionId: fixture.id, database: "app", schema: nil) + try await fixture.manager.withScopedDriver( + scope: scope, route: .sessionDriver, cancellation: .untracked, ownsSessionState: ownsSessionState + ) { _ in () } + } + + private func nextStatementRefusal(_ fixture: Fixture) async -> DatabaseError? { + do { + try await runNextStatement(fixture) + return nil + } catch { + return error as? DatabaseError + } + } + + @Test("A recovery that replaces a driver whose session set something up refuses the next statement once") + func recoveryReportsTheSessionStateItEnded() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["CREATE TABLE #orders (id int)"], on: fixture) + #expect(manager.sessionStateLedger.state(heldBy: fixture.driver, for: fixture.id) == .temporaryObjects) + + try await recover(fixture) + + #expect(fixture.session?.driver !== fixture.driver) + let refusal = await nextStatementRefusal(fixture) + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .temporaryObjects) + ) + #expect(refusal?.localizedDescription.contains(expected) == true) + try await runNextStatement(fixture) + await cleanUp(fixture) + } + + @Test("A recovery that ends a transaction and what the session set up names both in one message") + func recoveryReportsTransactionAndStateInOneMessage() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .inTransaction + try await runUserStatements(["SET XACT_ABORT ON"], on: fixture) + #expect(manager.sessionTransactions.holdsTransaction(connectionId: fixture.id, container: "app")) + + try await recover(fixture) + + let refusal = await nextStatementRefusal(fixture) + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: true, state: .sessionSettings) + ) + #expect(refusal?.localizedDescription.contains(expected) == true) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + await cleanUp(fixture) + } + + // A grid save, an Explain or an MCP statement takes the report as the user's statement would, as a lost + // transaction's report always went. + @Test("The first turn that owns the session after the loss is refused once, whatever work it is, and the next runs") + func firstOwningTurnTakesTheStateReport() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["CREATE TABLE #orders (id int)"], on: fixture) + + try await recover(fixture) + + let scope = DatabaseScope(connectionId: fixture.id, database: "app", schema: nil) + let error = await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .protectedWrite) { _ in () } + } + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .temporaryObjects) + ) + #expect(error?.localizedDescription.contains(expected) == true) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + try await runNextStatement(fixture) + await cleanUp(fixture) + } + + @Test("A reconnect the user asks for reports a lane's lost transaction but not what its session set up") + func manualReconnectReportsNoSessionState() async throws { + let fixture = makeFixture() + let manager = fixture.manager + let logs = MockDatabaseDriver(connection: fixture.connection) + logs.sessionTransactionStateToReturn = .idle + let audit = MockDatabaseDriver(connection: fixture.connection) + audit.sessionTransactionStateToReturn = .inTransaction + manager.sessionLanes.park(logs, database: "logs", for: fixture.id) + manager.sessionLanes.park(audit, database: "audit", for: fixture.id) + let logsLane = SessionTransactionTracker.Key(connectionId: fixture.id, lane: "logs") + let auditLane = SessionTransactionTracker.Key(connectionId: fixture.id, lane: "audit") + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .temporaryObjects), on: logs, for: logsLane) + manager.sessionStateLedger.record(SQLSessionStateEffect(creates: .sessionSettings), on: audit, for: auditLane) + // A session that stopped answering moves the pooled transport, as a rebuilt tunnel does. + manager.updateSession(fixture.id) { $0.liveness = .unreachable(nil) } + + try? await manager.reconnectSession(fixture.id) + + #expect(logs.disconnectCallCount == 1) + #expect(audit.disconnectCallCount == 1) + #expect(!manager.sessionLanes.hasLoss(database: "logs", for: fixture.id)) + #expect( + manager.sessionLanes.takeLoss(database: "audit", for: fixture.id) + == SessionLoss(transaction: true, state: []) + ) + await cleanUp(fixture) + } + + @Test("A recovery of a session whose statements set nothing up refuses nothing") + func recoveryOfASessionThatSetNothingUpReportsNothing() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["SELECT * FROM orders", "UPDATE orders SET total = 0 WHERE id = 1"], on: fixture) + #expect(manager.sessionStateLedger.state(heldBy: fixture.driver, for: fixture.id).isEmpty) + + try await recover(fixture) + + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + try await runNextStatement(fixture) + await cleanUp(fixture) + } + + @Test("A startup command the user runs again is not reported, since the reconnect runs it too") + func startupCommandRerunByTheUserIsNotReported() async throws { + let fixture = makeFixture { $0.startupCommands = "SET NOCOUNT ON" } + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["SET NOCOUNT ON;"], on: fixture) + #expect(manager.sessionStateLedger.state(heldBy: fixture.driver, for: fixture.id).isEmpty) + + try await recover(fixture) + + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + try await runNextStatement(fixture) + await cleanUp(fixture) + } + + @Test("A metadata read on the session driver leaves the lost session state's report for the user's statement") + func metadataReadLeavesTheStateReport() async throws { + let fixture = makeFixture() + let manager = fixture.manager + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements(["DECLARE c CURSOR FOR SELECT 1"], on: fixture) + + try await recover(fixture) + try await runNextStatement(fixture, ownsSessionState: false) + + #expect(manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + let refusal = await nextStatementRefusal(fixture) + #expect(refusal != nil) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: fixture.id)) + await cleanUp(fixture) + } + + @Test("The report names kinds only, never a statement, a name or a value it carried") + func stateMessageNamesNoStatementText() async throws { + let fixture = makeFixture() + fixture.driver.sessionTransactionStateToReturn = .idle + try await runUserStatements( + ["SET app.key = 'secret'", "EXEC sp_set_session_context 'api_token', 'hunter2'"], on: fixture + ) + + try await recover(fixture) + + let refusal = await nextStatementRefusal(fixture) + let message = refusal?.localizedDescription ?? "" + let expected = DatabaseManager.sessionLostMessage( + database: "app", loss: SessionLoss(transaction: false, state: .sessionSettings) + ) + #expect(message.contains(expected)) + for leaked in ["secret", "app.key", "api_token", "hunter2", "sp_set_session_context"] { + #expect(!message.contains(leaked)) + } await cleanUp(fixture) } @@ -544,7 +738,7 @@ struct SessionRecoveryLoopTests { #expect(fixture.manager.sessionLanes.parkedDriver(for: fixture.id, database: "logs") == nil) #expect(lane.disconnectCallCount == 1) - #expect(fixture.manager.sessionLanes.hasTransactionLoss(database: "logs", for: fixture.id)) + #expect(fixture.manager.sessionLanes.hasLoss(database: "logs", for: fixture.id)) #expect(fixture.session?.driver === fixture.driver) await cleanUp(fixture) } @@ -587,7 +781,7 @@ struct SessionRecoveryLoopTests { #expect(manager.sessionLanes.parkedDriver(for: id, database: "logs") === replacementLane) #expect(replacementLane.disconnectCallCount == 0) - #expect(!manager.sessionLanes.hasTransactionLoss(database: "logs", for: id)) + #expect(!manager.sessionLanes.hasLoss(database: "logs", for: id)) #expect(fixture.session?.driver !== fixture.driver) #expect(fixture.session?.liveness == .live) await cleanUp(fixture) diff --git a/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift b/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift index 1d8ab421c1..2e741ec1a0 100644 --- a/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift +++ b/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift @@ -110,7 +110,7 @@ struct SwitchDatabasePooledConnectionTests { #expect(driver.disconnectCallCount == 0) #expect(manager.sessionTransactions.holdsTransaction(connectionId: connection.id, container: "app")) - #expect(!manager.sessionLanes.hasTransactionLoss(database: "app", for: connection.id)) + #expect(!manager.sessionLanes.hasLoss(database: "app", for: connection.id)) #expect(manager.session(for: connection.id)?.resolvedBrowseDatabase == "app") #expect(manager.session(for: connection.id)?.status == .connected) } @@ -145,7 +145,7 @@ struct SwitchDatabasePooledConnectionTests { try? await manager.switchDatabase(to: "unreachable", for: connection.id, persist: false) #expect(driver.disconnectCallCount >= 1) - #expect(manager.sessionLanes.hasTransactionLoss(database: "app", for: connection.id)) + #expect(manager.sessionLanes.hasLoss(database: "app", for: connection.id)) } @Test("A switch that reconnects a connection that stopped answering closes its pooled connections") diff --git a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift index e8b611a013..4a5284387e 100644 --- a/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift +++ b/apple/TableProTests/Core/Plugins/PluginMetadataRegistryCuratedCapabilityTests.swift @@ -158,6 +158,22 @@ private final class MockTrinoPlugin: NSObject, TableProPlugin, DriverPlugin { } } +private final class MockPostgreSQLPlugin: NSObject, TableProPlugin, DriverPlugin { + static let pluginName = "Mock PostgreSQL" + static let pluginVersion = "1.0.0" + static let pluginDescription = "Stands in for the bundled PostgreSQL plugin" + static let capabilities: [PluginCapability] = [.databaseDriver] + + static let databaseTypeId = "PostgreSQL" + static let databaseDisplayName = "PostgreSQL" + static let iconName = "postgresql-icon" + static let defaultPort = 5_432 + + func createDriver(config: DriverConnectionConfig) -> any PluginDatabaseDriver { + fatalError("Not used in tests") + } +} + private final class MockClickHousePlugin: NSObject, TableProPlugin, DriverPlugin { static let pluginName = "Mock ClickHouse" static let pluginVersion = "1.0.0" @@ -369,8 +385,8 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(snapshot.capabilities.supportsRawFilter) } - /// None of these has a `DriverPlugin` static, so a plugin that registers would reset them to the struct - /// defaults: counts on every open that read every partition, a header sort CQL refuses, and Match Any. + // None of these has a `DriverPlugin` static, so a plugin that registers would reset them to the struct + // defaults: counts on every open that read every partition, a header sort CQL refuses, and Match Any. @Test("Cassandra keeps its count, sort and filter limits when its plugin registers") func cassandraKeepsItsCuratedQueryLimits() { let built = PluginMetadataRegistry.shared.buildMetadataSnapshot(from: MockCassandraPlugin.self) @@ -433,8 +449,8 @@ struct PluginMetadataRegistryCuratedCapabilityTests { } } - /// Measured: `ADD … NOT NULL` and `COMMENT ON COLUMN` are syntax errors, `ALTER … TYPE` is refused, and only a - /// primary key column can be renamed, so the Structure tab offers adding and dropping a column and nothing else. + // Measured: `ADD … NOT NULL` and `COMMENT ON COLUMN` are syntax errors, `ALTER … TYPE` is refused, and only a + // primary key column can be renamed, so the Structure tab offers adding and dropping a column and nothing else. @Test("Cassandra and ScyllaDB offer only the column edits CQL can run", arguments: ["Cassandra", "ScyllaDB"]) func cassandraStructureEdits(typeId: String) throws { let snapshot = try #require(PluginMetadataRegistry.shared.snapshot(forRegisteredTypeId: typeId)) @@ -480,8 +496,8 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.connection.defaultLocalSocketPath == MySQLLocalSocket.defaultPath) } - /// A MySQL-protocol session opened with no database has none at all, so it lists nothing until - /// one is chosen. Databend's session falls back to its `default` database instead. + // A MySQL-protocol session opened with no database has none at all, so it lists nothing until + // one is chosen. Databend's session falls back to its `default` database instead. @Test("Only the MySQL engines with no default database require a selected database to browse") func browsingRequiresSelectedDatabasePerEngine() { let registry = PluginMetadataRegistry.shared @@ -591,6 +607,49 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.capabilities.verifiesServerWithSystemTrust == true) } + @Test("PostgreSQL keeps system trust, and TLS asked for without a mode stays Required, when its plugin registers") + func postgreSQLKeepsItsTLSCapabilities() { + let built = PluginMetadataRegistry.shared.buildMetadataSnapshot(from: MockPostgreSQLPlugin.self) + + #expect(built.capabilities.verifiesServerWithSystemTrust == true) + #expect(built.capabilities.requestedTLSVerifiesIdentity == false) + } + + @Test("Trino and ClickHouse keep Verify Identity for TLS asked for without a mode when their plugins register") + func requestedTLSSurvivesRegistration() { + let registry = PluginMetadataRegistry.shared + + #expect(registry.buildMetadataSnapshot(from: MockTrinoPlugin.self).capabilities.requestedTLSVerifiesIdentity) + #expect(registry.buildMetadataSnapshot(from: MockClickHousePlugin.self).capabilities.requestedTLSVerifiesIdentity) + #expect(registry.buildMetadataSnapshot(from: MockHanaPlugin.self).capabilities.requestedTLSVerifiesIdentity) + } + + @Test( + "The libpq engines check the system trust store, while TLS asked for without a mode stays Required", + arguments: ["PostgreSQL", "Redshift", "CockroachDB", "KingbaseES"] + ) + func libpqEnginesSplitTheTwoTLSFlags(typeId: String) throws { + let curated = try #require( + PluginMetadataRegistry.shared.builtInDefaults().first { $0.typeId == typeId }?.snapshot + ) + + #expect(curated.capabilities.verifiesServerWithSystemTrust == true) + #expect(curated.capabilities.requestedTLSVerifiesIdentity == false) + } + + @Test( + "The engines that already verified the system trust store keep both TLS flags", + arguments: ["SAP HANA", "Kafka", "Trino", "Presto", "Qdrant", "DuckDB", "ClickHouse"] + ) + func systemTrustEnginesKeepBothTLSFlags(typeId: String) throws { + let curated = try #require( + PluginMetadataRegistry.shared.builtInDefaults().first { $0.typeId == typeId }?.snapshot + ) + + #expect(curated.capabilities.verifiesServerWithSystemTrust == true) + #expect(curated.capabilities.requestedTLSVerifiesIdentity == true) + } + @Test("etcd offers no generic SSL Mode, since its driver reads only its own TLS Mode field") @MainActor func etcdDeclaresNoGenericSSL() { @@ -624,6 +683,7 @@ struct PluginMetadataRegistryCuratedCapabilityTests { #expect(built.capabilities.filtersTypedByReadRows == false) #expect(built.capabilities.tlsImpliedPorts.isEmpty) #expect(built.capabilities.verifiesServerWithSystemTrust == false) + #expect(built.capabilities.requestedTLSVerifiesIdentity == false) #expect(built.capabilities.supportsPerConnectionCertificatePaths == true) #expect(built.capabilities.newDatabaseFileExtensions.isEmpty) #expect(built.schema.implicitSchemaName == nil) diff --git a/apple/TableProTests/Core/Services/DateFormattingServiceTests.swift b/apple/TableProTests/Core/Services/DateFormattingServiceTests.swift index ecec6574f7..1fc6accabd 100644 --- a/apple/TableProTests/Core/Services/DateFormattingServiceTests.swift +++ b/apple/TableProTests/Core/Services/DateFormattingServiceTests.swift @@ -369,3 +369,62 @@ struct DateFormattingServiceTimeZoneTests { #expect(written == wire) } } + +// Work off the main actor builds its own renderer, so it has to print exactly what the service prints. +@MainActor +struct DateDisplayRendererTests { + private struct Sample: Sendable { + let text: String + let columnType: ColumnType? + } + + private static var samples: [Sample] { + let shapes: [ColumnType?] = [ + .datetime(rawType: "DATETIME"), + .datetime(rawType: "DATETIME(6)"), + .date(rawType: "DATE"), + .timestamp(rawType: "TIME"), + .timestamp(rawType: "TIMETZ"), + .timestamp(rawType: "TIMESTAMPTZ"), + nil + ] + let texts = DatabaseDateParserTests.wireSpellings + [ + "2024-12-31 23:59:59-03:30", + "2024-12-31 23:59:59+05:45", + "2024-12-31 23:59:59Z", + "2024-03-15 09:30:00.000000", + "2024-03-15 09:30:00.999999999", + "09:30:00.123456+05:30", + "not-a-date" + ] + return texts.flatMap { text in shapes.map { Sample(text: text, columnType: $0) } } + } + + private static let instants: [Date] = [0, 1_700_000_000, 1_709_294_400.5, -86_400].map(Date.init(timeIntervalSince1970:)) + + // The service hands its calls to a renderer, so this pins the same text on and off the main actor; the literal tests above pin the text. + @Test("A renderer off the main actor prints the same text as the service on the main actor, under every format") + func rendererMatchesTheService() async { + let samples = Self.samples + let original = DateFormattingService.shared.currentFormat + var printed: [DateFormatOption: [String?]] = [:] + for format in DateFormatOption.allCases { + DateFormattingService.shared.updateFormat(format) + printed[format] = samples.map { DateFormattingService.shared.format(dateString: $0.text, columnType: $0.columnType) } + + Self.instants.map { Optional(DateFormattingService.shared.format($0)) } + } + DateFormattingService.shared.updateFormat(original) + + for format in DateFormatOption.allCases { + let rendered = await Self.render(samples, instants: Self.instants, format: format) + #expect(rendered == printed[format], "\(format)") + } + } + + @concurrent + nonisolated private static func render(_ samples: [Sample], instants: [Date], format: DateFormatOption) async -> [String?] { + let renderer = DateDisplayRenderer(format: format) + return samples.map { renderer.format(dateString: $0.text, columnType: $0.columnType) } + + instants.map { Optional(renderer.format($0)) } + } +} diff --git a/apple/TableProTests/Core/Services/Query/DatabaseTreeMetadataServiceTests.swift b/apple/TableProTests/Core/Services/Query/DatabaseTreeMetadataServiceTests.swift index 9513e89f5e..456bc825e2 100644 --- a/apple/TableProTests/Core/Services/Query/DatabaseTreeMetadataServiceTests.swift +++ b/apple/TableProTests/Core/Services/Query/DatabaseTreeMetadataServiceTests.swift @@ -628,4 +628,66 @@ struct DatabaseTreeMetadataServicePooledScopeTests { #expect(MetadataConnectionPool.shared.pooledDriverCount(for: connection.id) == 1) await tearDown(connection) } + + // Completion reads on the interactive connection, so a Refresh of rows already on screen must not queue there. + private func twoWorkloadSession() -> (DatabaseConnection, interactive: MockDatabaseDriver, bulk: MockDatabaseDriver) { + let connection = TestFixtures.makeConnection(database: "shop", type: .postgresql) + var session = ConnectionSession(connection: connection, driver: MockDatabaseDriver(connection: connection)) + session.status = .connected + session.browseDatabase = "shop" + session.browseSchema = "public" + DatabaseManager.shared.injectSession(session, for: connection.id) + let interactive = MockDatabaseDriver(connection: connection) + let bulk = MockDatabaseDriver(connection: connection) + for (driver, workload) in [(interactive, MetadataConnectionPool.Workload.interactive), (bulk, .bulk)] { + MetadataConnectionPool.shared.injectEntry( + driver, scope: DatabaseScope(connectionId: connection.id, database: "shop", schema: "public"), workload: workload + ) + MetadataConnectionPool.shared.injectEntry( + driver, scope: DatabaseScope(connectionId: connection.id, database: "shop", schema: nil), workload: workload + ) + } + return (connection, interactive, bulk) + } + + @Test("A Refresh of tables already on screen reads on the bulk connection") + func refreshOfLoadedTablesReadsOnTheBulkConnection() async { + let (connection, interactive, bulk) = twoWorkloadSession() + let service = DatabaseTreeMetadataService.shared + + await service.loadTables(connectionId: connection.id, database: "shop", schema: "public") + #expect(interactive.fetchScopedTablesCallCount == 1) + await service.refreshTableObjects(connectionId: connection.id, database: "shop", schema: "public") + + #expect(interactive.fetchScopedTablesCallCount == 1) + #expect(bulk.fetchScopedTablesCallCount == 1) + await tearDown(connection) + } + + @Test("A Refresh of a schema list already on screen reads on the bulk connection") + func refreshOfLoadedSchemaListReadsOnTheBulkConnection() async { + let (connection, interactive, bulk) = twoWorkloadSession() + let service = DatabaseTreeMetadataService.shared + + await service.loadSchemas(connectionId: connection.id, database: "shop") + let firstReads = interactive.fetchSchemasCallCount + #expect(firstReads == 1) + await service.refreshSchemas(connectionId: connection.id, database: "shop") + + #expect(interactive.fetchSchemasCallCount == firstReads) + #expect(bulk.fetchSchemasCallCount == 1) + await tearDown(connection) + } + + @Test("A Retry with nothing on screen stays on the interactive connection") + func retryWithNothingOnScreenStaysInteractive() async { + let (connection, interactive, bulk) = twoWorkloadSession() + let service = DatabaseTreeMetadataService.shared + + await service.refreshTableObjects(connectionId: connection.id, database: "shop", schema: "public") + + #expect(interactive.fetchScopedTablesCallCount == 1) + #expect(bulk.fetchScopedTablesCallCount == 0) + await tearDown(connection) + } } diff --git a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift index 739be16695..3e3be93641 100644 --- a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift +++ b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolCapacityTests.swift @@ -14,40 +14,6 @@ import Foundation import TableProTestSupport import Testing -/// Stands in for opening a pooled connection, remembering each driver by the database it was -/// opened for. -@MainActor -private final class CountingOpener { - private let connectDelaySeconds: Double - private(set) var opened: [MockDatabaseDriver] = [] - private(set) var openedByDatabase: [String: MockDatabaseDriver] = [:] - private(set) var openOrder: [String] = [] - private(set) var peakOpenCount = 0 - var afterOpen: ((DatabaseScope) -> Void)? - var deadOnArrival: Set = [] - - init(connectDelaySeconds: Double = 0) { - self.connectDelaySeconds = connectDelaySeconds - } - - var openCount: Int { - opened.filter { $0.disconnectCallCount == 0 }.count - } - - func open(_ scope: DatabaseScope) async throws -> DatabaseDriver { - let driver = MockDatabaseDriver() - driver.connectDelaySeconds = connectDelaySeconds - driver.hasLostConnection = deadOnArrival.remove(scope.database) != nil - opened.append(driver) - openedByDatabase[scope.database] = driver - openOrder.append(scope.database) - peakOpenCount = max(peakOpenCount, openCount) - try await driver.connect() - afterOpen?(scope) - return driver - } -} - @MainActor private final class StartedTaskBox { var task: Task? diff --git a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolSchemaTests.swift b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolSchemaTests.swift new file mode 100644 index 0000000000..5726f59284 --- /dev/null +++ b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolSchemaTests.swift @@ -0,0 +1,399 @@ +// +// MetadataConnectionPoolSchemaTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +private struct SchemaSighting: Equatable, Sendable { + let driver: ObjectIdentifier + let schema: String? + + init(_ driver: DatabaseDriver) { + self.driver = ObjectIdentifier(driver) + schema = (driver as? MockDatabaseDriver)?.currentSchema + } +} + +private struct NestedSighting: Sendable { + let outer: SchemaSighting + let inner: SchemaSighting +} + +@MainActor +private final class SchemaLeaseLog { + private(set) var entries: [String] = [] + + func record(_ entry: String) { + entries.append(entry) + } +} + +// Each schema of a database held its own pooled connection, so work across three schemas at the cap evicted and redialed. +@Suite("MetadataConnectionPool schemas", .serialized) +@MainActor +struct MetadataConnectionPoolSchemaTests { + private let limit = MetadataConnectionPool.maxPerConnection + + private func makeSession(startupCommands: String? = nil) -> DatabaseConnection { + var connection = TestFixtures.makeConnection(database: "shop") + connection.startupCommands = startupCommands + var session = ConnectionSession(connection: connection, driver: MockDatabaseDriver(connection: connection)) + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + return connection + } + + private func scope(_ connection: DatabaseConnection, _ schema: String?) -> DatabaseScope { + DatabaseScope(connectionId: connection.id, database: "shop", schema: schema) + } + + private func probe(_ pool: MetadataConnectionPool, _ scope: DatabaseScope) async throws -> SchemaSighting { + try await pool.withDriver(scope: scope) { driver in SchemaSighting(driver) } + } + + @Test("Leases on three schemas of one database, one after another, open one connection") + func schemasOfOneDatabaseShareAnIdleConnection() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + for schema in ["a", "b", "c"] { + _ = try await probe(pool, scope(connection, schema)) + } + + #expect(opener.opened.count == 1) + #expect(pool.pooledDriverCount(for: connection.id) == 1) + } + + @Test("Each lease reads on its own schema, so a shared connection is moved before every change of schema") + func leaseRunsOnItsOwnSchema() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + var sightings: [SchemaSighting] = [] + for schema in ["a", "b", "a"] { + sightings.append(try await probe(pool, scope(connection, schema))) + } + let schemas = sightings.map { $0.schema } + let drivers = Set(sightings.map { $0.driver }) + + #expect(schemas == ["a", "b", "a"]) + #expect(drivers.count == 1) + #expect(opener.opened.first?.switchSchemaCallCount == 3) + } + + // One connection per database would make a schema's reads wait behind another schema's long lease. + @Test("A lease on another schema opens its own connection while the first schema's lease still runs", .timeLimit(.minutes(1))) + func busySchemaOpensASiblingForAnotherSchema() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + let gate = AsyncGate() + let held = scope(connection, "a") + let holding = Task { @MainActor in + try await pool.withDriver(scope: held) { driver in + await gate.wait() + return ObjectIdentifier(driver) + } + } + await gate.arrived(1) + + let sibling = try await probe(pool, scope(connection, "b")) + #expect(opener.opened.count == 2) + + await gate.open() + let heldOn = try await holding.value + #expect(sibling.driver != heldOn) + #expect(sibling.schema == "b") + } + + @Test("Two leases on one schema at once queue on one connection") + func sameSchemaStillQueuesOnItsConnection() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + let gate = AsyncGate() + let log = SchemaLeaseLog() + let place = scope(connection, "a") + let first = Task { @MainActor in + try await pool.withDriver(scope: place) { driver in + await gate.wait() + await log.record("first") + return ObjectIdentifier(driver) + } + } + await gate.arrived(1) + let second = Task { @MainActor in + try await pool.withDriver(scope: place) { driver in + await log.record("second") + return ObjectIdentifier(driver) + } + } + try await waitUntil { pool.leaseCount(for: connection.id) == 2 } + #expect(opener.opened.count == 1) + + await gate.open() + let firstRanOn = try await first.value + let secondRanOn = try await second.value + + #expect(firstRanOn == secondRanOn) + #expect(log.entries == ["first", "second"]) + #expect(opener.opened.count == 1) + } + + // Data Compare and Object Copy lease one side inside the other, and both sides stream at once. + @Test( + "A lease nested in another schema's lease gets its own connection, past the limit when it is full", + .timeLimit(.minutes(1)) + ) + func nestedLeaseOnAnotherSchemaGetsItsOwnConnection() async throws { + let connection = makeSession() + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let opener = CountingOpener() + let pool = MetadataConnectionPool.isolatedForTesting(openDriver: { try await opener.open($0) }) + defer { pool.closeAll(connectionId: connection.id) } + + let gate = AsyncGate() + let fills = (1.. NativeDumpCommand { let tool = try #require(NativeDumpRegistry.descriptor(for: type)?.commandLineTool) let effective = overrideConnection ?? connection(type: type) + var request = NativeDumpDescriptor.Request( + connection: effective, + database: database, + fileURL: fileURL, + password: password, + scope: scope, + localFilePath: localFilePath ?? effective.database, + serverVersion: serverVersion + ) + request.systemTrustAnchorsFile = systemTrustAnchorsFile let built = try NativeDumpService.buildCommand( kind: kind, tool: tool, @@ -60,15 +71,7 @@ struct NativeDumpRegistryTests { flavor: flavor, versionText: toolVersionText ), - request: NativeDumpDescriptor.Request( - connection: effective, - database: database, - fileURL: fileURL, - password: password, - scope: scope, - localFilePath: localFilePath ?? effective.database, - serverVersion: serverVersion - ) + request: request ) if let folder = built.clientPluginFolder { try? FileManager.default.removeItem(at: folder.url) @@ -125,8 +128,8 @@ struct NativeDumpRegistryTests { #expect(try #require(NativeDumpRegistry.descriptor(for: .sqlite)).archiveFormat.fileExtension == "sql") } - /// Anything in `argv` is readable by every process on the machine through `ps`, so no - /// descriptor may put a password there. + // Anything in `argv` is readable by every process on the machine through `ps`, so no + // descriptor may put a password there. @Test("No engine puts the password in the argument list") func passwordNeverReachesArgv() throws { for type in [DatabaseType.postgresql, .mysql, .mongodb, .sqlite] { @@ -149,7 +152,7 @@ struct NativeDumpRegistryTests { #expect(built.arguments.last == "sales") } - /// `mysqldump` writes SQL to standard output, so the caller has to redirect it to the file. + // `mysqldump` writes SQL to standard output, so the caller has to redirect it to the file. @Test("MySQL is redirected through standard output in both directions") func mysqlRedirects() throws { let backup = try command(.mysql, kind: .backup) @@ -163,7 +166,7 @@ struct NativeDumpRegistryTests { #expect(!restore.arguments.contains("--single-transaction")) } - /// `pg_dump -Fc` is told the path and writes it itself, so nothing is redirected. + // `pg_dump -Fc` is told the path and writes it itself, so nothing is redirected. @Test("PostgreSQL writes its own file") func postgresWritesItsOwnFile() throws { let built = try command(.postgresql) @@ -171,8 +174,8 @@ struct NativeDumpRegistryTests { #expect(built.redirectedFileURL == nil) } - /// `mongodump` reads a password from neither the environment nor standard input, so the only - /// channel left is a config file, which must be owner-only and must not survive the process. + // `mongodump` reads a password from neither the environment nor standard input, so the only + // channel left is a config file, which must be owner-only and must not survive the process. @Test("MongoDB writes an owner-only credentials file and points at it") func mongoUsesACredentialsFile() throws { let built = try command(.mongodb) @@ -352,7 +355,7 @@ struct NativeDumpRegistryTests { ) } - /// The database is a file the tool opens, so there is nothing to authenticate to. + // The database is a file the tool opens, so there is nothing to authenticate to. @Test("SQLite passes the file path and no network arguments") func sqliteUsesTheFilePath() throws { let file = connection(type: .sqlite, host: "", port: 0, database: "/tmp/app.sqlite", username: "") @@ -390,9 +393,9 @@ struct NativeDumpRegistryTests { #expect(!FileManager.default.fileExists(atPath: marker.path)) } - /// libSQL claims the SQLite descriptor and keeps its path in a plugin-declared additional - /// field, leaving `database` empty. Reading `database` handed `sqlite3` an empty path, which - /// exits 0 after writing a 52-byte file the result sheet then reported as a backup. + // libSQL claims the SQLite descriptor and keeps its path in a plugin-declared additional + // field, leaving `database` empty. Reading `database` handed `sqlite3` an empty path, which + // exits 0 after writing a 52-byte file the result sheet then reported as a backup. @Test("libSQL dumps the file its driver opens, not the empty database field") func libsqlUsesItsResolvedPath() throws { var remote = connection(type: .libsql, host: "", port: 0, database: "", username: "") @@ -437,8 +440,8 @@ struct NativeDumpRegistryTests { } } - /// A libSQL connection to a Turso URL has no local file, so `sqlite3` cannot reach it at all. - /// Answering by type alone is what offered Backup Dump on one and wrote nothing. + // A libSQL connection to a Turso URL has no local file, so `sqlite3` cannot reach it at all. + // Answering by type alone is what offered Backup Dump on one and wrote nothing. @Test("A connection with no local file is not offered a dump") func remoteFileBackedConnectionIsUnsupported() { var remote = connection(type: .libsql, host: "", port: 0, database: "", username: "") @@ -595,9 +598,9 @@ struct NativeDumpRegistryTests { #expect(fromDirectory.arguments.starts(with: ["--no-password", "-h", "/var/run/postgresql", "-p", "5434"])) } - /// Measured, MariaDB 12.3.3 answers any `--ssl-mode` with `unknown variable` and exit 7, and - /// MySQL 8.4.11 answers `--ssl` with `unknown option` and exit 2, so neither spelling may reach - /// the other family's tool in either direction (#3046). + // Measured, MariaDB 12.3.3 answers any `--ssl-mode` with `unknown variable` and exit 7, and + // MySQL 8.4.11 answers `--ssl` with `unknown option` and exit 2, so neither spelling may reach + // the other family's tool in either direction (#3046). @Test("Neither client family is ever handed the other one's SSL flags", arguments: [ NativeDumpKind.backup, .restore ]) @@ -614,8 +617,8 @@ struct NativeDumpRegistryTests { #expect(!mysql.arguments.contains("--skip-ssl")) } - /// SSL off is the other half of the same defect: the old code sent `--ssl-mode=DISABLED`, which - /// MariaDB rejects exactly as it rejects the rest. + // SSL off is the other half of the same defect: the old code sent `--ssl-mode=DISABLED`, which + // MariaDB rejects exactly as it rejects the rest. @Test("An SSL-off connection is disabled in the tool's own spelling") func sslDisabledFollowsTheResolvedTool() throws { let maria = try command(.mysql, flavor: .mariadb) @@ -626,8 +629,8 @@ struct NativeDumpRegistryTests { #expect(mysql.arguments.contains("--ssl-mode=DISABLED")) } - /// `--ssl-cert` implies `--ssl` on MariaDB, so a connection whose SSL is off must not carry the - /// certificate the form still holds. + // `--ssl-cert` implies `--ssl` on MariaDB, so a connection whose SSL is off must not carry the + // certificate the form still holds. @Test("Certificate paths reach the tool, and only while SSL is on") func certificatePathsFollowTheMode() throws { var secured = connection(type: .mysql, sslMode: .verifyCa, sslEnabled: true) @@ -647,8 +650,8 @@ struct NativeDumpRegistryTests { #expect(!disabled.arguments.contains { $0.hasPrefix("--ssl-key") }) } - /// A tool that answered nothing is not guessed at once the connection asks for encryption: - /// MariaDB accepts `--loose-ssl-mode=REQUIRED` and ignores it, which is a cleartext dump. + // A tool that answered nothing is not guessed at once the connection asks for encryption: + // MariaDB accepts `--loose-ssl-mode=REQUIRED` and ignores it, which is a cleartext dump. @Test("An unidentified client refuses an encrypted connection rather than guessing") func unidentifiedToolRefusesEncryptedModes() throws { for mode in [SSLMode.required, .verifyCa, .verifyIdentity] { @@ -662,8 +665,8 @@ struct NativeDumpRegistryTests { #expect(!built.arguments.contains { $0.hasPrefix("--ssl") }) } - /// `mysqldump` 8.0 reads a table no MariaDB server and no MySQL before 8.0 has, and exits 2 - /// after writing part of the file. The flag that skips it is MySQL's own and backup only. + // `mysqldump` 8.0 reads a table no MariaDB server and no MySQL before 8.0 has, and exits 2 + // after writing part of the file. The flag that skips it is MySQL's own and backup only. @Test("A MySQL 8 dump tool skips column statistics on a server that has none") func columnStatisticsFlagFollowsTheServer() throws { let mysql8 = "mysqldump Ver 8.4.11 for macos26.6 on arm64 (Homebrew)" @@ -691,9 +694,9 @@ struct NativeDumpRegistryTests { #expect(!mariaTool.arguments.contains("--skip-column-statistics")) } - /// `my_getopt` parses past a positional argument, so a database named `--no-data` was read as - /// the option: measured on MySQL 8.4.11 and MariaDB 12.3.3, that dumped a different database - /// with no rows and exited 0, which the result sheet reports as a successful backup. + // `my_getopt` parses past a positional argument, so a database named `--no-data` was read as + // the option: measured on MySQL 8.4.11 and MariaDB 12.3.3, that dumped a different database + // with no rows and exited 0, which the result sheet reports as a successful backup. @Test("The option terminator comes before the database, in both directions", arguments: [ NativeDumpKind.backup, .restore ]) @@ -715,9 +718,9 @@ struct NativeDumpRegistryTests { #expect(Array(built.arguments[terminator...]) == ["--", "sales", "orders", "customers"]) } - /// libpq falls back to `~/.postgresql/root.crt` when no root certificate is named, so a - /// Verify CA connection that opens in the app could never be dumped: measured with pg_dump - /// 17.11, it fails with `root certificate file "..." does not exist`. + // libpq falls back to `~/.postgresql/root.crt` when no root certificate is named, so a + // Verify CA connection that opens in the app could never be dumped: measured with pg_dump + // 17.11, it fails with `root certificate file "..." does not exist`. @Test("PostgreSQL sends the whole SSL configuration, not just the mode") func postgresSendsCertificatePaths() throws { var secured = connection(type: .postgresql, sslMode: .verifyCa, sslEnabled: true) @@ -739,8 +742,8 @@ struct NativeDumpRegistryTests { #expect(disabled.environment["PGSSLKEY"] == nil) } - /// The CA belongs to the modes that verify one, which is the rule the live connection already - /// follows in `LibPQConnectionString`. + // The CA belongs to the modes that verify one, which is the rule the live connection already + // follows in `LibPQConnectionString`. @Test("A required connection sends its client certificate but not a CA it does not check") func postgresRequiredKeepsTheClientCertificate() throws { var secured = connection(type: .postgresql, sslMode: .required, sslEnabled: true) @@ -752,8 +755,96 @@ struct NativeDumpRegistryTests { #expect(built.environment["PGSSLCERT"] == "/certs/client.pem") } - /// MariaDB 11.0 renamed every client and some builds ship no `mysql`-prefixed symlink, so both - /// spellings have to be tried before reporting the tool missing. + private func dumpRequest(_ connection: DatabaseConnection) -> NativeDumpDescriptor.Request { + NativeDumpDescriptor.Request( + connection: connection, + database: "sales", + fileURL: URL(fileURLWithPath: "/tmp/out.dump"), + password: nil + ) + } + + @Test("Verify Identity with no CA file hands pg_dump this Mac's trusted roots") + func verifyIdentityWithoutACAGetsTheSystemAnchors() throws { + let verifying = connection(type: .postgresql, sslMode: .verifyIdentity, sslEnabled: true) + let tool = try #require(NativeDumpRegistry.descriptor(for: .postgresql)?.commandLineTool) + let roots = URL(fileURLWithPath: "/tmp/tablepro-trust-test/roots.pem") + + #expect(tool.needsSystemTrustAnchors(dumpRequest(verifying)) == NativeDumpRegistry.postgresNeedsSystemTrustAnchors( + dumpRequest(verifying) + )) + let built = try command(.postgresql, connection: verifying, systemTrustAnchorsFile: roots) + #expect(built.environment["PGSSLMODE"] == "verify-full") + #expect(built.environment["PGSSLROOTCERT"] == roots.path) + + var withCA = verifying + withCA.sslConfig.caCertificatePath = "/certs/ca.pem" + let chosen = try command(.postgresql, connection: withCA, systemTrustAnchorsFile: roots) + #expect(chosen.environment["PGSSLROOTCERT"] == "/certs/ca.pem") + } + + @Test("Verify CA with no CA file still names no root file, as libpq refuses this Mac's roots there") + func verifyCAWithoutACAStillSetsNoRootCertificate() throws { + let verifyCA = connection(type: .postgresql, sslMode: .verifyCa, sslEnabled: true) + let roots = URL(fileURLWithPath: "/tmp/tablepro-trust-test/roots.pem") + + let built = try command(.postgresql, connection: verifyCA, systemTrustAnchorsFile: roots) + + #expect(built.environment["PGSSLMODE"] == "verify-ca") + #expect(built.environment["PGSSLROOTCERT"] == nil) + #expect(!NativeDumpRegistry.postgresNeedsSystemTrustAnchors(dumpRequest(verifyCA), fileExists: { _ in false })) + } + + @Test("pg_dump gets this Mac's roots only for Verify Identity over TCP with no CA file and no root.crt of libpq's own") + func postgresSystemAnchorsFollowTheDriverRule() { + let home = ["HOME": "/Users/test"] + let rootCrt = "/Users/test/.postgresql/root.crt" + func needs(_ connection: DatabaseConnection, rootCrtExists: Bool = false) -> Bool { + NativeDumpRegistry.postgresNeedsSystemTrustAnchors( + dumpRequest(connection), + environment: home, + fileExists: { rootCrtExists && $0 == rootCrt } + ) + } + let verifying = connection(type: .postgresql, sslMode: .verifyIdentity, sslEnabled: true) + var withCA = verifying + withCA.sslConfig.caCertificatePath = "/certs/ca.pem" + var socket = verifying + socket.additionalFields[MySQLLocalSocket.fieldKey] = "/tmp/.s.PGSQL.5432" + + #expect(needs(verifying)) + #expect(needs(connection(type: .redshift, sslMode: .verifyIdentity, sslEnabled: true))) + #expect(!needs(verifying, rootCrtExists: true)) + #expect(!needs(withCA)) + #expect(!needs(socket)) + for mode in [SSLMode.disabled, .preferred, .required, .verifyCa] { + #expect(!needs(connection(type: .postgresql, sslMode: mode, sslEnabled: mode != .disabled))) + } + #expect(!NativeDumpRegistry.postgresNeedsSystemTrustAnchors( + dumpRequest(verifying), + environment: ["HOME": "/Users/test", "PGSSLROOTCERT": "/etc/corp-ca.pem"], + fileExists: { _ in false } + )) + #expect(NativeDumpRegistry.postgresSSLEnvironment( + verifying.sslConfig, inherited: ["PGSSLROOTCERT": "/etc/corp-ca.pem"] + )["PGSSLROOTCERT"] == "/etc/corp-ca.pem") + #expect(NativeDumpRegistry.postgresDefaultRootCertificatePath(environment: home) == rootCrt) + #expect(NativeDumpRegistry.postgresDefaultRootCertificatePath(environment: [:]) + == NSHomeDirectory() + "/.postgresql/root.crt") + } + + @Test("Only the PostgreSQL tools ask for this Mac's trusted roots", arguments: [ + DatabaseType.mysql, .mariadb, .mongodb, .sqlite, .mssql + ]) + func otherToolsNeverAskForTheSystemAnchors(type: DatabaseType) throws { + let tool = try #require(NativeDumpRegistry.descriptor(for: type)?.commandLineTool) + let verifying = connection(type: type, sslMode: .verifyIdentity, sslEnabled: true) + + #expect(!tool.needsSystemTrustAnchors(dumpRequest(verifying))) + } + + // MariaDB 11.0 renamed every client and some builds ship no `mysql`-prefixed symlink, so both + // spellings have to be tried before reporting the tool missing. @Test("MySQL tries both the mysql and mariadb tool names") func mysqlTriesBothToolNames() throws { let tool = try #require(NativeDumpRegistry.descriptor(for: .mysql)?.commandLineTool) @@ -786,8 +877,8 @@ struct NativeDumpRegistryTests { #expect(try configurationValue("password", in: contents) == #"p\nuri: mongodb://evil.example.com"#) } - /// A progress bar showing a percentage of a number nobody measured is worse than an - /// indeterminate one, so an engine with no cheap size answer returns nil. + // A progress bar showing a percentage of a number nobody measured is worse than an + // indeterminate one, so an engine with no cheap size answer returns nil. @Test("Only the engines with a cheap size query offer a determinate progress bar") func sizeQueryCoverage() { #expect(NativeDumpService.sizeQuery(for: .postgresql) != nil) diff --git a/apple/TableProTests/Helpers/CountingOpener.swift b/apple/TableProTests/Helpers/CountingOpener.swift new file mode 100644 index 0000000000..400640dd95 --- /dev/null +++ b/apple/TableProTests/Helpers/CountingOpener.swift @@ -0,0 +1,42 @@ +// +// CountingOpener.swift +// TableProTests +// + +import Foundation +@testable import TablePro + +// Stands in for opening a pooled connection, remembering each driver by the database it was opened for. +@MainActor +final class CountingOpener { + private let connectDelaySeconds: Double + private(set) var opened: [MockDatabaseDriver] = [] + private(set) var openedByDatabase: [String: MockDatabaseDriver] = [:] + private(set) var openOrder: [String] = [] + private(set) var peakOpenCount = 0 + var configure: ((MockDatabaseDriver) -> Void)? + var afterOpen: ((DatabaseScope) -> Void)? + var deadOnArrival: Set = [] + + init(connectDelaySeconds: Double = 0) { + self.connectDelaySeconds = connectDelaySeconds + } + + var openCount: Int { + opened.filter { $0.disconnectCallCount == 0 }.count + } + + func open(_ scope: DatabaseScope) async throws -> DatabaseDriver { + let driver = MockDatabaseDriver() + driver.connectDelaySeconds = connectDelaySeconds + driver.hasLostConnection = deadOnArrival.remove(scope.database) != nil + configure?(driver) + opened.append(driver) + openedByDatabase[scope.database] = driver + openOrder.append(scope.database) + peakOpenCount = max(peakOpenCount, openCount) + try await driver.connect() + afterOpen?(scope) + return driver + } +} diff --git a/apple/TableProTests/Helpers/ScopeRecordingKeychain.swift b/apple/TableProTests/Helpers/ScopeRecordingKeychain.swift index 12d980e421..c433bd475c 100644 --- a/apple/TableProTests/Helpers/ScopeRecordingKeychain.swift +++ b/apple/TableProTests/Helpers/ScopeRecordingKeychain.swift @@ -9,6 +9,7 @@ final class ScopeRecordingKeychain: KeychainStoring, @unchecked Sendable { private var local: [String: String] = [:] private var unreadableKeys: Set = [] private var failingKeys: Set = [] + private var failingSyncedRemovals: Set = [] func seedSynced(_ value: String, forKey key: String) { lock.lock() @@ -40,6 +41,18 @@ final class ScopeRecordingKeychain: KeychainStoring, @unchecked Sendable { failingKeys.insert(key) } + func failSyncedRemoval(of key: String) { + lock.lock() + defer { lock.unlock() } + failingSyncedRemovals.insert(key) + } + + func allowSyncedRemoval(of key: String) { + lock.lock() + defer { lock.unlock() } + failingSyncedRemovals.remove(key) + } + func allowWrites(to key: String) { lock.lock() defer { lock.unlock() } @@ -102,4 +115,14 @@ final class ScopeRecordingKeychain: KeychainStoring, @unchecked Sendable { synced[key] = nil } } + + // As the real keychain does, refused while no local value exists, so the secret is never lost on this Mac. + @discardableResult + func removeSyncedCopy(forKey key: String) -> Bool { + lock.lock() + defer { lock.unlock() } + guard local[key] != nil, !failingSyncedRemovals.contains(key) else { return false } + synced[key] = nil + return true + } } diff --git a/apple/TableProTests/Models/DatabaseTypeTests.swift b/apple/TableProTests/Models/DatabaseTypeTests.swift index 33b8797bea..fa788e1d7a 100644 --- a/apple/TableProTests/Models/DatabaseTypeTests.swift +++ b/apple/TableProTests/Models/DatabaseTypeTests.swift @@ -145,7 +145,7 @@ struct DatabaseTypeTests { func testCACertificateRequirement() { #expect(DatabaseType.trino.requiresCACertificate(for: .verifyIdentity) == false) #expect(DatabaseType.clickhouse.requiresCACertificate(for: .verifyIdentity) == false) - #expect(DatabaseType.postgresql.requiresCACertificate(for: .verifyIdentity) == true) + #expect(DatabaseType.postgresql.requiresCACertificate(for: .verifyIdentity) == false) #expect(DatabaseType.mysql.requiresCACertificate(for: .verifyIdentity) == true) #expect(DatabaseType(rawValue: "FutureDB").requiresCACertificate(for: .verifyIdentity) == true) } diff --git a/apple/TableProTests/ViewModels/ConnectionFormChildObservationTests.swift b/apple/TableProTests/ViewModels/ConnectionFormChildObservationTests.swift index 4d0b82aeca..1672d26628 100644 --- a/apple/TableProTests/ViewModels/ConnectionFormChildObservationTests.swift +++ b/apple/TableProTests/ViewModels/ConnectionFormChildObservationTests.swift @@ -76,4 +76,12 @@ struct ConnectionFormChildObservationTests { replacement.type = .postgresql #expect(counter.sends == 1) } + + @Test("a new connection saved Local only has nothing in iCloud Keychain to remove") + func newLocalOnlyConnectionRemovesNothing() { + let coordinator = ConnectionFormCoordinator(connectionId: nil) + coordinator.advanced.localOnly = true + + #expect(!coordinator.removesSyncedSecretsOnSave) + } } diff --git a/apple/TableProTests/ViewModels/SSLPaneViewModelTests.swift b/apple/TableProTests/ViewModels/SSLPaneViewModelTests.swift index 5a3e12f451..9cff10a01a 100644 --- a/apple/TableProTests/ViewModels/SSLPaneViewModelTests.swift +++ b/apple/TableProTests/ViewModels/SSLPaneViewModelTests.swift @@ -55,9 +55,13 @@ struct SSLPaneViewModelTests { } private func verifyIdentityIssues(for type: DatabaseType, caCertPath: String = "") -> [String] { + issues(for: type, mode: .verifyIdentity, caCertPath: caCertPath) + } + + private func issues(for type: DatabaseType, mode: SSLMode, caCertPath: String = "") -> [String] { let coordinator = ConnectionFormCoordinator(connectionId: nil) coordinator.network.type = type - coordinator.ssl.select(.verifyIdentity) + coordinator.ssl.select(mode) coordinator.ssl.caCertPath = caCertPath return coordinator.ssl.validationIssues } @@ -72,9 +76,20 @@ struct SSLPaneViewModelTests { @Test("Verify Identity without a CA file is refused for engines that need one") func fileTrustEnginesStillRequireACAFile() { - #expect(!DatabaseType.postgresql.verifiesServerWithSystemTrust) - #expect(verifyIdentityIssues(for: .postgresql).count == 1) - #expect(verifyIdentityIssues(for: .postgresql, caCertPath: "/tmp/ca.pem").isEmpty) + #expect(!DatabaseType.mysql.verifiesServerWithSystemTrust) + #expect(verifyIdentityIssues(for: .mysql).count == 1) + #expect(verifyIdentityIssues(for: .mysql, caCertPath: "/tmp/ca.pem").isEmpty) + } + + @Test( + "The libpq engines save Verify Identity without a CA file and still need one for Verify CA", + arguments: [DatabaseType.postgresql, .redshift, .cockroachdb, .kingbasees] + ) + func libpqEnginesNeedACAFileOnlyForVerifyCA(type: DatabaseType) { + #expect(type.verifiesServerWithSystemTrust) + #expect(verifyIdentityIssues(for: type).isEmpty) + #expect(issues(for: type, mode: .verifyCa).count == 1) + #expect(issues(for: type, mode: .verifyCa, caCertPath: "/tmp/ca.pem").isEmpty) } @Test("A Trino port of 443 escalates the type default to Verify Identity, and another port reverts it") diff --git a/apple/TableProTests/ViewModels/WelcomeViewModelTests.swift b/apple/TableProTests/ViewModels/WelcomeViewModelTests.swift index e4afcb5559..6404c8dd2c 100644 --- a/apple/TableProTests/ViewModels/WelcomeViewModelTests.swift +++ b/apple/TableProTests/ViewModels/WelcomeViewModelTests.swift @@ -332,6 +332,42 @@ final class WelcomeViewModelTests { #expect(viewModel.listState == .noFilterMatch) } + @Test func testExcludingFromSyncAsksFirst() throws { + let prod = DatabaseConnection(name: "Prod", type: .postgresql) + save([prod]) + viewModel.loadConnections() + + viewModel.perform(.setIncludedInSync([prod.id], false)) + + #expect(viewModel.showExcludeFromSyncConfirmation) + #expect(viewModel.connectionsToExcludeFromSync.map(\.id) == [prod.id]) + #expect(try #require(connectionStorage.loadConnection(id: prod.id)).localOnly == false) + } + + @Test func testConfirmingExcludeFromSyncMarksTheConnectionLocalOnly() throws { + let prod = DatabaseConnection(name: "Prod", type: .postgresql) + save([prod]) + viewModel.loadConnections() + viewModel.perform(.setIncludedInSync([prod.id], false)) + + viewModel.confirmExcludeFromSync() + + #expect(viewModel.connectionsToExcludeFromSync.isEmpty) + #expect(try #require(connectionStorage.loadConnection(id: prod.id)).localOnly) + } + + @Test func testIncludingInSyncNeedsNoConfirmation() throws { + var prod = DatabaseConnection(name: "Prod", type: .postgresql) + prod.localOnly = true + save([prod]) + viewModel.loadConnections() + + viewModel.perform(.setIncludedInSync([prod.id], true)) + + #expect(!viewModel.showExcludeFromSyncConfirmation) + #expect(try #require(connectionStorage.loadConnection(id: prod.id)).localOnly == false) + } + @Test func testDeletingTheLastConnectionClearsTheSearchItCanNoLongerRun() { let only = DatabaseConnection(name: "Only", type: .mysql, sortOrder: 0) save([only]) diff --git a/apple/TableProTests/Views/DataGrid/DataGridCommandTests.swift b/apple/TableProTests/Views/DataGrid/DataGridCommandTests.swift index f2f53cd346..565e4eba21 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridCommandTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridCommandTests.swift @@ -798,6 +798,67 @@ struct DataGridHeaderMenuTests { ]) } + @Test("The list Filter Values builds off the main actor is the one the main actor builds") + func distinctValuesOffTheMainActor() async throws { + let rows = TableRows.from( + queryRows: [ + [.text("2024-03-15 09:30:00.123456"), .text("1700000000"), .bytes(Data("signup".utf8)), .text("b")], + [.text("2024-03-15 09:30:00.123457"), .text("1700000000"), .bytes(Data([0x89, 0x50])), .null], + [.text("2024-03-15 09:30:00.123456"), .null, .bytes(Data("signup".utf8)), .text("a\rb")], + ], + columns: ["created", "stamp", "payload", "note"], + columnTypes: [ + .datetime(rawType: "DATETIME(6)"), + .integer(rawType: "BIGINT"), + .blob(rawType: "VARBINARY(255)"), + .text(rawType: "TEXT"), + ] + ) + let grid = CommandGrid(rows) + grid.model.setDisplayFormat(.unixTimestamp, for: try #require(grid.model.columns.column(at: 1))) + grid.model.setDisplayFormat(.text, for: try #require(grid.model.columns.column(at: 2))) + let created = try #require(grid.model.columns.column(at: 0)) + #expect(grid.commands.distinctValues(for: created).map { $0.count } == [2, 1]) + + for column in grid.model.columns.columns { + let offMain = await ColumnValueTally.distinctValuesOffTheMainActor( + grid.model.rows, + dataIndex: column.dataIndex, + columnType: column.type, + displayFormat: grid.model.displayFormat(for: column), + databaseType: grid.model.configuration.traits.databaseType, + dateFormat: DateFormattingService.shared.currentFormat + ) + #expect(offMain == grid.commands.distinctValues(for: column), "\(column.name)") + } + } + + @Test("Filter Values asks whether to stop every 4,096 rows and gives up when told to") + func valueTallyChecksForCancellation() { + let rows = TableRows.from( + queryRows: (0..<10_000).map { [PluginCellValue.text("\($0 % 7)")] }, + columns: ["v"], + columnTypes: [.text(rawType: "TEXT")] + ) + let dates = DateDisplayRenderer(format: .iso8601) + var checks = 0 + + let counted = ColumnValueTally.tally( + rows, dataIndex: 0, columnType: .text(rawType: "TEXT"), displayFormat: nil, databaseType: nil, dates: dates + ) { + checks += 1 + return false + } + let stopped = ColumnValueTally.tally( + rows, dataIndex: 0, columnType: .text(rawType: "TEXT"), displayFormat: nil, databaseType: nil, dates: dates, + isCancelled: { true } + ) + + #expect(counted?.order.count == 7) + #expect(checks == 3) + #expect(stopped == nil) + } + @Test("Fill Column is offered on a writable column that is not a key") func fillColumnOffer() { var configuration = CommandGrid.editable() diff --git a/apple/TableProTests/Views/DataGrid/DataGridControllerTests.swift b/apple/TableProTests/Views/DataGrid/DataGridControllerTests.swift index e423818a3e..abcad752eb 100644 --- a/apple/TableProTests/Views/DataGrid/DataGridControllerTests.swift +++ b/apple/TableProTests/Views/DataGrid/DataGridControllerTests.swift @@ -222,7 +222,7 @@ struct DataGridControllerTests { // MARK: - Find @Test("A find match moves the active cell to the match's column, so the arrow keys continue from it") - func findMatchSetsActiveColumn() { + func findMatchSetsActiveColumn() async { let grid = HostedDataGrid(rows: GridFixtureRows.people) defer { grid.close() } grid.model.setSelection(cell(0, 0)) @@ -230,6 +230,7 @@ struct DataGridControllerTests { grid.model.showFind() grid.model.find.term = "dee" grid.model.findQueryDidChange() + await grid.model.findSearchSettled() #expect(grid.model.selection.activeCell == GridCoord(row: 3, displayColumn: 1)) #expect(grid.main.selectedRowIndexes == IndexSet(integer: 3)) diff --git a/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift b/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift index ce095016eb..cee195562e 100644 --- a/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift +++ b/apple/TableProTests/Views/Main/ResultRowsBuildOffMainTests.swift @@ -160,6 +160,19 @@ struct ResultRowsBuildOffMainTests { #expect(result.tableRows.value(at: 1, column: 1) == .text("j•••@example.com")) } + @Test("A batch with more results than a run keeps shows the last ones and says how many there were") + func batchKeepsTheLastResults() async throws { + let count = RunResultPlan.maxRowResults + 2 + let answered = (1...count).map { ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["\($0)", "x@example.com"]]) } + + let results = try await runBatch(answeredWith: answered) + + #expect(results.count == RunResultPlan.maxRowResults) + #expect(results.first?.tableRows.value(at: 0, column: 0) == .text("3")) + #expect(results.last?.tableRows.value(at: 0, column: 0) == .text("\(count)")) + #expect(results.allSatisfy { $0.statusMessage?.contains(count.formatted()) == true }) + } + @Test("A batch's numbered result sets are each masked and keep their order") func batchNumberedResultsAreMasked() async throws { let first = ScriptAnsweringDriver.resultSet(columns: Self.columns, rows: [["1", "jane@example.com"]]) diff --git a/apple/TableProTests/Views/Results/InvisibleCharacterSettingTests.swift b/apple/TableProTests/Views/Results/InvisibleCharacterSettingTests.swift index 511e3343f5..9dbd8a9d6c 100644 --- a/apple/TableProTests/Views/Results/InvisibleCharacterSettingTests.swift +++ b/apple/TableProTests/Views/Results/InvisibleCharacterSettingTests.swift @@ -39,23 +39,23 @@ extension GlobalStateSuites { "Find matches a lone CR as the space the cell shows while marks are off", .appSettings { $0.dataGrid.showInvisibleCharacters = false } ) - func findReadsTheUnmarkedText() { + func findReadsTheUnmarkedText() async { let (model, owner) = makeModel(values: ["a\rb"]) defer { withExtendedLifetime(owner) {} } - #expect(find("a b", in: model).count == 1) + #expect(await find("a b", in: model).count == 1) } @Test( "Find does not read a marked CR as a space", .appSettings { $0.dataGrid.showInvisibleCharacters = true } ) - func findReadsTheMarkedText() { + func findReadsTheMarkedText() async { let (model, owner) = makeModel(values: ["a\rb"]) defer { withExtendedLifetime(owner) {} } - #expect(find("a b", in: model).isEmpty) - #expect(find("b", in: model).count == 1) + #expect(await find("a b", in: model).isEmpty) + #expect(await find("b", in: model).count == 1) } // The model holds its owner weakly, so the caller keeps the owner alive. @@ -74,10 +74,11 @@ extension GlobalStateSuites { return (model, owner) } - private func find(_ term: String, in model: DataGridModel) -> [GridFindState.Match] { + private func find(_ term: String, in model: DataGridModel) async -> [GridFindState.Match] { model.showFind() model.find.term = term model.findQueryDidChange() + await model.findSearchSettled() return model.find.matches } } diff --git a/apple/project.yml b/apple/project.yml index 41c0ec9abe..7c0b67e1d3 100644 --- a/apple/project.yml +++ b/apple/project.yml @@ -188,7 +188,7 @@ targetTemplates: PostgreSQLLinkage: dependencies: - package: TableProCore - products: [TableProCoreTypes, TableProGeometry, TableProKerberos, TableProLogRedaction, TableProPostgreSQLCore, TableProSQLVocabulary] + products: [TableProCoreTypes, TableProGeometry, TableProKerberos, TableProLogRedaction, TableProPostgreSQLCore, TableProSQLVocabulary, TableProTLSClientIdentity] settings: base: HEADER_SEARCH_PATHS: @@ -412,7 +412,7 @@ targets: - package: Yams - package: PostHog - package: TableProCore - products: [TableProAutocomplete, TableProConnectionLibrary, TableProCoreTypes, TableProEtcdCore, TableProFirebaseCore, TableProGeometry, TableProGoogleCloud, TableProGzip, TableProHTTP, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProNumberFormatting, TableProProcess, TableProQdrantCore, TableProRedisCore, TableProSQLGrammar, TableProSQLQuoting, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTabular, TableProTabularIO, TableProTheme, TableProWeaviateCore] + products: [TableProAutocomplete, TableProConnectionLibrary, TableProCoreTypes, TableProEtcdCore, TableProFirebaseCore, TableProGeometry, TableProGoogleCloud, TableProGzip, TableProHTTP, TableProImport, TableProInfluxDBCore, TableProLogRedaction, TableProNumberFormatting, TableProProcess, TableProQdrantCore, TableProRedisCore, TableProSQLGrammar, TableProSQLQuoting, TableProSQLVocabulary, TableProSSHTransport, TableProSyncTransport, TableProTLSClientIdentity, TableProTabular, TableProTabularIO, TableProTheme, TableProWeaviateCore] - package: TableProMacCore products: [TableProCloudAPI, TableProMCP, TableProSSHConfig, TableProUtilities] # Writes nothing for a version CHANGELOG.md has no section for, so a 0.0.0-dev build ships no notes. diff --git a/apple/scripts/probes/check-system-trust-anchors.sh b/apple/scripts/probes/check-system-trust-anchors.sh new file mode 100755 index 0000000000..075a0c1fb5 --- /dev/null +++ b/apple/scripts/probes/check-system-trust-anchors.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# +# Whether the PEM that SystemTrustAnchors builds from this Mac's trust settings lets the bundled OpenSSL verify a live +# server the way libpq's verify-full does: SSL_VERIFY_PEER plus SSL_set1_host. The bundled OpenSSL's own default store +# is checked too. Its OPENSSLDIR is the libs builder's temporary folder, so that check is expected to fail with +# "unable to get local issuer certificate". +# +# Builds the shipping SystemTrustAnchors.swift with a small main that writes the PEM, and a TLS client linked against +# apple/Libs/libssl.a and libcrypto.a (run apple/scripts/download-libs.sh first, or set LIBS_DIR). Needs the network. +# A host is any TLS server whose chain should verify against this Mac's roots; port 443 unless one is given. +# +# Usage: apple/scripts/probes/check-system-trust-anchors.sh [host[:port]]... (default: www.postgresql.org) +# Exit 0: every host verifies with the exported roots. Exit 1: one does not, or the probe could not run. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +LIBS="${LIBS_DIR:-$ROOT/Libs}" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +for lib in libssl.a libcrypto.a; do + if [[ ! -f "$LIBS/$lib" ]]; then + echo "Missing $LIBS/$lib. Run apple/scripts/download-libs.sh first." >&2 + exit 1 + fi +done + +cat > "$WORK/main.swift" << 'EOF' +import Foundation + +let source = SecuritySystemTrustSource() +let start = Date() +let pem = try SystemTrustAnchors.pem(from: source) +let elapsed = Int(Date().timeIntervalSince(start) * 1_000) +try pem.write(to: URL(fileURLWithPath: CommandLine.arguments[1])) +let roots = try SystemTrustAnchors.trustedRoots(from: source).count +print("exported \(roots) roots, \(pem.count) bytes, \(elapsed) ms") +EOF + +cat > "$WORK/check.c" << 'EOF' +#include +#include +#include +#include +#include + +const void *TLS_client_method(void); +void *SSL_CTX_new(const void *method); +void SSL_CTX_free(void *ctx); +int SSL_CTX_set_default_verify_paths(void *ctx); +int SSL_CTX_load_verify_locations(void *ctx, const char *file, const char *dir); +void SSL_CTX_set_verify(void *ctx, int mode, void *callback); +void *SSL_new(void *ctx); +void SSL_free(void *ssl); +int SSL_set_fd(void *ssl, int fd); +long SSL_ctrl(void *ssl, int cmd, long larg, void *parg); +int SSL_set1_host(void *ssl, const char *host); +int SSL_connect(void *ssl); +long SSL_get_verify_result(const void *ssl); +const char *X509_verify_cert_error_string(long n); + +// SSL_set_tlsext_host_name is a macro over SSL_ctrl; 55 is SSL_CTRL_SET_TLSEXT_HOSTNAME. +static int check(const char *label, const char *pem, const char *host, const char *port) { + void *ctx = SSL_CTX_new(TLS_client_method()); + if (pem) { + if (SSL_CTX_load_verify_locations(ctx, pem, NULL) != 1) { + printf(" %-17s could not load %s\n", label, pem); + return 0; + } + } else { + SSL_CTX_set_default_verify_paths(ctx); + } + SSL_CTX_set_verify(ctx, 1, NULL); + + struct addrinfo hints = {0}, *res = NULL; + hints.ai_socktype = SOCK_STREAM; + if (getaddrinfo(host, port, &hints, &res) != 0) { + printf(" %-17s cannot resolve %s\n", label, host); + return 0; + } + int fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); + if (connect(fd, res->ai_addr, res->ai_addrlen) != 0) { + printf(" %-17s cannot connect to %s:%s\n", label, host, port); + freeaddrinfo(res); + return 0; + } + freeaddrinfo(res); + void *ssl = SSL_new(ctx); + SSL_set_fd(ssl, fd); + SSL_ctrl(ssl, 55, 0, (void *)host); + SSL_set1_host(ssl, host); + int handshake = SSL_connect(ssl); + long verify = SSL_get_verify_result(ssl); + printf(" %-17s handshake %s, verify %ld (%s)\n", label, handshake == 1 ? "ok" : "failed", verify, + X509_verify_cert_error_string(verify)); + SSL_free(ssl); + close(fd); + SSL_CTX_free(ctx); + return handshake == 1 && verify == 0; +} + +int main(int argc, char **argv) { + if (argc < 4) return 2; + check("default store", NULL, argv[2], argv[3]); + return check("exported roots", argv[1], argv[2], argv[3]) ? 0 : 1; +} +EOF + +arch="$(uname -m)" +xcrun swiftc -swift-version 6 -target "$arch-apple-macos13.0" -o "$WORK/export-roots" \ + "$ROOT/Packages/TableProCore/Sources/TableProTLSClientIdentity/SystemTrustAnchors.swift" "$WORK/main.swift" +clang -arch "$arch" -mmacosx-version-min=13.0 -o "$WORK/check" "$WORK/check.c" \ + "$LIBS/libssl.a" "$LIBS/libcrypto.a" -lz + +"$WORK/export-roots" "$WORK/roots.pem" + +status=0 +for target in "${@:-www.postgresql.org}"; do + host="${target%%:*}" + port="443" + [[ "$target" == *:* ]] && port="${target##*:}" + echo "== $host:$port" + "$WORK/check" "$WORK/roots.pem" "$host" "$port" || status=1 +done +exit "$status" From fbba6172a271e4e007bb5b2988512ec31e54bf61 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 11 Nov 2026 22:06:45 +0700 Subject: [PATCH 5/5] docs(macos): add the changelog fragments for #4200 --- apple/changelog.d/4200.added.md | 1 + apple/changelog.d/4200.changed.md | 1 + apple/changelog.d/4200.fixed.1.md | 1 + apple/changelog.d/4200.fixed.2.md | 1 + apple/changelog.d/4200.fixed.3.md | 1 + apple/changelog.d/4200.fixed.4.md | 1 + apple/changelog.d/4200.fixed.5.md | 1 + apple/changelog.d/4200.fixed.6.md | 1 + apple/changelog.d/4200.fixed.md | 1 + apple/changelog.d/4200.security.md | 1 + 10 files changed, 10 insertions(+) create mode 100644 apple/changelog.d/4200.added.md create mode 100644 apple/changelog.d/4200.changed.md create mode 100644 apple/changelog.d/4200.fixed.1.md create mode 100644 apple/changelog.d/4200.fixed.2.md create mode 100644 apple/changelog.d/4200.fixed.3.md create mode 100644 apple/changelog.d/4200.fixed.4.md create mode 100644 apple/changelog.d/4200.fixed.5.md create mode 100644 apple/changelog.d/4200.fixed.6.md create mode 100644 apple/changelog.d/4200.fixed.md create mode 100644 apple/changelog.d/4200.security.md diff --git a/apple/changelog.d/4200.added.md b/apple/changelog.d/4200.added.md new file mode 100644 index 0000000000..5ea239390d --- /dev/null +++ b/apple/changelog.d/4200.added.md @@ -0,0 +1 @@ +PostgreSQL connections set to Verify Identity check the server against this Mac's trusted roots without a CA file. diff --git a/apple/changelog.d/4200.changed.md b/apple/changelog.d/4200.changed.md new file mode 100644 index 0000000000..0cebd82ff0 --- /dev/null +++ b/apple/changelog.d/4200.changed.md @@ -0,0 +1 @@ +Run All shows statements that return no rows as one result, and keeps the last 100 results that return rows. diff --git a/apple/changelog.d/4200.fixed.1.md b/apple/changelog.d/4200.fixed.1.md new file mode 100644 index 0000000000..c57c00f578 --- /dev/null +++ b/apple/changelog.d/4200.fixed.1.md @@ -0,0 +1 @@ +Filter Values on a large column froze the window while it listed the values. diff --git a/apple/changelog.d/4200.fixed.2.md b/apple/changelog.d/4200.fixed.2.md new file mode 100644 index 0000000000..bd442f13c9 --- /dev/null +++ b/apple/changelog.d/4200.fixed.2.md @@ -0,0 +1 @@ +Grid edits to a row after another row's change saved out of order, failing saves such as a swap of unique values. diff --git a/apple/changelog.d/4200.fixed.3.md b/apple/changelog.d/4200.fixed.3.md new file mode 100644 index 0000000000..eb198d3b4d --- /dev/null +++ b/apple/changelog.d/4200.fixed.3.md @@ -0,0 +1 @@ +A sidebar Refresh held up autocomplete, and each schema of a database kept its own background connection. diff --git a/apple/changelog.d/4200.fixed.4.md b/apple/changelog.d/4200.fixed.4.md new file mode 100644 index 0000000000..9922a5daf1 --- /dev/null +++ b/apple/changelog.d/4200.fixed.4.md @@ -0,0 +1 @@ +After a dropped connection reconnected, statements ran without the temporary tables or settings the old session had. diff --git a/apple/changelog.d/4200.fixed.5.md b/apple/changelog.d/4200.fixed.5.md new file mode 100644 index 0000000000..ffb422e736 --- /dev/null +++ b/apple/changelog.d/4200.fixed.5.md @@ -0,0 +1 @@ +JSON import read UTF-16 and UTF-32 files, and files holding a single object, whole into memory. diff --git a/apple/changelog.d/4200.fixed.6.md b/apple/changelog.d/4200.fixed.6.md new file mode 100644 index 0000000000..cf51aee607 --- /dev/null +++ b/apple/changelog.d/4200.fixed.6.md @@ -0,0 +1 @@ +A chat reply cut off at the model's output limit gave no sign of it, and a tool call cut with it could run. diff --git a/apple/changelog.d/4200.fixed.md b/apple/changelog.d/4200.fixed.md new file mode 100644 index 0000000000..e2f2be9b9e --- /dev/null +++ b/apple/changelog.d/4200.fixed.md @@ -0,0 +1 @@ +Typing a Find term in a large grid froze the window while every cell was searched. diff --git a/apple/changelog.d/4200.security.md b/apple/changelog.d/4200.security.md new file mode 100644 index 0000000000..8497b232cf --- /dev/null +++ b/apple/changelog.d/4200.security.md @@ -0,0 +1 @@ +Marking a connection Local only left its passwords in iCloud Keychain on your other devices.