From 8526b15cb9dc57487f180b82144592728136ed67 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Tue, 10 Nov 2026 13:46:01 +0700 Subject: [PATCH 1/3] fix(macos): stop the data loss, wrong writes and gate bypasses the code audit found --- .claude/rules/data-sync-storage.md | 3 + apple/Packages/TableProCore/Package.swift | 5 + .../KeywordUppercaseHelper.swift | 99 +---- .../DocumentPathPlacement.swift | 126 +++++++ .../DuckDBFetchBudget.swift | 85 +++++ .../TableProEtcdCore/EtcdGuardedWrite.swift | 134 +++++++ .../TableProSQLGrammar/SQLOperatorRun.swift | 48 +++ .../SnowflakeSessionRenewal.swift | 22 ++ .../SyncMetadataStorage.swift | 16 + .../DelimitedFieldReader.swift | 21 ++ .../KeywordUppercaseHelperTests.swift | 110 ++++-- .../DocumentPathPlacementTests.swift | 95 +++++ .../DuckDBBoundedStreamTests.swift | 44 +++ .../DuckDBFetchBudgetTests.swift | 70 ++++ .../EtcdGuardedWriteTests.swift | 77 ++++ .../PluginExportDocumentTests.swift | 75 ++++ .../SQLOperatorRunTests.swift | 48 +++ .../SnowflakeSessionRenewalTests.swift | 96 +++++ .../SourceEditor+Coordinator.swift | 13 + .../SourceEditor/TextBindingSync.swift | 24 ++ .../SourceEditorBindingSyncTests.swift | 38 ++ .../Outside/MCPServerConfiguration.swift | 37 +- .../Outside/MCPServerToolNameTests.swift | 63 ++++ .../CSVImportPlugin/CSVImportParsing.swift | 26 ++ .../CSVImportPlugin/CSVImportPlugin.swift | 48 ++- .../ColumnTypeCorpus/Corpus/mariadb.json | 3 +- .../ColumnTypeCorpus/Corpus/mysql.json | 3 +- .../DuckDBDriverPlugin/DuckDBConnection.swift | 332 +++++++++++------ .../DuckDBDriverPlugin/DuckDBPlugin.swift | 28 +- .../DuckDBResultReader.swift | 72 ++-- .../ElasticsearchMappingFlattener.swift | 23 +- .../ElasticsearchPluginDriver+Execution.swift | 42 ++- .../ElasticsearchPluginDriver.swift | 12 +- .../ElasticsearchStatementGenerator.swift | 113 +++++- .../EtcdDriverPlugin/EtcdCommandParser.swift | 48 ++- .../EtcdDriverPlugin/EtcdHttpClient.swift | 4 + .../EtcdDriverPlugin/EtcdPluginDriver.swift | 38 ++ .../EtcdStatementGenerator.swift | 42 ++- .../CSVImportRowShapeTests.swift | 149 ++++++++ .../SQLExportBitLiteralTests.swift | 50 +++ .../XLSXWriterCellTypeTests.swift | 173 +++++++++ .../MongoDBStatementGenerator.swift | 19 +- .../DuckDBStagingDatabase.swift | 165 ++++++--- .../ParquetExportPlugin.swift | 25 +- .../ParquetExportPlugin/ParquetStaging.swift | 51 +++ .../ParquetTypeMapper.swift | 40 +- .../LibPQPluginConnection.swift | 35 +- .../PostgreSQLPluginDriver.swift | 20 +- .../PostgreSQLTableDDL.swift | 175 +++++++++ .../LibPQStreamHeaderTests.swift | 47 +++ .../PostgreSQLTableDDLTests.swift | 170 +++++++++ .../RedisCommandParser.swift | 31 +- .../RedisConditionalWrite.swift | 46 +++ .../RedisPluginDriver+Operations.swift | 10 +- .../RedisDriverPlugin/RedisPluginDriver.swift | 28 +- .../RedisStatementGenerator.swift | 66 +++- .../RedisTransactionOutcome.swift | 14 +- .../RedisBinaryValueTests.swift | 9 +- .../RedisCommandParserTests.swift | 42 ++- .../RedisConditionalWriteTests.swift | 119 ++++++ .../RedisStatementGeneratorTests.swift | 58 ++- .../SQLExportRowValueEncoder.swift | 19 + .../SnowflakeConnection.swift | 114 ++++-- .../SurrealCellCoder.swift | 41 ++- .../SurrealDBPluginDriver.swift | 27 +- .../SurrealStatementGenerator.swift | 26 +- .../PluginExportDocument.swift | 95 ++--- .../XLSXExportPlugin/XLSXExportPlugin.swift | 7 +- .../Plugins/XLSXExportPlugin/XLSXWriter.swift | 152 +++++++- apple/TablePro/AppDelegate.swift | 1 + apple/TablePro/Core/AI/AIProvider.swift | 15 + .../TablePro/Core/AI/AnthropicProvider.swift | 5 +- .../Core/AI/Chat/ChatToolCallRepair.swift | 73 ++++ .../TablePro/Core/AI/Chat/ChatTransport.swift | 7 +- .../AI/ClaudeAgent/ClaudeAgentMCPBridge.swift | 17 +- .../AI/ClaudeAgent/ClaudeAgentProvider.swift | 2 +- apple/TablePro/Core/AI/GeminiProvider.swift | 10 +- .../Core/AI/OpenAICompatibleProvider.swift | 34 +- .../Core/AI/OpenAIResponsesProvider.swift | 5 +- apple/TablePro/Core/AI/SSEEventStream.swift | 11 +- .../ChangeTracking/DataChangeManager.swift | 101 +++++- .../Core/ChangeTracking/PendingChanges.swift | 62 +++- .../SQLStatementGenerator.swift | 87 +++-- .../QueryExecutionCoordinator+Batches.swift | 12 +- .../QueryExecutionCoordinator+Helpers.swift | 19 +- ...yExecutionCoordinator+MultiStatement.swift | 10 +- ...QueryExecutionCoordinator+Parameters.swift | 28 +- .../QueryExecutionCoordinator.swift | 69 +++- .../RowEditingCoordinator+SaveChanges.swift | 117 ++++-- .../CrossEngine/CrossEngineValueCoercer.swift | 34 +- .../Core/DataWrite/CascadeDeletePreview.swift | 4 +- .../DataWrite/RowChangeStatementFactory.swift | 1 + .../Access/DatabaseAccessBridge+Scripts.swift | 3 +- .../Access/DatabaseAccessBridge.swift | 114 +++++- .../Database/DatabaseManager+Health.swift | 15 + .../Database/DatabaseManager+Metadata.swift | 1 + .../DatabaseManager+ScopedDriver.swift | 183 ++++++++-- .../Database/DatabaseManager+Sessions.swift | 13 +- .../TablePro/Core/Database/SessionLanes.swift | 60 +++ .../Core/Database/TriggerEditing.swift | 178 +++++++-- .../DatabaseSearch/DatabaseSearchReader.swift | 4 +- .../Completions/MCPCompletionProvider.swift | 2 + .../MCPCompletionSchemaSource.swift | 5 + apple/TablePro/Core/MCP/MCPAuthPolicy.swift | 8 + apple/TablePro/Core/MCP/MCPTokenStore.swift | 142 ++++++-- .../Outside/MCPRemoteToolCoordinator.swift | 10 + .../Core/Plugins/PluginDriverAdapter.swift | 21 +- .../Plugins/PluginManager+Registration.swift | 4 +- .../StructureChangeManager.swift | 75 ++++ .../Core/SchemaTracking/StructureRebase.swift | 167 +++++++++ .../Services/Formatting/SQLTokenizer.swift | 53 ++- .../Infrastructure/ConnectionWorkspace.swift | 1 + .../MainSplitViewController.swift | 2 +- .../TabPersistenceCoordinator.swift | 6 + .../Infrastructure/TabWindowController.swift | 3 + .../Masking/MaskingExportDataSource.swift | 113 +++--- .../Core/Services/Query/QueryExecutor.swift | 6 +- .../Services/Query/RowOperationsManager.swift | 21 +- .../Core/Services/Query/RowParser.swift | 7 +- .../Shortcuts/Shared/IntentDataError.swift | 3 + .../Core/Shortcuts/Shared/RowPayload.swift | 20 +- .../Core/Storage/ConnectionStorage.swift | 224 ++++++++++-- .../Storage/ConnectionStoreIntegrity.swift | 93 ++++- .../Storage/IntegrityStampedFileStore.swift | 5 +- .../Core/Storage/KeychainHelper.swift | 164 +++++++-- .../SyncCoordinator+DeferredDeletions.swift | 45 +++ .../TablePro/Core/Sync/SyncCoordinator.swift | 10 +- .../Core/Utilities/SQL/QueryClassifier.swift | 4 +- .../Utilities/SQL/SQLParameterExtractor.swift | 189 +++------- .../TablePro/Models/Query/QueryTabState.swift | 3 + .../TablePro/Resources/Localizable.xcstrings | 131 ++++++- .../AIChatViewModel+Streaming.swift | 32 +- .../ConnectionFormCoordinator.swift | 226 ++++++------ .../ViewModels/SSLPaneViewModel.swift | 3 +- .../Views/Editor/EditorEventRouter.swift | 10 + .../Views/Editor/SQLEditorCoordinator.swift | 4 +- ...ainContentCoordinator+AIQueryActions.swift | 1 + .../MainContentCoordinator+Alerts.swift | 2 + .../MainContentCoordinator+ExecuteAll.swift | 34 +- .../MainContentCoordinator+Explain.swift | 12 + .../MainContentCoordinator+Navigation.swift | 9 +- .../MainContentCoordinator+TabSwitch.swift | 31 +- .../Main/MainContentCommandActions.swift | 2 + .../Views/Main/MainContentCoordinator.swift | 81 ++++- ...abaseTreeOutlineCoordinator+Commands.swift | 7 +- .../DatabaseTreeOutlineCoordinator.swift | 7 +- .../StructureEditingSession+Apply.swift | 56 ++- .../StructureEditingSession+Loading.swift | 35 ++ .../StructureRebuildPlanRunner.swift | 21 +- .../Views/Structure/TriggerDetailView.swift | 48 ++- .../Views/Structure/TriggerEditorView.swift | 28 +- .../MongoDBStringFieldWriteTests.swift | 77 ++++ .../AI/ClaudeAgentConnectionScopeTests.swift | 62 ++++ .../Core/AI/ProviderStreamFailureTests.swift | 108 ++++++ .../KeywordVocabularyParityTests.swift | 5 +- .../EditedRowChangeTrackingTests.swift | 252 +++++++++++++ .../SQLStatementGeneratorBitColumnTests.swift | 174 +++++++++ .../CrossEngineValueCoercerTests.swift | 45 ++- .../CredentialLeaseRoutingTests.swift | 21 ++ .../ProtectedWriteDeadlineTests.swift | 176 +++++++++ .../Database/ScopedDriverRoutingTests.swift | 17 +- .../Core/Database/SessionLanesTests.swift | 249 ++++++++++++- .../Database/SessionRecoveryLoopTests.swift | 43 +++ .../SwitchDatabasePooledConnectionTests.swift | 17 + .../Database/TriggerInfoMappingTests.swift | 169 +++++++++ .../MCPCompletionApprovalGateTests.swift | 166 +++++++++ .../MCPCompletionProviderTests.swift | 10 + .../Core/MCP/MCPTokenStoreRecoveryTests.swift | 234 ++++++++++++ .../Core/MCP/MCPTokenStoreTests.swift | 4 +- .../MCP/Prompts/PromptsHandlerTests.swift | 1 + .../PluginDriverAdapterBrowseProbeTests.swift | 116 ++++++ .../StructureChangeRebaseTests.swift | 148 ++++++++ .../TabPersistenceSessionGateTests.swift | 81 +++++ .../Services/Masking/MaskingExportTests.swift | 26 ++ .../Services/RowPasteForeignTextTests.swift | 178 +++++++++ .../Services/SQLFormatterServiceTests.swift | 21 ++ .../Core/Services/SQLTokenizerTests.swift | 50 +++ .../RowPayloadCSVEmptyCellTests.swift | 62 ++++ .../Storage/ConnectionSecretScopeTests.swift | 341 ++++++++++++++++++ .../ConnectionStoreTagRemovalTests.swift | 122 +++++++ .../Core/Storage/KeychainItemScopeTests.swift | 201 +++++++++++ .../Core/Sync/SyncCategoryRefetchTests.swift | 101 +++++- .../SQL/EtcdCommandParserTests.swift | 62 +++- .../SQL/QueryClassifierHardeningTests.swift | 13 + .../SQLParameterExtractorTests.swift | 58 +++ .../Helpers/HeldExecutionGate.swift | 29 ++ .../AIChatStoppedToolRoundTests.swift | 290 +++++++++++++++ .../Views/Main/AuthorizedRunTargetTests.swift | 73 ++++ .../Views/Main/ParkedEditSnapshotTests.swift | 199 ++++++++++ .../Views/Main/RunDiscardGuardTests.swift | 55 +++ .../Views/Main/SaveAttributionTests.swift | 131 +++++++ .../FailedContainerSwitchCommandTests.swift | 77 ++++ ...StructureRebuildOpenTransactionTests.swift | 88 +++++ .../probes/check-duckdb-streaming-results.sh | 221 ++++++++++++ .../probes/check-parquet-export-values.sh | 224 ++++++++++++ 195 files changed, 11135 insertions(+), 1349 deletions(-) create mode 100644 apple/Packages/TableProCore/Sources/TableProDocumentPath/DocumentPathPlacement.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProDuckDBCore/DuckDBFetchBudget.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdGuardedWrite.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLOperatorRun.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProSnowflakeCore/SnowflakeSessionRenewal.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProDocumentPathTests/DocumentPathPlacementTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBBoundedStreamTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBFetchBudgetTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdGuardedWriteTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLOperatorRunTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProSnowflakeCoreTests/SnowflakeSessionRenewalTests.swift create mode 100644 apple/Packages/TableProMacCore/Tests/TableProMCPTests/Outside/MCPServerToolNameTests.swift create mode 100644 apple/Plugins/FormatPluginTests/CSVImportRowShapeTests.swift create mode 100644 apple/Plugins/FormatPluginTests/SQLExportBitLiteralTests.swift create mode 100644 apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift create mode 100644 apple/Plugins/ParquetExportPlugin/ParquetStaging.swift create mode 100644 apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift create mode 100644 apple/Plugins/PostgreSQLDriverTests/LibPQStreamHeaderTests.swift create mode 100644 apple/Plugins/PostgreSQLDriverTests/PostgreSQLTableDDLTests.swift create mode 100644 apple/Plugins/RedisDriverPlugin/RedisConditionalWrite.swift create mode 100644 apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift create mode 100644 apple/TablePro/Core/AI/Chat/ChatToolCallRepair.swift create mode 100644 apple/TablePro/Core/SchemaTracking/StructureRebase.swift create mode 100644 apple/TablePro/Core/Sync/Extensions/SyncCoordinator+DeferredDeletions.swift create mode 100644 apple/TableProMobile/TableProMobileTests/Drivers/MongoDBStringFieldWriteTests.swift create mode 100644 apple/TableProTests/Core/AI/ClaudeAgentConnectionScopeTests.swift create mode 100644 apple/TableProTests/Core/AI/ProviderStreamFailureTests.swift create mode 100644 apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift create mode 100644 apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift create mode 100644 apple/TableProTests/Core/Database/ProtectedWriteDeadlineTests.swift create mode 100644 apple/TableProTests/Core/MCP/Completions/MCPCompletionApprovalGateTests.swift create mode 100644 apple/TableProTests/Core/MCP/MCPTokenStoreRecoveryTests.swift create mode 100644 apple/TableProTests/Core/Plugins/PluginDriverAdapterBrowseProbeTests.swift create mode 100644 apple/TableProTests/Core/SchemaTracking/StructureChangeRebaseTests.swift create mode 100644 apple/TableProTests/Core/Services/Infrastructure/TabPersistenceSessionGateTests.swift create mode 100644 apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift create mode 100644 apple/TableProTests/Core/Shortcuts/RowPayloadCSVEmptyCellTests.swift create mode 100644 apple/TableProTests/Core/Storage/ConnectionSecretScopeTests.swift create mode 100644 apple/TableProTests/Core/Storage/ConnectionStoreTagRemovalTests.swift create mode 100644 apple/TableProTests/Core/Storage/KeychainItemScopeTests.swift create mode 100644 apple/TableProTests/Helpers/HeldExecutionGate.swift create mode 100644 apple/TableProTests/ViewModels/AIChatStoppedToolRoundTests.swift create mode 100644 apple/TableProTests/Views/Main/AuthorizedRunTargetTests.swift create mode 100644 apple/TableProTests/Views/Main/ParkedEditSnapshotTests.swift create mode 100644 apple/TableProTests/Views/Main/RunDiscardGuardTests.swift create mode 100644 apple/TableProTests/Views/Main/SaveAttributionTests.swift create mode 100644 apple/TableProTests/Views/Sidebar/FailedContainerSwitchCommandTests.swift create mode 100644 apple/TableProTests/Views/Structure/StructureRebuildOpenTransactionTests.swift create mode 100755 apple/scripts/probes/check-duckdb-streaming-results.sh create mode 100755 apple/scripts/probes/check-parquet-export-values.sh diff --git a/.claude/rules/data-sync-storage.md b/.claude/rules/data-sync-storage.md index abd611cd4e..bb1ad2dfd1 100644 --- a/.claude/rules/data-sync-storage.md +++ b/.claude/rules/data-sync-storage.md @@ -178,6 +178,9 @@ This path holds user data. A silent wrong answer here looks exactly like an empt | Approved environment variable references, per connection, names and destination digest | UserDefaults, device-local, never synced | `ConnectionApprovalStore.envVarReferences` | | Approved sign-ins with this Mac's own credentials (Entra, AWS profile or SSO, Google ADC or OAuth, Kerberos, `~/.pgpass`), per connection, field values and destination digest | UserDefaults `com.TablePro.localCredentialApprovals`, plus `com.TablePro.localCredentialApprovalsMigrated` for the one-time approval of this Mac's rows; device-local, never synced | `ConnectionApprovalStore.localCredentials` | | Snowflake SSO and MFA tokens, per host and user. A bare account's `NAME.USER` item stays for 0.83.1, which reads only that key | Keychain, device-local | `SnowflakeTokenCache` | +| Whether `connections.json` was ever given an integrity tag, so a deleted tag reads as modified | Keychain `com.TablePro.connectionStoreStamped.`, device-local | `ConnectionStoreIntegrity` | +| Whether Local only connections' secrets were copied to device-only items | UserDefaults `com.TablePro.localOnlySecretsKeptOnDevice`, device-local, never synced | `ConnectionStorage` | +| Deletions the server reported for a sync category that is off, applied when the category is next read | UserDefaults, device-local, never synced | `SyncMetadataStorage.deferredDeletions` | | Where a connection came from (`origin`): this Mac, imported, iCloud | `connections.json`, never synced or exported | `ConnectionStorage` | | Table load timings (no identifying data, never uploaded) | JSON lines | `TableLoadHistoryStore` | | Reopen Last Session list (`LastSession.json`, plus `LastOpenConnections.json` holding only the ids that reconnect, which older builds read) | JSON | `LastOpenConnectionsStorage` | diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index d756a12c8c..7b85a5049e 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -577,6 +577,11 @@ let package = Package( path: "Tests/TableProMemcachedCoreTests", swiftSettings: pluginCoreSettings ), + .testTarget( + name: "TableProDocumentPathTests", + dependencies: ["TableProDocumentPath"], + path: "Tests/TableProDocumentPathTests" + ), .testTarget( name: "TableProRedisCoreTests", dependencies: ["TableProRedisCore"], diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/KeywordUppercaseHelper.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/KeywordUppercaseHelper.swift index 5ba31dc999..fb8772eac6 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/KeywordUppercaseHelper.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/KeywordUppercaseHelper.swift @@ -1,4 +1,5 @@ import Foundation +import TableProSQLGrammar public enum KeywordUppercaseHelper { public static func isWordBoundary(_ string: String) -> Bool { @@ -11,99 +12,39 @@ public enum KeywordUppercaseHelper { } } - /// Checks if a UTF-16 character is part of a SQL identifier (a-z, A-Z, 0-9, _). static func isWordCharacter(_ ch: unichar) -> Bool { (ch >= 0x41 && ch <= 0x5A) || (ch >= 0x61 && ch <= 0x7A) || (ch >= 0x30 && ch <= 0x39) || ch == 0x5F } - /// Scans backwards up to 2,000 characters to determine if `position` is inside - /// a protected context (string literal, comment, backtick identifier, dollar-quote). - /// Keywords inside protected contexts should NOT be uppercased. - static func isInsideProtectedContext(_ text: NSString, at position: Int) -> Bool { - let scanStart = max(0, position - 2_000) - var inSingleQuote = false - var inDoubleQuote = false - var inBacktick = false - var inLineComment = false - var inBlockComment = false - var inDollarQuote = false - var i = scanStart - - while i < position { - let ch = text.character(at: i) - - if inBlockComment { - if ch == 0x2A && i + 1 < position && text.character(at: i + 1) == 0x2F { - inBlockComment = false - i += 2 - continue - } - i += 1 - continue - } - if inLineComment { - if ch == 0x0A { inLineComment = false } - i += 1 - continue - } - if inDollarQuote { - if ch == 0x24 && i + 1 < position && text.character(at: i + 1) == 0x24 { - inDollarQuote = false - i += 2 - continue - } - i += 1 + // Read from the start of the text with the session's grammar: a literal can open any distance back, and where it + // ends is the engine's rule. + static func isInsideProtectedContext(_ text: NSString, at position: Int, grammar: SQLLexicalGrammar) -> Bool { + let length = text.length + let runsExecutableComments = grammar.contains(.executableComments) + var index = 0 + while index < position { + // The server runs the body of `/*! ... */`, so it is code. + if runsExecutableComments, let opener = SqlLexer.executableCommentOpenerLength(text, at: index, length: length) { + index += opener continue } - - if ch == 0x5C && (inSingleQuote || inDoubleQuote) { - i += 2 + guard let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) else { + index += 1 continue } - - switch ch { - case 0x27: if !inDoubleQuote && !inBacktick { inSingleQuote.toggle() } - case 0x22: if !inSingleQuote && !inBacktick { inDoubleQuote.toggle() } - case 0x60: if !inSingleQuote && !inDoubleQuote { inBacktick.toggle() } - case 0x23: - if !inSingleQuote && !inDoubleQuote && !inBacktick { - inLineComment = true - } - case 0x2D: - if !inSingleQuote && !inDoubleQuote && !inBacktick && - i + 1 < position && text.character(at: i + 1) == 0x2D { - inLineComment = true - i += 2 - continue - } - case 0x2F: - if !inSingleQuote && !inDoubleQuote && !inBacktick && - i + 1 < position && text.character(at: i + 1) == 0x2A { - inBlockComment = true - i += 2 - continue - } - case 0x24: - if !inSingleQuote && !inDoubleQuote && !inBacktick && - i + 1 < position && text.character(at: i + 1) == 0x24 { - inDollarQuote.toggle() - i += 2 - continue - } - default: break - } - i += 1 + if SQLContextAnalyzer.holds(span, cursor: position) { return true } + index = max(span.end, index + 1) } - - return inSingleQuote || inDoubleQuote || inBacktick || inLineComment || inBlockComment || inDollarQuote + return false } - /// Nil when the word is not a keyword, sits in a protected context, or is already in the target case. + // Nil when the word is not a keyword, sits in a comment, a literal or a quoted name, or is already in the target case. public static func keywordBeforePosition( _ text: NSString, at position: Int, - uppercase: Bool = true + uppercase: Bool = true, + grammar: SQLLexicalGrammar ) -> (word: String, folded: String, range: NSRange)? { var wordStart = position while wordStart > 0 { @@ -117,7 +58,7 @@ public enum KeywordUppercaseHelper { let word = text.substring(with: NSRange(location: wordStart, length: wordLength)) guard SQLKeywords.keywordSet.contains(word.lowercased()) else { return nil } - guard !isInsideProtectedContext(text, at: wordStart) else { return nil } + guard !isInsideProtectedContext(text, at: wordStart, grammar: grammar) else { return nil } let folded = uppercase ? word.uppercased() : word.lowercased() guard folded != word else { return nil } diff --git a/apple/Packages/TableProCore/Sources/TableProDocumentPath/DocumentPathPlacement.swift b/apple/Packages/TableProCore/Sources/TableProDocumentPath/DocumentPathPlacement.swift new file mode 100644 index 0000000000..c9ef6d3d6e --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDocumentPath/DocumentPathPlacement.swift @@ -0,0 +1,126 @@ +// +// DocumentPathPlacement.swift +// TableProDocumentPath +// +// Builds the partial document that sets one leaf of a stored JSON document. +// + +import Foundation + +public enum DocumentPathPlacement { + public struct InsideArray: Error, Equatable { + public let path: String + } + + @discardableResult + public static func set(_ value: Any, atKeys keys: [String], in document: inout [String: Any]) -> Bool { + guard let key = keys.first else { return false } + guard keys.count > 1 else { + document[key] = value + return true + } + var child: [String: Any] + switch document[key] { + case nil: + child = [:] + case let existing as [String: Any]: + child = existing + default: + return false + } + guard set(value, atKeys: Array(keys.dropFirst()), in: &child) else { return false } + document[key] = child + return true + } + + public static func value(in document: [String: Any], atKeys keys: [String]) -> Any? { + var current: Any = document + for key in keys { + guard let dictionary = current as? [String: Any], let next = dictionary[key] else { return nil } + current = next + } + return keys.isEmpty ? nil : current + } + + // A stored document can spell one field as nested objects, as a literal dotted key, or as a + // mix, and a merge matches keys literally. The value goes to every spelling the document + // already holds; a field it does not hold yet is added under the deepest object that exists, + // split where `keys` splits. + public static func place( + _ value: Any, + atKeys keys: [String], + in partial: inout [String: Any], + following stored: [String: Any] + ) throws { + var atoms: [String] = [] + var boundaries: Set = [] + for key in keys { + atoms.append(contentsOf: key.split(separator: ".", omittingEmptySubsequences: false).map(String.init)) + boundaries.insert(atoms.count) + } + guard !atoms.isEmpty else { return } + let path = atoms.joined(separator: ".") + + let spellings = try storedSpellings(of: atoms[...], in: stored, path: path) + guard spellings.isEmpty else { + for spelling in spellings { + set(value, atKeys: spelling, in: &partial) + } + return + } + + var consumed: [String] = [] + var start = 0 + var level = stored + while let step = firstObject(in: level, atoms: atoms, from: start) { + consumed.append(step.key) + start = step.end + level = step.object + } + + var group: [String] = [] + for index in start.., + in level: [String: Any], + path: String + ) throws -> [[String]] { + var spellings: [[String]] = [] + for end in atoms.indices.map({ $0 + 1 }) { + let key = atoms[atoms.startIndex.. (key: String, end: Int, object: [String: Any])? { + guard start < atoms.count - 1 else { return nil } + for end in (start + 1).. Int { + let room = limit - kept + guard count > room else { + kept += max(0, count) + return max(0, count) + } + kept = limit + isExceeded = true + return room + } +} + +// What happens to the rest of a result the reader stops in the middle of. +public enum DuckDBUnreadRows: Sendable, Equatable { + case drain + case abandon + case abandonWhenIdle + + // Dropping an unfinished statement that may write aborts the transaction it runs in, so only a session known to + // hold none drops one. + public func abandons(sessionIsIdle: () -> Bool) -> Bool { + switch self { + case .drain: false + case .abandon: true + case .abandonWhenIdle: sessionIsIdle() + } + } +} + +// The session settings that change how DuckDB spells a value as text, which a second connection rendering the first +// one's rows has to share. +public enum DuckDBRenderSettings { + public static let names = ["TimeZone", "Calendar"] + + public static let readQuery = + "SELECT name, value FROM duckdb_settings() WHERE name IN ('TimeZone', 'Calendar')" + + public static func assignments(from current: [String: String], to wanted: [String: String]) -> [String] { + names.compactMap { name in + guard let value = wanted[name], current[name] != value else { return nil } + return "SET \(name) = '\(value.replacingOccurrences(of: "'", with: "''", options: .literal))'" + } + } +} + +// Hands a row batch to a stream whose buffer is bounded. A buffering policy alone drops what does not fit, so the +// producer keeps the refused batch and offers it again once the consumer has read one. +public enum DuckDBBoundedStream { + public static let bufferedBatches = 64 + + // Returns false when the stream has ended and the producer should stop. + public static func deliver( + _ element: Element, + to continuation: AsyncThrowingStream.Continuation, + pause: () -> Void = { _ = usleep(2_000) } + ) -> Bool { + var pending = element + while true { + switch continuation.yield(pending) { + case .enqueued: + return true + case .dropped(let refused): + pending = refused + pause() + case .terminated: + return false + @unknown default: + return false + } + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdGuardedWrite.swift b/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdGuardedWrite.swift new file mode 100644 index 0000000000..dc160454e5 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProEtcdCore/EtcdGuardedWrite.swift @@ -0,0 +1,134 @@ +import Foundation + +// A put replaces whatever the key holds, and a put followed by a delete is two requests. A +// transaction checks the keys and writes in one step: a new key only where none exists, and a +// rename only onto a free name and only while the source is still what was read. +public struct EtcdTxnRequest: Encodable, Equatable, Sendable { + public struct Compare: Encodable, Equatable, Sendable { + public let key: Data + public let target: String + public let result: String + public var createRevision: String? + public var modRevision: String? + public var version: String? + + private enum CodingKeys: String, CodingKey { + case key + case target + case result + case createRevision = "create_revision" + case modRevision = "mod_revision" + case version + } + } + + public struct Put: Encodable, Equatable, Sendable { + public let key: Data + public let value: Data + public var lease: String? + } + + public struct DeleteRange: Encodable, Equatable, Sendable { + public let key: Data + } + + public struct Range: Encodable, Equatable, Sendable { + public let key: Data + public let countOnly: Bool + + private enum CodingKeys: String, CodingKey { + case key + case countOnly = "count_only" + } + } + + public struct Operation: Encodable, Equatable, Sendable { + public var requestPut: Put? + public var requestDeleteRange: DeleteRange? + public var requestRange: Range? + + private enum CodingKeys: String, CodingKey { + case requestPut = "request_put" + case requestDeleteRange = "request_delete_range" + case requestRange = "request_range" + } + } + + public let compare: [Compare] + public let success: [Operation] + public let failure: [Operation] + + public static func create(key: Data, value: Data, lease: Int64?) -> EtcdTxnRequest { + EtcdTxnRequest( + compare: [absent(key)], + success: [Operation(requestPut: Put(key: key, value: value, lease: lease.map { String($0) }))], + failure: [countOf(key)] + ) + } + + // Without a revision to hold the source to, it only has to still exist. + public static func move( + from source: Data, + to target: Data, + value: Data, + lease: Int64?, + modRevision: Int64? + ) -> EtcdTxnRequest { + let sourceUnchanged = modRevision.map { + Compare(key: source, target: "MOD", result: "EQUAL", modRevision: String($0)) + } ?? Compare(key: source, target: "VERSION", result: "GREATER", version: "0") + return EtcdTxnRequest( + compare: [absent(target), sourceUnchanged], + success: [ + Operation(requestPut: Put(key: target, value: value, lease: lease.map { String($0) })), + Operation(requestDeleteRange: DeleteRange(key: source)) + ], + failure: [countOf(target)] + ) + } + + private static func absent(_ key: Data) -> Compare { + Compare(key: key, target: "CREATE", result: "EQUAL", createRevision: "0") + } + + private static func countOf(_ key: Data) -> Operation { + Operation(requestRange: Range(key: key, countOnly: true)) + } +} + +public struct EtcdTxnResponse: Decodable, Sendable { + public enum Outcome: Equatable, Sendable { + case applied + case targetExists + case sourceChanged + } + + struct Header: Decodable, Sendable { + let revision: String? + } + + struct Response: Decodable, Sendable { + struct RangeResult: Decodable, Sendable { + let count: String? + } + + let responseRange: RangeResult? + + private enum CodingKeys: String, CodingKey { + case responseRange = "response_range" + } + } + + let header: Header? + let succeeded: Bool? + let responses: [Response]? + + public var revision: String? { header?.revision } + + // The failure branch counts the target key, which tells a taken name from a changed source. + public var outcome: Outcome { + if succeeded == true { return .applied } + let targetCount = responses?.first?.responseRange?.count.flatMap { Int64($0) } ?? 0 + return targetCount > 0 ? .targetExists : .sourceChanged + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLOperatorRun.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLOperatorRun.swift new file mode 100644 index 0000000000..545b13ee0a --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLOperatorRun.swift @@ -0,0 +1,48 @@ +import Foundation + +// PostgreSQL's operator rule, from its scanner: an operator is a run of operator characters, because a user can define +// one, so pgvector's `<#>` and PostGIS's `&&&` are single operators that no list holds. +public enum SQLOperatorRun { + private static let plus = UInt16(UnicodeScalar("+").value) + private static let operatorUnits: Set = Set("~!@#^&|`?+-*/%<>=".utf16) + private static let signKeepers: Set = Set("~!@#^&|`?%".utf16) + + // Where the operator starting at `index` ends, or nil when the grammar has no such rule or `index` starts none. + public static func end(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Int? { + let length = text.length + guard grammar.readsOperatorRuns, index >= 0, index < length, + continuesRun(at: index, in: text, grammar: grammar) + else { + return nil + } + var end = index + 1 + while end < length, continuesRun(at: end, in: text, grammar: grammar) { + end += 1 + } + // `a=-1` is `=` and a sign: a run ends in `+` or `-` only when one of these characters is in it. + let keepsSign = (index.. 1, isSign(text.character(at: end - 1)) { + end -= 1 + } + } + return end + } + + // A comment opener ends the run, as does a backtick where it quotes a name. + private static func continuesRun(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Bool { + operatorUnits.contains(text.character(at: index)) + && SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) == nil + } + + private static func isSign(_ unit: UInt16) -> Bool { + unit == plus || unit == SqlLexer.dash + } +} + +public extension SQLLexicalGrammar { + // `E'...'` marks an engine that lexes with PostgreSQL's scanner, the one with operator runs. + var readsOperatorRuns: Bool { + contains(.escapeStringPrefix) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSnowflakeCore/SnowflakeSessionRenewal.swift b/apple/Packages/TableProCore/Sources/TableProSnowflakeCore/SnowflakeSessionRenewal.swift new file mode 100644 index 0000000000..5609475cc9 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSnowflakeCore/SnowflakeSessionRenewal.swift @@ -0,0 +1,22 @@ +import Foundation + +// A session token expires while a statement is still running. The request the server refused is +// sent again as it was, with the renewed token: a submit keeps its request id, which the server +// deduplicates on, and a poll keeps its result URL. The statement itself is never submitted anew. +public enum SnowflakeSessionRenewal { + public static func send( + currentToken: () -> String?, + isExpired: (Response) -> Bool, + renew: (_ expiredToken: String) async throws -> Void, + request: (_ token: String) async throws -> Response + ) async throws -> (response: Response, token: String)? { + guard let token = currentToken() else { return nil } + let response = try await request(token) + guard isExpired(response) else { return (response, token) } + + // `renew` does nothing when another request on the shared session already replaced the token. + try await renew(token) + guard let renewed = currentToken() else { return nil } + return (try await request(renewed), renewed) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncMetadataStorage.swift b/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncMetadataStorage.swift index 6b57ffe520..546eb9078f 100644 --- a/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncMetadataStorage.swift +++ b/apple/Packages/TableProCore/Sources/TableProSyncTransport/SyncMetadataStorage.swift @@ -195,6 +195,20 @@ public final class SyncMetadataStorage: @unchecked Sendable { userDefaults.set(owners.map(\.uuidString).sorted(), forKey: key("ownersKeptOffSync")) } + // MARK: - Deferred Deletions + + // Record names the server reported deleted while this device did not read their type. + public var deferredDeletions: Set { + get { Set(userDefaults.stringArray(forKey: key("deferredDeletions")) ?? []) } + set { + guard !newValue.isEmpty else { + userDefaults.removeObject(forKey: key("deferredDeletions")) + return + } + userDefaults.set(newValue.sorted(), forKey: key("deferredDeletions")) + } + } + // MARK: - Last Sync Date public var lastSyncDate: Date? { @@ -237,6 +251,7 @@ public final class SyncMetadataStorage: @unchecked Sendable { private func forgetServerPosition() { saveToken(nil) + deferredDeletions = [] userDefaults.removeObject(forKey: key("lastSyncDate")) } @@ -262,6 +277,7 @@ public final class SyncMetadataStorage: @unchecked Sendable { public func clearAll() { saveToken(nil) + deferredDeletions = [] zoneState = .unknown userDefaults.removeObject(forKey: key("lastSyncDate")) userDefaults.removeObject(forKey: key("lastAccountId")) diff --git a/apple/Packages/TableProCore/Sources/TableProTabularIO/DelimitedFieldReader.swift b/apple/Packages/TableProCore/Sources/TableProTabularIO/DelimitedFieldReader.swift index 90b495ef5e..563094ac29 100644 --- a/apple/Packages/TableProCore/Sources/TableProTabularIO/DelimitedFieldReader.swift +++ b/apple/Packages/TableProCore/Sources/TableProTabularIO/DelimitedFieldReader.swift @@ -66,6 +66,27 @@ public struct DelimitedFieldReader: Sendable { return result } + // An empty field and a quoted empty field read as the same text; only the quoting tells "no value" from "". + public func fieldsWithQuoting( + in base: UnsafePointer, + range: Range + ) -> [(text: String, isQuoted: Bool)] { + var scratch: [UInt8] = [] + var result: [(text: String, isQuoted: Bool)] = [] + let encoding = dialect.encoding.readsInPlace ? dialect.encoding : .utf8 + let end = range.upperBound + var index = range.lowerBound + while true { + let isQuoted = index < end && base[index] == dialect.quote + let next = readField(in: base, from: index, end: end, scratch: &scratch) { content in + result.append((text: TabularTextCodec.string(from: content, encoding: encoding), isQuoted: isQuoted)) + return true + } + guard let resume = next.resume else { return result } + index = resume + } + } + private struct FieldEnd { let resume: Int? let shouldContinue: Bool diff --git a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/KeywordUppercaseHelperTests.swift b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/KeywordUppercaseHelperTests.swift index a4ea58badf..9de5cba03a 100644 --- a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/KeywordUppercaseHelperTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/KeywordUppercaseHelperTests.swift @@ -1,5 +1,6 @@ import Foundation @testable import TableProAutocomplete +import TableProSQLGrammar import Testing struct KeywordUppercaseHelperTests { @@ -78,31 +79,31 @@ struct KeywordUppercaseHelperTests { func insideSingleQuote() { let text: NSString = "SELECT 'hello select " // Position 20 is after "select " inside the string - #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 20)) + #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 20, grammar: .ansi)) } @Test("Outside single-quoted string") func outsideSingleQuote() { let text: NSString = "SELECT 'hello' select " - #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 22)) + #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 22, grammar: .ansi)) } @Test("Inside double-quoted identifier") func insideDoubleQuote() { let text: NSString = "SELECT \"select" - #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 14)) + #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 14, grammar: .ansi)) } @Test("Inside backtick identifier") func insideBacktick() { let text: NSString = "SELECT `select" - #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 14)) + #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 14, grammar: .backtickQuotes)) } @Test("Backslash-escaped quote does not end string") func backslashEscapedQuote() { let text: NSString = "SELECT 'it\\'s select" - #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 20)) + #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 20, grammar: .backslashEscapesInSingleQuotes)) } // MARK: - isInsideProtectedContext: Comments @@ -110,37 +111,37 @@ struct KeywordUppercaseHelperTests { @Test("Inside line comment (--)") func insideLineComment() { let text: NSString = "SELECT -- select" - #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 16)) + #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 16, grammar: .ansi)) } @Test("After line comment on new line") func afterLineCommentNewLine() { let text: NSString = "-- comment\nselect" - #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 17)) + #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 17, grammar: .ansi)) } @Test("Inside block comment") func insideBlockComment() { let text: NSString = "SELECT /* select" - #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 16)) + #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 16, grammar: .ansi)) } @Test("After closed block comment") func afterClosedBlockComment() { let text: NSString = "/* comment */ select" - #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 19)) + #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 19, grammar: .ansi)) } @Test("Inside MySQL hash comment (#)") func insideMySQLHashComment() { let text: NSString = "SELECT # select" - #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 15)) + #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 15, grammar: .hashLineComments)) } @Test("Hash inside string is not a comment") func hashInsideStringNotComment() { let text: NSString = "SELECT 'test#' select" - #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 21)) + #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 21, grammar: .ansi)) } // MARK: - isInsideProtectedContext: Dollar-Quoting (PostgreSQL) @@ -148,19 +149,19 @@ struct KeywordUppercaseHelperTests { @Test("Inside dollar-quoted string") func insideDollarQuote() { let text: NSString = "SELECT $$ select" - #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 16)) + #expect(KeywordUppercaseHelper.isInsideProtectedContext(text, at: 16, grammar: .taggedDollarQuotes)) } @Test("After closed dollar-quoted string") func afterClosedDollarQuote() { let text: NSString = "$$ body $$ select" - #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 17)) + #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 17, grammar: .taggedDollarQuotes)) } @Test("Single dollar is not a quote") func singleDollarNotQuote() { let text: NSString = "SELECT $5 select" - #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 16)) + #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 16, grammar: .taggedDollarQuotes)) } // MARK: - isInsideProtectedContext: Edge Cases @@ -168,19 +169,19 @@ struct KeywordUppercaseHelperTests { @Test("Position 0 is never protected") func positionZeroNotProtected() { let text: NSString = "select" - #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 0)) + #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 0, grammar: .ansi)) } @Test("Empty string at position 0") func emptyStringNotProtected() { let text: NSString = "" - #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 0)) + #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 0, grammar: .ansi)) } @Test("Nested quotes: double inside single are ignored") func nestedDoubleInSingle() { let text: NSString = "SELECT '\"hello\"' select" - #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 23)) + #expect(!KeywordUppercaseHelper.isInsideProtectedContext(text, at: 23, grammar: .ansi)) } // MARK: - keywordBeforePosition @@ -188,7 +189,7 @@ struct KeywordUppercaseHelperTests { @Test("Detects lowercase keyword") func detectsLowercaseKeyword() { let text: NSString = "select " - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 6) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 6, grammar: .ansi) #expect(result != nil) #expect(result?.word == "select") #expect(result?.range == NSRange(location: 0, length: 6)) @@ -197,14 +198,14 @@ struct KeywordUppercaseHelperTests { @Test("Returns nil for already uppercase keyword") func alreadyUppercaseReturnsNil() { let text: NSString = "SELECT " - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 6) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 6, grammar: .ansi) #expect(result == nil) } @Test("Detects mixed-case keyword") func detectsMixedCase() { let text: NSString = "Select " - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 6) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 6, grammar: .ansi) #expect(result != nil) #expect(result?.word == "Select") } @@ -212,42 +213,42 @@ struct KeywordUppercaseHelperTests { @Test("Returns nil for non-keyword") func nonKeywordReturnsNil() { let text: NSString = "foobar " - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 6) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 6, grammar: .ansi) #expect(result == nil) } @Test("Returns nil for keyword inside string literal") func keywordInsideStringReturnsNil() { let text: NSString = "'select" - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 7) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 7, grammar: .ansi) #expect(result == nil) } @Test("Returns nil for keyword inside comment") func keywordInsideCommentReturnsNil() { let text: NSString = "-- select" - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 9) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 9, grammar: .ansi) #expect(result == nil) } @Test("Returns nil for keyword inside dollar-quote") func keywordInsideDollarQuoteReturnsNil() { let text: NSString = "$$ select" - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 9) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 9, grammar: .taggedDollarQuotes) #expect(result == nil) } @Test("Returns nil for keyword inside hash comment") func keywordInsideHashCommentReturnsNil() { let text: NSString = "# select" - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 8) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 8, grammar: .hashLineComments) #expect(result == nil) } @Test("Detects keyword after other text") func keywordAfterOtherText() { let text: NSString = "SELECT * from " - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 13) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 13, grammar: .ansi) #expect(result != nil) #expect(result?.word == "from") #expect(result?.range == NSRange(location: 9, length: 4)) @@ -256,14 +257,14 @@ struct KeywordUppercaseHelperTests { @Test("Returns nil for partial identifier containing keyword") func identifierContainingKeywordReturnsNil() { let text: NSString = "select_count " - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 12) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 12, grammar: .ansi) #expect(result == nil) } @Test("Returns nil at position 0") func positionZeroReturnsNil() { let text: NSString = "select" - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 0) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 0, grammar: .ansi) #expect(result == nil) } @@ -271,7 +272,7 @@ struct KeywordUppercaseHelperTests { func keywordBeforeParen() { // "count" is not a keyword, but "where" is let text2: NSString = "where(" - let result2 = KeywordUppercaseHelper.keywordBeforePosition(text2, at: 5) + let result2 = KeywordUppercaseHelper.keywordBeforePosition(text2, at: 5, grammar: .ansi) #expect(result2 != nil) #expect(result2?.word == "where") } @@ -279,14 +280,14 @@ struct KeywordUppercaseHelperTests { @Test("Returns nil for empty word (consecutive spaces)") func emptyWordReturnsNil() { let text: NSString = "SELECT " - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 8) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 8, grammar: .ansi) #expect(result == nil) } @Test("Keyword with digits is not a keyword") func keywordWithDigitsNotKeyword() { let text: NSString = "select2 " - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 7) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: 7, grammar: .ansi) #expect(result == nil) } @@ -298,8 +299,53 @@ struct KeywordUppercaseHelperTests { "not", "and", "or", "as", "set", "into", "values", "begin", "commit", "rollback"] for kw in keywords { let text = kw as NSString - let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: text.length) + let result = KeywordUppercaseHelper.keywordBeforePosition(text, at: text.length, grammar: .ansi) #expect(result != nil, "Expected '\(kw)' to be detected as a keyword") } } + + // MARK: - Engine grammar + + private func folds(_ text: String, _ grammar: SQLLexicalGrammar) -> Bool { + let source = text as NSString + return KeywordUppercaseHelper.keywordBeforePosition(source, at: source.length, grammar: grammar) != nil + } + + @Test("A backslash before a closing quote keeps the literal open only where the engine escapes with it") + func backslashFollowsTheGrammar() { + let text = "VALUES ('C:\\', 'order from" + #expect(!folds(text, .ansi)) + #expect(folds(text, .backslashEscapesInSingleQuotes)) + } + + @Test("A literal that opens more than 2,000 characters back still protects the word") + func longLiteral() { + let text = "SELECT '" + String(repeating: "x ", count: 2_000) + "from" + #expect(!folds(text, .ansi)) + } + + @Test("A bracket name protects the word only where brackets quote") + func bracketName() { + #expect(!folds("SELECT [group", .bracketQuotedIdentifiers)) + #expect(folds("SELECT a[group", .ansi)) + } + + @Test("A tagged dollar quote protects the word, and an inner $$ does not end it") + func taggedDollarQuote() { + #expect(!folds("DO $fn$ begin $$ select", .taggedDollarQuotes)) + #expect(folds("DO $fn$ x $fn$ select", .taggedDollarQuotes)) + } + + @Test("A # starts a comment only where the engine reads one") + func hashFollowsTheGrammar() { + #expect(folds("SELECT a #> b from", .ansi)) + #expect(!folds("SELECT a # b from", .hashLineComments)) + } + + @Test("The body of an executable comment is code") + func executableComment() { + let mySQL: SQLLexicalGrammar = [.executableComments, .backtickQuotes] + #expect(folds("/*!50001 select", mySQL)) + #expect(!folds("/* select", mySQL)) + } } diff --git a/apple/Packages/TableProCore/Tests/TableProDocumentPathTests/DocumentPathPlacementTests.swift b/apple/Packages/TableProCore/Tests/TableProDocumentPathTests/DocumentPathPlacementTests.swift new file mode 100644 index 0000000000..a5972c15cf --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProDocumentPathTests/DocumentPathPlacementTests.swift @@ -0,0 +1,95 @@ +import Foundation +import Testing + +@testable import TableProDocumentPath + +struct DocumentPathPlacementTests { + private func placed( + _ value: Any, + _ keys: [String], + following stored: [String: Any] + ) throws -> NSDictionary { + var partial: [String: Any] = [:] + try DocumentPathPlacement.place(value, atKeys: keys, in: &partial, following: stored) + return partial as NSDictionary + } + + @Test("a field stored as a nested object is set inside that object") + func nestedSpelling() throws { + let stored: [String: Any] = ["address": ["city": "Hanoi", "zip": "100000"]] + let partial = try placed("Hue", ["address", "city"], following: stored) + #expect(partial == ["address": ["city": "Hue"]] as NSDictionary) + } + + @Test("a field stored under a literal dotted key is set under that key") + func dottedSpelling() throws { + let stored: [String: Any] = ["address.city": "Hanoi"] + let partial = try placed("Hue", ["address", "city"], following: stored) + #expect(partial == ["address.city": "Hue"] as NSDictionary) + } + + @Test("a field the document holds under both spellings is set under both") + func bothSpellings() throws { + let stored: [String: Any] = ["address": ["city": "Hanoi"], "address.city": "Hue"] + let partial = try placed("Da Nang", ["address", "city"], following: stored) + #expect(partial == ["address": ["city": "Da Nang"], "address.city": "Da Nang"] as NSDictionary) + } + + @Test("a mixed spelling is followed level by level") + func mixedSpelling() throws { + let stored: [String: Any] = ["a": ["b.c": 1]] + let partial = try placed(2, ["a", "b", "c"], following: stored) + #expect(partial == ["a": ["b.c": 2]] as NSDictionary) + } + + @Test("a field the document lacks is nested where its keys split") + func missingField() throws { + let partial = try placed("Hue", ["address", "city"], following: ["name": "x"]) + #expect(partial == ["address": ["city": "Hue"]] as NSDictionary) + } + + @Test("a key the schema declares with a dot stays one key") + func dottedSchemaKey() throws { + let partial = try placed(3, ["metrics", "cpu.usage"], following: [:]) + #expect(partial == ["metrics": ["cpu.usage": 3]] as NSDictionary) + } + + @Test("a missing field joins the deepest object the document already has") + func deepestExistingObject() throws { + let stored: [String: Any] = ["address": ["geo": ["lat": 1]]] + let partial = try placed(2, ["address", "geo", "lon"], following: stored) + #expect(partial == ["address": ["geo": ["lon": 2]]] as NSDictionary) + } + + @Test("clearing a field writes null at the spelling the document holds") + func nullValue() throws { + let partial = try placed(NSNull(), ["address", "city"], following: ["address.city": "Hanoi"]) + #expect(partial == ["address.city": NSNull()] as NSDictionary) + } + + @Test("a field inside an array of objects cannot be placed") + func insideArray() { + let stored: [String: Any] = ["variants": [["sku": "a"], ["sku": "b"]]] + #expect(throws: DocumentPathPlacement.InsideArray(path: "variants.sku")) { + _ = try placed("c", ["variants", "sku"], following: stored) + } + } + + @Test("two fields of one object share it in the partial document") + func siblings() throws { + let stored: [String: Any] = ["address": ["city": "Hanoi", "zip": "1"]] + var partial: [String: Any] = [:] + try DocumentPathPlacement.place("Hue", atKeys: ["address", "city"], in: &partial, following: stored) + try DocumentPathPlacement.place("2", atKeys: ["address", "zip"], in: &partial, following: stored) + #expect(partial as NSDictionary == ["address": ["city": "Hue", "zip": "2"]] as NSDictionary) + } + + @Test("set nests a value and refuses to pass through a scalar") + func setAndRead() { + var document: [String: Any] = ["a": 1] + #expect(DocumentPathPlacement.set("x", atKeys: ["b", "c"], in: &document)) + #expect(DocumentPathPlacement.value(in: document, atKeys: ["b", "c"]) as? String == "x") + #expect(!DocumentPathPlacement.set("y", atKeys: ["a", "c"], in: &document)) + #expect(DocumentPathPlacement.value(in: document, atKeys: ["a", "c"]) == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBBoundedStreamTests.swift b/apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBBoundedStreamTests.swift new file mode 100644 index 0000000000..1aa3be5881 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBBoundedStreamTests.swift @@ -0,0 +1,44 @@ +import Foundation +import TableProDuckDBCore +import Testing + +@Test func aSlowConsumerReceivesEveryBatchThroughASmallBuffer() async throws { + let (stream, continuation) = AsyncThrowingStream.makeStream(bufferingPolicy: .bufferingOldest(2)) + Thread.detachNewThread { + for value in 0..<500 { + guard DuckDBBoundedStream.deliver(value, to: continuation, pause: { _ = usleep(100) }) else { break } + } + continuation.finish() + } + + var received: [Int] = [] + for try await value in stream { + received.append(value) + } + #expect(received == Array(0..<500)) +} + +@Test func theProducerWaitsWhileTheBufferIsFull() { + let (stream, continuation) = AsyncThrowingStream.makeStream(bufferingPolicy: .bufferingOldest(2)) + #expect(DuckDBBoundedStream.deliver(1, to: continuation)) + #expect(DuckDBBoundedStream.deliver(2, to: continuation)) + + var pauses = 0 + let delivered = DuckDBBoundedStream.deliver(3, to: continuation) { + pauses += 1 + if pauses == 3 { continuation.finish() } + } + #expect(!delivered) + #expect(pauses == 3) + _ = stream +} + +@Test func theProducerStopsWhenTheStreamIsGone() { + var stream: AsyncThrowingStream? + var held: AsyncThrowingStream.Continuation? + stream = AsyncThrowingStream(bufferingPolicy: .bufferingOldest(1)) { held = $0 } + let continuation = held + #expect(continuation.map { DuckDBBoundedStream.deliver(1, to: $0) } == true) + stream = nil + #expect(continuation.map { DuckDBBoundedStream.deliver(2, to: $0, pause: {}) } == false) +} diff --git a/apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBFetchBudgetTests.swift b/apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBFetchBudgetTests.swift new file mode 100644 index 0000000000..e1d952da0c --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProDuckDBCoreTests/DuckDBFetchBudgetTests.swift @@ -0,0 +1,70 @@ +import Foundation +import TableProDuckDBCore +import Testing + +@Test func aResultPastFiveMillionRowsIsReadWholeWithoutALimit() { + var budget = DuckDBFetchBudget(limit: .max) + var kept = 0 + for _ in 0..<5_860 { + kept += budget.take(2_048) + } + #expect(kept == 12_001_280) + #expect(!budget.isExceeded) +} + +@Test func aResultOfExactlyTheLimitIsNotTruncated() { + var budget = DuckDBFetchBudget(limit: 4_096) + #expect(budget.take(2_048) == 2_048) + #expect(budget.take(2_048) == 2_048) + #expect(!budget.isExceeded) + #expect(budget.take(0) == 0) + #expect(!budget.isExceeded) +} + +@Test func theFirstRowPastTheLimitStopsTheRead() { + var budget = DuckDBFetchBudget(limit: 1_000) + #expect(budget.take(2_048) == 1_000) + #expect(budget.isExceeded) + #expect(budget.kept == 1_000) + #expect(budget.take(2_048) == 0) +} + +@Test func aLimitInsideALaterChunkKeepsOnlyItsHead() { + var budget = DuckDBFetchBudget(limit: 3_000) + #expect(budget.take(2_048) == 2_048) + #expect(!budget.isExceeded) + #expect(budget.take(2_048) == 952) + #expect(budget.isExceeded) +} + +@Test func anUnfinishedStatementIsDroppedOnlyOutsideATransaction() { + #expect(DuckDBUnreadRows.abandonWhenIdle.abandons { true }) + #expect(!DuckDBUnreadRows.abandonWhenIdle.abandons { false }) + #expect(DuckDBUnreadRows.abandon.abandons { false }) + #expect(!DuckDBUnreadRows.drain.abandons { true }) +} + +@Test func theSessionIsNotProbedWhenTheAnswerCannotChangeTheOutcome() { + var probes = 0 + _ = DuckDBUnreadRows.drain.abandons { probes += 1; return true } + _ = DuckDBUnreadRows.abandon.abandons { probes += 1; return true } + #expect(probes == 0) +} + +@Test func theRenderingConnectionTakesOnlyTheSettingsThatDiffer() { + let statements = DuckDBRenderSettings.assignments( + from: ["TimeZone": "UTC", "Calendar": "gregorian"], + to: ["TimeZone": "Asia/Tokyo", "Calendar": "gregorian"] + ) + #expect(statements == ["SET TimeZone = 'Asia/Tokyo'"]) +} + +@Test func aSettingTheSessionDoesNotReportIsLeftAlone() { + #expect(DuckDBRenderSettings.assignments(from: [:], to: [:]).isEmpty) + #expect(DuckDBRenderSettings.assignments(from: ["TimeZone": "UTC"], to: [:]).isEmpty) +} + +@Test func aSettingValueIsQuotedAsALiteral() { + let statements = DuckDBRenderSettings.assignments(from: [:], to: ["TimeZone": "it's"]) + #expect(statements == ["SET TimeZone = 'it''s'"]) +} diff --git a/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdGuardedWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdGuardedWriteTests.swift new file mode 100644 index 0000000000..ec7c5defd9 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProEtcdCoreTests/EtcdGuardedWriteTests.swift @@ -0,0 +1,77 @@ +import Foundation +import Testing + +@testable import TableProEtcdCore + +struct EtcdGuardedWriteTests { + private func json(_ request: EtcdTxnRequest) throws -> NSDictionary { + let data = try JSONEncoder().encode(request) + return try #require(try JSONSerialization.jsonObject(with: data) as? NSDictionary) + } + + private func base64(_ text: String) -> String { + Data(text.utf8).base64EncodedString() + } + + private func outcome(_ body: String) throws -> EtcdTxnResponse.Outcome { + try JSONDecoder().decode(EtcdTxnResponse.self, from: Data(body.utf8)).outcome + } + + @Test("A new key is put only where no key exists") + func createRequest() throws { + let request = EtcdTxnRequest.create(key: Data("/app/b".utf8), value: Data("v".utf8), lease: 42) + let expected: NSDictionary = [ + "compare": [ + ["key": base64("/app/b"), "target": "CREATE", "result": "EQUAL", "create_revision": "0"] + ], + "success": [["request_put": ["key": base64("/app/b"), "value": base64("v"), "lease": "42"]]], + "failure": [["request_range": ["key": base64("/app/b"), "count_only": true]]] + ] + #expect(try json(request) == expected) + } + + @Test("A rename puts the new key and deletes the old one in one transaction, guarded on both") + func moveRequest() throws { + let request = EtcdTxnRequest.move( + from: Data("/app/a".utf8), to: Data("/app/b".utf8), value: Data("v".utf8), lease: nil, modRevision: 17 + ) + let expected: NSDictionary = [ + "compare": [ + ["key": base64("/app/b"), "target": "CREATE", "result": "EQUAL", "create_revision": "0"], + ["key": base64("/app/a"), "target": "MOD", "result": "EQUAL", "mod_revision": "17"] + ], + "success": [ + ["request_put": ["key": base64("/app/b"), "value": base64("v")]], + ["request_delete_range": ["key": base64("/app/a")]] + ], + "failure": [["request_range": ["key": base64("/app/b"), "count_only": true]]] + ] + #expect(try json(request) == expected) + } + + @Test("A rename with no revision to compare still requires the source to exist") + func moveWithoutRevision() throws { + let request = EtcdTxnRequest.move( + from: Data("a".utf8), to: Data("b".utf8), value: Data(), lease: nil, modRevision: nil + ) + #expect(request.compare.last == EtcdTxnRequest.Compare( + key: Data("a".utf8), target: "VERSION", result: "GREATER", version: "0" + )) + } + + @Test("A transaction that ran reports applied") + func appliedOutcome() throws { + #expect(try outcome(#"{"header":{"revision":"9"},"succeeded":true,"responses":[{"response_put":{}}]}"#) == .applied) + } + + @Test("A failed transaction whose target key exists reports the taken name") + func targetExistsOutcome() throws { + #expect(try outcome(#"{"header":{"revision":"9"},"responses":[{"response_range":{"count":"1"}}]}"#) == .targetExists) + } + + @Test("A failed transaction whose target key is free reports the changed source") + func sourceChangedOutcome() throws { + #expect(try outcome(#"{"header":{"revision":"9"},"responses":[{"response_range":{"header":{}}}]}"#) == .sourceChanged) + #expect(try outcome(#"{"header":{"revision":"9"}}"#) == .sourceChanged) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginExportDocumentTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginExportDocumentTests.swift index 2a6ece1604..16896c7355 100644 --- a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginExportDocumentTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginExportDocumentTests.swift @@ -71,6 +71,81 @@ struct PluginExportDocumentTests { #expect(count == 2) } + private final class PullCount: @unchecked Sendable { + private let lock = NSLock() + private var count = 0 + + func increment() -> Int { + lock.lock() + defer { lock.unlock() } + count += 1 + return count + } + + var value: Int { + lock.lock() + defer { lock.unlock() } + return count + } + } + + private func endlessRows(_ pulls: PullCount) -> AsyncThrowingStream { + AsyncThrowingStream(unfolding: { + pulls.increment() == 1 + ? .header(PluginStreamHeader(columns: ["v"], columnTypeNames: ["VARCHAR"])) + : .rows([["42"]]) + }) + } + + @Test("The legacy adapter reads a row batch only when its consumer asks for one") + func wrappingPullsOnDemand() async throws { + let pulls = PullCount() + var iterator = PluginExportStreamElement.wrapping(endlessRows(pulls)).makeAsyncIterator() + _ = try await iterator.next() + _ = try await iterator.next() + _ = try await iterator.next() + // A pump would have run ahead of a consumer that stopped here. + try await Task.sleep(for: .milliseconds(50)) + #expect(pulls.value == 3) + } + + @Test("The cell adapter over the legacy adapter still reads on demand") + func cellsPullOnDemand() async throws { + let pulls = PullCount() + let stream = PluginExportStreamElement.cells(PluginExportStreamElement.wrapping(endlessRows(pulls))) + var iterator = stream.makeAsyncIterator() + _ = try await iterator.next() + _ = try await iterator.next() + try await Task.sleep(for: .milliseconds(50)) + #expect(pulls.value == 2) + } + + @Test("A failure in the inner stream reaches the consumer") + func wrappingForwardsFailure() async { + let failing = AsyncThrowingStream { continuation in + continuation.yield(.rows([["1"]])) + continuation.finish(throwing: PluginExportError.encodingFailed) + } + await #expect(throws: PluginExportError.self) { + for try await _ in PluginExportStreamElement.wrapping(failing) {} + } + } + + @Test("A document with a field outside the columns fails the cell stream") + func cellsRefuseAnUnknownField() async { + let stream = AsyncThrowingStream { continuation in + continuation.yield(.header(PluginStreamHeader(columns: ["v"], columnTypeNames: ["VARCHAR"]))) + let document = PluginExportDocument(fields: [ + PluginExportField(name: "other", canonicalJSON: #""1""#, typeName: "VARCHAR", cellValue: "1") + ]) + continuation.yield(.rows([["1"]], documents: [document])) + continuation.finish() + } + await #expect(throws: PluginExportError.self) { + for try await _ in PluginExportStreamElement.cells(stream) {} + } + } + @Test("Cell consumers retain every row from document streams") func cellStream() async throws { let stream = AsyncThrowingStream { continuation in diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLOperatorRunTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLOperatorRunTests.swift new file mode 100644 index 0000000000..6676dc8221 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLOperatorRunTests.swift @@ -0,0 +1,48 @@ +import Foundation +import TableProSQLGrammar +import Testing + +struct SQLOperatorRunTests { + private static let postgreSQL = SQLLexicalReadings.resolve(databaseTypeId: "PostgreSQL", declared: nil, session: nil) + .execution + private static let mySQL = SQLLexicalReadings.resolve(databaseTypeId: "MySQL", declared: nil, session: nil) + .execution + + private func run(_ text: String, at index: Int = 1, _ grammar: SQLLexicalGrammar = postgreSQL) -> String? { + let source = text as NSString + return SQLOperatorRun.end(at: index, in: source, grammar: grammar).map { + source.substring(with: NSRange(location: index, length: $0 - index)) + } + } + + @Test( + "A run of operator characters is one operator", + arguments: ["<#>", "<%", "%>", "&&&", "?-|", "<<|", "@-@", "#>>", "||", "=", "-"] + ) + func wholeRun(symbol: String) { + #expect(run("a\(symbol)b") == symbol) + } + + @Test("A trailing sign is dropped unless the run holds a character that keeps it") + func trailingSign() { + #expect(run("a=-1") == "=") + #expect(run("a>=+-1") == ">=") + #expect(run("a@-b") == "@-") + #expect(run("a|-b") == "|-") + } + + @Test("A comment opener ends the run") + func commentEndsTheRun() { + #expect(run("a+-- c") == "+") + #expect(run("ab", Self.mySQL) == nil) + #expect(run("a<#>b", .ansi) == nil) + #expect(run("ab") == nil) + #expect(run("a::b") == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSnowflakeCoreTests/SnowflakeSessionRenewalTests.swift b/apple/Packages/TableProCore/Tests/TableProSnowflakeCoreTests/SnowflakeSessionRenewalTests.swift new file mode 100644 index 0000000000..8b9a8f1644 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSnowflakeCoreTests/SnowflakeSessionRenewalTests.swift @@ -0,0 +1,96 @@ +import Foundation +import TableProSnowflakeCore +import Testing + +struct SnowflakeSessionRenewalTests { + private final class Session { + var token: String? = "t1" + var renewals = 0 + var sent: [String] = [] + } + + private func send( + _ session: Session, + expiredTokens: Set, + beforeRenewalCheck: () -> Void = {} + ) async throws -> (response: String, token: String)? { + try await SnowflakeSessionRenewal.send( + currentToken: { session.token }, + isExpired: { (response: String) in response == "expired" }, + renew: { expired in + guard session.token == expired else { return } + session.renewals += 1 + session.token = "t\(session.renewals + 1)" + }, + request: { token in + session.sent.append("poll /queries/q1/result as \(token)") + guard expiredTokens.contains(token) else { return "done" } + beforeRenewalCheck() + return "expired" + } + ) + } + + @Test("A request the server accepts is sent once and nothing is renewed") + func acceptedRequest() async throws { + let session = Session() + let sent = try await send(session, expiredTokens: []) + #expect(sent?.response == "done") + #expect(sent?.token == "t1") + #expect(session.renewals == 0) + #expect(session.sent.count == 1) + } + + @Test("An expired token is renewed and the same request goes out again with the new token") + func expiredTokenResendsTheSameRequest() async throws { + let session = Session() + let sent = try await send(session, expiredTokens: ["t1"]) + #expect(sent?.response == "done") + #expect(sent?.token == "t2") + #expect(session.renewals == 1) + #expect(session.sent == ["poll /queries/q1/result as t1", "poll /queries/q1/result as t2"]) + } + + @Test("A token another request already renewed is not renewed a second time") + func alreadyRenewedTokenIsReused() async throws { + let session = Session() + let sent = try await send(session, expiredTokens: ["t1"]) { session.token = "other" } + #expect(sent?.token == "other") + #expect(session.renewals == 0) + } + + @Test("A token that is still refused after renewal returns the refusal, without a third attempt") + func stillExpiredAfterRenewal() async throws { + let session = Session() + let sent = try await send(session, expiredTokens: ["t1", "t2"]) + #expect(sent?.response == "expired") + #expect(session.renewals == 1) + #expect(session.sent.count == 2) + } + + @Test("No session token sends nothing") + func missingToken() async throws { + let session = Session() + session.token = nil + #expect(try await send(session, expiredTokens: []) == nil) + #expect(session.sent.isEmpty) + } + + @Test("A failed renewal surfaces and the request is not sent again") + func failedRenewal() async { + struct RenewalFailed: Error {} + let session = Session() + await #expect(throws: RenewalFailed.self) { + _ = try await SnowflakeSessionRenewal.send( + currentToken: { "t1" }, + isExpired: { (_: String) in true }, + renew: { _ in throw RenewalFailed() }, + request: { token in + session.sent.append(token) + return "expired" + } + ) + } + #expect(session.sent == ["t1"]) + } +} diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditor+Coordinator.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditor+Coordinator.swift index 93b58efaa2..b19f9c59c6 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditor+Coordinator.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/SourceEditor+Coordinator.swift @@ -78,6 +78,12 @@ public extension SourceEditor { name: TextView.textDidChangeNotification, object: controller.textView ) + NotificationCenter.default.addObserver( + self, + selector: #selector(textViewShouldFlushWriteback(_:)), + name: TextViewController.flushTextWritebackNotification, + object: controller.textView + ) // Needs to be put on the main runloop or SwiftUI gets mad about updating state during view updates. NotificationCenter.default @@ -155,6 +161,13 @@ public extension SourceEditor { textSync.editorTextDidChange(textView) } + @objc func textViewShouldFlushWriteback(_ notification: Notification) { + guard let textView = notification.object as? TextView else { + return + } + textSync.flushPendingWriteback(from: textView) + } + @objc func textControllerCursorsDidUpdate(_ notification: Notification) { guard let controller = notification.object as? TextViewController else { return diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/TextBindingSync.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/TextBindingSync.swift index 741c441061..60b3403155 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/TextBindingSync.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/SourceEditor/TextBindingSync.swift @@ -60,10 +60,22 @@ final class TextBindingSync { guard case .binding(let currentBinding) = self.text else { return } let newText = textView.string self.lastSyncedText = newText + self.writebackTask = nil currentBinding.wrappedValue = newText } } + // A run or save reads the binding, which trails the editor by the debounce on a large document. + func flushPendingWriteback(from textView: TextView) { + guard let pending = writebackTask else { return } + pending.cancel() + writebackTask = nil + guard case .binding(let binding) = text else { return } + let newText = textView.string + lastSyncedText = newText + binding.wrappedValue = newText + } + /// Pushes an external binding change down into the text view. The editor's /// content wins while one of its own edits is still in flight, so user /// typing is never clobbered by a stale binding snapshot. @@ -82,9 +94,21 @@ final class TextBindingSync { if let lastSyncedText, (newValue as NSString).isEqual(to: lastSyncedText) { return } writebackTask?.cancel() + writebackTask = nil phase.applyRepresentableValue { controller.setText(newValue) } lastSyncedText = newValue } } + +public extension TextViewController { + static let flushTextWritebackNotification = Notification.Name( + "TableProEditorKit.flushTextWritebackNotification" + ) + + func flushPendingTextWriteback() { + guard let textView else { return } + NotificationCenter.default.post(name: Self.flushTextWritebackNotification, object: textView) + } +} diff --git a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/SourceEditorBindingSyncTests.swift b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/SourceEditorBindingSyncTests.swift index e3bcfa45bc..99bea4bb35 100644 --- a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/SourceEditorBindingSyncTests.swift +++ b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/SourceEditorBindingSyncTests.swift @@ -109,6 +109,44 @@ final class SourceEditorBindingSyncTests { #expect(coordinator.textSync.lastSyncedText == largeText) } + @MainActor + @Test func test_flushWritesADebouncedLargeDocumentEditAtOnce() { + var bound = "" + var setterCallCount = 0 + let coordinator = makeCoordinator(get: { bound }, set: { bound = $0; setterCallCount += 1 }) + coordinator.setController(controller) + let largeText = String(repeating: "a", count: 500_001) + controller.textView.setText(largeText) + coordinator.textViewDidChangeText( + Notification(name: TextView.textDidChangeNotification, object: controller.textView) + ) + #expect(bound == "") + + controller.flushPendingTextWriteback() + + #expect(bound == largeText) + #expect(coordinator.textSync.lastSyncedText == largeText) + #expect(coordinator.textSync.writebackTask == nil) + + let writesAfterFlush = setterCallCount + controller.flushPendingTextWriteback() + #expect(setterCallCount == writesAfterFlush) + } + + @MainActor + @Test func test_flushWithoutAPendingWritebackLeavesTheBindingAlone() { + var bound = "external" + var setterCallCount = 0 + let coordinator = makeCoordinator(get: { bound }, set: { bound = $0; setterCallCount += 1 }) + coordinator.setController(controller) + controller.textView.setText("not yet pushed down") + + controller.flushPendingTextWriteback() + + #expect(setterCallCount == 0) + #expect(bound == "external") + } + @MainActor @Test func test_syncSkipsStaleBindingSnapshotWhileEditIsInFlight() { var bound = "" diff --git a/apple/Packages/TableProMacCore/Sources/TableProMCP/Outside/MCPServerConfiguration.swift b/apple/Packages/TableProMacCore/Sources/TableProMCP/Outside/MCPServerConfiguration.swift index c3f039fa74..b4cd3f20ed 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProMCP/Outside/MCPServerConfiguration.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProMCP/Outside/MCPServerConfiguration.swift @@ -30,9 +30,42 @@ public struct MCPServerConfiguration: Codable, Equatable, Identifiable, Sendable self.allowedConnectionIds = allowedConnectionIds } - public var toolNamespace: String { "ext__\(id.uuidString.lowercased())__" } + // Provider APIs take a function name of at most 64 characters from letters, digits, "_" and "-". + public static let maxToolNameLength = 64 - public func toolName(for remoteName: String) -> String { toolNamespace + remoteName } + public var toolNamespace: String { + "ext__\(id.uuidString.lowercased().replacingOccurrences(of: "-", with: ""))_" + } + + // The name a provider sees. A remote name that already fits is kept as it is; any other is cut + // down to the allowed characters and tagged with a hash of the original, so the same remote + // tool gets the same name in every session and two remote names never fold into one silently. + public func toolName(for remoteName: String) -> String { + let budget = Self.maxToolNameLength - toolNamespace.count + let allowed = String(remoteName.unicodeScalars.map { Self.isToolNameScalar($0) ? Character($0) : "_" }) + if allowed == remoteName, !remoteName.isEmpty, remoteName.count <= budget { + return toolNamespace + remoteName + } + let tag = "_" + Self.nameTag(remoteName) + return toolNamespace + allowed.prefix(budget - tag.count) + tag + } + + private static func isToolNameScalar(_ scalar: Unicode.Scalar) -> Bool { + switch scalar { + case "a"..."z", "A"..."Z", "0"..."9", "_", "-": return true + default: return false + } + } + + // FNV-1a, 32 bit: stable across launches, which `Hasher` is not. + private static func nameTag(_ name: String) -> String { + var hash: UInt32 = 2_166_136_261 + for byte in name.utf8 { + hash = (hash ^ UInt32(byte)) &* 16_777_619 + } + let hex = String(hash, radix: 16) + return String(repeating: "0", count: 8 - hex.count) + hex + } public func allows(connectionId: UUID?) -> Bool { guard let connectionId else { return false } diff --git a/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Outside/MCPServerToolNameTests.swift b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Outside/MCPServerToolNameTests.swift new file mode 100644 index 0000000000..2dc160abcf --- /dev/null +++ b/apple/Packages/TableProMacCore/Tests/TableProMCPTests/Outside/MCPServerToolNameTests.swift @@ -0,0 +1,63 @@ +import Foundation +@testable import TableProMCP +import Testing + +struct MCPServerToolNameTests { + private let server = MCPServerConfiguration( + name: "Runbooks", + endpoint: URL(string: "https://a.example/mcp") ?? URL(fileURLWithPath: "/") + ) + + private func isAcceptedByProviders(_ name: String) -> Bool { + let allowed = Set("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-") + return !name.isEmpty + && name.count <= MCPServerConfiguration.maxToolNameLength + && name.allSatisfy { allowed.contains($0) } + } + + @Test("A short plain name is kept after the namespace") + func shortNameIsKept() { + #expect(server.toolName(for: "search") == server.toolNamespace + "search") + #expect(isAcceptedByProviders(server.toolName(for: "search"))) + } + + @Test("A 26 character name still fits whole") + func longestWholeNameFits() { + let name = "create_pull_request_review" + #expect(server.toolName(for: name) == server.toolNamespace + name) + #expect(server.toolName(for: name).count == 64) + } + + @Test( + "Any remote name becomes one a provider accepts", + arguments: [ + "create_pull_request_review_comment", + "repo.search", + "files/read", + "tìm kiếm", + "a b c", + "", + String(repeating: "x", count: 300) + ] + ) + func everyNameIsAccepted(remoteName: String) { + let name = server.toolName(for: remoteName) + #expect(isAcceptedByProviders(name)) + #expect(name.hasPrefix(server.toolNamespace)) + } + + @Test("The same remote name always maps to the same name") + func aliasIsStable() { + let first = server.toolName(for: "repo.search") + let second = server.toolName(for: "repo.search") + #expect(first == second) + } + + @Test("Names that differ only in a replaced character or past the cut stay distinct") + func foldedNamesStayDistinct() { + #expect(server.toolName(for: "repo.search") != server.toolName(for: "repo/search")) + #expect(server.toolName(for: "repo.search") != server.toolName(for: "repo_search")) + let long = String(repeating: "x", count: 40) + #expect(server.toolName(for: long + "a") != server.toolName(for: long + "b")) + } +} diff --git a/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift b/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift index 1a8a41c965..e81c968cb2 100644 --- a/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift +++ b/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift @@ -66,6 +66,32 @@ enum CSVImportParsing { return row } + struct RowShapeError: LocalizedError, Equatable { + let fieldCount: Int + let columnCount: Int + + var errorDescription: String? { + String( + format: String( + localized: "The row has %1$lld fields where %2$lld are expected. A field that holds the delimiter must be quoted.", + bundle: .main + ), + Int64(fieldCount), + Int64(columnCount) + ) + } + } + + // A short row is padded, as it always was: nothing in the file is lost. Only a surplus field + // that still holds something after trimming would be cut. + static func shapeError(fields: [String], columnCount: Int, options: CSVImportOptions) -> RowShapeError? { + guard fields.count > columnCount else { return nil } + let surplusIsEmpty = fields[columnCount...].allSatisfy { field in + options.trimWhitespace ? field.trimmingCharacters(in: .whitespaces).isEmpty : field.isEmpty + } + return surplusIsEmpty ? nil : RowShapeError(fieldCount: fields.count, columnCount: columnCount) + } + static func importFieldType(for type: CSVTypeInferrer.InferredType) -> PluginImportFieldType { switch type { case .integer: return .integer diff --git a/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift b/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift index b52fcf2174..0a3b2d9ae3 100644 --- a/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift +++ b/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift @@ -46,6 +46,8 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi AnyView(CSVImportOptionsView(plugin: self)) } + private typealias MalformedRow = (line: Int, error: CSVImportParsing.RowShapeError) + private static let detectionPrefixBytes = 1_048_576 private static let batchSize = 500 @@ -91,14 +93,19 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi let lineOffset = hasHeader ? 2 : 1 var cursor = 0 + let configuration = runConfiguration + var pendingFailure: PluginImportError? + var malformedCount = 0 + var malformedErrors: [PluginImportResult.ImportStatementError] = [] let outcome = try await RowImportRunner.run( - configuration: runConfiguration, + configuration: configuration, sink: sink, progress: progress ) { + if let pendingFailure { throw pendingFailure } guard cursor < dataRanges.count else { return nil } let end = min(cursor + Self.batchSize, dataRanges.count) - let batch = self.parseBatch( + let parsed = self.parseBatch( in: data, parser: parser, ranges: dataRanges[cursor.. 0 { progress.incrementStatement(by: blankRows) } cursor = end + guard let first = malformed.first else { return batch } + guard configuration.errorHandling == .skipAndContinue else { + // The rows above the malformed one go in first, as they do when the database refuses a row. + batch.removeAll { $0.line > first.line } + pendingFailure = .statementFailed( + statement: "row \(first.line)", line: first.line, underlyingError: first.error) + return batch + } + malformedCount += malformed.count + for entry in malformed where malformedErrors.count < configuration.maxRecordedErrors { + malformedErrors.append(.init( + statement: "row \(entry.line)", line: entry.line, errorMessage: entry.error.localizedDescription)) + } return batch } + let errors = (outcome.errors + malformedErrors).sorted { $0.line < $1.line } return PluginImportResult( executedStatements: outcome.inserted, executionTime: Date().timeIntervalSince(startTime), - skippedStatements: outcome.skipped, - errors: outcome.errors + skippedStatements: outcome.skipped + malformedCount, + errors: Array(errors.prefix(configuration.maxRecordedErrors)) ) } @@ -150,19 +173,26 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi startIndex: Int, lineOffset: Int, columnNames: [String] - ) -> [(line: Int, row: [String: PluginCellValue])] { - data.withUnsafeBytes { raw -> [(line: Int, row: [String: PluginCellValue])] in - guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return [] } + ) -> (rows: [RowImportRunner.Entry], malformed: [MalformedRow]) { + data.withUnsafeBytes { raw -> (rows: [RowImportRunner.Entry], malformed: [MalformedRow]) in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return ([], []) } let buffer = UnsafeBufferPointer(start: base, count: raw.count) - var out: [(line: Int, row: [String: PluginCellValue])] = [] + var out: [RowImportRunner.Entry] = [] + var malformed: [MalformedRow] = [] out.reserveCapacity(ranges.count) for (offset, range) in ranges.enumerated() { let fields = parser.parseRow(buffer, range: range) if CSVImportParsing.isBlank(fields) { continue } let line = startIndex + offset + lineOffset + if let error = CSVImportParsing.shapeError( + fields: fields, columnCount: columnNames.count, options: settings + ) { + malformed.append((line, error)) + continue + } out.append((line, CSVImportParsing.row(fields: fields, columnNames: columnNames, options: settings))) } - return out + return (out, malformed) } } diff --git a/apple/Plugins/ColumnTypeCorpus/Corpus/mariadb.json b/apple/Plugins/ColumnTypeCorpus/Corpus/mariadb.json index 5913fc3011..3e1f30eba0 100644 --- a/apple/Plugins/ColumnTypeCorpus/Corpus/mariadb.json +++ b/apple/Plugins/ColumnTypeCorpus/Corpus/mariadb.json @@ -218,8 +218,7 @@ "values": [{"sql": "NULL", "cell": null}, {"sql": "b'10101010'", "cell": "170"}, {"sql": "b'1'", "cell": "1"}], "edit": {"row": 2, "text": "5", "cell": "5"}, "known": { - "edit": {"defect": "the grid saves a BIT cell as text, and the server stores the text's bytes: 5 saves as 53", "matches": "reads back \"53\""}, - "sql": {"defect": "the dump writes a BIT value as quoted text, which restores as the text's bytes", "matches": "the replayed dump reads"} + "edit": {"defect": "the grid saves a BIT cell as text, and the server stores the text's bytes: 5 saves as 53", "matches": "reads back \"53\""} } }, { diff --git a/apple/Plugins/ColumnTypeCorpus/Corpus/mysql.json b/apple/Plugins/ColumnTypeCorpus/Corpus/mysql.json index 2a84fa621a..2a3c520b25 100644 --- a/apple/Plugins/ColumnTypeCorpus/Corpus/mysql.json +++ b/apple/Plugins/ColumnTypeCorpus/Corpus/mysql.json @@ -218,8 +218,7 @@ "values": [{"sql": "NULL", "cell": null}, {"sql": "b'10101010'", "cell": "170"}, {"sql": "b'1'", "cell": "1"}], "edit": {"row": 2, "text": "5", "cell": "5"}, "known": { - "edit": {"defect": "the grid saves a BIT cell as text, and the server stores the text's bytes: 5 saves as 53", "matches": "reads back \"53\""}, - "sql": {"defect": "the dump writes a BIT value as quoted text, which restores as the text's bytes", "matches": "the replayed dump reads"} + "edit": {"defect": "the grid saves a BIT cell as text, and the server stores the text's bytes: 5 saves as 53", "matches": "reads back \"53\""} } }, { diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift index b201bc0689..321389ab22 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift @@ -6,6 +6,7 @@ import CDuckDB import Foundation import os +import TableProDuckDBCore import TableProPluginKit /// Everything a reopen needs to put the connection back where it was. Held from the first open, @@ -39,6 +40,9 @@ actor DuckDBConnectionActor { private var database: duckdb_database? private var connection: duckdb_connection? + // Renders the rows of a result the session's own connection is still producing. + private var renderConnection: duckdb_connection? + private var renderSettings: [String: String] = [:] private var resultReader: DuckDBResultReader? private var openSpec: DuckDBOpenSpec? @@ -174,19 +178,20 @@ actor DuckDBConnectionActor { } func executeQuery(_ query: String) throws -> DuckDBRawResult { - try ensureOpen() - guard let conn = connection else { - throw DuckDBPluginError.notConnected - } - - let startTime = Date() - noteActivity(query) - var resolved = try resolvedResult(query: query, parameters: [], connection: conn) - defer { duckdb_destroy_result(&resolved.result) } - return try Self.extractResult(from: &resolved.result, schema: resolved.schema, rowsChanged: resolved.rowsChanged, startTime: startTime) + try executePrepared(query, parameters: []) } func executePrepared(_ query: String, parameters: [PluginCellValue]) throws -> DuckDBRawResult { + try run(query, parameters: parameters, limit: PluginRowLimits.emergencyMax, unread: .drain) + } + + func executeBounded(_ query: String, parameters: [PluginCellValue], rowCap: Int) throws -> DuckDBRawResult { + try run(query, parameters: parameters, limit: rowCap, unread: .abandonWhenIdle) + } + + private func run( + _ query: String, parameters: [PluginCellValue], limit: Int, unread: DuckDBUnreadRows + ) throws -> DuckDBRawResult { try ensureOpen() guard let conn = connection else { throw DuckDBPluginError.notConnected @@ -194,9 +199,9 @@ actor DuckDBConnectionActor { let startTime = Date() noteActivity(query) - var resolved = try resolvedResult(query: query, parameters: parameters, connection: conn) - defer { duckdb_destroy_result(&resolved.result) } - return try Self.extractResult(from: &resolved.result, schema: resolved.schema, rowsChanged: resolved.rowsChanged, startTime: startTime) + var opened = try openResult(query: query, parameters: parameters, connection: conn, unread: unread) + defer { duckdb_destroy_result(&opened.result) } + return try extractResult(from: opened, limit: limit, startTime: startTime) } func streamQuery( @@ -209,20 +214,20 @@ actor DuckDBConnectionActor { } noteActivity(query) - var resolved = try resolvedResult(query: query, parameters: [], connection: conn) - defer { duckdb_destroy_result(&resolved.result) } - try Self.streamResultRows(&resolved.result, schema: resolved.schema, continuation: continuation) + var opened = try openResult(query: query, parameters: [], connection: conn, unread: .abandon) + defer { duckdb_destroy_result(&opened.result) } + try streamResultRows(opened, continuation: continuation) } // The state check, the BEGIN, the statement and the ROLLBACK share one actor turn, so nothing lands in between. // The statement text is not tracked: the ROLLBACK leaves the session as idle as it was found. - func executeReadOnly(_ query: String, parameters: [PluginCellValue]) throws -> DuckDBRawResult { + func executeReadOnly(_ query: String, parameters: [PluginCellValue], rowCap: Int? = nil) throws -> DuckDBRawResult { let conn = try readOnlyConnection() let startTime = Date() return try DuckDBReadOnlyStatement.run(on: conn, session: sessionTransactionState()) { - var resolved = try resolvedResult(query: query, parameters: parameters, connection: conn) - defer { duckdb_destroy_result(&resolved.result) } - return try Self.extractResult(from: &resolved.result, schema: resolved.schema, rowsChanged: resolved.rowsChanged, startTime: startTime) + var opened = try openResult(query: query, parameters: parameters, connection: conn, unread: .abandon) + defer { duckdb_destroy_result(&opened.result) } + return try extractResult(from: opened, limit: rowCap ?? PluginRowLimits.emergencyMax, startTime: startTime) } } @@ -232,9 +237,9 @@ actor DuckDBConnectionActor { ) throws { let conn = try readOnlyConnection() try DuckDBReadOnlyStatement.run(on: conn, session: sessionTransactionState()) { - var resolved = try resolvedResult(query: query, parameters: [], connection: conn) - defer { duckdb_destroy_result(&resolved.result) } - try Self.streamResultRows(&resolved.result, schema: resolved.schema, continuation: continuation) + var opened = try openResult(query: query, parameters: [], connection: conn, unread: .abandon) + defer { duckdb_destroy_result(&opened.result) } + try streamResultRows(opened, continuation: continuation) } } @@ -324,9 +329,14 @@ actor DuckDBConnectionActor { duckdb_close(&db) throw DuckDBPluginError.connectionFailed("Failed to create DuckDB connection") } + var renderConn: duckdb_connection? do { - resultReader = try DuckDBResultReader(connection: conn) + guard duckdb_connect(openedDB, &renderConn) != DuckDBError, let renderConn else { + throw DuckDBPluginError.connectionFailed("Failed to create DuckDB connection") + } + resultReader = try DuckDBResultReader(renderConnection: renderConn) } catch { + if renderConn != nil { duckdb_disconnect(&renderConn) } var failedConnection: duckdb_connection? = conn duckdb_disconnect(&failedConnection) duckdb_close(&db) @@ -334,12 +344,18 @@ actor DuckDBConnectionActor { } database = db connection = conn + renderConnection = renderConn + renderSettings = [:] liveConnection.set(conn) } private func closeHandle() { liveConnection.set(nil) resultReader = nil + if renderConnection != nil { + duckdb_disconnect(&renderConnection) + renderConnection = nil + } if connection != nil { duckdb_disconnect(&connection) connection = nil @@ -430,7 +446,9 @@ actor DuckDBConnectionActor { throw DuckDBPluginError.queryFailed(Self.errorMessage(from: &raw)) } let schema = Self.schema(of: &raw) - let result = try Self.extractResult(from: &raw, schema: schema, rowsChanged: 0, startTime: Date()) + let rowCount = duckdb_row_count(&raw) + let opened = OpenedResult(result: raw, schema: schema, rowsChanged: 0, rowCount: rowCount, abandonsUnreadRows: true) + let result = try extractResult(from: opened, limit: PluginRowLimits.emergencyMax, startTime: Date()) guard let row = result.rows.first, row.count >= 2 else { throw DuckDBPluginError.queryFailed("DuckDB reported no session state") } @@ -469,9 +487,9 @@ actor DuckDBConnectionActor { private func runInternalQuery(_ query: String) throws -> DuckDBRawResult { guard let conn = connection else { throw DuckDBPluginError.notConnected } let startTime = Date() - var resolved = try resolvedResult(query: query, parameters: [], connection: conn) - defer { duckdb_destroy_result(&resolved.result) } - return try Self.extractResult(from: &resolved.result, schema: resolved.schema, rowsChanged: resolved.rowsChanged, startTime: startTime) + var opened = try openResult(query: query, parameters: [], connection: conn, unread: .drain) + defer { duckdb_destroy_result(&opened.result) } + return try extractResult(from: opened, limit: PluginRowLimits.emergencyMax, startTime: startTime) } @discardableResult @@ -479,42 +497,24 @@ actor DuckDBConnectionActor { try runInternalQuery(statement) } - // MARK: - Result rendering + // MARK: - Execution - private func resolvedResult( - query: String, parameters: [PluginCellValue], connection: duckdb_connection - ) throws -> (result: duckdb_result, schema: ColumnSchema, rowsChanged: idx_t) { - let executed = try Self.runPrepared(query, parameters: parameters, on: connection) - var result = executed.result - let schema = Self.schema(of: &result) - var rowsChanged = duckdb_rows_changed(&result) - if rowsChanged == 0, duckdb_result_return_type(result) == DUCKDB_RESULT_TYPE_QUERY_RESULT { - switch executed.statementType { - case DUCKDB_STATEMENT_TYPE_INSERT, DUCKDB_STATEMENT_TYPE_UPDATE, DUCKDB_STATEMENT_TYPE_DELETE: - // RETURNING carries the changed rows themselves instead of a separate count. - rowsChanged = duckdb_row_count(&result) - default: - break - } - } - guard DuckDBLogicalType.requiresTextProjection(anyOf: schema.logicalTypes), - duckdb_row_count(&result) > 0 else { - return (result, schema, rowsChanged) - } - defer { duckdb_destroy_result(&result) } - try Task.checkCancellation() - guard let resultReader else { throw DuckDBPluginError.notConnected } - let projected = try resultReader.render(result, columns: schema.projectionColumns, on: connection) - return (projected, schema, rowsChanged) - } + private struct OpenedResult { + var result: duckdb_result + let schema: ColumnSchema + let rowsChanged: idx_t + // Nil for a streaming result, whose length is unknown until it ends. + let rowCount: idx_t? + let abandonsUnreadRows: Bool - // MARK: - Execution + var isStreaming: Bool { rowCount == nil } + } - private static func runPrepared( - _ sql: String, parameters: [PluginCellValue], on connection: duckdb_connection - ) throws -> (result: duckdb_result, statementType: duckdb_statement_type) { + private func openResult( + query: String, parameters: [PluginCellValue], connection: duckdb_connection, unread: DuckDBUnreadRows + ) throws -> OpenedResult { var stmtOpt: duckdb_prepared_statement? - let prepareState = duckdb_prepare(connection, sql, &stmtOpt) + let prepareState = duckdb_prepare(connection, query, &stmtOpt) defer { duckdb_destroy_prepare(&stmtOpt) } guard prepareState != DuckDBError, let stmt = stmtOpt else { @@ -525,17 +525,74 @@ actor DuckDBConnectionActor { throw DuckDBPluginError.queryFailed(message) } - try bind(parameters, to: stmt) + try Self.bind(parameters, to: stmt) // Result metadata reports EXECUTE, so affected counts need the prepared statement's original kind. let statementType = duckdb_prepared_statement_type(stmt) + // Only a SELECT streams. A statement that changes rows takes effect when its result ends, so it runs whole. + let streams = statementType == DUCKDB_STATEMENT_TYPE_SELECT + + // Both read the session, which a streaming result keeps busy from here until it ends. + alignRenderSession(for: stmt, on: connection) + let abandonsUnreadRows = streams && unread.abandons { sessionTransactionState() == .idle } var result = duckdb_result() - guard duckdb_execute_prepared(stmt, &result) != DuckDBError else { - let message = errorMessage(from: &result) + let state = streams ? duckdb_execute_prepared_streaming(stmt, &result) : duckdb_execute_prepared(stmt, &result) + guard state != DuckDBError else { + let message = Self.errorMessage(from: &result) duckdb_destroy_result(&result) throw DuckDBPluginError.queryFailed(message) } - return (result, statementType) + + let schema = Self.schema(of: &result) + guard !duckdb_result_is_streaming(result) else { + return OpenedResult( + result: result, schema: schema, rowsChanged: 0, rowCount: nil, abandonsUnreadRows: abandonsUnreadRows + ) + } + let rowCount = duckdb_row_count(&result) + var rowsChanged = duckdb_rows_changed(&result) + if rowsChanged == 0, duckdb_result_return_type(result) == DUCKDB_RESULT_TYPE_QUERY_RESULT { + switch statementType { + case DUCKDB_STATEMENT_TYPE_INSERT, DUCKDB_STATEMENT_TYPE_UPDATE, DUCKDB_STATEMENT_TYPE_DELETE: + // RETURNING carries the changed rows themselves instead of a separate count. + rowsChanged = rowCount + default: + break + } + } + return OpenedResult( + result: result, schema: schema, rowsChanged: rowsChanged, rowCount: rowCount, abandonsUnreadRows: true + ) + } + + private static let typesSpelledBySession: Set = [.timestampTz, .list, .array, .structure, .map, .union] + + // TIMESTAMPTZ text follows the session's time zone and calendar, which each connection holds for itself. + private func alignRenderSession(for statement: duckdb_prepared_statement, on connection: duckdb_connection) { + guard let resultReader else { return } + let followsSession = (0.. DuckDBRawResult { - let rowCount = duckdb_row_count(&result) - let resultTypes = storageTypes(of: &result) - - let maxRows = min(rowCount, UInt64(PluginRowLimits.emergencyMax)) + private func extractResult(from opened: OpenedResult, limit: Int, startTime: Date) throws -> DuckDBRawResult { var rows: [[PluginCellValue]] = [] - rows.reserveCapacity(Int(maxRows)) + if let rowCount = opened.rowCount { + rows.reserveCapacity(Int(min(rowCount, UInt64(max(0, limit))))) + } - try readChunks(of: result, types: resultTypes, limit: maxRows) { batch in - rows.append(contentsOf: batch) - return true + var budget = DuckDBFetchBudget(limit: limit) + let reachedEnd = try readRows(of: opened) { batch in + let kept = budget.take(batch.count) + rows.append(contentsOf: kept == batch.count ? batch[...] : batch.prefix(kept)) + return !budget.isExceeded + } + if !reachedEnd, opened.isStreaming, !opened.abandonsUnreadRows { + try Self.drain(opened.result) } return DuckDBRawResult( - columns: schema.names, - columnTypeNames: schema.typeNames, + columns: opened.schema.names, + columnTypeNames: opened.schema.typeNames, rows: rows, - rowsAffected: Int(rowsChanged), + rowsAffected: Int(opened.rowsChanged), executionTime: Date().timeIntervalSince(startTime), - isTruncated: rowCount > UInt64(PluginRowLimits.emergencyMax) + isTruncated: budget.isExceeded ) } - private static func streamResultRows( - _ result: inout duckdb_result, - schema: ColumnSchema, + private func streamResultRows( + _ opened: OpenedResult, continuation: AsyncThrowingStream.Continuation ) throws { - let rowCount = duckdb_row_count(&result) - let resultTypes = storageTypes(of: &result) - continuation.yield(.header(PluginStreamHeader( - columns: schema.names, - columnTypeNames: schema.typeNames, - estimatedRowCount: Int(rowCount) + columns: opened.schema.names, + columnTypeNames: opened.schema.typeNames, + estimatedRowCount: opened.rowCount.map { Int($0) } ))) - let maxRows = min(rowCount, UInt64(PluginRowLimits.emergencyMax)) - if rowCount > UInt64(PluginRowLimits.emergencyMax) { - logger.warning("streamQuery truncating result from \(rowCount) to \(maxRows) rows") - } - var cancelled = false - try readChunks(of: result, types: resultTypes, limit: maxRows) { batch in + try readRows(of: opened) { batch in cancelled = Task.isCancelled - if !cancelled { continuation.yield(.rows(batch)) } + || !DuckDBBoundedStream.deliver(PluginStreamElement.rows(batch), to: continuation) return !cancelled } if cancelled { @@ -658,19 +704,49 @@ actor DuckDBConnectionActor { } } - private static func storageTypes(of result: inout duckdb_result) -> [duckdb_type] { - (0.. Bool) throws -> Bool { + var source = opened.result + let types = Self.storageTypes(of: &source) + let needsText = DuckDBLogicalType.requiresTextProjection(anyOf: opened.schema.logicalTypes) + var projection: duckdb_prepared_statement? + defer { duckdb_destroy_prepare(&projection) } + + while true { + // A statement that ran whole has already taken effect, so its rows are still handed back. + if opened.isStreaming { try Task.checkCancellation() } + var chunk = duckdb_fetch_chunk(source) + guard let fetched = chunk else { + if let message = duckdb_result_error(&source) { + throw DuckDBPluginError.queryFailed(String(cString: message)) + } + return true + } + defer { duckdb_destroy_data_chunk(&chunk) } + guard duckdb_data_chunk_get_size(fetched) > 0 else { continue } + + let batch: [[PluginCellValue]] + if needsText { + guard let resultReader else { throw DuckDBPluginError.notConnected } + if projection == nil { + projection = try resultReader.prepareProjection(of: source, columns: opened.schema.projectionColumns) + } + guard let projection else { throw DuckDBPluginError.notConnected } + var rendered = try resultReader.render(fetched, of: source, with: projection) + defer { duckdb_destroy_result(&rendered) } + batch = try Self.renderedRows(of: &rendered) + } else { + batch = Self.rows(of: fetched, types: types) + } + guard body(batch) else { return false } + } } - private static func readChunks( - of result: duckdb_result, - types: [duckdb_type], - limit: UInt64, - body: ([[PluginCellValue]]) -> Bool - ) throws { + // Completes a statement whose rows are no longer wanted, so that what it changed is kept. + private static func drain(_ result: duckdb_result) throws { var source = result - var remaining = limit - while remaining > 0 { + while true { var chunk = duckdb_fetch_chunk(source) guard chunk != nil else { if let message = duckdb_result_error(&source) { @@ -678,17 +754,39 @@ actor DuckDBConnectionActor { } return } - defer { duckdb_destroy_data_chunk(&chunk) } - let size = min(duckdb_data_chunk_get_size(chunk), remaining) - let vectors = types.indices.map { duckdb_data_chunk_get_vector(chunk, idx_t($0)) } - var batch: [[PluginCellValue]] = [] - batch.reserveCapacity(Int(size)) - for row in 0.. [[PluginCellValue]] { + let types = storageTypes(of: &rendered) + var batch: [[PluginCellValue]] = [] + while true { + var chunk = duckdb_fetch_chunk(rendered) + guard let fetched = chunk else { + if let message = duckdb_result_error(&rendered) { + throw DuckDBPluginError.queryFailed(String(cString: message)) + } + return batch } - remaining -= size - guard body(batch) else { return } + defer { duckdb_destroy_data_chunk(&chunk) } + batch.append(contentsOf: rows(of: fetched, types: types)) + } + } + + private static func storageTypes(of result: inout duckdb_result) -> [duckdb_type] { + (0.. [[PluginCellValue]] { + let size = duckdb_data_chunk_get_size(chunk) + let vectors = types.indices.map { duckdb_data_chunk_get_vector(chunk, idx_t($0)) } + var batch: [[PluginCellValue]] = [] + batch.reserveCapacity(Int(size)) + for row in 0.. PluginCellValue { diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift index 4e262f044a..19da1b5586 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBPlugin.swift @@ -715,6 +715,18 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return Self.queryResult(try await connectionActor.executeReadOnly(query, parameters: parameters)) } + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + guard rowCap > 0 else { return nil } + return Self.queryResult(try await connectionActor.executeBounded(query, parameters: [], rowCap: rowCap)) + } + + func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { + guard let cap = rowCap, cap > 0 else { + return Self.queryResult(try await connectionActor.executePrepared(query, parameters: parameters ?? [])) + } + return Self.queryResult(try await connectionActor.executeBounded(query, parameters: parameters ?? [], rowCap: cap)) + } + func executeUserQuery( query: String, rowCap: Int?, @@ -724,16 +736,8 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { guard context.readOnly, context.origin == .external else { return try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) } - let raw = Self.queryResult(try await connectionActor.executeReadOnly(query, parameters: parameters ?? [])) - guard let cap = rowCap, cap > 0, raw.rows.count > cap else { return raw } - return PluginQueryResult( - columns: raw.columns, - columnTypeNames: raw.columnTypeNames, - rows: Array(raw.rows.prefix(cap)), - rowsAffected: raw.rowsAffected, - executionTime: raw.executionTime, - isTruncated: true - ) + let cap = rowCap.flatMap { $0 > 0 ? $0 : nil } + return Self.queryResult(try await connectionActor.executeReadOnly(query, parameters: parameters ?? [], rowCap: cap)) } private static func queryResult(_ rawResult: DuckDBRawResult) -> PluginQueryResult { @@ -756,7 +760,7 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { func streamRows(query: String) -> AsyncThrowingStream { let actor = connectionActor - return AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + return AsyncThrowingStream(bufferingPolicy: .bufferingOldest(DuckDBBoundedStream.bufferedBatches)) { continuation in Task { do { try await actor.streamQuery(query, continuation: continuation) @@ -770,7 +774,7 @@ final class DuckDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { func streamRows(query: String, context: PluginStatementContext) -> AsyncThrowingStream { guard context.readOnly, context.origin == .external else { return streamRows(query: query) } let actor = connectionActor - return AsyncThrowingStream(bufferingPolicy: .unbounded) { continuation in + return AsyncThrowingStream(bufferingPolicy: .bufferingOldest(DuckDBBoundedStream.bufferedBatches)) { continuation in Task { do { try await actor.streamReadOnly(query, continuation: continuation) diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBResultReader.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBResultReader.swift index 16ea644357..fa1e894823 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBResultReader.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBResultReader.swift @@ -1,14 +1,18 @@ import CDuckDB import Foundation -// A table function lets DuckDB render an existing result without running the user's statement again. +// DuckDB spells a value as text only through a query. A streaming result keeps its own connection busy until it ends +// (DUCKDB_ERROR_RESOURCE_IN_USE), so a second connection renders each fetched chunk through a table function. final class DuckDBResultReader: @unchecked Sendable { private let name = "_tablepro_result_" + UUID().uuidString.replacingOccurrences(of: "-", with: "") + private let connection: duckdb_connection private let connectionID: idx_t private let lock = NSLock() private var source: duckdb_result? + private var pendingChunk: duckdb_data_chunk? - init(connection: duckdb_connection) throws { + init(renderConnection connection: duckdb_connection) throws { + self.connection = connection var context: duckdb_client_context? duckdb_connection_get_client_context(connection, &context) defer { duckdb_destroy_client_context(&context) } @@ -39,23 +43,42 @@ final class DuckDBResultReader: @unchecked Sendable { } } + func send(_ sql: String) -> Bool { + var result = duckdb_result() + defer { duckdb_destroy_result(&result) } + return duckdb_query(connection, sql, &result) == DuckDBSuccess + } + + func prepareProjection( + of result: duckdb_result, + columns: [(name: String, type: DuckDBLogicalType?)] + ) throws -> duckdb_prepared_statement { + setSource(result, chunk: nil) + defer { setSource(nil, chunk: nil) } + + let sql = DuckDBProjectedQuery.build(function: name, columns: columns) + var statement: duckdb_prepared_statement? + guard duckdb_prepare(connection, sql, &statement) == DuckDBSuccess, let prepared = statement else { + let message = statement.flatMap { duckdb_prepare_error($0) }.map { String(cString: $0) } + ?? "Could not render the DuckDB result" + duckdb_destroy_prepare(&statement) + throw DuckDBPluginError.queryFailed(message) + } + return prepared + } + + // The chunk has to outlive the call: the rendered result is materialized before this returns. func render( - _ result: duckdb_result, - columns: [(name: String, type: DuckDBLogicalType?)], - on connection: duckdb_connection + _ chunk: duckdb_data_chunk, + of result: duckdb_result, + with projection: duckdb_prepared_statement ) throws -> duckdb_result { - lock.lock() - source = result - lock.unlock() - defer { - lock.lock() - source = nil - lock.unlock() - } + // DuckDB binds the function again on every execution, so the result stays readable for its column types. + setSource(result, chunk: chunk) + defer { setSource(nil, chunk: nil) } - let sql = DuckDBProjectedQuery.build(function: name, columns: columns) var rendered = duckdb_result() - guard duckdb_query(connection, sql, &rendered) == DuckDBSuccess else { + guard duckdb_execute_prepared(projection, &rendered) == DuckDBSuccess else { let message = duckdb_result_error(&rendered).map { String(cString: $0) } ?? "Could not render the DuckDB result" duckdb_destroy_result(&rendered) @@ -64,6 +87,13 @@ final class DuckDBResultReader: @unchecked Sendable { return rendered } + private func setSource(_ result: duckdb_result?, chunk: duckdb_data_chunk?) { + lock.lock() + source = result + pendingChunk = chunk + lock.unlock() + } + private func bind(_ info: duckdb_bind_info) { var context: duckdb_client_context? duckdb_table_function_get_client_context(info, &context) @@ -88,24 +118,16 @@ final class DuckDBResultReader: @unchecked Sendable { } duckdb_bind_add_result_column(info, DuckDBProjectedQuery.sourceAlias(at: Int(column)), type) } - duckdb_bind_set_cardinality(info, duckdb_row_count(&source), true) } private func scan(_ info: duckdb_function_info, into output: duckdb_data_chunk) { lock.lock() defer { lock.unlock() } - guard var source else { - duckdb_function_set_error(info, "No result is being read") - duckdb_data_chunk_set_size(output, 0) - return - } - var chunk = duckdb_fetch_chunk(source) - defer { duckdb_destroy_data_chunk(&chunk) } - guard let chunk else { - if let message = duckdb_result_error(&source) { duckdb_function_set_error(info, message) } + guard let chunk = pendingChunk else { duckdb_data_chunk_set_size(output, 0) return } + pendingChunk = nil for column in 0.. [ElasticsearchColumn] { var columns: [ElasticsearchColumn] = [] - collect(properties: properties, prefix: "", nestedPaths: [], into: &columns) + collect(properties: properties, prefix: "", pathKeys: [], nestedPaths: [], into: &columns) return columns.sorted { $0.name.localizedCaseInsensitiveCompare($1.name) == .orderedAscending } } private static func collect( properties: [String: Any], prefix: String, + pathKeys: [String], nestedPaths: [String], into columns: inout [ElasticsearchColumn] ) { @@ -84,7 +96,8 @@ enum ElasticsearchMappingFlattener { name: path, type: type ?? "object", hasKeywordSubfield: hasKeyword, - nestedPaths: nestedPaths + nestedPaths: nestedPaths, + pathKeys: pathKeys + [key] )) continue } @@ -94,12 +107,14 @@ enum ElasticsearchMappingFlattener { name: path, type: nestedTypeName, hasKeywordSubfield: false, - nestedPaths: nestedPaths + nestedPaths: nestedPaths, + pathKeys: pathKeys + [key] )) } collect( properties: children, prefix: path, + pathKeys: pathKeys + [key], nestedPaths: isNested ? nestedPaths + [path] : nestedPaths, into: &columns ) diff --git a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver+Execution.swift b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver+Execution.swift index 45c023ced1..63a0b38d19 100644 --- a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver+Execution.swift +++ b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver+Execution.swift @@ -7,6 +7,7 @@ import Foundation import os +import TableProDocumentPath import TableProPluginKit extension ElasticsearchPluginDriver { @@ -276,12 +277,16 @@ extension ElasticsearchPluginDriver { conn: ElasticsearchConnection, startTime: Date ) async throws -> PluginQueryResult { - guard let request = ElasticsearchStatementGenerator.decode(statement) else { + guard let decoded = ElasticsearchStatementGenerator.decode(statement) else { throw ElasticsearchError.invalidResponse("Invalid write request") } + let request = try await alignedToStoredDocument(decoded, conn: conn) let response = try await conn.request(method: request.method, path: request.path, body: request.body) guard (200..<300).contains(response.statusCode) else { + if response.statusCode == 409, let reason = Self.conflictReason(for: decoded) { + throw ElasticsearchError.serverError(reason) + } throw mapWriteError(response) } @@ -295,6 +300,41 @@ extension ElasticsearchPluginDriver { ) } + // A partial update merges by literal key, so a field the document spells `a.b` and one it + // spells `{"a": {"b": ...}}` need different requests. Only the stored document says which. + private func alignedToStoredDocument( + _ request: ElasticsearchWriteRequest, + conn: ElasticsearchConnection + ) async throws -> ElasticsearchWriteRequest { + guard let path = ElasticsearchStatementGenerator.storedDocumentPath(for: request) else { return request } + let response = try await conn.request(method: "GET", path: path) + guard response.statusCode == 200, + let json = response.json as? [String: Any], + let stored = json["_source"] as? [String: Any] + else { return request } + do { + return try ElasticsearchStatementGenerator.aligned( + request, + toStored: stored, + seqNo: (json["_seq_no"] as? NSNumber)?.intValue, + primaryTerm: (json["_primary_term"] as? NSNumber)?.intValue + ) + } catch let failure as DocumentPathPlacement.InsideArray { + throw ElasticsearchError.serverError(String( + format: String(localized: "'%@' is a field of an array of objects in this document. Change it with a query."), + failure.path + )) + } + } + + private static func conflictReason(for request: ElasticsearchWriteRequest) -> String? { + if ElasticsearchStatementGenerator.isCreateOnly(request) { + return String(localized: "A document with this _id already exists.") + } + guard !request.leafPaths.isEmpty else { return nil } + return String(localized: "The document changed on the server after it was read. Refresh and try again.") + } + // MARK: - Console private func executeConsole( diff --git a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift index 692c0a9130..2d51700557 100644 --- a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift +++ b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift @@ -249,7 +249,12 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked insertedRowIndices: Set ) throws -> [PluginRowWrite]? { let typeNames = columnTypeNames(for: columns, index: table) - let generator = ElasticsearchStatementGenerator(index: table, columns: columns, columnTypeNames: typeNames) + let generator = ElasticsearchStatementGenerator( + index: table, + columns: columns, + columnTypeNames: typeNames, + columnPathKeys: columnPathKeys(index: table) + ) return try generator.generateRowWrites( from: changes, insertedRowData: insertedRowData, @@ -278,6 +283,11 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked lock.withLock { _mappingCache.removeAll() } } + private func columnPathKeys(index: String) -> [String: [String]] { + let cached = lock.withLock { _mappingCache[index] } ?? [] + return Dictionary(cached.map { ($0.name, $0.pathKeys) }) { first, _ in first } + } + private func columnTypeNames(for columns: [String], index: String) -> [String] { let cached = lock.withLock { _mappingCache[index] } ?? [] var lookup: [String: String] = [:] diff --git a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift index c83ef06702..0bfc05bcd3 100644 --- a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift +++ b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift @@ -6,6 +6,7 @@ // import Foundation +import TableProDocumentPath import TableProNumberFormatting import TableProPluginKit @@ -13,11 +14,24 @@ struct ElasticsearchWriteRequest: Equatable { let method: String let path: String let body: String? + // The fields a partial update sets, each as its mapping keys. The stored document decides how + // each one is spelled, so the request is aligned to it when it runs. + let leafPaths: [[String]] + + init(method: String, path: String, body: String?, leafPaths: [[String]] = []) { + self.method = method + self.path = path + self.body = body + self.leafPaths = leafPaths + } } struct ElasticsearchStatementGenerator { static let writeTag = "ELASTICSEARCH_WRITE:" private static let refreshQuery = "?refresh=true" + // Plain `PUT _doc/` replaces a document that already has the id. + private static let createOnlyQuery = "&op_type=create" + private static let updateEndpoint = "_update" let index: String let columns: [String] @@ -29,11 +43,13 @@ struct ElasticsearchStatementGenerator { /// through its parent and a value typed into a leaf is refused rather than sent. Each leaf maps /// to the outermost array it belongs to, which is the column that writes it. private let nestedParentByLeaf: [String: String] + private let columnPathKeys: [String: [String]] - init(index: String, columns: [String], columnTypeNames: [String]) { + init(index: String, columns: [String], columnTypeNames: [String], columnPathKeys: [String: [String]] = [:]) { self.index = index self.columns = columns self.columnTypeNames = columnTypeNames + self.columnPathKeys = columnPathKeys self.nestedParentByLeaf = Self.nestedParents(columns: columns, typeNames: columnTypeNames) } @@ -112,16 +128,19 @@ struct ElasticsearchStatementGenerator { var document: [String: Any] = [:] for column in columns where !metaColumns.contains(column) && nestedParentByLeaf[column] == nil { guard let value = values[column], let text = value.asText else { continue } - document[column] = jsonValue(text, for: column) + let json = jsonValue(text, for: column) + guard DocumentPathPlacement.set(json, atKeys: pathKeys(for: column), in: &document) else { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.notJSONReason) + } } - guard let body = serialize(document) else { + guard let body = Self.serialize(document) else { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.notJSONReason) } let explicitId = values[ElasticsearchMappingFlattener.idColumn]?.asText if let id = explicitId, !id.isEmpty { - return .init(method: "PUT", path: docPath(id: id), body: body) + return .init(method: "PUT", path: docPath(id: id) + Self.createOnlyQuery, body: body) } return .init(method: "POST", path: "/\(encodedIndex)/_doc\(Self.refreshQuery)", body: body) } @@ -188,6 +207,7 @@ struct ElasticsearchStatementGenerator { } var doc: [String: Any] = [:] + var leafPaths: [[String]] = [] for cellChange in change.cellChanges { let column = cellChange.columnName if metaColumns.contains(column) { @@ -198,20 +218,65 @@ struct ElasticsearchStatementGenerator { rowIndex: change.rowIndex, reason: Self.nestedLeafReason(leaf: column, parent: parent) ) } - if let text = cellChange.newValue.asText { - guard !Self.isShortened(cellChange.newValue), !Self.isShortened(cellChange.oldValue) else { - throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.shortenedValueReason(column)) - } - doc[column] = jsonValue(text, for: column) - } else { - doc[column] = NSNull() + if !cellChange.newValue.isNull, + Self.isShortened(cellChange.newValue) || Self.isShortened(cellChange.oldValue) { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.shortenedValueReason(column)) + } + let value: Any = cellChange.newValue.asText.map { jsonValue($0, for: column) } ?? NSNull() + let keys = pathKeys(for: column) + guard DocumentPathPlacement.set(value, atKeys: keys, in: &doc) else { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.notJSONReason) } + leafPaths.append(keys) } - guard let body = serialize(["doc": doc]) else { + guard let body = Self.serialize(["doc": doc]) else { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.notJSONReason) } - return .init(method: "POST", path: "/\(encodedIndex)/_update/\(encodePathComponent(id))\(Self.refreshQuery)", body: body) + return .init( + method: "POST", + path: "/\(encodedIndex)/\(Self.updateEndpoint)/\(encodePathComponent(id))\(Self.refreshQuery)", + body: body, + leafPaths: leafPaths + ) + } + + // MARK: - Stored Document Alignment + + static func isCreateOnly(_ request: ElasticsearchWriteRequest) -> Bool { + request.path.hasSuffix(createOnlyQuery) + } + + static func storedDocumentPath(for request: ElasticsearchWriteRequest) -> String? { + guard !request.leafPaths.isEmpty else { return nil } + let parts = (request.path.split(separator: "?", maxSplits: 1).first ?? "").split(separator: "/") + guard parts.count == 3, parts[1] == updateEndpoint else { return nil } + return "/\(parts[0])/_doc/\(parts[2])" + } + + static func aligned( + _ request: ElasticsearchWriteRequest, + toStored stored: [String: Any], + seqNo: Int?, + primaryTerm: Int? + ) throws -> ElasticsearchWriteRequest { + guard let body = request.body, + let parsed = try? JSONSerialization.jsonObject(with: Data(body.utf8)) as? [String: Any], + let doc = parsed["doc"] as? [String: Any] + else { return request } + + var placed: [String: Any] = [:] + for keys in request.leafPaths { + guard let value = DocumentPathPlacement.value(in: doc, atKeys: keys) else { return request } + try DocumentPathPlacement.place(value, atKeys: keys, in: &placed, following: stored) + } + guard let alignedBody = serialize(["doc": placed]) else { return request } + + var path = request.path + if let seqNo, let primaryTerm { + path += "&if_seq_no=\(seqNo)&if_primary_term=\(primaryTerm)" + } + return ElasticsearchWriteRequest(method: request.method, path: path, body: alignedBody) } // MARK: - DELETE @@ -262,6 +327,10 @@ struct ElasticsearchStatementGenerator { return id } + private func pathKeys(for column: String) -> [String] { + columnPathKeys[column] ?? column.split(separator: ".").map(String.init) + } + private func docPath(id: String) -> String { "/\(encodedIndex)/_doc/\(encodePathComponent(id))\(Self.refreshQuery)" } @@ -301,7 +370,7 @@ struct ElasticsearchStatementGenerator { return text } - private func serialize(_ object: [String: Any]) -> String? { + private static func serialize(_ object: [String: Any]) -> String? { guard JSONSerialization.isValidJSONObject(object), let data = try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) else { return nil } @@ -312,7 +381,11 @@ struct ElasticsearchStatementGenerator { let b64Method = Data(request.method.utf8).base64EncodedString() let b64Path = Data(request.path.utf8).base64EncodedString() let b64Body = Data((request.body ?? "").utf8).base64EncodedString() - return "\(writeTag)\(b64Method):\(b64Path):\(b64Body)" + let statement = "\(writeTag)\(b64Method):\(b64Path):\(b64Body)" + guard !request.leafPaths.isEmpty, + let leafData = try? JSONSerialization.data(withJSONObject: request.leafPaths) + else { return statement } + return "\(statement):\(leafData.base64EncodedString())" } static func decode(_ statement: String) -> ElasticsearchWriteRequest? { @@ -323,7 +396,15 @@ struct ElasticsearchStatementGenerator { let path = decodeBase64(parts[1]) else { return nil } let body = decodeBase64(parts[2]) - return ElasticsearchWriteRequest(method: method, path: path, body: (body?.isEmpty ?? true) ? nil : body) + let leafPaths = parts.count > 3 + ? Data(base64Encoded: parts[3]).flatMap { try? JSONSerialization.jsonObject(with: $0) as? [[String]] } + : nil + return ElasticsearchWriteRequest( + method: method, + path: path, + body: (body?.isEmpty ?? true) ? nil : body, + leafPaths: leafPaths ?? [] + ) } static func isTaggedStatement(_ statement: String) -> Bool { diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdCommandParser.swift b/apple/Plugins/EtcdDriverPlugin/EtcdCommandParser.swift index d079020050..3e021e148c 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdCommandParser.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdCommandParser.swift @@ -12,6 +12,9 @@ enum EtcdOperation { // KV case get(key: Data, prefix: Bool, limit: Int64?, keysOnly: Bool, sortOrder: EtcdSortOrder?, sortTarget: EtcdSortTarget?) case put(key: Data, value: Data, leaseId: Int64?) + // Not etcdctl verbs. A put that refuses an existing key, and a rename that refuses a taken name. + case create(key: Data, value: Data, leaseId: Int64?) + case move(from: Data, to: Data, value: Data, leaseId: Int64?, modRevision: Int64?) case del(key: Data, prefix: Bool) case watch(key: Data, prefix: Bool, timeout: TimeInterval) @@ -57,9 +60,11 @@ extension EtcdOperation { case .get, .watch, .leaseTimetolive, .leaseList, .memberList, .endpointStatus, .endpointHealth, .userList, .roleList: return .read - case .put, .leaseGrant, .leaseKeepAlive, .authEnable, .userAdd, .roleAdd, .userGrantRole, .userRevokeRole: + case .put, .create, .leaseGrant, .leaseKeepAlive, .authEnable, .userAdd, .roleAdd, + .userGrantRole, .userRevokeRole: return .write - case .del, .compaction, .leaseRevoke, .authDisable, .userDelete, .roleDelete: + // A move takes the key away from its name, as a del does. + case .move, .del, .compaction, .leaseRevoke, .authDisable, .userDelete, .roleDelete: return .destructive case .unknown: return nil @@ -82,6 +87,8 @@ extension EtcdOperation { switch self { case .get: return "get" case .put: return "put" + case .create: return "create" + case .move: return "mv" case .del: return "del" case .watch: return "watch" case .leaseGrant: return "lease grant" @@ -173,6 +180,8 @@ struct EtcdCommandParser { switch command { case "get": return try parseGet(remaining) case "put": return try parsePut(remaining) + case "create": return try parseCreate(remaining) + case "mv": return try parseMove(remaining) case "del", "delete": return try parseDel(remaining) case "watch": return try parseWatch(remaining) case "lease": return try parseLease(remaining) @@ -258,6 +267,41 @@ struct EtcdCommandParser { return .put(key: key, value: value, leaseId: leaseId) } + private static func parseCreate(_ tokens: [EtcdCommandToken]) throws -> EtcdOperation { + var flags = ParsedFlags() + let positional = flags.parse(from: tokens) + guard positional.count >= 2 else { + throw EtcdParseError.missingArgument("create requires key and value") + } + return .create( + key: positional[0].bytes, + value: positional[1].bytes, + leaseId: try flags.value(for: "lease").map(parseLeaseId) + ) + } + + private static func parseMove(_ tokens: [EtcdCommandToken]) throws -> EtcdOperation { + var flags = ParsedFlags() + let positional = flags.parse(from: tokens) + guard positional.count >= 3 else { + throw EtcdParseError.missingArgument("mv requires the key, its new name and the value") + } + var modRevision: Int64? + if let text = flags.value(for: "mod-rev") { + guard let parsed = Int64(text) else { + throw EtcdParseError.invalidArgument("--mod-rev must be an integer") + } + modRevision = parsed + } + return .move( + from: positional[0].bytes, + to: positional[1].bytes, + value: positional[2].bytes, + leaseId: try flags.value(for: "lease").map(parseLeaseId), + modRevision: modRevision + ) + } + private static func parseDel(_ tokens: [EtcdCommandToken]) throws -> EtcdOperation { var flags = ParsedFlags() let positional = flags.parse(from: tokens) diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdHttpClient.swift b/apple/Plugins/EtcdDriverPlugin/EtcdHttpClient.swift index b2c939dc1f..892dea24a8 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdHttpClient.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdHttpClient.swift @@ -561,6 +561,10 @@ internal final class EtcdHttpClient: @unchecked Sendable { try await post(path: apiPath("kv/deleterange"), body: req) } + func txnRequest(_ req: EtcdTxnRequest) async throws -> EtcdTxnResponse { + try await post(path: apiPath("kv/txn"), body: req) + } + // MARK: - Lease Operations func leaseGrant(ttl: Int64) async throws -> EtcdLeaseGrantResponse { diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift index dc1d869384..60e0518d0d 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift @@ -452,6 +452,18 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { case .put(let key, let value, let leaseId): return try await dispatchPut(key: key, value: value, leaseId: leaseId, client: client, startTime: startTime) + case .create(let key, let value, let leaseId): + return try await dispatchGuardedWrite( + .create(key: key, value: value, lease: leaseId), + key: key, value: value, client: client, startTime: startTime + ) + + case .move(let source, let target, let value, let leaseId, let modRevision): + return try await dispatchGuardedWrite( + .move(from: source, to: target, value: value, lease: leaseId, modRevision: modRevision), + key: target, value: value, client: client, startTime: startTime + ) + case .del(let key, let prefix): return try await dispatchDel(key: key, prefix: prefix, client: client, startTime: startTime) @@ -611,6 +623,32 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { ) } + private func dispatchGuardedWrite( + _ request: EtcdTxnRequest, key: Data, value: Data, + client: EtcdHttpClient, startTime: Date + ) async throws -> PluginQueryResult { + let response = try await client.txnRequest(request) + switch response.outcome { + case .applied: + return PluginQueryResult( + columns: ["Key", "Value", "Revision"], + columnTypeNames: ["String", "String", "Int64"], + rows: [[.etcd(key), .etcd(value), .text(response.revision ?? "unknown")]], + rowsAffected: 1, + executionTime: Date().timeIntervalSince(startTime) + ) + case .targetExists: + throw EtcdGuardedWriteRefusal(message: String( + format: String(localized: "A key named '%@' already exists."), + EtcdCommandText.text(key) ?? EtcdCommandText.quoted(key) + )) + case .sourceChanged: + throw EtcdGuardedWriteRefusal( + message: String(localized: "The key changed on the server after it was read. Refresh and try again.") + ) + } + } + private func dispatchDel( key: Data, prefix: Bool, client: EtcdHttpClient, startTime: Date diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdStatementGenerator.swift b/apple/Plugins/EtcdDriverPlugin/EtcdStatementGenerator.swift index d014660c31..6f9f687ba0 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdStatementGenerator.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdStatementGenerator.swift @@ -14,6 +14,7 @@ struct EtcdStatementGenerator { var keyColumnIndex: Int? { columns.firstIndex(of: "Key") } private var valueColumnIndex: Int? { columns.firstIndex(of: "Value") } private var leaseColumnIndex: Int? { columns.firstIndex(of: "Lease") } + private var modRevisionColumnIndex: Int? { columns.firstIndex(of: "ModRevision") } // etcd stores no NULL, so a NULL written to Value means an empty value, and to Lease, no lease. private static let writableColumns: Set = ["Key", "Value", "Lease"] @@ -82,7 +83,7 @@ struct EtcdStatementGenerator { // A key typed without a leading slash lands under the table's prefix. let fullKey = !prefix.isEmpty && k.first != UInt8(ascii: "/") ? Data(prefix.utf8) + k : k - return [Self.putCommand(key: fullKey, value: value ?? Data(), lease: leaseId)] + return [Self.writeCommand("create", arguments: [fullKey, value ?? Data()], lease: leaseId)] } private func updateCommands(for change: PluginRowChange) throws -> [String] { @@ -94,8 +95,6 @@ struct EtcdStatementGenerator { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.serverOwnedReason(column)) } - var commands: [String] = [] - let keyChange = change.cellChanges.first { $0.columnName == "Key" } let valueChange = change.cellChanges.first { $0.columnName == "Value" } let leaseChange = change.cellChanges.first { $0.columnName == "Lease" } @@ -109,24 +108,29 @@ struct EtcdStatementGenerator { let editedLease = leaseChange.map { $0.newValue.asText ?? "" } let lease = editedLease ?? originalCell(at: leaseColumnIndex, of: change)?.asText - if valueChange != nil || shouldDeleteOriginalKey { - let newValue = valueChange.map { $0.newValue.etcdBytes ?? Data() } ?? originalValue(of: change) - commands.append(Self.putCommand(key: newKey, value: newValue, lease: lease)) - if shouldDeleteOriginalKey { - commands.append(EtcdCommandText.command("del", arguments: [originalKey])) - } - } else if editedLease != nil { - commands.append(Self.putCommand(key: newKey, value: originalValue(of: change), lease: lease)) + let newValue = valueChange.map { $0.newValue.etcdBytes ?? Data() } ?? originalValue(of: change) + if shouldDeleteOriginalKey { + // One transaction: a put and a del sent apart replace a key already at the new name + // and leave both keys behind when the del fails. + let revision = originalCell(at: modRevisionColumnIndex, of: change)?.asText.flatMap { Int64($0) } + return [Self.writeCommand( + "mv", + arguments: [originalKey, newKey, newValue], + lease: lease, + flags: revision.map { ["--mod-rev=\($0)"] } ?? [] + )] } - - return commands + guard valueChange != nil || editedLease != nil else { return [] } + return [Self.writeCommand("put", arguments: [newKey, newValue], lease: lease)] } - private static func putCommand(key: Data, value: Data, lease: String?) -> String { + private static func writeCommand(_ name: String, arguments: [Data], lease: String?, flags: [String] = []) -> String { guard let lease, !lease.isEmpty, lease != "0" else { - return EtcdCommandText.command("put", arguments: [key, value]) + return EtcdCommandText.command(name, arguments: arguments, flags: flags) } - return EtcdCommandText.command("put", arguments: [key, value], flags: ["--lease=" + EtcdCommandText.quoted(lease)]) + return EtcdCommandText.command( + name, arguments: arguments, flags: flags + ["--lease=" + EtcdCommandText.quoted(lease)] + ) } // A new row's NULL in a server-owned column leaves the value to etcd, so only a value set in one counts. @@ -165,3 +169,9 @@ struct EtcdStatementGenerator { return originalRow[columnIndex] } } + +struct EtcdGuardedWriteRefusal: Error, PluginDriverError { + let message: String + + var pluginErrorMessage: String { message } +} diff --git a/apple/Plugins/FormatPluginTests/CSVImportRowShapeTests.swift b/apple/Plugins/FormatPluginTests/CSVImportRowShapeTests.swift new file mode 100644 index 0000000000..1c0b39b90b --- /dev/null +++ b/apple/Plugins/FormatPluginTests/CSVImportRowShapeTests.swift @@ -0,0 +1,149 @@ +// +// CSVImportRowShapeTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +private final class RowRecordingSink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "mock" + let targetTable: String? = "people" + + private(set) var rows: [[String: PluginCellValue]] = [] + private(set) var rolledBack = false + private(set) var committed = false + + func execute(statement: String) async throws {} + func insertRow(_ values: [String: PluginCellValue]) async throws { rows.append(values) } + func insertRows(_ rows: [[String: PluginCellValue]]) async throws { self.rows.append(contentsOf: rows) } + func deleteAllRowsFromTargetTable() async throws {} + func beginTransaction() async throws {} + func commitTransaction() async throws { committed = true } + func rollbackTransaction() async throws { rolledBack = true } + func disableForeignKeyChecks() async throws {} + func enableForeignKeyChecks() async throws {} +} + +private final class CSVFileSource: PluginImportSource, @unchecked Sendable { + private let url: URL + + init(url: URL) { self.url = url } + + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + AsyncThrowingStream { $0.finish() } + } + + func fileURL() -> URL { url } + func fileSizeBytes() -> Int64 { 0 } +} + +@Suite("CSV import reports a row with more fields than columns", .serialized) +struct CSVImportRowShapeTests { + private func runImport( + _ text: String, + errorHandling: ImportErrorHandling + ) async throws -> (Result, RowRecordingSink) { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("csv-shape-\(UUID().uuidString).csv") + try Data(text.utf8).write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + + let plugin = CSVImportPlugin() + let original = plugin.settings + defer { plugin.settings = original } + var options = CSVImportOptions() + options.errorHandling = errorHandling + plugin.settings = options + + let sink = RowRecordingSink() + do { + let result = try await plugin.performImport( + source: CSVFileSource(url: url), + sink: sink, + progress: PluginImportProgress(progress: Progress()) + ) + return (.success(result), sink) + } catch { + return (.failure(error), sink) + } + } + + private static let unquotedComma = "id,name,city\n1,Ada,Paris\n2,Smith, John,NY\n3,Lin,Hanoi\n" + + @Test("A row with an extra field stops the import and names its line") + func extraFieldStopsTheImport() async throws { + let (outcome, sink) = try await runImport(Self.unquotedComma, errorHandling: .stopAndRollback) + guard case .failure(let error) = outcome else { + Issue.record("The import went ahead over a row it had to cut") + return + } + #expect(error.localizedDescription.contains("line 3")) + #expect(error.localizedDescription.contains("4 fields")) + #expect(sink.rolledBack) + #expect(!sink.rows.contains { $0["id"] == .text("2") }) + #expect(!sink.rows.contains { $0["id"] == .text("3") }) + } + + @Test("Stop and commit keeps the rows above the malformed one") + func stopAndCommitKeepsEarlierRows() async throws { + let (outcome, sink) = try await runImport(Self.unquotedComma, errorHandling: .stopAndCommit) + guard case .failure = outcome else { + Issue.record("The import went ahead over a row it had to cut") + return + } + #expect(sink.committed) + #expect(sink.rows.map { $0["id"] } == [.text("1")]) + } + + @Test("Skip and continue leaves the row out and reports it") + func skipModeRecordsTheRow() async throws { + let (outcome, sink) = try await runImport(Self.unquotedComma, errorHandling: .skipAndContinue) + let result = try outcome.get() + #expect(result.executedStatements == 2) + #expect(result.skippedStatements == 1) + #expect(result.errors.map { $0.line } == [3]) + #expect(sink.rows.map { $0["id"] } == [.text("1"), .text("3")]) + } + + @Test("A row that lacks trailing fields imports with them NULL") + func shortRowIsPadded() async throws { + let (outcome, sink) = try await runImport( + "id,name,city\n1,Ada,Paris\n2,Smith\n", errorHandling: .stopAndRollback) + let result = try outcome.get() + #expect(result.executedStatements == 2) + #expect(result.skippedStatements == 0) + #expect(result.errors.isEmpty) + #expect(sink.rows.count == 2) + #expect(sink.rows.last?["name"] == .text("Smith")) + #expect(sink.rows.last?["city"] == .null) + } + + @Test("Trailing delimiters and blank lines are not errors") + func trailingDelimitersImport() async throws { + let (outcome, sink) = try await runImport( + "id,name\n1,Ada,,\n\n2,Lin\n", errorHandling: .stopAndRollback) + let result = try outcome.get() + #expect(result.executedStatements == 2) + #expect(result.skippedStatements == 0) + #expect(sink.rows.count == 2) + } + + @Test("Only a surplus field that holds something is an error") + func shapeCheck() { + let options = CSVImportOptions() + var trimming = CSVImportOptions() + trimming.trimWhitespace = true + #expect(CSVImportParsing.shapeError(fields: ["1", "a"], columnCount: 2, options: options) == nil) + #expect(CSVImportParsing.shapeError(fields: ["1", "a", ""], columnCount: 2, options: options) == nil) + #expect(CSVImportParsing.shapeError(fields: ["1"], columnCount: 2, options: options) == nil) + #expect(CSVImportParsing.shapeError(fields: ["1", "a", "x"], columnCount: 2, options: options) + == CSVImportParsing.RowShapeError(fieldCount: 3, columnCount: 2)) + #expect(CSVImportParsing.shapeError(fields: ["1", "a", " "], columnCount: 2, options: options) + == CSVImportParsing.RowShapeError(fieldCount: 3, columnCount: 2)) + #expect(CSVImportParsing.shapeError(fields: ["1", "a", " ", ""], columnCount: 2, options: trimming) == nil) + #expect(CSVImportParsing.shapeError(fields: ["1", "a", " x "], columnCount: 2, options: trimming) + == CSVImportParsing.RowShapeError(fieldCount: 3, columnCount: 2)) + } +} diff --git a/apple/Plugins/FormatPluginTests/SQLExportBitLiteralTests.swift b/apple/Plugins/FormatPluginTests/SQLExportBitLiteralTests.swift new file mode 100644 index 0000000000..86d15e7355 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/SQLExportBitLiteralTests.swift @@ -0,0 +1,50 @@ +import Foundation +import TableProPluginKit +import Testing + +// MySQL stores a quoted literal's bytes in a BIT column: '5' restores as 53. +struct SQLExportBitLiteralTests { + private func encoder(databaseTypeId: String, types: [String] = ["BIT", "VARCHAR"]) -> SQLExportRowValueEncoder { + SQLExportRowValueEncoder( + columns: ["b", "s"], + columnTypeNames: types, + excludedColumnNames: [], + databaseTypeId: databaseTypeId, + escapeStringLiteral: { $0.replacingOccurrences(of: "'", with: "''") } + ) + } + + @Test("A MySQL-family BIT value is written as a bare number", arguments: ["MySQL", "MariaDB", "TiDB", "OceanBase"]) + func bitIsANumber(databaseTypeId: String) { + let row = encoder(databaseTypeId: databaseTypeId).render([.text("5"), .text("5")]) + #expect(row == " (5, '5')") + } + + @Test("A BIT(64) value keeps every digit") + func widestBit() { + let row = encoder(databaseTypeId: "MySQL", types: ["BIT(64)", "VARCHAR"]) + .render([.text("18446744073709551615"), .text("x")]) + #expect(row == " (18446744073709551615, 'x')") + } + + @Test("NULL stays NULL and text that is no number stays quoted") + func otherValues() { + let mysql = encoder(databaseTypeId: "MySQL") + #expect(mysql.render([.null, .text("a")]) == " (NULL, 'a')") + #expect(mysql.render([.text("-1"), .text("a")]) == " ('-1', 'a')") + #expect(mysql.render([.text("1e3"), .text("a")]) == " ('1e3', 'a')") + #expect(mysql.render([.text(""), .text("a")]) == " ('', 'a')") + } + + @Test("A PostgreSQL bit string stays a quoted literal") + func postgresBitStaysQuoted() { + let row = encoder(databaseTypeId: "PostgreSQL", types: ["bit", "varchar"]).render([.text("101"), .text("x")]) + #expect(row == " ('101', 'x')") + } + + @Test("StarRocks BITMAP is not BIT") + func bitmapStaysQuoted() { + let row = encoder(databaseTypeId: "StarRocks", types: ["BITMAP", "VARCHAR"]).render([.text("7"), .text("x")]) + #expect(row == " ('7', 'x')") + } +} diff --git a/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift b/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift new file mode 100644 index 0000000000..3d9980f0bb --- /dev/null +++ b/apple/Plugins/FormatPluginTests/XLSXWriterCellTypeTests.swift @@ -0,0 +1,173 @@ +// +// XLSXWriterCellTypeTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct XLSXWriterCellTypeTests { + private func sheetXML( + columnTypeNames: [String], + row: [PluginCellValue], + databaseTypeId: String = "" + ) async throws -> String { + let writer = XLSXWriter(databaseTypeId: databaseTypeId) + writer.beginSheet( + name: "t", + columns: columnTypeNames.indices.map { "c\($0)" }, + columnTypeNames: columnTypeNames, + includeHeader: false, + convertNullToEmpty: true + ) + writer.addRows([row], convertNullToEmpty: true) + writer.finishSheet() + let destination = FileManager.default.temporaryDirectory + .appendingPathComponent("\(UUID().uuidString).xlsx") + defer { try? FileManager.default.removeItem(at: destination) } + try await writer.write(to: destination) + let written = try Data(contentsOf: destination) + let archive = try #require(String(bytes: written, encoding: .isoLatin1)) + let start = try #require(archive.range(of: "")) + let end = try #require(archive.range(of: "")) + return String(archive[start.upperBound ..< end.lowerBound]) + } + + @Test("Text that reads as a number stays text in a text column", arguments: [ + "Infinity", "Nan", "inf", "1E5", "+5", "42", "1234567890123456789", "0x1F" + ]) + func textColumnKeepsText(value: String) { + #expect(XLSXWriter.cell(forText: value, numbers: .init(typeName: "varchar(32)")) == .string(value)) + #expect(XLSXWriter.cell(forText: value, numbers: .text) == .string(value)) + #expect(XLSXWriter.cell(forText: value, numbers: .init(typeName: "Nullable(String)")) == .string(value)) + } + + @Test("A numeric column writes number cells") + func numericColumnWritesNumbers() { + #expect(XLSXWriter.cell(forText: "42", numbers: .init(typeName: "int4")) == .number("42")) + #expect(XLSXWriter.cell(forText: "-0.50", numbers: .init(typeName: "numeric(10,2)")) == .number("-0.50")) + #expect(XLSXWriter.cell(forText: "0", numbers: .init(typeName: "BIGINT")) == .number("0")) + #expect(XLSXWriter.cell(forText: "1.5e-7", numbers: .init(typeName: "double precision")) == .number("1.5e-7")) + #expect(XLSXWriter.cell(forText: "123456789012345", numbers: .exact) == .number("123456789012345")) + #expect(XLSXWriter.cell(forText: "1000000000000000000", numbers: .exact) == .number("1000000000000000000")) + } + + @Test("A number is written in the spelling a workbook reads") + func numberSpellingIsNormalized() { + #expect(XLSXWriter.cell(forText: "+5", numbers: .exact) == .number("5")) + #expect(XLSXWriter.cell(forText: ".5", numbers: .exact) == .number("0.5")) + #expect(XLSXWriter.cell(forText: "5.", numbers: .exact) == .number("5")) + } + + @Test("An exact number past 15 significant digits stays text, so no digit is lost", arguments: [ + "1234567890123456789", "-9223372036854775808", "1234567890123.4567", "0.1234567890123456" + ]) + func wideExactNumberStaysText(value: String) { + #expect(XLSXWriter.cell(forText: value, numbers: .init(typeName: "bigint")) == .string(value)) + #expect(XLSXWriter.cell(forText: value, numbers: .init(typeName: "decimal(38,10)")) == .string(value)) + } + + @Test("A floating-point column keeps its number cells at full precision") + func approximateColumnKeepsNumbers() { + #expect(XLSXWriter.cell(forText: "0.30000000000000004", numbers: .init(typeName: "float8")) + == .number("0.30000000000000004")) + } + + @Test("What no workbook number can hold stays text in a numeric column", arguments: [ + "NaN", "Infinity", "-Infinity", "1e999", "0x1F", "12abc", "007", "-007" + ]) + func unrepresentableValueStaysText(value: String) { + #expect(XLSXWriter.cell(forText: value, numbers: .init(typeName: "double")) == .string(value)) + #expect(XLSXWriter.cell(forText: value, numbers: .init(typeName: "int")) == .string(value)) + } + + @Test("Numeric type names from every engine are recognised", arguments: [ + "UInt64", "Nullable(UInt8)", "LowCardinality(Nullable(Int32))", "Decimal(18, 4)", "UBIGINT", "HUGEINT", + "bigserial", "INT UNSIGNED", "NUMBER(10)", "BIGNUMERIC", "YEAR", "int8" + ]) + func exactTypeNames(typeName: String) { + #expect(XLSXWriter.ColumnNumbers(typeName: typeName) == .exact) + } + + @Test("Floating-point type names are recognised", arguments: [ + "Float64", "Nullable(Float32)", "double precision", "REAL", "BINARY_DOUBLE", "float4" + ]) + func approximateTypeNames(typeName: String) { + #expect(XLSXWriter.ColumnNumbers(typeName: typeName) == .approximate) + } + + @Test("Text type names are recognised", arguments: [ + "varchar(32)", "TEXT", "character varying", "FixedString(8)", "NVARCHAR2(10)", "CLOB", "bpchar", + "Enum8('a' = 1)", "jsonb", "LowCardinality(String)" + ]) + func textTypeNames(typeName: String) { + #expect(XLSXWriter.ColumnNumbers(typeName: typeName) == .text) + } + + @Test("A bit column is a number where the engine reads it as one", arguments: [ + "SQL Server", "MySQL", "MariaDB", "" + ]) + func bitIsNumericOutsidePostgres(databaseTypeId: String) async throws { + let xml = try await sheetXML( + columnTypeNames: ["bit", "BIT(8)"], row: [.text("1"), .text("170")], databaseTypeId: databaseTypeId) + #expect(xml == "1170") + } + + @Test("A bit column is text on the PostgreSQL family", arguments: [ + "PostgreSQL", "Redshift", "CockroachDB", "KingbaseES" + ]) + func bitIsTextOnPostgres(databaseTypeId: String) async throws { + let xml = try await sheetXML( + columnTypeNames: ["bit(8)", "bit varying(8)", "varbit"], + row: [.text("10101010"), .text("1"), .text("11")], + databaseTypeId: databaseTypeId) + #expect(xml == "" + + "10101010" + + "1" + + "11" + + "") + } + + @Test("A missing or unknown type name is untyped", arguments: ["", " ", "interval", "geometry", "variant"]) + func untypedTypeNames(typeName: String) { + #expect(XLSXWriter.ColumnNumbers(typeName: typeName) == .untyped) + } + + @Test("An untyped column keeps ordinary numbers numeric", arguments: ["42", "-7", "0", "3.14", "0.50", "123456789012345"]) + func untypedColumnWritesPlainNumbers(value: String) { + #expect(XLSXWriter.cell(forText: value, numbers: .untyped) == .number(value)) + } + + @Test("An untyped column keeps ids, codes and padded values as text", arguments: [ + "1234567890123456", "1234567890123456789", "007", "02134", "1E5", "1e5", "+5", ".5", "5.", + "Infinity", "Nan", "inf", "0x1F", "1000000000000000000" + ]) + func untypedColumnKeepsText(value: String) { + #expect(XLSXWriter.cell(forText: value, numbers: .untyped) == .string(value)) + } + + @Test("A sheet given no type names still writes ordinary numbers as numbers") + func sheetWithoutTypeNames() async throws { + let xml = try await sheetXML(columnTypeNames: [], row: [.text("12"), .text("007"), .text("1E5")]) + #expect(xml == "" + + "12" + + "007" + + "1E5" + + "") + } + + @Test("The sheet types each cell from its column") + func sheetUsesColumnTypes() async throws { + let xml = try await sheetXML( + columnTypeNames: ["int4", "text", "text", "bigint"], + row: [.text("7"), .text("1E5"), .text("Infinity"), .text("1234567890123456789")] + ) + #expect(xml == "" + + "7" + + "1E5" + + "Infinity" + + "1234567890123456789" + + "") + } +} diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift index a50d109e07..e3ff61ce49 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift @@ -590,7 +590,7 @@ nonisolated struct MongoDBStatementGenerator { /// strict JSON. A stored string that merely starts with `[` and ends with `]` would otherwise run /// as code the moment its row is duplicated or its delete is undone. private func jsonValue(for value: String, field: String, provenance: Provenance) throws -> String { - if declaredKinds[field] == .string { + if declaredKinds[field] == .string || spellsAnotherKindInAStringField(value, field: field) { return "\"\(escapeJsonString(value))\"" } if value == "true" || value == "false" || value == "null" { @@ -611,6 +611,23 @@ nonisolated struct MongoDBStatementGenerator { return "\"\(escapeJsonString(value))\"" } + // A cell shows the string "90210" and the number 90210 as the same text. In a field that held + // strings and never a number, boolean or null, text of that shape is a string like the rest. + private func spellsAnotherKindInAStringField(_ value: String, field: String) -> Bool { + guard let held = heldKinds(of: field), held.contains(.string) else { return false } + let spelled: Set + if value == "true" || value == "false" { + spelled = [.boolean] + } else if value == "null" { + spelled = [.null] + } else if MongoDBJsonNumber.isValid(value) { + spelled = [.int32, .int64, .double, .decimal128] + } else { + return false + } + return held.isDisjoint(with: spelled) + } + /// A bare JSON number is stored as int32 or double, which silently retypes a column that /// holds int64 or decimal128, and a date or an ObjectId written as its text is a string. /// Extended JSON is the only way to keep the original type. diff --git a/apple/Plugins/ParquetExportPlugin/DuckDBStagingDatabase.swift b/apple/Plugins/ParquetExportPlugin/DuckDBStagingDatabase.swift index 3650a6c1ad..6db1a077da 100644 --- a/apple/Plugins/ParquetExportPlugin/DuckDBStagingDatabase.swift +++ b/apple/Plugins/ParquetExportPlugin/DuckDBStagingDatabase.swift @@ -8,11 +8,7 @@ import Foundation import TableProPluginKit import TableProSQLQuoting -/// An in-memory DuckDB used only to encode Parquet. -/// -/// It never opens the user's database and never sees their connection: rows arrive already streamed -/// from whichever engine the export is reading, are staged in one table here, and are written out -/// by DuckDB's own Parquet writer. The instance dies with the export. +// An in-memory DuckDB that only encodes Parquet. It never opens the user's database. final class DuckDBStagingDatabase { enum StagingError: LocalizedError { case openFailed @@ -28,16 +24,22 @@ final class DuckDBStagingDatabase { } } + struct TextFallback: Equatable { + let column: String + let type: String + let valueCount: Int + } + private static let stagingTable = "export_rows" private var database: duckdb_database? private var connection: duckdb_connection? + private var appender: duckdb_appender? - /// The staged column types, kept so every inserted value can be cast into its own column. - private var columnTypes: [String] = [] + private var columns: [String] = [] + private var targetTypes: [String] = [] - /// Where DuckDB spills. An in-memory database holds the staged table in RAM until told - /// otherwise, so exporting a table larger than memory would fail rather than take longer. + // An in-memory database keeps the staged table in RAM unless it has somewhere to spill. private let spillDirectory: URL init() throws { @@ -59,67 +61,144 @@ final class DuckDBStagingDatabase { } deinit { + if appender != nil { duckdb_appender_destroy(&appender) } if connection != nil { duckdb_disconnect(&connection) } if database != nil { duckdb_close(&database) } try? FileManager.default.removeItem(at: spillDirectory) } + // Values are staged as the text or bytes they arrived as. The typed cast happens once, in + // copyToParquet, where a column that cannot take it can still be written as text. func createTable(columns: [String], types: [String]) throws { guard !columns.isEmpty else { throw StagingError.queryFailed("no columns") } let definitions = zip(columns, types) - .map { "\(SQLQuoting.identifier($0.0)) \($0.1)" } + .map { "\(SQLQuoting.identifier($0.0)) \(ParquetStaging.stagedType(for: $0.1))" } .joined(separator: ", ") try run("CREATE TABLE \(SQLQuoting.identifier(Self.stagingTable)) (\(definitions))") - columnTypes = types + self.columns = columns + targetTypes = types } - // TRY_CAST writes null for a value its column cannot read, where an implicit cast fails the whole statement. - func insert(rows: [[PluginCellValue]], columnCount: Int) throws { - guard !rows.isEmpty, columnCount > 0 else { return } - var statement = "INSERT INTO \(SQLQuoting.identifier(Self.stagingTable)) VALUES " - var tuples: [String] = [] - tuples.reserveCapacity(rows.count) + func insert(rows: [[PluginCellValue]]) throws { + guard !rows.isEmpty, !targetTypes.isEmpty else { return } + let appender = try openAppender() for row in rows { - var values: [String] = [] - values.reserveCapacity(columnCount) - for index in 0 ..< columnCount { - values.append(castLiteral(row[safe: index] ?? .null, columnIndex: index)) + for (index, type) in targetTypes.enumerated() { + let value = index < row.count ? row[index] : .null + try check(append(value, isBinary: type == ParquetStaging.binaryType, to: appender), appender) } - tuples.append("(\(values.joined(separator: ", ")))") + try check(duckdb_appender_end_row(appender), appender) } - statement += tuples.joined(separator: ", ") - try run(statement) + try check(duckdb_appender_flush(appender), appender) } - private func castLiteral(_ value: PluginCellValue, columnIndex: Int) -> String { - let rendered = literal(value) - guard rendered != "NULL" else { return rendered } - guard let type = columnTypes[safe: columnIndex], type != "VARCHAR", type != "BLOB" else { - return rendered + @discardableResult + func copyToParquet(fileURL: URL, compression: String, rowGroupSize: Int) throws -> [TextFallback] { + try closeAppender() + let table = SQLQuoting.identifier(Self.stagingTable) + let rejected = try rejectedValueCounts(in: table) + let projection = columns.indices.map { index -> String in + let name = SQLQuoting.identifier(columns[index]) + guard ParquetStaging.isCast(targetTypes[index]), rejected[index] == nil else { return name } + return "CAST(\(name) AS \(targetTypes[index])) AS \(name)" } - return "TRY_CAST(\(rendered) AS \(type))" - } - - func copyToParquet(fileURL: URL, compression: String, rowGroupSize: Int) throws { let path = SQLQuoting.stringLiteral(fileURL.path(percentEncoded: false)) try run(""" - COPY \(SQLQuoting.identifier(Self.stagingTable)) TO \(path) + COPY (SELECT \(projection.joined(separator: ", ")) FROM \(table)) TO \(path) (FORMAT PARQUET, COMPRESSION \(compression), ROW_GROUP_SIZE \(rowGroupSize)) """) + return rejected.keys.sorted().map { index in + TextFallback(column: columns[index], type: targetTypes[index], valueCount: rejected[index] ?? 0) + } } // MARK: - Private - private func literal(_ value: PluginCellValue) -> String { + private func rejectedValueCounts(in table: String) throws -> [Int: Int] { + let castColumns = columns.indices.filter { ParquetStaging.isCast(targetTypes[$0]) } + guard !castColumns.isEmpty else { return [:] } + let counts = castColumns.map { index in + let predicate = ParquetStaging.rejectionPredicate( + column: SQLQuoting.identifier(columns[index]), type: targetTypes[index]) + return "count(*) FILTER (WHERE \(predicate))" + } + guard let connection else { throw StagingError.openFailed } + var result = duckdb_result() + defer { duckdb_destroy_result(&result) } + guard duckdb_query(connection, "SELECT \(counts.joined(separator: ", ")) FROM \(table)", &result) + == DuckDBSuccess else { + throw StagingError.queryFailed( + duckdb_result_error(&result).map { String(cString: $0) } ?? "unknown error") + } + var rejected: [Int: Int] = [:] + for (position, index) in castColumns.enumerated() { + let count = Int(duckdb_value_int64(&result, idx_t(position), 0)) + if count > 0 { rejected[index] = count } + } + return rejected + } + + private func openAppender() throws -> duckdb_appender { + if let appender { return appender } + guard let connection else { throw StagingError.openFailed } + var created: duckdb_appender? + guard duckdb_appender_create(connection, nil, Self.stagingTable, &created) == DuckDBSuccess, + let created else { + let message = created.flatMap { duckdb_appender_error($0) }.map { String(cString: $0) } + duckdb_appender_destroy(&created) + throw StagingError.queryFailed(message ?? "could not open the staging table") + } + appender = created + return created + } + + private func closeAppender() throws { + guard let open = appender else { return } + let state = duckdb_appender_close(open) + let message = state == DuckDBSuccess ? nil : duckdb_appender_error(open).map { String(cString: $0) } + duckdb_appender_destroy(&appender) + appender = nil + if state != DuckDBSuccess { throw StagingError.queryFailed(message ?? "unknown error") } + } + + private func check(_ state: duckdb_state, _ appender: duckdb_appender) throws { + guard state != DuckDBSuccess else { return } + throw StagingError.queryFailed( + duckdb_appender_error(appender).map { String(cString: $0) } ?? "unknown error") + } + + private func append(_ value: PluginCellValue, isBinary: Bool, to appender: duckdb_appender) -> duckdb_state { switch value { case .null: - return "NULL" - case .text(let text): - return SQLQuoting.stringLiteral(text) + return duckdb_append_null(appender) + case .text(var text): + return text.withUTF8 { appendBytes(UnsafeRawBufferPointer($0), isBinary: isBinary, to: appender) } case .bytes(let data): - let hex = data.map { String(format: "%02X", $0) }.joined() - return hex.isEmpty ? "NULL" : "'\\x\(hex)'::BLOB" + guard isBinary else { + var text = ParquetStaging.text(forBytes: data) + return text.withUTF8 { appendBytes(UnsafeRawBufferPointer($0), isBinary: false, to: appender) } + } + return data.withUnsafeBytes { appendBytes($0, isBinary: true, to: appender) } + } + } + + private func appendBytes( + _ bytes: UnsafeRawBufferPointer, + isBinary: Bool, + to appender: duckdb_appender + ) -> duckdb_state { + guard let base = bytes.baseAddress, !bytes.isEmpty else { + var none: UInt8 = 0 + return withUnsafePointer(to: &none) { pointer in + isBinary + ? duckdb_append_blob(appender, pointer, 0) + : duckdb_append_varchar_length(appender, UnsafeRawPointer(pointer).assumingMemoryBound(to: CChar.self), 0) + } + } + if isBinary { + return duckdb_append_blob(appender, base, idx_t(bytes.count)) } + return duckdb_append_varchar_length(appender, base.assumingMemoryBound(to: CChar.self), idx_t(bytes.count)) } private func run(_ sql: String) throws { @@ -133,9 +212,3 @@ final class DuckDBStagingDatabase { duckdb_destroy_result(&result) } } - -private extension Array { - subscript(safe index: Int) -> Element? { - indices.contains(index) ? self[index] : nil - } -} diff --git a/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift b/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift index b374e65d25..90eeda01e8 100644 --- a/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift +++ b/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift @@ -32,8 +32,7 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP typealias Settings = ParquetExportOptions static let settingsStorageId = "parquet" - /// Rows staged per `INSERT`. DuckDB binds an appender-sized batch happily; this only bounds how - /// many rows are held in Swift at once. + // Bounds how many rows are held in Swift between appender flushes. private static let stagingBatchSize = 2_000 @Published var settings = ParquetExportOptions() { @@ -63,7 +62,7 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP var warnings: [String] = [] try await ParquetTableFiles.write(tables, destination: destination, progress: progress) { table, fileURL in - try await writeTable(table, dataSource: dataSource, to: fileURL, progress: progress) + warnings += try await writeTable(table, dataSource: dataSource, to: fileURL, progress: progress) } if tables.count > 1 { @@ -80,7 +79,7 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP dataSource: any PluginExportDataSource, to fileURL: URL, progress: PluginExportProgress - ) async throws { + ) async throws -> [String] { let columnInfo = (try? await dataSource.fetchColumns( table: table.name, databaseName: table.databaseName)) ?? [] let declaredTypes = Dictionary( @@ -96,9 +95,10 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP switch element { case .header(let header): columns = header.columns - let types = columns.map { column in - ParquetTypeMapper.duckDBType( - forColumnType: declaredTypes[column] ?? "", databaseTypeId: dataSource.databaseTypeId) + let types = columns.enumerated().map { index, column in + let streamed = index < header.columnTypeNames.count ? header.columnTypeNames[index] : "" + return ParquetTypeMapper.duckDBType( + forColumnType: declaredTypes[column] ?? streamed, databaseTypeId: dataSource.databaseTypeId) } try staging.createTable(columns: columns, types: types) created = true @@ -108,7 +108,7 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP batch.append(row) progress.incrementRow() guard batch.count >= Self.stagingBatchSize else { continue } - try staging.insert(rows: batch, columnCount: columns.count) + try staging.insert(rows: batch) batch.removeAll(keepingCapacity: true) } } @@ -119,12 +119,17 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP String(format: String(localized: "%@ returned no columns."), table.name)) } if !batch.isEmpty { - try staging.insert(rows: batch, columnCount: columns.count) + try staging.insert(rows: batch) } - try staging.copyToParquet( + let fallbacks = try staging.copyToParquet( fileURL: fileURL, compression: settings.compression.duckDBValue, rowGroupSize: settings.rowGroupSize ) + return fallbacks.map { fallback in + String( + format: String(localized: "%1$@.%2$@ was written as text: %3$lld values do not fit %4$@."), + table.name, fallback.column, Int64(fallback.valueCount), fallback.type) + } } } diff --git a/apple/Plugins/ParquetExportPlugin/ParquetStaging.swift b/apple/Plugins/ParquetExportPlugin/ParquetStaging.swift new file mode 100644 index 0000000000..853fa49e1a --- /dev/null +++ b/apple/Plugins/ParquetExportPlugin/ParquetStaging.swift @@ -0,0 +1,51 @@ +// +// ParquetStaging.swift +// ParquetExportPlugin +// + +import Foundation + +// Kept free of DuckDB so a test bundle can compile it. +enum ParquetStaging { + static let textType = "VARCHAR" + static let binaryType = "BLOB" + + static func stagedType(for targetType: String) -> String { + targetType == binaryType ? binaryType : textType + } + + static func isCast(_ targetType: String) -> Bool { + targetType != textType && targetType != binaryType + } + + // True for a staged value the target type would null or change: TRY_CAST rounds 1.7 into a + // BIGINT and drops the time of day going into a DATE, and neither is a failed cast. + static func rejectionPredicate(column: String, type: String) -> String { + let failed = "TRY_CAST(\(column) AS \(type)) IS NULL" + let changed: String? + switch type { + case "BIGINT", "UBIGINT": + changed = "TRY_CAST(\(column) AS DOUBLE) IS DISTINCT FROM CAST(TRY_CAST(\(column) AS \(type)) AS DOUBLE)" + case "DATE": + changed = "TRY_CAST(\(column) AS TIMESTAMP) IS DISTINCT FROM CAST(TRY_CAST(\(column) AS DATE) AS TIMESTAMP)" + default: + changed = nil + } + guard let changed else { return "\(column) IS NOT NULL AND \(failed)" } + return "\(column) IS NOT NULL AND (\(failed) OR \(changed))" + } + + // The spelling DuckDB gives a BLOB cast to VARCHAR, for bytes that arrive in a text column. + static func text(forBytes data: Data) -> String { + var text = "" + text.reserveCapacity(data.count) + for byte in data { + if (0x20 ... 0x7E).contains(byte), byte != 0x5C, byte != 0x27, byte != 0x22 { + text.unicodeScalars.append(Unicode.Scalar(byte)) + } else { + text += String(format: "\\x%02X", byte) + } + } + return text + } +} diff --git a/apple/Plugins/ParquetExportPlugin/ParquetTypeMapper.swift b/apple/Plugins/ParquetExportPlugin/ParquetTypeMapper.swift index d505a4c0a1..6ab10742af 100644 --- a/apple/Plugins/ParquetExportPlugin/ParquetTypeMapper.swift +++ b/apple/Plugins/ParquetExportPlugin/ParquetTypeMapper.swift @@ -6,34 +6,31 @@ import Foundation import TableProPluginKit -/// Maps a source column's declared type to the DuckDB type the staging table uses. -/// -/// Parquet is a typed format, so writing every column as a string would produce a file that reads -/// back with no numbers, no dates and no booleans. The source engine's own type name is the only -/// thing that says what a column holds, because a streamed value arrives as text either way. +// A streamed value arrives as text whatever it holds, so the declared type decides the column's +// Parquet type. A column whose values do not fit it is written as text (DuckDBStagingDatabase). public enum ParquetTypeMapper { - /// The DuckDB type for a column, or `VARCHAR` when nothing better is known. A value that fails - /// to cast becomes null rather than failing the export, which is what `TRY_CAST` gives. public static func duckDBType(forColumnType typeName: String, databaseTypeId: String) -> String { let base = baseName(typeName) if let moneyType = fixedPointMoneyTypes[databaseTypeId]?[base] { return moneyType } - if integerTypes.contains(base) { return "BIGINT" } + if integerTypes.contains(base) { + return base == "bigint" && isUnsigned(typeName) ? "UBIGINT" : "BIGINT" + } if exactNumericTypes.contains(base) { guard !enginesIgnoringDeclaredPrecision.contains(databaseTypeId) else { return "DOUBLE" } return exactNumericType(declaredAs: typeName) } if approximateNumericTypes.contains(base) { return "DOUBLE" } if booleanTypes.contains(base) { return "BOOLEAN" } - if dateTypes.contains(base) { return "DATE" } + if base == "bit" { return isSingleBit(typeName) ? "BOOLEAN" : "VARCHAR" } + if dateTypes.contains(base) { + return enginesWhereDateCarriesTime.contains(databaseTypeId) ? "TIMESTAMP" : "DATE" + } if timestampTypes.contains(base) { return "TIMESTAMP" } if timeTypes.contains(base) { return "TIME" } if binaryTypes.contains(base) { return "BLOB" } return "VARCHAR" } - /// A type name carries its width in parentheses (`VARCHAR(64)`, `NUMERIC(10,2)`) and sometimes a - /// modifier after a space (`INT UNSIGNED`, `TIMESTAMP WITH TIME ZONE`). Only the first word - /// before either matters here. public static func baseName(_ typeName: String) -> String { let trimmed = typeName.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() let withoutArgs = trimmed.split(separator: "(", maxSplits: 1).first.map(String.init) ?? trimmed @@ -44,6 +41,21 @@ public enum ParquetTypeMapper { .map(String.init) ?? withoutArgs } + private static func words(_ typeName: String) -> [Substring] { + typeName.lowercased().split { !$0.isLetter } + } + + private static func isUnsigned(_ typeName: String) -> Bool { + words(typeName).contains("unsigned") + } + + // `bit` and `bit(1)` hold one flag. `bit(8)` and `bit varying` hold a string of them. + private static func isSingleBit(_ typeName: String) -> Bool { + guard !words(typeName).contains("varying") else { return false } + guard let width = typeArguments(typeName).first else { return true } + return Int(width) == 1 + } + private static func exactNumericType(declaredAs typeName: String) -> String { let arguments = typeArguments(typeName) guard let precision = arguments.first.flatMap({ Int($0) }), @@ -85,7 +97,9 @@ public enum ParquetTypeMapper { "float", "float4", "float8", "double", "real", "binary_float", "binary_double" ] - private static let booleanTypes: Set = ["bool", "boolean", "bit"] + private static let booleanTypes: Set = ["bool", "boolean"] + + private static let enginesWhereDateCarriesTime: Set = ["Oracle"] private static let dateTypes: Set = ["date"] diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift index 1e1159596d..f818f69baf 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift @@ -1409,6 +1409,22 @@ final class LibPQPluginConnection: @unchecked Sendable { let cancelsOutput: Bool } + // In single-row mode a result with no rows arrives as one zero-row PGRES_TUPLES_OK, which carries the same row + // description every PGRES_SINGLE_TUPLE does. + static func carriesRowDescription(_ status: ExecStatusType) -> Bool { + status == PGRES_SINGLE_TUPLE || status == PGRES_TUPLES_OK + } + + func streamHeader(from result: OpaquePointer) -> (header: PluginStreamHeader, columnOids: [UInt32]) { + let metadata = readColumnMetadata(from: result) + let header = PluginStreamHeader( + columns: metadata.columns, + columnTypeNames: metadata.columnTypeNames, + estimatedRowCount: nil + ) + return (header, metadata.columnOids) + } + private func streamResults(_ run: StreamRun, singleStatement: Bool) throws { let conn = run.conn let continuation = run.continuation @@ -1441,19 +1457,14 @@ final class LibPQPluginConnection: @unchecked Sendable { while let result = PQgetResult(conn) { let status = PQresultStatus(result) - if status == PGRES_SINGLE_TUPLE { - if !headerSent { - let metadata = readColumnMetadata(from: result) - columnOids = metadata.columnOids - - continuation.yield(.header(PluginStreamHeader( - columns: metadata.columns, - columnTypeNames: metadata.columnTypeNames, - estimatedRowCount: nil - ))) - headerSent = true - } + if !headerSent, Self.carriesRowDescription(status) { + let described = streamHeader(from: result) + columnOids = described.columnOids + continuation.yield(.header(described.header)) + headerSent = true + } + if status == PGRES_SINGLE_TUPLE { let numFields = Int(PQnfields(result)) var row: [PluginCellValue] = [] row.reserveCapacity(numFields) diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift index 4494740a86..332e0df495 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift @@ -413,10 +413,16 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { schema: resolvedSchema, table: table ) + let shapeQuery = PostgreSQLTableDDL.shapeQuery( + schema: resolvedSchema, table: table, capabilities: versionedCapabilities + ) + async let columnsResult = catalogRead(columnsQuery) async let constraintsResult = catalogRead(constraintsQuery) + async let shapeResult = try? catalogRead(shapeQuery) let (cols, cons) = try await (columnsResult, constraintsResult) + let shapeRows = await shapeResult /// `pg_attribute` covers views and materialized views as well as tables, so a view reached /// here as a `CREATE TABLE` of its columns. The Structure tab's DDL and every other caller @@ -440,10 +446,17 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { var parts = columnDefs parts.append(contentsOf: constraints) + let shape = PostgreSQLTableDDL.shape(fromRows: shapeRows?.rows.map { $0.map(\.asText) }) + var partitions: [PostgreSQLTableDDL.Partition] = [] + if shape.partitionKey != nil { + let tree = try await catalogRead(PostgreSQLTableDDL.partitionTreeQuery(schema: resolvedSchema, table: table)) + partitions = tree.rows.compactMap { PostgreSQLTableDDL.partition(from: $0.map(\.asText)) } + } + let quotedSchema = quoteIdentifier(resolvedSchema) - return "CREATE TABLE \(quotedSchema).\(quotedTable) (\n " + - parts.joined(separator: ",\n ") + - "\n);" + return PostgreSQLTableDDL.script( + name: "\(quotedSchema).\(quotedTable)", definitions: parts, shape: shape, partitions: partitions + ) } /// `pg_get_indexdef` is what `pg_dump` itself emits, and it round-trips an expression key, an @@ -452,6 +465,7 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { /// spanning two schemas attaches each index to the right one. func fetchIndexDDL(table: String, schema: String?) async throws -> [String] { try await fetchStandaloneIndexes(table: table, schema: schema ?? core.currentSchema).definitions + .map(PostgreSQLTableDDL.indexCoveringPartitions) } func fetchStandaloneIndexes(table: String, schema: String) async throws -> PostgreSQLStandaloneIndexes { diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift new file mode 100644 index 0000000000..e4728a714a --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift @@ -0,0 +1,175 @@ +// +// PostgreSQLTableDDL.swift +// PostgreSQLDriverPlugin +// + +import Foundation +import TableProPostgreSQLCore + +// What a table's CREATE statement says outside its column list. The table listing hides the partitions of a +// partitioned table, so the parent's script creates them: without them a restore has nowhere to put a row. +enum PostgreSQLTableDDL { + struct Shape: Equatable { + var isUnlogged = false + var partitionKey: String? + var partitionOf: String? + var partitionBound: String? + } + + struct Partition: Equatable { + let name: String + let parent: String + let bound: String + let partitionKey: String? + let isUnlogged: Bool + } + + // `relispartition`, `relpartbound` and `pg_get_partkeydef` are PostgreSQL 10. + static func shapeQuery(schema: String, table: String, capabilities: PostgreSQLCapabilities) -> String { + let partitionColumns = capabilities.hasDeclarativePartitioning ? """ + CASE WHEN c.relkind = 'p' THEN pg_catalog.pg_get_partkeydef(c.oid) END, + CASE WHEN c.relispartition THEN pg_catalog.pg_get_expr(c.relpartbound, c.oid) END, + CASE WHEN c.relispartition THEN ( + SELECT pg_catalog.quote_ident(pn.nspname) || '.' || pg_catalog.quote_ident(p.relname) + FROM pg_catalog.pg_inherits i + JOIN pg_catalog.pg_class p ON p.oid = i.inhparent + JOIN pg_catalog.pg_namespace pn ON pn.oid = p.relnamespace + WHERE i.inhrelid = c.oid + LIMIT 1) END + """ : "NULL::text, NULL::text, NULL::text" + return """ + SELECT c.relpersistence::text, + \(partitionColumns) + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE c.relname = \(PostgreSQLObjectQueries.quoteLiteral(table)) + AND n.nspname = \(PostgreSQLObjectQueries.quoteLiteral(schema)) + """ + } + + // Every partition below the table, a parent before its own partitions. A foreign table attached as a partition + // is left out: CREATE TABLE cannot make one. + static func partitionTreeQuery(schema: String, table: String) -> String { + """ + WITH RECURSIVE parts AS ( + SELECT i.inhrelid AS relid, i.inhparent AS parent, 1 AS depth + FROM pg_catalog.pg_inherits i + JOIN pg_catalog.pg_class root ON root.oid = i.inhparent + JOIN pg_catalog.pg_namespace rn ON rn.oid = root.relnamespace + WHERE root.relname = \(PostgreSQLObjectQueries.quoteLiteral(table)) + AND rn.nspname = \(PostgreSQLObjectQueries.quoteLiteral(schema)) + AND root.relkind = 'p' + UNION ALL + SELECT i.inhrelid, i.inhparent, parts.depth + 1 + FROM pg_catalog.pg_inherits i + JOIN parts ON parts.relid = i.inhparent + ) + SELECT pg_catalog.quote_ident(cn.nspname) || '.' || pg_catalog.quote_ident(c.relname), + pg_catalog.quote_ident(pn.nspname) || '.' || pg_catalog.quote_ident(p.relname), + pg_catalog.pg_get_expr(c.relpartbound, c.oid), + CASE WHEN c.relkind = 'p' THEN pg_catalog.pg_get_partkeydef(c.oid) END, + c.relpersistence::text + FROM parts + JOIN pg_catalog.pg_class c ON c.oid = parts.relid + JOIN pg_catalog.pg_namespace cn ON cn.oid = c.relnamespace + JOIN pg_catalog.pg_class p ON p.oid = parts.parent + JOIN pg_catalog.pg_namespace pn ON pn.oid = p.relnamespace + WHERE c.relispartition + AND c.relkind IN ('r', 'p') + ORDER BY parts.depth, cn.nspname, c.relname + """ + } + + // A server that reports version 10 without these catalog functions fails the read; the table is then plain. + static func shape(fromRows rows: [[String?]]?) -> Shape { + rows?.first.map(shape(from:)) ?? Shape() + } + + static func shape(from row: [String?]) -> Shape { + func text(_ index: Int) -> String? { + guard row.indices.contains(index), let value = row[index], !value.isEmpty else { return nil } + return value + } + return Shape( + isUnlogged: text(0) == "u", + partitionKey: text(1), + partitionOf: text(3), + partitionBound: text(2) + ) + } + + static func partition(from row: [String?]) -> Partition? { + guard row.count >= 5, + let name = row[0], !name.isEmpty, + let parent = row[1], !parent.isEmpty, + let bound = row[2], !bound.isEmpty + else { return nil } + let key = row[3].flatMap { $0.isEmpty ? nil : $0 } + return Partition(name: name, parent: parent, bound: bound, partitionKey: key, isUnlogged: row[4] == "u") + } + + static func createTable(name: String, definitions: [String], shape: Shape) -> String { + let partitionBy = shape.partitionKey.map { " PARTITION BY \($0)" } ?? "" + if let parent = shape.partitionOf, let bound = shape.partitionBound { + let isUnlogged = shape.isUnlogged && shape.partitionKey == nil + return "\(keyword(isUnlogged: isUnlogged)) \(name) PARTITION OF \(parent) \(bound)\(partitionBy);" + } + // PostgreSQL 18 refuses an unlogged partitioned table, and before it the flag did nothing on the parent. + let isUnlogged = shape.isUnlogged && shape.partitionKey == nil + return "\(keyword(isUnlogged: isUnlogged)) \(name) (\n " + + definitions.joined(separator: ",\n ") + + "\n)\(partitionBy);" + } + + static func createPartition(_ partition: Partition) -> String { + createTable( + name: partition.name, + definitions: [], + shape: Shape( + isUnlogged: partition.isUnlogged, + partitionKey: partition.partitionKey, + partitionOf: partition.parent, + partitionBound: partition.bound + ) + ) + } + + static func script(name: String, definitions: [String], shape: Shape, partitions: [Partition]) -> String { + ([createTable(name: name, definitions: definitions, shape: shape)] + partitions.map(createPartition)) + .joined(separator: "\n\n") + } + + // `pg_get_indexdef` writes ON ONLY for an index over a partitioned table, and replayed that builds nothing on the + // partitions and leaves the index invalid. Without it the server builds each partition's index itself. + static func indexCoveringPartitions(_ definition: String) -> String { + guard let keyword = ["CREATE UNIQUE INDEX ", "CREATE INDEX "].first(where: { definition.hasPrefix($0) }) else { + return definition + } + var index = definition.index(definition.startIndex, offsetBy: keyword.count) + if index < definition.endIndex, definition[index] == "\"" { + index = definition.index(after: index) + while index < definition.endIndex { + let next = definition.index(after: index) + if definition[index] == "\"" { + guard next < definition.endIndex, definition[next] == "\"" else { + index = next + break + } + index = definition.index(after: next) + } else { + index = next + } + } + } else { + index = definition[index...].firstIndex(of: " ") ?? definition.endIndex + } + let marker = " ON ONLY " + guard definition[index...].hasPrefix(marker) else { return definition } + let rest = definition[definition.index(index, offsetBy: marker.count)...] + return String(definition[.. String { + isUnlogged ? "CREATE UNLOGGED TABLE" : "CREATE TABLE" + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQStreamHeaderTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQStreamHeaderTests.swift new file mode 100644 index 0000000000..7537dacf9a --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQStreamHeaderTests.swift @@ -0,0 +1,47 @@ +import CLibPQ +import Foundation +import TableProPluginKit +import Testing + +struct LibPQStreamHeaderTests { + private func connection() -> LibPQPluginConnection { + LibPQPluginConnection(host: "localhost", port: 5_432, user: "postgres", password: nil, database: "postgres") + } + + private func emptyResult(columns: [(name: String, oid: Oid)]) throws -> OpaquePointer { + let result = try #require(PQmakeEmptyPGresult(nil, PGRES_TUPLES_OK)) + let names = columns.map { strdup($0.name) } + defer { names.forEach { free($0) } } + var attributes = zip(names, columns).map { name, column in + PGresAttDesc(name: name, tableid: 0, columnid: 0, format: 0, typid: column.oid, typlen: -1, atttypmod: -1) + } + #expect(PQsetResultAttrs(result, Int32(attributes.count), &attributes) != 0) + return result + } + + @Test("The result that ends a stream describes its rows, so an empty one still has a header") + func zeroRowResultCarriesTheRowDescription() { + #expect(LibPQPluginConnection.carriesRowDescription(PGRES_TUPLES_OK)) + #expect(LibPQPluginConnection.carriesRowDescription(PGRES_SINGLE_TUPLE)) + } + + @Test("A command and a failure describe no rows") + func otherResultsCarryNone() { + #expect(!LibPQPluginConnection.carriesRowDescription(PGRES_COMMAND_OK)) + #expect(!LibPQPluginConnection.carriesRowDescription(PGRES_FATAL_ERROR)) + #expect(!LibPQPluginConnection.carriesRowDescription(PGRES_COPY_OUT)) + } + + @Test("A zero-row result yields its column names and types") + func headerOfAnEmptyResult() throws { + let result = try emptyResult(columns: [(name: "id", oid: 23), (name: "name", oid: 25)]) + defer { PQclear(result) } + #expect(PQntuples(result) == 0) + + let described = connection().streamHeader(from: result) + + #expect(described.header.columns == ["id", "name"]) + #expect(described.header.columnTypeNames == ["integer", "text"]) + #expect(described.columnOids == [23, 25]) + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/PostgreSQLTableDDLTests.swift b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLTableDDLTests.swift new file mode 100644 index 0000000000..2664e5b315 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLTableDDLTests.swift @@ -0,0 +1,170 @@ +import Foundation +import TableProPluginKit +import TableProPostgreSQLCore +import Testing + +struct PostgreSQLTableDDLTests { + private static let modern = PostgreSQLCapabilities(serverVersion: 170_011) + private static let columns = ["id integer NOT NULL", "created_at date NOT NULL"] + + @Test("A plain table keeps the statement it always had") + func plainTable() { + let ddl = PostgreSQLTableDDL.script( + name: "\"public\".\"orders\"", definitions: Self.columns, shape: PostgreSQLTableDDL.Shape(), partitions: [] + ) + #expect(ddl == """ + CREATE TABLE "public"."orders" ( + id integer NOT NULL, + created_at date NOT NULL + ); + """) + } + + @Test("A partitioned table says how it is partitioned and creates its partitions") + func partitionedTable() { + let shape = PostgreSQLTableDDL.shape(from: ["p", "RANGE (created_at)", nil, nil]) + let partitions = [ + ["public.events_2024", "public.events", "FOR VALUES FROM ('2024-01-01') TO ('2025-01-01')", nil, "p"], + ["public.events_rest", "public.events", "DEFAULT", nil, "p"] + ].compactMap { PostgreSQLTableDDL.partition(from: $0) } + + let ddl = PostgreSQLTableDDL.script( + name: "\"public\".\"events\"", definitions: Self.columns, shape: shape, partitions: partitions + ) + + #expect(ddl == """ + CREATE TABLE "public"."events" ( + id integer NOT NULL, + created_at date NOT NULL + ) PARTITION BY RANGE (created_at); + + CREATE TABLE public.events_2024 PARTITION OF public.events FOR VALUES FROM ('2024-01-01') TO ('2025-01-01'); + + CREATE TABLE public.events_rest PARTITION OF public.events DEFAULT; + """) + } + + @Test("A partition that is partitioned again carries its own key") + func subPartition() throws { + let partition = try #require(PostgreSQLTableDDL.partition( + from: ["public.events_eu", "public.events", "FOR VALUES IN ('eu')", "HASH (id)", "p"] + )) + #expect(PostgreSQLTableDDL.createPartition(partition) + == "CREATE TABLE public.events_eu PARTITION OF public.events FOR VALUES IN ('eu') PARTITION BY HASH (id);") + } + + @Test("A partition asked for by name is created as a partition of its parent") + func partitionByName() { + let shape = PostgreSQLTableDDL.shape( + from: ["p", nil, "FOR VALUES FROM (1) TO (100)", "public.events"] + ) + let ddl = PostgreSQLTableDDL.script( + name: "\"public\".\"events_1\"", definitions: Self.columns, shape: shape, partitions: [] + ) + #expect(ddl == "CREATE TABLE \"public\".\"events_1\" PARTITION OF public.events FOR VALUES FROM (1) TO (100);") + } + + @Test("An unlogged table and an unlogged partition stay unlogged") + func unlogged() throws { + let shape = PostgreSQLTableDDL.shape(from: ["u", nil, nil, nil]) + #expect(PostgreSQLTableDDL.createTable(name: "t", definitions: ["id integer"], shape: shape) + == "CREATE UNLOGGED TABLE t (\n id integer\n);") + let partition = try #require(PostgreSQLTableDDL.partition(from: ["s.p1", "s.p", "DEFAULT", "", "u"])) + #expect(PostgreSQLTableDDL.createPartition(partition) == "CREATE UNLOGGED TABLE s.p1 PARTITION OF s.p DEFAULT;") + } + + @Test("A partitioned table is never written unlogged, and its partitions keep theirs") + func unloggedPartitionedParent() throws { + let shape = PostgreSQLTableDDL.shape(from: ["u", "RANGE (id)", nil, nil]) + #expect(PostgreSQLTableDDL.createTable(name: "t", definitions: ["id integer"], shape: shape) + == "CREATE TABLE t (\n id integer\n) PARTITION BY RANGE (id);") + + let leaf = try #require(PostgreSQLTableDDL.partition(from: ["s.p1", "s.p", "DEFAULT", nil, "u"])) + #expect(PostgreSQLTableDDL.createPartition(leaf) == "CREATE UNLOGGED TABLE s.p1 PARTITION OF s.p DEFAULT;") + + let branch = try #require(PostgreSQLTableDDL.partition(from: ["s.p2", "s.p", "DEFAULT", "HASH (id)", "u"])) + #expect(PostgreSQLTableDDL.createPartition(branch) + == "CREATE TABLE s.p2 PARTITION OF s.p DEFAULT PARTITION BY HASH (id);") + } + + @Test("A shape read that failed or found nothing leaves a plain table") + func failedShapeRead() { + #expect(PostgreSQLTableDDL.shape(fromRows: nil) == PostgreSQLTableDDL.Shape()) + #expect(PostgreSQLTableDDL.shape(fromRows: []) == PostgreSQLTableDDL.Shape()) + #expect(PostgreSQLTableDDL.shape(fromRows: [["u", nil, nil, nil]]).isUnlogged) + } + + @Test("A row with no catalog answer reads as a plain table") + func missingShape() { + #expect(PostgreSQLTableDDL.shape(from: []) == PostgreSQLTableDDL.Shape()) + #expect(PostgreSQLTableDDL.shape(from: ["p", "", "", ""]) == PostgreSQLTableDDL.Shape()) + #expect(PostgreSQLTableDDL.partition(from: ["s.p1", "s.p", nil, nil, "p"]) == nil) + } + + @Test("Before PostgreSQL 10 the shape read names no partition catalog") + func legacyShapeQuery() { + let sql = PostgreSQLTableDDL.shapeQuery( + schema: "public", table: "orders", capabilities: PostgreSQLCapabilities(serverVersion: 90_124) + ) + #expect(sql.contains("c.relpersistence")) + #expect(!sql.contains("relispartition")) + #expect(!sql.contains("pg_get_partkeydef")) + #expect(!sql.contains("relpartbound")) + } + + @Test("The shape read asks for the key, the bound and the parent") + func modernShapeQuery() { + let sql = PostgreSQLTableDDL.shapeQuery(schema: "public", table: "orders", capabilities: Self.modern) + #expect(sql.contains("pg_catalog.pg_get_partkeydef(c.oid)")) + #expect(sql.contains("pg_catalog.pg_get_expr(c.relpartbound, c.oid)")) + #expect(sql.contains("i.inhrelid = c.oid")) + } + + @Test("Both reads call built-ins through pg_catalog and quote the names they are given") + func readsAreQualifiedAndQuoted() { + let hostile = "o'rders" + let statements = [ + PostgreSQLTableDDL.shapeQuery(schema: hostile, table: hostile, capabilities: Self.modern), + PostgreSQLTableDDL.partitionTreeQuery(schema: hostile, table: hostile) + ] + for sql in statements { + #expect(PostgreSQLCatalogQualificationTests.unqualifiedCalls(in: sql).isEmpty, "\(sql)") + #expect(sql.contains("'o''rders'")) + #expect(!sql.contains("'o'rders'")) + } + } + + @Test("The partition tree lists a parent before its own partitions") + func partitionTreeOrder() { + let sql = PostgreSQLTableDDL.partitionTreeQuery(schema: "public", table: "events") + #expect(sql.contains("WITH RECURSIVE")) + #expect(sql.contains("ORDER BY parts.depth")) + #expect(sql.contains("c.relkind IN ('r', 'p')")) + } + + @Test("An index over a partitioned table is written so the server builds it on every partition") + func indexCoversPartitions() { + #expect(PostgreSQLTableDDL.indexCoveringPartitions( + "CREATE INDEX events_created_idx ON ONLY public.events USING btree (created_at)" + ) == "CREATE INDEX events_created_idx ON public.events USING btree (created_at)") + #expect(PostgreSQLTableDDL.indexCoveringPartitions( + "CREATE UNIQUE INDEX \"Events Key\" ON ONLY public.events USING btree (id, created_at)" + ) == "CREATE UNIQUE INDEX \"Events Key\" ON public.events USING btree (id, created_at)") + } + + @Test("Only the ONLY the server wrote is removed") + func indexNamesAreNotRewritten() { + let plain = "CREATE INDEX orders_idx ON public.orders USING btree (id)" + #expect(PostgreSQLTableDDL.indexCoveringPartitions(plain) == plain) + + let named = "CREATE INDEX \"i ON ONLY x\" ON public.orders USING btree (id)" + #expect(PostgreSQLTableDDL.indexCoveringPartitions(named) == named) + + let quoted = "CREATE INDEX \"a\"\"b ON ONLY c\" ON ONLY public.events USING btree (id)" + #expect(PostgreSQLTableDDL.indexCoveringPartitions(quoted) + == "CREATE INDEX \"a\"\"b ON ONLY c\" ON public.events USING btree (id)") + + let predicate = "CREATE INDEX i ON public.orders USING btree (id) WHERE (note = ' ON ONLY ')" + #expect(PostgreSQLTableDDL.indexCoveringPartitions(predicate) == predicate) + } +} diff --git a/apple/Plugins/RedisDriverPlugin/RedisCommandParser.swift b/apple/Plugins/RedisDriverPlugin/RedisCommandParser.swift index c08fdbf66e..a63a93da3c 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisCommandParser.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisCommandParser.swift @@ -66,6 +66,7 @@ enum RedisOperation { case configSet(parameter: String, value: String) case slowlogGet(count: Int?) case command(args: [RedisArgument]) + case mustApply(args: [RedisArgument]) // Multi case multi @@ -89,13 +90,14 @@ struct RedisSetOptions { var pxat: Int? var nx: Bool = false var xx: Bool = false + var keepTTL: Bool = false static let valueOptions: Set = ["EX", "PX", "EXAT", "PXAT"] - static let flagOptions: Set = ["NX", "XX"] + static let flagOptions: Set = ["NX", "XX", "KEEPTTL"] /// The driver rebuilds SET from the parsed options alone, so an option this type cannot carry - /// would be dropped on the way to the server: `SET k v KEEPTTL` would clear the very TTL it - /// asks to keep, and report OK. Anything unrecognised is sent verbatim instead. + /// would be dropped on the way to the server: `SET k v GET` would answer OK in place of the + /// old value. Anything unrecognised is sent verbatim instead. static func modelsEveryOption(in tokens: [RedisArgument]) -> Bool { var index = 0 while index < tokens.count { @@ -166,6 +168,7 @@ struct RedisCommandParser { let args = Array(tokens.dropFirst()) if command == "DB" { return try parseInDatabase(args) } if command == "OUTSIDEBLOCK" { return try parseOutsideBlock(args) } + if command == RedisConditionalWrite.keyword { return try parseMustApply(args) } if let typedCount = typedArgumentCount[command], args.count > typedCount { return .command(args: tokens) @@ -258,6 +261,25 @@ struct RedisCommandParser { ) } + // The wrapper sends its command straight to the server, past every typed case and past the + // host's tiering of the command word, so it takes only the two writes whose reply it reads. + private static func parseMustApply(_ args: [RedisArgument]) throws -> RedisOperation { + let refusal = RedisParseError.invalidArgument( + String(localized: "MUSTAPPLY takes only RENAMENX or SET with NX") + ) + switch args.first?.text.uppercased() { + case "RENAMENX": + guard args.count == 3 else { throw refusal } + case "SET": + let options = Array(args.dropFirst(3)) + guard args.count > 3, RedisSetOptions.modelsEveryOption(in: options), + try parseSetOptions(options)?.nx == true else { throw refusal } + default: + throw refusal + } + return .mustApply(args: args) + } + /// Only INFO and SLOWLOG GET keep their own grids; anything else goes to the server as typed. private static func parseOutsideBlock(_ args: [RedisArgument]) throws -> RedisOperation { guard !args.isEmpty else { throw RedisParseError.emptySyntax } @@ -1115,6 +1137,9 @@ struct RedisCommandParser { case "XX": options.xx = true hasOption = true + case "KEEPTTL": + options.keepTTL = true + hasOption = true default: break } diff --git a/apple/Plugins/RedisDriverPlugin/RedisConditionalWrite.swift b/apple/Plugins/RedisDriverPlugin/RedisConditionalWrite.swift new file mode 100644 index 0000000000..8d41c646ca --- /dev/null +++ b/apple/Plugins/RedisDriverPlugin/RedisConditionalWrite.swift @@ -0,0 +1,46 @@ +// +// RedisConditionalWrite.swift +// RedisDriverPlugin +// +// A write that must not land on a key that already exists. +// + +import Foundation + +// `RENAMENX` answers 0 and `SET ... NX` answers nil when the key they would create is already +// there, and neither is an error to the server. A save that sends one has to fail on that answer, +// so the grid wraps the command in `MUSTAPPLY` and the driver reads the reply. +enum RedisConditionalWrite { + static let keyword = "MUSTAPPLY" + + static func statement(_ command: String) -> String { + "\(keyword) \(command)" + } + + static func applied(_ reply: RedisReply) -> Bool { + switch reply { + case .null: return false + case .integer(let value): return value != 0 + default: return true + } + } + + static var refusedMessage: String { + String(localized: "A key with this name already exists.") + } + + static func command(in statement: String) -> String? { + let prefix = "\(keyword) " + return statement.hasPrefix(prefix) ? String(statement.dropFirst(prefix.count)) : nil + } +} + +extension RedisOperation { + var mustApply: Bool { + switch self { + case .mustApply: return true + case .inDatabase(_, let operation): return operation.mustApply + default: return false + } + } +} diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift index de2e746b55..c02cd0baa2 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift @@ -25,7 +25,7 @@ extension RedisPluginDriver { return try await runOperation(operation, connection: conn, startTime: startTime) } catch let queued as RedisQueuedCommand { guard operation.queuedCommandAnswer == .reportQueued else { throw queued } - recordQueued(queued.command) + recordQueued(queued.command, mustApply: operation.mustApply) return buildStatusResult(Self.queuedStatus, startTime: startTime) } } @@ -73,6 +73,13 @@ extension RedisPluginDriver { .multi, .exec, .discard: return try await executeServerOperation(operation, connection: conn, startTime: startTime) + case .mustApply(let args): + let reply = try await conn.run(args.asRedisArguments) + guard RedisConditionalWrite.applied(reply) else { + throw RedisPluginError(code: 0, message: RedisConditionalWrite.refusedMessage) + } + return buildStatusResult("OK", startTime: startTime) + case .inDatabase(let database, let operation): return try await conn.withDatabase(database) { try await runOperation(operation, connection: conn, startTime: startTime) @@ -113,6 +120,7 @@ extension RedisPluginDriver { if let pxat = opts.pxat { args += ["PXAT", String(pxat)].asRedisArguments } if opts.nx { args.append("NX".redisArgument) } if opts.xx { args.append("XX".redisArgument) } + if opts.keepTTL { args.append("KEEPTTL".redisArgument) } } try await conn.run(args) return buildStatusResult("OK", startTime: startTime) diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift index 9c4dc2922a..1e832eec3c 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift @@ -48,6 +48,9 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private let queuedCommandsLock = NSLock() private var queuedCommands: [String] = [] + // Counted past the label cap: a conditional write EXEC skipped has to fail the save wherever it sits. + private var queuedCount = 0 + private var conditionalPositions: Set = [] var serverVersion: String? { redisConnection?.serverVersion() @@ -212,26 +215,34 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { /// result rather than an error: the block is theirs to end, and `EXEC` will report every reply. static let queuedStatus = "QUEUED" - func recordQueued(_ command: String) { + func recordQueued(_ command: String, mustApply: Bool = false) { queuedCommandsLock.lock() + if mustApply { conditionalPositions.insert(queuedCount) } + queuedCount += 1 if queuedCommands.count < Self.maxRecordedQueuedCommands { queuedCommands.append(command) } queuedCommandsLock.unlock() } - private func takeQueuedCommands() -> [String] { + private func takeQueuedCommands() -> (labels: [String], conditional: Set) { queuedCommandsLock.lock() defer { queuedCommandsLock.unlock() } - let recorded = queuedCommands - queuedCommands = [] + let recorded = (labels: queuedCommands, conditional: conditionalPositions) + resetQueuedCommands() return recorded } private func clearQueuedCommands() { queuedCommandsLock.lock() - queuedCommands = [] + resetQueuedCommands() queuedCommandsLock.unlock() } + private func resetQueuedCommands() { + queuedCommands = [] + queuedCount = 0 + conditionalPositions = [] + } + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { try await execute(query: query) } @@ -432,7 +443,9 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { guard let conn = redisConnection else { throw RedisPluginError.notConnected } let queued = takeQueuedCommands() let reply = try await conn.run(["EXEC"]) - let failed = RedisTransactionOutcome.failures(inExecReply: reply, queuedCommands: queued) + let failed = RedisTransactionOutcome.failures( + inExecReply: reply, queuedCommands: queued.labels, conditionalPositions: queued.conditional + ) guard failed.isEmpty else { throw RedisTransactionError(failed: failed) } } @@ -625,7 +638,8 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let generator = RedisStatementGenerator( namespaceName: table, columns: columns, - deleteBatching: redisConnection?.partitionsKeyspace == true ? .perHashSlot : .singleCommand + deleteBatching: redisConnection?.partitionsKeyspace == true ? .perHashSlot : .singleCommand, + keepsTTLOnSet: RedisStatementGenerator.serverKeepsTTLOnSet(version: redisConnection?.serverVersion()) ) let writes = try generator.generateRowWrites( from: changes, insertedRowData: insertedRowData, diff --git a/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift b/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift index f65e5e1035..22922a53b4 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift @@ -24,6 +24,13 @@ struct RedisStatementGenerator { let namespaceName: String let columns: [String] var deleteBatching: RedisDeleteBatching = .singleCommand + // `SET ... KEEPTTL` needs Redis 6.0. An older server gets the expiry put back from the grid. + var keepsTTLOnSet = true + + static func serverKeepsTTLOnSet(version: String?) -> Bool { + guard let major = version?.split(separator: ".").first.flatMap({ Int($0) }) else { return true } + return major >= 6 + } /// Index of the "Key" column (used as primary identifier, like MongoDB's "_id") var keyColumnIndex: Int? { @@ -156,15 +163,22 @@ struct RedisStatementGenerator { ttl = seconds } - var commands = [generateInsertCommand(key: k, encodedValue: value ?? RedisArgumentCodec.quote(""), type: typeName)] - if let ttlSeconds = ttl, ttlSeconds > 0 { - commands.append("EXPIRE \(RedisArgumentCodec.quote(k)) \(ttlSeconds)") + let encodedValue = value ?? RedisArgumentCodec.quote("") + let expiry = ttl.flatMap { $0 > 0 ? $0 : nil } + guard let collectionCommand = collectionInsertCommand(key: k, encodedValue: encodedValue, type: typeName) else { + let expiryOption = expiry.map { " EX \($0)" } ?? "" + let set = "SET \(RedisArgumentCodec.quote(k)) \(encodedValue)\(expiryOption) NX" + return [RedisConditionalWrite.statement(set)] + } + var commands = [collectionCommand] + if let expiry { + commands.append("EXPIRE \(RedisArgumentCodec.quote(k)) \(expiry)") } return commands } - /// Generate the appropriate Redis command based on the data type - private func generateInsertCommand(key: String, encodedValue: String, type: String) -> String { + // Nil for a string, which is written by a `SET` that carries its own expiry and key check. + private func collectionInsertCommand(key: String, encodedValue: String, type: String) -> String? { let quotedKey = RedisArgumentCodec.quote(key) switch type { case "hash": @@ -181,7 +195,7 @@ struct RedisStatementGenerator { case "zset": return "ZADD \(quotedKey) 0 \(encodedValue)" default: - return "SET \(quotedKey) \(encodedValue)" + return nil } } @@ -215,8 +229,9 @@ struct RedisStatementGenerator { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.unaddressableKeyReason) } - var commands: [String] = [] - var effectiveKey = key + var valueCommands: [String] = [] + var ttlCommands: [String] = [] + var rename: String? if let keyChange = change.cellChanges.first(where: { $0.columnName == "Key" }) { guard let newKey = keyChange.newValue.asText else { @@ -225,19 +240,23 @@ struct RedisStatementGenerator { ) } if newKey != key { - commands.append("RENAME \(RedisArgumentCodec.quote(key)) \(RedisArgumentCodec.quote(newKey))") + rename = RedisConditionalWrite.statement( + "RENAMENX \(RedisArgumentCodec.quote(key)) \(RedisArgumentCodec.quote(newKey))" + ) } - effectiveKey = newKey } + let changesTTL = change.cellChanges.contains { $0.columnName == "TTL" } for cellChange in change.cellChanges { switch cellChange.columnName { case "Key": continue case "Value": - commands.append(try valueCommand(setting: cellChange.newValue, of: change, key: effectiveKey)) + valueCommands += try self.valueCommands( + setting: cellChange.newValue, of: change, key: key, restoringTTL: !changesTTL + ) case "TTL": - commands.append(try ttlCommand(setting: cellChange.newValue, of: change, key: effectiveKey)) + ttlCommands.append(try ttlCommand(setting: cellChange.newValue, of: change, key: key)) default: throw PluginRowWriteRefusal( rowIndex: change.rowIndex, @@ -246,12 +265,19 @@ struct RedisStatementGenerator { } } - return commands + // The rename goes last and names the key the row was read from. A refused rename then leaves + // every earlier command on that key, never on the key that was already at the new name. + return valueCommands + ttlCommands + (rename.map { [$0] } ?? []) } /// Only a string's value is the whole of what the grid shows. A collection shows a preview, /// and a `SET` over it would replace the structure with that text. - private func valueCommand(setting newValue: PluginCellValue, of change: PluginRowChange, key: String) throws -> String { + private func valueCommands( + setting newValue: PluginCellValue, + of change: PluginRowChange, + key: String, + restoringTTL: Bool + ) throws -> [String] { guard let encodedValue = Self.encodedArgument(newValue) else { throw PluginRowWriteRefusal( rowIndex: change.rowIndex, @@ -273,7 +299,17 @@ struct RedisStatementGenerator { ) ) } - return "SET \(RedisArgumentCodec.quote(key)) \(encodedValue)" + let set = "SET \(RedisArgumentCodec.quote(key)) \(encodedValue)" + guard !keepsTTLOnSet else { return ["\(set) KEEPTTL"] } + guard restoringTTL, let seconds = originalTTL(of: change), seconds > 0 else { return [set] } + return [set, "EXPIRE \(RedisArgumentCodec.quote(key)) \(seconds)"] + } + + private func originalTTL(of change: PluginRowChange) -> Int? { + guard let ttlIndex = ttlColumnIndex, let originalRow = change.originalRow, ttlIndex < originalRow.count else { + return nil + } + return originalRow[ttlIndex].asText.flatMap { Int($0) } } private func ttlCommand(setting newValue: PluginCellValue, of change: PluginRowChange, key: String) throws -> String { diff --git a/apple/Plugins/RedisDriverPlugin/RedisTransactionOutcome.swift b/apple/Plugins/RedisDriverPlugin/RedisTransactionOutcome.swift index cc065325d8..d15b8530eb 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisTransactionOutcome.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisTransactionOutcome.swift @@ -58,11 +58,19 @@ enum RedisTransactionOutcome { /// refusal, `-ERR EXEC without MULTI` for one `RESET` already ended, and a nil reply for one /// whose `WATCH` was broken. None of them applied anything, so none of them names a failure /// here. - static func failures(inExecReply reply: RedisReply, queuedCommands: [String]) -> [RedisFailedCommand] { + static func failures( + inExecReply reply: RedisReply, + queuedCommands: [String], + conditionalPositions: Set = [] + ) -> [RedisFailedCommand] { guard case .array(let elements) = reply else { return [] } return elements.enumerated().compactMap { index, element in - guard let message = element.errorMessage else { return nil } - return RedisFailedCommand(label: label(at: index, in: queuedCommands), message: message) + let label = label(at: index, in: queuedCommands) + if let message = element.errorMessage { + return RedisFailedCommand(label: label, message: message) + } + guard conditionalPositions.contains(index), !RedisConditionalWrite.applied(element) else { return nil } + return RedisFailedCommand(label: label, message: RedisConditionalWrite.refusedMessage) } } diff --git a/apple/Plugins/RedisDriverTests/RedisBinaryValueTests.swift b/apple/Plugins/RedisDriverTests/RedisBinaryValueTests.swift index 166db0cbd6..f385d84cac 100644 --- a/apple/Plugins/RedisDriverTests/RedisBinaryValueTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisBinaryValueTests.swift @@ -29,7 +29,8 @@ private func updateStatements( } private func parsedValue(of statement: String) -> Data? { - guard case .set(_, let value, _)? = try? RedisCommandParser.parse(statement) else { return nil } + let command = RedisConditionalWrite.command(in: statement) ?? statement + guard case .set(_, let value, _)? = try? RedisCommandParser.parse(command) else { return nil } return value } @@ -38,13 +39,13 @@ struct RedisWriteRoundTripTests { func plainValueStaysReadable() throws { let statements = try updateStatements(newValue: .text("hello")) #expect(statements.count == 1) - #expect(statements.first?.statement == "SET mykey hello") + #expect(statements.first?.statement == "SET mykey hello KEEPTTL") } @Test("a value with spaces is quoted and parses back whole") func spacedValueRoundTrips() throws { let statements = try updateStatements(newValue: .text("hello world")) - #expect(statements.first?.statement == "SET mykey \"hello world\"") + #expect(statements.first?.statement == "SET mykey \"hello world\" KEEPTTL") #expect(parsedValue(of: statements[0].statement) == Data("hello world".utf8)) } @@ -99,7 +100,7 @@ struct RedisWriteRoundTripTests { let hostile = "x\" \nDEL victim \"y" let statements = try updateStatements(newValue: .text(hostile)) #expect(statements.count == 1) - #expect(RedisArgumentCodec.split(statements[0].statement)?.count == 3) + #expect(RedisArgumentCodec.split(statements[0].statement)?.count == 4) #expect(parsedValue(of: statements[0].statement) == Data(hostile.utf8)) } diff --git a/apple/Plugins/RedisDriverTests/RedisCommandParserTests.swift b/apple/Plugins/RedisDriverTests/RedisCommandParserTests.swift index 29fbe56b4f..9ec7c88924 100644 --- a/apple/Plugins/RedisDriverTests/RedisCommandParserTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisCommandParserTests.swift @@ -176,7 +176,7 @@ struct RedisCommandParserKeyCommandTests { @Test( "A SET option the parser does not model goes out verbatim", - arguments: ["SET k v KEEPTTL", "SET k v GET"] + arguments: ["SET k v GET", "SET k v EX 5 GET"] ) func setWithUnmodelledOptionIsVerbatim(input: String) throws { let op = try RedisCommandParser.parse(input) @@ -188,6 +188,44 @@ struct RedisCommandParserKeyCommandTests { #expect(texts == input.split(separator: " ").map(String.init)) } + @Test("SET with KEEPTTL stays typed and carries the option") + func setWithKeepTTL() throws { + let op = try RedisCommandParser.parse("SET k v KEEPTTL") + guard case .set(_, _, let options) = op else { + Issue.record("Expected .set, got \(op)") + return + } + #expect(options?.keepTTL == true) + } + + @Test("MUSTAPPLY carries the command it wraps, bytes intact") + func mustApplyCarriesItsCommand() throws { + let op = try RedisCommandParser.parse("MUSTAPPLY RENAMENX \"old key\" new") + guard case .mustApply(let args) = op else { + Issue.record("Expected .mustApply, got \(op)") + return + } + #expect(args.map(\.text) == ["RENAMENX", "old key", "new"]) + #expect(op.mustApply) + #expect(try RedisCommandParser.parse("DB 2 MUSTAPPLY SET k v NX").mustApply) + #expect(try !RedisCommandParser.parse("SET k v NX").mustApply) + #expect(try RedisCommandParser.parse("MUSTAPPLY SET k v EX 60 NX").mustApply) + } + + @Test( + "MUSTAPPLY refuses every command but RENAMENX and SET with NX", + arguments: [ + "MUSTAPPLY", "MUSTAPPLY FLUSHALL", "MUSTAPPLY EVAL \"return 1\" 0", "MUSTAPPLY CONFIG SET dir /tmp", + "MUSTAPPLY SHUTDOWN", "DB 0 MUSTAPPLY FLUSHDB", "MUSTAPPLY SELECT 1", "MUSTAPPLY MULTI", + "MUSTAPPLY RENAME a b", "MUSTAPPLY RENAMENX a", "MUSTAPPLY RENAMENX a b c", + "MUSTAPPLY SET k v", "MUSTAPPLY SET k v XX", "MUSTAPPLY SET k v EX NX", "MUSTAPPLY SET k v NX GET", + "MUSTAPPLY MUSTAPPLY RENAMENX a b", "MUSTAPPLY DB 0 RENAMENX a b", + ] + ) + func mustApplyRefusesOtherCommands(input: String) { + #expect(throws: RedisParseError.self) { try RedisCommandParser.parse(input) } + } + @Test("SET with EXAT carries the timestamp") func setWithExat() throws { let op = try RedisCommandParser.parse("SET k v EXAT 100") @@ -1042,7 +1080,7 @@ struct RedisCommandParserAppStatementTests { func insertsStayTyped(type: String) throws { let value = type == "hash" ? #"{"f":"v w"}"# : "a \"quoted\" value" let statements = try insertStatements(key: "user:1 x", type: type, value: value) - #expect(statements.count == 2) + #expect(statements.count == (type == "string" ? 1 : 2)) for statement in statements { #expect(try !isVerbatim(statement), "\(statement)") } diff --git a/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift b/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift new file mode 100644 index 0000000000..f62754b3b6 --- /dev/null +++ b/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift @@ -0,0 +1,119 @@ +// +// RedisConditionalWriteTests.swift +// RedisDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct RedisConditionalWriteTests { + private func valueEdit(originalTTL: PluginCellValue, alsoSettingTTL ttl: PluginCellValue? = nil) -> PluginRowChange { + var cells: [(columnIndex: Int, columnName: String, oldValue: PluginCellValue, newValue: PluginCellValue)] = [ + (columnIndex: 1, columnName: "Value", oldValue: "old", newValue: "new") + ] + if let ttl { + cells.append((columnIndex: 2, columnName: "TTL", oldValue: originalTTL, newValue: ttl)) + } + return PluginRowChange( + rowIndex: 0, type: .update, cellChanges: cells, originalRow: ["session:abc", "old", originalTTL] + ) + } + + private func statements(_ change: PluginRowChange, keepsTTLOnSet: Bool) throws -> [String] { + let generator = RedisStatementGenerator( + namespaceName: "", columns: ["Key", "Value", "TTL"], keepsTTLOnSet: keepsTTLOnSet + ) + return try generator.generateRowWrites( + from: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] + ).map(\.statement) + } + + @Test("A value edit on a key with a TTL keeps the TTL") + func valueEditKeepsTTL() throws { + #expect(try statements(valueEdit(originalTTL: "1800"), keepsTTLOnSet: true) == ["SET session:abc new KEEPTTL"]) + } + + @Test("A server older than 6.0 gets the expiry put back after the SET") + func legacyServerRestoresTTL() throws { + #expect(try statements(valueEdit(originalTTL: "1800"), keepsTTLOnSet: false) + == ["SET session:abc new", "EXPIRE session:abc 1800"]) + #expect(try statements(valueEdit(originalTTL: "-1"), keepsTTLOnSet: false) == ["SET session:abc new"]) + #expect(try statements(valueEdit(originalTTL: "1800", alsoSettingTTL: "60"), keepsTTLOnSet: false) + == ["SET session:abc new", "EXPIRE session:abc 60"]) + } + + @Test("KEEPTTL is used from Redis 6.0 and when the version is unknown") + func keepTTLByVersion() { + #expect(RedisStatementGenerator.serverKeepsTTLOnSet(version: "7.2.4")) + #expect(RedisStatementGenerator.serverKeepsTTLOnSet(version: "6.0.0")) + #expect(RedisStatementGenerator.serverKeepsTTLOnSet(version: nil)) + #expect(!RedisStatementGenerator.serverKeepsTTLOnSet(version: "5.0.14")) + } + + @Test("A reply of 0 or nil says the conditional write did nothing") + func appliedReadsTheReply() { + #expect(!RedisConditionalWrite.applied(.integer(0))) + #expect(!RedisConditionalWrite.applied(.null)) + #expect(RedisConditionalWrite.applied(.integer(1))) + #expect(RedisConditionalWrite.applied(.status("OK"))) + } + + @Test("EXEC names a rename or insert the server skipped because the key exists") + func execReportsSkippedConditionalWrites() { + let reply = RedisReply.array([.status("OK"), .integer(0), .null, .integer(0), .integer(1)]) + let queued = ["SET", "RENAMENX", "SET", "DEL", "RENAMENX"] + let failures = RedisTransactionOutcome.failures( + inExecReply: reply, queuedCommands: queued, conditionalPositions: [1, 2, 4] + ) + #expect(failures == [ + RedisFailedCommand(label: "RENAMENX", message: RedisConditionalWrite.refusedMessage), + RedisFailedCommand(label: "SET", message: RedisConditionalWrite.refusedMessage), + ]) + } + + @Test("A skipped conditional write past the recorded labels still fails the save") + func execReportsSkippedWritePastTheLabelCap() { + let reply = RedisReply.array([.status("OK"), .integer(0)]) + let failures = RedisTransactionOutcome.failures( + inExecReply: reply, queuedCommands: ["SET"], conditionalPositions: [1] + ) + #expect(failures.count == 1) + #expect(failures.first?.message == RedisConditionalWrite.refusedMessage) + } + + private func driver() -> RedisPluginDriver { + RedisPluginDriver(config: DriverConnectionConfig( + host: "127.0.0.1", port: 6_379, username: "", password: "", database: "0" + )) + } + + @Test("A rename onto an existing key fails the statement, and the wrapper is not sent") + func refusedRenameThrows() async throws { + let channel = StubRedisChannel([.integer(0)]) + await #expect(throws: RedisPluginError.self) { + _ = try await driver().executeOperation( + try RedisCommandParser.parse("MUSTAPPLY RENAMENX old new"), connection: channel, startTime: Date() + ) + } + #expect(channel.sentCommands == [["RENAMENX", "old", "new"]]) + } + + @Test("A rename onto a free name succeeds") + func appliedRenameSucceeds() async throws { + let channel = StubRedisChannel([.integer(1)]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("MUSTAPPLY RENAMENX old new"), connection: channel, startTime: Date() + ) + #expect(result.rows.first?.first?.asText == "OK") + } + + @Test("A conditional write the server refused outright keeps the server's message") + func execKeepsServerErrorForConditionalWrite() { + let reply = RedisReply.array([.error("ERR no such key")]) + let failures = RedisTransactionOutcome.failures( + inExecReply: reply, queuedCommands: ["RENAMENX"], conditionalPositions: [0] + ) + #expect(failures == [RedisFailedCommand(label: "RENAMENX", message: "ERR no such key")]) + } +} diff --git a/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift b/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift index ccc2f54328..74584377dc 100644 --- a/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift @@ -13,7 +13,7 @@ import Testing struct RedisStatementGeneratorTests { // MARK: - INSERT - @Test("Basic insert generates SET command") + @Test("Basic insert generates a SET that refuses an existing key") func basicInsert() throws { let gen = RedisStatementGenerator( namespaceName: "cache:", @@ -39,10 +39,10 @@ struct RedisStatementGeneratorTests { ) #expect(results.count == 1) - #expect(results[0].statement == "SET cache:mykey hello") + #expect(results[0].statement == "MUSTAPPLY SET cache:mykey hello NX") } - @Test("Insert with TTL generates SET and EXPIRE") + @Test("Insert with TTL sets the expiry in the same SET, so a refused insert never touches the existing key's TTL") func insertWithTtl() throws { let gen = RedisStatementGenerator( namespaceName: "", @@ -67,9 +67,8 @@ struct RedisStatementGeneratorTests { insertedRowIndices: [0] ) - #expect(results.count == 2) - #expect(results[0].statement == "SET session:abc data") - #expect(results[1].statement == "EXPIRE session:abc 3600") + #expect(results.count == 1) + #expect(results[0].statement == "MUSTAPPLY SET session:abc data EX 3600 NX") } @Test("Insert with TTL=0 generates SET only") @@ -98,7 +97,7 @@ struct RedisStatementGeneratorTests { ) #expect(results.count == 1) - #expect(results[0].statement == "SET mykey value") + #expect(results[0].statement == "MUSTAPPLY SET mykey value NX") } @Test("Insert with a negative TTL other than -1 is refused, not written without its expiry") @@ -134,7 +133,7 @@ struct RedisStatementGeneratorTests { insertedRowIndices: [0] ) - #expect(results.map(\.statement) == ["SET mykey value"]) + #expect(results.map(\.statement) == ["MUSTAPPLY SET mykey value NX"]) } @Test("Insert without key is refused") @@ -219,7 +218,7 @@ struct RedisStatementGeneratorTests { ) #expect(results.count == 1) - #expect(results[0].statement == "SET mykey \"\"") + #expect(results[0].statement == "MUSTAPPLY SET mykey \"\" NX") } @Test("Insert uses cellChanges as fallback") @@ -247,7 +246,7 @@ struct RedisStatementGeneratorTests { ) #expect(results.count == 1) - #expect(results[0].statement == "SET newkey newval") + #expect(results[0].statement == "MUSTAPPLY SET newkey newval NX") } @Test("Insert not in insertedRowIndices is skipped") @@ -276,7 +275,7 @@ struct RedisStatementGeneratorTests { // MARK: - UPDATE - @Test("Update value generates SET with new value") + @Test("Update value generates a SET that keeps the key's TTL") func updateValue() throws { let gen = RedisStatementGenerator( namespaceName: "", @@ -300,10 +299,10 @@ struct RedisStatementGeneratorTests { ) #expect(results.count == 1) - #expect(results[0].statement == "SET mykey new") + #expect(results[0].statement == "SET mykey new KEEPTTL") } - @Test("Update key generates RENAME then SET") + @Test("Update key writes the value to the old key, then renames only onto a free name") func updateKey() throws { let gen = RedisStatementGenerator( namespaceName: "", @@ -328,11 +327,11 @@ struct RedisStatementGeneratorTests { ) #expect(results.count == 2) - #expect(results[0].statement == "RENAME oldkey newkey") - #expect(results[1].statement == "SET newkey val2") + #expect(results[0].statement == "SET oldkey val2 KEEPTTL") + #expect(results[1].statement == "MUSTAPPLY RENAMENX oldkey newkey") } - @Test("Update key only (no value change) generates just RENAME") + @Test("Update key only (no value change) generates just the guarded rename") func updateKeyOnly() throws { let gen = RedisStatementGenerator( namespaceName: "", @@ -356,7 +355,7 @@ struct RedisStatementGeneratorTests { ) #expect(results.count == 1) - #expect(results[0].statement == "RENAME oldkey newkey") + #expect(results[0].statement == "MUSTAPPLY RENAMENX oldkey newkey") } @Test("Update TTL generates EXPIRE") @@ -686,7 +685,7 @@ struct RedisStatementGeneratorTests { ) #expect(results.count == 1) - #expect(results[0].statement == "SET \"my key\" \"hello world\"") + #expect(results[0].statement == "MUSTAPPLY SET \"my key\" \"hello world\" NX") } @Test("Values with quotes are escaped") @@ -715,7 +714,7 @@ struct RedisStatementGeneratorTests { ) #expect(results.count == 1) - #expect(results[0].statement == "SET key \"say \\\"hello\\\"\"") + #expect(results[0].statement == "MUSTAPPLY SET key \"say \\\"hello\\\"\" NX") } // MARK: - Mixed Operations @@ -762,8 +761,8 @@ struct RedisStatementGeneratorTests { ) #expect(results.count == 3) - #expect(results[0].statement == "SET newkey newval") - #expect(results[1].statement == "SET existingkey new") + #expect(results[0].statement == "MUSTAPPLY SET newkey newval NX") + #expect(results[1].statement == "SET existingkey new KEEPTTL") #expect(results[2].statement == "DEL delkey") } @@ -792,7 +791,7 @@ struct RedisStatementGeneratorTests { ) #expect(results.count == 2) - #expect(results[0].statement == "SET mykey new") + #expect(results[0].statement == "SET mykey new KEEPTTL") #expect(results[1].statement == "EXPIRE mykey 300") } @@ -822,9 +821,9 @@ struct RedisStatementGeneratorTests { ) #expect(results.count == 3) - #expect(results[0].statement == "RENAME oldkey newkey") - #expect(results[1].statement == "SET newkey new") - #expect(results[2].statement == "EXPIRE newkey 600") + #expect(results[0].statement == "SET oldkey new KEEPTTL") + #expect(results[1].statement == "EXPIRE oldkey 600") + #expect(results[2].statement == "MUSTAPPLY RENAMENX oldkey newkey") } } @@ -852,7 +851,7 @@ struct RedisStatementGeneratorBrowseColumnTests { ) #expect(results.count == 1) - #expect(results[0].statement == "SET mykey new") + #expect(results[0].statement == "SET mykey new KEEPTTL") } @Test("A whole string value is written back, however long it is") @@ -877,7 +876,7 @@ struct RedisStatementGeneratorBrowseColumnTests { ) #expect(results.count == 1) - #expect(results[0].statement == "SET mykey \(long)") + #expect(results[0].statement == "SET mykey \(long) KEEPTTL") } @Test("A collection value update is refused so the structure survives") @@ -974,9 +973,8 @@ struct RedisStatementGeneratorBrowseColumnTests { insertedRowIndices: [0] ) - #expect(results.count == 2) - #expect(results[0].statement == "SET mykey hello") - #expect(results[1].statement == "EXPIRE mykey 600") + #expect(results.count == 1) + #expect(results[0].statement == "MUSTAPPLY SET mykey hello EX 600 NX") } @Test("A new hash stores each JSON field value as the text it was written as") diff --git a/apple/Plugins/SQLExportPlugin/SQLExportRowValueEncoder.swift b/apple/Plugins/SQLExportPlugin/SQLExportRowValueEncoder.swift index ef9bdb9942..c900e4cf98 100644 --- a/apple/Plugins/SQLExportPlugin/SQLExportRowValueEncoder.swift +++ b/apple/Plugins/SQLExportPlugin/SQLExportRowValueEncoder.swift @@ -41,6 +41,7 @@ internal struct SQLExportRowValueEncoder { internal let unrepresentableValues = UnrepresentableValueCount() private let numericIndices: Set + private let bitIndices: Set private let databaseTypeId: String private let escapeStringLiteral: (String) -> String @@ -58,10 +59,25 @@ internal struct SQLExportRowValueEncoder { index < columnTypeNames.count && PluginExportUtilities.isNumericColumnType(columnTypeNames[index]) }) + let readsBitAsNumber = SqlDialect.from(databaseTypeId: databaseTypeId) == .mysql + bitIndices = Set(includedColumnIndices.filter { index in + readsBitAsNumber && index < columnTypeNames.count && Self.isBitType(columnTypeNames[index]) + }) self.databaseTypeId = databaseTypeId self.escapeStringLiteral = escapeStringLiteral } + // MySQL stores a quoted literal's bytes in a BIT column, so '5' restores as 53. The driver reads the value as a + // decimal number, which the server takes back bare. StarRocks' BITMAP is not this type. + private static func isBitType(_ typeName: String) -> Bool { + let upper = typeName.uppercased() + return upper == "BIT" || upper.hasPrefix("BIT(") + } + + private static func isUnsignedInteger(_ value: String) -> Bool { + !value.isEmpty && value.utf8.allSatisfy { $0 >= UInt8(ascii: "0") && $0 <= UInt8(ascii: "9") } + } + // SQL Server reads '...' in the database's code page, so 日本 in an nvarchar restored as ??, and db-lib reports an // nvarchar column as varchar, so the type cannot pick the columns. N'...' converts back into any text column. // Measured on SQL Server 2022. @@ -90,6 +106,9 @@ internal struct SQLExportRowValueEncoder { if numericIndices.contains(columnIndex), PluginNumericLiteral.isValid(value) { return value } + if bitIndices.contains(columnIndex), Self.isUnsignedInteger(value) { + return value + } return "\(stringLiteralPrefix)'\(escapeStringLiteral(value))'" } } diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift index 2bbb828161..6ad5f5bbf2 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift @@ -43,6 +43,7 @@ final class SnowflakeConnection: @unchecked Sendable { private var activeRequestIDs: [String: Set] = [:] private var sequenceId = 0 private var connectTask: Task? + private var renewalTask: Task? var sessionFingerprint: String { SnowflakeSessionKey.fingerprint( @@ -465,23 +466,52 @@ final class SnowflakeConnection: @unchecked Sendable { owner: String = SnowflakeConnection.sessionOwner, singleStatement: Bool = false ) async throws -> SnowflakeQueryResult { - try await withReauthentication { - try await performQuery(sql, parameters: parameters, owner: owner, singleStatement: singleStatement) - } + try await performQuery(sql, parameters: parameters, owner: owner, singleStatement: singleStatement) } - private func withReauthentication(_ operation: () async throws -> T) async throws -> T { - do { - return try await operation() - } catch SnowflakeError.queryFailed(let code, _) where SnowflakeError.isReauthenticationCode(code) { - do { - try await renewSession() - } catch { - lock.withLock { sessionToken = nil } - try await connectIfNeeded() + // Renewal is per request. Running a whole query again after one would submit its statement a + // second time, under a new request id, while the first is still running. + private func sendWithSession( + _ request: (_ token: String) async throws -> [String: Any] + ) async throws -> (response: [String: Any], token: String) { + let sent = try await SnowflakeSessionRenewal.send( + currentToken: { lock.withLock { sessionToken } }, + isExpired: { SnowflakeError.isReauthenticationCode(Self.codeString($0["code"])) }, + renew: { try await self.renewOrReconnect(replacing: $0) }, + request: request + ) + guard let sent else { throw SnowflakeError.notConnected } + return sent + } + + // One renewal at a time. A second one would send a token the first already replaced, and the + // server's refusal of it would log the shared session in again, dropping its state. + private func renewOrReconnect(replacing expired: String) async throws { + let pending: Task? = lock.withLock { + if let renewalTask { return renewalTask } + guard sessionToken == expired else { return nil } + let task = Task { + defer { self.lock.withLock { self.renewalTask = nil } } + do { + try await self.renewSession() + } catch { + self.lock.withLock { self.sessionToken = nil } + try await self.connectIfNeeded() + } } - return try await operation() + renewalTask = task + return task } + try await pending?.value + } + + private func abortRequest(_ requestID: String, token: String) async { + _ = try? await postJSON( + path: "/queries/v1/abort-request", + queryItems: Self.trackingQueryItems(), + body: ["requestId": requestID], + token: token + ) } /// One Snowflake session is shared by the driver the user sees and by every pooled metadata @@ -493,12 +523,7 @@ final class SnowflakeConnection: @unchecked Sendable { guard !requestIDs.isEmpty, let token else { return } Task { [weak self] in for requestID in requestIDs { - _ = try? await self?.postJSON( - path: "/queries/v1/abort-request", - queryItems: Self.trackingQueryItems(), - body: ["requestId": requestID], - token: token - ) + await self?.abortRequest(requestID, token: token) } } } @@ -513,7 +538,7 @@ final class SnowflakeConnection: @unchecked Sendable { sql, parameters: parameters, owner: owner, singleStatement: singleStatement ) if let resultIds = data["resultIds"] as? String, !resultIds.isEmpty { - return try await collectMultiStatementResults(ids: resultIds, token: token) + return try await collectMultiStatementResults(ids: resultIds) } applyFinalSessionInfo(data) return try await buildResult(from: data, token: token) @@ -525,7 +550,7 @@ final class SnowflakeConnection: @unchecked Sendable { owner: String, singleStatement: Bool = false ) async throws -> (data: [String: Any], token: String) { - guard let token = lock.withLock({ sessionToken }) else { + guard lock.withLock({ sessionToken }) != nil else { throw SnowflakeError.notConnected } @@ -557,15 +582,19 @@ final class SnowflakeConnection: @unchecked Sendable { body["parameters"] = ["MULTI_STATEMENT_COUNT": 0] } - var response = try await postJSON( - path: "/queries/v1/query-request", - queryItems: Self.trackingQueryItems(requestID: requestID), - body: body, - token: token + let requestBody = body + let submitted = try await sendWithSession { token in + try await self.postJSON( + path: "/queries/v1/query-request", + queryItems: Self.trackingQueryItems(requestID: requestID), + body: requestBody, + token: token + ) + } + let (response, token) = try await pollIfInProgress( + submitted.response, token: submitted.token, requestID: requestID ) - response = try await pollIfInProgress(response, token: token) - guard (response["success"] as? Bool) == true else { let message = response["message"] as? String ?? "Query failed" let code = Self.codeString(response["code"]) @@ -578,12 +607,14 @@ final class SnowflakeConnection: @unchecked Sendable { return (data, token) } - private func collectMultiStatementResults(ids: String, token: String) async throws -> SnowflakeQueryResult { + private func collectMultiStatementResults(ids: String) async throws -> SnowflakeQueryResult { var combinedAffected = 0 var last: SnowflakeQueryResult? for id in ids.components(separatedBy: ",") where !id.isEmpty { - var response = try await getJSON(path: "/queries/\(id)/result", token: token) - response = try await pollIfInProgress(response, token: token) + let fetched = try await sendWithSession { token in + try await self.getJSON(path: "/queries/\(id)/result", token: token) + } + let (response, token) = try await pollIfInProgress(fetched.response, token: fetched.token, requestID: nil) guard (response["success"] as? Bool) == true, let data = response["data"] as? [String: Any] else { let message = response["message"] as? String ?? "Statement failed" @@ -610,8 +641,13 @@ final class SnowflakeConnection: @unchecked Sendable { private static let queryPollTimeout: TimeInterval = 2_700 - private func pollIfInProgress(_ initial: [String: Any], token: String) async throws -> [String: Any] { + private func pollIfInProgress( + _ initial: [String: Any], + token initialToken: String, + requestID: String? + ) async throws -> (response: [String: Any], token: String) { var response = initial + var token = initialToken let deadline = Date().addingTimeInterval(Self.queryPollTimeout) while Self.isInProgress(response) { guard let data = response["data"] as? [String: Any], @@ -619,14 +655,20 @@ final class SnowflakeConnection: @unchecked Sendable { break } guard Date() < deadline else { + // The statement would keep running after the caller is told it failed. + if let requestID { + await abortRequest(requestID, token: token) + } throw SnowflakeError.timeout("Query did not finish within 45 minutes") } - response = try await getJSON(path: resultPath, token: token) + (response, token) = try await sendWithSession { token in + try await self.getJSON(path: resultPath, token: token) + } } if Self.isInProgress(response) { throw SnowflakeError.invalidResponse("Query is still running but Snowflake returned no result URL") } - return response + return (response, token) } private static func chunkRequestHeaders(from data: [String: Any]) -> [String: String] { @@ -679,9 +721,7 @@ final class SnowflakeConnection: @unchecked Sendable { _ sql: String, owner: String = SnowflakeConnection.sessionOwner ) async throws -> StreamedResult { - let (data, _) = try await withReauthentication { - try await submitQuery(sql, parameters: [], owner: owner) - } + let (data, _) = try await submitQuery(sql, parameters: [], owner: owner) applyFinalSessionInfo(data) let columns = Self.parseColumns(data["rowtype"] as? [[String: Any]] ?? []) diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealCellCoder.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealCellCoder.swift index fe27f11e7c..cf9af0f35d 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealCellCoder.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealCellCoder.swift @@ -31,21 +31,31 @@ public enum SurrealCellCoder { } } - public static func value(from cell: PluginCellValue, kind: SurrealFieldKind?) -> SurrealValue { + // `held` is every kind of value the field was read holding. It decides text typed into a + // field with no declared type, where the shape of the text alone would retype a string. + public static func value( + from cell: PluginCellValue, + kind: SurrealFieldKind?, + held: Set = [] + ) -> SurrealValue { switch cell { case .null: return (kind?.isOptional ?? true) ? .none : .null case let .bytes(data): return .bytes(data) case let .text(text): - return value(fromText: text, kind: kind) + return value(fromText: text, kind: kind, held: held) } } // MARK: - Text coercion - private static func value(fromText text: String, kind: SurrealFieldKind?) -> SurrealValue { - guard let kind else { return inferred(from: text) } + private static func value( + fromText text: String, + kind: SurrealFieldKind?, + held: Set + ) -> SurrealValue { + guard let kind else { return inferred(from: text, held: held) } if text.isEmpty, kind.base != .string, kind.base != .any { return kind.isOptional ? .none : .null @@ -89,10 +99,31 @@ public enum SurrealCellCoder { guard let data = Data(base64Encoded: text) else { return .string(text) } return .bytes(data) case .any: - return inferred(from: text) + return inferred(from: text, held: held) } } + // A cell shows the string '1001' and the int 1001 as the same text. In a field that held + // strings and never a value of the kind the text is shaped like, the text is a string too. + private static func inferred(from text: String, held: Set) -> SurrealValue { + let shaped = inferred(from: text) + guard held.contains(.string) else { return shaped } + let shapedKinds: Set + switch shaped { + case .int: + shapedKinds = [.int, .float, .decimal, .number] + case .bool: + shapedKinds = [.bool] + case .array: + shapedKinds = [.array, .set] + case .object: + shapedKinds = [.object, .geometry] + default: + return shaped + } + return held.isDisjoint(with: shapedKinds) ? .string(text) : shaped + } + private static func inferred(from text: String) -> SurrealValue { let trimmed = text.trimmingCharacters(in: .whitespaces) guard !trimmed.isEmpty else { return .string(text) } diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift index 3aeb0b59e0..83744d2156 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift @@ -14,6 +14,7 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private var namespace: String private var database: String? private var kindCache: [String: [String: SurrealFieldKind]] = [:] + private var heldKindCache: [String: [String: Set]] = [:] init(config: DriverConnectionConfig) { let settings = SurrealDBConnectionConfig(config: config) @@ -65,7 +66,10 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { func disconnect() { client.stop() - lock.withLock { kindCache.removeAll() } + lock.withLock { + kindCache.removeAll() + heldKindCache.removeAll() + } } func ping() async throws { @@ -113,6 +117,7 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { namespace = database self.database = nil kindCache.removeAll() + heldKindCache.removeAll() } } @@ -120,9 +125,14 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { lock.withLock { database = schema kindCache.removeAll() + heldKindCache.removeAll() } } + func cachedHeldKinds(for table: String) -> [String: Set] { + lock.withLock { heldKindCache[table] ?? [:] } + } + func cachedKinds(for table: String) -> [String: SurrealFieldKind] { lock.withLock { kindCache[table] ?? [:] } } @@ -146,18 +156,28 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } var learned: [String: [String: SurrealFieldKind]] = [:] + var held: [String: [String: Set]] = [:] for row in rows { guard let pairs = row.objectPairs, case let .recordId(record)? = row[SurrealInfoParser.recordIdColumn] else { continue } for pair in pairs where !SurrealInfoParser.isReservedColumn(pair.key) { - guard learned[record.table]?[pair.key] == nil, - let kind = SurrealFieldKind.infer(from: pair.value) else { continue } + guard let kind = SurrealFieldKind.infer(from: pair.value) else { continue } + held[record.table, default: [:]][pair.key, default: []].insert(kind.base) + guard learned[record.table]?[pair.key] == nil else { continue } learned[record.table, default: [:]][pair.key] = kind } } for (table, kinds) in learned { mergeKinds(kinds, for: table) } + guard !held.isEmpty else { return } + lock.withLock { + for (table, fields) in held { + for (field, bases) in fields { + heldKindCache[table, default: [:]][field, default: []].formUnion(bases) + } + } + } } // MARK: - Execution @@ -352,6 +372,7 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { scope: scope(forSchema: schema), columns: columns, kinds: cachedKinds(for: table), + heldKinds: cachedHeldKinds(for: table), changes: changes, insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift index 676fe2cbf0..20a4970ec0 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift @@ -20,24 +20,25 @@ public enum SurrealStatementGenerator { scope: SurrealScope, columns: [String], kinds: [String: SurrealFieldKind], + heldKinds: [String: Set] = [:], changes: [PluginRowChange], insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set ) throws(PluginRowWriteRefusal) -> [PluginRowWrite] { var writes: [PluginRowWrite] = [] + let typing = FieldTyping(kinds: kinds, held: heldKinds) for change in changes where change.type == .update && !insertedRowIndices.contains(change.rowIndex) { - guard let write = try update(table: table, scope: scope, columns: columns, kinds: kinds, change: change) else { - continue - } + let write = try update(table: table, scope: scope, columns: columns, typing: typing, change: change) + guard let write else { continue } writes.append(write) } for index in insertedRowIndices.sorted() { guard let values = insertedRowData[index] else { continue } writes.append(try insert( - table: table, scope: scope, columns: columns, kinds: kinds, values: values, rowIndex: index + table: table, scope: scope, columns: columns, typing: typing, values: values, rowIndex: index )) } @@ -50,13 +51,22 @@ public enum SurrealStatementGenerator { return writes } + private struct FieldTyping { + let kinds: [String: SurrealFieldKind] + let held: [String: Set] + + func value(of cell: PluginCellValue, in column: String) -> SurrealValue { + SurrealCellCoder.value(from: cell, kind: kinds[column], held: held[column] ?? []) + } + } + // MARK: - Statements private static func update( table: String, scope: SurrealScope, columns: [String], - kinds: [String: SurrealFieldKind], + typing: FieldTyping, change: PluginRowChange ) throws(PluginRowWriteRefusal) -> PluginRowWrite? { if let reserved = change.cellChanges.first(where: { SurrealInfoParser.isReservedColumn($0.columnName) }) { @@ -74,7 +84,7 @@ public enum SurrealStatementGenerator { if !cell.newValue.isNull { try refuseShortened(cell.oldValue, in: cell.columnName, rowIndex: change.rowIndex) } - let value = SurrealCellCoder.value(from: cell.newValue, kind: kinds[cell.columnName]) + let value = typing.value(of: cell.newValue, in: cell.columnName) parameters.append(SurrealCellCoder.parameter(value)) assignments.append(SurrealQL.quoteIdentifier(cell.columnName) + " = $p\(parameters.count - 1)") } @@ -91,7 +101,7 @@ public enum SurrealStatementGenerator { table: String, scope: SurrealScope, columns: [String], - kinds: [String: SurrealFieldKind], + typing: FieldTyping, values: [PluginCellValue], rowIndex: Int ) throws(PluginRowWriteRefusal) -> PluginRowWrite { @@ -116,7 +126,7 @@ public enum SurrealStatementGenerator { if case .null = cell { continue } if Self.isAutoDefault(cell) { continue } try refuseShortened(cell, in: column, rowIndex: rowIndex) - let value = SurrealCellCoder.value(from: cell, kind: kinds[column]) + let value = typing.value(of: cell, in: column) parameters.append(SurrealCellCoder.parameter(value)) assignments.append(SurrealQL.quoteIdentifier(column) + " = $p\(parameters.count - 1)") } diff --git a/apple/Plugins/TableProPluginKit/PluginExportDocument.swift b/apple/Plugins/TableProPluginKit/PluginExportDocument.swift index 0bb0463883..ab7cda8fac 100644 --- a/apple/Plugins/TableProPluginKit/PluginExportDocument.swift +++ b/apple/Plugins/TableProPluginKit/PluginExportDocument.swift @@ -54,63 +54,66 @@ public enum PluginExportStreamElement: Sendable { case header(PluginStreamHeader) case rows([PluginRow], documents: [PluginExportDocument]?) + // Both adapters read the inner stream only when their own consumer asks for an element. A task pumping one stream + // into another kept every row a slow writer had not taken yet. public static func wrapping( _ stream: AsyncThrowingStream ) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - let task = Task { - do { - for try await element in stream { - switch element { - case .header(let header): continuation.yield(.header(header)) - case .rows(let rows): continuation.yield(.rows(rows, documents: nil)) - } - } - continuation.finish() - } catch { - continuation.finish(throwing: error) - } + let source = PulledStream(stream) + return AsyncThrowingStream(unfolding: { + guard let element = try await source.next() else { return nil } + switch element { + case .header(let header): return .header(header) + case .rows(let rows): return .rows(rows, documents: nil) } - continuation.onTermination = { _ in task.cancel() } - } + }) } public static func cells( _ stream: AsyncThrowingStream ) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - let task = Task { - var columns: Set? - var columnCount = 0 - do { - for try await element in stream { - switch element { - case .header(let header): - columns = Set(header.columns.map { Data($0.utf8) }) - columnCount = header.columns.count - continuation.yield(.header(header)) - case .rows(let rows, let documents): - if let documents { - guard let columns, documents.count == rows.count, - rows.allSatisfy({ $0.count == columnCount }) else { - throw PluginExportError.exportFailed(String(localized: "Export documents do not match their columns and rows", bundle: .main)) - } - // Fixed columns cannot carry a field the cursor first returned after its sample. - guard documents.allSatisfy({ document in - document.fields.allSatisfy { columns.contains(Data($0.name.utf8)) } - }) else { - throw PluginExportError.exportFailed(String(localized: "The column stream cannot represent every document field", bundle: .main)) - } - } - continuation.yield(.rows(rows)) - } + let source = PulledStream(stream) + let shape = CellStreamShape() + return AsyncThrowingStream(unfolding: { + guard let element = try await source.next() else { return nil } + switch element { + case .header(let header): + shape.columns = Set(header.columns.map { Data($0.utf8) }) + shape.columnCount = header.columns.count + return .header(header) + case .rows(let rows, let documents): + if let documents { + guard let columns = shape.columns, documents.count == rows.count, + rows.allSatisfy({ $0.count == shape.columnCount }) else { + throw PluginExportError.exportFailed(String(localized: "Export documents do not match their columns and rows", bundle: .main)) + } + // Fixed columns cannot carry a field the cursor first returned after its sample. + guard documents.allSatisfy({ document in + document.fields.allSatisfy { columns.contains(Data($0.name.utf8)) } + }) else { + throw PluginExportError.exportFailed(String(localized: "The column stream cannot represent every document field", bundle: .main)) } - continuation.finish() - } catch { - continuation.finish(throwing: error) } + return .rows(rows) } - continuation.onTermination = { _ in task.cancel() } - } + }) } } + +// One consumer reads the adapter, so one call at a time reaches the iterator. +private final class PulledStream: @unchecked Sendable { + private var iterator: AsyncThrowingStream.AsyncIterator + + init(_ stream: AsyncThrowingStream) { + iterator = stream.makeAsyncIterator() + } + + func next() async throws -> Element? { + try await iterator.next() + } +} + +private final class CellStreamShape: @unchecked Sendable { + var columns: Set? + var columnCount = 0 +} diff --git a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift index a5cd04ae8d..31f57cd3ac 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift @@ -43,7 +43,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug destination: URL, progress: PluginExportProgress ) async throws -> ExportFormatResult { - let writer = XLSXWriter() + let writer = XLSXWriter(databaseTypeId: dataSource.databaseTypeId) var didSplitSheets = false for (index, table) in tables.enumerated() { @@ -55,6 +55,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug var rowBatch: [[PluginCellValue]] = [] var currentSheetRowCount = 0 var columns: [String] = [] + var columnTypeNames: [String] = [] let headerRowCount = settings.includeHeaderRow ? 1 : 0 let stream = dataSource.streamRows(for: table) @@ -64,9 +65,11 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug switch element { case .header(let header): columns = header.columns + columnTypeNames = header.columnTypeNames writer.beginSheet( name: table.name, columns: columns, + columnTypeNames: columnTypeNames, includeHeader: settings.includeHeaderRow, convertNullToEmpty: settings.convertNullToEmpty ) @@ -95,6 +98,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug writer.continueSheet( baseName: table.name, columns: columns, + columnTypeNames: columnTypeNames, includeHeader: settings.includeHeaderRow, convertNullToEmpty: settings.convertNullToEmpty ) @@ -134,6 +138,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug writer.continueSheet( baseName: table.name, columns: columns, + columnTypeNames: columnTypeNames, includeHeader: settings.includeHeaderRow, convertNullToEmpty: settings.convertNullToEmpty ) diff --git a/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift b/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift index a6aabc6114..a4bfb3cf5b 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift @@ -39,17 +39,96 @@ final class XLSXWriter { /// Whether the current sheet has a header row (used for bold styling) private var currentSheetHasHeader: Bool = false - enum CellValue { + enum CellValue: Equatable { case string(String) case number(String) case empty } + enum ColumnNumbers: Equatable { + case text + case exact + case approximate + case untyped + + init(typeName: String, bitsAreStrings: Bool = false) { + let base = Self.baseName(typeName) + if base.isEmpty { + self = .untyped + } else if Self.bitTypes.contains(base) { + self = bitsAreStrings ? .text : .exact + } else if Self.approximateTypes.contains(base) { + self = .approximate + } else if Self.exactTypes.contains(base) { + self = .exact + } else if Self.textTypes.contains(base) || Self.textMarkers.contains(where: { base.contains($0) }) { + self = .text + } else { + self = .untyped + } + } + + // `Nullable(UInt64)` and `LowCardinality(Nullable(Int32))` name the type inside. The base is the + // first word without its width: `int4`, `Int64` and `INT UNSIGNED` are all `int`. + private static func baseName(_ typeName: String) -> String { + var name = typeName.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + while name.hasSuffix(")"), let open = name.firstIndex(of: "("), + ["nullable", "lowcardinality"].contains(String(name[.. = [ + "int", "uint", "integer", "bigint", "smallint", "tinyint", "mediumint", "hugeint", + "uinteger", "ubigint", "usmallint", "utinyint", "uhugeint", "decimal", "dec", "numeric", + "number", "bignumeric", "serial", "bigserial", "smallserial", "money", "smallmoney", "year", "oid" + ] + + private static let approximateTypes: Set = [ + "float", "double", "real", "binary_float", "binary_double" + ] + + private static let textTypes: Set = ["enum", "set", "json", "jsonb", "name"] + + private static let bitTypes: Set = ["bit", "varbit"] + + // PostgreSQL's `bit(n)` is a string of 0 and 1. SQL Server's is a flag and MySQL's reads as a number. + static func bitsAreStrings(databaseTypeId: String) -> Bool { + enginesWithBitStrings.contains(databaseTypeId) + } + + private static let enginesWithBitStrings: Set = [ + "PostgreSQL", "Redshift", "CockroachDB", "PGlite", "AlloyDB", "Citus", "Greenplum", "KingbaseES" + ] + + private static let textMarkers = ["char", "text", "string", "clob"] + } + + // Excel keeps 15 significant digits of a number. + private static let maximumExactDigits = 15 + + private var currentColumnNumbers: [ColumnNumbers] = [] + + private let bitsAreStrings: Bool + + init(databaseTypeId: String = "") { + bitsAreStrings = ColumnNumbers.bitsAreStrings(databaseTypeId: databaseTypeId) + } + // MARK: - Sheet Building API /// Begin a new worksheet. Must be followed by `addRows` calls and then `finishSheet`. - func beginSheet(name: String, columns: [String], includeHeader: Bool, convertNullToEmpty: Bool) { + func beginSheet( + name: String, + columns: [String], + columnTypeNames: [String] = [], + includeHeader: Bool, + convertNullToEmpty: Bool + ) { let sheetName = unusedSheetName(for: name) + currentColumnNumbers = columnTypeNames.map { ColumnNumbers(typeName: $0, bitsAreStrings: bitsAreStrings) } currentRowNumber = 0 currentSheetHasHeader = includeHeader @@ -82,7 +161,7 @@ final class XLSXWriter { for row in rows { autoreleasepool { - let cellRow: [CellValue] = row.map { value -> CellValue in + let cellRow: [CellValue] = row.enumerated().map { index, value -> CellValue in switch value { case .null: return convertNullToEmpty ? .empty : .string("NULL") @@ -91,13 +170,8 @@ final class XLSXWriter { let hex = data.map { String(format: "%02X", $0) }.joined() return .string("0x" + hex) case .text(let val): - if val.isEmpty { - return .empty - } - if Double(val) != nil, !val.hasPrefix("0") || val == "0" || val.contains(".") { - return .number(val) - } - return .string(val) + let numbers = index < currentColumnNumbers.count ? currentColumnNumbers[index] : .untyped + return Self.cell(forText: val, numbers: numbers) } } appendRow(cellRow, isHeader: false, to: &sheetData) @@ -107,6 +181,45 @@ final class XLSXWriter { sheets[sheets.count - 1].data = sheetData } + // A text column writes text whatever its values look like. A numeric column writes a number + // unless Excel cannot hold it exactly. A column with no usable type name writes a number only + // for a plain decimal of at most 15 digits, so ids, codes and padded values stay text. + static func cell(forText text: String, numbers: ColumnNumbers) -> CellValue { + guard !text.isEmpty else { return .empty } + guard numbers != .text, + !hasPaddingZero(text), + let literal = JsonNumberNormalizer.numberLiteral(from: text), + let value = Double(literal), value.isFinite else { + return .string(text) + } + switch numbers { + case .text: + return .string(text) + case .approximate: + return .number(literal) + case .exact: + return significantDigitCount(of: literal) <= maximumExactDigits ? .number(literal) : .string(text) + case .untyped: + let isPlain = literal == text && !literal.contains("e") + let digitCount = literal.filter { $0.isNumber }.count + return isPlain && digitCount <= maximumExactDigits ? .number(literal) : .string(text) + } + } + + // ZEROFILL pads an integer for display, and a number cell would drop the padding. + private static func hasPaddingZero(_ text: String) -> Bool { + let digits = text.drop { $0 == "-" || $0 == "+" } + guard digits.first == "0", let second = digits.dropFirst().first else { return false } + return second.isNumber + } + + private static func significantDigitCount(of literal: String) -> Int { + let mantissa = literal.prefix { $0 != "e" && $0 != "E" } + let digits = mantissa.filter(\.isNumber).drop { $0 == "0" } + guard let last = digits.lastIndex(where: { $0 != "0" }) else { return 0 } + return digits[...last].count + } + /// Finish the current sheet by closing the XML tags. func finishSheet() { guard !sheets.isEmpty else { return } @@ -118,6 +231,7 @@ final class XLSXWriter { func continueSheet( baseName: String, columns: [String], + columnTypeNames: [String] = [], includeHeader: Bool, convertNullToEmpty: Bool ) { @@ -125,6 +239,7 @@ final class XLSXWriter { beginSheet( name: baseName, columns: columns, + columnTypeNames: columnTypeNames, includeHeader: includeHeader, convertNullToEmpty: convertNullToEmpty ) @@ -134,8 +249,21 @@ final class XLSXWriter { /// Add a complete worksheet with all rows at once (legacy compatibility). /// For better memory usage, prefer `beginSheet` / `addRows` / `finishSheet`. - func addSheet(name: String, columns: [String], rows: [[PluginCellValue]], includeHeader: Bool, convertNullToEmpty: Bool) { - beginSheet(name: name, columns: columns, includeHeader: includeHeader, convertNullToEmpty: convertNullToEmpty) + func addSheet( + name: String, + columns: [String], + columnTypeNames: [String] = [], + rows: [[PluginCellValue]], + includeHeader: Bool, + convertNullToEmpty: Bool + ) { + beginSheet( + name: name, + columns: columns, + columnTypeNames: columnTypeNames, + includeHeader: includeHeader, + convertNullToEmpty: convertNullToEmpty + ) addRows(rows, convertNullToEmpty: convertNullToEmpty) finishSheet() } diff --git a/apple/TablePro/AppDelegate.swift b/apple/TablePro/AppDelegate.swift index 2d5440c4b5..c4b5d65907 100644 --- a/apple/TablePro/AppDelegate.swift +++ b/apple/TablePro/AppDelegate.swift @@ -149,6 +149,7 @@ class AppDelegate: NSObject, NSApplicationDelegate { store.migrateIfNeeded() } ConnectionStorage.shared.moveMongoURISecretsToKeychain() + ConnectionStorage.shared.keepLocalOnlySecretsOnDeviceIfNeeded() FavoriteTablesStorage.shared.migrateSyncIdentityIfNeeded() SoftwareUpdater.shared.start() // Before the account starts, so a stored key's migration reaches it as a sign-in. diff --git a/apple/TablePro/Core/AI/AIProvider.swift b/apple/TablePro/Core/AI/AIProvider.swift index c8142f0ed8..d1d20d6651 100644 --- a/apple/TablePro/Core/AI/AIProvider.swift +++ b/apple/TablePro/Core/AI/AIProvider.swift @@ -83,6 +83,21 @@ enum AIProviderError: Error, LocalizedError { return message } + static func streamError(in json: [String: Any]) -> AIProviderError? { + guard let error = json["error"], !(error is NSNull) else { return nil } + if let message = error as? String { + return .streamingFailed(message) + } + let fields = error as? [String: Any] + if let message = fields?["message"] as? String, !message.isEmpty { + return .streamingFailed(message) + } + if let status = (fields?["status"] ?? fields?["code"] ?? fields?["type"]) as? String, !status.isEmpty { + return .streamingFailed(status) + } + return .streamingFailed(String(localized: "The provider reported an error.")) + } + var isRetryable: Bool { switch self { case .invalidEndpoint, .authenticationFailed, .notFound: diff --git a/apple/TablePro/Core/AI/AnthropicProvider.swift b/apple/TablePro/Core/AI/AnthropicProvider.swift index 56971a036a..0019e43fda 100644 --- a/apple/TablePro/Core/AI/AnthropicProvider.swift +++ b/apple/TablePro/Core/AI/AnthropicProvider.swift @@ -307,9 +307,8 @@ final class AnthropicProvider: ChatTransport { } static func decodeStreamLine(_ line: String) -> [String: Any]? { - guard line.hasPrefix("data: ") else { return nil } - let jsonString = String(line.dropFirst(6)) - guard jsonString != "[DONE]", + guard let jsonString = SSEEventStream.dataPayload(line), + jsonString != "[DONE]", let data = jsonString.data(using: .utf8), let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return nil } diff --git a/apple/TablePro/Core/AI/Chat/ChatToolCallRepair.swift b/apple/TablePro/Core/AI/Chat/ChatToolCallRepair.swift new file mode 100644 index 0000000000..63302e7804 --- /dev/null +++ b/apple/TablePro/Core/AI/Chat/ChatToolCallRepair.swift @@ -0,0 +1,73 @@ +// +// ChatToolCallRepair.swift +// TablePro +// + +import Foundation + +// Providers reject a request in which a tool call has no result. A stopped round, or a transcript +// saved by a build that left one open, must still be sendable. +enum ChatToolCallRepair { + static let interruptedContent = "This tool call was interrupted before it returned a result." + static let stoppedBeforeRunContent = "The user stopped the response before this tool call ran." + + static func answeringEveryToolCall(_ turns: [ChatTurnWire]) -> [ChatTurnWire] { + var repaired: [ChatTurnWire] = [] + repaired.reserveCapacity(turns.count) + var owed: [ToolResultBlock] = [] + + for (index, original) in turns.enumerated() { + var turn = original + if !owed.isEmpty { + if turn.role == .user, !toolResultIds(in: turn).isEmpty { + turn.blocks.insert(contentsOf: owed.map { .toolResult($0) }, at: 0) + } else { + repaired.append(ChatTurnWire(role: .user, blocks: owed.map { .toolResult($0) })) + } + owed = [] + } + repaired.append(turn) + guard turn.role == .assistant else { continue } + owed = unansweredResults(for: turn, next: index + 1 < turns.count ? turns[index + 1] : nil) + } + + if !owed.isEmpty { + repaired.append(ChatTurnWire(role: .user, blocks: owed.map { .toolResult($0) })) + } + return repaired + } + + static func interruptedResults(for blocks: [ToolUseBlock]) -> [ToolResultBlock] { + blocks.map { block in + switch block.approvalState { + case .approved, .pending: + return ToolResultBlock(toolUseId: block.id, content: stoppedBeforeRunContent, isError: true) + case .cancelled: + return ToolResultBlock(toolUseId: block.id, content: "User cancelled this tool call.", isError: true) + case .denied(let reason): + return ToolResultBlock(toolUseId: block.id, content: reason, isError: true) + } + } + } + + private static func unansweredResults(for turn: ChatTurnWire, next: ChatTurnWire?) -> [ToolResultBlock] { + var answered = toolResultIds(in: turn) + if let next, next.role == .user { + answered.formUnion(toolResultIds(in: next)) + } + var owed: [ToolResultBlock] = [] + for block in turn.blocks { + guard case .toolUse(let call) = block.kind, !answered.contains(call.id) else { continue } + answered.insert(call.id) + owed.append(ToolResultBlock(toolUseId: call.id, content: interruptedContent, isError: true)) + } + return owed + } + + private static func toolResultIds(in turn: ChatTurnWire) -> Set { + Set(turn.blocks.compactMap { block -> String? in + guard case .toolResult(let result) = block.kind else { return nil } + return result.toolUseId + }) + } +} diff --git a/apple/TablePro/Core/AI/Chat/ChatTransport.swift b/apple/TablePro/Core/AI/Chat/ChatTransport.swift index 7c5ded77b3..5ef3b2230a 100644 --- a/apple/TablePro/Core/AI/Chat/ChatTransport.swift +++ b/apple/TablePro/Core/AI/Chat/ChatTransport.swift @@ -32,6 +32,9 @@ struct ChatTransportOptions: Sendable { /// was answered with the other's context, across connections included, while their local /// transcripts stayed correctly separate. var sessionId: UUID + // The connection the conversation is attached to. A transport that hands the model its own + // database tools scopes them to this one, and hands out none without it. + var connectionId: UUID? init( model: String, @@ -40,7 +43,8 @@ struct ChatTransportOptions: Sendable { temperature: Double? = nil, tools: [ChatToolSpec] = [], reasoningEffort: ReasoningEffort? = nil, - sessionId: UUID = UUID() + sessionId: UUID = UUID(), + connectionId: UUID? = nil ) { self.model = model self.systemPrompt = systemPrompt @@ -49,6 +53,7 @@ struct ChatTransportOptions: Sendable { self.tools = tools self.reasoningEffort = reasoningEffort self.sessionId = sessionId + self.connectionId = connectionId } } diff --git a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentMCPBridge.swift b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentMCPBridge.swift index 12d6ebba27..ee01dce6bc 100644 --- a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentMCPBridge.swift +++ b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentMCPBridge.swift @@ -13,7 +13,7 @@ struct ClaudeAgentToolGrant: Sendable, Equatable { } protocol ClaudeAgentMCPBridging: Sendable { - func makeGrant() async -> ClaudeAgentToolGrant? + func makeGrant(connectionId: UUID?) async -> ClaudeAgentToolGrant? func release(_ grant: ClaudeAgentToolGrant?) async } @@ -23,14 +23,21 @@ struct ClaudeAgentMCPBridge: ClaudeAgentMCPBridging { private static let tokenLifetime: TimeInterval = 15 * 60 private let permissions: TokenPermissions - private let connectionAccess: ConnectionAccess - init(permissions: TokenPermissions = .readOnly, connectionAccess: ConnectionAccess = .all) { + init(permissions: TokenPermissions = .readOnly) { self.permissions = permissions - self.connectionAccess = connectionAccess } - func makeGrant() async -> ClaudeAgentToolGrant? { + static func connectionAccess(for connectionId: UUID?) -> ConnectionAccess? { + guard let connectionId else { return nil } + return .limited([connectionId]) + } + + func makeGrant(connectionId: UUID?) async -> ClaudeAgentToolGrant? { + guard let connectionAccess = Self.connectionAccess(for: connectionId) else { + Self.logger.info("No connection attached; Claude Agent runs without database tools") + return nil + } guard let endpoint = await Self.resolveEndpoint() else { return nil } guard let tokenStore = await MCPServerManager.shared.tokenStore else { return nil } diff --git a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift index 1cd17a46a9..e21593ca8f 100644 --- a/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift +++ b/apple/TablePro/Core/AI/ClaudeAgent/ClaudeAgentProvider.swift @@ -31,7 +31,7 @@ final class ClaudeAgentProvider: ChatTransport { let task = Task { var grant: ClaudeAgentToolGrant? do { - grant = await toolBridge.makeGrant() + grant = await toolBridge.makeGrant(connectionId: options.connectionId) let prompt = AgentPromptRenderer.renderPrompt(turns: turns, includingSystemPrompt: nil) let lines = try cli.streamLines( arguments: Self.inferenceArguments( diff --git a/apple/TablePro/Core/AI/GeminiProvider.swift b/apple/TablePro/Core/AI/GeminiProvider.swift index 849332fcf5..b59e1a7663 100644 --- a/apple/TablePro/Core/AI/GeminiProvider.swift +++ b/apple/TablePro/Core/AI/GeminiProvider.swift @@ -47,7 +47,10 @@ final class GeminiProvider: ChatTransport { buildRequest: { [self] in try buildStreamRequest(turns: turns, options: options) }, decodeLine: { Self.decodeStreamLine($0) }, makeState: { GeminiStreamState() }, - parse: { Self.parseChunk($0, state: &$1, idGenerator: { UUID().uuidString }) }, + parse: { json, state in + if let error = AIProviderError.streamError(in: json) { throw error } + return Self.parseChunk(json, state: &state, idGenerator: { UUID().uuidString }) + }, finalEvents: { state in state.finalUsageEvent().map { [$0] } ?? [] } ) } @@ -275,9 +278,8 @@ final class GeminiProvider: ChatTransport { /// Decodes one Gemini SSE line. Returns nil for non-data lines. static func decodeStreamLine(_ line: String) -> [String: Any]? { - guard line.hasPrefix("data: ") else { return nil } - let jsonString = String(line.dropFirst(6)) - guard let data = jsonString.data(using: .utf8), + guard let jsonString = SSEEventStream.dataPayload(line), + let data = jsonString.data(using: .utf8), let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return nil } return json diff --git a/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift b/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift index abdcfa61c6..87cba934b7 100644 --- a/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift +++ b/apple/TablePro/Core/AI/OpenAICompatibleProvider.swift @@ -89,23 +89,36 @@ final class OpenAICompatibleProvider: ChatTransport { buildRequest: { [self] in try buildChatCompletionRequest(turns: turns, options: options) }, decodeLine: { Self.decodeStreamLine($0, providerType: providerType) }, makeState: { OpenAIStreamState() }, - parse: { Self.parseChunk($0, state: &$1).events }, - finalEvents: { state in - var state = state - return [state.flushReasoningEnd(), state.finalUsageEvent()].compactMap { $0 } - } + parse: { try Self.streamEvents(from: $0, state: &$1) }, + finalEvents: { try Self.finalStreamEvents($0) } ) } + static func streamEvents( + from json: [String: Any], + state: inout OpenAIStreamState + ) throws -> [ChatStreamEvent] { + if let error = AIProviderError.streamError(in: json) { throw error } + return parseChunk(json, state: &state).events + } + + static func finalStreamEvents(_ state: OpenAIStreamState) throws -> [ChatStreamEvent] { + guard state.receivedOutput else { + throw AIProviderError.streamingFailed( + String(localized: "The provider ended the response without sending any content.") + ) + } + var state = state + return [state.flushReasoningEnd(), state.finalUsageEvent()].compactMap { $0 } + } + static func decodeStreamLine(_ line: String, providerType: AIProviderType) -> [String: Any]? { let jsonString: String if providerType == .ollama { guard !line.isEmpty else { return nil } jsonString = line } else { - guard line.hasPrefix("data: ") else { return nil } - let payload = String(line.dropFirst(6)) - guard payload != "[DONE]" else { return nil } + guard let payload = SSEEventStream.dataPayload(line), payload != "[DONE]" else { return nil } jsonString = payload } guard let data = jsonString.data(using: .utf8), @@ -176,6 +189,10 @@ final class OpenAICompatibleProvider: ChatTransport { if shouldBreak, !state.toolCallIndexToId.isEmpty { events.append(contentsOf: state.flushToolUseEnds()) } + let finished = (firstChoice?["finish_reason"] as? String)?.isEmpty == false + if !events.isEmpty || finished || shouldBreak { + state.receivedOutput = true + } return (events, shouldBreak) } @@ -634,6 +651,7 @@ struct OpenAIStreamState { var toolCallIndexToId: [Int: String] = [:] var toolCallOrder: [Int] = [] var reasoningBlockID: String? + var receivedOutput = false mutating func flushReasoningEnd() -> ChatStreamEvent? { guard let id = reasoningBlockID else { return nil } diff --git a/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift b/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift index 08cf8c9cca..e246479c13 100644 --- a/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift +++ b/apple/TablePro/Core/AI/OpenAIResponsesProvider.swift @@ -294,9 +294,8 @@ final class OpenAIResponsesProvider: ChatTransport { } static func decodeStreamLine(_ line: String) -> [String: Any]? { - guard line.hasPrefix("data: ") else { return nil } - let payload = String(line.dropFirst(6)) - guard payload != "[DONE]", + guard let payload = SSEEventStream.dataPayload(line), + payload != "[DONE]", let data = payload.data(using: .utf8), let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return nil } diff --git a/apple/TablePro/Core/AI/SSEEventStream.swift b/apple/TablePro/Core/AI/SSEEventStream.swift index 1770c6faa5..40ddde4444 100644 --- a/apple/TablePro/Core/AI/SSEEventStream.swift +++ b/apple/TablePro/Core/AI/SSEEventStream.swift @@ -7,6 +7,13 @@ import Foundation import TableProHTTP enum SSEEventStream { + // WHATWG event stream: the field ends at the colon, and one space after it is optional. + static func dataPayload(_ line: String) -> String? { + guard line.hasPrefix("data:") else { return nil } + let value = line.dropFirst(5) + return String(value.first == " " ? value.dropFirst() : value) + } + static func make( session: URLSession, treatForbiddenAsAuthFailure: Bool = false, @@ -14,7 +21,7 @@ enum SSEEventStream { decodeLine: @escaping @Sendable (String) -> [String: Any]?, makeState: @escaping @Sendable () -> State, parse: @escaping @Sendable ([String: Any], inout State) throws -> [ChatStreamEvent], - finalEvents: @escaping @Sendable (State) -> [ChatStreamEvent] = { _ in [] }, + finalEvents: @escaping @Sendable (State) throws -> [ChatStreamEvent] = { _ in [] }, refreshOnUnauthorized: (@Sendable () async throws -> Void)? = nil ) -> AsyncThrowingStream { AsyncThrowingStream { continuation in @@ -48,7 +55,7 @@ enum SSEEventStream { let events = try parse(json, &state) for event in events { continuation.yield(event) } } - for final in finalEvents(state) { + for final in try finalEvents(state) { continuation.yield(final) } continuation.finish() diff --git a/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift b/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift index bb5169b889..55cb981c82 100644 --- a/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift +++ b/apple/TablePro/Core/ChangeTracking/DataChangeManager.swift @@ -45,7 +45,8 @@ struct UndoResult { final class DataChangeManager: ObservableObject, ChangeManaging { nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "DataChangeManager") - @Published private(set) var pending = PendingChanges() + // Not @Published: the wrapper has no in-place access, so each mutation copied every pending change. + private(set) var pending = PendingChanges() @Published var hasChanges: Bool = false @Published var reloadVersion: Int = 0 @@ -136,6 +137,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { func clearChanges() { discardCoalescedUndoRun() + objectWillChange.send() pending.clear() hasChanges = false reloadVersion += 1 @@ -168,6 +170,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { self.columnTypeNames = columnTypeNames discardUndoHistory() + objectWillChange.send() pending.clear() hasChanges = false @@ -320,6 +323,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { wasExpression: pending.isExpression(rowID: rowID, columnIndex: columnIndex), isExpression: isExpression ) + objectWillChange.send() let recorded = pending.recordCellChange( rowID: rowID, columnIndex: columnIndex, @@ -419,6 +423,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { for (rowID, cells) in values { loadedValues[rowID, default: [:]].merge(cells) { _, loaded in loaded } } + objectWillChange.send() pending.adoptLoadedValues(values) } @@ -428,10 +433,13 @@ final class DataChangeManager: ObservableObject, ChangeManaging { } func recordRowDeletion(rowID: RowID, originalRow: [PluginCellValue], absentColumns: Set = []) { - let originalRow = completedImage(originalRow, of: rowID) - pending.recordRowDeletion(rowID: rowID, originalRow: originalRow, absentColumns: absentColumns) + let image = pending.imageAsRead(ofRow: rowID, current: originalRow, absentColumns: absentColumns) + let rowAsRead = completedImage(image.row, of: rowID) + let absentAsRead = image.absentColumns + objectWillChange.send() + pending.recordRowDeletion(rowID: rowID, originalRow: rowAsRead, absentColumns: absentAsRead) registerUndo(actionName: String(localized: "Delete Row")) { target in - target.applyDataUndo(.rowDeletion(rowID: rowID, originalRow: originalRow, absentColumns: absentColumns)) + target.applyDataUndo(.rowDeletion(rowID: rowID, originalRow: rowAsRead, absentColumns: absentAsRead)) } hasChanges = true } @@ -448,13 +456,26 @@ final class DataChangeManager: ObservableObject, ChangeManaging { } return } - let rows = rows.map { (rowID: $0.rowID, originalRow: completedImage($0.originalRow, of: $0.rowID)) } - for (rowID, originalRow) in rows { - pending.recordRowDeletion(rowID: rowID, originalRow: originalRow, absentColumns: absentColumns[rowID] ?? []) + var batchData: [(rowID: RowID, originalRow: [PluginCellValue])] = [] + var absentAsRead: [RowID: Set] = [:] + batchData.reserveCapacity(rows.count) + for row in rows { + let image = pending.imageAsRead( + ofRow: row.rowID, current: row.originalRow, absentColumns: absentColumns[row.rowID] ?? [] + ) + batchData.append((rowID: row.rowID, originalRow: completedImage(image.row, of: row.rowID))) + if !image.absentColumns.isEmpty { absentAsRead[row.rowID] = image.absentColumns } + } + objectWillChange.send() + for (rowID, originalRow) in batchData { + pending.recordRowDeletion( + rowID: rowID, originalRow: originalRow, absentColumns: absentAsRead[rowID] ?? [] + ) } - let batchData = rows + let deleted = batchData + let deletedAbsentColumns = absentAsRead registerUndo(actionName: String(localized: "Delete Rows")) { target in - target.applyDataUndo(.batchRowDeletion(rows: batchData, absentColumns: absentColumns)) + target.applyDataUndo(.batchRowDeletion(rows: deleted, absentColumns: deletedAbsentColumns)) } hasChanges = true } @@ -466,6 +487,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { absentColumns: Set = [], expressionColumns: Set = [] ) { + objectWillChange.send() pending.recordRowInsertion( rowID: rowID, values: values, absentColumns: absentColumns, expressionColumns: expressionColumns ) @@ -475,9 +497,45 @@ final class DataChangeManager: ObservableObject, ChangeManaging { hasChanges = true } + struct InsertedRow { + let location: InsertedRowLocation + let values: [PluginCellValue] + var absentColumns: Set = [] + var expressionColumns: Set = [] + } + + // One publish and one undo step for the whole batch, so a paste of many rows stays linear. + func recordBatchRowInsertion(rows: [InsertedRow]) { + guard rows.count > 1 else { + if let row = rows.first { + recordRowInsertion( + rowID: row.location.rowID, values: row.values, + absentColumns: row.absentColumns, expressionColumns: row.expressionColumns + ) + } + return + } + objectWillChange.send() + for row in rows { + pending.recordRowInsertion( + rowID: row.location.rowID, values: row.values, + absentColumns: row.absentColumns, expressionColumns: row.expressionColumns + ) + } + registerUndo(actionName: String(localized: "Insert Rows")) { target in + target.applyDataUndo(.batchRowInsertion( + rows: rows.map(\.location), rowValues: rows.map(\.values), + rowAbsentColumns: rows.map(\.absentColumns), rowExpressionColumns: rows.map(\.expressionColumns) + )) + } + hasChanges = true + } + // MARK: - Undo Operations func undoRowDeletion(rowID: RowID) { + guard pending.isRowDeleted(rowID) else { return } + objectWillChange.send() guard pending.undoRowDeletion(rowID: rowID) else { return } hasChanges = !pending.isEmpty } @@ -487,6 +545,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { guard !validRows.isEmpty else { return } let rowAbsentColumns = validRows.map { pending.insertedAbsentColumns(forRow: $0.rowID) } let rowExpressionColumns = validRows.map { pending.insertedExpressionColumns(forRow: $0.rowID) } + objectWillChange.send() let rowValues = pending.undoBatchRowInsertion( rowIDs: validRows.map(\.rowID), columnCount: columns.count ) @@ -502,6 +561,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { // MARK: - Core Undo Application private func applyDataUndo(_ action: UndoAction) { + objectWillChange.send() switch action { case .cellEdit(let rowID, let columnIndex, let columnName, let previousValue, let newValue, let originalRow, let absence, let expression): @@ -547,15 +607,15 @@ final class DataChangeManager: ObservableObject, ChangeManaging { )) } - if let updateChange = pending.change(forRow: rowID, type: .update) { - if updateChange.cellChanges.contains(where: { $0.columnIndex == columnIndex }) { - pending.revertUpdateCell( - rowID: rowID, columnIndex: columnIndex, - columnName: columnName, previousValue: previousValue, - previousIsAbsent: absence.wasAbsent, - previousIsExpression: expression.wasExpression - ) - } + let updateHoldsCell = pending.change(forRow: rowID, type: .update)? + .cellChanges.contains { $0.columnIndex == columnIndex } ?? false + if updateHoldsCell { + pending.revertUpdateCell( + rowID: rowID, columnIndex: columnIndex, + columnName: columnName, previousValue: previousValue, + previousIsAbsent: absence.wasAbsent, + previousIsExpression: expression.wasExpression + ) } else if pending.change(forRow: rowID, type: .insert) != nil { pending.updateInsertedCellDirectly( rowID: rowID, columnIndex: columnIndex, @@ -808,7 +868,8 @@ final class DataChangeManager: ObservableObject, ChangeManaging { func getOriginalValues() -> [(rowID: RowID, columnIndex: Int, value: PluginCellValue, isAbsent: Bool)] { var originals: [(rowID: RowID, columnIndex: Int, value: PluginCellValue, isAbsent: Bool)] = [] - for change in pending.changes where change.type == .update { + let updates = pending.changes.filter { $0.type == .update } + pending.supersededUpdates.values + for change in updates { for cellChange in change.cellChanges { originals.append(( rowID: change.rowID, @@ -822,6 +883,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { } func discardChanges() { + objectWillChange.send() pending.clear() hasChanges = false reloadVersion += 1 @@ -851,6 +913,7 @@ final class DataChangeManager: ObservableObject, ChangeManaging { self.rowMatchPolicy = rowMatchPolicy self.columnTypeNames = columnTypeNames discardUndoHistory() + objectWillChange.send() pending.restore(from: state) self.hasChanges = !pending.isEmpty } diff --git a/apple/TablePro/Core/ChangeTracking/PendingChanges.swift b/apple/TablePro/Core/ChangeTracking/PendingChanges.swift index 24cc158aac..aeb641a715 100644 --- a/apple/TablePro/Core/ChangeTracking/PendingChanges.swift +++ b/apple/TablePro/Core/ChangeTracking/PendingChanges.swift @@ -24,6 +24,9 @@ struct PendingChanges: Equatable { private(set) var modifiedCells: [RowID: Set] = [:] private(set) var insertedRowData: [RowID: [PluginCellValue]] = [:] + // A deleted row's edits, parked so the delete keys on the row as read and undoing it brings them back. + private(set) var supersededUpdates: [RowID: RowChange] = [:] + private var changeIndex: [RowChangeKey: Int] = [:] var isEmpty: Bool { changes.isEmpty } @@ -51,6 +54,26 @@ struct PendingChanges: Equatable { RowChange.adoptLoadedValues(values, into: &changes) } + // `current` is the row as the grid holds it, pending edits included. + func imageAsRead( + ofRow rowID: RowID, + current: [PluginCellValue], + absentColumns: Set + ) -> (row: [PluginCellValue], absentColumns: Set) { + guard let update = change(forRow: rowID, type: .update) else { return (current, absentColumns) } + var row = current + var absent = absentColumns + for cellChange in update.cellChanges where row.indices.contains(cellChange.columnIndex) { + row[cellChange.columnIndex] = cellChange.oldValue + if cellChange.oldIsAbsent { + absent.insert(cellChange.columnIndex) + } else { + absent.remove(cellChange.columnIndex) + } + } + return (row, absent) + } + func change(forRow rowID: RowID, type: ChangeType) -> RowChange? { guard let idx = changeIndex[RowChangeKey(rowID: rowID, type: type)] else { return nil } return changes[idx] @@ -118,17 +141,14 @@ struct PendingChanges: Equatable { return true } - let updateKey = RowChangeKey(rowID: rowID, type: .update) - if let updateIdx = changeIndex[updateKey] { + if let updateIdx = changeIndex[RowChangeKey(rowID: rowID, type: .update)] { mergeUpdateCell(at: updateIdx, cellChange: cellChange) } else { - let row = RowChange( + appendChange(RowChange( rowID: rowID, type: .update, cellChanges: [cellChange], originalRow: originalRow, absentColumns: absence.originalRow - ) - changes.append(row) - changeIndex[updateKey] = changes.count - 1 + )) modifiedCells[rowID, default: []].insert(columnIndex) } return true @@ -136,9 +156,13 @@ struct PendingChanges: Equatable { mutating func recordRowDeletion(rowID: RowID, originalRow: [PluginCellValue], absentColumns: Set = []) { guard !deletedRowIDs.contains(rowID) else { return } - removeChange(rowID: rowID, type: .update) + let image = imageAsRead(ofRow: rowID, current: originalRow, absentColumns: absentColumns) + if let updateIdx = changeIndex[RowChangeKey(rowID: rowID, type: .update)] { + supersededUpdates[rowID] = changes[updateIdx] + removeChangeAt(updateIdx) + } modifiedCells.removeValue(forKey: rowID) - appendChange(RowChange(rowID: rowID, type: .delete, originalRow: originalRow, absentColumns: absentColumns)) + appendChange(RowChange(rowID: rowID, type: .delete, originalRow: image.row, absentColumns: image.absentColumns)) deletedRowIDs.insert(rowID) } @@ -170,6 +194,13 @@ struct PendingChanges: Equatable { guard deletedRowIDs.contains(rowID) else { return false } removeChange(rowID: rowID, type: .delete) deletedRowIDs.remove(rowID) + let updateKey = RowChangeKey(rowID: rowID, type: .update) + if let update = supersededUpdates.removeValue(forKey: rowID), changeIndex[updateKey] == nil { + // Appended as it was, so it keeps the place in the save order its first edit had. + changes.append(update) + changeIndex[updateKey] = changes.count - 1 + modifiedCells[rowID] = Set(update.cellChanges.map(\.columnIndex)) + } return true } @@ -237,17 +268,14 @@ struct PendingChanges: Equatable { return } - let updateKey = RowChangeKey(rowID: rowID, type: .update) - if let updateIdx = changeIndex[updateKey] { + if let updateIdx = changeIndex[RowChangeKey(rowID: rowID, type: .update)] { mergeUpdateCell(at: updateIdx, cellChange: cellChange) } else { - let row = RowChange( + appendChange(RowChange( rowID: rowID, type: .update, cellChanges: [cellChange], originalRow: originalRow, absentColumns: absence.originalRow - ) - changes.append(row) - changeIndex[updateKey] = changes.count - 1 + )) modifiedCells[rowID, default: []].insert(columnIndex) } } @@ -382,6 +410,7 @@ struct PendingChanges: Equatable { insertedRowIDs.removeAll() modifiedCells.removeAll() insertedRowData.removeAll() + supersededUpdates.removeAll() } mutating func restore(from snapshot: TabChangeSnapshot) { @@ -390,7 +419,9 @@ struct PendingChanges: Equatable { insertedRowIDs = snapshot.insertedRowIDs modifiedCells = snapshot.modifiedCells insertedRowData = snapshot.insertedRowData - nextSequence = (changes.map(\.sequence).max() ?? -1) + 1 + supersededUpdates = snapshot.supersededUpdates + let sequences = changes.map(\.sequence) + supersededUpdates.values.map(\.sequence) + nextSequence = (sequences.max() ?? -1) + 1 rebuildChangeIndex() } @@ -401,6 +432,7 @@ struct PendingChanges: Equatable { snap.insertedRowIDs = insertedRowIDs snap.modifiedCells = modifiedCells snap.insertedRowData = insertedRowData + snap.supersededUpdates = supersededUpdates snap.primaryKeyColumns = primaryKeyColumns snap.columns = columns return snap diff --git a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift index ce4d9ee853..162b793ee8 100644 --- a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift +++ b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift @@ -49,6 +49,8 @@ struct SQLStatementGenerator { /// Set only by a restore that writes a row back with the identity it had: PostgreSQL refuses an explicit value /// for a `GENERATED ALWAYS` column unless the INSERT says so. let insertOverridesSystemValue: Bool + // BIT columns of a family that reads them as a number. Text bound to one is stored and compared as its bytes. + private let numberBitColumns: Set private let quoteIdentifierFn: (String) -> String private let rowUpdateFn: ((_ table: String, _ assignments: String, _ predicate: String) -> String?)? private let rowDeleteFn: ((_ table: String, _ predicate: String?) -> String?)? @@ -61,6 +63,7 @@ struct SQLStatementGenerator { databaseType: DatabaseType, generatedColumns: Set = [], rowMatchPolicy: RowMatchPolicy = .none, + columnTypeNames: [String: String] = [:], parameterStyle: ParameterStyle? = nil, insertOverridesSystemValue: Bool = false, dialect: SQLDialectDescriptor? = nil, @@ -79,6 +82,7 @@ struct SQLStatementGenerator { self.databaseType = databaseType self.parameterStyle = parameterStyle ?? Self.defaultParameterStyle(for: databaseType) self.insertOverridesSystemValue = insertOverridesSystemValue + self.numberBitColumns = Self.numberBitColumns(in: columnTypeNames, databaseType: databaseType) if let quoteIdentifier { self.quoteIdentifierFn = quoteIdentifier } else { @@ -182,6 +186,39 @@ struct SQLStatementGenerator { } } + private static func numberBitColumns(in typeNames: [String: String], databaseType: DatabaseType) -> Set { + let family = SQLTypeFamily.of(databaseType) + guard family.readsBitsAsDecimal else { return [] } + let bitColumns = typeNames.filter { _, typeName in + if case .bitString = SQLTypeParser.parse(typeName, family: family).kind { return true } + return false + } + return Set(bitColumns.keys) + } + + private func bind(_ value: PluginCellValue, toColumn column: String?, into parameters: inout [Any?]) -> String { + guard let column, numberBitColumns.contains(column), case .text(let text) = value else { + parameters.append(value.asAny) + return placeholder(at: parameters.count - 1) + } + parameters.append(Self.decimalBitText(text)) + return "CAST(\(placeholder(at: parameters.count - 1)) AS UNSIGNED)" + } + + // b'101' and 0b101 are SQL, not values: bound as text they would cast to 0. + static func decimalBitText(_ text: String) -> String { + let trimmed = text.trimmingCharacters(in: .whitespaces) + let digits: Substring + if trimmed.count > 3, trimmed.hasPrefix("b'") || trimmed.hasPrefix("B'"), trimmed.hasSuffix("'") { + digits = trimmed.dropFirst(2).dropLast() + } else if trimmed.hasPrefix("0b") { + digits = trimmed.dropFirst(2) + } else { + return text + } + return UInt64(digits, radix: 2).map { String($0) } ?? text + } + // MARK: - INSERT Generation private func generateInsertSQL(for change: RowChange, insertedRowData: [RowID: [PluginCellValue]]) @@ -215,8 +252,7 @@ struct SQLStatementGenerator { case .text(let expression) where expressionColumns.contains(index): placeholderParts.append(expression) default: - bindParameters.append(value.asAny) - placeholderParts.append(placeholder(at: bindParameters.count - 1)) + placeholderParts.append(bind(value, toColumn: columnName, into: &bindParameters)) } } @@ -246,9 +282,8 @@ struct SQLStatementGenerator { var bindParameters: [Any?] = [] let columnList = insertColumns.map(quoteIdentifierFn).joined(separator: ", ") - let placeholders = values.map { value -> String in - bindParameters.append(value.asAny) - return placeholder(at: bindParameters.count - 1) + let placeholders = zip(insertColumns, values).map { column, value in + bind(value, toColumn: column, into: &bindParameters) }.joined(separator: ", ") let sql = @@ -266,9 +301,8 @@ struct SQLStatementGenerator { var bindParameters: [Any?] = [] let columnList = insertColumns.map(quoteIdentifierFn).joined(separator: ", ") let rowTuples = rows.map { values -> String in - let placeholders = values.map { value -> String in - bindParameters.append(value.asAny) - return placeholder(at: bindParameters.count - 1) + let placeholders = zip(insertColumns, values).map { column, value in + bind(value, toColumn: column, into: &bindParameters) }.joined(separator: ", ") return "(\(placeholders))" }.joined(separator: ", ") @@ -284,11 +318,9 @@ struct SQLStatementGenerator { guard !keyColumns.isEmpty, !keys.isEmpty, keys.allSatisfy({ $0.count == keyColumns.count }) else { return nil } var bindParameters: [Any?] = [] - let quotedKey = keyColumns.map(quoteIdentifierFn) let selects = keys.enumerated().map { position, values -> String in - let conditions = zip(quotedKey, values).map { column, value -> String in - bindParameters.append(value.asAny) - return "\(column) = \(placeholder(at: bindParameters.count - 1))" + let conditions = zip(keyColumns, values).map { column, value -> String in + "\(quoteIdentifierFn(column)) = \(bind(value, toColumn: column, into: &bindParameters))" }.joined(separator: " AND ") return "SELECT \(position), COUNT(*) FROM \(qualifiedTableName) WHERE \(conditions)" } @@ -307,8 +339,7 @@ struct SQLStatementGenerator { var bindParameters: [Any?] = [] func bound(_ column: String, _ value: PluginCellValue) -> String { - bindParameters.append(value.asAny) - return "\(quoteIdentifierFn(column)) = \(placeholder(at: bindParameters.count - 1))" + "\(quoteIdentifierFn(column)) = \(bind(value, toColumn: column, into: &bindParameters))" } let assignments = zip(setColumns, values).map(bound).joined(separator: ", ") let conditions = zip(keyColumns, keyValues).map(bound).joined(separator: " AND ") @@ -350,8 +381,7 @@ struct SQLStatementGenerator { case .text(let expression) where cellChange.newIsExpression: return expression default: - parameters.append(cellChange.newValue.asAny) - return placeholder(at: parameters.count - 1) + return bind(cellChange.newValue, toColumn: cellChange.columnName, into: ¶meters) } }.joined(separator: ", ") @@ -377,8 +407,8 @@ struct SQLStatementGenerator { case .text(let expression) where cellChange.newIsExpression: return "\(quoteIdentifierFn(cellChange.columnName)) = \(expression)" default: - parameters.append(cellChange.newValue.asAny) - return "\(quoteIdentifierFn(cellChange.columnName)) = \(placeholder(at: parameters.count - 1))" + let bound = bind(cellChange.newValue, toColumn: cellChange.columnName, into: ¶meters) + return "\(quoteIdentifierFn(cellChange.columnName)) = \(bound)" } }.joined(separator: ", ") @@ -402,7 +432,7 @@ struct SQLStatementGenerator { return nil } - terms.append(RowMatchTerm(matched: quoteIdentifierFn(pkColumn), value: pkValue)) + terms.append(RowMatchTerm(matched: quoteIdentifierFn(pkColumn), value: pkValue, column: pkColumn)) } guard !terms.isEmpty else { return nil } @@ -428,7 +458,11 @@ struct SQLStatementGenerator { for: columnName, quoted: quoteIdentifierFn(columnName), value: value, databaseType: databaseType ) // Spelled out: a bare `nil` here is PluginCellValue's own nil literal, `.null`, which binds rather than reads IS NULL. - terms.append(RowMatchTerm(matched: matched, value: value.isNull ? Optional.none : value)) + terms.append(RowMatchTerm( + matched: matched, + value: value.isNull ? Optional.none : value, + column: plainlyMatchedColumn(columnName) + )) } guard !terms.isEmpty else { return nil } @@ -444,6 +478,12 @@ struct SQLStatementGenerator { private struct RowMatchTerm { let matched: String let value: PluginCellValue? + // Nil when `matched` is not the bare column, so its value is bound as it is. + let column: String? + } + + private func plainlyMatchedColumn(_ column: String) -> String? { + rowMatchPolicy.textColumns.contains(column) ? nil : column } /// The conditions that pick one row. SQLite keeps the number 5 and the text '5' apart in a column with no @@ -460,9 +500,11 @@ struct SQLStatementGenerator { private func conditions(_ terms: [RowMatchTerm], parameters: inout [Any?]) -> String { terms.map { term in guard let value = term.value else { return "\(term.matched) IS NULL" } + guard matchesEitherWay(value), case .text(let text) = value else { + return "\(term.matched) = \(bind(value, toColumn: term.column, into: ¶meters))" + } parameters.append(value.asAny) let bound = placeholder(at: parameters.count - 1) - guard matchesEitherWay(value), case .text(let text) = value else { return "\(term.matched) = \(bound)" } parameters.append(value.asAny) let read = placeholder(at: parameters.count - 1) return "(\(term.matched) IN (\(text), \(bound)) AND CAST(\(term.matched) AS TEXT) = \(read))" @@ -578,7 +620,8 @@ struct SQLStatementGenerator { value: match.boundValue ?? .null, databaseType: databaseType ), - value: match.boundValue + value: match.boundValue, + column: plainlyMatchedColumn(match.column) ) } let joined = rowCondition(terms, parameters: ¶meters) diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift index 71a653a965..f451693062 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Batches.swift @@ -100,9 +100,10 @@ extension QueryExecutionCoordinator { func executeBatches( _ batches: [ExecutableBatch], parameters: [QueryParameter], - bypassRowLimit: Bool = false + bypassRowLimit: Bool = false, + onTab target: UUID? = nil ) { - guard let (selectedTab, index) = parent.tabManager.selectedTabAndIndex, + guard let (selectedTab, index) = parent.executionTab(target), !parent.tabExecution.isExecuting(selectedTab.id) else { return } if let firstMissing = parameters.first(where: { @@ -197,9 +198,11 @@ extension QueryExecutionCoordinator { let startLines = BatchErrorText.lines(of: batches.map(\.range.location), in: query) return zip(batches, startLines).map { batch, startLine in let bindsParameters = !parameters.isEmpty && batch.acceptsBindParameters - && !SQLParameterExtractor.extractParameters(from: batch.sql).isEmpty + && !SQLParameterExtractor.extractParameters(from: batch.sql, grammar: grammar).isEmpty let conversion = bindsParameters - ? SQLParameterExtractor.convertToNativeStyle(sql: batch.sql, parameters: parameters, style: style) + ? SQLParameterExtractor.convertToNativeStyle( + sql: batch.sql, parameters: parameters, style: style, grammar: grammar + ) : nil let isCommitPoint = batch.statements.count == 1 && batch.statements.allSatisfy { BatchCommitStatement.matches($0.sql, grammar: grammar) } @@ -455,6 +458,7 @@ extension QueryExecutionCoordinator { } } parent.seedBufferFromActiveResult(tabId: tabId) + parent.dropParkedEdits(ofBackgroundTab: tabId) if parent.tabManager.selectedTabId == tabId { parent.toolbarState.isResultsCollapsed = false parent.toolbarState.recordQueryTiming(timing, for: tabId) diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift index 81630c60f9..1b775282a5 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Helpers.swift @@ -174,6 +174,15 @@ extension QueryExecutionCoordinator { return definitionChanged ? [] : previous } + private func storeKeys(_ keys: [String], viewRowEditing: PluginViewRowEditing?, sameRelation: Bool, at index: Int) { + parent.tabManager.mutate(at: index) { tab in + tab.tableContext.primaryKeyColumns = keys + // Cached and inline metadata carry no answer, so the last one stands unless the relation changed. + let kept = sameRelation ? tab.tableContext.viewRowEditing : nil + tab.tableContext.adoptViewAnswer(viewRowEditing ?? kept, tabType: tab.tabType) + } + } + func applyPhase1Result( // swiftlint:disable:this function_parameter_count tabId: UUID, columns: [String], @@ -334,12 +343,7 @@ extension QueryExecutionCoordinator { definitionChanged: definitionChanged ) - parent.tabManager.mutate(at: idx) { tab in - tab.tableContext.primaryKeyColumns = resolvedPKs - // Cached and inline metadata carry no answer, so the last one stands unless the relation changed. - let kept = tableName == previousTableName ? tab.tableContext.viewRowEditing : nil - tab.tableContext.adoptViewAnswer(metadata?.viewRowEditing ?? kept, tabType: tab.tabType) - } + storeKeys(resolvedPKs, viewRowEditing: metadata?.viewRowEditing, sameRelation: tableName == previousTableName, at: idx) captureOrigin( tabIndex: idx, tabId: tabId, @@ -359,7 +363,9 @@ extension QueryExecutionCoordinator { rowMatchPolicy: resolved.rowMatchPolicy, columnTypeNames: newTableRows.columnTypeNames ) + parent.changeManagerMatchesSelectedResult() } + parent.dropParkedEdits(ofBackgroundTab: tabId) recordHistory( QueryHistoryRecordRequest( @@ -436,6 +442,7 @@ extension QueryExecutionCoordinator { } } parent.seedBufferFromActiveResult(tabId: tabId) + parent.dropParkedEdits(ofBackgroundTab: tabId) parent.toolbarState.isResultsCollapsed = false recordHistory( diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift index 7c725a4621..e90dd0ded3 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift @@ -10,9 +10,12 @@ import TableProSQLGrammar extension QueryExecutionCoordinator { func executeMultipleStatements( _ statements: [SQLStatementScanner.ExecutableStatement], - bypassRowLimit: Bool = false + bypassRowLimit: Bool = false, + onTab target: UUID? = nil ) { - executeMultipleStatementsWithParameters(statements, parameters: [], bypassRowLimit: bypassRowLimit) + executeMultipleStatementsWithParameters( + statements, parameters: [], bypassRowLimit: bypassRowLimit, onTab: target + ) } // The rest of a run, cut again once a statement changed how the session reads literals, in tab coordinates. @@ -220,6 +223,9 @@ extension QueryExecutionCoordinator { if parent.tabManager.selectedTabId == tabId { parent.changeManager.clearChangesAndUndoHistory() + parent.changeManagerMatchesSelectedResult() + } else { + parent.dropParkedEdits(ofBackgroundTab: tabId) } } } diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift index 106b20e0d8..ac3f5b8ec1 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift @@ -100,7 +100,7 @@ private struct ExecutedStatementHistory { extension QueryExecutionCoordinator { func detectAndReconcileParameters(sql: String, existing: [QueryParameter]) -> [QueryParameter] { - QueryExecutor.detectAndReconcileParameters(sql: sql, existing: existing) + QueryExecutor.detectAndReconcileParameters(sql: sql, existing: existing, grammar: parent.lexicalGrammar) } func executeQueryWithParameters( @@ -108,9 +108,10 @@ extension QueryExecutionCoordinator { parameters: [QueryParameter], bypassRowLimit: Bool = false, anchor: StatementAnchor? = nil, - install: ResultInstall = .newResult + install: ResultInstall = .newResult, + onTab target: UUID? = nil ) { - guard let (_, index) = parent.tabManager.selectedTabAndIndex else { return } + guard let (_, index) = parent.executionTab(target) else { return } let missing = parameters.filter { !$0.isNull && $0.value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty @@ -131,7 +132,8 @@ extension QueryExecutionCoordinator { let conversion = SQLParameterExtractor.convertToNativeStyle( sql: sql, parameters: parameters, - style: style + style: style, + grammar: parent.lexicalGrammar ) paramLog.info("Executing parameterized query: \(conversion.sql.prefix(100), privacy: .private) with \(conversion.values.count) parameters") @@ -143,7 +145,8 @@ extension QueryExecutionCoordinator { bypassRowLimit: bypassRowLimit, originalSQL: sql, anchor: anchor, - install: install + install: install, + onTab: target ) } @@ -156,9 +159,10 @@ extension QueryExecutionCoordinator { bypassRowLimit: Bool = false, originalSQL: String? = nil, anchor: StatementAnchor? = nil, - install: ResultInstall = .newResult + install: ResultInstall = .newResult, + onTab target: UUID? = nil ) { - guard let (selectedTab, index) = parent.tabManager.selectedTabAndIndex, + guard let (selectedTab, index) = parent.executionTab(target), !parent.tabExecution.isExecuting(selectedTab.id) else { return } guard let scope = parent.scope(for: selectedTab) else { @@ -311,9 +315,10 @@ extension QueryExecutionCoordinator { func executeMultipleStatementsWithParameters( _ statements: [SQLStatementScanner.ExecutableStatement], parameters: [QueryParameter], - bypassRowLimit: Bool = false + bypassRowLimit: Bool = false, + onTab target: UUID? = nil ) { - guard let (selectedTab, index) = parent.tabManager.selectedTabAndIndex, + guard let (selectedTab, index) = parent.executionTab(target), !parent.tabExecution.isExecuting(selectedTab.id) else { return } let missing = parameters.filter { @@ -481,10 +486,10 @@ extension QueryExecutionCoordinator { let sql = statement.sql let parameterNames = parameters.isEmpty || !statement.acceptsBindParameters ? [] - : SQLParameterExtractor.extractParameters(from: sql) + : SQLParameterExtractor.extractParameters(from: sql, grammar: grammar) let conversion = parameterNames.isEmpty ? nil - : SQLParameterExtractor.convertToNativeStyle(sql: sql, parameters: parameters, style: style) + : SQLParameterExtractor.convertToNativeStyle(sql: sql, parameters: parameters, style: style, grammar: grammar) let executableSQL = conversion?.sql ?? sql let bounded = resolveStatement(sql: executableSQL, tabType: tabType, bypassLimit: bypassRowLimit) return PreparedStatement( @@ -821,6 +826,7 @@ extension QueryExecutionCoordinator { } } parent.seedBufferFromActiveResult(tabId: tabId) + parent.dropParkedEdits(ofBackgroundTab: tabId) if parent.tabManager.selectedTabId == tabId { parent.toolbarState.isResultsCollapsed = false parent.toolbarState.recordQueryTiming(timing, for: tabId) diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator.swift index 3677faa438..9911aa248a 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator.swift @@ -7,6 +7,13 @@ import Combine import Foundation import TableProSQLGrammar +// The tab an approval prompt was shown for, and the database and schema it was bound to. +struct RunTarget { + let tabId: UUID + let databaseName: String + let schemaName: String? +} + @MainActor final class QueryExecutionCoordinator: ObservableObject { unowned let parent: MainContentCoordinator @@ -28,7 +35,7 @@ final class QueryExecutionCoordinator: ObservableObject { if AppSettingsManager.shared.editor.queryParametersEnabled, parent.bindsNamedParameters { let combinedSQL = SQLParameterExtractor.parameterSource(of: statements) - let detectedNames = SQLParameterExtractor.extractParameters(from: combinedSQL) + let detectedNames = SQLParameterExtractor.extractParameters(from: combinedSQL, grammar: parent.lexicalGrammar) if !detectedNames.isEmpty { let reconciled = detectAndReconcileParameters( @@ -63,7 +70,7 @@ final class QueryExecutionCoordinator: ObservableObject { install: ResultInstall = .newResult ) { guard !parent.isShowingSafeModePrompt, let route = executionRoute(for: batches) else { return } - guard !refuses(route, tabIndex: index) else { return } + guard !refuses(route, tabIndex: index), let target = runTarget(at: index) else { return } parent.isShowingSafeModePrompt = true let request = makeExecuteRequest( statements: batches.flatMap(\.statements), @@ -73,7 +80,7 @@ final class QueryExecutionCoordinator: ObservableObject { defer { parent.isShowingSafeModePrompt = false } switch await parent.executionGate.authorize(request) { case .authorized: - guard parent.rereadCanStart(install.source) else { return } + guard parent.rereadCanStart(install.source), stillHolds(target) else { return } switch route { case .single(let only): parent.executeQueryInternal( @@ -81,19 +88,45 @@ final class QueryExecutionCoordinator: ObservableObject { bypassRowLimit: bypassRowLimit, anchor: install.anchor ?? StatementAnchor(only), viewport: install.viewport, - source: install.source + source: install.source, + onTab: target.tabId ) case .statements(let statements): - executeMultipleStatements(statements, bypassRowLimit: bypassRowLimit) + executeMultipleStatements(statements, bypassRowLimit: bypassRowLimit, onTab: target.tabId) case .batches(let batches): - executeBatches(batches, parameters: [], bypassRowLimit: bypassRowLimit) + executeBatches(batches, parameters: [], bypassRowLimit: bypassRowLimit, onTab: target.tabId) case .needsBatchDriver: break } case .denied(let reason, _): - parent.tabManager.mutate(at: index) { $0.execution.errorMessage = reason } + parent.tabManager.mutate(tabId: target.tabId) { $0.execution.errorMessage = reason } + } + } + } + + private func runTarget(at index: Int) -> RunTarget? { + guard parent.tabManager.tabs.indices.contains(index) else { return nil } + let tab = parent.tabManager.tabs[index] + return RunTarget( + tabId: tab.id, databaseName: tab.tableContext.databaseName, schemaName: tab.tableContext.schemaName + ) + } + + // The approval prompt is not window-modal: the tab can be closed or repointed at another database before the answer. + private func stillHolds(_ target: RunTarget) -> Bool { + guard let tab = parent.tabManager.tabs.first(where: { $0.id == target.tabId }) else { return false } + // A tab bound to no database follows the connection, which a queued switch may move meanwhile. + let moved = !target.databaseName.isEmpty + && (tab.tableContext.databaseName != target.databaseName || tab.tableContext.schemaName != target.schemaName) + guard !moved else { + parent.tabManager.mutate(tabId: target.tabId) { + $0.execution.errorMessage = String( + localized: "This tab moved to another database while the query waited for approval. Run it again." + ) } + return false } + return true } private func makeExecuteRequest( @@ -121,9 +154,8 @@ final class QueryExecutionCoordinator: ObservableObject { install: ResultInstall = .newResult ) { guard !parent.isShowingSafeModePrompt, let route = executionRoute(for: batches) else { return } - guard !refuses(route, tabIndex: index) else { return } + guard !refuses(route, tabIndex: index), let target = runTarget(at: index) else { return } parent.isShowingSafeModePrompt = true - let tabId = parent.tabManager.tabs[index].id let request = makeExecuteRequest( statements: batches.flatMap(\.statements), extraCapabilities: extraCapabilities @@ -132,15 +164,16 @@ final class QueryExecutionCoordinator: ObservableObject { defer { parent.isShowingSafeModePrompt = false } switch await parent.executionGate.authorize(request) { case .authorized: - guard parent.rereadCanStart(install.source) else { return } + guard parent.rereadCanStart(install.source), stillHolds(target) else { return } executeParameterizedAfterSafeMode( route, parameters: parameters, bypassRowLimit: bypassRowLimit, - install: install + install: install, + tabId: target.tabId ) case .denied(let reason, _): - parent.tabManager.mutate(tabId: tabId) { $0.execution.errorMessage = reason } + parent.tabManager.mutate(tabId: target.tabId) { $0.execution.errorMessage = reason } } } } @@ -149,7 +182,8 @@ final class QueryExecutionCoordinator: ObservableObject { _ route: QueryExecutionRoute, parameters: [QueryParameter], bypassRowLimit: Bool, - install: ResultInstall + install: ResultInstall, + tabId: UUID ) { switch route { case .single(let only): @@ -158,12 +192,15 @@ final class QueryExecutionCoordinator: ObservableObject { parameters: parameters, bypassRowLimit: bypassRowLimit, anchor: install.anchor ?? StatementAnchor(only), - install: install + install: install, + onTab: tabId ) case .statements(let statements): - executeMultipleStatementsWithParameters(statements, parameters: parameters, bypassRowLimit: bypassRowLimit) + executeMultipleStatementsWithParameters( + statements, parameters: parameters, bypassRowLimit: bypassRowLimit, onTab: tabId + ) case .batches(let batches): - executeBatches(batches, parameters: parameters, bypassRowLimit: bypassRowLimit) + executeBatches(batches, parameters: parameters, bypassRowLimit: bypassRowLimit, onTab: tabId) case .needsBatchDriver: break } diff --git a/apple/TablePro/Core/Coordinators/RowEditingCoordinator+SaveChanges.swift b/apple/TablePro/Core/Coordinators/RowEditingCoordinator+SaveChanges.swift index d01ac53574..bcb80872e1 100644 --- a/apple/TablePro/Core/Coordinators/RowEditingCoordinator+SaveChanges.swift +++ b/apple/TablePro/Core/Coordinators/RowEditingCoordinator+SaveChanges.swift @@ -12,6 +12,25 @@ import TableProSQLGrammar private let saveChangesLogger = Logger(subsystem: "com.TablePro", category: "RowEditingCoordinator") +// What the user staged, without the pre-images: a value loaded while a prompt is up completes those and edits nothing. +struct PlannedEdits: Equatable { + private let changes: [RowChange] + private let deletedRowIDs: Set + private let insertedRowIDs: Set + private let insertedRowData: [RowID: [PluginCellValue]] + + init(_ pending: PendingChanges) { + changes = pending.changes.map { change in + var change = change + change.originalRow = nil + return change + } + deletedRowIDs = pending.deletedRowIDs + insertedRowIDs = pending.insertedRowIDs + insertedRowData = pending.insertedRowData + } +} + extension RowEditingCoordinator { /// The plan carries the scope it was built for, so it runs where its statements were /// generated however long the destructive-delete sheet and the authorization prompt stay @@ -23,12 +42,13 @@ extension RowEditingCoordinator { tableOperationOptions: inout [DatabaseTreeTableRef: TableOperationOptions] ) { endInspectorEditRun() + // Taken now: the slot is a handoff, and a save still in flight keeps its own waiter when a second one starts. + let waiter = parent.takeSaveCompletion() let hasEditedCells = parent.changeManager.hasChanges let hasPendingTableOps = !pendingTruncates.isEmpty || !pendingDeletes.isEmpty guard hasEditedCells || hasPendingTableOps else { - parent.saveCompletionContinuation?.resume(returning: true) - parent.saveCompletionContinuation = nil + waiter?.resume(returning: true) return } @@ -42,13 +62,13 @@ extension RowEditingCoordinator { $0.execution.errorMessage = String(localized: "A save is already running in this window.") } } - parent.saveCompletionContinuation?.resume(returning: false) - parent.saveCompletionContinuation = nil + waiter?.resume(returning: false) return } + let savingTabId = parent.tabManager.selectedTabId guard parent.selectedTabScope != nil else { - failSave(message: String(localized: "Not connected to database")) + failSave(message: String(localized: "Not connected to database"), tabId: savingTabId, waiter: waiter) return } @@ -63,13 +83,17 @@ extension RowEditingCoordinator { failSave( message: [error.localizedDescription, (error as? DataWriteError)?.recoverySuggestion] .compactMap { $0 } - .joined(separator: " ") + .joined(separator: " "), + tabId: savingTabId, + waiter: waiter ) return } guard !plan.isEmpty else { - failSave(message: String(localized: "Could not generate SQL for changes.")) + failSave( + message: String(localized: "Could not generate SQL for changes."), tabId: savingTabId, waiter: waiter + ) return } @@ -89,17 +113,19 @@ extension RowEditingCoordinator { deletedRowCount: parent.changeManager.deletedRowIDs.count, tableName: plan.rowOperations.first(where: { $0.kind == .delete })?.target.table ) - let confirmingTabId = parent.tabManager.selectedTabId - let confirmingChanges = parent.changeManager.pending + let confirmingChanges = PlannedEdits(parent.changeManager.pending) Task { [weak self, parent] in - guard let self else { return } + guard let self else { + waiter?.resume(returning: false) + return + } if deleteConfirmation.isRequired { let preview = await previewCascade(of: plan) // The window stays live while counting. A tab switched to holds the change manager this // save would clear, and an edit made meanwhile is not in the plan, so either ends the save. - let planIsCurrent = parent.tabManager.selectedTabId == confirmingTabId - && parent.changeManager.pending == confirmingChanges + let planIsCurrent = parent.tabManager.selectedTabId == savingTabId + && PlannedEdits(parent.changeManager.pending) == confirmingChanges var choice = BulkDeleteAlert.Choice.cancel if planIsCurrent { choice = await BulkDeleteAlert.present(deleteConfirmation.with(preview), window: parent.contentWindow) @@ -114,9 +140,8 @@ extension RowEditingCoordinator { ) } endSaveInFlight() - parent.saveCompletionContinuation?.resume(returning: false) - parent.saveCompletionContinuation = nil - if !planIsCurrent, let index = parent.tabManager.tabs.firstIndex(where: { $0.id == confirmingTabId }) { + waiter?.resume(returning: false) + if !planIsCurrent, let index = parent.tabManager.tabs.firstIndex(where: { $0.id == savingTabId }) { parent.tabManager.mutate(at: index) { $0.execution.errorMessage = String( localized: "The changes were edited while the save was checking other tables. Save again to include them." @@ -146,12 +171,37 @@ extension RowEditingCoordinator { ) switch decision { case .authorized: + // The approval prompt is not window-modal. A tab switched to holds the change manager this save + // would clear, and an edit made meanwhile is not in the plan, so either ends the save. + let editsAreCurrent = !hasEditedCells + || (parent.tabManager.selectedTabId == savingTabId + && PlannedEdits(parent.changeManager.pending) == confirmingChanges) + guard editsAreCurrent else { + if hasPendingTableOps { + restorePendingTableOperations( + connectionId: connId, + truncates: snapshotTruncates, + deletes: snapshotDeletes, + options: snapshotOptions + ) + } + failSave( + message: String( + localized: "The tab or its edits changed while the save waited for approval. Nothing was saved." + ), + tabId: savingTabId, + waiter: waiter + ) + return + } var truncs = snapshotTruncates var dels = snapshotDeletes var opts = snapshotOptions executeCommitPlan( plan, scope: plan.scope, + savingTabId: savingTabId, + waiter: waiter, clearTableOps: hasPendingTableOps, pendingTruncates: &truncs, pendingDeletes: &dels, @@ -166,7 +216,7 @@ extension RowEditingCoordinator { options: snapshotOptions ) } - failSave(message: reason) + failSave(message: reason, tabId: savingTabId, waiter: waiter) } } } @@ -178,6 +228,8 @@ extension RowEditingCoordinator { private func executeCommitPlan( _ plan: DataWritePlan, scope: DatabaseScope, + savingTabId: UUID?, + waiter: CheckedContinuation?, clearTableOps: Bool, pendingTruncates: inout Set, pendingDeletes: inout Set, @@ -186,8 +238,7 @@ extension RowEditingCoordinator { let validSteps = plan.steps.filter { !$0.statement.sql.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } guard !validSteps.isEmpty else { endSaveInFlight() - parent.saveCompletionContinuation?.resume(returning: true) - parent.saveCompletionContinuation = nil + waiter?.resume(returning: true) return } @@ -209,12 +260,14 @@ extension RowEditingCoordinator { } let route = DatabaseManager.shared.executionRoute(for: scope) - let savingTabId = parent.tabManager.selectedTabId let serverOwned = serverOwnedColumns(of: savingTabId) let columnTypeNames = parent.changeManager.columnTypeNames Task { [weak self, parent] in - guard let self else { return } + guard let self else { + waiter?.resume(returning: false) + return + } let overallStartTime = Date() let operationStart = ContinuousClock.Instant.now @@ -248,8 +301,7 @@ extension RowEditingCoordinator { ) endSaveInFlight() - parent.saveCompletionContinuation?.resume(returning: true) - parent.saveCompletionContinuation = nil + waiter?.resume(returning: true) reportSaveFinished( .succeeded(OperationSummary(rowsAffected: run.results.reduce(0) { $0 + $1.rowsAffected })), connection: conn, @@ -317,7 +369,9 @@ extension RowEditingCoordinator { message: String( format: String(localized: "Save failed: %@"), DatabaseWriteRejectionDiagnosis.formatted(error) - ) + ), + tabId: savingTabId, + waiter: waiter ) } } @@ -632,13 +686,13 @@ extension RowEditingCoordinator { ) } - private func failSave(message: String) { + // Written to the tab that started the save: the one in front may be another by the time it fails. + private func failSave(message: String, tabId: UUID?, waiter: CheckedContinuation?) { endSaveInFlight() - if let index = parent.tabManager.selectedTabIndex { - parent.tabManager.mutate(at: index) { $0.execution.errorMessage = message } + if let tabId { + parent.tabManager.mutate(tabId: tabId) { $0.execution.errorMessage = message } } - parent.saveCompletionContinuation?.resume(returning: false) - parent.saveCompletionContinuation = nil + waiter?.resume(returning: false) } private func restorePendingTableOperations( @@ -674,3 +728,10 @@ fileprivate extension RowEditingCoordinator { ) } } + +extension MainContentCoordinator { + func takeSaveCompletion() -> CheckedContinuation? { + defer { saveCompletionContinuation = nil } + return saveCompletionContinuation + } +} diff --git a/apple/TablePro/Core/CrossEngine/CrossEngineValueCoercer.swift b/apple/TablePro/Core/CrossEngine/CrossEngineValueCoercer.swift index 3e00035544..c39008be0e 100644 --- a/apple/TablePro/Core/CrossEngine/CrossEngineValueCoercer.swift +++ b/apple/TablePro/Core/CrossEngine/CrossEngineValueCoercer.swift @@ -137,7 +137,8 @@ internal struct CrossEngineValueCoercer: Sendable { /// only by accident, because its `:` makes the array parse fail, so the bug was invisible /// until a row held an empty one and `jsonb_typeof` came back `array`. private func json(_ value: PluginCellValue, source: CanonicalTypeKind?) -> PluginCellValue { - guard sourceFamily == .postgres, case .array = source, case .text(let text) = value else { return value } + guard sourceFamily == .postgres, case .array(let elementKind) = source, + case .text(let text) = value else { return value } guard text.hasPrefix("{"), text.hasSuffix("}") else { return value } guard let elements = PostgresArrayLiteralCodec.parse(text) else { return value } var items: [String] = [] @@ -147,7 +148,7 @@ internal struct CrossEngineValueCoercer: Sendable { case .null: items.append("null") case .value(let raw): - items.append(Self.jsonScalar(raw)) + items.append(Self.jsonScalar(raw, element: elementKind)) /// A case this build has never seen cannot be rendered as JSON, and a wrong guess would /// be written into every row. The value is left as the array literal it arrived as. @unknown default: @@ -198,15 +199,24 @@ internal struct CrossEngineValueCoercer: Sendable { return try? NSRegularExpression(pattern: pattern, options: [.caseInsensitive]) }() - private static func jsonScalar(_ raw: String) -> String { - if raw == "true" || raw == "false" { return raw } - if Int64(raw) != nil || Double(raw) != nil { return raw } - let escaped = raw - .replacingOccurrences(of: "\\", with: "\\\\", options: .literal) - .replacingOccurrences(of: "\"", with: "\\\"", options: .literal) - .replacingOccurrences(of: "\n", with: "\\n", options: .literal) - .replacingOccurrences(of: "\r", with: "\\r", options: .literal) - .replacingOccurrences(of: "\t", with: "\\t", options: .literal) - return "\"\(escaped)\"" + // The element's type decides, never the shape of the text: a `text[]` holding 02134, inf or + // true holds strings, and written bare they are invalid JSON or a different type. + private static func jsonScalar(_ raw: String, element: CanonicalTypeKind) -> String { + switch element { + case .boolean: + switch raw.lowercased() { + case "t", "true": return "true" + case "f", "false": return "false" + default: break + } + case .integer, .decimal, .floatingPoint: + if let literal = JsonNumberNormalizer.numberLiteral(from: raw) { return literal } + case .json: + let parsed = try? JSONSerialization.jsonObject(with: Data(raw.utf8), options: [.fragmentsAllowed]) + if parsed != nil { return raw } + default: + break + } + return "\"\(PluginExportUtilities.escapeJSONString(raw))\"" } } diff --git a/apple/TablePro/Core/DataWrite/CascadeDeletePreview.swift b/apple/TablePro/Core/DataWrite/CascadeDeletePreview.swift index 56dcb2ba84..f69de6844a 100644 --- a/apple/TablePro/Core/DataWrite/CascadeDeletePreview.swift +++ b/apple/TablePro/Core/DataWrite/CascadeDeletePreview.swift @@ -238,7 +238,9 @@ extension CascadeDeletePreviewer.Environment { @MainActor func scalar(_ query: String, on route: ScopedDriverRoute) async throws -> String? { - try await manager.withCancellableRead(scope: scope, route: route) { driver in + try await manager.withCancellableRead( + scope: scope, route: route, ownsSessionState: route != countRoute + ) { driver in try await driver.execute(query: query).rows.first?.first?.asText } } diff --git a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift index 704394140b..968b67c8df 100644 --- a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift +++ b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift @@ -606,6 +606,7 @@ struct RowChangeStatementFactory { databaseType: databaseType, generatedColumns: generatedColumns ?? self.generatedColumns, rowMatchPolicy: rowMatchPolicy, + columnTypeNames: columnTypeNames, insertOverridesSystemValue: insertOverridesSystemValue, dialect: PluginManager.shared.sqlDialect(for: databaseType), quoteIdentifier: pluginDriver?.quoteIdentifier diff --git a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift index 4f2890779d..1206c4838b 100644 --- a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift +++ b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge+Scripts.swift @@ -111,7 +111,8 @@ extension DatabaseAccessBridge { route: route, policy: policy, owner: owner, - timeoutSeconds: timeoutSeconds + timeoutSeconds: timeoutSeconds, + onLateCompletion: { CatalogChangeService.post(statementsRan) } ) { driver in try await ScriptBatchRun.run( batches, startLines: startLines, rowCap: rowCap, driver: driver, progress: progress, context: context diff --git a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift index ec818feb5c..e0c3b5f772 100644 --- a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift +++ b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift @@ -393,7 +393,8 @@ internal actor DatabaseAccessBridge { route: route, policy: policy, owner: owner, - timeoutSeconds: timeoutSeconds + timeoutSeconds: timeoutSeconds, + onLateCompletion: { CatalogChangeService.post(statementRan) } ) { driver in let answer = try await driver.executeUserQuery( query: sql, @@ -457,7 +458,8 @@ internal actor DatabaseAccessBridge { route: route, policy: policy, owner: owner, - timeoutSeconds: timeoutSeconds + timeoutSeconds: timeoutSeconds, + onLateCompletion: { CatalogChangeService.post(statementRan) } ) { driver in try await driver.explainPlan(statement: statement, variantId: variantId) } @@ -647,10 +649,23 @@ internal actor DatabaseAccessBridge { policy: DriverCancellationPolicy, owner: DriverLeaseOwner, timeoutSeconds: Int, + onLateCompletion: (@Sendable () -> Void)? = nil, _ body: @Sendable @escaping (DatabaseDriver) async throws -> Output ) async throws -> Output { let connectionId = scope.connectionId let manager = await databaseManager + if policy == .protectedWrite { + return try await Self.awaitProtectedWrite( + timeout: .seconds(timeoutSeconds), + timeoutSeconds: timeoutSeconds, + onLateCompletion: onLateCompletion + ) { enterDriver in + try await manager.withScopedDriver(scope: scope, route: route, cancellation: policy) { driver in + try enterDriver() + return try await body(driver) + } + } + } return try await withThrowingTaskGroup(of: Output.self) { group in group.addTask { try await manager.withScopedDriver( @@ -667,12 +682,7 @@ internal actor DatabaseAccessBridge { owner: owner, on: connectionId, delivery: .immediate ) } - throw DatabaseAccessError.timeout( - String( - format: String(localized: "Query timed out after %d seconds"), - timeoutSeconds - ) - ) + throw Self.plainTimeout(seconds: timeoutSeconds) } guard let first = try await group.next() else { throw DatabaseAccessError.dataSourceError("No result from query execution") @@ -682,6 +692,94 @@ internal actor DatabaseAccessBridge { } } + private static func plainTimeout(seconds: Int) -> DatabaseAccessError { + .timeout(String(format: String(localized: "Query timed out after %d seconds"), seconds)) + } + + private enum ProtectedWritePhase: Sendable { + case queued + case inDriver + case abandoned + } + + // A protected write is never aborted once the driver has it. Past the deadline, or after the + // caller cancels, such a write runs on and the caller is told the outcome is unknown: a plain + // timeout would read as "nothing happened" and invite a retry that applies it twice. A write + // still waiting for its turn is refused instead, so nothing runs after the caller was told no. + // `work` calls `enterDriver` as the first thing inside its driver turn. + internal static func awaitProtectedWrite( + timeout: Duration, + timeoutSeconds: Int, + onLateCompletion: (@Sendable () -> Void)? = nil, + _ work: @Sendable @escaping (_ enterDriver: @escaping @Sendable () throws -> Void) async throws -> Output + ) async throws -> Output { + let phase = OSAllocatedUnfairLock(initialState: ProtectedWritePhase.queued) + let abandonIfQueued: @Sendable () -> Bool = { + phase.withLock { current -> Bool in + guard current == .queued else { return current == .abandoned } + current = .abandoned + return true + } + } + let (outcomes, continuation) = AsyncThrowingStream.makeStream() + let deadline = Task { + do { + try await Task.sleep(for: timeout) + } catch { + return false + } + continuation.yield(nil) + return true + } + let workTask = Task { + do { + let output = try await work { + let entered = phase.withLock { current -> Bool in + guard current == .queued else { return false } + current = .inDriver + return true + } + guard entered else { throw CancellationError() } + } + continuation.yield(output) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + deadline.cancel() + let ranInDriver = phase.withLock { $0 == .inDriver } + if await deadline.value, ranInDriver { + onLateCompletion?() + } + } + return try await withTaskCancellationHandler { + for try await outcome in outcomes { + if let outcome { return outcome } + if abandonIfQueued() { + workTask.cancel() + throw plainTimeout(seconds: timeoutSeconds) + } + throw DatabaseAccessError.timeout( + String( + format: String(localized: """ + The statement did not finish within %d seconds and is still running. \ + It may have been applied; check before running it again. + """), + timeoutSeconds + ), + context: ["outcome": "unknown"] + ) + } + try Task.checkCancellation() + throw DatabaseAccessError.dataSourceError("No result from query execution") + } onCancel: { + if abandonIfQueued() { + workTask.cancel() + } + deadline.cancel() + } + } + /// The text an external client's statement reaches the driver as. /// /// Trailing separators come off and a terminator that belongs to the statement stays, as the editor decides it: diff --git a/apple/TablePro/Core/Database/DatabaseManager+Health.swift b/apple/TablePro/Core/Database/DatabaseManager+Health.swift index e41bf3d010..b42ed9eae7 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Health.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Health.swift @@ -40,6 +40,7 @@ extension DatabaseManager { pingInterval: { await AppSettingsManager.shared.general.connectionHealthCheck.interval }, pingHandler: { [weak self] in guard let self else { return false } + await self.closeIdleOpenedLanes(for: connectionId) if await self.queriesInFlight[connectionId] != nil { Self.logger.debug("Ping skipped, operation in flight for \(connectionId)") return true @@ -178,6 +179,19 @@ extension DatabaseManager { } } + // Read before the driver is closed and replaced, which prunes the hold. The user's next statement there + // is refused once, as on a lane, instead of running outside the transaction it was written for. + internal func noteTransactionLost(with driver: DatabaseDriver?, for connectionId: UUID) { + guard let driver, let session = activeSessions[connectionId] else { return } + let lost = sessionTransactions.holds(for: connectionId).filter { sessionTransactions.driver(for: $0) === driver } + for hold in lost { + sessionLanes.markTransactionLost( + database: hold.key.lane ?? session.resolvedBrowseDatabase, + for: connectionId + ) + } + } + internal func reconnectVerdict(for error: Error) -> ConnectionFailureClassifier.ReconnectVerdict { if let tlsError = error as? SSLHandshakeError { return ConnectionFailureClassifier.reconnectVerdict(for: tlsError) @@ -229,6 +243,7 @@ extension DatabaseManager { generation: Int ) async throws -> ReconnectResult? { let connectionId = session.connection.id + noteTransactionLost(with: session.driver, for: connectionId) session.driver?.disconnect() // Resolved as the connect resolved it; a reference that lost its approval needs a person. let connection = try await envVarReferenceConsent.resolve(session.connection, trigger: .background) diff --git a/apple/TablePro/Core/Database/DatabaseManager+Metadata.swift b/apple/TablePro/Core/Database/DatabaseManager+Metadata.swift index 1fe2288dc4..d10ea0b38b 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Metadata.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Metadata.swift @@ -55,6 +55,7 @@ extension DatabaseManager { route: metadataRoute(for: scope), workload: workload, cancellation: .untracked, + ownsSessionState: false, body ) } diff --git a/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift b/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift index 5f3389174b..84ae516c57 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+ScopedDriver.swift @@ -50,11 +50,21 @@ extension DatabaseManager { metadataRoute(for: scope) } - /// SQL the user owns stays on the session driver, which holds their transaction, - /// their temp tables and the handle Stop cancels. The pool is the fallback only for - /// engines that cannot change database on a live connection, where the alternative - /// is querying whichever database the connection happens to be on. + /// SQL the user owns stays on a session connection, which holds their transaction, their temp + /// tables and the handle Stop cancels. It never runs on a pooled connection: the pool closes an + /// idle entry, and a transaction the user left open there is rolled back with it. A database + /// with no session connection yet gets one when its turn comes. func executionRoute(for scope: DatabaseScope) -> ScopedDriverRoute { + route(for: scope, opensLane: true) + } + + /// A table read holds nothing between statements, so a database nobody has run SQL on is read + /// from the pool instead of opening a session connection for it. + func tableReadRoute(for scope: DatabaseScope) -> ScopedDriverRoute { + route(for: scope, opensLane: false) + } + + private func route(for scope: DatabaseScope, opensLane: Bool) -> ScopedDriverRoute { guard let session = activeSessions[scope.connectionId] else { return .unavailable(String(localized: "Not connected to database")) } @@ -70,10 +80,18 @@ extension DatabaseManager { /// the transaction, temp tables and settings they left there. /// A database whose connection ended with a transaction still open takes the session path /// once more, where that loss is reported before anything runs on a replacement. - if usesDatabaseLanes(session), - sessionLanes.parkedDriver(for: scope.connectionId, database: scope.database) != nil - || sessionLanes.hasTransactionLoss(database: scope.database, for: scope.connectionId) { - return .sessionDriver + if usesDatabaseLanes(session) { + if sessionLanes.parkedDriver(for: scope.connectionId, database: scope.database) != nil { + return .sessionDriver + } + // A table read depends on no transaction, so a lost one is not its to hear about. + if opensLane, sessionLanes.hasTransactionLoss(database: scope.database, for: scope.connectionId) { + return .sessionDriver + } + // A temporary table lives in the session that made it, which a new connection is not. + if opensLane, !scope.isSessionBound { + return .sessionDriver + } } guard canPool(session), !scope.isSessionBound else { return .unavailable( @@ -97,6 +115,7 @@ extension DatabaseManager { route: ScopedDriverRoute, workload: MetadataConnectionPool.Workload = .interactive, cancellation: DriverCancellationPolicy, + ownsSessionState: Bool = true, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T { let leased = trackedLease(for: scope.connectionId, cancellation: cancellation, body) @@ -108,7 +127,7 @@ extension DatabaseManager { scope: scope, workload: workload, leased ) case .sessionDriver: - return try await withPinnedSessionDriver(scope: scope, leased) + return try await withPinnedSessionDriver(scope: scope, ownsSessionState: ownsSessionState, leased) } } @@ -117,12 +136,19 @@ extension DatabaseManager { func withCancellableRead( scope: DatabaseScope, route: ScopedDriverRoute, + ownsSessionState: Bool = true, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T { let owner = DriverLeaseOwner() let connectionId = scope.connectionId return try await withTaskCancellationHandler { - try await withScopedDriver(scope: scope, route: route, cancellation: .cancellableRead(owner), body) + try await withScopedDriver( + scope: scope, + route: route, + cancellation: .cancellableRead(owner), + ownsSessionState: ownsSessionState, + body + ) } onCancel: { Task { @MainActor in try? DatabaseManager.shared.cancelRunningQuery(owner: owner, on: connectionId, delivery: .background) @@ -145,7 +171,7 @@ extension DatabaseManager { guard rowCountRunsWithPage(scope, filtered: filtered) else { return try await withMetadataDriver(scope: scope, workload: .bulk, body) } - return try await withCancellableRead(scope: scope, route: executionRoute(for: scope), body) + return try await withCancellableRead(scope: scope, route: tableReadRoute(for: scope), body) } /// A table tab's read is a SELECT the app built from the tab's own table, so it depends on @@ -165,7 +191,7 @@ extension DatabaseManager { _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T { let leased = trackedLease(for: scope.connectionId, cancellation: cancellation, body) - var route = executionRoute(for: scope) + var route = tableReadRoute(for: scope) while true { switch route { case .unavailable(let message): @@ -343,36 +369,46 @@ extension DatabaseManager { private func withPinnedSessionDriver( scope: DatabaseScope, + ownsSessionState: Bool, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> T { - try await withSessionDriverTurn(scope: scope) { driver in + try await withSessionDriverTurn(scope: scope, ownsSessionState: ownsSessionState) { driver in try await pin(driver, to: scope) return try await body(driver) } } - /// The route is asked again through `executionRoute` itself, so it reads the same inputs the + /// The route is asked again through `tableReadRoute` itself, so it reads the same inputs the /// caller's first answer did, the browsed database among them. The driver's own connection is no /// substitute: it carries the database name the connection resolved to, not the one browsed. private func withTableReadSessionDriver( scope: DatabaseScope, _ body: @Sendable @escaping (DatabaseDriver) async throws -> T ) async throws -> TableReadTurn { - try await withSessionDriverTurn(scope: scope) { driver in - let route = executionRoute(for: scope) + try await withSessionDriverTurn(scope: scope, ownsSessionState: false) { driver in + let route = tableReadRoute(for: scope) guard route == .sessionDriver else { return .moved(route) } try await pin(driver, to: scope) return .ran(try await body(driver)) } } + /// `ownsSessionState` is false for a table read or a metadata read, which depend on nothing the + /// session holds. Such a turn neither opens a session connection nor takes the report of a lost + /// transaction, which is kept for the user's next statement. private func withSessionDriverTurn( scope: DatabaseScope, + ownsSessionState: Bool, _ turn: (DatabaseDriver) async throws -> R ) async throws -> R { let connectionId = scope.connectionId let lane = activeSessions[connectionId].flatMap { laneDatabase(for: scope, in: $0) } let startsOnBrowsed = lane == nil || lane == activeSessions[connectionId]?.resolvedBrowseDatabase + /// Decided before the turn is queued. Work written against a connection that was there when + /// it queued and is gone when its turn comes is refused, never moved onto a new one: the + /// transaction it meant to continue ended with the connection it was on. + let startsWithoutLane = !startsOnBrowsed && !scope.isSessionBound + && lane.flatMap { sessionLanes.parkedDriver(for: connectionId, database: $0) } == nil if startsOnBrowsed { /// Outside the gate on purpose. A verification that has to reconnect runs the whole /// reconnect, which restores the schema and the database on the new driver, and doing that @@ -396,28 +432,36 @@ extension DatabaseManager { /// A database whose own connection is gone, its entry closed, falls back to the browsed /// driver only to be turned away: a table read re-routes to the pool and anything else /// is refused by `pin` before it runs a statement there. - guard isUsable(connectionId), - let driver = sessionDriver(for: connectionId, laneDatabase: lane) - ?? activeSessions[connectionId]?.driver - else { + guard isUsable(connectionId), let browsedDriver = activeSessions[connectionId]?.driver else { throw DatabaseError.notConnected } guard let lane else { - // A driver replaced while it held a transaction, which a Vault lease rotation can do. - if let browsed = activeSessions[connectionId]?.resolvedBrowseDatabase, - sessionLanes.takeTransactionLoss(database: browsed, for: connectionId) { - throw DatabaseError.connectionFailed(Self.parkedTransactionLostMessage(database: browsed)) + // A driver replaced while it held a transaction, by a reconnect or a Vault lease rotation. + if ownsSessionState, let lost = sessionLanes.takeAnyTransactionLoss(for: connectionId) { + throw DatabaseError.connectionFailed(Self.parkedTransactionLostMessage(database: lost)) } - return try await recordingTransaction(on: driver, for: .init(connectionId: connectionId, lane: nil)) { - try await turn(driver) + return try await recordingTransaction(on: browsedDriver, for: .init(connectionId: connectionId, lane: nil)) { + try await turn(browsedDriver) } } + let driver: DatabaseDriver + if let held = sessionDriver(for: connectionId, laneDatabase: lane) { + driver = held + } else if ownsSessionState, !scope.isSessionBound, + startsWithoutLane || sessionLanes.wasClosedIdle(database: lane, for: connectionId), + !sessionLanes.hasTransactionLoss(database: lane, for: connectionId) { + driver = try await openedLaneDriver(database: lane, for: connectionId) + } else { + driver = browsedDriver + } let laneDriver = sessionLanes.isParked(driver, for: connectionId) ? try await verifiedParkedDriver(driver, database: lane, for: connectionId) : driver - if sessionLanes.takeTransactionLoss(database: lane, for: connectionId) { + if ownsSessionState, sessionLanes.takeTransactionLoss(database: lane, for: connectionId) { throw DatabaseError.connectionFailed(Self.parkedTransactionLostMessage(database: lane)) } + sessionLanes.noteOpenedUse(database: lane, for: connectionId) + defer { sessionLanes.noteOpenedUse(database: lane, for: connectionId) } return try await recordingTransaction(on: laneDriver, for: .init(connectionId: connectionId, lane: lane)) { try await turn(laneDriver) } @@ -495,6 +539,7 @@ extension DatabaseManager { /// Read before the ping: a ping that finds the socket closed leaves libpq reporting no /// transaction state at all, which would read as nothing to lose. let held = await driver.heldSessionTransactionState() + let wasOpened = sessionLanes.isOpened(database: database, for: connectionId) sessionLanes.beginVerifying(driver) defer { sessionLanes.endVerifying(driver) } do { @@ -520,10 +565,94 @@ extension DatabaseManager { } sessionLanes.park(reopened, database: database, for: connectionId) sessionLanes.markVerified(reopened) + if wasOpened { + sessionLanes.markOpened(reopened, database: database, for: connectionId) + } return reopened } } + /// Opens the session connection of a database the user runs SQL on without having browsed it. + /// It is parked like a database the user switched away from, so the transaction bar, the close + /// warnings and the loss report all cover it. + /// + /// A switch or a closed entry elsewhere on the connection supersedes the open, which is then + /// tried once more. A switch onto this very database that finished first is the connection to + /// run on, and the one opened here is closed. + private func openedLaneDriver(database: String, for connectionId: UUID) async throws -> DatabaseDriver { + // Here as well as on the health tick, which does not run with health checks set to Never. + await closeIdleOpenedLanes(for: connectionId) + let sessionStartedAt = activeSessions[connectionId]?.connectedAt + for _ in 0 ..< 2 { + guard !MetadataConnectionPool.shared.isReplacingTransport(for: connectionId) else { + throw DatabaseError.connectionFailed(Self.recoveringConnectMessage) + } + let generation = sessionLanes.generation(for: connectionId) + let opened = try await sessionLanes.open( + DatabaseScope(connectionId: connectionId, database: database, schema: nil) + ) + guard !MetadataConnectionPool.shared.isReplacingTransport(for: connectionId), + let session = activeSessions[connectionId], + session.connectedAt == sessionStartedAt, + session.driver != nil + else { + opened.disconnect() + throw DatabaseError.notConnected + } + if let current = sessionDriver(for: connectionId, laneDatabase: database) { + opened.disconnect() + return current + } + guard sessionLanes.generation(for: connectionId) == generation else { + opened.disconnect() + continue + } + sessionLanes.park(opened, database: database, for: connectionId) + sessionLanes.markVerified(opened) + sessionLanes.markOpened(opened, database: database, for: connectionId) + return opened + } + throw DatabaseError.notConnected + } + + /// Closes the connections opened for SQL on databases the user never browsed that nobody has + /// used for `idleFor`. One that is running work or holds a transaction stays. Whatever else the + /// session held goes with it unreported: on PostgreSQL its temp tables, `SET` values and + /// advisory locks. + internal func closeIdleOpenedLanes( + for connectionId: UUID, + idleFor: TimeInterval = SessionLanes.openedIdleTimeout, + now: Date = Date() + ) async { + for lane in sessionLanes.openedLanes(for: connectionId) where now.timeIntervalSince(lane.lastUsed) >= idleFor { + _ = await closeOpenedLaneIfIdle(lane.database, driver: lane.driver, for: connectionId) + } + } + + /// Asked inside the lane's own turn, so no statement runs between the answer and the close. Only + /// a driver that says it is idle is closed: one that cannot say may be holding a transaction. + private func closeOpenedLaneIfIdle( + _ database: String, + driver: DatabaseDriver, + for connectionId: UUID + ) async -> Bool { + let key = SessionDriverGate.Key(connectionId: connectionId, database: database) + guard !sessionDriverGate.isHeld(key) else { return false } + let closed = try? await sessionDriverGate.withExclusiveAccess(key) { () async -> Bool in + guard sessionLanes.isOpened(database: database, for: connectionId), + sessionLanes.parkedDriver(for: connectionId, database: database) === driver, + !sessionTransactions.holdsTransaction(connectionId: connectionId, container: database) + else { return false } + let state = await Self.boundedTransactionState(of: driver, clock: ContinuousClock()) + guard state == .idle, + sessionLanes.parkedDriver(for: connectionId, database: database) === driver + else { return false } + sessionLanes.closeIdle(database: database, for: connectionId) + return true + } + return closed ?? false + } + static func parkedTransactionLostMessage(database: String) -> String { String( format: String( diff --git a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift index 9269f652cc..aeb3174daa 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift @@ -528,8 +528,8 @@ extension DatabaseManager { let sessionStartedAt = session.connectedAt let generation = sessionLanes.beginSwitch(for: connectionId) - let target: DatabaseDriver - let freshlyOpened: Bool + var target: DatabaseDriver + var freshlyOpened: Bool if let parkedDriver = await answeringParkedDriver(database: database, for: connectionId) { target = parkedDriver freshlyOpened = false @@ -547,6 +547,13 @@ extension DatabaseManager { if freshlyOpened { target.disconnect() } throw CancellationError() } + // A statement on this database opened its connection while this open was in flight, and may hold + // a transaction there. That connection is the one to browse on; the one opened here is closed. + if freshlyOpened, let parkedMeanwhile = sessionLanes.parkedDriver(for: connectionId, database: database) { + target.disconnect() + target = parkedMeanwhile + freshlyOpened = false + } _ = sessionLanes.unpark(database: database, for: connectionId) sessionLanes.park(leavingDriver, database: leavingDatabase, for: connectionId) @@ -656,6 +663,8 @@ extension DatabaseManager { let previousBrowseSchema = previous.browseSchema let previousSavedSchema = appSettingsStorage.loadLastSchema(for: connectionId) + // Noted under the database being left, before the session names the target. + noteTransactionLost(with: previous.driver, for: connectionId) updateSession(connectionId) { session in session.connection.database = database session.browseDatabase = database diff --git a/apple/TablePro/Core/Database/SessionLanes.swift b/apple/TablePro/Core/Database/SessionLanes.swift index 6debc2a838..ebcd95ee09 100644 --- a/apple/TablePro/Core/Database/SessionLanes.swift +++ b/apple/TablePro/Core/Database/SessionLanes.swift @@ -37,6 +37,21 @@ internal final class SessionLanes { /// is refused once, so work meant to continue that transaction does not carry on without it. private var lostTransactions: [UUID: Set] = [:] + private struct Opened { + weak var driver: AnyObject? + var lastUsed: Date + } + + /// Connections opened for SQL on a database the user never browsed. Unlike a database the user + /// switched away from, nothing on screen closes one, so it is closed once idle. Browsing its + /// database takes it off this list. + private var opened: [UUID: [String: Opened]] = [:] + /// Databases whose opened connection was closed for being idle. Work queued for one reopens it: + /// no transaction was lost. The session's temp tables, `SET` values and advisory locks were. + private var closedIdle: [UUID: Set] = [:] + + internal static let openedIdleTimeout: TimeInterval = MetadataConnectionPool.idleTimeout + /// Opens the connection for a database the session has not browsed yet. Replaced under test, /// where plugins never load and no server is there to dial. internal var opener: Opener @@ -67,6 +82,10 @@ internal final class SessionLanes { internal func park(_ driver: DatabaseDriver, database: String, for connectionId: UUID) { let replaced = parked[connectionId]?[database].flatMap { $0 !== driver ? $0 : nil } parked[connectionId, default: [:]][database] = driver + closedIdle[connectionId]?.remove(database) + if opened[connectionId]?[database]?.driver !== driver { + opened[connectionId]?.removeValue(forKey: database) + } if let replaced { replaced.disconnect() onClose?(connectionId) @@ -79,6 +98,7 @@ internal final class SessionLanes { if parked[connectionId]?.isEmpty == true { parked.removeValue(forKey: connectionId) } + opened[connectionId]?.removeValue(forKey: database) return driver } @@ -91,6 +111,7 @@ internal final class SessionLanes { if supersedingOpens { generations[connectionId, default: 0] &+= 1 } + closedIdle[connectionId]?.remove(database) guard let driver = unpark(database: database, for: connectionId) else { return } Self.logger.info("closing parked connection connId=\(connectionId, privacy: .public)") verifiedAt.removeValue(forKey: ObjectIdentifier(driver)) @@ -104,6 +125,8 @@ internal final class SessionLanes { internal func closeAll(for connectionId: UUID, keepingLostTransactions: Bool = false) { generations[connectionId, default: 0] &+= 1 let drivers = parked.removeValue(forKey: connectionId) ?? [:] + opened.removeValue(forKey: connectionId) + closedIdle.removeValue(forKey: connectionId) if !keepingLostTransactions { lostTransactions.removeValue(forKey: connectionId) } @@ -198,4 +221,41 @@ internal final class SessionLanes { internal func takeTransactionLoss(database: String, for connectionId: UUID) -> Bool { lostTransactions[connectionId]?.remove(database) != nil } + + // A connection with one driver for every database lost the transaction whichever name it was noted under. + internal func takeAnyTransactionLoss(for connectionId: UUID) -> String? { + lostTransactions.removeValue(forKey: connectionId)?.min() + } + + internal func markOpened(_ driver: DatabaseDriver, database: String, for connectionId: UUID, at date: Date = Date()) { + guard parked[connectionId]?[database] === driver else { return } + opened[connectionId, default: [:]][database] = Opened(driver: driver, lastUsed: date) + } + + internal func isOpened(database: String, for connectionId: UUID) -> Bool { + opened[connectionId]?[database] != nil + } + + internal func noteOpenedUse(database: String, for connectionId: UUID, at date: Date = Date()) { + opened[connectionId]?[database]?.lastUsed = date + } + + /// Least recently used first. + internal func openedLanes(for connectionId: UUID) -> [(database: String, driver: DatabaseDriver, lastUsed: Date)] { + (opened[connectionId] ?? [:]) + .compactMap { database, entry in + guard let driver = parked[connectionId]?[database], entry.driver === driver else { return nil } + return (database, driver, entry.lastUsed) + } + .sorted { $0.lastUsed < $1.lastUsed } + } + + internal func closeIdle(database: String, for connectionId: UUID) { + close(database: database, for: connectionId, supersedingOpens: false) + closedIdle[connectionId, default: []].insert(database) + } + + internal func wasClosedIdle(database: String, for connectionId: UUID) -> Bool { + closedIdle[connectionId]?.contains(database) ?? false + } } diff --git a/apple/TablePro/Core/Database/TriggerEditing.swift b/apple/TablePro/Core/Database/TriggerEditing.swift index 0aca0f9356..c3744ad7e4 100644 --- a/apple/TablePro/Core/Database/TriggerEditing.swift +++ b/apple/TablePro/Core/Database/TriggerEditing.swift @@ -16,14 +16,91 @@ enum TriggerEditingError: LocalizedError { case notConnected case denied(String) case dropUnavailable + case notCreateTrigger + case definitionUnavailable + case originalDropped(reason: String, definition: String) var errorDescription: String? { switch self { case .notConnected: String(localized: "Not connected to database") case let .denied(reason): reason case .dropUnavailable: String(localized: "This database cannot drop triggers") + case .notCreateTrigger: + String( + localized: """ + The text has no CREATE TRIGGER statement. Saving it would drop the trigger without \ + creating it again, so nothing was run. + """ + ) + case .definitionUnavailable: + String( + localized: """ + The trigger's full definition could not be read, so it could not be restored if the \ + edit failed. Nothing was run. + """ + ) + case let .originalDropped(reason, _): + String( + format: String( + localized: """ + The trigger was dropped, the new definition failed, and the original could not be \ + created again. The trigger no longer exists. Copy the original definition to \ + recreate it. Error: %@ + """ + ), + reason + ) + } + } + + var droppedDefinition: String? { + if case let .originalDropped(_, definition) = self { return definition } + return nil + } +} + +enum TriggerStatementShape { + private static let objectKinds: Set = [ + "TRIGGER", "TABLE", "VIEW", "PROCEDURE", "FUNCTION", "INDEX", "EVENT", "DATABASE", "SCHEMA", + "SEQUENCE", "TYPE", "ROLE", "USER", "PACKAGE" + ] + private static let headerReach = 8 + + static func createsTrigger(_ sql: String, databaseType: DatabaseType) -> Bool { + let code = SQLCodeProjection.code( + of: sql, + grammar: databaseType.lexicalGrammar, + revealingExecutableComments: true + ) + let words = code + .split { !($0.isLetter || $0.isNumber || $0 == "_") } + .map { $0.uppercased() } + .filter { word in !word.allSatisfy(\.isNumber) } + for (index, word) in words.enumerated() where word == "CREATE" { + let header = words[(index + 1)...].prefix(headerReach) + if header.first(where: { objectKinds.contains($0) }) == "TRIGGER" { return true } + } + return false + } +} + +struct TriggerApplyPlan: Equatable, Sendable { + let strategy: TriggerApplyStrategy + let dropSQL: String? + + var dropsFirst: Bool { + switch strategy { + case .transactional(let dropFirst): dropFirst + case .dropThenCreate: true + case .direct: false } } + + // What the gate previews, in the order they run. + func gatedStatements(for sql: String) -> [String] { + guard dropsFirst, let dropSQL else { return [] } + return [dropSQL, sql] + } } enum TriggerApplyStrategy: Equatable { @@ -57,6 +134,20 @@ enum TriggerEditing { originalDefinition: String?, gate: any ExecutionGate = ExecutionGateProvider.shared ) async throws { + let plan = try await DatabaseManager.shared.withMetadataDriver(scope: scope) { driver in + TriggerApplyPlan( + strategy: TriggerApplyStrategy.resolve( + isEdit: isEdit, + usesReplace: driver.triggerEditUsesReplace, + transactionalDDL: driver.supportsTransactionalDDL + ), + dropSQL: originalName.flatMap { driver.generateDropTriggerSQL(name: $0, table: tableName) } + ) + } + let restore = try restoreDefinition( + for: plan, sql: sql, originalDefinition: originalDefinition, databaseType: connection.type + ) + let decision = await gate.authorize( OperationRequest( connectionId: connection.id, @@ -67,7 +158,8 @@ enum TriggerEditing { capabilities: .interactiveUser, operationDescription: isEdit ? String(localized: "Edit Trigger") - : String(localized: "Create Trigger") + : String(localized: "Create Trigger"), + statements: plan.gatedStatements(for: sql) ) ) guard case .authorized = decision else { @@ -80,30 +172,57 @@ enum TriggerEditing { .changed(CatalogChange(connectionId: connection.id, database: scope.database, kinds: .triggers)) ) } - try await withSchemaChangeDriver(scope: scope) { driver in - let strategy = TriggerApplyStrategy.resolve( - isEdit: isEdit, - usesReplace: driver.triggerEditUsesReplace, - transactionalDDL: driver.supportsTransactionalDDL - ) - let dropSQL = originalName.flatMap { driver.generateDropTriggerSQL(name: $0, table: tableName) } - switch strategy { - case let .transactional(dropFirst): - try await runInTransaction(driver: driver, dropSQL: dropFirst ? dropSQL : nil, sql: sql) - case .dropThenCreate: - guard let dropSQL else { throw TriggerEditingError.dropUnavailable } - try await runDropThenCreate(driver: driver, dropSQL: dropSQL, sql: sql, rollback: originalDefinition) - case .direct: - _ = try await driver.execute(query: sql) + do { + try await withSchemaChangeDriver(scope: scope) { driver in + switch plan.strategy { + case let .transactional(dropFirst): + try await runInTransaction(driver: driver, dropSQL: dropFirst ? plan.dropSQL : nil, sql: sql) + case .dropThenCreate: + guard let dropSQL = plan.dropSQL, let restore else { throw TriggerEditingError.dropUnavailable } + try await runDropThenCreate(driver: driver, dropSQL: dropSQL, sql: sql, rollback: restore) + case .direct: + _ = try await driver.execute(query: sql) + } + } + } catch let error as TriggerEditingError { + if error.droppedDefinition != nil { + AppCommands.shared.objectChanged.send( + DatabaseObjectChange(connectionId: connection.id, scope: scope, name: tableName, kind: .structure) + ) } + throw error } + if plan.dropsFirst, let dropSQL = plan.dropSQL { + await recordHistory(dropSQL, scope: scope, connection: connection, executionTime: 0) + } await recordHistory(sql, scope: scope, connection: connection, executionTime: Date().timeIntervalSince(startedAt)) AppCommands.shared.objectChanged.send( DatabaseObjectChange(connectionId: connection.id, scope: scope, name: tableName, kind: .structure) ) } + // Refuses, before the gate and before any statement, an edit that would drop the trigger and leave nothing. + nonisolated static func restoreDefinition( + for plan: TriggerApplyPlan, + sql: String, + originalDefinition: String?, + databaseType: DatabaseType + ) throws -> String? { + guard plan.dropsFirst else { return nil } + guard plan.dropSQL != nil else { throw TriggerEditingError.dropUnavailable } + guard TriggerStatementShape.createsTrigger(sql, databaseType: databaseType) else { + throw TriggerEditingError.notCreateTrigger + } + guard plan.strategy == .dropThenCreate else { return nil } + guard let originalDefinition, + TriggerStatementShape.createsTrigger(originalDefinition, databaseType: databaseType) + else { + throw TriggerEditingError.definitionUnavailable + } + return originalDefinition + } + static func drop( scope: DatabaseScope, connection: DatabaseConnection, @@ -174,24 +293,23 @@ enum TriggerEditing { } } - static func runDropThenCreate(driver: DatabaseDriver, dropSQL: String, sql: String, rollback: String?) async throws { + static func runDropThenCreate(driver: DatabaseDriver, dropSQL: String, sql: String, rollback: String) async throws { _ = try await driver.execute(query: dropSQL) do { _ = try await driver.execute(query: sql) } catch { - if let rollback { - do { - _ = try await driver.execute(query: rollback) - logger.error("Trigger edit failed; restored original definition: \(error.publicLogShape, privacy: .public)") - } catch let rollbackError { - logger.error( - """ - Trigger edit failed and rollback failed, trigger may be missing: \ - edit=\(error.publicLogShape, privacy: .public) \ - rollback=\(rollbackError.publicLogShape, privacy: .public) - """ - ) - } + do { + _ = try await driver.execute(query: rollback) + logger.error("Trigger edit failed; restored original definition: \(error.publicLogShape, privacy: .public)") + } catch let rollbackError { + logger.error( + """ + Trigger edit failed and rollback failed, trigger is missing: \ + edit=\(error.publicLogShape, privacy: .public) \ + rollback=\(rollbackError.publicLogShape, privacy: .public) + """ + ) + throw TriggerEditingError.originalDropped(reason: error.localizedDescription, definition: rollback) } throw error } diff --git a/apple/TablePro/Core/DatabaseSearch/DatabaseSearchReader.swift b/apple/TablePro/Core/DatabaseSearch/DatabaseSearchReader.swift index ab47a6daac..4e23754acf 100644 --- a/apple/TablePro/Core/DatabaseSearch/DatabaseSearchReader.swift +++ b/apple/TablePro/Core/DatabaseSearch/DatabaseSearchReader.swift @@ -59,7 +59,9 @@ internal final class DatabaseSearchLiveReader: DatabaseSearchReading { let engine = DatabaseSearchColumnPlanner.Engine(databaseType: databaseType, dialect: dialect) let rowCap = Self.rowCap let scope = DatabaseScope(connectionId: connection.id, database: database, schema: target.schema) - return try await manager.withCancellableRead(scope: scope, route: manager.metadataRoute(for: scope)) { driver in + return try await manager.withCancellableRead( + scope: scope, route: manager.metadataRoute(for: scope), ownsSessionState: false + ) { driver in let classifier = ColumnTypeClassifier(family: family) let columns = try await driver.fetchColumns(table: target.name, schema: target.schema).map { info in let type = classifier.classify(rawTypeName: info.typeNameForClassification) diff --git a/apple/TablePro/Core/MCP/Completions/MCPCompletionProvider.swift b/apple/TablePro/Core/MCP/Completions/MCPCompletionProvider.swift index cbd8d460e4..5a8c62e32d 100644 --- a/apple/TablePro/Core/MCP/Completions/MCPCompletionProvider.swift +++ b/apple/TablePro/Core/MCP/Completions/MCPCompletionProvider.swift @@ -148,6 +148,8 @@ public actor MCPCompletionProvider { guard case .resolved(let connection) = MCPConnectionReferenceMatcher.resolve(reference, in: connections) else { return nil } + // Asked on every request and ahead of the name cache, so a denial or a revoked grant applies at once. + guard await source.isApproved(principal: principal, connectionId: connection.id) else { return nil } return connection } diff --git a/apple/TablePro/Core/MCP/Completions/MCPCompletionSchemaSource.swift b/apple/TablePro/Core/MCP/Completions/MCPCompletionSchemaSource.swift index 00463a621a..54326daedd 100644 --- a/apple/TablePro/Core/MCP/Completions/MCPCompletionSchemaSource.swift +++ b/apple/TablePro/Core/MCP/Completions/MCPCompletionSchemaSource.swift @@ -54,6 +54,7 @@ public enum MCPConnectionReferenceMatcher { public protocol MCPCompletionSchemaSource: Sendable { func connections(principal: MCPPrincipal) async -> [MCPConnectionDescriptor] + func isApproved(principal: MCPPrincipal, connectionId: UUID) async -> Bool func databases(connectionId: UUID) async -> [String] func schemas(connectionId: UUID, database: String?) async -> [String] func tables(connectionId: UUID, database: String?, schema: String?) async -> [String] @@ -82,6 +83,10 @@ public struct MCPBridgeCompletionSchemaSource: MCPCompletionSchemaSource { } } + public func isApproved(principal: MCPPrincipal, connectionId: UUID) async -> Bool { + await services.authPolicy.isAllowedWithoutPrompt(principal: principal, connectionId: connectionId) + } + public func databases(connectionId: UUID) async -> [String] { guard await isConnected(connectionId) else { return [] } guard let payload = try? await services.connectionBridge.listDatabases(connectionId: connectionId) else { diff --git a/apple/TablePro/Core/MCP/MCPAuthPolicy.swift b/apple/TablePro/Core/MCP/MCPAuthPolicy.swift index 6f55c5bbe0..1a0e67e84e 100644 --- a/apple/TablePro/Core/MCP/MCPAuthPolicy.swift +++ b/apple/TablePro/Core/MCP/MCPAuthPolicy.swift @@ -164,6 +164,14 @@ public actor MCPAuthPolicy { return readable } + // For a caller that cannot wait on a prompt: anything short of an outright allow is a no. + func isAllowedWithoutPrompt(principal: MCPPrincipal, connectionId: UUID) async -> Bool { + guard let decision = try? await authorize(principal: principal, tool: "", connectionId: connectionId), + case .allowed = decision + else { return false } + return true + } + func resolveAndAuthorize( principal: MCPPrincipal, tool: MCPToolName, diff --git a/apple/TablePro/Core/MCP/MCPTokenStore.swift b/apple/TablePro/Core/MCP/MCPTokenStore.swift index 4a01a9a1f3..8ca297d81a 100644 --- a/apple/TablePro/Core/MCP/MCPTokenStore.swift +++ b/apple/TablePro/Core/MCP/MCPTokenStore.swift @@ -8,6 +8,7 @@ import TableProUtilities enum MCPTokenStoreError: Error, Sendable, Equatable { case randomGenerationFailed(OSStatus) case keychainUnavailable(OSStatus) + case storedTokensUnreadable var message: String { switch self { @@ -15,6 +16,8 @@ enum MCPTokenStoreError: Error, Sendable, Equatable { return "Secure random generation failed (OSStatus \(status))" case .keychainUnavailable(let status): return "Keychain unavailable (OSStatus \(status))" + case .storedTokensUnreadable: + return "The saved MCP tokens could not be read, so a new token cannot be saved" } } } @@ -172,6 +175,10 @@ actor MCPTokenStore { private static let logger = Logger(subsystem: "com.TablePro", category: "MCPTokenStore") private var tokens: [MCPAuthToken] = [] + // Stored entries this build cannot decode, kept as JSON so a save writes them back. + private var undecodedEntries: [Data] = [] + // Set while the stored list could not be read: saving then would replace tokens nobody loaded. + private var storedTokensUnread = false private let credentialStore: any MCPTokenCredentialStoring private let legacyStorageUrl: URL private var lastSavedAt: ContinuousClock.Instant = .now @@ -210,6 +217,11 @@ actor MCPTokenStore { let name = (!isBridgeCredential && name == Self.stdioBridgeTokenName) ? name + " (client)" : name + // The bridge credential is dropped at every load, so it alone may live in memory only. + if !isBridgeCredential, !adoptStoredTokensIfUnread() { + Self.logger.error("Refused to mint an MCP token: the stored list is still unreadable") + throw MCPTokenStoreError.storedTokensUnreadable + } let plaintext = "tp_" + Self.base64UrlEncode(try Self.randomBytes(count: 32)) let saltBase64 = try Self.randomBytes(count: 16).base64EncodedString() let hash = Self.computeHash(salt: saltBase64, plaintext: plaintext) @@ -239,6 +251,7 @@ actor MCPTokenStore { } func resolve(bearerToken: String) -> Result { + adoptStoredTokensIfUnread() let match = tokens.firstIndex { token in Self.constantTimeCompare(Self.computeHash(salt: token.salt, plaintext: bearerToken), token.tokenHash) } @@ -307,28 +320,89 @@ actor MCPTokenStore { } func loadFromDisk() { - tokens = readCredentialStore() ?? migrateLegacyFileIfPresent() - - let isStaleBridge: (MCPAuthToken) -> Bool = { - $0.isBridgeCredential || $0.name == Self.stdioBridgeTokenName + undecodedEntries = [] + switch credentialStore.read() { + case .found(let data): + storedTokensUnread = false + let stored = Self.decodeStored(data) + tokens = stored.tokens + undecodedEntries = stored.undecoded + Self.logger.info("Loaded \(stored.tokens.count) MCP tokens from the keychain") + case .missing: + storedTokensUnread = false + tokens = migrateLegacyFileIfPresent() + case .unreadable: + storedTokensUnread = true + tokens = [] + Self.logger.error("MCP tokens could not be read; keeping new tokens in memory until they can") + return } - let staleCount = tokens.filter(isStaleBridge).count + + let staleCount = tokens.filter(Self.isStaleBridge).count guard staleCount > 0 else { return } - tokens.removeAll(where: isStaleBridge) + tokens.removeAll(where: Self.isStaleBridge) save() Self.logger.info("Cleaned up \(staleCount) stale bridge token(s)") } - private func readCredentialStore() -> [MCPAuthToken]? { - guard let data = credentialStore.read() else { return nil } - do { - let decoded = try Self.decoder.decode([MCPAuthToken].self, from: data) - Self.logger.info("Loaded \(decoded.count) MCP tokens from the keychain") - return decoded - } catch { - Self.logger.error("Failed to decode MCP tokens: \(error.publicLogShape, privacy: .public)") - return [] + private static func isStaleBridge(_ token: MCPAuthToken) -> Bool { + token.isBridgeCredential || token.name == stdioBridgeTokenName + } + + // One entry this build cannot decode costs that entry only, and it is written back untouched. + static func decodeStored(_ data: Data) -> (tokens: [MCPAuthToken], undecoded: [Data]) { + guard let entries = try? JSONSerialization.jsonObject(with: data) as? [Any] else { + logger.error("Stored MCP tokens are not a JSON list; nothing in them can be recovered") + return ([], []) + } + var tokens: [MCPAuthToken] = [] + var undecoded: [Data] = [] + for entry in entries { + guard let fields = entry as? [String: Any], + let entryData = try? JSONSerialization.data(withJSONObject: fields, options: [.sortedKeys]) + else { continue } + do { + tokens.append(try decoder.decode(MCPAuthToken.self, from: entryData)) + } catch { + logger.error("Kept an MCP token this build cannot decode: \(error.publicLogShape, privacy: .public)") + if fields["isBridgeCredential"] as? Bool != true { + undecoded.append(entryData) + } + } + } + return (tokens, undecoded) + } + + static func encodeStored(tokens: [MCPAuthToken], undecoded: [Data]) throws -> Data { + let data = try encoder.encode(tokens) + guard !undecoded.isEmpty else { return data } + var entries = try JSONSerialization.jsonObject(with: data) as? [Any] ?? [] + for entry in undecoded { + entries.append(try JSONSerialization.jsonObject(with: entry)) + } + return try JSONSerialization.data(withJSONObject: entries, options: [.sortedKeys]) + } + + // Returns false while the stored list is still unreadable. Once it reads, the tokens minted in + // the meantime are merged over it, so neither side is lost. + @discardableResult + private func adoptStoredTokensIfUnread() -> Bool { + guard storedTokensUnread else { return true } + let stored: (tokens: [MCPAuthToken], undecoded: [Data]) + switch credentialStore.read() { + case .unreadable: + return false + case .missing: + stored = ([], []) + case .found(let data): + stored = Self.decodeStored(data) } + storedTokensUnread = false + let minted = Set(tokens.map(\.id)) + tokens = stored.tokens.filter { !minted.contains($0.id) && !Self.isStaleBridge($0) } + tokens + undecodedEntries = stored.undecoded + Self.logger.info("Read the stored MCP tokens after an earlier failure") + return true } private func migrateLegacyFileIfPresent() -> [MCPAuthToken] { @@ -358,8 +432,12 @@ actor MCPTokenStore { private func save() { lastSavedAt = .now + guard adoptStoredTokensIfUnread() else { + Self.logger.error("MCP tokens not saved: the stored list is still unreadable") + return + } do { - let data = try Self.encoder.encode(tokens) + let data = try Self.encodeStored(tokens: tokens, undecoded: undecodedEntries) guard credentialStore.write(data) else { Self.logger.error("Failed to write MCP tokens to the keychain") return @@ -419,8 +497,14 @@ actor MCPTokenStore { } } +enum MCPTokenBlob: Sendable, Equatable { + case found(Data) + case missing + case unreadable +} + protocol MCPTokenCredentialStoring: Sendable { - func read() -> Data? + func read() -> MCPTokenBlob @discardableResult func write(_ data: Data) -> Bool func delete() @@ -436,12 +520,17 @@ struct MCPTokenKeychainStore: MCPTokenCredentialStoring { AppStorageEnvironment.shared.isIsolated ? AppStorageEnvironment.shared.keychain : nil } - func read() -> Data? { + func read() -> MCPTokenBlob { if let isolatedStore = Self.isolatedStore { - guard case .found(let encoded) = isolatedStore.readStringResult(forKey: Self.isolatedKey) else { - return nil + switch isolatedStore.readStringResult(forKey: Self.isolatedKey) { + case .found(let encoded): + // A value that is not base64 never becomes readable; treat it like any other corrupt blob. + return .found(Data(base64Encoded: encoded) ?? Data()) + case .notFound: + return .missing + case .locked, .userCancelled, .authFailed, .error: + return .unreadable } - return Data(base64Encoded: encoded) } var query = Self.baseQuery() @@ -450,13 +539,12 @@ struct MCPTokenKeychainStore: MCPTokenCredentialStoring { var result: AnyObject? let status = SecItemCopyMatching(query as CFDictionary, &result) - guard status == errSecSuccess else { - if status != errSecItemNotFound { - Self.logger.error("Keychain read failed (OSStatus \(status, privacy: .public))") - } - return nil + if status == errSecItemNotFound { return .missing } + guard status == errSecSuccess, let data = result as? Data else { + Self.logger.error("Keychain read failed (OSStatus \(status, privacy: .public))") + return .unreadable } - return result as? Data + return .found(data) } @discardableResult diff --git a/apple/TablePro/Core/MCP/Outside/MCPRemoteToolCoordinator.swift b/apple/TablePro/Core/MCP/Outside/MCPRemoteToolCoordinator.swift index 94b97a1681..f27f0b181b 100644 --- a/apple/TablePro/Core/MCP/Outside/MCPRemoteToolCoordinator.swift +++ b/apple/TablePro/Core/MCP/Outside/MCPRemoteToolCoordinator.swift @@ -95,6 +95,16 @@ internal final class MCPRemoteToolCoordinator { let adapter = MCPRemoteToolAdapter(server: configuration, tool: tool) { remoteName, arguments in try await client.callTool(name: remoteName, arguments: arguments) } + // Two remote tools under one name would leave the model calling whichever registered last. + guard !registered.contains(adapter.name) else { + Self.logger.error( + """ + Skipped a tool of MCP server \(configuration.id, privacy: .public): \ + its name is already taken by another tool of that server + """ + ) + continue + } guard registry.register(adapter) else { continue } registered.insert(adapter.name) } diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift index 76352f37fc..8e0083c5fb 100644 --- a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift @@ -30,6 +30,8 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor /// Read from the driver when its connect is adopted, so it outlives the `disconnect()` Test Connection /// makes before anyone asks for it. var connectionWarning: String? + // Only a yes is kept, so the driver's own code is not asked again on every read. + var buildsBrowseQueries = false } let connection: DatabaseConnection @@ -60,17 +62,20 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor var schemaPluginDriver: any PluginDatabaseDriver { pluginDriver } var queryBuildingPluginDriver: (any PluginDatabaseDriver)? { - // Expose plugin driver for query building dispatch if it implements the hooks. - // SQL drivers without custom pagination (MySQL, PostgreSQL, etc.) return nil - // from buildBrowseQuery and use standard SQL query rewriting instead. - guard pluginDriver.buildBrowseQuery( - table: "_probe", sortColumns: [], columns: [], limit: 1, offset: 0 - ) != nil else { - return nil - } + if state.withLock({ $0.buildsBrowseQueries }) { return pluginDriver } + guard Self.buildsBrowseQueries(pluginDriver) else { return nil } + state.withLock { $0.buildsBrowseQueries = true } return pluginDriver } + // Asks the schema-aware overload: its default forwards to the older one, so it answers for a driver + // that implements either. A driver that leaves browsing to the app returns nil from both. + static func buildsBrowseQueries(_ driver: any PluginDatabaseDriver) -> Bool { + driver.buildBrowseQuery( + table: "_probe", schema: nil, sortColumns: [], columns: [], limit: 1, offset: 0 + ) != nil + } + var rowWritingPluginDriver: (any PluginDatabaseDriver)? { pluginDriver.capabilities.contains(.rowWrites) ? pluginDriver : queryBuildingPluginDriver } diff --git a/apple/TablePro/Core/Plugins/PluginManager+Registration.swift b/apple/TablePro/Core/Plugins/PluginManager+Registration.swift index 8a17ae7da9..3d53726531 100644 --- a/apple/TablePro/Core/Plugins/PluginManager+Registration.swift +++ b/apple/TablePro/Core/Plugins/PluginManager+Registration.swift @@ -290,9 +290,7 @@ extension PluginManager { } let config = DriverConnectionConfig(host: "", port: 0, username: "", password: "", database: "") let driver = plugin.createDriver(config: config) - let result: (any PluginDatabaseDriver)? = - driver.buildBrowseQuery(table: "_probe", sortColumns: [], columns: [], limit: 1, offset: 0) != nil - ? driver : nil + let result: (any PluginDatabaseDriver)? = PluginDriverAdapter.buildsBrowseQueries(driver) ? driver : nil if hasFinishedInitialLoad { queryBuildingDriverCache[typeId] = .some(result) } diff --git a/apple/TablePro/Core/SchemaTracking/StructureChangeManager.swift b/apple/TablePro/Core/SchemaTracking/StructureChangeManager.swift index dc271f8c80..87b4c54dc8 100644 --- a/apple/TablePro/Core/SchemaTracking/StructureChangeManager.swift +++ b/apple/TablePro/Core/SchemaTracking/StructureChangeManager.swift @@ -888,6 +888,81 @@ final class StructureChangeManager: ObservableObject, ChangeManaging { currentPrimaryKey = workingPrimaryKey } + // MARK: - Rebase + + // After a save that stopped part way on an engine that commits each DDL statement: what the read shows as done + // is dropped, the rest is staged against the table as it is. False, touching nothing, when the baseline held. + @discardableResult + func rebaseStagedChanges(onto baseline: StructureRebase.Baseline) -> Bool { + guard !isHeldForSave else { return false } + endTableCommentRun() + endTypingRun() + + let rebasedColumns = StructureRebase.entities( + onto: baseline.columns, in: self, using: Self.columnOperations, \.name, \.id + ) + let rebasedIndexes = StructureRebase.entities( + onto: baseline.indexes, in: self, using: Self.indexOperations, \.name, \.id + ) + let rebasedForeignKeys = StructureRebase.entities( + onto: baseline.foreignKeys, in: self, using: Self.foreignKeyOperations, \.name, \.id + ) + let rebasedCheckConstraints = StructureRebase.entities( + onto: baseline.checkConstraints, in: self, using: Self.checkConstraintOperations, \.name, \.id + ) + + let baselineMoved = rebasedColumns.current != currentColumns + || rebasedIndexes.current != currentIndexes + || rebasedForeignKeys.current != currentForeignKeys + || rebasedCheckConstraints.current != currentCheckConstraints + || baseline.primaryKey != currentPrimaryKey + || baseline.comment.original != tableComment.original + || baseline.options.original != tableOptions.original + guard baselineMoved else { return false } + + currentColumns = rebasedColumns.current + workingColumns = StructureRebase.keepingKeyFlags(of: rebasedColumns.working, from: workingColumns) + currentIndexes = rebasedIndexes.current + workingIndexes = rebasedIndexes.working + currentForeignKeys = rebasedForeignKeys.current + workingForeignKeys = rebasedForeignKeys.working + currentCheckConstraints = rebasedCheckConstraints.current + workingCheckConstraints = rebasedCheckConstraints.working + currentPrimaryKey = baseline.primaryKey + adopt(rebasedColumns) + adopt(rebasedIndexes) + adopt(rebasedForeignKeys) + adopt(rebasedCheckConstraints) + + let stagedComment = tableComment.hasChanges ? tableComment.text : nil + tableComment = baseline.comment + if let stagedComment { tableComment.text = stagedComment } + tableCommentRunStart = nil + restageTableComment() + + let stagedOptions = tableOptions.hasChanges ? tableOptions.current : nil + tableOptions = baseline.options + if let stagedOptions, baseline.options.original != nil { tableOptions.current = stagedOptions } + restageTableOptions() + + indexesAddedClustered = indexesAddedClustered.filter { id in !currentIndexes.contains { $0.id == id } } + changeOrder.removeAll { pendingChanges[$0] == nil } + undoManager.removeAllActions() + workingCopyDidChange() + reloadVersion += 1 + return true + } + + private func adopt(_ rebased: StructureRebase.Entities) { + for key in rebased.staleKeys { + pendingChanges.removeValue(forKey: key) + } + for staged in rebased.changes { + pendingChanges[staged.key] = staged.change + trackChangeKey(staged.key) + } + } + // MARK: - Undo/Redo Operations func undo() { diff --git a/apple/TablePro/Core/SchemaTracking/StructureRebase.swift b/apple/TablePro/Core/SchemaTracking/StructureRebase.swift new file mode 100644 index 0000000000..9134449a8f --- /dev/null +++ b/apple/TablePro/Core/SchemaTracking/StructureRebase.swift @@ -0,0 +1,167 @@ +// +// StructureRebase.swift +// TablePro +// + +import Foundation + +// Matches one kind of structure definition, as read after a save that stopped part way, to the staged edits. +@MainActor +enum StructureRebase { + struct Entities { + let current: [Entity] + let working: [Entity] + let staleKeys: [SchemaChangeIdentifier] + let changes: [(key: SchemaChangeIdentifier, change: SchemaChange)] + } + + struct Baseline { + let columns: [EditableColumnDefinition] + let indexes: [EditableIndexDefinition] + let foreignKeys: [EditableForeignKeyDefinition] + let checkConstraints: [EditableCheckConstraintDefinition] + let primaryKey: [String] + let comment: ObjectCommentDraft + let options: TableOptionsDraft + + init( + columns: [ColumnInfo], + indexes: [IndexInfo], + foreignKeys: [ForeignKeyInfo], + checkConstraints: [CheckConstraintInfo] = [], + primaryKey: [String], + tableComment: String? = nil, + tableOptions: TableOptions? = nil, + autoIncrementConfirmed: Bool = true + ) { + var loaded = columns.map { EditableColumnDefinition.from($0) } + if !primaryKey.isEmpty { + for index in loaded.indices { + loaded[index].isPrimaryKey = primaryKey.contains(loaded[index].name) + } + } + self.columns = loaded + self.indexes = indexes.map { EditableIndexDefinition.from($0) } + self.foreignKeys = EditableForeignKeyDefinition.grouping(foreignKeys).sorted { $0.name < $1.name } + self.checkConstraints = checkConstraints.map { EditableCheckConstraintDefinition.from($0) } + self.primaryKey = primaryKey + comment = ObjectCommentDraft(original: tableComment) + options = TableOptionsDraft(original: tableOptions, isAutoIncrementConfirmed: autoIncrementConfirmed) + } + } + + // A column's key flag is the table's key change, so the one the user set outlives the adopted definition. + static func keepingKeyFlags( + of rebased: [EditableColumnDefinition], + from working: [EditableColumnDefinition] + ) -> [EditableColumnDefinition] { + let flags = Dictionary(working.map { ($0.id, $0.isPrimaryKey) }, uniquingKeysWith: { first, _ in first }) + return rebased.map { column in + guard let flag = flags[column.id] else { return column } + var kept = column + kept.isPrimaryKey = flag + return kept + } + } + + static func entities( + onto loaded: [Entity], + in manager: StructureChangeManager, + using operations: SchemaEntityOperations, + _ name: KeyPath, + _ id: WritableKeyPath + ) -> Entities { + let working = manager[keyPath: operations.working] + let deleted = working.map(\.id).filter { manager.pendingChanges[operations.identifier($0)]?.isDelete == true } + return entities( + loaded: loaded, + earlier: manager[keyPath: operations.current], + working: working, + deleted: Set(deleted), + using: operations, + name, + id + ) + } + + static func entities( + loaded: [Entity], + earlier: [Entity], + working: [Entity], + deleted: Set, + using operations: SchemaEntityOperations, + _ name: KeyPath, + _ id: WritableKeyPath + ) -> Entities { + // The name a definition has after the save, then the one it had before: a rename that ran is found under + // the first, one that did not under the second. + var idsAfterSave: [String: UUID] = [:] + for entity in working where !deleted.contains(entity.id) { + let entityName = entity[keyPath: name] + if !entityName.isEmpty, idsAfterSave[entityName] == nil { idsAfterSave[entityName] = entity.id } + } + var idsBeforeSave: [String: UUID] = [:] + for entity in earlier { + let entityName = entity[keyPath: name] + if !entityName.isEmpty, idsBeforeSave[entityName] == nil { idsBeforeSave[entityName] = entity.id } + } + var used = Set() + let current: [Entity] = loaded.map { entity in + let entityName = entity[keyPath: name] + var candidates = [idsAfterSave[entityName], idsBeforeSave[entityName]].compactMap { $0 } + if entityName.isEmpty { + // A definition with no name is the one it was when nothing else about it changed. + candidates = earlier.filter { held in + var probe = entity + probe[keyPath: id] = held.id + return probe == held + } + .map(\.id) + } + guard let kept = candidates.first(where: { !used.contains($0) }) else { return entity } + used.insert(kept) + var matched = entity + matched[keyPath: id] = kept + return matched + } + + let currentByID = Dictionary(current.map { ($0.id, $0) }, uniquingKeysWith: { first, _ in first }) + let earlierByID = Dictionary(earlier.map { ($0.id, $0) }, uniquingKeysWith: { first, _ in first }) + var rebasedWorking: [Entity] = [] + var changes: [(key: SchemaChangeIdentifier, change: SchemaChange)] = [] + for entity in working { + let key = operations.identifier(entity.id) + guard let baseline = currentByID[entity.id] else { + // A deletion that ran leaves nothing to show; anything else the table lacks is still to be added. + guard !deleted.contains(entity.id) else { continue } + rebasedWorking.append(entity) + changes.append((key: key, change: operations.addition(entity))) + continue + } + if deleted.contains(entity.id) { + rebasedWorking.append(entity) + changes.append((key: key, change: operations.deletion(baseline))) + continue + } + // An addition the table now has ran. An edit is still owed only where the table differs from it. + guard let before = earlierByID[entity.id], + operations.needsStatement(before, entity), + operations.needsStatement(baseline, entity) + else { + rebasedWorking.append(baseline) + continue + } + rebasedWorking.append(entity) + changes.append((key: key, change: operations.modification(baseline, entity))) + } + let shown = Set(rebasedWorking.map(\.id)) + rebasedWorking.append(contentsOf: current.filter { !shown.contains($0.id) }) + + return Entities( + current: current, + working: rebasedWorking, + staleKeys: (working + earlier).map { operations.identifier($0.id) }, + changes: changes + ) + } +} diff --git a/apple/TablePro/Core/Services/Formatting/SQLTokenizer.swift b/apple/TablePro/Core/Services/Formatting/SQLTokenizer.swift index 2debcf7357..5c9c02383c 100644 --- a/apple/TablePro/Core/Services/Formatting/SQLTokenizer.swift +++ b/apple/TablePro/Core/Services/Formatting/SQLTokenizer.swift @@ -396,6 +396,12 @@ private struct Lexer { readWord() return } + if let end = SQLOperatorRun.end(at: index, in: text, grammar: grammar), end - index > 1 { + let start = index + index = end + emit(.operator, from: start) + return + } if readPrefixedName(unit) { return } readOperatorOrPunctuation(unit) } @@ -420,6 +426,11 @@ private struct Lexer { emit(.placeholder, from: start) case .quoted: let opener = text.character(at: start) + if span.isTerminated, grammar.isQuote(opener), let end = accountHostEnd(at: index) { + index = end + emit(.identifier, from: start) + return + } emit(opener == Self.backtick || opener == Self.openBracket ? .identifier : .string, from: start) } } @@ -526,6 +537,11 @@ private struct Lexer { while index < length, continuesWord(at: index) { index += 1 } + if let end = accountHostEnd(at: index) { + index = end + emit(.identifier, from: start) + return + } let word = substring(from: start, to: index) let upper = word.uppercased() if tokenizer.isKeyword(upper) { @@ -559,7 +575,37 @@ private struct Lexer { return span } - // `@name`, `@@version`, `:name`, `$1`, `?1` and T-SQL's `#temp` and `##global` are one token each. + // `user@host` is one name: MySQL's ``'app'@'%'`` and `` `root`@`localhost` `` and Oracle's `emp@remote` take no + // space at the `@`, and a user name that spells a keyword keeps its case. Not where `@` is an operator character. + private func accountHostEnd(at position: Int) -> Int? { + guard !grammar.readsOperatorRuns, position + 1 < length, text.character(at: position) == Self.at else { + return nil + } + let host = position + 1 + if let span = quotedName(at: host) { return span.end } + guard isWordPart(text.character(at: host)), text.character(at: host) != Self.at else { return nil } + var end = host + while end < length { + let isInnerDot = text.character(at: end) == Self.dot && end + 1 < length + && isWordPart(text.character(at: end + 1)) + guard isInnerDot || continuesWord(at: end) else { break } + end += 1 + } + return end + } + + private func quotedName(at position: Int) -> SQLNonCodeSpan.Span? { + guard position < length, grammar.isQuote(text.character(at: position)), + let span = SQLNonCodeSpan.span(at: position, in: text, grammar: grammar), span.kind == .quoted, + span.isTerminated + else { + return nil + } + return span + } + + // `@name`, `@@version`, `:name`, `$1`, `?1` and T-SQL's `#temp` and `##global` are one token each, and so is + // MySQL's quoted user variable, `` @`my var` ``. private mutating func readPrefixedName(_ unit: UInt16) -> Bool { let start = index switch unit { @@ -582,6 +628,11 @@ private struct Lexer { emit(.placeholder, from: start) return true case Self.at, Self.hash: + if unit == Self.at, !grammar.readsOperatorRuns, let span = quotedName(at: index + 1) { + index = span.end + emit(.placeholder, from: start) + return true + } var cursor = index + 1 if cursor < length, text.character(at: cursor) == unit { cursor += 1 diff --git a/apple/TablePro/Core/Services/Infrastructure/ConnectionWorkspace.swift b/apple/TablePro/Core/Services/Infrastructure/ConnectionWorkspace.swift index 4f922750c2..640d218993 100644 --- a/apple/TablePro/Core/Services/Infrastructure/ConnectionWorkspace.swift +++ b/apple/TablePro/Core/Services/Infrastructure/ConnectionWorkspace.swift @@ -404,6 +404,7 @@ internal final class ConnectionWorkspace { sessionState?.coordinator.teardown() sessionState = nil session = nil + MainContentCoordinator.endTabSessionIfUnhosted(for: connectionId, closing: self) undoManager.removeAllActions() agentUndoManager.removeAllActions() /// Once nothing hosts this connection any more, its agent sessions stop: a stream, a tool diff --git a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController.swift b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController.swift index 341d3bd2fe..a651e9a1b2 100644 --- a/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController.swift +++ b/apple/TablePro/Core/Services/Infrastructure/MainSplitViewController.swift @@ -633,7 +633,7 @@ internal final class MainSplitViewController: NSSplitViewController { /// the coordinator itself rather than which workspace is on screen keeps this exact when a /// background workspace loses its session. let releasedCoordinator = workspace.sessionState?.coordinator - workspace.sessionState?.coordinator.teardown() + workspace.sessionState?.coordinator.teardown(endsTabSession: false) workspace.sessionState = nil workspace.session = nil if let releasedCoordinator, toolbarOwner?.coordinator === releasedCoordinator { diff --git a/apple/TablePro/Core/Services/Infrastructure/TabPersistenceCoordinator.swift b/apple/TablePro/Core/Services/Infrastructure/TabPersistenceCoordinator.swift index 1d8cc113dd..72da8fa986 100644 --- a/apple/TablePro/Core/Services/Infrastructure/TabPersistenceCoordinator.swift +++ b/apple/TablePro/Core/Services/Infrastructure/TabPersistenceCoordinator.swift @@ -58,6 +58,12 @@ internal final class TabPersistenceCoordinator: ObservableObject { hasObservedTabs = true } + // The gate is one session's knowledge of the disk. Left open after the connection's last window closed, a + // window later opened on content alone, which skips the restore, would write its one tab over the saved set. + internal func endSession() { + hasObservedTabs = false + } + // MARK: - Save /// An automatic save never deletes, and never runs before a restore has told it what is on diff --git a/apple/TablePro/Core/Services/Infrastructure/TabWindowController.swift b/apple/TablePro/Core/Services/Infrastructure/TabWindowController.swift index 45e2288952..df586931db 100644 --- a/apple/TablePro/Core/Services/Infrastructure/TabWindowController.swift +++ b/apple/TablePro/Core/Services/Infrastructure/TabWindowController.swift @@ -288,6 +288,9 @@ internal final class TabWindowController: NSWindowController, NSWindowDelegate { if let splitVC = window.contentViewController as? MainSplitViewController { for workspace in splitVC.workspaces.workspaces { workspace.sessionState?.coordinator.handleWindowWillClose() + if workspace.sessionState == nil { + MainContentCoordinator.endTabSessionIfUnhosted(for: workspace.connectionId, closing: workspace) + } } } Self.lifecycleLogger.info("[close] windowWillClose seq=\(seq) handleWindowWillClose ms=\(Int(Date().timeIntervalSince(t0) * 1_000))") diff --git a/apple/TablePro/Core/Services/Masking/MaskingExportDataSource.swift b/apple/TablePro/Core/Services/Masking/MaskingExportDataSource.swift index ae2854f08f..667b61b15e 100644 --- a/apple/TablePro/Core/Services/Masking/MaskingExportDataSource.swift +++ b/apple/TablePro/Core/Services/Masking/MaskingExportDataSource.swift @@ -55,41 +55,34 @@ final class MaskingExportDataSource: PluginExportDataSource, @unchecked Sendable table: String? ) -> AsyncThrowingStream { let policy = self.policy - return AsyncThrowingStream { continuation in - let task = Task { - var header: PluginStreamHeader? - do { - for try await element in stream { - switch element { - case .header(let value): - header = value - continuation.yield(element) - case .rows(let rows, let documents): - guard let header else { - throw PluginExportError.exportFailed(String(localized: "Rows arrived before their columns", bundle: .main)) - } - let masks = policy.columnMasks(table: table, columns: header.columns, typeNames: header.columnTypeNames) - if let documents, documents.count != rows.count { - throw PluginExportError.exportFailed(String(localized: "Documents do not match their rows", bundle: .main)) - } - let masked = documents?.enumerated().map { index, document in - ExportDocumentMasking.document( - document, columns: header.columns, masks: masks, - policy: policy, table: table, row: rows[index] - ) - } - continuation.yield(.rows(masks.rows(rows), documents: masked)) - @unknown default: - throw PluginExportError.encodingFailed - } - } - continuation.finish() - } catch { - continuation.finish(throwing: error) + let source = PulledMaskingSource(stream) + let state = MaskingStreamState() + // Read on demand: a task pumping into a second stream kept every row a slow writer had not taken. + return AsyncThrowingStream(unfolding: { + guard let element = try await source.next() else { return nil } + switch element { + case .header(let value): + state.header = value + return element + case .rows(let rows, let documents): + guard let header = state.header else { + throw PluginExportError.exportFailed(String(localized: "Rows arrived before their columns", bundle: .main)) } + let masks = policy.columnMasks(table: table, columns: header.columns, typeNames: header.columnTypeNames) + if let documents, documents.count != rows.count { + throw PluginExportError.exportFailed(String(localized: "Documents do not match their rows", bundle: .main)) + } + let masked = documents?.enumerated().map { index, document in + ExportDocumentMasking.document( + document, columns: header.columns, masks: masks, + policy: policy, table: table, row: rows[index] + ) + } + return .rows(masks.rows(rows), documents: masked) + @unknown default: + throw PluginExportError.encodingFailed } - continuation.onTermination = { _ in task.cancel() } - } + }) } private func sourceTable(_ name: String) -> String? { @@ -105,30 +98,22 @@ final class MaskingExportDataSource: PluginExportDataSource, @unchecked Sendable table: String?, policy: MaskingPolicy ) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - let task = Task { - var masks: ColumnMasks? - do { - for try await element in stream { - switch element { - case .header(let header): - masks = policy.columnMasks(table: table, columns: header.columns, typeNames: header.columnTypeNames) - continuation.yield(element) - case .rows(let rows): - // Without a header there are no column names to match, so no row goes out unmatched. - guard let masks else { - throw PluginExportError.exportFailed(String(localized: "Rows arrived before their columns", bundle: .main)) - } - continuation.yield(.rows(masks.rows(rows))) - } - } - continuation.finish() - } catch { - continuation.finish(throwing: error) + let source = PulledMaskingSource(stream) + let state = MaskingStreamState() + return AsyncThrowingStream(unfolding: { + guard let element = try await source.next() else { return nil } + switch element { + case .header(let header): + state.masks = policy.columnMasks(table: table, columns: header.columns, typeNames: header.columnTypeNames) + return element + case .rows(let rows): + // Without a header there are no column names to match, so no row goes out unmatched. + guard let masks = state.masks else { + throw PluginExportError.exportFailed(String(localized: "Rows arrived before their columns", bundle: .main)) } + return .rows(masks.rows(rows)) } - continuation.onTermination = { _ in task.cancel() } - } + }) } func fetchTableDDL(table: String, databaseName: String) async throws -> String { @@ -278,3 +263,21 @@ enum ExportDocumentMasking { return PluginExportField(name: field.name, canonicalJSON: json, typeName: "VARCHAR", cellValue: value) } } + +// One consumer reads a masked stream, so one call at a time reaches the iterator. +private final class PulledMaskingSource: @unchecked Sendable { + private var iterator: AsyncThrowingStream.AsyncIterator + + init(_ stream: AsyncThrowingStream) { + iterator = stream.makeAsyncIterator() + } + + func next() async throws -> Element? { + try await iterator.next() + } +} + +private final class MaskingStreamState: @unchecked Sendable { + var header: PluginStreamHeader? + var masks: ColumnMasks? +} diff --git a/apple/TablePro/Core/Services/Query/QueryExecutor.swift b/apple/TablePro/Core/Services/Query/QueryExecutor.swift index df02729eb1..13c42a9140 100644 --- a/apple/TablePro/Core/Services/Query/QueryExecutor.swift +++ b/apple/TablePro/Core/Services/Query/QueryExecutor.swift @@ -1,6 +1,7 @@ import Foundation import os import TableProPluginKit +import TableProSQLGrammar private let queryExecutorLog = Logger(subsystem: "com.TablePro", category: "QueryExecutor") @@ -472,9 +473,10 @@ final class QueryExecutor { static func detectAndReconcileParameters( sql: String, - existing: [QueryParameter] + existing: [QueryParameter], + grammar: SQLLexicalGrammar = SQLParameterExtractor.unspecifiedGrammar ) -> [QueryParameter] { - let detectedNames = SQLParameterExtractor.extractParameters(from: sql) + let detectedNames = SQLParameterExtractor.extractParameters(from: sql, grammar: grammar) guard !detectedNames.isEmpty else { return [] } let existingByName = Dictionary( diff --git a/apple/TablePro/Core/Services/Query/RowOperationsManager.swift b/apple/TablePro/Core/Services/Query/RowOperationsManager.swift index 2d6592b141..6fba4619aa 100644 --- a/apple/TablePro/Core/Services/Query/RowOperationsManager.swift +++ b/apple/TablePro/Core/Services/Query/RowOperationsManager.swift @@ -487,7 +487,10 @@ final class RowOperationsManager { into tableRows: inout TableRows ) -> PasteRowsResult { var pastedRowInfo: [PastedRowInfo] = [] - var insertedIndices = IndexSet() + var recorded: [DataChangeManager.InsertedRow] = [] + pastedRowInfo.reserveCapacity(parsedRows.count) + recorded.reserveCapacity(parsedRows.count) + let firstIndex = tableRows.count // Pasted server-owned values skip the cell-edit check and the generator drops them, so the grid showed unsaved values. let serverOwned = tableRows.columns.enumerated().filter { _, name in @@ -508,14 +511,18 @@ final class RowOperationsManager { absentColumns.remove(index) expressionColumns.remove(index) } - insertedIndices.insert(tableRows.count) - let inserted = appendInsertedRow( - values: rowValues, absentColumns: absentColumns, expressionColumns: expressionColumns, to: &tableRows - ) - pastedRowInfo.append(PastedRowInfo(rowID: inserted.rowID, values: rowValues)) + let rowID = RowID.inserted(UUID()) + let storageIndex = tableRows.count + tableRows.appendInsertedRow(id: rowID, values: rowValues, absentColumns: absentColumns) + recorded.append(DataChangeManager.InsertedRow( + location: InsertedRowLocation(rowID: rowID, storageIndex: storageIndex), + values: rowValues, absentColumns: absentColumns, expressionColumns: expressionColumns + )) + pastedRowInfo.append(PastedRowInfo(rowID: rowID, values: rowValues)) } + changeManager.recordBatchRowInsertion(rows: recorded) - let delta: Delta = insertedIndices.isEmpty ? .none : .rowsInserted(insertedIndices) + let delta: Delta = recorded.isEmpty ? .none : .rowsInserted(IndexSet(integersIn: firstIndex.. String? { let trimmed = rawValue.trimmingCharacters(in: .whitespaces) @@ -74,7 +75,7 @@ struct TSVRowParser: RowDataParser { return nil } - return trimmed + return rawValue } } diff --git a/apple/TablePro/Core/Shortcuts/Shared/IntentDataError.swift b/apple/TablePro/Core/Shortcuts/Shared/IntentDataError.swift index 1a9d26d7c7..610da2a87d 100644 --- a/apple/TablePro/Core/Shortcuts/Shared/IntentDataError.swift +++ b/apple/TablePro/Core/Shortcuts/Shared/IntentDataError.swift @@ -16,6 +16,7 @@ nonisolated enum IntentDataError: Error, CustomLocalizedStringResourceConvertibl case jsonArrayContainsNonObject case invalidJSONShape case csvMissingHeader + case csvFieldCountMismatch(record: Int, found: Int, expected: Int) case malformedPayload(String) case tooManyRows(Int) @@ -49,6 +50,8 @@ nonisolated enum IntentDataError: Error, CustomLocalizedStringResourceConvertibl return LocalizedStringResource("Provide a JSON object or an array of objects.") case .csvMissingHeader: return LocalizedStringResource("The CSV data has no header row.") + case .csvFieldCountMismatch(let record, let found, let expected): + return "Record \(record) of the CSV data has \(found) fields, but the header has \(expected)." case .malformedPayload(let message): return "The data could not be read: \(message)" case .tooManyRows(let limit): diff --git a/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift b/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift index 2ff528de24..99fd7d2c56 100644 --- a/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift +++ b/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift @@ -86,29 +86,33 @@ nonisolated enum RowPayload { throw IntentDataError.invalidJSONShape } + // An unquoted empty field is no value, so the column takes its default or NULL. A quoted "" is an empty string. static func parseCSV(_ text: String) throws -> [PayloadRow] { let records = try csvRecords(in: text) - guard let header = records.first, !header.allSatisfy(\.isEmpty) else { + guard let header = records.first?.map({ $0.text }), !header.allSatisfy(\.isEmpty) else { throw IntentDataError.csvMissingHeader } - return records.dropFirst().compactMap { record in - guard !(record.count == 1 && record[0].isEmpty) else { return nil } + return try records.enumerated().dropFirst().compactMap { line, record in + guard !(record.count == 1 && record[0].text.isEmpty && !record[0].isQuoted) else { return nil } + guard record.count == header.count else { + throw IntentDataError.csvFieldCountMismatch(record: line + 1, found: record.count, expected: header.count) + } var values: [String: PayloadValue] = [:] - for (index, column) in header.enumerated() where !column.isEmpty { - let field = index < record.count ? record[index] : "" - values[column] = .text(field) + for (column, field) in zip(header, record) where !column.isEmpty { + guard field.isQuoted || !field.text.isEmpty else { continue } + values[column] = .text(field.text) } return PayloadRow(values: values) } } - private static func csvRecords(in text: String) throws -> [[String]] { + private static func csvRecords(in text: String) throws -> [[(text: String, isQuoted: Bool)]] { let dialect = DelimitedDialect() let reader = DelimitedFieldReader(dialect: dialect) return try Array(text.utf8).withUnsafeBufferPointer { buffer in guard let base = buffer.baseAddress else { return [] } let index = try DelimitedRowIndexer.index(buffer, dialect: dialect, contentStart: 0) - return (0.. Bool { - _ = loadConnections() + let previous = loadConnections() guard !lastLoadFailed else { Self.logger.error("Refused to save over connections.json: it could not be read") return false } - return persist(connections, byUser: byUser) + guard persist(connections, byUser: byUser) else { return false } + moveSecretsWhoseScopeChanged(from: previous, to: connections) + return true } @discardableResult @@ -623,7 +625,7 @@ final class ConnectionStorage { @discardableResult func savePassword(_ password: String, for connectionId: UUID) -> Bool { let key = "com.TablePro.password.\(connectionId.uuidString)" - return keychain.writeString(password, forKey: key) + return writeSecret(password, forKey: key, connectionId: connectionId) } func loadPassword(for connectionId: UUID) -> String? { @@ -633,7 +635,7 @@ final class ConnectionStorage { func deletePassword(for connectionId: UUID) { let key = "com.TablePro.password.\(connectionId.uuidString)" - keychain.delete(forKey: key) + deleteSecret(forKey: key, connectionId: connectionId) } // MARK: - SSH Password Storage @@ -641,7 +643,7 @@ final class ConnectionStorage { @discardableResult func saveSSHPassword(_ password: String, for connectionId: UUID) -> Bool { let key = "com.TablePro.sshpassword.\(connectionId.uuidString)" - return keychain.writeString(password, forKey: key) + return writeSecret(password, forKey: key, connectionId: connectionId) } func loadSSHPassword(for connectionId: UUID) -> String? { @@ -651,7 +653,7 @@ final class ConnectionStorage { func deleteSSHPassword(for connectionId: UUID) { let key = "com.TablePro.sshpassword.\(connectionId.uuidString)" - keychain.delete(forKey: key) + deleteSecret(forKey: key, connectionId: connectionId) } // MARK: - Key Passphrase Storage @@ -659,7 +661,7 @@ final class ConnectionStorage { @discardableResult func saveKeyPassphrase(_ passphrase: String, for connectionId: UUID) -> Bool { let key = "com.TablePro.keypassphrase.\(connectionId.uuidString)" - return keychain.writeString(passphrase, forKey: key) + return writeSecret(passphrase, forKey: key, connectionId: connectionId) } func loadKeyPassphrase(for connectionId: UUID) -> String? { @@ -669,7 +671,7 @@ final class ConnectionStorage { func deleteKeyPassphrase(for connectionId: UUID) { let key = "com.TablePro.keypassphrase.\(connectionId.uuidString)" - keychain.delete(forKey: key) + deleteSecret(forKey: key, connectionId: connectionId) } // MARK: - SSL Client Key Passphrase Storage @@ -677,7 +679,7 @@ final class ConnectionStorage { @discardableResult func saveSSLClientKeyPassphrase(_ passphrase: String, for connectionId: UUID) -> Bool { let key = "com.TablePro.sslkeypassphrase.\(connectionId.uuidString)" - return keychain.writeString(passphrase, forKey: key) + return writeSecret(passphrase, forKey: key, connectionId: connectionId) } func loadSSLClientKeyPassphrase(for connectionId: UUID) -> String? { @@ -687,7 +689,7 @@ final class ConnectionStorage { func deleteSSLClientKeyPassphrase(for connectionId: UUID) { let key = "com.TablePro.sslkeypassphrase.\(connectionId.uuidString)" - keychain.delete(forKey: key) + deleteSecret(forKey: key, connectionId: connectionId) } // MARK: - Plugin Secure Field Storage @@ -695,7 +697,7 @@ final class ConnectionStorage { @discardableResult func savePluginSecureField(_ value: String, fieldId: String, for connectionId: UUID) -> Bool { let key = "com.TablePro.plugin.\(fieldId).\(connectionId.uuidString)" - return keychain.writeString(value, forKey: key) + return writeSecret(value, forKey: key, connectionId: connectionId) } func loadPluginSecureField(fieldId: String, for connectionId: UUID) -> String? { @@ -705,7 +707,7 @@ final class ConnectionStorage { func deletePluginSecureField(fieldId: String, for connectionId: UUID) { let key = "com.TablePro.plugin.\(fieldId).\(connectionId.uuidString)" - keychain.delete(forKey: key) + deleteSecret(forKey: key, connectionId: connectionId) } func deleteAllPluginSecureFields(for connectionId: UUID, fieldIds: [String]) { @@ -719,7 +721,7 @@ final class ConnectionStorage { @discardableResult func saveTOTPSecret(_ secret: String, for connectionId: UUID) -> Bool { let key = "com.TablePro.totpsecret.\(connectionId.uuidString)" - return keychain.writeString(secret, forKey: key) + return writeSecret(secret, forKey: key, connectionId: connectionId) } func loadTOTPSecret(for connectionId: UUID) -> String? { @@ -729,14 +731,14 @@ final class ConnectionStorage { func deleteTOTPSecret(for connectionId: UUID) { let key = "com.TablePro.totpsecret.\(connectionId.uuidString)" - keychain.delete(forKey: key) + deleteSecret(forKey: key, connectionId: connectionId) } // MARK: - Cloudflare Service Token Storage func saveCloudflareTokenId(_ tokenId: String, for connectionId: UUID) { let key = "com.TablePro.cloudflaretokenid.\(connectionId.uuidString)" - keychain.writeString(tokenId, forKey: key) + writeSecret(tokenId, forKey: key, connectionId: connectionId) } func loadCloudflareTokenId(for connectionId: UUID) -> String? { @@ -746,12 +748,12 @@ final class ConnectionStorage { func deleteCloudflareTokenId(for connectionId: UUID) { let key = "com.TablePro.cloudflaretokenid.\(connectionId.uuidString)" - keychain.delete(forKey: key) + deleteSecret(forKey: key, connectionId: connectionId) } func saveCloudflareTokenSecret(_ tokenSecret: String, for connectionId: UUID) { let key = "com.TablePro.cloudflaretokensecret.\(connectionId.uuidString)" - keychain.writeString(tokenSecret, forKey: key) + writeSecret(tokenSecret, forKey: key, connectionId: connectionId) } func loadCloudflareTokenSecret(for connectionId: UUID) -> String? { @@ -761,14 +763,14 @@ final class ConnectionStorage { func deleteCloudflareTokenSecret(for connectionId: UUID) { let key = "com.TablePro.cloudflaretokensecret.\(connectionId.uuidString)" - keychain.delete(forKey: key) + deleteSecret(forKey: key, connectionId: connectionId) } // MARK: - Cloud SQL Auth Proxy Credential Storage func saveCloudSQLProxyServiceAccountKey(_ key: String, for connectionId: UUID) { let storageKey = "com.TablePro.cloudsqlproxyserviceaccountkey.\(connectionId.uuidString)" - keychain.writeString(key, forKey: storageKey) + writeSecret(key, forKey: storageKey, connectionId: connectionId) } func loadCloudSQLProxyServiceAccountKey(for connectionId: UUID) -> String? { @@ -778,14 +780,14 @@ final class ConnectionStorage { func deleteCloudSQLProxyServiceAccountKey(for connectionId: UUID) { let storageKey = "com.TablePro.cloudsqlproxyserviceaccountkey.\(connectionId.uuidString)" - keychain.delete(forKey: storageKey) + deleteSecret(forKey: storageKey, connectionId: connectionId) } // MARK: - SOCKS Proxy Password Storage func saveSOCKSProxyPassword(_ password: String, for connectionId: UUID) { let key = "com.TablePro.socksproxypassword.\(connectionId.uuidString)" - keychain.writeString(password, forKey: key) + writeSecret(password, forKey: key, connectionId: connectionId) } func loadSOCKSProxyPassword(for connectionId: UUID) -> String? { @@ -795,7 +797,162 @@ final class ConnectionStorage { func deleteSOCKSProxyPassword(for connectionId: UUID) { let key = "com.TablePro.socksproxypassword.\(connectionId.uuidString)" - keychain.delete(forKey: key) + deleteSecret(forKey: key, connectionId: connectionId) + } + + // MARK: - Secret Scope + + private static let secretKinds = [ + "password", "sshpassword", "keypassphrase", "sslkeypassphrase", "totpsecret", + "cloudflaretokenid", "cloudflaretokensecret", "cloudsqlproxyserviceaccountkey", "socksproxypassword" + ] + private static let localOnlySecretsMigratedKey = "com.TablePro.localOnlySecretsKeptOnDevice" + + private func secretKeys(for connection: DatabaseConnection) -> [String] { + let id = connection.id.uuidString + var fieldIds = Self.secureFieldIds(for: connection.type) + if !fieldIds.contains(ConnectionSecretOptions.mongoSessionTokenFieldId) { + fieldIds.append(ConnectionSecretOptions.mongoSessionTokenFieldId) + } + return Self.secretKinds.map { "com.TablePro.\($0).\(id)" } + + fieldIds.map { "com.TablePro.plugin.\($0).\(id)" } + } + + private func keepsSecretsOnDevice(_ connectionId: UUID) -> Bool { + loadConnection(id: connectionId).map { !$0.participatesInSync } ?? false + } + + private func readsOwnItemFirst(_ connectionId: UUID) -> Bool { + if let staging, staging.connectionId == connectionId { + return staging.deviceOnly + } + return keepsSecretsOnDevice(connectionId) + } + + @discardableResult + private func writeSecret(_ value: String, forKey key: String, connectionId: UUID) -> Bool { + let staged = staging?.connectionId == connectionId + let deviceOnly = staged ? staging?.deviceOnly ?? false : keepsSecretsOnDevice(connectionId) + let written = keychain.writeString(value, forKey: key, deviceOnly: deviceOnly) + if staged { + staging?.edits.deletedKeys.removeAll { $0 == key } + if !written { + staging?.edits.writeFailed = true + } + } + return written + } + + private func deleteSecret(forKey key: String, connectionId: UUID) { + guard let current = staging, current.connectionId == connectionId else { + keychain.delete(forKey: key) + return + } + guard !current.keptKeys.contains(key), !current.edits.deletedKeys.contains(key) else { return } + staging?.edits.deletedKeys.append(key) + } + + // Rewrites each stored secret into the scope the connection now has. Going device-only keeps the copy iCloud + // Keychain holds: removing it would remove it from every device. False when a secret could not be read or written. + @discardableResult + private func rewriteSecrets(of connection: DatabaseConnection, readingOwnItemFirst: Bool, deviceOnly: Bool) -> Bool { + var complete = true + for key in secretKeys(for: connection) { + switch keychain.readStringResult(forKey: key, deviceOnly: readingOwnItemFirst) { + case .found(let value): + if !keychain.writeString(value, forKey: key, deviceOnly: deviceOnly) { + complete = false + } + case .notFound: + break + case .locked, .userCancelled, .authFailed, .error: + complete = false + } + } + return complete + } + + // Each direction reads the value the connection was using, so the toggle never changes a secret. + private func moveSecretsWhoseScopeChanged(from previous: [DatabaseConnection], to current: [DatabaseConnection]) { + let wasSynced = Dictionary( + previous.map { ($0.id, $0.participatesInSync) }, + uniquingKeysWith: { first, _ in first } + ) + for connection in current { + guard let synced = wasSynced[connection.id], synced != connection.participatesInSync else { continue } + rewriteSecrets(of: connection, readingOwnItemFirst: !synced, deviceOnly: synced) + } + } + + // Builds before this one wrote a Local only connection's secrets wherever the password sync setting pointed. + func keepLocalOnlySecretsOnDeviceIfNeeded() { + guard !defaults.bool(forKey: Self.localOnlySecretsMigratedKey) else { return } + let connections = loadConnections() + guard !lastLoadFailed else { return } + var complete = true + for connection in connections where !connection.participatesInSync { + // A plugin that is not registered yet has secure fields this pass cannot name. + let knowsFields = PluginMetadataRegistry.shared.snapshot(for: connection.type) != nil + if !rewriteSecrets(of: connection, readingOwnItemFirst: true, deviceOnly: true) || !knowsFields { + complete = false + } + } + if complete { + defaults.set(true, forKey: Self.localOnlySecretsMigratedKey) + } else { + Self.logger.notice("Not every Local only secret has a copy on this Mac yet; retrying next launch") + } + } + + // MARK: - Staged Secret Edits + + struct StagedSecretEdits { + fileprivate(set) var deletedKeys: [String] = [] + fileprivate(set) var writeFailed = false + } + + private struct SecretStaging { + let connectionId: UUID + let deviceOnly: Bool + let keptKeys: Set + var edits = StagedSecretEdits() + } + + private var staging: SecretStaging? + + private var unreadableSecretsSeen: Set? + + // The keys a read inside `body` could not read. A form runs its prefill here, so the field left empty and the + // key it must not clear come from one read. + func recordingUnreadableSecrets(_ body: () -> Void) -> Set { + let outer = unreadableSecretsSeen + unreadableSecretsSeen = [] + body() + let seen = unreadableSecretsSeen ?? [] + unreadableSecretsSeen = outer.map { $0.union(seen) } + return seen + } + + // Inside `body`, this connection's secret writes take `deviceOnly` and report a failure, and its deletes wait + // for `applyStagedDeletions`, which the caller runs once the record is on disk. A key in `keeping` is never deleted. + func stagingSecretEdits( + for connectionId: UUID, + deviceOnly: Bool, + keeping keptKeys: Set = [], + _ body: () -> Void + ) -> StagedSecretEdits { + let outer = staging + staging = SecretStaging(connectionId: connectionId, deviceOnly: deviceOnly, keptKeys: keptKeys) + body() + let edits = staging?.edits ?? StagedSecretEdits() + staging = outer + return edits + } + + func applyStagedDeletions(_ edits: StagedSecretEdits) { + for key in edits.deletedKeys { + keychain.delete(forKey: key) + } } // MARK: - Stored Secret State @@ -813,19 +970,19 @@ final class ConnectionStorage { } func passwordState(for connectionId: UUID) -> StoredSecretState { - secretState(forKey: "com.TablePro.password.\(connectionId.uuidString)") + secretState(forKey: "com.TablePro.password.\(connectionId.uuidString)", connectionId: connectionId) } func sshPasswordState(for connectionId: UUID) -> StoredSecretState { - secretState(forKey: "com.TablePro.sshpassword.\(connectionId.uuidString)") + secretState(forKey: "com.TablePro.sshpassword.\(connectionId.uuidString)", connectionId: connectionId) } func keyPassphraseState(for connectionId: UUID) -> StoredSecretState { - secretState(forKey: "com.TablePro.keypassphrase.\(connectionId.uuidString)") + secretState(forKey: "com.TablePro.keypassphrase.\(connectionId.uuidString)", connectionId: connectionId) } - private func secretState(forKey key: String) -> StoredSecretState { - switch keychain.readStringResult(forKey: key) { + private func secretState(forKey key: String, connectionId: UUID) -> StoredSecretState { + switch readSecret(forKey: key, connectionId: connectionId) { case .found(let value): return value.isEmpty ? .absent : .stored case .notFound: @@ -835,13 +992,24 @@ final class ConnectionStorage { } } + private func readSecret(forKey key: String, connectionId: UUID) -> KeychainStringResult { + let result = keychain.readStringResult(forKey: key, deviceOnly: readsOwnItemFirst(connectionId)) + switch result { + case .found, .notFound: + break + case .locked, .userCancelled, .authFailed, .error: + unreadableSecretsSeen?.insert(key) + } + return result + } + private struct SecretContext { let label: String let connectionId: UUID } private func resolveString(_ context: SecretContext, forKey key: String) -> String? { - keychain.readStringResult(forKey: key) + readSecret(forKey: key, connectionId: context.connectionId) .value(label: "\(context.label) (connId=\(context.connectionId.uuidString))", logger: Self.logger) } diff --git a/apple/TablePro/Core/Storage/ConnectionStoreIntegrity.swift b/apple/TablePro/Core/Storage/ConnectionStoreIntegrity.swift index 606a46b617..69356f97dd 100644 --- a/apple/TablePro/Core/Storage/ConnectionStoreIntegrity.swift +++ b/apple/TablePro/Core/Storage/ConnectionStoreIntegrity.swift @@ -18,15 +18,26 @@ import Security protocol IntegrityKeySource: Sendable { /// The key this device signs its connection store with, or nil when no key can be obtained. func key() -> SymmetricKey? + // Whether this device ever tagged the store, kept where only TablePro can write. + func hasStamped(_ storeId: String) -> Bool + @discardableResult + func recordStamped(_ storeId: String) -> Bool +} + +extension IntegrityKeySource { + func hasStamped(_ storeId: String) -> Bool { false } + + @discardableResult + func recordStamped(_ storeId: String) -> Bool { false } } struct ConnectionStoreIntegrity: Sendable { enum Verdict: Equatable { /// The tag matches, so the file is the one TablePro last wrote. case trusted - /// No tag yet. An install that predates this check, or a fresh store. + /// No tag yet, and none was ever written. An install that predates this check, or a fresh store. case unstamped - /// A tag exists and does not match. The file changed outside TablePro. + /// The tag does not match, or is gone from a store this device tagged. The file changed outside TablePro. case modified /// No key, so nothing can be said either way. Treated as untrusted rather than assumed /// good, for the same reason an unreadable certificate authority fails a connection. @@ -45,14 +56,27 @@ struct ConnectionStoreIntegrity: Sendable { fileURL.appendingPathExtension("hmac") } + static func storeId(for fileURL: URL) -> String { + SHA256.hash(data: Data(fileURL.standardizedFileURL.path.utf8)) + .map { String(format: "%02x", $0) } + .joined() + } + func verify(_ data: Data, fileURL: URL) -> Verdict { guard let key = keySource.key() else { return .unavailable } + let storeId = Self.storeId(for: fileURL) guard let storedTag = try? Data(contentsOf: Self.tagURL(for: fileURL)), !storedTag.isEmpty else { - return .unstamped + // Deleting the tag must not turn an edited store back into one to adopt. + return keySource.hasStamped(storeId) ? .modified : .unstamped } let expected = Data(HMAC.authenticationCode(for: data, using: key)) - return Self.constantTimeEquals(expected, storedTag) ? .trusted : .modified + guard Self.constantTimeEquals(expected, storedTag) else { return .modified } + // A tag an older build wrote has no record yet. + if !keySource.hasStamped(storeId) { + keySource.recordStamped(storeId) + } + return .trusted } @discardableResult @@ -61,6 +85,9 @@ struct ConnectionStoreIntegrity: Sendable { let tag = Data(HMAC.authenticationCode(for: data, using: key)) do { try tag.write(to: Self.tagURL(for: fileURL), options: .atomic) + if !keySource.recordStamped(Self.storeId(for: fileURL)) { + Self.logger.error("Could not record that the connection store is tagged") + } return true } catch { Self.logger.error("Could not write the connection store tag: \(error.localizedDescription)") @@ -94,10 +121,12 @@ struct ConnectionStoreIntegrity: Sendable { struct KeychainIntegrityKeySource: IntegrityKeySource { private static let service = "com.TablePro" private static let account = "com.TablePro.connectionStoreIntegrityKey" + private static let stampAccountPrefix = "com.TablePro.connectionStoreStamped." private static let byteCount = 32 private static let lock = NSLock() nonisolated(unsafe) private static var cached: SymmetricKey? + nonisolated(unsafe) private static var knownStamps: Set = [] func key() -> SymmetricKey? { if let isolatedStore = Self.isolatedStore { @@ -117,7 +146,51 @@ struct KeychainIntegrityKeySource: IntegrityKeySource { return created } - private static func baseQuery() -> [String: Any] { + func hasStamped(_ storeId: String) -> Bool { + if let isolatedStore = Self.isolatedStore { + return StoredIntegrityKeySource(store: isolatedStore).hasStamped(storeId) + } + + Self.lock.lock() + defer { Self.lock.unlock() } + + if Self.knownStamps.contains(storeId) { return true } + var query = Self.baseQuery(account: Self.stampAccountPrefix + storeId) + query[kSecMatchLimit as String] = kSecMatchLimitOne + let status = SecItemCopyMatching(query as CFDictionary, nil) + if status == errSecSuccess { + Self.knownStamps.insert(storeId) + } + // A record that cannot be read is not proof the store was never tagged. + return status != errSecItemNotFound + } + + @discardableResult + func recordStamped(_ storeId: String) -> Bool { + if let isolatedStore = Self.isolatedStore { + return StoredIntegrityKeySource(store: isolatedStore).recordStamped(storeId) + } + + Self.lock.lock() + defer { Self.lock.unlock() } + + if Self.knownStamps.contains(storeId) { return true } + var addQuery = Self.baseQuery(account: Self.stampAccountPrefix + storeId) + addQuery[kSecValueData as String] = Data([1]) + addQuery[kSecAttrAccessible as String] = kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly + + let status = SecItemAdd(addQuery as CFDictionary, nil) + guard status == errSecSuccess || status == errSecDuplicateItem else { + ConnectionStoreIntegrity.logger.error( + "Could not record the connection store tag (OSStatus \(status))" + ) + return false + } + Self.knownStamps.insert(storeId) + return true + } + + private static func baseQuery(account: String = Self.account) -> [String: Any] { [ kSecClass as String: kSecClassGenericPassword, kSecAttrService as String: service, @@ -172,6 +245,7 @@ struct KeychainIntegrityKeySource: IntegrityKeySource { /// every store it writes reads back untrusted. struct StoredIntegrityKeySource: IntegrityKeySource { private static let storageKey = "connectionStoreIntegrity" + private static let stampKeyPrefix = "connectionStoreStamped." private static let byteCount = 32 private let store: any KeychainStoring @@ -194,4 +268,13 @@ struct StoredIntegrityKeySource: IntegrityKeySource { } return SymmetricKey(data: bytes) } + + func hasStamped(_ storeId: String) -> Bool { + store.readStringResult(forKey: Self.stampKeyPrefix + storeId) != .notFound + } + + @discardableResult + func recordStamped(_ storeId: String) -> Bool { + store.writeString("1", forKey: Self.stampKeyPrefix + storeId) + } } diff --git a/apple/TablePro/Core/Storage/IntegrityStampedFileStore.swift b/apple/TablePro/Core/Storage/IntegrityStampedFileStore.swift index 6c733a1876..857199fdb0 100644 --- a/apple/TablePro/Core/Storage/IntegrityStampedFileStore.swift +++ b/apple/TablePro/Core/Storage/IntegrityStampedFileStore.swift @@ -73,8 +73,9 @@ final class IntegrityStampedFileStore { isTrusted = false break } - /// An install that predates the tag. Adopt the file as it stands, which is the only - /// option without a prior baseline, and stamp it so later edits are detectable. + /// An install that predates the tag: this device never tagged the store. Adopt the file as it + /// stands, which is the only option without a prior baseline, and stamp it so later edits are + /// detectable. A tag that went missing afterwards reads as modified instead. integrity.stamp(data, fileURL: fileURL) isTrusted = true case .modified: diff --git a/apple/TablePro/Core/Storage/KeychainHelper.swift b/apple/TablePro/Core/Storage/KeychainHelper.swift index f522b21760..e45b42dfee 100644 --- a/apple/TablePro/Core/Storage/KeychainHelper.swift +++ b/apple/TablePro/Core/Storage/KeychainHelper.swift @@ -28,10 +28,53 @@ enum KeychainStringResult: Sendable, Equatable { protocol KeychainStoring: Sendable { @discardableResult func writeString(_ value: String, forKey key: String) -> Bool + // deviceOnly keeps the item out of iCloud Keychain whatever the password sync setting says. + @discardableResult + func writeString(_ value: String, forKey key: String, deviceOnly: Bool) -> Bool func readStringResult(forKey key: String) -> KeychainStringResult + // deviceOnly reads this Mac's own item before one iCloud Keychain delivered. + func readStringResult(forKey key: String, deviceOnly: Bool) -> KeychainStringResult func delete(forKey key: String) } +extension KeychainStoring { + func readStringResult(forKey key: String, deviceOnly: Bool) -> KeychainStringResult { + readStringResult(forKey: key) + } + + @discardableResult + func writeString(_ value: String, forKey key: String, deviceOnly: Bool) -> Bool { + writeString(value, forKey: key) + } +} + +protocol KeychainItemStore: Sendable { + func add(_ attributes: [String: Any]) -> OSStatus + func update(_ query: [String: Any], attributes: [String: Any]) -> OSStatus + func copyData(_ query: [String: Any]) -> (status: OSStatus, data: Data?) + func delete(_ query: [String: Any]) -> OSStatus +} + +struct SystemKeychainItemStore: KeychainItemStore { + func add(_ attributes: [String: Any]) -> OSStatus { + SecItemAdd(attributes as CFDictionary, nil) + } + + func update(_ query: [String: Any], attributes: [String: Any]) -> OSStatus { + SecItemUpdate(query as CFDictionary, attributes as CFDictionary) + } + + func copyData(_ query: [String: Any]) -> (status: OSStatus, data: Data?) { + var result: AnyObject? + let status = SecItemCopyMatching(query as CFDictionary, &result) + return (status, result as? Data) + } + + func delete(_ query: [String: Any]) -> OSStatus { + SecItemDelete(query as CFDictionary) + } +} + final class KeychainHelper: KeychainStoring { static let shared = KeychainHelper() static let passwordSyncEnabledKey = "com.TablePro.keychainPasswordSyncEnabled" @@ -56,62 +99,93 @@ final class KeychainHelper: KeychainStoring { // Off for a credential bound to this Mac, which another Mac must never receive through iCloud. private let followsPasswordSync: Bool + private let passwordSyncSetting: @Sendable () -> Bool + private let items: any KeychainItemStore private var isPasswordSyncEnabled: Bool { - followsPasswordSync && AppStorageEnvironment.shared.defaults.bool(forKey: Self.passwordSyncEnabledKey) + followsPasswordSync && passwordSyncSetting() + } + + static func storedPasswordSyncSetting() -> Bool { + AppStorageEnvironment.shared.defaults.bool(forKey: passwordSyncEnabledKey) } - init(service: String = "com.TablePro", followsPasswordSync: Bool = true) { + init( + service: String = "com.TablePro", + followsPasswordSync: Bool = true, + passwordSyncSetting: @escaping @Sendable () -> Bool = { KeychainHelper.storedPasswordSyncSetting() }, + items: any KeychainItemStore = SystemKeychainItemStore() + ) { self.service = service self.followsPasswordSync = followsPasswordSync + self.passwordSyncSetting = passwordSyncSetting + self.items = items } // MARK: - Data API @discardableResult func write(_ data: Data, forKey key: String) -> Bool { - let synchronizable = isPasswordSyncEnabled - let accessible = accessibility(forSync: synchronizable) + write(data, forKey: key, deviceOnly: false) + } - var addQuery = baseQuery(forKey: key) - addQuery[kSecValueData as String] = data - addQuery[kSecAttrAccessible as String] = accessible - if synchronizable { - addQuery[kSecAttrSynchronizable as String] = true - } + // Synchronizable is part of an item's identity, so a key can hold a synced and a local item at once. The write + // goes to the one the setting selects. A synced item is never removed here: deleting it reaches every device. + @discardableResult + func write(_ data: Data, forKey key: String, deviceOnly: Bool) -> Bool { + let synchronizable = !deviceOnly && isPasswordSyncEnabled + let accessible = accessibility(forSync: synchronizable) + let target = itemQuery(forKey: key, synchronizable: synchronizable) - var status = SecItemAdd(addQuery as CFDictionary, nil) - - if status == errSecDuplicateItem { - var search = baseQuery(forKey: key) - search[kSecAttrSynchronizable as String] = kSecAttrSynchronizableAny - let attributes: [String: Any] = [ - kSecValueData as String: data, - kSecAttrSynchronizable as String: synchronizable, - kSecAttrAccessible as String: accessible - ] - status = SecItemUpdate(search as CFDictionary, attributes as CFDictionary) + var status = items.update(target, attributes: [ + kSecValueData as String: data, + kSecAttrAccessible as String: accessible + ]) + if status == errSecItemNotFound { + var attributes = target + attributes[kSecValueData as String] = data + attributes[kSecAttrAccessible as String] = accessible + status = items.add(attributes) } if status != errSecSuccess { log(status: status, operation: "write", key: key) return false } + + if synchronizable { + let stale = items.delete(itemQuery(forKey: key, synchronizable: false)) + if stale != errSecSuccess, stale != errSecItemNotFound { + log(status: stale, operation: "remove the local copy", key: key) + } + } return true } func read(forKey key: String) -> KeychainResult { - var query = baseQuery(forKey: key) - query[kSecAttrSynchronizable as String] = kSecAttrSynchronizableAny - query[kSecReturnData as String] = true - query[kSecMatchLimit as String] = kSecMatchLimitOne + read(forKey: key, deviceOnly: false) + } - var result: AnyObject? - let status = SecItemCopyMatching(query as CFDictionary, &result) + func read(forKey key: String, deviceOnly: Bool) -> KeychainResult { + var preferred = itemQuery(forKey: key, synchronizable: !deviceOnly && isPasswordSyncEnabled) + preferred[kSecReturnData as String] = true + preferred[kSecMatchLimit as String] = kSecMatchLimitOne + let first = items.copyData(preferred) + guard first.status == errSecItemNotFound else { + return result(for: first, key: key) + } - switch status { + var fallback = baseQuery(forKey: key) + fallback[kSecAttrSynchronizable as String] = kSecAttrSynchronizableAny + fallback[kSecReturnData as String] = true + fallback[kSecMatchLimit as String] = kSecMatchLimitOne + return result(for: items.copyData(fallback), key: key) + } + + private func result(for copied: (status: OSStatus, data: Data?), key: String) -> KeychainResult { + switch copied.status { case errSecSuccess: - if let data = result as? Data { + if let data = copied.data { return .found(data) } return .notFound @@ -127,8 +201,8 @@ final class KeychainHelper: KeychainStoring { Self.logger.warning("Keychain auth failed for '\(key, privacy: .public)'") return .authFailed default: - log(status: status, operation: "read", key: key) - return .error(status) + log(status: copied.status, operation: "read", key: key) + return .error(copied.status) } } @@ -136,7 +210,7 @@ final class KeychainHelper: KeychainStoring { var query = baseQuery(forKey: key) query[kSecAttrSynchronizable as String] = kSecAttrSynchronizableAny - let status = SecItemDelete(query as CFDictionary) + let status = items.delete(query) if status != errSecSuccess, status != errSecItemNotFound { log(status: status, operation: "delete", key: key) } @@ -150,7 +224,16 @@ final class KeychainHelper: KeychainStoring { Self.logger.error("UTF-8 encode failed for '\(key, privacy: .public)'") return false } - return write(data, forKey: key) + return write(data, forKey: key, deviceOnly: false) + } + + @discardableResult + func writeString(_ value: String, forKey key: String, deviceOnly: Bool) -> Bool { + guard let data = value.data(using: .utf8) else { + Self.logger.error("UTF-8 encode failed for '\(key, privacy: .public)'") + return false + } + return write(data, forKey: key, deviceOnly: deviceOnly) } func readString(forKey key: String) -> String? { @@ -161,7 +244,11 @@ final class KeychainHelper: KeychainStoring { } func readStringResult(forKey key: String) -> KeychainStringResult { - switch read(forKey: key) { + readStringResult(forKey: key, deviceOnly: false) + } + + func readStringResult(forKey key: String, deviceOnly: Bool) -> KeychainStringResult { + switch read(forKey: key, deviceOnly: deviceOnly) { case .found(let data): guard let value = String(data: data, encoding: .utf8) else { Self.logger.error("UTF-8 decode failed for '\(key, privacy: .public)'") @@ -178,6 +265,15 @@ final class KeychainHelper: KeychainStoring { // MARK: - Private + // No synchronizable key matches local items only, in the file-based keychain too. + private func itemQuery(forKey key: String, synchronizable: Bool) -> [String: Any] { + var query = baseQuery(forKey: key) + if synchronizable { + query[kSecAttrSynchronizable as String] = true + } + return query + } + private func baseQuery(forKey key: String) -> [String: Any] { var query: [String: Any] = [ kSecClass as String: kSecClassGenericPassword, diff --git a/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+DeferredDeletions.swift b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+DeferredDeletions.swift new file mode 100644 index 0000000000..272a90623b --- /dev/null +++ b/apple/TablePro/Core/Sync/Extensions/SyncCoordinator+DeferredDeletions.swift @@ -0,0 +1,45 @@ +import CloudKit +import Foundation +import TableProSyncTransport + +// A deletion of a category this Mac does not sync is passed by the change token and never sent again, so its +// record name is kept until the category is read. +extension SyncCoordinator { + // Deleting a profile removes keychain secrets no pull can bring back, so a late deletion never does it. + private static let typesNeverDeletedLate: Set = [.sshProfile, .credentialProfile] + + struct DeferredDeletions { + var due: [CKRecord.ID] = [] + var settled: Set = [] + } + + func deferredDeletions(dueIn result: PullResult, settings: SyncSettings, zoneID: CKRecordZone.ID) -> DeferredDeletions { + let deferred = metadataStorage.deferredDeletions + guard !deferred.isEmpty else { return DeferredDeletions() } + // A record the server sends again was saved after its deletion. + let live = Set(result.changedRecords.map(\.recordID.recordName)) + var deletions = DeferredDeletions(settled: deferred.intersection(live)) + for name in deferred.subtracting(live) { + guard let parsed = SyncRecordType.parse(recordName: name) else { + deletions.settled.insert(name) + continue + } + guard settings.syncs(parsed.type) else { continue } + deletions.settled.insert(name) + let editedHere = changeTracker.dirtyRecords(for: parsed.type).contains(parsed.id) + guard !editedHere, !Self.typesNeverDeletedLate.contains(parsed.type) else { continue } + deletions.due.append(SyncRecordMapper.recordID(type: parsed.type, id: parsed.id, in: zoneID)) + } + return deletions + } + + func settleDeferredDeletions(_ deletions: DeferredDeletions, after result: PullResult, settings: SyncSettings) { + let skipped = result.deletedRecordIDs.map(\.recordName).filter { name in + SyncRecordType.parse(recordName: name).map { !settings.syncs($0.type) } == true + } + guard !skipped.isEmpty || !deletions.settled.isEmpty else { return } + metadataStorage.deferredDeletions = metadataStorage.deferredDeletions + .subtracting(deletions.settled) + .union(skipped) + } +} diff --git a/apple/TablePro/Core/Sync/SyncCoordinator.swift b/apple/TablePro/Core/Sync/SyncCoordinator.swift index 89e1aa7e17..917a6eaf64 100644 --- a/apple/TablePro/Core/Sync/SyncCoordinator.swift +++ b/apple/TablePro/Core/Sync/SyncCoordinator.swift @@ -217,6 +217,7 @@ final class SyncCoordinator: ObservableObject { /// save into a zone another device recreated fail as an unknown record. if nextZone == .removed, zone != .removed { recordCache.removeAll() + metadataStorage.deferredDeletions = [] metadataStorage.saveToken(nil) } metadataStorage.zoneState = nextZone @@ -374,6 +375,7 @@ final class SyncCoordinator: ObservableObject { // Clear token to force a full fetch on first sync after enabling metadataStorage.saveToken(nil) + metadataStorage.deferredDeletions = [] /// Turning sync on is the person's choice to upload, a removed zone included. metadataStorage.zoneState = .unknown @@ -400,6 +402,7 @@ final class SyncCoordinator: ObservableObject { } metadataStorage.zoneState = .unknown metadataStorage.saveToken(nil) + metadataStorage.deferredDeletions = [] recordCache.removeAll() markAllLocalDataDirty() await markSQLFavoritesDirty() @@ -725,7 +728,9 @@ final class SyncCoordinator: ObservableObject { echoGuard: SyncEchoGuard? = nil ) async -> Bool { let settings = settings ?? services.appSettingsStorage.loadSync() - let deletedRecordIDs = result.deletedRecordIDs.filter { recordID in + let zoneID = await transport.currentZoneID + let deferred = deferredDeletions(dueIn: result, settings: settings, zoneID: zoneID) + let deletedRecordIDs = (result.deletedRecordIDs + deferred.due).filter { recordID in echoGuard?.withholdsDeletion(recordID, tracker: changeTracker) != true } let pendingEdits = await recordNamesWithPendingEdits() @@ -764,7 +769,8 @@ final class SyncCoordinator: ObservableObject { echoGuard?.withholds(record.recordID, tracker: changeTracker) != true && SyncRecordType(rawValue: record.recordType).map { settings.syncs($0) } == true }) - recordCache.remove(result.deletedRecordIDs) + recordCache.remove(result.deletedRecordIDs + deferred.due) + settleDeferredDeletions(deferred, after: result, settings: settings) Self.logger.info( "Pull completed: \(result.changedRecords.count) changed, \(result.deletedRecordIDs.count) deleted" diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift index 478203b237..c326a505df 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -209,7 +209,7 @@ private extension QueryClassifier { return QueryClassification(tier: .write, reachesFilesystemOrExecutesCode: touchesUnsafeSurface) } - /// `DB ` and `OUTSIDEBLOCK ` run the command they wrap, so that + /// `DB `, `OUTSIDEBLOCK ` and `MUSTAPPLY ` run the command they wrap, so that /// command decides the tier: read as the bare prefix, `DB 0 FLUSHDB` would pass as an ordinary /// write. A prefix with nothing after it is left whole, which classifies as a write. The words are /// read the way the driver reads them, so a quoted `"FLUSHALL"` is the command it runs. @@ -218,7 +218,7 @@ private extension QueryClassifier { while let first = rest.first?.uppercased() { if first == "DB", rest.count > 2 { rest = rest.dropFirst(2) - } else if first == "OUTSIDEBLOCK", rest.count > 1 { + } else if first == "OUTSIDEBLOCK" || first == "MUSTAPPLY", rest.count > 1 { rest = rest.dropFirst() } else { break diff --git a/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift b/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift index 72b629544d..1dd8f10e35 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift @@ -8,18 +8,23 @@ import TableProPluginKit import TableProSQLGrammar enum SQLParameterExtractor { - /// The text the parameter panel reads names from: every statement that can bind one, joined. - /// - /// A definition is left out, because its `:name` is never a bind parameter: `:NEW` in a trigger body is a - /// pseudo-record, and rewriting it into a placeholder stores a broken trigger. + // The text the parameter panel reads names from: every statement that can bind one, joined. A definition is left + // out: `:NEW` in a trigger body is a pseudo-record, and rewriting it into a placeholder stores a broken trigger. static func parameterSource(of statements: [SQLStatementScanner.ExecutableStatement]) -> String { statements.filter(\.acceptsBindParameters).map(\.sql).joined(separator: "; ") } - static func extractParameters(from sql: String) -> [String] { + // The grammar a caller that names none is read with: the quotes and escapes this reader took for every engine + // before it read grammars. + static let unspecifiedGrammar: SQLLexicalGrammar = [ + .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backslashEscapesInBackticks, + .backtickQuotes, .taggedDollarQuotes, + ] + + static func extractParameters(from sql: String, grammar: SQLLexicalGrammar = unspecifiedGrammar) -> [String] { var result: [String] = [] var seen = Set() - scan(sql: sql) { name, _ in + scan(sql: sql, grammar: grammar) { name, _ in if !seen.contains(name) { seen.insert(name) result.append(name) @@ -31,7 +36,8 @@ enum SQLParameterExtractor { static func convertToNativeStyle( sql: String, parameters: [QueryParameter], - style: ParameterStyle + style: ParameterStyle, + grammar: SQLLexicalGrammar = unspecifiedGrammar ) -> (sql: String, values: [Any?]) { let nsSQL = sql as NSString let length = nsSQL.length @@ -43,7 +49,7 @@ enum SQLParameterExtractor { var dollarIndex = 1 var lastCopied = 0 - scan(sql: sql) { name, range in + scan(sql: sql, grammar: grammar) { name, range in if lastCopied < range.location { resultSQL += nsSQL.substring(with: NSRange(location: lastCopied, length: range.location - lastCopied)) } @@ -74,156 +80,43 @@ enum SQLParameterExtractor { // MARK: - Private - private static let singleQuote = UInt16(UnicodeScalar("'").value) - private static let doubleQuote = UInt16(UnicodeScalar("\"").value) - private static let backtick = UInt16(UnicodeScalar("`").value) - private static let colonChar = UInt16(UnicodeScalar(":").value) - private static let dash = UInt16(UnicodeScalar("-").value) - private static let slash = UInt16(UnicodeScalar("/").value) - private static let star = UInt16(UnicodeScalar("*").value) - private static let newline = UInt16(UnicodeScalar("\n").value) - private static let backslash = UInt16(UnicodeScalar("\\").value) - private static let underscore = UInt16(UnicodeScalar("_").value) - private static let dollarChar = UInt16(UnicodeScalar("$").value) - - private static func isIdentifierStart(_ ch: UInt16) -> Bool { - (ch >= 0x41 && ch <= 0x5A) || (ch >= 0x61 && ch <= 0x7A) || ch == underscore - } - - private static func isIdentifierChar(_ ch: UInt16) -> Bool { - isIdentifierStart(ch) || (ch >= 0x30 && ch <= 0x39) - } + private static let colon = UInt16(UnicodeScalar(":").value) + private static let closeBracket = UInt16(UnicodeScalar("]").value) + // A `:` written against the end of a name, a literal, a subscript, a call or another `:` is the engine's own + // syntax: a cast (`a::int`), a path (`src:customer`), a slice (`arr[lo:hi]`), a typed slot (`{id:UInt32}`). private static func scan( sql: String, + grammar: SQLLexicalGrammar, onParameter: (_ name: String, _ range: NSRange) -> Void ) { - let nsSQL = sql as NSString - let length = nsSQL.length - guard length > 0 else { return } - - var inString = false - var stringCharVal: UInt16 = 0 - var inLineComment = false - var inBlockComment = false - var i = 0 - - while i < length { - let ch = nsSQL.character(at: i) - - if inLineComment { - if ch == newline { inLineComment = false } - i += 1 - continue - } - - if inBlockComment { - if ch == star && i + 1 < length && nsSQL.character(at: i + 1) == slash { - inBlockComment = false - i += 2 - continue - } - i += 1 - continue - } - - if !inString && ch == dash && i + 1 < length && nsSQL.character(at: i + 1) == dash { - inLineComment = true - i += 2 - continue - } - - if !inString && ch == slash && i + 1 < length && nsSQL.character(at: i + 1) == star { - inBlockComment = true - i += 2 - continue - } - - if inString && ch == backslash && i + 1 < length { - i += 2 + let text = sql as NSString + let length = text.length + var index = 0 + var followsOperand = false + + while index < length { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + index = max(span.end, index + 1) + followsOperand = !span.kind.isComment continue } - - if ch == singleQuote || ch == doubleQuote || ch == backtick { - if !inString { - inString = true - stringCharVal = ch - } else if ch == stringCharVal { - if i + 1 < length && nsSQL.character(at: i + 1) == stringCharVal { - i += 1 - } else { - inString = false - } + let unit = text.character(at: index) + if unit == colon, !followsOperand, index + 1 < length, + SqlDollarQuote.isIdentifierStart(text.character(at: index + 1)) { + var nameEnd = index + 2 + while nameEnd < length, SqlDollarQuote.isIdentifierPart(text.character(at: nameEnd)) { + nameEnd += 1 } - i += 1 + let name = text.substring(with: NSRange(location: index + 1, length: nameEnd - index - 1)) + onParameter(name, NSRange(location: index, length: nameEnd - index)) + index = nameEnd + followsOperand = true continue } - - if !inString && ch == dollarChar { - let tagStart = i + 1 - if tagStart < length && nsSQL.character(at: tagStart) == dollarChar { - var j = tagStart + 1 - while j < length - 1 { - if nsSQL.character(at: j) == dollarChar && nsSQL.character(at: j + 1) == dollarChar { - i = j + 2 - break - } - j += 1 - } - if i < tagStart + 1 { i = length } - continue - } - var tagEnd = tagStart - while tagEnd < length && isIdentifierChar(nsSQL.character(at: tagEnd)) { - tagEnd += 1 - } - if tagEnd > tagStart && tagEnd < length && nsSQL.character(at: tagEnd) == dollarChar { - let tagLen = tagEnd - i + 1 - let openTag = nsSQL.substring(with: NSRange(location: i, length: tagLen)) - var j = tagEnd + 1 - var found = false - while j <= length - tagLen { - if nsSQL.character(at: j) == dollarChar { - let candidate = nsSQL.substring(with: NSRange(location: j, length: tagLen)) - // PostgreSQL compares the tag bytes; `String ==` would take a canonically - // equivalent tag such as `$\u{212A}$` (KELVIN SIGN) as the close of `$K$`. - if candidate.utf16.elementsEqual(openTag.utf16) { - i = j + tagLen - found = true - break - } - } - j += 1 - } - if !found { i = length } - continue - } - } - - if !inString && ch == colonChar { - if i + 1 < length && nsSQL.character(at: i + 1) == colonChar { - i += 2 - while i < length && isIdentifierChar(nsSQL.character(at: i)) { - i += 1 - } - continue - } - - if i + 1 < length && isIdentifierStart(nsSQL.character(at: i + 1)) { - let nameStart = i + 1 - var nameEnd = nameStart - while nameEnd < length && isIdentifierChar(nsSQL.character(at: nameEnd)) { - nameEnd += 1 - } - let paramRange = NSRange(location: i, length: nameEnd - i) - let name = nsSQL.substring(with: NSRange(location: nameStart, length: nameEnd - nameStart)) - onParameter(name, paramRange) - i = nameEnd - continue - } - } - - i += 1 + followsOperand = unit == colon || unit == closeBracket || unit == SqlLexer.closeParen + || SQLNonCodeSpan.isWordUnit(unit) + index += 1 } } } diff --git a/apple/TablePro/Models/Query/QueryTabState.swift b/apple/TablePro/Models/Query/QueryTabState.swift index 764cd1df8f..796c43ee88 100644 --- a/apple/TablePro/Models/Query/QueryTabState.swift +++ b/apple/TablePro/Models/Query/QueryTabState.swift @@ -174,6 +174,9 @@ struct TabChangeSnapshot: Equatable { var insertedRowIDs: Set var modifiedCells: [RowID: Set] var insertedRowData: [RowID: [PluginCellValue]] + var supersededUpdates: [RowID: RowChange] = [:] + // The result these edits were made on. They name its rows, so they are void on any other. + var resultSetId: UUID? var primaryKeyColumns: [String] var columns: [String] diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 8ecb808b3c..4aaa6d788d 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -634,6 +634,10 @@ } } }, + "'%@' is a field of an array of objects in this document. Change it with a query." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "'%@' is document metadata and cannot be edited." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -7801,6 +7805,10 @@ } } }, + "%1$@.%2$@ was written as text: %3$lld values do not fit %4$@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "%1$@/%2$@ rows" : { "localizations" : { "fr" : { @@ -15791,6 +15799,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "A document with this _id already exists." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A domain's base type, or a range's subtype" : { "localizations" : { "fr" : { @@ -16120,6 +16132,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "A key named '%@' already exists." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A key needs a name." : { "extractionState" : "manual", "generatesSymbol" : false, @@ -16136,6 +16152,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "A key with this name already exists." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A Kubernetes resource is required, such as service/postgres" : { "localizations" : { "fr" : { @@ -16501,6 +16521,9 @@ } } } + }, + "A plan works on up to 3 devices at a time. Removing one signs TablePro out there." : { + }, "A point's id cannot change. Insert a new row with the new id instead." : { "extractionState" : "manual", @@ -16508,9 +16531,6 @@ }, "A publish to the team library is still running. Try again when it finishes." : { - }, - "A plan works on up to 3 devices at a time. Removing one signs TablePro out there." : { - }, "a query" : { "localizations" : { @@ -17499,6 +17519,9 @@ }, "A transaction is open on this connection, and its uncommitted rows aren't in these counts." : { + }, + "A transaction is open on this connection, and this change cannot run inside it. Nothing was run. Commit or roll back the transaction, then try again." : { + }, "A transaction is open on this connection. Commit or roll it back in TablePro, then run the shortcut again." : { @@ -51779,6 +51802,9 @@ } } } + }, + "Copy Original Definition" : { + }, "Copy Output" : { "localizations" : { @@ -54596,6 +54622,9 @@ } } } + }, + "Could not save this connection's passwords to the Keychain. Your changes were not saved." : { + }, "Could not search the referenced table" : { "localizations" : { @@ -112835,6 +112864,10 @@ } } }, + "MUSTAPPLY takes only RENAMENX or SET with NX" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "My Queries" : { "localizations" : { "fr" : { @@ -144849,6 +144882,16 @@ }, "Record %@ has no fields list." : { + }, + "Record %lld of the CSV data has %lld fields, but the header has %lld." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Record %1$lld of the CSV data has %2$lld fields, but the header has %3$lld." + } + } + } }, "Record Access" : { "extractionState" : "manual", @@ -155786,6 +155829,9 @@ }, "Running query in “%1$@” on %2$@" : { + }, + "Running the query will replace this result and discard all unsaved changes." : { + }, "Running Threads" : { "localizations" : { @@ -168745,15 +168791,15 @@ } } } - }, - "Signing in with credentials from this Mac was not approved." : { - }, "Signing in" : { }, "Signing in by email? Type the 6-digit code from the email in the sign-in sheet. The link in the email opens your browser instead." : { + }, + "Signing in with credentials from this Mac was not approved." : { + }, "Signing in…" : { "localizations" : { @@ -170174,6 +170220,9 @@ } } } + }, + "Some of these changes were applied before the save failed. The table was read again, and only the changes that were not applied are still staged." : { + }, "Some passwords were not read. You can enter them in the connection editor after import." : { "localizations" : { @@ -187600,6 +187649,16 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The definition of trigger “%@” could not be read, so it cannot be edited: %@" : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "The definition of trigger “%1$@” could not be read, so it cannot be edited: %2$@" + } + } + } + }, "The descriptor set names an unknown field type %llu." : { }, @@ -187772,6 +187831,10 @@ } } }, + "The document changed on the server after it was read. Refresh and try again." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The document has not loaded, so there is nothing to save it over." : { "localizations" : { "fr" : { @@ -188989,6 +189052,10 @@ }, "The key %1$@ matched %2$@ rows, so the row was not written." : { + }, + "The key changed on the server after it was read. Refresh and try again." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The key that protects your save history is not available." : { "localizations" : { @@ -190855,6 +190922,12 @@ } } } + }, + "The provider ended the response without sending any content." : { + + }, + "The provider reported an error." : { + }, "The query contains a NUL character (U+0000) and was not run. Remove it and try again." : { "extractionState" : "manual", @@ -191526,6 +191599,10 @@ } } }, + "The row has %1$lld fields where %2$lld are expected. A field that holds the delimiter must be quoted." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The row has no __name__, so the document it came from is unknown. Reload the collection." : { "extractionState" : "manual", "generatesSymbol" : false @@ -193793,6 +193870,9 @@ } } } + }, + "The statement did not finish within %d seconds and is still running. It may have been applied; check before running it again." : { + }, "The statement has %1$lld placeholders but %2$lld values were given." : { "extractionState" : "manual", @@ -194006,6 +194086,9 @@ } } } + }, + "The tab or its edits changed while the save waited for approval. Nothing was saved." : { + }, "The table %@ was already created with different columns. Choose another name." : { "localizations" : { @@ -194656,9 +194739,6 @@ } } } - }, - "The text at byte %lld is not valid UTF-8." : { - }, "The team library asked TablePro to download it again." : { @@ -194674,6 +194754,9 @@ }, "The team library refused the change. It may be too large or hold a field the library never stores." : { + }, + "The text at byte %lld is not valid UTF-8." : { + }, "The text could not be parsed as JSON." : { "localizations" : { @@ -194757,6 +194840,9 @@ }, "The text encoding is invalid." : { + }, + "The text has no CREATE TRIGGER statement. Saving it would drop the trigger without creating it again, so nothing was run." : { + }, "The text is not valid JSON." : { "extractionState" : "manual", @@ -195034,6 +195120,12 @@ } } } + }, + "The trigger was dropped, the new definition failed, and the original could not be created again. The trigger no longer exists. Copy the original definition to recreate it. Error: %@" : { + + }, + "The trigger's full definition could not be read, so it could not be restored if the edit failed. Nothing was run." : { + }, "The triggers on %1$@ in %2$@ could not be listed: %3$@" : { "localizations" : { @@ -201647,6 +201739,9 @@ } } } + }, + "This tab moved to another database while the query waited for approval. Run it again." : { + }, "This table has no options to change." : { @@ -217859,6 +217954,15 @@ } } } + }, + "You were signed out. Sign in again." : { + + }, + "You're on Free" : { + + }, + "You're signed in" : { + }, "You've used TablePro on %lld days" : { "localizations" : { @@ -217899,15 +218003,6 @@ } } } - }, - "You were signed out. Sign in again." : { - - }, - "You're on Free" : { - - }, - "You're signed in" : { - }, "Your account is not allowed to read the source of %@" : { "extractionState" : "manual", diff --git a/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift b/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift index 25000d75a0..208ee830fb 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel+Streaming.swift @@ -194,7 +194,7 @@ extension AIChatViewModel { let toolSpecs = await MainActor.run { (registry ?? ChatToolRegistry.shared).specs(in: scope) } - var workingTurns = chatMessages + var workingTurns = ChatToolCallRepair.answeringEveryToolCall(chatMessages) var executedRoundtrips = 0 while true { @@ -236,7 +236,13 @@ extension AIChatViewModel { assistantID: currentAssistantID, registry: registry ) - guard !Task.isCancelled else { return } + guard !Task.isCancelled else { + await self.closeInterruptedToolRound( + assistantID: currentAssistantID, + results: ChatToolCallRepair.interruptedResults(for: approvals.blocks) + ) + return + } let toolUseBlocks = approvals.blocks let approvedBlocks = toolUseBlocks.filter { @@ -250,12 +256,18 @@ extension AIChatViewModel { explicitlyApproved: approvals.explicitlyApproved, registry: registry ) - guard !Task.isCancelled else { return } - let toolResultBlocks = Self.synthesizeResults( for: toolUseBlocks, executed: executedResults ) + // A statement that ran before Stop still ran: its result stays in the transcript. + guard !Task.isCancelled else { + await self.closeInterruptedToolRound( + assistantID: currentAssistantID, + results: toolResultBlocks + ) + return + } let continuation = await self.completeToolRoundtrip( assistantIDForRound: currentAssistantID, toolUseBlocks: toolUseBlocks, @@ -364,7 +376,8 @@ extension AIChatViewModel { systemPrompt: systemPrompt, tools: toolSpecs, reasoningEffort: resolved.config.reasoningEffort, - sessionId: sessionId + sessionId: sessionId, + connectionId: connection?.id ) ) @@ -578,6 +591,15 @@ extension AIChatViewModel { } } + // Placed right after the calls it answers: the user may already have sent the next message. + func closeInterruptedToolRound(assistantID: UUID, results: [ToolResultBlock]) { + guard !results.isEmpty, let index = messages.firstIndex(where: { $0.id == assistantID }) else { return } + finalizeStreamingMessage(id: assistantID) + let turn = ChatTurn(wire: ChatTurnWire(role: .user, blocks: results.map { .toolResult($0) })) + messages.insert(turn, at: index + 1) + persistCurrentConversation() + } + func completeToolRoundtrip( assistantIDForRound: UUID, toolUseBlocks: [ToolUseBlock], diff --git a/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift b/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift index 9a269f1f66..13ef2193b5 100644 --- a/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift +++ b/apple/TablePro/Views/ConnectionForm/ConnectionFormCoordinator.swift @@ -64,6 +64,9 @@ final class ConnectionFormCoordinator: ObservableObject { @Published private var temporaryTestIds: Set = [] + // Filled by the reads that prefill the form: a field left empty because its secret could not be read is not a cleared one. + private var unreadableSecretKeys: Set = [] + private var childChangeForwarding: AnyCancellable? let services: AppServices @@ -222,18 +225,20 @@ final class ConnectionFormCoordinator: ObservableObject { "[trace] load found existing id=\(existing.id.uuidString, privacy: .public) name='\(existing.name, privacy: .private(mask: .hash))' promptForPassword=\(existing.promptForPassword)" ) originalConnection = existing - network.load(from: existing) - auth.load(from: existing, storage: storage) - ssh.load(from: existing, storage: storage) - cloudflareTunnel.load(from: existing, storage: storage) - cloudSQLProxy.load(from: existing, storage: storage) - socksProxy.load(from: existing, storage: storage) - tunnelCommand.load(from: existing) - ssl.load(from: existing) - customization.load(from: existing) - advanced.load(from: existing) - aiRules.load(from: existing) - maskingRules.load(from: existing) + unreadableSecretKeys = storage.recordingUnreadableSecrets { + network.load(from: existing) + auth.load(from: existing, storage: storage) + ssh.load(from: existing, storage: storage) + cloudflareTunnel.load(from: existing, storage: storage) + cloudSQLProxy.load(from: existing, storage: storage) + socksProxy.load(from: existing, storage: storage) + tunnelCommand.load(from: existing) + ssl.load(from: existing) + customization.load(from: existing) + advanced.load(from: existing) + aiRules.load(from: existing) + maskingRules.load(from: existing) + } normalizeTransport() } hasLoadedData = true @@ -435,19 +440,13 @@ final class ConnectionFormCoordinator: ObservableObject { var edits = buildEdits() edits.additionalFields["promptForPassword"] = auth.effectivePromptForPassword ? "true" : nil + var secureFieldValues: [(fieldId: String, value: String)] = [] for field in secureFieldsOwnedByForm() { - if let value = edits.additionalFields[field.id], !value.isEmpty { - storage.savePluginSecureField(value, fieldId: field.id, for: finalId) - } else { - storage.deletePluginSecureField(fieldId: field.id, for: finalId) - } - edits.additionalFields.removeValue(forKey: field.id) + secureFieldValues.append((field.id, edits.additionalFields.removeValue(forKey: field.id) ?? "")) } // An installed plugin older than the field still must not leave the token in the record. let sessionTokenId = ConnectionSecretOptions.mongoSessionTokenFieldId - if let token = edits.additionalFields.removeValue(forKey: sessionTokenId), !token.isEmpty { - storage.savePluginSecureField(token, fieldId: sessionTokenId, for: finalId) - } + let sessionToken = edits.additionalFields.removeValue(forKey: sessionTokenId) ?? "" var connectionToSave = edits.applied(to: baseConnection(id: finalId)) @@ -456,57 +455,79 @@ final class ConnectionFormCoordinator: ObservableObject { /// connection in place with nothing left to authenticate it. var clearedSecrets: [() -> Void] = [] - /// A linked connection holds no secret of its own. That is the whole point: the password - /// exists once, under the profile, so rotating it is one edit rather than one per - /// connection. Any secret the connection had before the link goes with it, or an encrypted - /// export still carries it and a duplicate still copies it. - if auth.usesCredentialProfile { - clearedSecrets.append { self.storage.deletePassword(for: finalId) } - /// The secure-field loop above wrote the form's values back under the connection id. - /// Anything the profile owns has to go with the password, or the connection keeps a - /// second stale copy that a duplicate carries and the driver can fall back to. - for fieldId in auth.selectedCredentialProfile?.secureFieldIds ?? [] { - clearedSecrets.append { self.storage.deletePluginSecureField(fieldId: fieldId, for: finalId) } + let stagedSecrets = storage.stagingSecretEdits( + for: finalId, + deviceOnly: !connectionToSave.participatesInSync, + keeping: unreadableSecretKeys + ) { + for field in secureFieldValues { + if field.value.isEmpty { + storage.deletePluginSecureField(fieldId: field.fieldId, for: finalId) + } else { + storage.savePluginSecureField(field.value, fieldId: field.fieldId, for: finalId) + } + } + if !sessionToken.isEmpty { + storage.savePluginSecureField(sessionToken, fieldId: sessionTokenId, for: finalId) } - } else if auth.effectivePromptForPassword || auth.clearsStoredPassword { - clearedSecrets.append { self.storage.deletePassword(for: finalId) } - } else if !auth.password.isEmpty { - storage.savePassword(auth.password, for: connectionToSave.id) - } - if ssh.state.enabled && ssh.state.profileId == nil { - if (ssh.state.authMethod == .password || ssh.state.authMethod == .keyboardInteractive) - && !ssh.state.password.isEmpty - { - storage.saveSSHPassword(ssh.state.password, for: connectionToSave.id) - } else if ssh.state.clearsStoredPassword { - clearedSecrets.append { self.storage.deleteSSHPassword(for: finalId) } + /// A linked connection holds no secret of its own. That is the whole point: the password + /// exists once, under the profile, so rotating it is one edit rather than one per + /// connection. Any secret the connection had before the link goes with it, or an encrypted + /// export still carries it and a duplicate still copies it. + if auth.usesCredentialProfile { + clearedSecrets.append { self.storage.deletePassword(for: finalId) } + /// The secure-field loop above wrote the form's values back under the connection id. + /// Anything the profile owns has to go with the password, or the connection keeps a + /// second stale copy that a duplicate carries and the driver can fall back to. + for fieldId in auth.selectedCredentialProfile?.secureFieldIds ?? [] { + clearedSecrets.append { self.storage.deletePluginSecureField(fieldId: fieldId, for: finalId) } + } + } else if auth.effectivePromptForPassword || auth.clearsStoredPassword { + clearedSecrets.append { self.storage.deletePassword(for: finalId) } + } else if !auth.password.isEmpty { + storage.savePassword(auth.password, for: finalId) } - if ssh.state.authMethod == .privateKey && !ssh.state.keyPassphrase.isEmpty { - storage.saveKeyPassphrase(ssh.state.keyPassphrase, for: connectionToSave.id) - } else if ssh.state.clearsStoredKeyPassphrase { - clearedSecrets.append { self.storage.deleteKeyPassphrase(for: finalId) } + + if ssh.state.enabled && ssh.state.profileId == nil { + if (ssh.state.authMethod == .password || ssh.state.authMethod == .keyboardInteractive) + && !ssh.state.password.isEmpty + { + storage.saveSSHPassword(ssh.state.password, for: finalId) + } else if ssh.state.clearsStoredPassword { + clearedSecrets.append { self.storage.deleteSSHPassword(for: finalId) } + } + if ssh.state.authMethod == .privateKey && !ssh.state.keyPassphrase.isEmpty { + storage.saveKeyPassphrase(ssh.state.keyPassphrase, for: finalId) + } else if ssh.state.clearsStoredKeyPassphrase { + clearedSecrets.append { self.storage.deleteKeyPassphrase(for: finalId) } + } + if ssh.state.totpMode == .autoGenerate && !ssh.state.totpSecret.isEmpty { + storage.saveTOTPSecret(ssh.state.totpSecret, for: finalId) + } else { + storage.deleteTOTPSecret(for: finalId) + } + } else { + storage.deleteSSHPassword(for: finalId) + storage.deleteKeyPassphrase(for: finalId) + storage.deleteTOTPSecret(for: finalId) } - if ssh.state.totpMode == .autoGenerate && !ssh.state.totpSecret.isEmpty { - storage.saveTOTPSecret(ssh.state.totpSecret, for: connectionToSave.id) + + if !ssl.clientKeyPassphrase.isEmpty && !ssl.clientKeyPath.trimmingCharacters(in: .whitespaces).isEmpty { + storage.saveSSLClientKeyPassphrase(ssl.clientKeyPassphrase, for: finalId) } else { - storage.deleteTOTPSecret(for: connectionToSave.id) + storage.deleteSSLClientKeyPassphrase(for: finalId) } - } else { - storage.deleteSSHPassword(for: connectionToSave.id) - storage.deleteKeyPassphrase(for: connectionToSave.id) - storage.deleteTOTPSecret(for: connectionToSave.id) - } - if !ssl.clientKeyPassphrase.isEmpty && !ssl.clientKeyPath.trimmingCharacters(in: .whitespaces).isEmpty { - storage.saveSSLClientKeyPassphrase(ssl.clientKeyPassphrase, for: connectionToSave.id) - } else { - storage.deleteSSLClientKeyPassphrase(for: connectionToSave.id) + cloudflareTunnel.save(to: finalId, storage: storage) + cloudSQLProxy.save(to: finalId, storage: storage) + socksProxy.save(to: finalId, storage: storage) } - - cloudflareTunnel.save(to: connectionToSave.id, storage: storage) - cloudSQLProxy.save(to: connectionToSave.id, storage: storage) - socksProxy.save(to: connectionToSave.id, storage: storage) + guard !stagedSecrets.writeFailed else { + saveError = String(localized: "Could not save this connection's passwords to the Keychain. Your changes were not saved.") + return + } + clearedSecrets.append { self.storage.applyStagedDeletions(stagedSecrets) } var savedConnections = storage.loadConnections() if isNew { @@ -804,50 +825,53 @@ final class ConnectionFormCoordinator: ObservableObject { let cloudSQLProxyState = tunnelStates.cloudSQLProxy let socksProxyState = tunnelStates.socksProxy - if !password.isEmpty && !promptForPassword { - services.connectionStorage.savePassword(password, for: testId) - } - if sshState.enabled && sshState.profileId == nil { - if (sshState.authMethod == .password || sshState.authMethod == .keyboardInteractive) - && !sshState.password.isEmpty - { - services.connectionStorage.saveSSHPassword(sshState.password, for: testId) - } - if sshState.authMethod == .privateKey && !sshState.keyPassphrase.isEmpty { - services.connectionStorage.saveKeyPassphrase(sshState.keyPassphrase, for: testId) + // A test's copies never leave this Mac, whatever the password sync setting says. + _ = services.connectionStorage.stagingSecretEdits(for: testId, deviceOnly: true) { + if !password.isEmpty && !promptForPassword { + services.connectionStorage.savePassword(password, for: testId) } - if sshState.totpMode == .autoGenerate && !sshState.totpSecret.isEmpty { - services.connectionStorage.saveTOTPSecret(sshState.totpSecret, for: testId) + if sshState.enabled && sshState.profileId == nil { + if (sshState.authMethod == .password || sshState.authMethod == .keyboardInteractive) + && !sshState.password.isEmpty + { + services.connectionStorage.saveSSHPassword(sshState.password, for: testId) + } + if sshState.authMethod == .privateKey && !sshState.keyPassphrase.isEmpty { + services.connectionStorage.saveKeyPassphrase(sshState.keyPassphrase, for: testId) + } + if sshState.totpMode == .autoGenerate && !sshState.totpSecret.isEmpty { + services.connectionStorage.saveTOTPSecret(sshState.totpSecret, for: testId) + } } - } - if !sslClientKeyPassphrase.isEmpty - && !sslClientKeyPath.trimmingCharacters(in: .whitespaces).isEmpty - { - services.connectionStorage.saveSSLClientKeyPassphrase(sslClientKeyPassphrase, for: testId) - } + if !sslClientKeyPassphrase.isEmpty + && !sslClientKeyPath.trimmingCharacters(in: .whitespaces).isEmpty + { + services.connectionStorage.saveSSLClientKeyPassphrase(sslClientKeyPassphrase, for: testId) + } - if cloudflareState.enabled && cloudflareState.authMethod == .serviceToken { - services.connectionStorage.saveCloudflareTokenId(cloudflareState.serviceTokenId, for: testId) - services.connectionStorage.saveCloudflareTokenSecret(cloudflareState.serviceTokenSecret, for: testId) - } + if cloudflareState.enabled && cloudflareState.authMethod == .serviceToken { + services.connectionStorage.saveCloudflareTokenId(cloudflareState.serviceTokenId, for: testId) + services.connectionStorage.saveCloudflareTokenSecret(cloudflareState.serviceTokenSecret, for: testId) + } - if cloudSQLProxyState.enabled && cloudSQLProxyState.authMode == .serviceAccountKey - && !cloudSQLProxyState.serviceAccountKeyJSON.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { - services.connectionStorage.saveCloudSQLProxyServiceAccountKey( - cloudSQLProxyState.serviceAccountKeyJSON, for: testId - ) - } + if cloudSQLProxyState.enabled && cloudSQLProxyState.authMode == .serviceAccountKey + && !cloudSQLProxyState.serviceAccountKeyJSON.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + services.connectionStorage.saveCloudSQLProxyServiceAccountKey( + cloudSQLProxyState.serviceAccountKeyJSON, for: testId + ) + } - if socksProxyState.enabled && !socksProxyState.password.isEmpty { - services.connectionStorage.saveSOCKSProxyPassword(socksProxyState.password, for: testId) - } + if socksProxyState.enabled && !socksProxyState.password.isEmpty { + services.connectionStorage.saveSOCKSProxyPassword(socksProxyState.password, for: testId) + } - for field in services.pluginManager.additionalConnectionFields(for: connectionType) - where field.isSecure - { - if let value = additionalFieldValues[field.id], !value.isEmpty { - services.connectionStorage.savePluginSecureField(value, fieldId: field.id, for: testId) + for field in services.pluginManager.additionalConnectionFields(for: connectionType) + where field.isSecure + { + if let value = additionalFieldValues[field.id], !value.isEmpty { + services.connectionStorage.savePluginSecureField(value, fieldId: field.id, for: testId) + } } } } diff --git a/apple/TablePro/Views/ConnectionForm/ViewModels/SSLPaneViewModel.swift b/apple/TablePro/Views/ConnectionForm/ViewModels/SSLPaneViewModel.swift index b13af4253b..c43a433f24 100644 --- a/apple/TablePro/Views/ConnectionForm/ViewModels/SSLPaneViewModel.swift +++ b/apple/TablePro/Views/ConnectionForm/ViewModels/SSLPaneViewModel.swift @@ -63,7 +63,8 @@ final class SSLPaneViewModel: ObservableObject { caCertPath = connection.sslConfig.caCertificatePath clientCertPath = connection.sslConfig.clientCertificatePath clientKeyPath = connection.sslConfig.clientKeyPath - clientKeyPassphrase = ConnectionStorage.shared.loadSSLClientKeyPassphrase(for: connection.id) ?? "" + let storage = coordinator?.value?.storage ?? .shared + clientKeyPassphrase = storage.loadSSLClientKeyPassphrase(for: connection.id) ?? "" } func resetForType(_ type: DatabaseType) { diff --git a/apple/TablePro/Views/Editor/EditorEventRouter.swift b/apple/TablePro/Views/Editor/EditorEventRouter.swift index fe18f4c44c..300d8e2b9d 100644 --- a/apple/TablePro/Views/Editor/EditorEventRouter.swift +++ b/apple/TablePro/Views/Editor/EditorEventRouter.swift @@ -7,6 +7,7 @@ // @preconcurrency import AppKit +import TableProEditorKit import TableProTextEngine @MainActor @@ -157,6 +158,15 @@ internal final class EditorEventRouter { coordinator.runStatementAtCursorAndAdvance() } + // A large document writes its text to the tab on a delay; a run or an AI action reads the tab. + internal func flushPendingText(forTab tabId: UUID, connectionId: UUID) { + for ref in editors.values { + guard let coordinator = ref.coordinator, + coordinator.tabID == tabId, coordinator.connectionId == connectionId else { continue } + coordinator.controller?.flushPendingTextWriteback() + } + } + /// Called when the Edit menu's "Clear Selection" item is chosen with the pointer. /// /// Only with the pointer. AppKit never dispatches an unmodified Escape as a key equivalent: diff --git a/apple/TablePro/Views/Editor/SQLEditorCoordinator.swift b/apple/TablePro/Views/Editor/SQLEditorCoordinator.swift index 61f0e35e5a..1ec3d48dc9 100644 --- a/apple/TablePro/Views/Editor/SQLEditorCoordinator.swift +++ b/apple/TablePro/Views/Editor/SQLEditorCoordinator.swift @@ -853,13 +853,15 @@ final class SQLEditorCoordinator: ObservableObject, TextViewCoordinator, TextVie guard !isUppercasing, keywordCase.rewritesTypedText, KeywordUppercaseHelper.isWordBoundary(string), + PluginManager.shared.editorLanguage(for: databaseType ?? .mysql) == .sql, (textView.textStorage.string as NSString).length < 500_000 else { return } let nsText = textView.textStorage.string as NSString guard let match = KeywordUppercaseHelper.keywordBeforePosition( nsText, at: range.location, - uppercase: keywordCase.prefersUppercase + uppercase: keywordCase.prefersUppercase, + grammar: lexicalGrammar ) else { return } let word = match.word diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AIQueryActions.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AIQueryActions.swift index 98aa44d8da..1dc5c1c0d3 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AIQueryActions.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+AIQueryActions.swift @@ -50,6 +50,7 @@ extension MainContentCoordinator { } func runAIQueryAction(_ action: AIQueryAction, target: AIQueryTarget = .selectionOrStatementAtCursor) { + if let tabId = tabManager.selectedTabId { flushEditorText(forTab: tabId) } guard aiQueryActionAvailability.isEnabled, let tab = tabManager.selectedTab, let request = aiQueryRequest(for: action, in: tab, target: target) else { return } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Alerts.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Alerts.swift index b3aacae28f..fda3245d62 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Alerts.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Alerts.swift @@ -41,6 +41,8 @@ extension MainContentCoordinator { return String(localized: "Navigating to another page will discard all unsaved changes.") case .filter: return String(localized: "Applying or clearing filters will reload data and discard all unsaved changes.") + case .run: + return String(localized: "Running the query will replace this result and discard all unsaved changes.") case .resultSwitch: return String(localized: "Showing another result will discard all unsaved changes.") case .resultClose: diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ExecuteAll.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ExecuteAll.swift index 520eea9ae1..dae3b24d50 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ExecuteAll.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+ExecuteAll.swift @@ -7,8 +7,38 @@ import Foundation extension MainContentCoordinator { func runAllStatements(extraCapabilities: CallerCapabilities = []) { - commitActiveGridEdit() - queryExecutionCoordinator.runAllStatements(extraCapabilities: extraCapabilities) + guard let tab = tabManager.selectedTabAndIndex?.tab, tab.tabType == .query, + !tabExecution.isExecuting(tab.id) else { return } + confirmRunDiscard(onTab: tab.id) { [weak self] in + guard let self else { return } + flushEditorText(forTab: tab.id) + queryExecutionCoordinator.runAllStatements(extraCapabilities: extraCapabilities) + } + } + + // A run authorized for a tab stays on it; nil is the selected tab. + func executionTab(_ tabId: UUID?) -> (tab: QueryTab, index: Int)? { + guard let tabId else { return tabManager.selectedTabAndIndex } + guard let index = tabManager.tabs.firstIndex(where: { $0.id == tabId }) else { return nil } + return (tabManager.tabs[index], index) + } + + // A run replaces the result the staged edits belong to. A tab with no result has none of its own: + // for a turn after a switch the change manager still holds the outgoing tab's. + func confirmRunDiscard(onTab tabId: UUID, then run: @escaping () -> Void) { + guard let tab = tabManager.selectedTabAndIndex?.tab, tab.id == tabId, tab.display.activeResultSet != nil else { + commitActiveGridEdit() + run() + return + } + confirmDiscardRestoringRowsIfNeeded(action: .run) { [weak self] confirmed in + guard let self, confirmed, tabManager.selectedTabId == tabId, !tabExecution.isExecuting(tabId) else { return } + run() + } + } + + func flushEditorText(forTab tabId: UUID) { + EditorEventRouter.shared.flushPendingText(forTab: tabId, connectionId: connectionId) } internal func dispatchBatches( diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Explain.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Explain.swift index 64385b570e..067f3496e6 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Explain.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Explain.swift @@ -15,6 +15,17 @@ import TableProSQLGrammar extension MainContentCoordinator { @discardableResult func runExplain(variant: ExplainVariant? = nil) -> Task? { + guard let tabId = tabManager.selectedTabId else { return nil } + // Nil when the discard prompt has to ask first: the plan follows the answer. + var started: Task? + confirmRunDiscard(onTab: tabId) { [weak self] in + started = self?.startExplain(variant: variant) + } + return started + } + + private func startExplain(variant: ExplainVariant?) -> Task? { + if let tabId = tabManager.selectedTabId { flushEditorText(forTab: tabId) } guard let (tab, index) = tabManager.selectedTabAndIndex else { return nil } guard !tabExecution.isExecuting(tab.id) else { traceExecutionBlocked(tabId: tab.id, site: "runExplain") @@ -197,6 +208,7 @@ extension MainContentCoordinator { } } seedBufferFromActiveResult(tabId: tabId) + dropParkedEdits(ofBackgroundTab: tabId) toolbarState.isResultsCollapsed = false recordHistory( diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift index dec5899f1f..d7eab33615 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift @@ -560,7 +560,10 @@ extension MainContentCoordinator { /// Each value comes from the live session, never from `toolbarState`: a database switch moves /// the session schema without touching the toolbar, so comparing against the toolbar skips /// the switch exactly when the session needs it. - func switchContainers(database: String?, schema: String?) async { + /// + /// False when a step failed or was cancelled: the session is still wherever it was. + @discardableResult + func switchContainers(database: String?, schema: String?) async -> Bool { let steps = ContainerSwitchPlanner.plan( database: database, schema: schema, @@ -579,17 +582,19 @@ extension MainContentCoordinator { didSwitch = true /// A schema belongs to a database, so a failed database switch stops the plan /// rather than applying the schema against whatever is still open. - guard await switchDatabase(to: name) else { return } + guard await switchDatabase(to: name) else { return false } case .schema(let name): guard name != session?.browseSchema || !isSettled else { continue } didSwitch = true await switchSchema(to: name) + guard services.databaseManager.session(for: connectionId)?.browseSchema == name else { return false } } } /// Already there is still a request to be there: the window may be showing another entry. if !didSwitch { browseCursorCoordinator.showBrowsedContainer() } + return true } private var schemaEntityName: String { diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TabSwitch.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TabSwitch.swift index 00257963f2..40e812529e 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TabSwitch.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+TabSwitch.swift @@ -43,8 +43,12 @@ extension MainContentCoordinator { let oldIndex = tabManager.tabs.firstIndex(where: { $0.id == oldId }) { // Also checks the tab's snapshot: on `hasChanges` alone an undo kept a stale snapshot that restored the undone edit on return. - if changeManager.hasChanges || tabManager.tabs[oldIndex].pendingChanges.hasChanges { - let savedState = changeManager.saveState() + // Only on the switch the result landed in: a mark left by a reselect this handler never saw is dropped. + if let selectedAtLanding = tabsGivenResultOffScreen.removeValue(forKey: oldId), selectedAtLanding == newTabId { + tabManager.mutate(at: oldIndex) { $0.pendingChanges = TabChangeSnapshot() } + } else if changeManager.hasChanges || tabManager.tabs[oldIndex].pendingChanges.hasChanges { + var savedState = changeManager.saveState() + savedState.resultSetId = tabManager.tabs[oldIndex].display.activeResultSetId tabManager.mutate(at: oldIndex) { $0.pendingChanges = savedState } } // One editor serves every query tab with no `.id(tab.id)`, so write only once this tab's restored caret has been consumed. @@ -72,13 +76,18 @@ extension MainContentCoordinator { let newIndex = tabManager.tabs.firstIndex(where: { $0.id == newId }) { let newTab = tabManager.tabs[newIndex] let newRows = tabSessionRegistry.tableRows(for: newId) + tabsGivenResultOffScreen.removeValue(forKey: newId) toolbarState.isTableTab = newTab.tabType == .table toolbarState.isResultsCollapsed = newTab.display.isResultsCollapsed syncQueryToolbarState(for: newTab) // Consumed once restored: a copy left on the tab went stale with the first undo and kept reporting undone edits. - let pendingState = newTab.pendingChanges + var pendingState = newTab.pendingChanges + if pendingState.hasChanges, pendingState.resultSetId != newTab.display.activeResultSetId { + pendingState = TabChangeSnapshot() + tabManager.mutate(at: newIndex) { $0.pendingChanges = pendingState } + } if pendingState.hasChanges { changeManager.restoreState( from: pendingState, @@ -137,6 +146,22 @@ extension MainContentCoordinator { } } + // Edits parked on a background tab name rows of the result it held; a result installed meanwhile makes them + // another table's on return. + func dropParkedEdits(ofBackgroundTab tabId: UUID) { + guard tabManager.selectedTabId != tabId else { return } + tabsGivenResultOffScreen[tabId] = .some(tabManager.selectedTabId) + tabManager.mutate(tabId: tabId) { tab in + if tab.pendingChanges.hasChanges { tab.pendingChanges = TabChangeSnapshot() } + } + } + + // The selected tab's change manager now describes the result on screen. + func changeManagerMatchesSelectedResult() { + guard let selected = tabManager.selectedTabId else { return } + tabsGivenResultOffScreen.removeValue(forKey: selected) + } + private func commitGridEdit(ofTab tabId: UUID) { existingGridModel(forTab: tabId)?.commitEditing() guard let tab = tabManager.tabs.first(where: { $0.id == tabId }) else { return } diff --git a/apple/TablePro/Views/Main/MainContentCommandActions.swift b/apple/TablePro/Views/Main/MainContentCommandActions.swift index ee0a6abd59..51561b8731 100644 --- a/apple/TablePro/Views/Main/MainContentCommandActions.swift +++ b/apple/TablePro/Views/Main/MainContentCommandActions.swift @@ -752,6 +752,8 @@ final class MainContentCommandActions: ObservableObject { return await withCheckedContinuation { continuation in coordinator.saveCompletionContinuation = continuation saveChanges() + // Still there when the command went somewhere other than the grid save, which reports nothing back. + coordinator.takeSaveCompletion()?.resume(returning: false) } } diff --git a/apple/TablePro/Views/Main/MainContentCoordinator.swift b/apple/TablePro/Views/Main/MainContentCoordinator.swift index 8d3cde5fab..cd6e6d40bd 100644 --- a/apple/TablePro/Views/Main/MainContentCoordinator.swift +++ b/apple/TablePro/Views/Main/MainContentCoordinator.swift @@ -25,6 +25,7 @@ enum DiscardAction { case navigation case displayOrder case columnVisibility + case run } struct DisplayFormatsCacheEntry { @@ -206,7 +207,7 @@ final class MainContentCoordinator: ObservableObject { // The cached view model keeps staged work after `usersRolesActions` is nilled; without this a background tab closes silently. internal var tabsWithStagedPrincipals: Set = [] - // `saveCompletionContinuation` is one slot: a second close gesture would overwrite it and strand the first task forever. + // A second close gesture on a tab whose save-then-close is still running is dropped. internal var tabClosesInFlight: Set = [] // Bumped when a published schema row changes, so the inspector re-reads it. @@ -227,6 +228,10 @@ final class MainContentCoordinator: ObservableObject { var dataGridEntries: [UUID: DataTabGridEntry] = [:] // Set while a tab switch still has the outgoing tab's edits in the change manager. var tabSwitchOutgoingTabId: UUID? + // Tabs given a result while off screen, with the tab selected at that moment. One still listed when the switch + // handler parks its edits on the way to that same tab got the result before the handler ran, so the edits in + // the change manager are the old result's. + var tabsGivenResultOffScreen: [UUID: UUID?] = [:] private(set) lazy var gridChangeState = AnyChangeManager(changeManager) let gridCellsEdited = PassthroughSubject() @@ -368,7 +373,7 @@ final class MainContentCoordinator: ObservableObject { // The rebuild a column drag asked for, held while the user reviews it. @Published internal var tableRebuildRequest: TableRebuildReviewRequest? - // For save-then-close callers. Set before `saveChanges`, resumed by `executeCommitStatements`. + // A handoff for save-then-close callers: set right before `saveChanges`, which takes it and resumes it when that save ends. internal var saveCompletionContinuation: CheckedContinuation? // MARK: - Window Lifecycle (driven by TabWindowController NSWindowDelegate) @@ -420,6 +425,26 @@ final class MainContentCoordinator: ObservableObject { _didActivate.withLock { $0 } } + // Also for a workspace that closes while it has no session, and so no coordinator to tear down. + // A workspace that lost its session has no coordinator and still reconnects into the tab set, so it holds the gate. + static func endTabSessionIfUnhosted( + for connectionId: UUID, + closing: ConnectionWorkspace? = nil, + hosted: [ConnectionWorkspace]? = nil + ) { + let workspaces = hosted ?? WindowManager.shared.workspaces(for: connectionId) + guard !workspaces.contains(where: { $0 !== closing }), !hostsLiveSession(for: connectionId) else { return } + TabPersistenceCoordinator.forConnection(connectionId).endSession() + } + + // Leaves out the throwaway instances SwiftUI builds while re-evaluating a body. + private static func hostsLiveSession(for connectionId: UUID) -> Bool { + activeCoordinators.values.contains { coordinator in + coordinator.connectionId == connectionId && !coordinator.isTearingDown + && (coordinator.isActivated || coordinator.hasRestoredTabs) + } + } + // Union across windows: saving one window's tabs erased torn-off ones. Deduped, since SwiftUI's throwaway coordinators repeat tabs. // The connection's saved tab set. A debug tab is never saved, so a window holding only debug tabs counts as empty. static func aggregatedTabs(for connectionId: UUID) -> [QueryTab] { @@ -828,7 +853,8 @@ final class MainContentCoordinator: ObservableObject { } // Never from `onDisappear`: a workspace switch unparents panes the connection comes back to. Releases synchronously, not via deinit. - func teardown() { + // `endsTabSession` is false when only the session went: the workspace stays and reconnects into the same tab set. + func teardown(endsTabSession: Bool = true) { let start = Date() Self.lifecycleLogger.info( "[close] MainContentCoordinator.teardown start connId=\(self.connection.id, privacy: .public) tabs=\(self.tabManager.tabs.count) windowId=\(self.windowId?.uuidString ?? "nil", privacy: .public)" @@ -837,6 +863,11 @@ final class MainContentCoordinator: ObservableObject { sheetWork.stopAll() unregisterFromPersistence() + if endsTabSession { + Self.endTabSessionIfUnhosted( + for: connectionId, closing: splitViewController?.workspaces.workspace(for: connectionId) + ) + } dropMovesAwaitingRestore(reason: "coordinator torn down") SessionRecoveryTracker.sync() if let observer = terminationObserver { @@ -963,37 +994,48 @@ final class MainContentCoordinator: ObservableObject { viewport: GridReloadIntent, trigger: TableLoadTrigger = .userInitiated, bypassRowLimit: Bool = false, extraCapabilities: CallerCapabilities = [] ) { - guard let (tab, index) = tabManager.selectedTabAndIndex else { return } + guard let tab = tabManager.selectedTabAndIndex?.tab else { return } guard !tabExecution.isExecuting(tab.id) else { traceExecutionBlocked(tabId: tab.id, site: "runQuery") return } - commitActiveGridEdit() if tab.tabType == .table { + commitActiveGridEdit() executeTableTabQueryDirectly(trigger: trigger, viewport: viewport) return } - // The offset lets each result point back at its statement. Both paths pass untrimmed text and let the scanner trim. - let target = selectionOrStatementAtCursor(in: tab.content.query) - executeResolvedSQL( - target.sql, tabIndex: index, bypassRowLimit: bypassRowLimit, - sourceOffset: target.offset, extraCapabilities: extraCapabilities - ) + let tabId = tab.id + confirmRunDiscard(onTab: tabId) { [weak self] in + guard let self else { return } + flushEditorText(forTab: tabId) + guard let (tab, index) = executionTab(tabId) else { return } + // The offset lets each result point back at its statement. Both paths pass untrimmed text and let the scanner trim. + let target = selectionOrStatementAtCursor(in: tab.content.query) + executeResolvedSQL( + target.sql, tabIndex: index, bypassRowLimit: bypassRowLimit, + sourceOffset: target.offset, extraCapabilities: extraCapabilities + ) + } } // Takes SQL, not a range: the editor text and the tab binding can briefly differ, and a range on the wrong one truncates silently. @discardableResult func runStatement(_ sql: String, sourceOffset: Int? = nil) -> Bool { - guard let (tab, index) = tabManager.selectedTabAndIndex, tab.tabType == .query else { return false } + guard let tab = tabManager.selectedTabAndIndex?.tab, tab.tabType == .query else { return false } guard !tabExecution.isExecuting(tab.id) else { traceExecutionBlocked(tabId: tab.id, site: "runStatement") return false } - commitActiveGridEdit() - return executeResolvedSQL(sql, tabIndex: index, bypassRowLimit: false, sourceOffset: sourceOffset) + // False when the discard prompt has to ask first: the run follows the answer, the caret stays. + var started = false + confirmRunDiscard(onTab: tab.id) { [weak self] in + guard let self, let (_, index) = executionTab(tab.id) else { return } + started = executeResolvedSQL(sql, tabIndex: index, bypassRowLimit: false, sourceOffset: sourceOffset) + } + return started } // Returns false when a parameter panel opens instead, so callers don't advance the caret on a run that didn't happen. @@ -1011,7 +1053,7 @@ final class MainContentCoordinator: ObservableObject { if services.appSettings.editor.queryParametersEnabled, bindsNamedParameters { let combinedSQL = SQLParameterExtractor.parameterSource(of: statements) - let detectedNames = SQLParameterExtractor.extractParameters(from: combinedSQL) + let detectedNames = SQLParameterExtractor.extractParameters(from: combinedSQL, grammar: lexicalGrammar) if !detectedNames.isEmpty { let reconciled = detectAndReconcileParameters( @@ -1097,7 +1139,9 @@ final class MainContentCoordinator: ObservableObject { } switch decision { case .authorized: - executeQueryInternal(sql, isAutoLoad: true, trigger: trigger, viewport: viewport, source: .sameSource) + executeQueryInternal( + sql, isAutoLoad: true, trigger: trigger, viewport: viewport, source: .sameSource, onTab: tabId + ) case .denied(let reason, _): traceNavigationAbandoned(tabId: tabId, outcome: .safeModeDenied) queryExecutionCoordinator.presentTabFailure(reason, announcing: reason, onTab: tabId) @@ -1174,9 +1218,10 @@ final class MainContentCoordinator: ObservableObject { bypassRowLimit: Bool = false, anchor: StatementAnchor? = nil, viewport: GridReloadIntent = .firstRow, - source: ResultSourceChange = .newSource + source: ResultSourceChange = .newSource, + onTab target: UUID? = nil ) { - guard let (selectedTab, index) = tabManager.selectedTabAndIndex else { return } + guard let (selectedTab, index) = executionTab(target) else { return } let (claim, lease) = beginTabExecution(for: selectedTab.id) diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Commands.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Commands.swift index e406897762..3d05770683 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Commands.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator+Commands.swift @@ -227,10 +227,11 @@ extension DatabaseTreeOutlineCoordinator { } /// A command that opens or edits an object has to reach the database that object lives in - /// first, which is what a click on the row would have done. - private func activateThen(_ ref: DatabaseTreeTableRef, _ body: @escaping @MainActor () -> Void) { + /// first, which is what a click on the row would have done. A failed or cancelled switch runs + /// nothing: the command would act on a same-named object where the session still is. + internal func activateThen(_ ref: DatabaseTreeTableRef, _ body: @escaping @MainActor () -> Void) { Task { @MainActor in - await activate(ref) + guard await activate(ref) else { return } body() } } diff --git a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator.swift b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator.swift index af9b8bcbac..c5afbe0e81 100644 --- a/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator.swift +++ b/apple/TablePro/Views/Sidebar/DatabaseTreeOutlineCoordinator.swift @@ -513,7 +513,7 @@ final class DatabaseTreeOutlineCoordinator: NSObject, NSTextFieldDelegate { openWork = Task { @MainActor in defer { openSelectionDepth -= 1 } await inFlight?.value - await activate(ref) + guard await activate(ref) else { return } mainCoordinator?.openTableTab( ref.table, schema: ref.schema, @@ -546,8 +546,9 @@ final class DatabaseTreeOutlineCoordinator: NSObject, NSTextFieldDelegate { windowState.select(tables: tables, rowCount: nodes.count) } - internal func activate(_ ref: DatabaseTreeTableRef) async { - await mainCoordinator?.switchContainers(database: ref.database, schema: ref.schema) + // False when the switch failed or was cancelled: the session is still on another database. + internal func activate(_ ref: DatabaseTreeTableRef) async -> Bool { + await mainCoordinator?.switchContainers(database: ref.database, schema: ref.schema) ?? true } internal func setActiveDatabase(_ database: String) { diff --git a/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift b/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift index c2f9c83325..b760819838 100644 --- a/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift +++ b/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift @@ -137,8 +137,11 @@ internal extension StructureEditingSession { /// are the ones it held, and only while they are still exactly those. guard let hold = changeManager.holdForSave() else { return .refused } isApplying = true - let outcome = await saveHeldChanges(hold.changes, coordinator: coordinator) + let (outcome, alterFailure) = await saveHeldChanges(hold.changes, coordinator: coordinator) let cleared = changeManager.releaseHold(hold, written: outcome == .applied) + if let alterFailure { + await settle(alterFailure, coordinator: coordinator) + } isApplying = false guard outcome == .applied else { return outcome } guard cleared else { @@ -154,10 +157,36 @@ internal extension StructureEditingSession { return outcome } + // A failed save that may have written; its error waits for the table to be read again. + private struct AlterFailure { + let failure: StructureApplyFailure + let startedAt: ContinuousClock.Instant + } + + // Runs with the hold released and the save still marked running, so nothing saves against the old baseline. + private func settle(_ alterFailure: AlterFailure, coordinator: MainContentCoordinator?) async { + let rebased = await rebaseAfterFailedSave() + present( + alterFailure.failure, + note: rebased ? Self.partlyAppliedNote : nil, + startedAt: alterFailure.startedAt, + coordinator: coordinator + ) + } + + private static var partlyAppliedNote: String { + String( + localized: """ + Some of these changes were applied before the save failed. The table was read again, \ + and only the changes that were not applied are still staged. + """ + ) + } + private func saveHeldChanges( _ changes: [SchemaChange], coordinator: MainContentCoordinator? - ) async -> StructureSaveOutcome { + ) async -> (StructureSaveOutcome, AlterFailure?) { let planStart = ContinuousClock.Instant.now let plan: StructureSavePlan do { @@ -169,7 +198,7 @@ internal extension StructureEditingSession { message: error.localizedDescription, window: coordinator?.contentWindow ) - return .failed(error.localizedDescription) + return (.failed(error.localizedDescription), nil) } switch plan { @@ -181,7 +210,10 @@ internal extension StructureEditingSession { /// The review sheet is that confirmation and shows the exact script, so the run it /// starts tells the gate it was confirmed rather than stacking the gate's own sheet over /// it for the same decision. The HIG's rule is one alert at a time. - return presentRebuildReview(prepared, preparedFrom: changes, startedAt: planStart, coordinator: coordinator) + let outcome = presentRebuildReview( + prepared, preparedFrom: changes, startedAt: planStart, coordinator: coordinator + ) + return (outcome, nil) case .alter(let script): return await applyAlterStatements(script, coordinator: coordinator) } @@ -195,7 +227,7 @@ internal extension StructureEditingSession { private func applyAlterStatements( _ script: SchemaChangeScript, coordinator: MainContentCoordinator? - ) async -> StructureSaveOutcome { + ) async -> (StructureSaveOutcome, AlterFailure?) { let operationStart = ContinuousClock.Instant.now do { @@ -206,20 +238,26 @@ internal extension StructureEditingSession { gate: executionGate ) report(.succeeded(OperationSummary()), startedAt: operationStart, coordinator: coordinator) - return .applied + return (.applied, nil) } catch { let failure = StructureApplyFailure(error) - present(failure, startedAt: operationStart, coordinator: coordinator) - return failure.outcome + // A gate refusal and a driver's refusal both stop before the first statement. + guard failure.reportsFailure, !(error is SchemaOperationRefusedError) else { + present(failure, startedAt: operationStart, coordinator: coordinator) + return (failure.outcome, nil) + } + return (failure.outcome, AlterFailure(failure: failure, startedAt: operationStart)) } } private func present( _ failure: StructureApplyFailure, + note: String? = nil, startedAt: ContinuousClock.Instant, coordinator: MainContentCoordinator? ) { - guard let message = failure.message else { return } + guard let reason = failure.message else { return } + let message = [reason, note].compactMap { $0 }.joined(separator: "\n\n") if failure.reportsFailure { report(.failed(reason: message), startedAt: startedAt, coordinator: coordinator) } diff --git a/apple/TablePro/Views/Structure/StructureEditingSession+Loading.swift b/apple/TablePro/Views/Structure/StructureEditingSession+Loading.swift index 7b08238b0c..bfa913a46e 100644 --- a/apple/TablePro/Views/Structure/StructureEditingSession+Loading.swift +++ b/apple/TablePro/Views/Structure/StructureEditingSession+Loading.swift @@ -172,6 +172,41 @@ internal extension StructureEditingSession { editor.refresh() } + // After a save that stopped part way: what the read shows as done is dropped, the rest stays staged. + func rebaseStagedChangesOntoFetchedDefinitions() -> Bool { + let pkFromIndexes = indexes.first(where: { $0.isPrimary })?.columns ?? [] + let pkFromColumns = columns.filter { $0.isPrimaryKey }.map { $0.name } + let moved = changeManager.rebaseStagedChanges( + onto: StructureRebase.Baseline( + columns: columns, + indexes: indexes, + foreignKeys: foreignKeys, + checkConstraints: checkConstraints, + primaryKey: pkFromIndexes.isEmpty ? pkFromColumns : pkFromIndexes, + tableComment: loadedTableComment, + tableOptions: loadedTableOptions, + autoIncrementConfirmed: autoIncrementRead == .confirmed + ) + ) + if moved { + editor.refresh() + } + return moved + } + + func rebaseAfterFailedSave() async -> Bool { + // Where DDL rolls back with its transaction, a failed save wrote nothing and the baseline still holds. + let rolledBack = try? await DatabaseManager.shared.withMetadataDriver(scope: scope) { driver in + DatabaseManager.wrapsDDLInTransaction(driver) + } + guard rolledBack == false else { return false } + isReloadingAfterSave = true + await reloadDefinitions() + isReloadingAfterSave = false + guard definitionsAreCurrent else { return false } + return rebaseStagedChangesOntoFetchedDefinitions() + } + func adoptFetchedBaseline() { guard !changeManager.hasChanges else { markStructureStale() diff --git a/apple/TablePro/Views/Structure/StructureRebuildPlanRunner.swift b/apple/TablePro/Views/Structure/StructureRebuildPlanRunner.swift index 4dcbc1ad50..a3ff46c722 100644 --- a/apple/TablePro/Views/Structure/StructureRebuildPlanRunner.swift +++ b/apple/TablePro/Views/Structure/StructureRebuildPlanRunner.swift @@ -34,9 +34,17 @@ enum StructureRebuildPlanRunner { case schemaChanged case verificationFailed(String) case executionFailed(String) + case sessionTransactionOpen var errorDescription: String? { switch self { + case .sessionTransactionOpen: + return String( + localized: """ + A transaction is open on this connection, and this change cannot run inside it. \ + Nothing was run. Commit or roll back the transaction, then try again. + """ + ) case .schemaChanged: return String( localized: """ @@ -138,10 +146,6 @@ enum StructureRebuildPlanRunner { throw RunError.schemaChanged } - for sql in plan.prologue { - _ = try? await driver.execute(query: sql) - } - /// Only the transaction this plan opened is ever rolled back, and none is opened over a /// transaction the session already holds: on the same shared session an app-owned /// `COMMIT` commits the user's pending work, and MySQL's `START TRANSACTION` commits it @@ -151,6 +155,15 @@ enum StructureRebuildPlanRunner { supportsTransactions: driver.supportsTransactions, sessionState: await driver.heldSessionTransactionState() ) + // Inside a transaction SQLite ignores `PRAGMA foreign_keys = off`, so the rebuild's DROP TABLE cascades. + if !plan.prologue.isEmpty, plan.isTransactional, owner == .session { + throw RunError.sessionTransactionOpen + } + + for sql in plan.prologue { + _ = try? await driver.execute(query: sql) + } + let usesTransaction = plan.isTransactional && owner.opensTransaction if usesTransaction { try await driver.beginTransaction(mode: .readWrite) diff --git a/apple/TablePro/Views/Structure/TriggerDetailView.swift b/apple/TablePro/Views/Structure/TriggerDetailView.swift index fb869f481a..a0885aac43 100644 --- a/apple/TablePro/Views/Structure/TriggerDetailView.swift +++ b/apple/TablePro/Views/Structure/TriggerDetailView.swift @@ -5,6 +5,7 @@ import Combine import SwiftUI +import TableProPluginKit @MainActor final class TriggerInspectorState: ObservableObject { @@ -96,6 +97,30 @@ enum TriggerListColumns { } } +enum TriggerEditSource { + // `statement` is only the action body on some engines, and an editor seeded with it saves the body in place of the trigger. + static func resolve( + _ trigger: TriggerInfo, + table: String, + databaseType: DatabaseType, + on driver: any DatabaseDriver + ) async throws -> String { + if let editable = try? await driver.fetchTriggerDefinition(name: trigger.name, table: table), !editable.isEmpty { + return editable + } + do { + let source = try await driver.fetchTriggerDDL(trigger) + if !source.isEmpty { return source } + } catch { + guard TriggerStatementShape.createsTrigger(trigger.statement, databaseType: databaseType) else { throw error } + } + guard TriggerStatementShape.createsTrigger(trigger.statement, databaseType: databaseType) else { + throw PluginObjectSourceError.unsupported(trigger.name) + } + return trigger.statement + } +} + private struct TriggerEditorSheetItem: Identifiable { let id = UUID() let mode: TriggerEditorView.Mode @@ -223,15 +248,22 @@ struct TriggerDetailView: View { Task { let scope = scope let tableName = tableName - let name = trigger.name - let fetched = try? await DatabaseManager.shared.withMetadataDriver(scope: scope) { driver in - try await driver.fetchTriggerDefinition(name: name, table: tableName) + let databaseType = connection.type + do { + let source = try await DatabaseManager.shared.withMetadataDriver(scope: scope) { driver in + try await TriggerEditSource.resolve(trigger, table: tableName, databaseType: databaseType, on: driver) + } + editorSheet = TriggerEditorSheetItem( + mode: .edit(originalName: trigger.name, originalDefinition: source), + sql: source + ) + } catch { + actionError = String( + format: String(localized: "The definition of trigger “%@” could not be read, so it cannot be edited: %@"), + trigger.name, + error.localizedDescription + ) } - let sql = (fetched ?? nil) ?? trigger.statement - editorSheet = TriggerEditorSheetItem( - mode: .edit(originalName: trigger.name, originalDefinition: trigger.statement), - sql: sql - ) } } diff --git a/apple/TablePro/Views/Structure/TriggerEditorView.swift b/apple/TablePro/Views/Structure/TriggerEditorView.swift index 2693da77dd..6dee7b596e 100644 --- a/apple/TablePro/Views/Structure/TriggerEditorView.swift +++ b/apple/TablePro/Views/Structure/TriggerEditorView.swift @@ -28,6 +28,7 @@ struct TriggerEditorView: View { @State private var editorConfiguration: SourceEditorConfiguration @State private var isApplying = false @State private var errorMessage: String? + @State private var droppedDefinition: String? @AppStorage("structureCodeFontSize", store: AppStorageEnvironment.shared.defaults) private var fontSize: Double = 13 init( @@ -71,6 +72,13 @@ struct TriggerEditorView: View { .font(.callout) .frame(maxWidth: .infinity, alignment: .leading) .padding(8) + if let droppedDefinition { + Button("Copy Original Definition") { + ClipboardService.shared.writeText(droppedDefinition) + } + .frame(maxWidth: .infinity, alignment: .leading) + .padding([.horizontal, .bottom], 8) + } } } .frame(minWidth: 560, idealWidth: 680, minHeight: 360, idealHeight: 460) @@ -84,7 +92,7 @@ struct TriggerEditorView: View { private var header: some View { HStack { - Text(isEdit ? "Edit Trigger" : "New Trigger") + Text(isOriginalEdit ? "Edit Trigger" : "New Trigger") .font(.headline) Spacer() Button("Cancel", role: .cancel) { onClose() } @@ -97,23 +105,28 @@ struct TriggerEditorView: View { .padding() } - private var isEdit: Bool { + private var isOriginalEdit: Bool { if case .edit = mode { return true } return false } + // Once a failed save has left the trigger dropped, the next save creates it; a second DROP would fail. + private var isEdit: Bool { + isOriginalEdit && droppedDefinition == nil + } + private func apply() { isApplying = true errorMessage = nil let originalName: String? let originalDefinition: String? switch mode { - case .create: - originalName = nil - originalDefinition = nil - case let .edit(name, definition): + case let .edit(name, definition) where droppedDefinition == nil: originalName = name originalDefinition = definition + case .create, .edit: + originalName = nil + originalDefinition = nil } Task { defer { isApplying = false } @@ -129,6 +142,9 @@ struct TriggerEditorView: View { ) onClose() } catch { + if let dropped = (error as? TriggerEditingError)?.droppedDefinition { + droppedDefinition = dropped + } errorMessage = error.localizedDescription } } diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBStringFieldWriteTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBStringFieldWriteTests.swift new file mode 100644 index 0000000000..44d54fc9eb --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBStringFieldWriteTests.swift @@ -0,0 +1,77 @@ +import Foundation +@testable import TableProMobile +import TableProPluginKit +import Testing + +struct MongoDBStringFieldWriteTests { + private static let locator = #"{"$oid":"507f1f77bcf86cd799439011"}"# + + private func statement( + editing old: String, + to new: String, + held: Set? + ) throws -> String { + var generator = MongoDBStatementGenerator(collectionName: "places", columns: ["_id", "zip"]) + generator.rowLocators = [0: Self.locator] + if let held { + generator.fieldKinds = MongoDBFieldKinds(["zip": held]) + } + let change = PluginRowChange( + rowIndex: 0, + type: .update, + cellChanges: [(1, "zip", .text(old), .text(new))], + originalRow: [.text("507f1f77bcf86cd799439011"), .text(old)] + ) + let writes = try generator.generateRowWrites( + from: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] + ) + return try #require(writes.first?.statement) + } + + @Test( + "Text that reads as a number, a boolean or null stays a string in a field that held only strings", + arguments: ["90211", "-1.5", "true", "false", "null"] + ) + func stringFieldKeepsStrings(text: String) throws { + let statement = try statement(editing: "90210", to: text, held: [.string]) + #expect(statement.contains(#""zip": "\#(text)""#)) + } + + @Test("A string field that also held nulls still stores number text as a string") + func stringAndNullField() throws { + let statement = try statement(editing: "90210", to: "90211", held: [.string, .null]) + #expect(statement.contains(#""zip": "90211""#)) + } + + @Test("A field that held numbers beside strings keeps reading number text as a number") + func mixedFieldInfersByShape() throws { + let statement = try statement(editing: "90210", to: "90211", held: [.string, .int32]) + #expect(statement.contains(#""zip": 90211"#)) + } + + @Test("A field that held booleans beside strings keeps reading true as a boolean") + func mixedBooleanField() throws { + let statement = try statement(editing: "x", to: "true", held: [.string, .boolean]) + #expect(statement.contains(#""zip": true"#)) + } + + @Test("A field nothing was read from keeps reading number text as a number") + func unknownFieldInfersByShape() throws { + let statement = try statement(editing: "90210", to: "90211", held: nil) + #expect(statement.contains(#""zip": 90211"#)) + } + + @Test("A new document stores number text as a string in a field that held only strings") + func insertIntoStringField() throws { + var generator = MongoDBStatementGenerator(collectionName: "places", columns: ["_id", "zip"]) + generator.fieldKinds = MongoDBFieldKinds(["zip": [.string]]) + let change = PluginRowChange( + rowIndex: 0, type: .insert, cellChanges: [(1, "zip", .null, .text("90211"))], originalRow: nil + ) + let writes = try generator.generateRowWrites( + from: [change], insertedRowData: [0: [.null, .text("90211")]], deletedRowIndices: [], insertedRowIndices: [0] + ) + let statement = try #require(writes.first?.statement) + #expect(statement.contains(#""zip": "90211""#)) + } +} diff --git a/apple/TableProTests/Core/AI/ClaudeAgentConnectionScopeTests.swift b/apple/TableProTests/Core/AI/ClaudeAgentConnectionScopeTests.swift new file mode 100644 index 0000000000..c589970a09 --- /dev/null +++ b/apple/TableProTests/Core/AI/ClaudeAgentConnectionScopeTests.swift @@ -0,0 +1,62 @@ +// +// ClaudeAgentConnectionScopeTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProMCP +import Testing + +private actor RecordingToolBridge: ClaudeAgentMCPBridging { + private(set) var requested: [UUID?] = [] + + func makeGrant(connectionId: UUID?) async -> ClaudeAgentToolGrant? { + requested.append(connectionId) + return nil + } + + func release(_ grant: ClaudeAgentToolGrant?) async {} +} + +struct ClaudeAgentConnectionScopeTests { + @Test("The token reaches the conversation's connection and no other") + func tokenIsLimitedToTheConversationConnection() throws { + let connectionId = UUID() + let access = try #require(ClaudeAgentMCPBridge.connectionAccess(for: connectionId)) + + #expect(access == .limited([connectionId])) + #expect(access.allows(connectionId)) + #expect(!access.allows(UUID())) + } + + @Test("A conversation with no connection gets no token") + func noConnectionMeansNoToken() async { + #expect(ClaudeAgentMCPBridge.connectionAccess(for: nil) == nil) + #expect(await ClaudeAgentMCPBridge().makeGrant(connectionId: nil) == nil) + } + + @Test("Transport options carry no connection unless the caller names one") + func optionsDefaultToNoConnection() { + #expect(ChatTransportOptions(model: "sonnet").connectionId == nil) + } + + @Test("The provider asks for a grant on the connection in its options") + func providerPassesTheConnectionToTheBridge() async throws { + let bridge = RecordingToolBridge() + let provider = ClaudeAgentProvider( + model: "sonnet", + cli: ClaudeAgentCLI(executableOverride: "/usr/bin/true"), + toolBridge: bridge + ) + let connectionId = UUID() + + let stream = provider.streamChat( + turns: [ChatTurnWire(role: .user, blocks: [.text("hi")])], + options: ChatTransportOptions(model: "sonnet", connectionId: connectionId) + ) + for try await _ in stream {} + + #expect(await bridge.requested == [connectionId]) + } +} diff --git a/apple/TableProTests/Core/AI/ProviderStreamFailureTests.swift b/apple/TableProTests/Core/AI/ProviderStreamFailureTests.swift new file mode 100644 index 0000000000..22379e0339 --- /dev/null +++ b/apple/TableProTests/Core/AI/ProviderStreamFailureTests.swift @@ -0,0 +1,108 @@ +// +// ProviderStreamFailureTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct ProviderStreamFailureTests { + private func streamingFailureMessage(_ error: (any Error)?) -> String? { + guard case .streamingFailed(let message) = error as? AIProviderError else { return nil } + return message + } + + @Test("A data line is read with or without the space after the colon") + func dataPayloadFraming() { + #expect(SSEEventStream.dataPayload("data: {\"a\":1}") == "{\"a\":1}") + #expect(SSEEventStream.dataPayload("data:{\"a\":1}") == "{\"a\":1}") + #expect(SSEEventStream.dataPayload("data: x") == " x") + #expect(SSEEventStream.dataPayload("data:") == "") + #expect(SSEEventStream.dataPayload("event: message_stop") == nil) + #expect(SSEEventStream.dataPayload(": keep-alive") == nil) + } + + @Test("Every SSE provider decodes a data line that has no space") + func providersDecodeLineWithoutSpace() { + let line = #"data:{"choices":[]}"# + #expect(OpenAICompatibleProvider.decodeStreamLine(line, providerType: .openAI) != nil) + #expect(OpenAICompatibleProvider.decodeStreamLine("data:[DONE]", providerType: .openAI) == nil) + #expect(AnthropicProvider.decodeStreamLine(#"data:{"type":"ping"}"#) != nil) + #expect(AnthropicProvider.decodeStreamLine("data:[DONE]") == nil) + #expect(OpenAIResponsesProvider.decodeStreamLine(#"data:{"type":"response.created"}"#) != nil) + #expect(OpenAIResponsesProvider.decodeStreamLine("data:[DONE]") == nil) + #expect(GeminiProvider.decodeStreamLine(#"data:{"candidates":[]}"#) != nil) + } + + @Test("An error chunk in a chat completions stream fails the stream with its message") + func errorChunkThrows() { + var state = OpenAIStreamState() + let error = #expect(throws: AIProviderError.self) { + _ = try OpenAICompatibleProvider.streamEvents( + from: ["error": ["message": "Rate limit reached", "code": 429]], + state: &state + ) + } + #expect(streamingFailureMessage(error) == "Rate limit reached") + } + + @Test("An error sent as a plain string fails the stream") + func stringErrorThrows() { + let error = AIProviderError.streamError(in: ["error": "model not loaded"]) + #expect(streamingFailureMessage(error) == "model not loaded") + } + + @Test("An error object without a message falls back to its status") + func errorStatusFallback() { + let error = AIProviderError.streamError(in: ["error": ["code": 429, "status": "RESOURCE_EXHAUSTED"]]) + #expect(streamingFailureMessage(error) == "RESOURCE_EXHAUSTED") + } + + @Test("A chunk with no error, or a null one, is not a failure") + func noErrorIsNotAFailure() throws { + #expect(AIProviderError.streamError(in: ["choices": [[String: Any]]()]) == nil) + #expect(AIProviderError.streamError(in: ["error": NSNull()]) == nil) + var state = OpenAIStreamState() + let events = try OpenAICompatibleProvider.streamEvents( + from: ["error": NSNull(), "choices": [["delta": ["content": "hi"]]]], + state: &state + ) + #expect(events.count == 1) + } + + @Test("A stream that ends having sent nothing is a failure") + func emptyStreamThrows() { + let error = #expect(throws: AIProviderError.self) { + _ = try OpenAICompatibleProvider.finalStreamEvents(OpenAIStreamState()) + } + #expect(streamingFailureMessage(error) != nil) + } + + @Test("A stream that sent text ends normally") + func textStreamEnds() throws { + var state = OpenAIStreamState() + _ = try OpenAICompatibleProvider.streamEvents( + from: ["choices": [["delta": ["content": "hi"]]]], + state: &state + ) + _ = try OpenAICompatibleProvider.finalStreamEvents(state) + } + + @Test("A stream with only a finish reason ends normally") + func finishReasonOnlyEnds() throws { + var state = OpenAIStreamState() + _ = try OpenAICompatibleProvider.streamEvents( + from: ["choices": [["delta": [String: Any](), "finish_reason": "stop"]]], + state: &state + ) + _ = try OpenAICompatibleProvider.finalStreamEvents(state) + } + + @Test("An Ollama stream that only reports done ends normally") + func ollamaDoneEnds() throws { + var state = OpenAIStreamState() + _ = try OpenAICompatibleProvider.streamEvents(from: ["done": true], state: &state) + _ = try OpenAICompatibleProvider.finalStreamEvents(state) + } +} diff --git a/apple/TableProTests/Core/Autocomplete/KeywordVocabularyParityTests.swift b/apple/TableProTests/Core/Autocomplete/KeywordVocabularyParityTests.swift index d3d08a695a..021236e9fd 100644 --- a/apple/TableProTests/Core/Autocomplete/KeywordVocabularyParityTests.swift +++ b/apple/TableProTests/Core/Autocomplete/KeywordVocabularyParityTests.swift @@ -9,6 +9,7 @@ import Foundation @testable import TablePro @testable import TableProAutocomplete +import TableProSQLGrammar import Testing struct KeywordVocabularyParityTests { @@ -110,9 +111,9 @@ struct KeywordVocabularyParityTests { let text = statement as NSString let endOfAdd = text.range(of: "add").location + 3 - #expect(KeywordUppercaseHelper.keywordBeforePosition(text, at: endOfAdd)?.word == "add") + #expect(KeywordUppercaseHelper.keywordBeforePosition(text, at: endOfAdd, grammar: .ansi)?.word == "add") let endOfUsers = text.range(of: "users").location + 5 - #expect(KeywordUppercaseHelper.keywordBeforePosition(text, at: endOfUsers) == nil) + #expect(KeywordUppercaseHelper.keywordBeforePosition(text, at: endOfUsers, grammar: .ansi) == nil) } } diff --git a/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift b/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift new file mode 100644 index 0000000000..5abc551188 --- /dev/null +++ b/apple/TableProTests/Core/ChangeTracking/EditedRowChangeTrackingTests.swift @@ -0,0 +1,252 @@ +// +// EditedRowChangeTrackingTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@MainActor +struct EditedRowDeletionTests { + private static let columns = ["code", "name"] + + private func makeManager() -> (DataChangeManager, UndoManager) { + let manager = DataChangeManager() + let undoManager = UndoManager() + undoManager.groupsByEvent = false + manager.undoManagerProvider = { undoManager } + manager.configureForTable( + tableName: "countries", + columns: Self.columns, + primaryKeyColumns: ["code"], + databaseType: .mysql, + generatedColumns: [] + ) + return (manager, undoManager) + } + + private func editCode(_ manager: DataChangeManager) { + manager.recordCellChange( + rowID: .existing(0), columnIndex: 0, columnName: "code", + oldValue: "US", newValue: "UK", originalRow: ["US", "United States"] + ) + } + + @Test("A delete of an edited row keys on the values the row was read with") + func deleteUsesTheRowAsRead() throws { + let (manager, _) = makeManager() + editCode(manager) + + manager.recordRowDeletion(rowID: .existing(0), originalRow: ["UK", "United States"]) + + let delete = try #require(manager.changes.first { $0.type == .delete }) + #expect(delete.originalRow == ["US", "United States"]) + #expect(manager.changes.count == 1) + + let statements = try manager.generateSQL() + #expect(statements.count == 1) + let parameters = String(describing: statements.first?.parameters ?? []) + #expect(parameters.contains("US")) + #expect(!parameters.contains("UK")) + } + + @Test("A batch delete reads each edited row's image the same way") + func batchDeleteUsesTheRowsAsRead() { + let (manager, _) = makeManager() + editCode(manager) + + manager.recordBatchRowDeletion(rows: [ + (rowID: .existing(0), originalRow: ["UK", "United States"]), + (rowID: .existing(1), originalRow: ["FR", "France"]) + ]) + + let edited = manager.changes.first { $0.rowID == .existing(0) } + let untouched = manager.changes.first { $0.rowID == .existing(1) } + #expect(edited?.originalRow == ["US", "United States"]) + #expect(untouched?.originalRow == ["FR", "France"]) + } + + @Test("Undoing the delete brings the row's edit back") + func undoingTheDeleteRestoresTheEdit() throws { + let (manager, undoManager) = makeManager() + editCode(manager) + manager.recordRowDeletion(rowID: .existing(0), originalRow: ["UK", "United States"]) + #expect(!manager.isCellModified(rowID: .existing(0), columnIndex: 0)) + + undoManager.undo() + + #expect(!manager.isRowDeleted(.existing(0))) + #expect(manager.isCellModified(rowID: .existing(0), columnIndex: 0)) + let update = try #require(manager.changes.first) + #expect(manager.changes.count == 1) + #expect(update.type == .update) + #expect(update.cellChanges.first?.oldValue == "US") + #expect(update.cellChanges.first?.newValue == "UK") + + undoManager.redo() + + #expect(manager.changes.count == 1) + #expect(manager.changes.first?.type == .delete) + #expect(manager.changes.first?.originalRow == ["US", "United States"]) + } + + @Test("A discard still knows the original of a cell edited before its row was deleted") + func originalValuesCoverADeletedRowsEdits() { + let (manager, _) = makeManager() + editCode(manager) + manager.recordRowDeletion(rowID: .existing(0), originalRow: ["UK", "United States"]) + + let originals = manager.getOriginalValues() + + #expect(originals.count == 1) + #expect(originals.first?.rowID == .existing(0)) + #expect(originals.first?.columnIndex == 0) + #expect(originals.first?.value == "US") + } + + @Test("A deleted row's parked edit survives a tab switch") + func parkedEditSurvivesSaveAndRestore() { + let (manager, _) = makeManager() + editCode(manager) + manager.recordRowDeletion(rowID: .existing(0), originalRow: ["UK", "United States"]) + let state = manager.saveState() + + let (restored, _) = makeManager() + restored.restoreState(from: state, tableName: "countries", databaseType: .mysql, generatedColumns: []) + restored.undoRowDeletion(rowID: .existing(0)) + + #expect(restored.isCellModified(rowID: .existing(0), columnIndex: 0)) + #expect(restored.changes.first?.type == .update) + } + + @Test("Deleting an edited row from the grid deletes the row the server holds") + func gridDeleteOfAnEditedRow() throws { + let (manager, _) = makeManager() + let operations = RowOperationsManager(changeManager: manager) + var tableRows = TableRows.from( + queryRows: [["US", "United States"], ["UK", "United Kingdom"]], + columns: Self.columns, + columnTypes: Array(repeating: .text(rawType: nil), count: Self.columns.count) + ) + let rowID = tableRows.rows[0].id + manager.recordCellChange( + rowID: rowID, columnIndex: 0, columnName: "code", + oldValue: "US", newValue: "UK", originalRow: Array(tableRows.rows[0].values) + ) + _ = tableRows.edit(row: 0, column: 0, value: "UK", isAbsent: false) + + _ = operations.deleteRows([rowID], selectedDisplayIndices: [0], displayCount: 2, tableRows: &tableRows) + + let delete = try #require(manager.changes.first { $0.type == .delete }) + #expect(delete.originalRow == ["US", "United States"]) + } +} + +@MainActor +struct SameRowCellRedoTests { + @Test("Redoing one cell of a row that still has another pending edit records it again") + func redoMergesIntoTheRowsUpdate() throws { + let manager = DataChangeManager() + let undoManager = UndoManager() + undoManager.groupsByEvent = false + manager.undoManagerProvider = { undoManager } + manager.configureForTable( + tableName: "users", + columns: ["id", "name", "email"], + primaryKeyColumns: ["id"], + databaseType: .mysql, + generatedColumns: [] + ) + let original: [PluginCellValue] = ["1", "Ann", "ann@old.com"] + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Ann", newValue: "Anna", originalRow: original + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "email", + oldValue: "ann@old.com", newValue: "ann@new.com", originalRow: original + ) + + undoManager.undo() + #expect(manager.getModifiedColumnsForRow(.existing(0)) == [1]) + + undoManager.redo() + + #expect(manager.getModifiedColumnsForRow(.existing(0)) == [1, 2]) + let update = try #require(manager.changes.first) + #expect(manager.changes.count == 1) + let email = try #require(update.cellChanges.first { $0.columnIndex == 2 }) + #expect(email.oldValue == "ann@old.com") + #expect(email.newValue == "ann@new.com") + + undoManager.undo() + undoManager.undo() + #expect(manager.changes.isEmpty) + #expect(!manager.hasChanges) + } +} + +@MainActor +struct UpdateSaveOrderTests { + @Test("An edit made after two deletes is written after them") + func updateKeepsItsPlaceAfterDeletes() throws { + let manager = DataChangeManager() + manager.configureForTable( + tableName: "users", + columns: ["id", "email"], + primaryKeyColumns: ["id"], + databaseType: .mysql, + generatedColumns: [] + ) + manager.recordRowDeletion(rowID: .existing(0), originalRow: ["1", "a@test.com"]) + manager.recordRowInsertion(rowID: .inserted(UUID()), values: ["9", "new@test.com"]) + manager.recordRowDeletion(rowID: .existing(1), originalRow: ["2", "b@test.com"]) + manager.recordCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "email", + oldValue: "c@test.com", newValue: "b@test.com", originalRow: ["3", "c@test.com"] + ) + + let verbs = try manager.generateSQL().map { statement in String(statement.sql.prefix(while: { $0 != " " })) } + + #expect(verbs == ["DELETE", "INSERT", "DELETE", "UPDATE"]) + } + + @Test("An update gets its own sequence number, and gets a later one when a redo records it again") + func updatesAreSequenced() throws { + var pending = PendingChanges() + pending.recordRowDeletion(rowID: .existing(0), originalRow: ["1", "a"]) + pending.recordCellChange( + rowID: .existing(1), columnIndex: 1, columnName: "email", + oldValue: "b", newValue: "a", originalRow: ["2", "b"] + ) + let delete = try #require(pending.change(forRow: .existing(0), type: .delete)) + let update = try #require(pending.change(forRow: .existing(1), type: .update)) + #expect(update.sequence > delete.sequence) + + pending.reapplyCellChange( + rowID: .existing(2), columnIndex: 1, columnName: "email", + originalDBValue: "c", newValue: "d", originalRow: ["3", "c"] + ) + let replayed = try #require(pending.change(forRow: .existing(2), type: .update)) + #expect(replayed.sequence > update.sequence) + } + + @Test("An edit parked under a delete keeps its place in the order when the delete is undone") + func parkedUpdateKeepsItsSequence() throws { + var pending = PendingChanges() + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "email", + oldValue: "a", newValue: "b", originalRow: ["1", "a"] + ) + let sequence = try #require(pending.change(forRow: .existing(0), type: .update)).sequence + pending.recordRowInsertion(rowID: .inserted(UUID()), values: ["9", "a"]) + pending.recordRowDeletion(rowID: .existing(0), originalRow: ["1", "b"]) + + _ = pending.undoRowDeletion(rowID: .existing(0)) + + #expect(pending.change(forRow: .existing(0), type: .update)?.sequence == sequence) + #expect(pending.isCellModified(rowID: .existing(0), columnIndex: 1)) + } +} diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift new file mode 100644 index 0000000000..984194c139 --- /dev/null +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift @@ -0,0 +1,174 @@ +// +// SQLStatementGeneratorBitColumnTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +struct SQLStatementGeneratorBitColumnTests { + private static let columns = ["id", "flags", "name"] + private static let typeNames = ["id": "INT", "flags": "BIT(8)", "name": "VARCHAR(40)"] + + private func makeGenerator( + primaryKeyColumns: [String] = ["id"], + databaseType: DatabaseType = .mysql, + typeNames: [String: String] = SQLStatementGeneratorBitColumnTests.typeNames + ) throws -> SQLStatementGenerator { + try SQLStatementGenerator( + tableName: "t", + columns: Self.columns, + primaryKeyColumns: primaryKeyColumns, + databaseType: databaseType, + columnTypeNames: typeNames, + dialect: nil + ) + } + + private func texts(_ statement: ParameterizedStatement?) -> [String] { + (statement?.parameters ?? []).map { ($0 as? String) ?? "NULL" } + } + + private func flagsEdit(to newValue: PluginCellValue) -> RowChange { + RowChange( + rowID: .existing(0), + type: .update, + cellChanges: [CellChange(columnIndex: 1, columnName: "flags", oldValue: "1", newValue: newValue)], + originalRow: ["7", "1", "Ann"] + ) + } + + @Test("An edited BIT cell is written as the number it reads as") + func updateWritesTheNumber() throws { + let statement = try makeGenerator().generateUpdateSQL(for: flagsEdit(to: "5")) + + #expect(statement?.sql == "UPDATE `t` SET `flags` = CAST(? AS UNSIGNED) WHERE `id` = ?") + #expect(texts(statement) == ["5", "7"]) + } + + @Test("A BIT cell set to NULL binds NULL as it is") + func nullStaysNull() throws { + let statement = try makeGenerator().generateUpdateSQL(for: flagsEdit(to: .null)) + + #expect(statement?.sql == "UPDATE `t` SET `flags` = ? WHERE `id` = ?") + } + + @Test("A keyless match compares a BIT column as a number, in UPDATE and DELETE") + func keylessMatchComparesTheNumber() throws { + let generator = try makeGenerator(primaryKeyColumns: []) + let match = "`id` = ? AND `flags` = CAST(? AS UNSIGNED) AND `name` = ?" + + let update = generator.generateUpdateSQL(for: flagsEdit(to: "5")) + #expect(update?.sql == "UPDATE `t` SET `flags` = CAST(? AS UNSIGNED) WHERE \(match)") + #expect(texts(update) == ["5", "7", "1", "Ann"]) + + let deletes = generator.generateStatements( + from: [RowChange(rowID: .existing(0), type: .delete, originalRow: ["7", "1", "Ann"])], + insertedRowData: [:], + deletedRowIDs: [.existing(0)], + insertedRowIDs: [] + ) + #expect(deletes.first?.sql == "DELETE FROM `t` WHERE (\(match))") + } + + @Test("A BIT primary key is matched as a number") + func bitPrimaryKey() throws { + let generator = try makeGenerator(primaryKeyColumns: ["flags"]) + let change = RowChange( + rowID: .existing(0), + type: .update, + cellChanges: [CellChange(columnIndex: 2, columnName: "name", oldValue: "Ann", newValue: "Bea")], + originalRow: ["7", "1", "Ann"] + ) + + #expect(generator.generateUpdateSQL(for: change)?.sql == "UPDATE `t` SET `name` = ? WHERE `flags` = CAST(? AS UNSIGNED)") + } + + @Test("A new row writes its BIT value as a number, from stored values and from an explicit column list") + func insertsWriteTheNumber() throws { + let generator = try makeGenerator() + let rowID = RowID.inserted(UUID()) + + let statements = generator.generateStatements( + from: [RowChange(rowID: rowID, type: .insert)], + insertedRowData: [rowID: ["__DEFAULT__", "5", "Ann"]], + deletedRowIDs: [], + insertedRowIDs: [rowID] + ) + #expect(statements.first?.sql == "INSERT INTO `t` (`flags`, `name`) VALUES (CAST(? AS UNSIGNED), ?)") + + let single = generator.insertStatement(columns: ["flags", "name"], values: ["5", "Ann"]) + #expect(single?.sql == "INSERT INTO `t` (`flags`, `name`) VALUES (CAST(? AS UNSIGNED), ?)") + + let many = generator.insertStatement(columns: ["flags"], rows: [["5"], [.null]]) + #expect(many?.sql == "INSERT INTO `t` (`flags`) VALUES (CAST(? AS UNSIGNED)), (?)") + } + + @Test("A keyed update and its key count write and match a BIT column as a number") + func keyedStatements() throws { + let generator = try makeGenerator() + + let update = generator.keyedUpdateStatement( + columns: ["flags"], values: ["5"], keyColumns: ["flags"], keyValues: ["1"] + ) + #expect(update?.sql == "UPDATE `t` SET `flags` = CAST(? AS UNSIGNED) WHERE `flags` = CAST(? AS UNSIGNED)") + + let count = generator.keyMatchCountStatement(keyColumns: ["flags"], keys: [["1"]]) + #expect(count?.sql == "SELECT 0, COUNT(*) FROM `t` WHERE `flags` = CAST(? AS UNSIGNED)") + } + + @Test("A bit literal typed into the cell is bound as its number") + func bitLiteralText() throws { + #expect(SQLStatementGenerator.decimalBitText("b'101'") == "5") + #expect(SQLStatementGenerator.decimalBitText("B'11111111'") == "255") + #expect(SQLStatementGenerator.decimalBitText("0b10") == "2") + #expect(SQLStatementGenerator.decimalBitText("170") == "170") + #expect(SQLStatementGenerator.decimalBitText("b'102'") == "b'102'") + + let statement = try makeGenerator().generateUpdateSQL(for: flagsEdit(to: "b'101'")) + #expect(texts(statement) == ["5", "7"]) + } + + @Test("Other columns, unknown types and engines whose BIT is a string of digits keep the plain placeholder") + func everythingElseIsUnchanged() throws { + let untyped = try makeGenerator(typeNames: [:]).generateUpdateSQL(for: flagsEdit(to: "5")) + #expect(untyped?.sql == "UPDATE `t` SET `flags` = ? WHERE `id` = ?") + + let postgres = try makeGenerator(databaseType: .postgresql).generateUpdateSQL(for: flagsEdit(to: "101")) + #expect(postgres != nil) + #expect(postgres?.sql.contains("CAST") == false) + } + + @Test("MariaDB writes a BIT cell the same way") + func mariaDB() throws { + let statement = try makeGenerator(databaseType: .mariadb).generateUpdateSQL(for: flagsEdit(to: "5")) + + #expect(statement?.sql == "UPDATE `t` SET `flags` = CAST(? AS UNSIGNED) WHERE `id` = ?") + } +} + +@MainActor +struct BitColumnSaveTests { + @Test("A grid edit of a BIT cell reaches the statement with its column type") + func managerPassesColumnTypes() throws { + let manager = DataChangeManager() + manager.configureForTable( + tableName: "t", + columns: ["id", "flags"], + primaryKeyColumns: ["id"], + databaseType: .mysql, + generatedColumns: [], + columnTypeNames: ["id": "INT", "flags": "BIT"] + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "flags", + oldValue: "1", newValue: "5", originalRow: ["7", "1"] + ) + + let statements = try manager.generateSQL() + + #expect(statements.first?.sql == "UPDATE `t` SET `flags` = CAST(? AS UNSIGNED) WHERE `id` = ?") + } +} diff --git a/apple/TableProTests/Core/CrossEngine/CrossEngineValueCoercerTests.swift b/apple/TableProTests/Core/CrossEngine/CrossEngineValueCoercerTests.swift index a75608ba10..895a80b358 100644 --- a/apple/TableProTests/Core/CrossEngine/CrossEngineValueCoercerTests.swift +++ b/apple/TableProTests/Core/CrossEngine/CrossEngineValueCoercerTests.swift @@ -151,11 +151,54 @@ struct CrossEngineValueCoercerTests { @Test func aPostgresArrayBecomesJson() { let subject = coercer([.json], sources: [arrayOfText], from: .postgres) - #expect(subject.coerce([.text("{1,2,3}")]) == [.text("[1,2,3]")]) + #expect(subject.coerce([.text("{1,2,3}")]) == [.text("[\"1\",\"2\",\"3\"]")]) #expect(subject.coerce([.text("{a,b}")]) == [.text("[\"a\",\"b\"]")]) #expect(subject.coerce([.text("{}")]) == [.text("[]")]) } + @Test + func aTextArrayKeepsEveryElementAString() { + let subject = coercer([.json], sources: [arrayOfText], from: .postgres) + #expect(subject.coerce([.text("{02134,inf}")]) == [.text("[\"02134\",\"inf\"]")]) + #expect(subject.coerce([.text("{10001,true}")]) == [.text("[\"10001\",\"true\"]")]) + #expect(subject.coerce([.text("{0x1F,+5,NaN,NULL}")]) == [.text("[\"0x1F\",\"+5\",\"NaN\",null]")]) + } + + @Test + func aNumericArrayWritesJsonNumbers() { + let integers = coercer([.json], sources: [.array(element: .integer(bytes: 4))], from: .postgres) + #expect(integers.coerce([.text("{1,-2,NULL}")]) == [.text("[1,-2,null]")]) + let decimals = coercer( + [.json], sources: [.array(element: .decimal(precision: nil, scale: nil))], from: .postgres) + #expect(decimals.coerce([.text("{1.50,12345678901234567890.5}")]) + == [.text("[1.50,12345678901234567890.5]")]) + } + + @Test + func aFloatThatJsonCannotSpellIsWrittenAsAString() { + let subject = coercer([.json], sources: [.array(element: .floatingPoint(bits: 64))], from: .postgres) + #expect(subject.coerce([.text("{1.5,NaN,Infinity,-Infinity}")]) + == [.text("[1.5,\"NaN\",\"Infinity\",\"-Infinity\"]")]) + } + + @Test + func aBooleanArrayWritesJsonBooleans() { + let subject = coercer([.json], sources: [.array(element: .boolean)], from: .postgres) + #expect(subject.coerce([.text("{t,f,NULL}")]) == [.text("[true,false,null]")]) + } + + @Test + func aJsonArrayKeepsItsElementsAsJson() { + let subject = coercer([.json], sources: [.array(element: .json)], from: .postgres) + #expect(subject.coerce([.text(#"{"{\"a\": 1}",7,"\"x\""}"#)]) == [.text(#"[{"a": 1},7,"x"]"#)]) + } + + @Test + func aControlCharacterInAnElementIsEscaped() { + let subject = coercer([.json], sources: [arrayOfText], from: .postgres) + #expect(subject.coerce([.text("{\"a\u{08}b\"}")]) == [.text("[\"a\\bb\"]")]) + } + /// A JSON object arrives with the same brackets an array literal uses, and it is already JSON. @Test func aJsonObjectIsNotMistakenForAnArray() { diff --git a/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift b/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift index 45f203a66f..81ec4f95e9 100644 --- a/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift +++ b/apple/TableProTests/Core/Database/CredentialLeaseRoutingTests.swift @@ -78,4 +78,25 @@ struct CredentialLeaseRoutingTests { #expect(error?.localizedDescription.contains(DatabaseManager.parkedTransactionLostMessage(database: "app")) == true) try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .protectedWrite) { _ in } } + + @Test("A metadata read on the session driver leaves the lost transaction's report for the user's statement") + func metadataReadLeavesTheLossReport() async throws { + let connection = TestFixtures.makeConnection(database: "", type: .mysql) + var session = ConnectionSession(connection: connection, driver: MockDatabaseDriver(connection: connection)) + session.status = .connected + let manager = DatabaseManager.shared + manager.injectSession(session, for: connection.id) + defer { manager.removeSession(for: connection.id) } + let scope = DatabaseScope(connectionId: connection.id, database: "", schema: nil) + #expect(manager.metadataRoute(for: scope) == .sessionDriver) + manager.sessionLanes.markTransactionLost(database: "", for: connection.id) + + try await manager.withMetadataDriver(scope: scope) { _ in } + + #expect(manager.sessionLanes.hasTransactionLoss(database: "", for: connection.id)) + await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .protectedWrite) { _ in } + } + #expect(!manager.sessionLanes.hasTransactionLoss(database: "", for: connection.id)) + } } diff --git a/apple/TableProTests/Core/Database/ProtectedWriteDeadlineTests.swift b/apple/TableProTests/Core/Database/ProtectedWriteDeadlineTests.swift new file mode 100644 index 0000000000..f18bfc632a --- /dev/null +++ b/apple/TableProTests/Core/Database/ProtectedWriteDeadlineTests.swift @@ -0,0 +1,176 @@ +// +// ProtectedWriteDeadlineTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +private actor Flag { + private(set) var isSet = false + func set() { isSet = true } +} + +struct ProtectedWriteDeadlineTests { + private struct WriteFailed: Error {} + + @Test("A write that finishes in time returns its result") + func finishesInTime() async throws { + let lateCompletion = Flag() + let value = try await DatabaseAccessBridge.awaitProtectedWrite( + timeout: .seconds(60), + timeoutSeconds: 60, + onLateCompletion: { Task { await lateCompletion.set() } }, + { enterDriver in + try enterDriver() + return 7 + } + ) + + #expect(value == 7) + #expect(await !lateCompletion.isSet) + } + + @Test("A write that fails in time throws its own error") + func failsInTime() async { + await #expect(throws: WriteFailed.self) { + _ = try await DatabaseAccessBridge.awaitProtectedWrite( + timeout: .seconds(60), + timeoutSeconds: 60, + { _ -> Int in throw WriteFailed() } + ) + } + } + + @Test("A write that outlives the deadline is reported as outcome unknown, not as a plain timeout") + func outlivesTheDeadline() async throws { + let gate = AsyncGate() + let finished = Flag() + let lateCompletion = Flag() + + let error = await #expect(throws: DatabaseAccessError.self) { + _ = try await DatabaseAccessBridge.awaitProtectedWrite( + timeout: .milliseconds(20), + timeoutSeconds: 30, + onLateCompletion: { Task { await lateCompletion.set() } }, + { enterDriver -> Int in + try enterDriver() + await gate.wait() + await finished.set() + return 1 + } + ) + } + + guard case .timeout(let detail, let context) = error else { + Issue.record("expected a timeout, got \(String(describing: error))") + return + } + #expect(context?["outcome"] == "unknown") + #expect(detail.contains("30")) + #expect(!detail.contains("timed out")) + #expect(await !finished.isSet) + + await gate.open() + try await waitUntil { await finished.isSet } + try await waitUntil { await lateCompletion.isSet } + } + + @Test("A write still queued at the deadline gets a plain timeout and never runs") + func queuedAtTheDeadlineIsRefused() async throws { + let queue = AsyncGate() + let ran = Flag() + let refused = Flag() + let lateCompletion = Flag() + + let error = await #expect(throws: DatabaseAccessError.self) { + _ = try await DatabaseAccessBridge.awaitProtectedWrite( + timeout: .milliseconds(20), + timeoutSeconds: 30, + onLateCompletion: { Task { await lateCompletion.set() } }, + { enterDriver -> Int in + await queue.wait() + do { + try enterDriver() + } catch { + await refused.set() + throw error + } + await ran.set() + return 1 + } + ) + } + + guard case .timeout(let detail, let context) = error else { + Issue.record("expected a timeout, got \(String(describing: error))") + return + } + #expect(context == nil) + #expect(detail.contains("timed out")) + + await queue.open() + try await waitUntil { await refused.isSet } + #expect(await !ran.isSet) + #expect(await !lateCompletion.isSet) + } + + @Test("A write still queued when the caller cancels is refused and never runs") + func queuedAtCancelIsRefused() async throws { + let queue = AsyncGate() + let ran = Flag() + let refused = Flag() + + let caller = Task { + try await DatabaseAccessBridge.awaitProtectedWrite( + timeout: .seconds(60), + timeoutSeconds: 60, + { enterDriver -> Int in + await queue.wait() + do { + try enterDriver() + } catch { + await refused.set() + throw error + } + await ran.set() + return 1 + } + ) + } + await queue.arrived(1) + caller.cancel() + await #expect(throws: CancellationError.self) { _ = try await caller.value } + + await queue.open() + try await waitUntil { await refused.isSet } + #expect(await !ran.isSet) + } + + @Test("A write already in the driver when the caller cancels runs to the end") + func inDriverAtCancelFinishes() async throws { + let gate = AsyncGate() + let finished = Flag() + + let caller = Task { + try await DatabaseAccessBridge.awaitProtectedWrite( + timeout: .seconds(60), + timeoutSeconds: 60, + { enterDriver -> Int in + try enterDriver() + await gate.wait() + await finished.set() + return 1 + } + ) + } + await gate.arrived(1) + caller.cancel() + await #expect(throws: CancellationError.self) { _ = try await caller.value } + + await gate.open() + try await waitUntil { await finished.isSet } + } +} diff --git a/apple/TableProTests/Core/Database/ScopedDriverRoutingTests.swift b/apple/TableProTests/Core/Database/ScopedDriverRoutingTests.swift index 145cc6f85f..3285cf70f2 100644 --- a/apple/TableProTests/Core/Database/ScopedDriverRoutingTests.swift +++ b/apple/TableProTests/Core/Database/ScopedDriverRoutingTests.swift @@ -127,14 +127,25 @@ struct ScopedDriverRoutingTests { } } - @Test("A reconnect-required engine runs a foreign database on a pooled connection") - func reconnectRequiredEngineOnAForeignDatabasePools() throws { + @Test("A reconnect-required engine never runs the user's SQL on a pooled connection") + func reconnectRequiredEngineKeepsUserSQLOffThePool() throws { let connection = Self.makeSession(type: .postgresql, browseDatabase: "inventory") defer { DatabaseManager.shared.removeSession(for: connection.id) } let foreign = Self.scope(connection, database: "orders") - #expect(DatabaseManager.shared.executionRoute(for: foreign) == .pooled) + #expect(DatabaseManager.shared.executionRoute(for: foreign) == .sessionDriver) + } + + @Test("A table read of a database with no session connection is served by the pool") + func tableReadOfAForeignDatabasePools() throws { + let connection = Self.makeSession(type: .postgresql, browseDatabase: "inventory") + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let foreign = Self.scope(connection, database: "orders") + + #expect(DatabaseManager.shared.tableReadRoute(for: foreign) == .pooled) + #expect(DatabaseManager.shared.tableReadRoute(for: Self.scope(connection, database: "inventory")) == .sessionDriver) } @Test("An engine that can neither pin nor pool reports the tab's database instead of guessing") diff --git a/apple/TableProTests/Core/Database/SessionLanesTests.swift b/apple/TableProTests/Core/Database/SessionLanesTests.swift index 09da6a754e..4b4e31a158 100644 --- a/apple/TableProTests/Core/Database/SessionLanesTests.swift +++ b/apple/TableProTests/Core/Database/SessionLanesTests.swift @@ -22,6 +22,8 @@ struct SessionLanesTests { var openCount = 0 var failure: Error? var schemaForNextOpen: String? + var inOrder: [MockDatabaseDriver] = [] + var holdsNextOpen: AsyncGate? func open(_ scope: DatabaseScope) async throws -> DatabaseDriver { openCount += 1 @@ -29,6 +31,11 @@ struct SessionLanesTests { let driver = MockDatabaseDriver() driver.currentSchema = schemaForNextOpen opened[scope.database] = driver + inOrder.append(driver) + if let gate = holdsNextOpen { + holdsNextOpen = nil + await gate.wait() + } return driver } } @@ -207,7 +214,226 @@ struct SessionLanesTests { #expect(harness.home.disconnectCallCount == 1) let scope = DatabaseScope(connectionId: harness.connection.id, database: "app", schema: nil) - #expect(DatabaseManager.shared.executionRoute(for: scope) == .pooled) + #expect(DatabaseManager.shared.sessionLanes.parkedDriver(for: harness.connection.id, database: "app") == nil) + #expect(DatabaseManager.shared.tableReadRoute(for: scope) == .pooled) + } + + @Test("SQL on a database never browsed runs on a session connection of its own, which tracks its transaction") + func userSQLOpensItsOwnSessionConnection() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let scope = DatabaseScope(connectionId: harness.connection.id, database: "logs", schema: nil) + + let route = manager.executionRoute(for: scope) + let ranOn = try await manager.withScopedDriver( + scope: scope, route: route, cancellation: .untracked + ) { driver -> ObjectIdentifier in + (driver as? MockDatabaseDriver)?.sessionTransactionStateToReturn = .inTransaction + return ObjectIdentifier(driver) + } + + let logs = try #require(harness.opener.opened["logs"]) + #expect(route == .sessionDriver) + #expect(ranOn == ObjectIdentifier(logs)) + #expect(manager.sessionLanes.parkedDriver(for: harness.connection.id, database: "logs") === logs) + #expect(manager.sessionTransactions.holdsTransaction(connectionId: harness.connection.id, container: "logs")) + #expect(session(harness)?.driver === harness.home) + #expect(session(harness)?.resolvedBrowseDatabase == "app") + + let ranAgainOn = try await manager.withScopedDriver( + scope: scope, route: manager.executionRoute(for: scope), cancellation: .untracked + ) { driver in ObjectIdentifier(driver) } + + #expect(ranAgainOn == ObjectIdentifier(logs)) + #expect(harness.opener.openCount == 1) + #expect(logs.disconnectCallCount == 0) + } + + private func run( + _ harness: Harness, + on database: String, + leaving state: PluginSessionTransactionState + ) async throws -> ObjectIdentifier { + let scope = DatabaseScope(connectionId: harness.connection.id, database: database, schema: nil) + return try await DatabaseManager.shared.withScopedDriver( + scope: scope, route: DatabaseManager.shared.executionRoute(for: scope), cancellation: .untracked + ) { driver -> ObjectIdentifier in + (driver as? MockDatabaseDriver)?.sessionTransactionStateToReturn = state + return ObjectIdentifier(driver) + } + } + + @Test("A connection opened for SQL on a database never browsed is closed once idle, and reopened on the next use") + func openedConnectionClosesWhenIdle() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + _ = try await run(harness, on: "logs", leaving: .idle) + let logs = try #require(harness.opener.opened["logs"]) + + await manager.closeIdleOpenedLanes(for: harness.connection.id) + #expect(logs.disconnectCallCount == 0) + + await manager.closeIdleOpenedLanes(for: harness.connection.id, idleFor: 0) + #expect(logs.disconnectCallCount == 1) + #expect(manager.sessionLanes.parkedDriver(for: harness.connection.id, database: "logs") == nil) + + let reopenedOn = try await run(harness, on: "logs", leaving: .idle) + #expect(harness.opener.openCount == 2) + #expect(reopenedOn != ObjectIdentifier(logs)) + } + + @Test("An idle connection that holds a transaction, or cannot say, is never closed") + func openedConnectionHoldingATransactionStays() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + _ = try await run(harness, on: "logs", leaving: .inTransaction) + _ = try await run(harness, on: "audit", leaving: .unknown) + + await manager.closeIdleOpenedLanes(for: harness.connection.id, idleFor: 0) + + #expect(harness.opener.opened["logs"]?.disconnectCallCount == 0) + #expect(harness.opener.opened["audit"]?.disconnectCallCount == 0) + #expect(manager.sessionLanes.parkedDatabases(for: harness.connection.id).sorted() == ["audit", "logs"]) + } + + @Test("The connection of a database the user browsed and left is not closed for being idle") + func browsedConnectionIsNotClosedWhenIdle() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + harness.home.sessionTransactionStateToReturn = .idle + try await manager.switchDatabase(to: "logs", for: harness.connection.id, persist: false) + _ = try await run(harness, on: "app", leaving: .idle) + + await manager.closeIdleOpenedLanes(for: harness.connection.id, idleFor: 0) + + #expect(harness.home.disconnectCallCount == 0) + #expect(manager.sessionLanes.parkedDriver(for: harness.connection.id, database: "app") === harness.home) + } + + @Test("A switch that lands while the database's first statement holds a new connection browses on that connection") + func switchAdoptsTheConnectionAStatementOpened() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + let hold = AsyncGate() + harness.opener.holdsNextOpen = hold + let switching = Task { @MainActor in + try await manager.switchDatabase(to: "logs", for: connectionId, persist: false) + } + await hold.arrived(1) + + let ranOn = try await run(harness, on: "logs", leaving: .inTransaction) + await hold.open() + try await switching.value + + let switchOpened = try #require(harness.opener.inOrder.first) + let statementOpened = try #require(harness.opener.inOrder.last) + #expect(harness.opener.inOrder.count == 2) + #expect(ranOn == ObjectIdentifier(statementOpened)) + #expect(session(harness)?.driver === statementOpened) + #expect(statementOpened.disconnectCallCount == 0) + #expect(switchOpened.disconnectCallCount == 1) + #expect(manager.sessionTransactions.holdsTransaction(connectionId: connectionId, container: "logs")) + #expect(manager.sessionLanes.parkedDriver(for: connectionId, database: "app") === harness.home) + } + + @Test("A statement whose database became the browsed one while it opened a connection runs on the browsed one") + func statementRunsOnTheConnectionASwitchInstalled() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let connectionId = harness.connection.id + let hold = AsyncGate() + harness.opener.holdsNextOpen = hold + let statement = Task { @MainActor in + try await self.run(harness, on: "logs", leaving: .idle) + } + await hold.arrived(1) + + try await manager.switchDatabase(to: "logs", for: connectionId, persist: false) + await hold.open() + let ranOn = try await statement.value + + let statementOpened = try #require(harness.opener.inOrder.first) + let switchOpened = try #require(harness.opener.inOrder.last) + #expect(ranOn == ObjectIdentifier(switchOpened)) + #expect(session(harness)?.driver === switchOpened) + #expect(statementOpened.disconnectCallCount == 1) + #expect(switchOpened.disconnectCallCount == 0) + } + + @Test("A switch elsewhere on the connection while a statement opens its connection does not fail the statement") + func unrelatedSwitchDoesNotFailTheStatement() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + let hold = AsyncGate() + harness.opener.holdsNextOpen = hold + let statement = Task { @MainActor in + try await self.run(harness, on: "logs", leaving: .idle) + } + await hold.arrived(1) + + _ = manager.sessionLanes.beginSwitch(for: harness.connection.id) + await hold.open() + let ranOn = try await statement.value + + let superseded = try #require(harness.opener.inOrder.first) + let parked = try #require(manager.sessionLanes.parkedDriver(for: harness.connection.id, database: "logs")) + #expect(harness.opener.openCount == 2) + #expect(superseded.disconnectCallCount == 1) + #expect(ranOn == ObjectIdentifier(parked)) + } + + @Test("A server that refuses the session connection fails the statement and parks nothing") + func refusedSessionConnectionFailsTheStatement() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + harness.opener.failure = DatabaseError.connectionFailed("too many connections") + let scope = DatabaseScope(connectionId: harness.connection.id, database: "logs", schema: nil) + + await #expect(throws: DatabaseError.self) { + try await DatabaseManager.shared.withScopedDriver( + scope: scope, route: .sessionDriver, cancellation: .untracked + ) { _ in () } + } + + #expect(DatabaseManager.shared.sessionLanes.parkedDatabases(for: harness.connection.id).isEmpty) + } + + @Test("A statement queued for a connection that is gone by its turn is refused, not moved onto a new one") + func statementQueuedForAGoneConnectionIsRefused() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + try await manager.switchDatabase(to: "logs", for: harness.connection.id, persist: false) + let scope = DatabaseScope(connectionId: harness.connection.id, database: "app", schema: nil) + let key = SessionDriverGate.Key(connectionId: harness.connection.id, database: "app") + let acquired = AsyncGate() + let release = AsyncGate() + let holder = Task { @MainActor in + try await manager.sessionDriverGate.withExclusiveAccess(key) { + await acquired.open() + await release.wait() + } + } + await acquired.wait() + let queued = Task { @MainActor in + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in () } + } + try await waitUntil { manager.sessionDriverGate.waiterCount(for: harness.connection.id) >= 1 } + + manager.sessionLanes.close(database: "app", for: harness.connection.id) + await release.open() + try await holder.value + + await #expect(throws: (any Error).self) { try await queued.value } + #expect(harness.opener.opened["app"] == nil) } @Test("Ending the session closes every database's connection") @@ -419,7 +645,26 @@ struct SessionLanesTests { cancellation: .untracked ) { _ in () } } - #expect(DatabaseManager.shared.executionRoute(for: scope) == .pooled) + #expect(!DatabaseManager.shared.sessionLanes.hasTransactionLoss(database: "app", for: harness.connection.id)) + } + + @Test("A table read leaves a lost transaction's report for the user's next statement") + func tableReadLeavesTheLossReport() async throws { + let harness = makeHarness() + defer { cleanUp(harness) } + let manager = DatabaseManager.shared + manager.sessionLanes.markTransactionLost(database: "app", for: harness.connection.id) + let scope = DatabaseScope(connectionId: harness.connection.id, database: "app", schema: nil) + + let ranOn = try await manager.withTableReadDriver(scope: scope, cancellation: .untracked) { driver in + ObjectIdentifier(driver) + } + + #expect(ranOn == ObjectIdentifier(harness.home)) + #expect(manager.sessionLanes.hasTransactionLoss(database: "app", for: harness.connection.id)) + await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in () } + } } @Test("Closing a database supersedes a reopen of it still in flight") diff --git a/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift b/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift index 38f6952f77..d076adf949 100644 --- a/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift +++ b/apple/TableProTests/Core/Database/SessionRecoveryLoopTests.swift @@ -298,6 +298,49 @@ struct SessionRecoveryLoopTests { await cleanUp(fixture) } + @Test("A recovery that replaces a driver holding a transaction refuses the next statement once") + func recoveryReportsTheTransactionItEnded() async throws { + let fixture = makeFixture() + let manager = fixture.manager + let scope = DatabaseScope(connectionId: fixture.id, database: "app", schema: nil) + fixture.driver.sessionTransactionStateToReturn = .inTransaction + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in () } + #expect(manager.sessionTransactions.holdsTransaction(connectionId: fixture.id, container: "app")) + + manager.beginRecovery(for: fixture.id, cause: .stoppedResponding) + let loop = try loopTask(fixture) + await runNextAttempt(fixture) + await loop.value + + #expect(fixture.session?.driver !== fixture.driver) + #expect(!manager.sessionTransactions.holdsTransaction(connectionId: fixture.id, container: "app")) + #expect(manager.sessionLanes.hasTransactionLoss(database: "app", for: fixture.id)) + let error = await #expect(throws: DatabaseError.self) { + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in () } + } + #expect(error?.localizedDescription.contains(DatabaseManager.parkedTransactionLostMessage(database: "app")) == true) + #expect(!manager.sessionLanes.hasTransactionLoss(database: "app", for: fixture.id)) + await cleanUp(fixture) + } + + @Test("A recovery that replaces an idle driver refuses nothing") + func recoveryOfAnIdleDriverReportsNothing() async throws { + let fixture = makeFixture() + let manager = fixture.manager + let scope = DatabaseScope(connectionId: fixture.id, database: "app", schema: nil) + fixture.driver.sessionTransactionStateToReturn = .idle + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in () } + + manager.beginRecovery(for: fixture.id, cause: .stoppedResponding) + let loop = try loopTask(fixture) + await runNextAttempt(fixture) + await loop.value + + #expect(fixture.session?.driver !== fixture.driver) + #expect(!manager.sessionLanes.hasTransactionLoss(database: "app", for: fixture.id)) + await cleanUp(fixture) + } + @Test("Cancel during the backoff stops the loop before it dials and marks the session lost") func cancelDuringBackoffStopsTheLoop() async throws { let fixture = makeFixture() diff --git a/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift b/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift index cf8b114869..a8e7e7852d 100644 --- a/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift +++ b/apple/TableProTests/Core/Database/SwitchDatabasePooledConnectionTests.swift @@ -93,6 +93,23 @@ struct SwitchDatabasePooledConnectionTests { #expect(!MetadataConnectionPool.shared.isReplacingTransport(for: connection.id)) } + @Test("A switch that reconnects reports the transaction the connection it closed was holding") + func reconnectingSwitchReportsTheTransactionItEnded() async throws { + let connection = makeSession() + defer { cleanUp(connection) } + let manager = DatabaseManager.shared + let driver = try #require(manager.session(for: connection.id)?.driver as? MockDatabaseDriver) + driver.sessionTransactionStateToReturn = .inTransaction + let scope = DatabaseScope(connectionId: connection.id, database: "app", schema: nil) + try await manager.withScopedDriver(scope: scope, route: .sessionDriver, cancellation: .untracked) { _ in } + #expect(manager.sessionTransactions.holdsTransaction(connectionId: connection.id, container: "app")) + + try? await manager.switchDatabase(to: "unreachable", for: connection.id, persist: false) + + #expect(driver.disconnectCallCount >= 1) + #expect(manager.sessionLanes.hasTransactionLoss(database: "app", for: connection.id)) + } + @Test("A switch that reconnects a connection that stopped answering closes its pooled connections") func recoveringSwitchClosesThePool() async { let connection = makeSession(stoppedAnswering: true) diff --git a/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift b/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift index c47fc806ea..e67a5599bd 100644 --- a/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift +++ b/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift @@ -268,3 +268,172 @@ struct TriggerApplyExecutionTests { #expect(sessionStub.executedQueries.isEmpty) } } + +struct TriggerStatementShapeTests { + @Test("A trigger's action body is not a CREATE TRIGGER statement") + func actionBodyIsRefused() { + #expect(!TriggerStatementShape.createsTrigger("UPDATE stats SET n = n + 1", databaseType: .mysql)) + #expect(!TriggerStatementShape.createsTrigger("BEGIN\n UPDATE stats SET n = n + 1;\nEND", databaseType: .mysql)) + #expect(!TriggerStatementShape.createsTrigger("CREATE PROCEDURE p() SELECT 'create trigger'", databaseType: .mysql)) + #expect(!TriggerStatementShape.createsTrigger("-- CREATE TRIGGER t\nUPDATE stats SET n = 1", databaseType: .mysql)) + } + + @Test("A full definition is accepted, with a definer, a replace clause or a function ahead of it") + func fullDefinitionIsAccepted() { + #expect(TriggerStatementShape.createsTrigger( + "CREATE DEFINER=`root`@`localhost` TRIGGER `bump` AFTER INSERT ON `orders` FOR EACH ROW\nUPDATE stats SET n = n + 1", + databaseType: .mysql + )) + #expect(TriggerStatementShape.createsTrigger("create or replace trigger t before insert on x begin null; end;", databaseType: .oracle)) + #expect(TriggerStatementShape.createsTrigger("CREATE TEMP TRIGGER t AFTER INSERT ON x BEGIN SELECT 1; END", databaseType: .sqlite)) + #expect(TriggerStatementShape.createsTrigger( + """ + CREATE OR REPLACE FUNCTION f() RETURNS trigger AS $$ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql; + CREATE TRIGGER t BEFORE INSERT ON x FOR EACH ROW EXECUTE FUNCTION f(); + """, + databaseType: .postgresql + )) + } +} + +@MainActor +struct TriggerEditSafetyTests { + private static let fullDefinition = "CREATE TRIGGER bump AFTER INSERT ON orders FOR EACH ROW\nUPDATE stats SET n = n + 1" + private static let body = "UPDATE stats SET n = n + 1" + + private func makeStubAndAdapter() -> (StubTriggerDriver, PluginDriverAdapter) { + let stub = StubTriggerDriver() + let connection = DatabaseConnection(name: "Test", type: .mysql) + return (stub, PluginDriverAdapter(connection: connection, pluginDriver: stub)) + } + + @Test("The editor is seeded with the full definition, never the action body") + func editorSourceIsTheFullDefinition() async throws { + let (_, adapter) = makeStubAndAdapter() + let trigger = TriggerInfo( + name: "bump", timing: "AFTER", event: "INSERT", statement: Self.body, table: "orders", + definition: Self.fullDefinition + ) + let source = try await TriggerEditSource.resolve(trigger, table: "orders", databaseType: .mysql, on: adapter) + #expect(source == Self.fullDefinition) + } + + @Test("A trigger with only an action body cannot be opened for editing") + func bodyOnlyTriggerIsRefused() async { + let (_, adapter) = makeStubAndAdapter() + let trigger = TriggerInfo(name: "bump", timing: "AFTER", event: "INSERT", statement: Self.body, table: "orders") + await #expect(throws: (any Error).self) { + _ = try await TriggerEditSource.resolve(trigger, table: "orders", databaseType: .mysql, on: adapter) + } + } + + @Test("A statement that is itself the full CREATE is still editable") + func fullStatementIsEditable() async throws { + let (_, adapter) = makeStubAndAdapter() + let trigger = TriggerInfo(name: "bump", timing: "AFTER", event: "INSERT", statement: Self.fullDefinition, table: "orders") + let source = try await TriggerEditSource.resolve(trigger, table: "orders", databaseType: .mysql, on: adapter) + #expect(source == Self.fullDefinition) + } + + @Test("A drop-first edit refuses text that would not recreate the trigger") + func dropFirstRefusesANonCreate() { + let plan = TriggerApplyPlan(strategy: .dropThenCreate, dropSQL: "DROP TRIGGER bump") + #expect(throws: TriggerEditingError.self) { + _ = try TriggerEditing.restoreDefinition( + for: plan, sql: Self.body, originalDefinition: Self.fullDefinition, databaseType: .mysql + ) + } + let transactional = TriggerApplyPlan(strategy: .transactional(dropFirst: true), dropSQL: "DROP TRIGGER bump") + #expect(throws: TriggerEditingError.self) { + _ = try TriggerEditing.restoreDefinition( + for: transactional, sql: Self.body, originalDefinition: nil, databaseType: .sqlite + ) + } + } + + @Test("A drop-then-create edit refuses to run without a definition to restore") + func dropThenCreateNeedsARestorableOriginal() throws { + let plan = TriggerApplyPlan(strategy: .dropThenCreate, dropSQL: "DROP TRIGGER bump") + #expect(throws: TriggerEditingError.self) { + _ = try TriggerEditing.restoreDefinition( + for: plan, sql: Self.fullDefinition, originalDefinition: Self.body, databaseType: .mysql + ) + } + #expect(throws: TriggerEditingError.self) { + _ = try TriggerEditing.restoreDefinition( + for: plan, sql: Self.fullDefinition, originalDefinition: nil, databaseType: .mysql + ) + } + let restore = try TriggerEditing.restoreDefinition( + for: plan, sql: Self.fullDefinition, originalDefinition: Self.fullDefinition, databaseType: .mysql + ) + #expect(restore == Self.fullDefinition) + } + + @Test("A replace or a create runs without a restore check") + func directNeedsNoRestore() throws { + let plan = TriggerApplyPlan(strategy: .direct, dropSQL: nil) + let restore = try TriggerEditing.restoreDefinition( + for: plan, sql: Self.body, originalDefinition: nil, databaseType: .oracle + ) + #expect(restore == nil) + #expect(plan.gatedStatements(for: Self.fullDefinition).isEmpty) + } + + @Test("The gate is shown the DROP ahead of the CREATE") + func gateSeesTheDrop() { + let plan = TriggerApplyPlan(strategy: .dropThenCreate, dropSQL: "DROP TRIGGER bump") + #expect(plan.gatedStatements(for: Self.fullDefinition) == ["DROP TRIGGER bump", Self.fullDefinition]) + } + + @Test("A failed restore reports that the trigger is gone and carries its definition") + func failedRestoreIsReported() async { + let (stub, adapter) = makeStubAndAdapter() + stub.throwOnQueryContaining = "CREATE TRIGGER" + var dropped: String? + do { + try await TriggerEditing.runDropThenCreate( + driver: adapter, dropSQL: "DROP TRIGGER bump", sql: "CREATE TRIGGER bump broken", rollback: Self.fullDefinition + ) + Issue.record("The edit was expected to fail") + } catch { + dropped = (error as? TriggerEditingError)?.droppedDefinition + } + #expect(dropped == Self.fullDefinition) + #expect(stub.executedQueries == ["DROP TRIGGER bump", "CREATE TRIGGER bump broken", Self.fullDefinition]) + } + + @Test("Saving an action body as an edit drops nothing") + func applyRunsNothingForAnActionBody() async throws { + let connection = TestFixtures.makeConnection(database: "app", type: .mysql) + let stub = StubTriggerDriver() + stub.dropToReturn = "DROP TRIGGER bump" + let adapter = PluginDriverAdapter(connection: connection, pluginDriver: stub) + try await adapter.connect() + let session = ConnectionSession(connection: connection, driver: adapter) + DatabaseManager.shared.injectSession(session, for: connection.id) + let scope = DatabaseScope(connectionId: connection.id, database: "app", schema: nil) + MetadataConnectionPool.shared.injectEntry(adapter, scope: scope) + defer { + MetadataConnectionPool.shared.closeAll(connectionId: connection.id) + DatabaseManager.shared.removeSession(for: connection.id) + } + let gate = RequestRecordingGate(allows: true) + + await #expect(throws: TriggerEditingError.self) { + try await TriggerEditing.apply( + scope: scope, + connection: connection, + tableName: "orders", + sql: Self.body, + isEdit: true, + originalName: "bump", + originalDefinition: Self.body, + gate: gate + ) + } + #expect(stub.executedQueries.isEmpty) + let requests = await gate.requests + #expect(requests.isEmpty) + } +} diff --git a/apple/TableProTests/Core/MCP/Completions/MCPCompletionApprovalGateTests.swift b/apple/TableProTests/Core/MCP/Completions/MCPCompletionApprovalGateTests.swift new file mode 100644 index 0000000000..1ac2aab4c7 --- /dev/null +++ b/apple/TableProTests/Core/MCP/Completions/MCPCompletionApprovalGateTests.swift @@ -0,0 +1,166 @@ +// +// MCPCompletionApprovalGateTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProMCP +import TableProMCPTestSupport +import TableProPluginKit +import Testing + +struct MCPCompletionApprovalGateTests { + private let connectionId = UUID() + + private func makePrincipal() -> MCPPrincipal { + MCPPrincipal( + tokenFingerprint: "fp", + tokenId: UUID(), + scopes: MCPScope.fullAccessSet, + connectionAccess: .all, + metadata: MCPPrincipalMetadata( + label: "token", + issuedAt: .distantPast, + expiresAt: nil, + isBridgeCredential: false + ) + ) + } + + private func makePolicy( + approval: MCPConnectionApproval, + policy: AIConnectionPolicy = .askEachTime, + presenter: RecordingApprovalPresenter + ) -> MCPAuthPolicy { + let snapshot = MCPConnectionAuthSnapshot( + policy: policy, + externalAccess: .readWrite, + name: "Test Connection", + databaseType: DatabaseType.postgresql.rawValue + ) + return MCPAuthPolicy( + connectionResolver: { _ in snapshot }, + connectionIdsProvider: { [] }, + approvalLedger: MCPApprovalLedger(clock: MCPTestClock(), store: MCPInMemoryApprovalStore()), + approvalSetting: MCPFixedApprovalReader(approval), + presenter: presenter + ) + } + + @Test("A connection nobody approved yet is refused without raising the prompt") + func unapprovedIsRefusedSilently() async { + let presenter = RecordingApprovalPresenter(answer: true) + let policy = makePolicy(approval: .oncePerConnection, presenter: presenter) + + let allowed = await policy.isAllowedWithoutPrompt(principal: makePrincipal(), connectionId: connectionId) + + #expect(!allowed) + #expect(await presenter.askedCount == 0) + } + + @Test("A remembered grant lets the silent check through") + func rememberedGrantIsAllowed() async throws { + let presenter = RecordingApprovalPresenter(answer: true) + let policy = makePolicy(approval: .oncePerConnection, presenter: presenter) + let principal = makePrincipal() + try await policy.resolveAndAuthorize(principal: principal, tool: "list_tables", connectionId: connectionId) + + #expect(await policy.isAllowedWithoutPrompt(principal: principal, connectionId: connectionId)) + } + + @Test("A client the user just denied stays refused") + func deniedClientIsRefused() async { + let presenter = RecordingApprovalPresenter(answer: false) + let policy = makePolicy(approval: .oncePerConnection, presenter: presenter) + let principal = makePrincipal() + await #expect(throws: DatabaseAccessError.self) { + try await policy.resolveAndAuthorize(principal: principal, tool: "list_tables", connectionId: connectionId) + } + + #expect(await !policy.isAllowedWithoutPrompt(principal: principal, connectionId: connectionId)) + #expect(await presenter.askedCount == 1) + } + + @Test("Ask Every Time never passes the silent check") + func everyTimeIsRefused() async throws { + let presenter = RecordingApprovalPresenter(answer: true) + let policy = makePolicy(approval: .everyTime, presenter: presenter) + let principal = makePrincipal() + try await policy.resolveAndAuthorize(principal: principal, tool: "list_tables", connectionId: connectionId) + + #expect(await !policy.isAllowedWithoutPrompt(principal: principal, connectionId: connectionId)) + } + + @Test("Always Allow and Never Ask pass the silent check") + func openSettingsAreAllowed() async { + let presenter = RecordingApprovalPresenter(answer: false) + let alwaysAllow = makePolicy(approval: .everyTime, policy: .alwaysAllow, presenter: presenter) + let neverAsk = makePolicy(approval: .alwaysApprove, presenter: presenter) + + #expect(await alwaysAllow.isAllowedWithoutPrompt(principal: makePrincipal(), connectionId: connectionId)) + #expect(await neverAsk.isAllowedWithoutPrompt(principal: makePrincipal(), connectionId: connectionId)) + } + + @Test("A connection set to Never is refused whatever the approval setting") + func neverPolicyIsRefused() async { + let presenter = RecordingApprovalPresenter(answer: true) + let policy = makePolicy(approval: .alwaysApprove, policy: .never, presenter: presenter) + + #expect(await !policy.isAllowedWithoutPrompt(principal: makePrincipal(), connectionId: connectionId)) + } + + @Test("Completing a database, schema or table of an unapproved connection reads nothing") + func unapprovedConnectionCompletesNothing() async { + let source = RecordingCompletionSource(connections: CompletionFixtures.connections) + await source.setDatabases(["warehouse"]) + await source.setSchemas(["public"]) + await source.setTables(["orders"]) + await source.setUnapproved([CompletionFixtures.analyticsId]) + let provider = MCPCompletionProvider(source: source) + let context = ["connection_id": CompletionFixtures.analyticsId.uuidString] + + for argument in ["database", "schema", MCPPromptArgument.tableArgumentName] { + let result = await provider.complete( + reference: .prompt(name: "explain_table"), + argumentName: argument, + argumentValue: "", + context: context, + principal: CompletionFixtures.principal() + ) + #expect(result.values.isEmpty) + } + + #expect(await source.databaseCalls.isEmpty) + #expect(await source.schemaCalls.isEmpty) + #expect(await source.tableCalls.isEmpty) + } + + @Test("Names cached while approved are not served once the approval is gone") + func cachedNamesNeedApproval() async { + let source = RecordingCompletionSource(connections: CompletionFixtures.connections) + await source.setTables(["orders"]) + let provider = MCPCompletionProvider(source: source, clock: MCPProtocolStubClock(), cacheDuration: 15) + let principal = CompletionFixtures.principal() + let context = ["connection": "Analytics"] + + let first = await provider.complete( + reference: .prompt(name: "explain_table"), + argumentName: MCPPromptArgument.tableArgumentName, + argumentValue: "", + context: context, + principal: principal + ) + #expect(first.values == ["orders"]) + + await source.setUnapproved([CompletionFixtures.analyticsId]) + let second = await provider.complete( + reference: .prompt(name: "explain_table"), + argumentName: MCPPromptArgument.tableArgumentName, + argumentValue: "", + context: context, + principal: principal + ) + #expect(second.values.isEmpty) + } +} diff --git a/apple/TableProTests/Core/MCP/Completions/MCPCompletionProviderTests.swift b/apple/TableProTests/Core/MCP/Completions/MCPCompletionProviderTests.swift index 5016a62907..6abdb60e1c 100644 --- a/apple/TableProTests/Core/MCP/Completions/MCPCompletionProviderTests.swift +++ b/apple/TableProTests/Core/MCP/Completions/MCPCompletionProviderTests.swift @@ -687,6 +687,8 @@ actor RecordingCompletionSource: MCPCompletionSchemaSource { private var storedSchemas: [String] = [] private var storedTables: [String] = [] + private var unapproved: Set = [] + private(set) var databaseCalls: [DatabaseCall] = [] private(set) var schemaCalls: [SchemaCall] = [] private(set) var tableCalls: [TableCall] = [] @@ -699,6 +701,14 @@ actor RecordingCompletionSource: MCPCompletionSchemaSource { storedConnections = connections } + func setUnapproved(_ connectionIds: Set) { + unapproved = connectionIds + } + + func isApproved(principal: MCPPrincipal, connectionId: UUID) async -> Bool { + !unapproved.contains(connectionId) + } + func setDatabases(_ databases: [String]) { storedDatabases = databases } diff --git a/apple/TableProTests/Core/MCP/MCPTokenStoreRecoveryTests.swift b/apple/TableProTests/Core/MCP/MCPTokenStoreRecoveryTests.swift new file mode 100644 index 0000000000..a7c3a442b7 --- /dev/null +++ b/apple/TableProTests/Core/MCP/MCPTokenStoreRecoveryTests.swift @@ -0,0 +1,234 @@ +// +// MCPTokenStoreRecoveryTests.swift +// TableProTests +// + +import Foundation +import TableProMCP +import Testing + +@testable import TablePro + +private final class ScriptedCredentialStore: MCPTokenCredentialStoring, @unchecked Sendable { + private let lock = NSLock() + private var stored: Data? + private var readable = true + private var writes = 0 + + init(stored: Data?) { + self.stored = stored + } + + func setReadable(_ value: Bool) { + lock.withLock { readable = value } + } + + var writeCount: Int { + lock.withLock { writes } + } + + var storedData: Data? { + lock.withLock { stored } + } + + func read() -> MCPTokenBlob { + lock.withLock { + guard readable else { return .unreadable } + return stored.map(MCPTokenBlob.found) ?? .missing + } + } + + @discardableResult + func write(_ data: Data) -> Bool { + lock.withLock { + stored = data + writes += 1 + } + return true + } + + func delete() { + lock.withLock { stored = nil } + } +} + +struct MCPTokenStoreRecoveryTests { + private func makeToken(name: String) -> MCPAuthToken { + MCPAuthToken( + id: UUID(), + name: name, + prefix: "tp_abc12", + tokenHash: "fakehash-\(name)", + salt: "fakesalt", + permissions: .readOnly, + connectionAccess: .all, + createdAt: Date(timeIntervalSince1970: 1_700_000_000), + lastUsedAt: nil, + expiresAt: nil, + isActive: true + ) + } + + private func blob(_ tokens: [MCPAuthToken], extra: [[String: Any]] = []) throws -> Data { + let encoded = try MCPTokenStore.encodeStored(tokens: tokens, undecoded: []) + var entries = try #require(try JSONSerialization.jsonObject(with: encoded) as? [Any]) + entries.append(contentsOf: extra) + return try JSONSerialization.data(withJSONObject: entries) + } + + private func futureEntry(name: String = "from-a-newer-build") -> [String: Any] { + [ + "id": UUID().uuidString, + "name": name, + "tokenHash": "hash", + "salt": "salt", + "permissions": "somethingNewer", + "createdAt": "2026-01-01T00:00:00Z", + "isActive": true + ] + } + + private func storedNames(_ store: ScriptedCredentialStore) throws -> [String] { + let data = try #require(store.storedData) + let entries = try #require(try JSONSerialization.jsonObject(with: data) as? [[String: Any]]) + return entries.compactMap { $0["name"] as? String } + } + + private func mint(_ store: MCPTokenStore, name: String, bridge: Bool = false) async throws -> String { + try await store.generate( + name: name, + permissions: .readOnly, + connectionAccess: .all, + expiresAt: nil, + isBridgeCredential: bridge, + extraScopes: [] + ).plaintext + } + + @Test("One entry this build cannot decode leaves the other tokens loaded") + func undecodableEntryKeepsTheRest() async throws { + let paired = makeToken(name: "paired") + let credentials = ScriptedCredentialStore(stored: try blob([paired], extra: [futureEntry()])) + let store = MCPTokenStore(credentialStore: credentials) + + await store.loadFromDisk() + + #expect(await store.list().map(\.id) == [paired.id]) + } + + @Test("Saving writes the undecodable entry back beside the others") + func undecodableEntrySurvivesASave() async throws { + let paired = makeToken(name: "paired") + let credentials = ScriptedCredentialStore(stored: try blob([paired], extra: [futureEntry()])) + let store = MCPTokenStore(credentialStore: credentials) + await store.loadFromDisk() + + _ = try await mint(store, name: "bridge", bridge: true) + + let names = try storedNames(credentials) + #expect(names.contains("paired")) + #expect(names.contains("from-a-newer-build")) + #expect(names.contains("bridge")) + } + + @Test("A failed read does not let the bridge mint overwrite the stored tokens") + func unreadableStoreIsNotOverwritten() async throws { + let paired = makeToken(name: "paired") + let original = try blob([paired]) + let credentials = ScriptedCredentialStore(stored: original) + credentials.setReadable(false) + let store = MCPTokenStore(credentialStore: credentials) + await store.loadFromDisk() + + let plaintext = try await mint(store, name: "bridge", bridge: true) + + #expect(credentials.writeCount == 0) + #expect(credentials.storedData == original) + #expect(await store.validate(bearerToken: plaintext) != nil) + } + + @Test("Once the read works again the stored tokens and the ones minted meanwhile are both kept") + func recoveryMergesBothSides() async throws { + let paired = makeToken(name: "paired") + let credentials = ScriptedCredentialStore(stored: try blob([paired])) + credentials.setReadable(false) + let store = MCPTokenStore(credentialStore: credentials) + await store.loadFromDisk() + _ = try await mint(store, name: "minted-while-unreadable", bridge: true) + + credentials.setReadable(true) + _ = try await mint(store, name: "minted-after") + + let names = Set(await store.list().map(\.name)) + #expect(names == ["paired", "minted-while-unreadable", "minted-after"]) + #expect(Set(try storedNames(credentials)) == names) + } + + @Test("Minting a client token while the stored list is unreadable fails and keeps nothing") + func mintFailsWhileUnreadable() async throws { + let credentials = ScriptedCredentialStore(stored: try blob([makeToken(name: "paired")])) + credentials.setReadable(false) + let store = MCPTokenStore(credentialStore: credentials) + await store.loadFromDisk() + + await #expect(throws: MCPTokenStoreError.storedTokensUnreadable) { + _ = try await mint(store, name: "new-client") + } + + #expect(await store.list().isEmpty) + #expect(credentials.writeCount == 0) + } + + @Test("A request from a paired client retries the read") + func authenticationRetriesTheRead() async throws { + let paired = makeToken(name: "paired") + let credentials = ScriptedCredentialStore(stored: try blob([paired])) + credentials.setReadable(false) + let store = MCPTokenStore(credentialStore: credentials) + await store.loadFromDisk() + #expect(await store.list().isEmpty) + + credentials.setReadable(true) + _ = await store.validate(bearerToken: "tp_unknown") + + #expect(await store.list().map(\.id) == [paired.id]) + } + + @Test("A bridge token left in the store is dropped when the read recovers") + func recoveryDropsStaleBridgeTokens() async throws { + let stale = MCPAuthToken( + id: UUID(), + name: MCPTokenStore.stdioBridgeTokenName, + prefix: "tp_old", + tokenHash: "old", + salt: "salt", + permissions: .readWrite, + connectionAccess: .all, + createdAt: Date(timeIntervalSince1970: 1_700_000_000), + lastUsedAt: nil, + expiresAt: nil, + isActive: true, + isBridgeCredential: true + ) + let credentials = ScriptedCredentialStore(stored: try blob([stale, makeToken(name: "paired")])) + credentials.setReadable(false) + let store = MCPTokenStore(credentialStore: credentials) + await store.loadFromDisk() + let current = try await store.generate( + name: MCPTokenStore.stdioBridgeTokenName, + permissions: .readWrite, + connectionAccess: .all, + expiresAt: nil, + isBridgeCredential: true, + extraScopes: [] + ).token + + credentials.setReadable(true) + _ = await store.validate(bearerToken: "tp_unknown") + + let ids = Set(await store.list().map(\.id)) + #expect(ids.contains(current.id)) + #expect(!ids.contains(stale.id)) + #expect(ids.count == 2) + } +} diff --git a/apple/TableProTests/Core/MCP/MCPTokenStoreTests.swift b/apple/TableProTests/Core/MCP/MCPTokenStoreTests.swift index 395c80f377..f65e8544e5 100644 --- a/apple/TableProTests/Core/MCP/MCPTokenStoreTests.swift +++ b/apple/TableProTests/Core/MCP/MCPTokenStoreTests.swift @@ -26,10 +26,10 @@ private final class InMemoryCredentialStore: MCPTokenCredentialStoring, @uncheck self.stored = seed } - func read() -> Data? { + func read() -> MCPTokenBlob { lock.lock() defer { lock.unlock() } - return stored + return stored.map(MCPTokenBlob.found) ?? .missing } @discardableResult diff --git a/apple/TableProTests/Core/MCP/Prompts/PromptsHandlerTests.swift b/apple/TableProTests/Core/MCP/Prompts/PromptsHandlerTests.swift index 1c698d6681..b0f9f1fd0d 100644 --- a/apple/TableProTests/Core/MCP/Prompts/PromptsHandlerTests.swift +++ b/apple/TableProTests/Core/MCP/Prompts/PromptsHandlerTests.swift @@ -275,6 +275,7 @@ struct FixedPromptSchemaSource: MCPCompletionSchemaSource { connections.filter { principal.connectionAccess.allows($0.id) } } + func isApproved(principal: MCPPrincipal, connectionId: UUID) async -> Bool { true } func databases(connectionId: UUID) async -> [String] { [] } func schemas(connectionId: UUID, database: String?) async -> [String] { [] } func tables(connectionId: UUID, database: String?, schema: String?) async -> [String] { [] } diff --git a/apple/TableProTests/Core/Plugins/PluginDriverAdapterBrowseProbeTests.swift b/apple/TableProTests/Core/Plugins/PluginDriverAdapterBrowseProbeTests.swift new file mode 100644 index 0000000000..6ae7cf1e4e --- /dev/null +++ b/apple/TableProTests/Core/Plugins/PluginDriverAdapterBrowseProbeTests.swift @@ -0,0 +1,116 @@ +// +// PluginDriverAdapterBrowseProbeTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +private final class SchemaAwareBrowseDriver: PluginDriverStub, @unchecked Sendable { + var supportsSchemas: Bool { false } + var supportsTransactions: Bool { false } + var currentSchema: String? { nil } + var serverVersion: String? { nil } + private(set) var browsedTables: [String] = [] + + func buildBrowseQuery( + table: String, + schema: String?, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int + ) -> String? { + browsedTables.append(table) + return "READ \(table) LIMIT \(limit)" + } + + func ping() async throws {} +} + +private final class LegacyBrowseDriver: PluginDriverStub, @unchecked Sendable { + var supportsSchemas: Bool { false } + var supportsTransactions: Bool { false } + var currentSchema: String? { nil } + var serverVersion: String? { nil } + + func buildBrowseQuery( + table: String, + sortColumns: [(columnIndex: Int, ascending: Bool)], + columns: [String], + limit: Int, + offset: Int + ) -> String? { + "READ \(table)" + } + + func ping() async throws {} +} + +private final class HostBrowsedDriver: PluginDriverStub, @unchecked Sendable { + var supportsSchemas: Bool { false } + var supportsTransactions: Bool { false } + var currentSchema: String? { nil } + var serverVersion: String? { nil } + + func ping() async throws {} +} + +@MainActor +struct PluginDriverAdapterBrowseProbeTests { + private func makeAdapter(_ driver: any PluginDatabaseDriver) -> PluginDriverAdapter { + PluginDriverAdapter(connection: TestFixtures.makeConnection(), pluginDriver: driver) + } + + @Test("A driver that builds its browse only through the schema-aware overload builds the app's queries") + func schemaAwareOverloadCounts() { + let driver = SchemaAwareBrowseDriver() + let adapter = makeAdapter(driver) + + #expect(adapter.queryBuildingPluginDriver === driver) + #expect(adapter.rowWritingPluginDriver === driver) + #expect(PluginDriverAdapter.buildsBrowseQueries(driver)) + } + + @Test("The table browse of a schema-aware driver is the driver's own statement") + func tableQueryComesFromTheDriver() { + let driver = SchemaAwareBrowseDriver() + let adapter = makeAdapter(driver) + + let query = adapter.queryBuildingPluginDriver?.buildBrowseQuery( + table: "events", schema: nil, sortColumns: [], columns: [], limit: 50, offset: 0 + ) + + #expect(query == "READ events LIMIT 50") + } + + @Test("A driver that answered once is not asked again on every read") + func probesOnce() { + let driver = SchemaAwareBrowseDriver() + let adapter = makeAdapter(driver) + + for _ in 0 ..< 5 { + #expect(adapter.queryBuildingPluginDriver != nil) + } + + #expect(driver.browsedTables == ["_probe"]) + } + + @Test("A driver with only the older overload still builds the app's queries") + func legacyOverloadStillCounts() { + let driver = LegacyBrowseDriver() + + #expect(makeAdapter(driver).queryBuildingPluginDriver === driver) + } + + @Test("A driver that leaves browsing to the app builds nothing") + func hostBrowsedDriverBuildsNothing() { + let adapter = makeAdapter(HostBrowsedDriver()) + + #expect(adapter.queryBuildingPluginDriver == nil) + #expect(adapter.rowWritingPluginDriver == nil) + } +} diff --git a/apple/TableProTests/Core/SchemaTracking/StructureChangeRebaseTests.swift b/apple/TableProTests/Core/SchemaTracking/StructureChangeRebaseTests.swift new file mode 100644 index 0000000000..c847c5ec7d --- /dev/null +++ b/apple/TableProTests/Core/SchemaTracking/StructureChangeRebaseTests.swift @@ -0,0 +1,148 @@ +// +// StructureChangeRebaseTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +// MySQL and Oracle commit each DDL statement as it runs, so a save that fails part way has already changed the table. +@MainActor +struct StructureChangeRebaseTests { + private func column(_ name: String) -> ColumnInfo { + ColumnInfo( + name: name, dataType: "INTEGER", isNullable: name != "id", isPrimaryKey: name == "id", + defaultValue: nil, extra: nil, charset: nil, collation: nil, comment: nil + ) + } + + private func manager() -> StructureChangeManager { + let manager = StructureChangeManager() + manager.loadSchema( + tableName: "notes", + columns: ["id", "a", "x"].map { column($0) }, + indexes: [], + foreignKeys: [], + primaryKey: ["id"] + ) + return manager + } + + private func workingColumn(_ name: String, in manager: StructureChangeManager) throws -> EditableColumnDefinition { + try #require(manager.workingColumns.first { $0.name == name }) + } + + private func stageRenameAddColumnAndIndex(in manager: StructureChangeManager) throws { + var renamed = try workingColumn("a", in: manager) + renamed.name = "b" + manager.updateColumn(id: renamed.id, with: renamed) + var added = try workingColumn("x", in: manager) + added.id = UUID() + added.name = "c" + manager.addColumn(added) + manager.addIndex( + EditableIndexDefinition( + id: UUID(), name: "idx_c", columns: ["c"], type: .btree, isUnique: false, isPrimary: false, comment: nil + ) + ) + } + + @Test("Statements that ran before the failure leave the staged edits") + func appliedChangesAreDropped() throws { + let manager = manager() + try stageRenameAddColumnAndIndex(in: manager) + #expect(manager.getChangesArray().count == 3) + + let moved = manager.rebaseStagedChanges( + onto: StructureRebase.Baseline( + columns: ["id", "b", "x", "c"].map { column($0) }, + indexes: [], + foreignKeys: [], + primaryKey: ["id"] + ) + ) + + #expect(moved) + let changes = manager.getChangesArray() + #expect(changes.count == 1) + if case .addIndex(let index)? = changes.first { + #expect(index.name == "idx_c") + } else { + Issue.record("The index that failed should be the one change left") + } + let baseline = manager.currentColumns.map { $0.name } + let shown = manager.workingColumns.map { $0.name } + #expect(baseline == ["id", "b", "x", "c"]) + #expect(shown == ["id", "b", "x", "c"]) + } + + @Test("A save that wrote nothing keeps every staged edit and its undo history") + func unchangedBaselineIsLeftAlone() throws { + let manager = manager() + try stageRenameAddColumnAndIndex(in: manager) + let before = manager.getChangesArray() + + let moved = manager.rebaseStagedChanges( + onto: StructureRebase.Baseline( + columns: ["id", "a", "x"].map { column($0) }, + indexes: [], + foreignKeys: [], + primaryKey: ["id"] + ) + ) + + #expect(!moved) + #expect(manager.getChangesArray() == before) + #expect(manager.canUndo) + } + + @Test("A deletion that ran is gone, and one that did not is still staged") + func deletionsFollowTheTable() throws { + let manager = manager() + try manager.deleteColumn(id: workingColumn("x", in: manager).id) + try manager.deleteColumn(id: workingColumn("a", in: manager).id) + + let moved = manager.rebaseStagedChanges( + onto: StructureRebase.Baseline( + columns: ["id", "a"].map { column($0) }, + indexes: [], + foreignKeys: [], + primaryKey: ["id"] + ) + ) + + #expect(moved) + let changes = manager.getChangesArray() + #expect(changes.count == 1) + if case .deleteColumn(let deleted)? = changes.first { + #expect(deleted.name == "a") + } else { + Issue.record("The deletion that did not run should be the one change left") + } + let shown = manager.workingColumns.map { $0.name } + #expect(shown == ["id", "a"]) + } + + @Test("A rename that did not run is staged against the table as it is") + func unappliedRenameStaysStaged() throws { + let manager = manager() + try stageRenameAddColumnAndIndex(in: manager) + + let moved = manager.rebaseStagedChanges( + onto: StructureRebase.Baseline( + columns: ["id", "a", "x", "c"].map { column($0) }, + indexes: [], + foreignKeys: [], + primaryKey: ["id"] + ) + ) + + #expect(moved) + let renames = manager.getChangesArray().compactMap { change -> String? in + guard case .modifyColumn(let old, let new) = change else { return nil } + return "\(old.name)>\(new.name)" + } + #expect(renames == ["a>b"]) + } +} diff --git a/apple/TableProTests/Core/Services/Infrastructure/TabPersistenceSessionGateTests.swift b/apple/TableProTests/Core/Services/Infrastructure/TabPersistenceSessionGateTests.swift new file mode 100644 index 0000000000..c4ff4ce90c --- /dev/null +++ b/apple/TableProTests/Core/Services/Infrastructure/TabPersistenceSessionGateTests.swift @@ -0,0 +1,81 @@ +import Foundation +import Testing + +@testable import TablePro + +@MainActor +struct TabPersistenceSessionGateTests { + @Test("Closing a connection's last window closes the gate its restore opened") + func lastTeardownClosesTheGate() { + let fixture = CoordinatorFixture() + let persistence = TabPersistenceCoordinator.forConnection(fixture.connection.id) + persistence.markObservedTabs() + #expect(fixture.coordinator.persistence === persistence) + + fixture.coordinator.teardown() + + #expect(!persistence.hasObservedTabs) + } + + @Test("A lost session keeps the gate open for the workspace that reconnects") + func sessionReleaseKeepsTheGate() { + let fixture = CoordinatorFixture() + let persistence = TabPersistenceCoordinator.forConnection(fixture.connection.id) + persistence.markObservedTabs() + + fixture.coordinator.teardown(endsTabSession: false) + + #expect(persistence.hasObservedTabs) + + // The workspace then closes with no coordinator left to tear down. + MainContentCoordinator.endTabSessionIfUnhosted(for: fixture.connection.id) + #expect(!persistence.hasObservedTabs) + } + + private func disconnectedWorkspace(for connection: DatabaseConnection) -> ConnectionWorkspace { + ConnectionWorkspace( + connectionId: connection.id, + payload: nil, + autoConnect: false, + payloadConnection: connection, + session: nil, + sessionState: nil, + trailingPaneState: nil, + phase: .idle + ) + } + + @Test("A disconnected workspace in another window holds the gate when a window closes") + func disconnectedWorkspaceElsewhereHoldsTheGate() { + let connection = TestFixtures.makeConnection() + let closing = disconnectedWorkspace(for: connection) + let remaining = disconnectedWorkspace(for: connection) + let persistence = TabPersistenceCoordinator.forConnection(connection.id) + persistence.markObservedTabs() + + MainContentCoordinator.endTabSessionIfUnhosted( + for: connection.id, closing: closing, hosted: [closing, remaining] + ) + #expect(persistence.hasObservedTabs) + + MainContentCoordinator.endTabSessionIfUnhosted(for: connection.id, closing: remaining, hosted: [remaining]) + #expect(!persistence.hasObservedTabs) + } + + @Test("The gate stays open while another window still hosts the connection") + func gateStaysOpenForARemainingWindow() { + let connection = TestFixtures.makeConnection() + let closing = CoordinatorFixture(connection: connection) + let remaining = CoordinatorFixture(connection: connection) + remaining.coordinator.registerEagerly() + remaining.coordinator.hasRestoredTabs = true + let persistence = TabPersistenceCoordinator.forConnection(connection.id) + persistence.markObservedTabs() + + closing.coordinator.teardown() + #expect(persistence.hasObservedTabs) + + remaining.coordinator.teardown() + #expect(!persistence.hasObservedTabs) + } +} diff --git a/apple/TableProTests/Core/Services/Masking/MaskingExportTests.swift b/apple/TableProTests/Core/Services/Masking/MaskingExportTests.swift index 6d2ff85d69..3e467e5515 100644 --- a/apple/TableProTests/Core/Services/Masking/MaskingExportTests.swift +++ b/apple/TableProTests/Core/Services/Masking/MaskingExportTests.swift @@ -6,6 +6,7 @@ // import Foundation +import os @testable import TablePro import TableProPluginKit import Testing @@ -94,6 +95,31 @@ struct MaskingExportTests { #expect(MaskingExportDataSource.wrapping(base, policy: .none) === base) } + @Test("A masked stream reads a row batch only when its consumer asks for one") + func maskedStreamPullsOnDemand() async throws { + let pulls = OSAllocatedUnfairLock(initialState: 0) + let endless = AsyncThrowingStream(unfolding: { + let pull = pulls.withLock { count -> Int in + count += 1 + return count + } + return pull == 1 + ? .header(PluginStreamHeader(columns: ["id", "email"], columnTypeNames: ["INT", "VARCHAR"])) + : .rows([[.text("1"), .text("a@example.com")]]) + }) + var iterator = MaskingExportDataSource.masking(endless, table: "customers", policy: policy).makeAsyncIterator() + _ = try await iterator.next() + let batch = try await iterator.next() + guard case .rows(let rows)? = batch else { + Issue.record("expected rows") + return + } + #expect(rows == [[.text("1"), .text("••••")]]) + // A pump would have run ahead of a consumer that stopped here. + try await Task.sleep(for: .milliseconds(50)) + #expect(pulls.withLock { $0 } == 2) + } + @Test("Rows that arrive before their header are refused rather than let through") func rowsBeforeHeaderAreRefused() async { let headless = FixedRowsDataSource([.rows([[.text("jane@example.com")]])]) diff --git a/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift b/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift new file mode 100644 index 0000000000..c4999b4236 --- /dev/null +++ b/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift @@ -0,0 +1,178 @@ +// +// RowPasteForeignTextTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@MainActor +struct RowPasteForeignTextTests { + private static let columns = ["id", "name", "email"] + + private func parse(_ text: String, primaryKey: [String] = ["id"]) throws -> [[PluginCellValue]] { + let schema = TableSchema(columns: Self.columns, primaryKeyColumns: primaryKey) + return try TSVRowParser().parse(text, schema: schema).get().map(\.values) + } + + @Test("A row whose first cell is empty keeps every value in its own column") + func leadingEmptyFieldStaysInPlace() throws { + let rows = try parse("\tAnn\tann@x.com\n\tBob\tbob@x.com") + + #expect(rows == [ + [.text("__DEFAULT__"), .text("Ann"), .text("ann@x.com")], + [.text("__DEFAULT__"), .text("Bob"), .text("bob@x.com")] + ]) + } + + @Test("An empty first cell reads as NULL when the table has no key to reset") + func leadingEmptyFieldIsNullWithoutAKey() throws { + let rows = try parse("\tAnn\tann@x.com", primaryKey: []) + + #expect(rows == [[.null, .text("Ann"), .text("ann@x.com")]]) + } + + @Test("Trailing empty cells stay NULL and blank lines are not rows") + func trailingEmptyFieldsAndBlankLines() throws { + let rows = try parse("7\tAnn\t\r\n\r\n8\t\tbob@x.com\n \n", primaryKey: []) + + #expect(rows == [ + [.text("7"), .text("Ann"), .null], + [.text("8"), .null, .text("bob@x.com")] + ]) + } + + @Test("A cell keeps its own leading and trailing spaces") + func cellSpacesAreKept() throws { + let rows = try parse("7\t indented\ttrailing ", primaryKey: []) + + #expect(rows == [[.text("7"), .text(" indented"), .text("trailing ")]]) + } + + @Test("Pasting spreadsheet rows with a blank first column lands each value under its header") + func pasteThroughTheManager() { + let changeManager = DataChangeManager() + changeManager.configureForTable( + tableName: "users", + columns: Self.columns, + primaryKeyColumns: ["id"], + databaseType: .mysql, + generatedColumns: [] + ) + let manager = RowOperationsManager(changeManager: changeManager) + let clipboard = RecordingClipboard() + clipboard.text = "\tAnn\tann@x.com" + var rows = TableRows.from( + queryRows: [], + columns: Self.columns, + columnTypes: Array(repeating: .text(rawType: nil), count: Self.columns.count) + ) + + let result = manager.pasteRowsFromClipboard( + columns: Self.columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard + ) + + #expect(result.pastedRows.first?.values == [.text("__DEFAULT__"), .text("Ann"), .text("ann@x.com")]) + } +} + +@MainActor +struct RowPasteBatchRecordingTests { + private static let columns = ["id", "name"] + + private func makeManagers() -> (RowOperationsManager, DataChangeManager, UndoManager) { + let changeManager = DataChangeManager() + let undoManager = UndoManager() + undoManager.groupsByEvent = false + changeManager.undoManagerProvider = { undoManager } + changeManager.configureForTable( + tableName: "users", + columns: Self.columns, + primaryKeyColumns: ["id"], + databaseType: .mysql, + generatedColumns: [] + ) + return (RowOperationsManager(changeManager: changeManager), changeManager, undoManager) + } + + private func paste(_ text: String, with manager: RowOperationsManager, into rows: inout TableRows) + -> RowOperationsManager.PasteRowsResult + { + let clipboard = RecordingClipboard() + clipboard.text = text + return manager.pasteRowsFromClipboard( + columns: Self.columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard + ) + } + + private func emptyRows() -> TableRows { + TableRows.from( + queryRows: [["1", "Zed"]], + columns: Self.columns, + columnTypes: Array(repeating: .text(rawType: nil), count: Self.columns.count) + ) + } + + @Test("A paste of several rows is one undo step, and its redo puts every row back") + func pasteIsOneUndoStep() { + let (manager, changeManager, undoManager) = makeManagers() + var rows = emptyRows() + var applied: [UndoResult] = [] + changeManager.onUndoApplied = { applied.append($0) } + + let result = paste("7\tAnn\n8\tBob\n9\tCy", with: manager, into: &rows) + + let pastedIDs = Set(result.pastedRows.map { $0.rowID }) + let pastedValues = result.pastedRows.map { $0.values } + #expect(result.delta == .rowsInserted(IndexSet(integersIn: 1..<4))) + #expect(changeManager.insertedRowIDs == pastedIDs) + #expect(rows.count == 4) + + undoManager.undo() + #expect(changeManager.insertedRowIDs.isEmpty) + #expect(!changeManager.hasChanges) + #expect(!undoManager.canUndo) + for step in applied { + _ = manager.applyUndoResult(step, tableRows: &rows) + } + applied.removeAll() + #expect(rows.count == 1) + + undoManager.redo() + #expect(changeManager.insertedRowIDs.count == 3) + for step in applied { + _ = manager.applyUndoResult(step, tableRows: &rows) + } + #expect(rows.count == 4) + let restoredValues = rows.rows.suffix(3).map { Array($0.values) } + #expect(restoredValues == pastedValues) + } + + @Test("The batch entry point records every row with its values, missing fields and expressions") + func batchInsertionRecordsEachRow() { + let (_, changeManager, _) = makeManagers() + let first = RowID.inserted(UUID()) + let second = RowID.inserted(UUID()) + + changeManager.recordBatchRowInsertion(rows: [ + DataChangeManager.InsertedRow( + location: InsertedRowLocation(rowID: first, storageIndex: 0), values: ["1", "now()"], + expressionColumns: [1] + ), + DataChangeManager.InsertedRow( + location: InsertedRowLocation(rowID: second, storageIndex: 1), values: ["2", nil], + absentColumns: [1] + ) + ]) + + let state = changeManager.saveState() + #expect(state.insertedRowData[first] == ["1", "now()"]) + #expect(state.insertedRowData[second] == ["2", nil]) + #expect(changeManager.expressionColumns(forRow: first) == [1]) + let secondChange = changeManager.changes.first { $0.rowID == second } + #expect(secondChange?.absentColumns == [1]) + #expect(changeManager.hasChanges) + } +} diff --git a/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift b/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift index f95186b3c1..7de7e4234b 100644 --- a/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift +++ b/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift @@ -2233,4 +2233,25 @@ struct SQLFormatterServiceTests { let first = try format(sql, options: options) #expect(try format(first, options: options) == first) } + + // MARK: - Account names and operator runs + + @Test("A MySQL account name and a quoted user variable keep their @ unspaced") + func mysqlAccountNames() throws { + #expect(try format("grant all on db.* to 'app'@'%'").hasSuffix("'app'@'%'")) + #expect(try format("create definer=`root`@`localhost` view v as select 1").contains("`root`@`localhost`")) + #expect(try format("create user admin@localhost").hasSuffix("admin@localhost")) + #expect(try format("select @`my var`").hasSuffix("@`my var`")) + } + + @Test( + "A PostgreSQL operator outside the vocabulary is not split", + arguments: ["<#>", "<%", "%>", "&&&", "?-|", "<<|", "@-@"] + ) + func postgresOperatorRun(symbol: String) throws { + let spaced = try formatter.format("select a \(symbol) b from t", dialect: .postgresql).formattedSQL + #expect(spaced.contains("a \(symbol) b")) + let glued = try formatter.format("select a\(symbol)b from t", dialect: .postgresql).formattedSQL + #expect(glued.contains("a \(symbol) b")) + } } diff --git a/apple/TableProTests/Core/Services/SQLTokenizerTests.swift b/apple/TableProTests/Core/Services/SQLTokenizerTests.swift index 0151f5f2ed..7cb207c89e 100644 --- a/apple/TableProTests/Core/Services/SQLTokenizerTests.swift +++ b/apple/TableProTests/Core/Services/SQLTokenizerTests.swift @@ -398,6 +398,56 @@ struct SQLTokenizerTests { #expect(code("#tmp ##global @@rowcount @p", .mssql).map(\.value) == ["#tmp", "##global", "@@rowcount", "@p"]) #expect(code("v$session SYS#1", .oracle).map(\.value) == ["v$session", "SYS#1"]) } + + // MARK: - Account names and operator runs + + @Test("An account name is one name, whatever quotes it wears") + func accountNames() { + #expect(code("grant all on db.* to 'app'@'%'", .mysql).last == SQLToken(type: .identifier, value: "'app'@'%'")) + #expect(code("definer=`root`@`localhost`", .mysql).map(\.value) == ["definer", "=", "`root`@`localhost`"]) + #expect( + code("to admin@localhost, app@10.0.0.1, 'app'@localhost", .mysql).map(\.value) + == ["to", "admin@localhost", ",", "app@10.0.0.1", ",", "'app'@localhost"] + ) + #expect(code("create user user@localhost", .mysql).last == SQLToken(type: .identifier, value: "user@localhost")) + #expect(code("select * from emp@remote.example.com", .oracle).last?.value == "emp@remote.example.com") + } + + @Test("A quoted user variable is one variable") + func quotedUserVariables() { + let tokens = code("select @`my var`, @'v'", .mysql) + #expect(tokens.map(\.value) == ["select", "@`my var`", ",", "@'v'"]) + #expect(tokens[1].type == .placeholder) + #expect(tokens[3].type == .placeholder) + } + + @Test("A variable after an operator or a comma is still a variable") + func variablesStayApart() { + #expect(code("set @a=@b, c=1", .mysql).map(\.value) == ["set", "@a", "=", "@b", ",", "c", "=", "1"]) + #expect(code("a<@b", .mssql).map(\.value) == ["a", "<", "@b"]) + } + + @Test( + "A PostgreSQL operator no list holds is one operator", + arguments: ["<#>", "<%", "%>", "&&&", "?-|", "<<|", "@-@", "<<->>", "|=|", "!=-"] + ) + func operatorRun(symbol: String) { + #expect(code("a\(symbol)b", .postgresql) == [ + SQLToken(type: .identifier, value: "a"), + SQLToken(type: .operator, value: symbol), + SQLToken(type: .identifier, value: "b"), + ]) + } + + @Test("A sign after a PostgreSQL operator stays a sign, and a comment ends the operator") + func operatorRunEnds() { + #expect( + code("a=-1 and b>=+2 and c*-d", .postgresql).map(\.value) + == ["a", "=", "-", "1", "and", "b", ">=", "+", "2", "and", "c", "*", "-", "d"] + ) + #expect(code("a <-> b -- c", .postgresql).map(\.value) == ["a", "<->", "b", "-- c"]) + #expect(code("a+/* c */b", .postgresql).map(\.value) == ["a", "+", "/* c */", "b"]) + } } // What the query fingerprint read on main, before the formatter read grammars: its hashes are stored, so the reading diff --git a/apple/TableProTests/Core/Shortcuts/RowPayloadCSVEmptyCellTests.swift b/apple/TableProTests/Core/Shortcuts/RowPayloadCSVEmptyCellTests.swift new file mode 100644 index 0000000000..fe2c47d8fe --- /dev/null +++ b/apple/TableProTests/Core/Shortcuts/RowPayloadCSVEmptyCellTests.swift @@ -0,0 +1,62 @@ +// +// RowPayloadCSVEmptyCellTests.swift +// TableProTests +// +// An unquoted empty CSV cell is no value; only a quoted "" is an empty string. +// + +import Foundation +import Testing + +@testable import TablePro + +struct RowPayloadCSVEmptyCellTests { + @Test("An unquoted empty cell leaves its column out of the row") + func emptyCellIsAbsent() throws { + let rows = try RowPayload.parseCSV("name,age,note\nBob,,\n") + + #expect(rows == [PayloadRow(values: ["name": .text("Bob")])]) + } + + @Test("A quoted empty cell is an empty string") + func quotedEmptyCellIsAnEmptyString() throws { + let rows = try RowPayload.parseCSV("name,age,note\nBob,,\"\"\n") + + #expect(rows == [PayloadRow(values: ["name": .text("Bob"), "note": .text("")])]) + } + + @Test("An empty cell is not inserted as an empty string") + func emptyCellIsNotInserted() throws { + let columns = [ + PayloadColumn(name: "name", isPrimaryKey: false, isServerOwned: false), + PayloadColumn(name: "age", isPrimaryKey: false, isServerOwned: false), + PayloadColumn(name: "note", isPrimaryKey: false, isServerOwned: false) + ] + let planned = try PayloadRowPlanner.values( + table: "people", + columns: columns, + rows: try RowPayload.parseCSV("name,age,note\nBob,,\n") + ) + + #expect(planned.count == 1) + let inserted = planned.first?.map { $0.column } + #expect(inserted == ["name"]) + } + + @Test("A record shorter or longer than the header is refused") + func fieldCountMismatchIsRefused() { + #expect(throws: IntentDataError.csvFieldCountMismatch(record: 3, found: 1, expected: 3)) { + _ = try RowPayload.parseCSV("name,age,note\nAda,36,x\nBob\n") + } + #expect(throws: IntentDataError.csvFieldCountMismatch(record: 2, found: 4, expected: 3)) { + _ = try RowPayload.parseCSV("name,age,note\nAda,36,x,extra\n") + } + } + + @Test("A blank line between records is skipped") + func blankLineIsSkipped() throws { + let rows = try RowPayload.parseCSV("name,age\nAda,36\n\nBob,41\n") + + #expect(rows.count == 2) + } +} diff --git a/apple/TableProTests/Core/Storage/ConnectionSecretScopeTests.swift b/apple/TableProTests/Core/Storage/ConnectionSecretScopeTests.swift new file mode 100644 index 0000000000..cf0bb3c235 --- /dev/null +++ b/apple/TableProTests/Core/Storage/ConnectionSecretScopeTests.swift @@ -0,0 +1,341 @@ +import Foundation +@testable import TablePro +import TableProSyncTransport +import Testing + +// One synced and one local value per key, as the keychain holds them. Nothing here removes a synced value except delete. +private final class ScopeRecordingKeychain: KeychainStoring, @unchecked Sendable { + private let lock = NSLock() + private var synced: [String: String] = [:] + private var local: [String: String] = [:] + private var unreadableKeys: Set = [] + private var failingKeys: Set = [] + + func seedSynced(_ value: String, forKey key: String) { + lock.lock() + defer { lock.unlock() } + synced[key] = value + } + + func seedLocal(_ value: String, forKey key: String) { + lock.lock() + defer { lock.unlock() } + local[key] = value + } + + func makeUnreadable(_ key: String) { + lock.lock() + defer { lock.unlock() } + unreadableKeys.insert(key) + } + + func makeReadable(_ key: String) { + lock.lock() + defer { lock.unlock() } + unreadableKeys.remove(key) + } + + func failWrites(to key: String) { + lock.lock() + defer { lock.unlock() } + failingKeys.insert(key) + } + + func syncedValue(forKey key: String) -> String? { + lock.lock() + defer { lock.unlock() } + return synced[key] + } + + func localValue(forKey key: String) -> String? { + lock.lock() + defer { lock.unlock() } + return local[key] + } + + @discardableResult + func writeString(_ value: String, forKey key: String) -> Bool { + writeString(value, forKey: key, deviceOnly: false) + } + + // Password sync is on in this model, so a write that is not device-only is a synced one. + @discardableResult + func writeString(_ value: String, forKey key: String, deviceOnly: Bool) -> Bool { + lock.lock() + defer { lock.unlock() } + guard !failingKeys.contains(key) else { return false } + if deviceOnly { + local[key] = value + } else { + synced[key] = value + local[key] = nil + } + return true + } + + func readStringResult(forKey key: String) -> KeychainStringResult { + readStringResult(forKey: key, deviceOnly: false) + } + + func readStringResult(forKey key: String, deviceOnly: Bool) -> KeychainStringResult { + lock.lock() + defer { lock.unlock() } + guard !unreadableKeys.contains(key) else { return .locked } + let value = deviceOnly ? local[key] ?? synced[key] : synced[key] ?? local[key] + return value.map { .found($0) } ?? .notFound + } + + func delete(forKey key: String) { + lock.lock() + defer { lock.unlock() } + synced[key] = nil + local[key] = nil + } +} + +@MainActor +struct ConnectionSecretScopeTests { + private let keychain = ScopeRecordingKeychain() + private let suite: EphemeralDefaults + private let storage: ConnectionStorage + + init() throws { + suite = try EphemeralDefaults() + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("secret-scope-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + storage = ConnectionStorage( + fileURL: directory.appendingPathComponent("connections.json"), + userDefaults: suite.defaults, + syncTracker: SyncChangeTracker(metadataStorage: SyncMetadataStorage(userDefaults: suite.defaults)), + keychain: keychain, + appEvents: AppEvents(), + integrity: ConnectionStoreIntegrity(keySource: StoredIntegrityKeySource(store: InMemoryKeychain())) + ) + } + + private func passwordKey(_ id: UUID) -> String { "com.TablePro.password.\(id.uuidString)" } + private func totpKey(_ id: UUID) -> String { "com.TablePro.totpsecret.\(id.uuidString)" } + private func sslKey(_ id: UUID) -> String { "com.TablePro.sslkeypassphrase.\(id.uuidString)" } + + private func makeConnection(localOnly: Bool) -> DatabaseConnection { + var connection = TestFixtures.makeConnection() + connection.localOnly = localOnly + return connection + } + + @Test("A Local only connection's secrets are written device-only") + func localOnlySecretsStayOnDevice() { + let connection = makeConnection(localOnly: true) + #expect(storage.addConnection(connection, password: "hunter2")) + storage.saveTOTPSecret("JBSWY3DP", for: connection.id) + storage.saveCloudSQLProxyServiceAccountKey("{}", for: connection.id) + let serviceAccountKey = "com.TablePro.cloudsqlproxyserviceaccountkey.\(connection.id.uuidString)" + + #expect(keychain.localValue(forKey: passwordKey(connection.id)) == "hunter2") + #expect(keychain.localValue(forKey: totpKey(connection.id)) == "JBSWY3DP") + #expect(keychain.localValue(forKey: serviceAccountKey) == "{}") + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == nil) + #expect(keychain.syncedValue(forKey: totpKey(connection.id)) == nil) + #expect(keychain.syncedValue(forKey: serviceAccountKey) == nil) + } + + @Test("A synced connection's secrets follow the password sync setting") + func syncedSecretsFollowTheSetting() { + let connection = makeConnection(localOnly: false) + #expect(storage.addConnection(connection, password: "hunter2")) + + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "hunter2") + #expect(keychain.localValue(forKey: passwordKey(connection.id)) == nil) + } + + @Test("Marking a connection Local only copies its secrets to this Mac and leaves the synced ones for other devices") + func markingLocalOnlyCopiesAndKeepsTheSyncedSecrets() { + let connection = makeConnection(localOnly: false) + #expect(storage.addConnection(connection, password: "hunter2")) + storage.saveSSLClientKeyPassphrase("passphrase", for: connection.id) + + let marked = storage.mutateConnections(ids: [connection.id]) { $0.localOnly = true } + #expect(marked) + + #expect(keychain.localValue(forKey: passwordKey(connection.id)) == "hunter2") + #expect(keychain.localValue(forKey: sslKey(connection.id)) == "passphrase") + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "hunter2") + #expect(keychain.syncedValue(forKey: sslKey(connection.id)) == "passphrase") + } + + @Test("A Local only connection reads and changes its own copy, never the one another device synced") + func localOnlyUsesItsOwnCopy() { + let connection = makeConnection(localOnly: true) + #expect(storage.saveConnections([connection])) + keychain.seedSynced("from-another-device", forKey: passwordKey(connection.id)) + + #expect(storage.loadPassword(for: connection.id) == "from-another-device") + storage.savePassword("mine", for: connection.id) + + #expect(storage.loadPassword(for: connection.id) == "mine") + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "from-another-device") + } + + @Test("Putting a connection back in sync keeps the password changed while it was Local only") + func unmarkingLocalOnlyKeepsTheNewerSecret() { + let connection = makeConnection(localOnly: false) + #expect(storage.addConnection(connection, password: "v1")) + let marked = storage.mutateConnections(ids: [connection.id]) { $0.localOnly = true } + #expect(marked) + storage.savePassword("v2", for: connection.id) + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "v1") + + let unmarked = storage.mutateConnections(ids: [connection.id]) { $0.localOnly = false } + #expect(unmarked) + + #expect(storage.loadPassword(for: connection.id) == "v2") + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "v2") + + storage.savePassword(storage.loadPassword(for: connection.id) ?? "", for: connection.id) + #expect(storage.loadPassword(for: connection.id) == "v2") + } + + @Test("Putting a connection back in sync leaves a secret only another device holds as it is") + func unmarkingLocalOnlyKeepsASyncedOnlySecret() { + let connection = makeConnection(localOnly: true) + #expect(storage.saveConnections([connection])) + keychain.seedSynced("from-another-device", forKey: passwordKey(connection.id)) + + let unmarked = storage.mutateConnections(ids: [connection.id]) { $0.localOnly = false } + #expect(unmarked) + + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "from-another-device") + } + + @Test("Marking a connection Local only copies the value it was using, not an older local one") + func markingLocalOnlyCopiesTheValueInUse() { + let connection = makeConnection(localOnly: false) + #expect(storage.saveConnections([connection])) + keychain.seedLocal("older-local", forKey: passwordKey(connection.id)) + keychain.seedSynced("in-use", forKey: passwordKey(connection.id)) + #expect(storage.loadPassword(for: connection.id) == "in-use") + + let marked = storage.mutateConnections(ids: [connection.id]) { $0.localOnly = true } + #expect(marked) + + #expect(storage.loadPassword(for: connection.id) == "in-use") + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "in-use") + } + + @Test("The launch move copies an older Local only connection's secrets and leaves the synced ones in place") + func launchMoveCopiesAndKeepsTheSyncedSecrets() { + let connection = makeConnection(localOnly: true) + #expect(storage.saveConnections([connection])) + keychain.seedSynced("hunter2", forKey: passwordKey(connection.id)) + keychain.seedSynced("JBSWY3DP", forKey: totpKey(connection.id)) + + storage.keepLocalOnlySecretsOnDeviceIfNeeded() + + #expect(keychain.localValue(forKey: passwordKey(connection.id)) == "hunter2") + #expect(keychain.localValue(forKey: totpKey(connection.id)) == "JBSWY3DP") + #expect(keychain.syncedValue(forKey: passwordKey(connection.id)) == "hunter2") + #expect(keychain.syncedValue(forKey: totpKey(connection.id)) == "JBSWY3DP") + } + + @Test("The launch move runs again when a secret could not be read") + func launchMoveRetriesAfterAnUnreadableSecret() { + let connection = makeConnection(localOnly: true) + #expect(storage.saveConnections([connection])) + keychain.seedSynced("hunter2", forKey: passwordKey(connection.id)) + keychain.makeUnreadable(passwordKey(connection.id)) + + storage.keepLocalOnlySecretsOnDeviceIfNeeded() + #expect(keychain.localValue(forKey: passwordKey(connection.id)) == nil) + + keychain.makeReadable(passwordKey(connection.id)) + storage.keepLocalOnlySecretsOnDeviceIfNeeded() + #expect(keychain.localValue(forKey: passwordKey(connection.id)) == "hunter2") + } + + @Test("A staged delete waits for the caller, so a failed record save removes nothing") + func stagedDeleteWaitsForTheRecord() { + let id = UUID() + keychain.seedSynced("JBSWY3DP", forKey: totpKey(id)) + + let staged = storage.stagingSecretEdits(for: id, deviceOnly: false) { + storage.deleteTOTPSecret(for: id) + } + #expect(keychain.syncedValue(forKey: totpKey(id)) == "JBSWY3DP") + + storage.applyStagedDeletions(staged) + #expect(keychain.syncedValue(forKey: totpKey(id)) == nil) + } + + @Test("A secret the prefill read could not read is never deleted by an empty field") + func unreadableSecretIsKept() { + let connection = makeConnection(localOnly: false) + let tokenKey = "com.TablePro.plugin.apiToken.\(connection.id.uuidString)" + keychain.seedSynced("JBSWY3DP", forKey: totpKey(connection.id)) + keychain.seedSynced("passphrase", forKey: sslKey(connection.id)) + keychain.seedSynced("token", forKey: tokenKey) + keychain.seedSynced("proxy", forKey: "com.TablePro.socksproxypassword.\(connection.id.uuidString)") + for key in [totpKey(connection.id), sslKey(connection.id), tokenKey] { + keychain.makeUnreadable(key) + } + + let unreadable = storage.recordingUnreadableSecrets { + #expect(storage.loadTOTPSecret(for: connection.id) == nil) + #expect(storage.loadSSLClientKeyPassphrase(for: connection.id) == nil) + #expect(storage.loadPluginSecureField(fieldId: "apiToken", for: connection.id) == nil) + #expect(storage.loadSOCKSProxyPassword(for: connection.id) == "proxy") + } + #expect(unreadable == [totpKey(connection.id), sslKey(connection.id), tokenKey]) + + // Readable again by the time of the save: only the prefill read counts. + for key in unreadable { + keychain.makeReadable(key) + } + let staged = storage.stagingSecretEdits(for: connection.id, deviceOnly: false, keeping: unreadable) { + storage.deleteTOTPSecret(for: connection.id) + storage.deleteSSLClientKeyPassphrase(for: connection.id) + storage.deletePluginSecureField(fieldId: "apiToken", for: connection.id) + storage.deleteSOCKSProxyPassword(for: connection.id) + } + storage.applyStagedDeletions(staged) + + #expect(keychain.syncedValue(forKey: totpKey(connection.id)) == "JBSWY3DP") + #expect(keychain.syncedValue(forKey: sslKey(connection.id)) == "passphrase") + #expect(keychain.syncedValue(forKey: tokenKey) == "token") + #expect(storage.loadSOCKSProxyPassword(for: connection.id) == nil) + } + + @Test("A staged write that the keychain refuses is reported") + func failedStagedWriteIsReported() { + let id = UUID() + keychain.failWrites(to: passwordKey(id)) + + let failed = storage.stagingSecretEdits(for: id, deviceOnly: false) { + storage.savePassword("hunter2", for: id) + storage.saveSOCKSProxyPassword("proxy", for: id) + } + let succeeded = storage.stagingSecretEdits(for: id, deviceOnly: false) { + storage.saveSOCKSProxyPassword("proxy", for: id) + } + + #expect(failed.writeFailed) + #expect(!succeeded.writeFailed) + } + + @Test("Staged writes take the scope the unsaved record will have") + func stagedWritesTakeTheGivenScope() { + let id = UUID() + let tokenSecretKey = "com.TablePro.cloudflaretokensecret.\(id.uuidString)" + + _ = storage.stagingSecretEdits(for: id, deviceOnly: true) { + storage.savePassword("hunter2", for: id) + storage.saveCloudflareTokenSecret("secret", for: id) + } + + #expect(keychain.localValue(forKey: passwordKey(id)) == "hunter2") + #expect(keychain.localValue(forKey: tokenSecretKey) == "secret") + #expect(keychain.syncedValue(forKey: passwordKey(id)) == nil) + #expect(keychain.syncedValue(forKey: tokenSecretKey) == nil) + } +} diff --git a/apple/TableProTests/Core/Storage/ConnectionStoreTagRemovalTests.swift b/apple/TableProTests/Core/Storage/ConnectionStoreTagRemovalTests.swift new file mode 100644 index 0000000000..0a2672d69d --- /dev/null +++ b/apple/TableProTests/Core/Storage/ConnectionStoreTagRemovalTests.swift @@ -0,0 +1,122 @@ +import CryptoKit +import Foundation +@testable import TablePro +import TableProSyncTransport +import Testing + +// A build from before the record existed: it holds the key and remembers nothing. +private struct KeyOnlySource: IntegrityKeySource { + let source: StoredIntegrityKeySource + + func key() -> SymmetricKey? { source.key() } +} + +@MainActor +struct ConnectionStoreTagRemovalTests { + private let keychain = InMemoryKeychain() + + private var integrity: ConnectionStoreIntegrity { + ConnectionStoreIntegrity(keySource: StoredIntegrityKeySource(store: keychain)) + } + + private func makeFileURL() throws -> URL { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("tablepro-tests") + .appendingPathComponent("tag-removal-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + return directory.appendingPathComponent("connections.json") + } + + @Test("A store this device tagged reads as modified once its tag is deleted") + func deletedTagIsNotAnUnstampedStore() throws { + let fileURL = try makeFileURL() + let data = Data(#"[{"name":"Prod"}]"#.utf8) + try #require(integrity.stamp(data, fileURL: fileURL)) + + try FileManager.default.removeItem(at: ConnectionStoreIntegrity.tagURL(for: fileURL)) + + #expect(integrity.verify(data, fileURL: fileURL) == .modified) + } + + @Test("An emptied tag on a tagged store reads as modified") + func emptiedTagIsModified() throws { + let fileURL = try makeFileURL() + let data = Data("[]".utf8) + try #require(integrity.stamp(data, fileURL: fileURL)) + + try Data().write(to: ConnectionStoreIntegrity.tagURL(for: fileURL)) + + #expect(integrity.verify(data, fileURL: fileURL) == .modified) + } + + @Test("A store that was never tagged still reads as unstamped") + func neverTaggedStoreIsUnstamped() throws { + let fileURL = try makeFileURL() + + #expect(integrity.verify(Data("[]".utf8), fileURL: fileURL) == .unstamped) + } + + @Test("A tag an older build wrote is recorded the first time it verifies") + func olderBuildTagIsRecordedOnVerify() throws { + let fileURL = try makeFileURL() + let data = Data(#"[{"name":"Prod"}]"#.utf8) + let older = ConnectionStoreIntegrity(keySource: KeyOnlySource(source: StoredIntegrityKeySource(store: keychain))) + try #require(older.stamp(data, fileURL: fileURL)) + + #expect(integrity.verify(data, fileURL: fileURL) == .trusted) + try FileManager.default.removeItem(at: ConnectionStoreIntegrity.tagURL(for: fileURL)) + + #expect(integrity.verify(data, fileURL: fileURL) == .modified) + } + + @Test("connections.json rewritten with its tag deleted is not trusted on the next load") + func rewrittenStoreWithoutTagIsUntrusted() throws { + let fileURL = try makeFileURL() + let suite = try EphemeralDefaults() + let tracker = SyncChangeTracker(metadataStorage: SyncMetadataStorage(userDefaults: suite.defaults)) + let first = ConnectionStorage( + fileURL: fileURL, + userDefaults: suite.defaults, + syncTracker: tracker, + keychain: keychain, + integrity: integrity + ) + var connection = TestFixtures.makeConnection() + connection.localOnly = true + #expect(first.saveConnections([connection])) + #expect(first.storeIsTrusted) + + var planted = connection + planted.name = "Planted" + try JSONEncoder().encode([StoredConnection(from: planted)]).write(to: fileURL) + try FileManager.default.removeItem(at: ConnectionStoreIntegrity.tagURL(for: fileURL)) + + let reloaded = ConnectionStorage( + fileURL: fileURL, + userDefaults: suite.defaults, + syncTracker: tracker, + keychain: keychain, + integrity: integrity + ) + #expect(reloaded.loadConnections().first?.name == "Planted") + #expect(!reloaded.storeIsTrusted) + } + + @Test("connections.json from before the tag existed is adopted and trusted") + func untaggedUpgradeIsAdopted() throws { + let fileURL = try makeFileURL() + let suite = try EphemeralDefaults() + try JSONEncoder().encode([StoredConnection(from: TestFixtures.makeConnection())]).write(to: fileURL) + + let storage = ConnectionStorage( + fileURL: fileURL, + userDefaults: suite.defaults, + syncTracker: SyncChangeTracker(metadataStorage: SyncMetadataStorage(userDefaults: suite.defaults)), + keychain: keychain, + integrity: integrity + ) + + #expect(storage.loadConnections().count == 1) + #expect(storage.storeIsTrusted) + } +} diff --git a/apple/TableProTests/Core/Storage/KeychainItemScopeTests.swift b/apple/TableProTests/Core/Storage/KeychainItemScopeTests.swift new file mode 100644 index 0000000000..71d6cf1c4b --- /dev/null +++ b/apple/TableProTests/Core/Storage/KeychainItemScopeTests.swift @@ -0,0 +1,201 @@ +import Foundation +import Security +@testable import TablePro +import Testing + +// Models the one rule the helper depends on: synchronizable is part of an item's identity. +private final class ScopedItemStore: KeychainItemStore, @unchecked Sendable { + struct ItemID: Hashable { + let account: String + let synchronizable: Bool + } + + private let lock = NSLock() + private var items: [ItemID: Data] = [:] + + func seed(_ value: String, account: String, synchronizable: Bool) { + lock.lock() + defer { lock.unlock() } + items[ItemID(account: account, synchronizable: synchronizable)] = Data(value.utf8) + } + + func value(account: String, synchronizable: Bool) -> String? { + lock.lock() + defer { lock.unlock() } + return items[ItemID(account: account, synchronizable: synchronizable)].map { String(decoding: $0, as: UTF8.self) } + } + + private func matches(_ query: [String: Any]) -> [ItemID] { + let account = query[kSecAttrAccount as String] as? String ?? "" + let scopes: [Bool] + if let scope = query[kSecAttrSynchronizable as String] { + scopes = (scope as? Bool).map { [$0] } ?? [false, true] + } else { + scopes = [false] + } + return scopes + .map { ItemID(account: account, synchronizable: $0) } + .filter { items[$0] != nil } + } + + func add(_ attributes: [String: Any]) -> OSStatus { + lock.lock() + defer { lock.unlock() } + let id = ItemID( + account: attributes[kSecAttrAccount as String] as? String ?? "", + synchronizable: attributes[kSecAttrSynchronizable as String] as? Bool ?? false + ) + guard items[id] == nil else { return errSecDuplicateItem } + items[id] = attributes[kSecValueData as String] as? Data + return errSecSuccess + } + + func update(_ query: [String: Any], attributes: [String: Any]) -> OSStatus { + lock.lock() + defer { lock.unlock() } + let found = matches(query) + guard !found.isEmpty else { return errSecItemNotFound } + // Two items for one key cannot both take one synchronizable value. + if attributes[kSecAttrSynchronizable as String] != nil, found.count > 1 { + return errSecDuplicateItem + } + for id in found { + items[id] = attributes[kSecValueData as String] as? Data + } + return errSecSuccess + } + + func copyData(_ query: [String: Any]) -> (status: OSStatus, data: Data?) { + lock.lock() + defer { lock.unlock() } + guard let id = matches(query).first else { return (errSecItemNotFound, nil) } + return (errSecSuccess, items[id]) + } + + func delete(_ query: [String: Any]) -> OSStatus { + lock.lock() + defer { lock.unlock() } + let found = matches(query) + guard !found.isEmpty else { return errSecItemNotFound } + for id in found { + items[id] = nil + } + return errSecSuccess + } +} + +struct KeychainItemScopeTests { + private let store = ScopedItemStore() + private let key = "com.TablePro.password.scope" + + private func helper(passwordSync: Bool, followsPasswordSync: Bool = true) -> KeychainHelper { + KeychainHelper( + service: "com.TablePro.tests.scope", + followsPasswordSync: followsPasswordSync, + passwordSyncSetting: { passwordSync }, + items: store + ) + } + + @Test("With password sync on, a write with a synced and a local item saves the synced one and drops the local one") + func writeWithBothItemsPresent() { + store.seed("old-local", account: key, synchronizable: false) + store.seed("old-synced", account: key, synchronizable: true) + let keychain = helper(passwordSync: true) + + #expect(keychain.writeString("new", forKey: key)) + + #expect(store.value(account: key, synchronizable: true) == "new") + #expect(store.value(account: key, synchronizable: false) == nil) + #expect(keychain.readString(forKey: key) == "new") + } + + @Test("With password sync off, a write with both items saves the local one and leaves the synced one alone") + func localWriteKeepsTheSyncedItem() { + store.seed("old-local", account: key, synchronizable: false) + store.seed("from-another-device", account: key, synchronizable: true) + let keychain = helper(passwordSync: false) + + #expect(keychain.writeString("new", forKey: key)) + + #expect(store.value(account: key, synchronizable: false) == "new") + #expect(store.value(account: key, synchronizable: true) == "from-another-device") + #expect(keychain.readString(forKey: key) == "new") + } + + @Test("With password sync off, saving a secret another device synced never deletes that device's item") + func localWriteNeverDeletesASyncedItem() { + store.seed("from-another-device", account: key, synchronizable: true) + + #expect(helper(passwordSync: false).writeString("new", forKey: key)) + + #expect(store.value(account: key, synchronizable: true) == "from-another-device") + #expect(store.value(account: key, synchronizable: false) == "new") + } + + @Test("With password sync on, a write moves an item saved before it was on into iCloud Keychain") + func writeMovesLocalItemToSynced() { + store.seed("old-local", account: key, synchronizable: false) + + #expect(helper(passwordSync: true).writeString("new", forKey: key)) + + #expect(store.value(account: key, synchronizable: true) == "new") + #expect(store.value(account: key, synchronizable: false) == nil) + } + + @Test("A device-only write stays out of iCloud Keychain and leaves the synced copy for other devices") + func deviceOnlyWriteKeepsSyncedCopy() { + store.seed("old-synced", account: key, synchronizable: true) + let keychain = helper(passwordSync: true) + + #expect(keychain.writeString("new", forKey: key, deviceOnly: true)) + + #expect(store.value(account: key, synchronizable: false) == "new") + #expect(store.value(account: key, synchronizable: true) == "old-synced") + } + + @Test("A device-only read returns this Mac's item over a synced one, whatever the setting") + func deviceOnlyReadPrefersTheLocalItem() { + store.seed("local", account: key, synchronizable: false) + store.seed("synced", account: key, synchronizable: true) + let keychain = helper(passwordSync: true) + + #expect(keychain.readStringResult(forKey: key, deviceOnly: true) == .found("local")) + #expect(keychain.readStringResult(forKey: key) == .found("synced")) + } + + @Test("A device-only read falls back to the synced item when this Mac has no copy yet") + func deviceOnlyReadFallsBackToTheSyncedItem() { + store.seed("synced", account: key, synchronizable: true) + + #expect(helper(passwordSync: false).readStringResult(forKey: key, deviceOnly: true) == .found("synced")) + } + + @Test("A keychain that never follows password sync writes local items only") + func deviceBoundKeychainNeverSyncs() { + #expect(helper(passwordSync: true, followsPasswordSync: false).writeString("new", forKey: key)) + + #expect(store.value(account: key, synchronizable: false) == "new") + #expect(store.value(account: key, synchronizable: true) == nil) + } + + @Test("A read returns the item the setting selects when both exist") + func readPrefersTheSelectedItem() { + store.seed("local", account: key, synchronizable: false) + store.seed("synced", account: key, synchronizable: true) + + #expect(helper(passwordSync: true).readString(forKey: key) == "synced") + #expect(helper(passwordSync: false).readString(forKey: key) == "local") + } + + @Test("Delete removes both items") + func deleteRemovesBothItems() { + store.seed("local", account: key, synchronizable: false) + store.seed("synced", account: key, synchronizable: true) + let keychain = helper(passwordSync: false) + + keychain.delete(forKey: key) + + #expect(keychain.readStringResult(forKey: key) == .notFound) + } +} diff --git a/apple/TableProTests/Core/Sync/SyncCategoryRefetchTests.swift b/apple/TableProTests/Core/Sync/SyncCategoryRefetchTests.swift index 52a9dc34cf..f3d157d2eb 100644 --- a/apple/TableProTests/Core/Sync/SyncCategoryRefetchTests.swift +++ b/apple/TableProTests/Core/Sync/SyncCategoryRefetchTests.swift @@ -56,6 +56,91 @@ struct SyncCategoryRefetchTests { #expect(pulls == [.full, .full, .delta]) } + @Test("A saved query deleted elsewhere while Saved Queries was off is removed when the category turns back on") + func deletionMadeWhileACategoryWasOffIsApplied() async throws { + let deleted = SQLFavorite(name: "Revenue", query: "SELECT 1") + let kept = SQLFavorite(name: "Churn", query: "SELECT 2") + let zoneID = SyncTestEnvironment.zoneID + let deletedRecord = SyncRecordMapper.toCKRecord(sqlFavorite: deleted, in: zoneID) + let server = ZoneServer( + zoneID: zoneID, + records: [deletedRecord, SyncRecordMapper.toCKRecord(sqlFavorite: kept, in: zoneID)], + token: try Self.makeChangeToken() + ) + _ = environment.metadata.adoptReadableRecordTypes( + SyncCoordinator.readableContent(for: saveSync(savedQueries: true)) + ) + let coordinator = makeCoordinator(server: server) + + #expect(await coordinator.runSyncCycle() == nil) + #expect(await environment.favorites.fetchFavorite(id: deleted.id) != nil) + + saveSync(savedQueries: false) + await server.delete(deletedRecord.recordID) + #expect(await coordinator.runSyncCycle() == nil) + #expect(await environment.favorites.fetchFavorite(id: deleted.id) != nil) + + saveSync(savedQueries: true) + #expect(await coordinator.runSyncCycle() == nil) + + #expect(await environment.favorites.fetchFavorite(id: deleted.id) == nil) + #expect(await environment.favorites.fetchFavorite(id: kept.id)?.name == "Churn") + #expect(environment.metadata.deferredDeletions.isEmpty) + } + + @Test("A saved query edited here after it was deleted elsewhere is kept when the category turns back on") + func editedRecordSurvivesALateDeletion() async throws { + let favorite = SQLFavorite(name: "Revenue", query: "SELECT 1") + let record = SyncRecordMapper.toCKRecord(sqlFavorite: favorite, in: SyncTestEnvironment.zoneID) + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID)) + let on = Self.syncSettings(savedQueries: true) + let deletion = PullResult(changedRecords: [], deletedRecordIDs: [record.recordID], newToken: nil) + + #expect(await coordinator.applyPullResult(pull([record]), settings: on)) + #expect(await coordinator.applyPullResult(deletion, settings: Self.syncSettings(savedQueries: false))) + environment.tracker.markDirty(.favorite, id: favorite.id.uuidString) + #expect(await coordinator.applyPullResult(pull([]), settings: on)) + + #expect(await environment.favorites.fetchFavorite(id: favorite.id) != nil) + #expect(environment.metadata.deferredDeletions.isEmpty) + } + + @Test("A saved query saved again on the server after its deletion is kept") + func recordSavedAgainSurvivesALateDeletion() async throws { + let favorite = SQLFavorite(name: "Revenue", query: "SELECT 1") + let record = SyncRecordMapper.toCKRecord(sqlFavorite: favorite, in: SyncTestEnvironment.zoneID) + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID)) + let on = Self.syncSettings(savedQueries: true) + let deletion = PullResult(changedRecords: [], deletedRecordIDs: [record.recordID], newToken: nil) + + #expect(await coordinator.applyPullResult(pull([record]), settings: on)) + #expect(await coordinator.applyPullResult(deletion, settings: Self.syncSettings(savedQueries: false))) + #expect(await coordinator.applyPullResult(pull([record]), settings: on)) + #expect(await coordinator.applyPullResult(pull([]), settings: on)) + + #expect(await environment.favorites.fetchFavorite(id: favorite.id) != nil) + } + + @Test("A credential profile deleted elsewhere while its category was off is never deleted late") + func profileIsNeverDeletedLate() async throws { + let recordID = SyncRecordMapper.recordID(type: .credentialProfile, id: UUID().uuidString, in: SyncTestEnvironment.zoneID) + let coordinator = environment.makeCoordinator(transport: ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID)) + var off = SyncSettings.default + off.syncCredentialProfiles = false + let deletion = PullResult(changedRecords: [], deletedRecordIDs: [recordID], newToken: nil) + + #expect(await coordinator.applyPullResult(deletion, settings: off)) + #expect(environment.metadata.deferredDeletions == [recordID.recordName]) + + let due = coordinator.deferredDeletions(dueIn: pull([]), settings: .default, zoneID: SyncTestEnvironment.zoneID) + #expect(due.due.isEmpty) + #expect(due.settled == [recordID.recordName]) + } + + private func pull(_ records: [CKRecord]) -> PullResult { + PullResult(changedRecords: records, deletedRecordIDs: [], newToken: nil) + } + @Test("Saved Queries off keeps the change token at launch and at the next launch") func categoryOffKeepsTheTokenAcrossRelaunch() async throws { #expect(environment.connections.saveConnections([DatabaseConnection(name: "Local")])) @@ -151,7 +236,8 @@ private actor ZoneServer: SyncTransport { } let currentZoneID: CKRecordZone.ID - private let zoneRecords: [CKRecord] + private var zoneRecords: [CKRecord] + private var undeliveredDeletions: [CKRecord.ID] = [] private let token: CKServerChangeToken private var firstDelta: [CKRecord] private let duringPull: @Sendable () -> Void @@ -171,6 +257,11 @@ private actor ZoneServer: SyncTransport { self.duringPull = duringPull } + func delete(_ recordID: CKRecord.ID) { + zoneRecords.removeAll { $0.recordID == recordID } + undeliveredDeletions.append(recordID) + } + func accountStatus() async throws -> CKAccountStatus { .available } @@ -200,10 +291,14 @@ private actor ZoneServer: SyncTransport { pulls.append(token == nil ? .full : .delta) duringPull() guard token != nil else { + undeliveredDeletions = [] return PullResult(changedRecords: zoneRecords, deletedRecordIDs: [], newToken: self.token) } - defer { firstDelta = [] } - return PullResult(changedRecords: firstDelta, deletedRecordIDs: [], newToken: self.token) + defer { + firstDelta = [] + undeliveredDeletions = [] + } + return PullResult(changedRecords: firstDelta, deletedRecordIDs: undeliveredDeletions, newToken: self.token) } } diff --git a/apple/TableProTests/Core/Utilities/SQL/EtcdCommandParserTests.swift b/apple/TableProTests/Core/Utilities/SQL/EtcdCommandParserTests.swift index 4a9d37721f..4b4a7a6637 100644 --- a/apple/TableProTests/Core/Utilities/SQL/EtcdCommandParserTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/EtcdCommandParserTests.swift @@ -265,6 +265,64 @@ struct EtcdCommandParserPutTests { } } +struct EtcdCommandParserGuardedWriteTests { + @Test("create parses the key, the value and the lease") + func create() throws { + let op = try EtcdCommandParser.parse("create /app/b value --lease=7b") + guard case .create(let key, let value, let leaseId) = op else { + Issue.record("Expected .create, got \(op)") + return + } + #expect(key == Data("/app/b".utf8)) + #expect(value == Data("value".utf8)) + #expect(leaseId == 0x7b) + } + + @Test("mv parses both keys, the value, the lease and the revision the source was read at") + func move() throws { + let line = EtcdCommandText.command( + "mv", + arguments: [Data("/app/a".utf8), Data("/app/b b".utf8), Data()], + flags: ["--mod-rev=17", "--lease=123"] + ) + guard case .move(let source, let target, let value, let leaseId, let modRevision) = try EtcdCommandParser.parse(line) else { + Issue.record("Expected .move for \(line)") + return + } + #expect(source == Data("/app/a".utf8)) + #expect(target == Data("/app/b b".utf8)) + #expect(value.isEmpty) + #expect(leaseId == 123) + #expect(modRevision == 17) + } + + @Test("mv keeps a key that starts with two dashes as a key") + func moveDashedKey() throws { + let line = EtcdCommandText.command( + "mv", arguments: [Data("--a".utf8), Data("b".utf8), Data("v".utf8)], flags: ["--mod-rev=2"] + ) + guard case .move(let source, let target, _, _, let modRevision) = try EtcdCommandParser.parse(line) else { + Issue.record("Expected .move for \(line)") + return + } + #expect(source == Data("--a".utf8)) + #expect(target == Data("b".utf8)) + #expect(modRevision == 2) + } + + @Test("mv without a revision carries none, and a malformed one is refused") + func moveRevision() throws { + guard case .move(_, _, _, _, let modRevision) = try EtcdCommandParser.parse("mv a b v") else { + Issue.record("Expected .move") + return + } + #expect(modRevision == nil) + #expect(throws: EtcdParseError.self) { try EtcdCommandParser.parse("mv a b v --mod-rev=x") } + #expect(throws: EtcdParseError.self) { try EtcdCommandParser.parse("mv a b") } + #expect(throws: EtcdParseError.self) { try EtcdCommandParser.parse("create a") } + } +} + struct EtcdCommandParserDelTests { @Test("Basic del parses key") func basicDel() throws { @@ -1104,11 +1162,11 @@ private let etcdReadLines = [ ] private let etcdWriteLines = [ "put /a v", "lease grant 60", "lease keep-alive 1", "auth enable", "user add bob pw", "role add readers", - "user grant-role bob readers", "user revoke-role bob readers", + "user grant-role bob readers", "user revoke-role bob readers", "create /a v", ] private let etcdDeleteLines = [ "del /a", #"del "" --prefix"#, "del /app/ --prefix", "compaction 5", "lease revoke 1", "auth disable", - "user delete bob", "role delete readers", + "user delete bob", "role delete readers", "mv /a /b v --mod-rev=3", "mv /a /b v", ] struct EtcdCommandParserStatementPolicyTests { diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift index ae76f45ff0..01da6e95ba 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierHardeningTests.swift @@ -644,6 +644,19 @@ struct QueryClassifierNonSqlTests { /// `OUTSIDEBLOCK` is the Server Dashboard's prefix for a command held out of the user's MULTI /// block. Read as the bare word, `OUTSIDEBLOCK FLUSHALL` passed as an ordinary write. + @Test("Redis MUSTAPPLY prefix is classified by the command it wraps") + func redisMustApplyPrefixIsClassifiedByItsCommand() { + #expect(QueryClassifier.classifyTier("MUSTAPPLY FLUSHALL", databaseType: .redis) == .destructive) + #expect(QueryClassifier.classifyTier("MUSTAPPLY SHUTDOWN", databaseType: .redis) == .destructive) + #expect(QueryClassifier.classifyTier("MUSTAPPLY CONFIG SET dir /tmp", databaseType: .redis) == .destructive) + #expect(QueryClassifier.classifyTier("DB 0 MUSTAPPLY FLUSHDB", databaseType: .redis) == .destructive) + #expect(QueryClassifier.classifyTier("MUSTAPPLY DB 0 FLUSHDB", databaseType: .redis) == .destructive) + #expect(QueryClassifier.reachesFilesystemOrExecutesCode("MUSTAPPLY EVAL \"return 1\" 0", databaseType: .redis)) + #expect(QueryClassifier.classifyTier("MUSTAPPLY RENAMENX a b", databaseType: .redis) == .write) + #expect(QueryClassifier.classifyTier("MUSTAPPLY SET k v NX", databaseType: .redis) == .write) + #expect(QueryClassifier.isWriteQuery("MUSTAPPLY", databaseType: .redis)) + } + @Test("Redis OUTSIDEBLOCK prefix is classified by the command it wraps") func redisOutsideBlockPrefix() { #expect(QueryClassifier.classifyTier("OUTSIDEBLOCK FLUSHALL", databaseType: .redis) == .destructive) diff --git a/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift b/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift index 3ffd478c3c..bed11927c4 100644 --- a/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift +++ b/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift @@ -6,6 +6,7 @@ import Foundation @testable import TablePro import TableProPluginKit +import TableProSQLGrammar import Testing struct SQLParameterExtractorTests { @@ -234,4 +235,61 @@ struct SQLParameterExtractorTests { #expect(result.sql == "$$ :id $$ WHERE id = ?") #expect(result.values.count == 1) } + + // MARK: - Engine grammar + + @Test( + "A colon written against a name, a literal, a subscript or a call is the engine's own syntax", + arguments: [ + "SELECT src:customer.name FROM raw", + "SELECT arr[lo:hi] FROM t", + "SELECT {id:UInt32}", + "SELECT \"src\":customer FROM raw", + "SELECT f(x):y FROM t", + "SELECT '{}'::jsonb, a::int", + ] + ) + func colonAfterOperandIsNotAParameter(sql: String) { + #expect(SQLParameterExtractor.extractParameters(from: sql, grammar: .ansi) == []) + } + + @Test + func parameterBesidePathAndCast() { + let sql = "SELECT src:customer, :id::int FROM raw WHERE a = :id AND b IN (:b, :c)" + #expect(SQLParameterExtractor.extractParameters(from: sql, grammar: .ansi) == ["id", "b", "c"]) + let result = SQLParameterExtractor.convertToNativeStyle( + sql: sql, + parameters: [QueryParameter(name: "id", value: "1")], + style: .dollar, + grammar: .ansi + ) + #expect(result.sql == "SELECT src:customer, $1::int FROM raw WHERE a = $2 AND b IN ($3, $4)") + } + + @Test + func backslashEndsALiteralOnlyWhereTheEngineEscapesWithIt() { + let sql = "SELECT 'C:\\' AS p, :id" + #expect(SQLParameterExtractor.extractParameters(from: sql, grammar: .ansi) == ["id"]) + #expect(SQLParameterExtractor.extractParameters(from: sql, grammar: .backslashEscapesInSingleQuotes) == []) + } + + @Test + func bracketNameHidesAColonOnlyWhereBracketsQuote() { + #expect(SQLParameterExtractor.extractParameters(from: "SELECT [a :b] FROM t", grammar: .bracketQuotedIdentifiers) == []) + #expect(SQLParameterExtractor.extractParameters(from: "SELECT x[ :b] FROM t", grammar: .ansi) == ["b"]) + } + + @Test + func hashCommentHidesAColonOnlyWhereTheEngineReadsOne() { + let sql = "SELECT 1 # :a\n, :b" + #expect(SQLParameterExtractor.extractParameters(from: sql, grammar: .hashLineComments) == ["b"]) + #expect(SQLParameterExtractor.extractParameters(from: sql, grammar: .ansi) == ["a", "b"]) + } + + @Test + func nestedCommentHidesAColon() { + let sql = "/* a /* b */ :x */ SELECT :y" + #expect(SQLParameterExtractor.extractParameters(from: sql, grammar: .nestedBlockComments) == ["y"]) + #expect(SQLParameterExtractor.extractParameters(from: sql, grammar: .ansi) == ["x", "y"]) + } } diff --git a/apple/TableProTests/Helpers/HeldExecutionGate.swift b/apple/TableProTests/Helpers/HeldExecutionGate.swift new file mode 100644 index 0000000000..bfafb00a60 --- /dev/null +++ b/apple/TableProTests/Helpers/HeldExecutionGate.swift @@ -0,0 +1,29 @@ +import Foundation +@testable import TablePro + +// Holds every request until the test answers, as an approval prompt that is not window-modal does. +@MainActor +internal final class HeldExecutionGate: ExecutionGate { + private(set) var askedCount = 0 + private var waiting: [(request: OperationRequest, continuation: CheckedContinuation)] = [] + + nonisolated internal func authorize(_ request: OperationRequest) async -> OperationDecision { + await hold(request) + } + + private func hold(_ request: OperationRequest) async -> OperationDecision { + askedCount += 1 + return await withCheckedContinuation { waiting.append((request, $0)) } + } + + internal func answer(allowing: Bool) async { + let held = waiting + waiting = [] + for entry in held { + let decision: OperationDecision = allowing + ? await AlwaysAllowGate().authorize(entry.request) + : .denied(reason: "Refused by the test") + entry.continuation.resume(returning: decision) + } + } +} diff --git a/apple/TableProTests/ViewModels/AIChatStoppedToolRoundTests.swift b/apple/TableProTests/ViewModels/AIChatStoppedToolRoundTests.swift new file mode 100644 index 0000000000..7c566967fb --- /dev/null +++ b/apple/TableProTests/ViewModels/AIChatStoppedToolRoundTests.swift @@ -0,0 +1,290 @@ +import Foundation +import TableProMCP +import TableProPluginKit +import TableProTestSupport +@testable import TablePro +import Testing + +struct ChatToolCallRepairTests { + private func call(_ id: String) -> ChatContentBlockWire { + .toolUse(ToolUseBlock(id: id, name: "execute_query", input: .object([:]))) + } + + private func result(_ id: String, _ content: String = "ok") -> ChatContentBlockWire { + .toolResult(ToolResultBlock(toolUseId: id, content: content)) + } + + private func resultIds(_ turn: ChatTurnWire) -> [String] { + turn.blocks.compactMap { block -> String? in + guard case .toolResult(let result) = block.kind else { return nil } + return result.toolUseId + } + } + + @Test("A transcript whose calls are all answered is sent as it is") + func answeredTranscriptIsUntouched() { + let turns = [ + ChatTurnWire(role: .user, blocks: [.text("count users")]), + ChatTurnWire(role: .assistant, blocks: [call("a")]), + ChatTurnWire(role: .user, blocks: [result("a")]), + ChatTurnWire(role: .assistant, blocks: [.text("42")]) + ] + + #expect(ChatToolCallRepair.answeringEveryToolCall(turns) == turns) + } + + @Test("An unanswered call followed by a new message gets a result turn between them") + func unansweredCallBeforeNewMessage() { + let turns = [ + ChatTurnWire(role: .user, blocks: [.text("count users")]), + ChatTurnWire(role: .assistant, blocks: [call("a"), call("b")]), + ChatTurnWire(role: .user, blocks: [.text("never mind")]) + ] + + let repaired = ChatToolCallRepair.answeringEveryToolCall(turns) + + #expect(repaired.count == 4) + #expect(repaired[2].role == .user) + #expect(resultIds(repaired[2]) == ["a", "b"]) + #expect(repaired[3] == turns[2]) + } + + @Test("An unanswered call at the end of the transcript gets a trailing result turn") + func unansweredCallAtTheEnd() { + let turns = [ + ChatTurnWire(role: .user, blocks: [.text("count users")]), + ChatTurnWire(role: .assistant, blocks: [call("a")]) + ] + + let repaired = ChatToolCallRepair.answeringEveryToolCall(turns) + + #expect(repaired.count == 3) + #expect(repaired[2].role == .user) + #expect(resultIds(repaired[2]) == ["a"]) + } + + @Test("A partly answered round gets only the missing result, ahead of the real ones") + func partlyAnsweredRound() { + let turns = [ + ChatTurnWire(role: .assistant, blocks: [call("a"), call("b")]), + ChatTurnWire(role: .user, blocks: [result("b")]) + ] + + let repaired = ChatToolCallRepair.answeringEveryToolCall(turns) + + #expect(repaired.count == 2) + #expect(resultIds(repaired[1]) == ["a", "b"]) + } + + @Test("A result recorded beside its call in the same turn counts as an answer") + func resultInTheSameTurn() { + let turns = [ + ChatTurnWire(role: .assistant, blocks: [call("a"), result("a")]), + ChatTurnWire(role: .user, blocks: [.text("thanks")]) + ] + + #expect(ChatToolCallRepair.answeringEveryToolCall(turns) == turns) + } + + @Test("The synthetic result is marked as an error") + func syntheticResultIsAnError() { + let repaired = ChatToolCallRepair.answeringEveryToolCall([ + ChatTurnWire(role: .assistant, blocks: [call("a")]) + ]) + + guard case .toolResult(let block) = repaired.last?.blocks.first?.kind else { + Issue.record("expected a tool result") + return + } + #expect(block.isError) + #expect(block.content == ChatToolCallRepair.interruptedContent) + } +} + +@MainActor +struct AIChatStoppedToolRoundTests { + private final class ScriptedTransport: ChatTransport, @unchecked Sendable { + private let lock = NSLock() + private let rounds: [[ChatStreamEvent]] + private var requests: [[ChatTurnWire]] = [] + + init(rounds: [[ChatStreamEvent]]) { + self.rounds = rounds + } + + var requestedTurns: [[ChatTurnWire]] { + lock.withLock { requests } + } + + func streamChat( + turns: [ChatTurnWire], + options: ChatTransportOptions + ) -> AsyncThrowingStream { + let index = lock.withLock { () -> Int in + requests.append(turns) + return requests.count - 1 + } + let events = index < rounds.count ? rounds[index] : [.textDelta("done")] + return AsyncThrowingStream { continuation in + for event in events { + continuation.yield(event) + } + continuation.finish() + } + } + + func fetchAvailableModels() async throws -> [AIModelInfo] { [] } + + func testConnection() async throws -> Bool { true } + } + + private struct GatedTool: ChatTool { + let name: String + let mode: ChatToolMode + let gate: AsyncGate + let description = "" + let inputSchema: JsonValue = .object(["type": .string("object")]) + + func execute(input: JsonValue, context: ChatToolContext) async throws -> ChatToolResult { + await gate.wait() + return ChatToolResult(content: "3 rows changed", isError: false) + } + } + + private static func toolRound(id: String, name: String) -> [ChatStreamEvent] { + [ + .toolUseStart(id: id, name: name, providerMetadata: nil), + .toolUseDelta(id: id, inputJSONDelta: "{}"), + .toolUseEnd(id: id) + ] + } + + private static let settings = AISettings( + enabled: true, + includeSchema: false, + includeCurrentQuery: false, + includeQueryResults: false, + maxToolRoundtrips: AISettings.defaultMaxToolRoundtrips, + maxToolRoundtripsEnabled: false, + chatMode: .agent + ) + + private func start( + _ viewModel: AIChatViewModel, + transport: ScriptedTransport, + tool: GatedTool, + chatMessages: [ChatTurnWire] = [] + ) -> ChatTurn { + let registry = ChatToolRegistry() + registry.register(tool) + let assistant = ChatTurn(role: .assistant, blocks: [], modelId: "test-model", providerId: nil) + viewModel.messages.append(assistant) + viewModel.streamingState = .streaming(assistantID: assistant.id) + viewModel.runStream( + chatMessages: chatMessages, + promptContext: nil, + resolved: AIProviderFactory.ResolvedProvider( + provider: transport, + model: "test-model", + config: AIProviderConfig(name: "Test", type: .claude) + ), + assistantID: assistant.id, + settings: Self.settings, + registry: registry + ) + return assistant + } + + private func turnAfter(_ turn: ChatTurn, in viewModel: AIChatViewModel) throws -> ChatTurn { + let index = try #require(viewModel.messages.firstIndex { $0.id == turn.id }) + try #require(index + 1 < viewModel.messages.count) + return viewModel.messages[index + 1] + } + + private func toolResults(in turn: ChatTurn) -> [ToolResultBlock] { + turn.blocks.compactMap { block -> ToolResultBlock? in + guard case .toolResult(let result) = block.kind else { return nil } + return result + } + } + + private func hasToolUse(_ turn: ChatTurn) -> Bool { + turn.blocks.contains { block in + if case .toolUse = block.kind { return true } + return false + } + } + + @Test("Stop while a tool call waits for approval leaves a result for that call") + func stopWhileAwaitingApproval() async throws { + let transport = ScriptedTransport(rounds: [Self.toolRound(id: "call_1", name: "writer")]) + let viewModel = AIChatViewModel() + let assistant = start( + viewModel, + transport: transport, + tool: GatedTool(name: "writer", mode: .write, gate: AsyncGate()) + ) + let task = viewModel.streamingTask + try await waitUntil { self.hasToolUse(assistant) } + + viewModel.cancelStream() + await task?.value + + let following = try turnAfter(assistant, in: viewModel) + let results = toolResults(in: following) + #expect(results.map(\.toolUseId) == ["call_1"]) + #expect(results.first?.isError == true) + #expect(following.role == .user) + } + + @Test("Stop while a tool runs keeps the result of what ran") + func stopWhileExecuting() async throws { + let gate = AsyncGate() + let transport = ScriptedTransport(rounds: [Self.toolRound(id: "call_1", name: "reader")]) + let viewModel = AIChatViewModel() + let assistant = start( + viewModel, + transport: transport, + tool: GatedTool(name: "reader", mode: .readOnly, gate: gate) + ) + let task = viewModel.streamingTask + await gate.arrived(1) + + viewModel.cancelStream() + await gate.open() + await task?.value + + let results = toolResults(in: try turnAfter(assistant, in: viewModel)) + #expect(results.map(\.toolUseId) == ["call_1"]) + #expect(results.first?.content == "3 rows changed") + #expect(transport.requestedTurns.count == 1) + } + + @Test("A saved transcript with an unanswered call is repaired before it is sent") + func unansweredHistoryIsRepairedOnTheWire() async throws { + let transport = ScriptedTransport(rounds: [[.textDelta("ok")]]) + let viewModel = AIChatViewModel() + let history = [ + ChatTurnWire(role: .user, blocks: [.text("drop it")]), + ChatTurnWire(role: .assistant, blocks: [ + .toolUse(ToolUseBlock(id: "call_9", name: "writer", input: .object([:]))) + ]), + ChatTurnWire(role: .user, blocks: [.text("actually, count the rows")]) + ] + _ = start( + viewModel, + transport: transport, + tool: GatedTool(name: "writer", mode: .write, gate: AsyncGate()), + chatMessages: history + ) + await viewModel.streamingTask?.value + + let sent = transport.requestedTurns.first ?? [] + try #require(sent.count == 4) + let answered = sent[2].blocks.compactMap { block -> String? in + guard case .toolResult(let result) = block.kind else { return nil } + return result.toolUseId + } + #expect(answered == ["call_9"]) + } +} diff --git a/apple/TableProTests/Views/Main/AuthorizedRunTargetTests.swift b/apple/TableProTests/Views/Main/AuthorizedRunTargetTests.swift new file mode 100644 index 0000000000..0163e4f39c --- /dev/null +++ b/apple/TableProTests/Views/Main/AuthorizedRunTargetTests.swift @@ -0,0 +1,73 @@ +import Foundation +import TableProPluginKit +import TableProTestSupport +import Testing + +@testable import TablePro + +@MainActor +struct AuthorizedRunTargetTests { + private func errorMessage(of tabId: UUID, in fixture: CoordinatorFixture) -> String? { + fixture.tabManager.tabs.first { $0.id == tabId }?.execution.errorMessage + } + + @Test("A run approved after a tab switch stays on the tab that asked for it") + func approvedRunStaysOnItsTab() async throws { + let fixture = CoordinatorFixture() + let gate = HeldExecutionGate() + fixture.coordinator.executionGate = gate + let asking = fixture.addTab(QueryTab(title: "A", query: "DELETE FROM orders", tabType: .query)) + let other = fixture.addTab(QueryTab(title: "B", query: "SELECT 1", tabType: .query), select: false) + + #expect(fixture.coordinator.runStatement("DELETE FROM orders")) + try await waitUntil { gate.askedCount == 1 } + fixture.tabManager.selectedTabId = other.id + await gate.answer(allowing: true) + + // No session is injected, so the run ends in an error on whichever tab it ran on. + try await waitUntil { + self.errorMessage(of: asking.id, in: fixture) != nil || self.errorMessage(of: other.id, in: fixture) != nil + } + #expect(errorMessage(of: asking.id, in: fixture) != nil) + #expect(errorMessage(of: other.id, in: fixture) == nil) + #expect(!fixture.coordinator.tabExecution.isExecuting(other.id)) + } + + @Test("A run is dropped when its tab moved to another database before the approval") + func approvedRunIsDroppedWhenTheScopeChanged() async throws { + let fixture = CoordinatorFixture() + let gate = HeldExecutionGate() + fixture.coordinator.executionGate = gate + let asking = fixture.addTab(QueryTab(title: "A", query: "DELETE FROM orders", tabType: .query)) + fixture.tabManager.mutate(tabId: asking.id) { $0.tableContext.databaseName = "staging" } + + #expect(fixture.coordinator.runStatement("DELETE FROM orders")) + try await waitUntil { gate.askedCount == 1 } + fixture.tabManager.mutate(tabId: asking.id) { $0.tableContext.databaseName = "production" } + await gate.answer(allowing: true) + try await waitUntil { !fixture.coordinator.isShowingSafeModePrompt } + + #expect(!fixture.coordinator.tabExecution.isExecuting(asking.id)) + #expect(errorMessage(of: asking.id, in: fixture)?.contains("moved to another database") == true) + #expect(fixture.tabManager.tabs.first { $0.id == asking.id }?.execution.lastExecutedAt == nil) + } + + @Test("A refusal lands on the tab that asked, wherever the selection went") + func refusalLandsOnTheAskingTab() async throws { + let fixture = CoordinatorFixture() + let gate = HeldExecutionGate() + fixture.coordinator.executionGate = gate + let asking = fixture.addTab(QueryTab(title: "A", query: "DELETE FROM orders", tabType: .query)) + let other = fixture.addTab(QueryTab(title: "B", query: "SELECT 1", tabType: .query), select: false) + + #expect(fixture.coordinator.runStatement("DELETE FROM orders")) + try await waitUntil { gate.askedCount == 1 } + fixture.tabManager.selectedTabId = other.id + fixture.tabManager.tabs.swapAt(0, 1) + await gate.answer(allowing: false) + try await waitUntil { !fixture.coordinator.isShowingSafeModePrompt } + + #expect(errorMessage(of: asking.id, in: fixture) == "Refused by the test") + #expect(errorMessage(of: other.id, in: fixture) == nil) + } +} diff --git a/apple/TableProTests/Views/Main/ParkedEditSnapshotTests.swift b/apple/TableProTests/Views/Main/ParkedEditSnapshotTests.swift new file mode 100644 index 0000000000..b25fe57889 --- /dev/null +++ b/apple/TableProTests/Views/Main/ParkedEditSnapshotTests.swift @@ -0,0 +1,199 @@ +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +@MainActor +struct ParkedEditSnapshotTests { + private func parkedEdits() -> TabChangeSnapshot { + var snapshot = TabChangeSnapshot() + snapshot.insertedRowIDs = [.inserted(UUID())] + snapshot.columns = ["id", "name"] + snapshot.primaryKeyColumns = ["id"] + return snapshot + } + + @Test("A result installed on a background tab drops the edits parked for its old result") + func backgroundResultDropsParkedEdits() throws { + let fixture = CoordinatorFixture() + let background = fixture.addTab(QueryTab(title: "A", query: "SELECT * FROM orders", tabType: .query)) + fixture.tabManager.mutate(tabId: background.id) { $0.pendingChanges = self.parkedEdits() } + fixture.addTab(QueryTab(title: "B", query: "SELECT 1", tabType: .query)) + + fixture.coordinator.queryExecutionCoordinator.presentMultiStatementResults( + tabId: background.id, + timing: PluginQueryTiming(total: 0), + totalRowsAffected: 0, + newResultSets: [ResultSet(label: "Result 1", tableRows: TableRows())], + sessionNotice: nil + ) + + let tab = try #require(fixture.tabManager.tabs.first { $0.id == background.id }) + #expect(!tab.pendingChanges.hasChanges) + } + + @Test("Edits parked on one result are not restored onto the result that replaced it") + func switchingBackDropsEditsOfAReplacedResult() throws { + let fixture = CoordinatorFixture() + let edited = fixture.addTab(QueryTab(title: "A", query: "SELECT * FROM users", tabType: .query)) + let other = fixture.addTab(QueryTab(title: "B", query: "SELECT 1", tabType: .query), select: false) + let users = ResultSet(label: "users", tableRows: TableRows()) + fixture.tabManager.mutate(tabId: edited.id) { $0.display.replaceUnpinnedResults(with: [users]) } + fixture.changeManager.configureForTable( + tableName: "users", + columns: ["id", "name"], + primaryKeyColumns: ["id"], + databaseType: .mysql, + generatedColumns: [], + triggerReload: false + ) + fixture.changeManager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "a", newValue: "z", originalRow: ["1", "a"] + ) + + fixture.tabManager.selectedTabId = other.id + fixture.coordinator.handleTabChange(from: edited.id, to: other.id, tabs: fixture.tabManager.tabs) + let parked = try #require(fixture.tabManager.tabs.first { $0.id == edited.id }).pendingChanges + #expect(parked.hasChanges) + #expect(parked.resultSetId == users.id) + + // Written straight to the tab, as a result landing in the turn before the switch handler ran would be. + let orders = ResultSet(label: "orders", tableRows: TableRows()) + fixture.tabManager.mutate(tabId: edited.id) { $0.display.replaceUnpinnedResults(with: [orders]) } + fixture.tabManager.selectedTabId = edited.id + fixture.coordinator.handleTabChange(from: other.id, to: edited.id, tabs: fixture.tabManager.tabs) + + #expect(!fixture.changeManager.hasChanges) + #expect(!(try #require(fixture.tabManager.tabs.first { $0.id == edited.id })).pendingChanges.hasChanges) + } + + @Test("A result that lands before the switch handler runs voids the edits still in the change manager") + func resultLandingBeforeTheSwitchHandlerVoidsTheEdits() throws { + let fixture = CoordinatorFixture() + let edited = fixture.addTab(QueryTab(title: "A", query: "SELECT * FROM users", tabType: .query)) + let other = fixture.addTab(QueryTab(title: "B", query: "SELECT 1", tabType: .query), select: false) + let users = ResultSet(label: "users", tableRows: TableRows()) + fixture.tabManager.mutate(tabId: edited.id) { $0.display.replaceUnpinnedResults(with: [users]) } + fixture.changeManager.configureForTable( + tableName: "users", + columns: ["id", "name"], + primaryKeyColumns: ["id"], + databaseType: .mysql, + generatedColumns: [], + triggerReload: false + ) + fixture.changeManager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "a", newValue: "z", originalRow: ["1", "a"] + ) + + // The selection moved, the handler has not run yet, and the run finishes in between. + fixture.tabManager.selectedTabId = other.id + fixture.coordinator.queryExecutionCoordinator.presentMultiStatementResults( + tabId: edited.id, + timing: PluginQueryTiming(total: 0), + totalRowsAffected: 0, + newResultSets: [ResultSet(label: "orders", tableRows: TableRows())], + sessionNotice: nil + ) + fixture.coordinator.handleTabChange(from: edited.id, to: other.id, tabs: fixture.tabManager.tabs) + #expect(!(try #require(fixture.tabManager.tabs.first { $0.id == edited.id })).pendingChanges.hasChanges) + + fixture.tabManager.selectedTabId = edited.id + fixture.coordinator.handleTabChange(from: other.id, to: edited.id, tabs: fixture.tabManager.tabs) + #expect(!fixture.changeManager.hasChanges) + } + + @Test("A tab reselected without the switch handler keeps the edits it stages afterwards") + func reselectWithoutTheHandlerKeepsLaterEdits() throws { + let fixture = CoordinatorFixture() + let edited = fixture.addTab(QueryTab(title: "A", query: "SELECT * FROM users", tabType: .query)) + let other = fixture.addTab(QueryTab(title: "B", query: "SELECT 1", tabType: .query), select: false) + let third = fixture.addTab(QueryTab(title: "C", query: "SELECT 2", tabType: .query), select: false) + + // A to B and back inside one update: a result lands on A in between, and no handler runs. + fixture.tabManager.selectedTabId = other.id + fixture.coordinator.dropParkedEdits(ofBackgroundTab: edited.id) + fixture.tabManager.selectedTabId = edited.id + fixture.changeManager.configureForTable( + tableName: "users", + columns: ["id", "name"], + primaryKeyColumns: ["id"], + databaseType: .mysql, + generatedColumns: [], + triggerReload: false + ) + fixture.changeManager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "a", newValue: "z", originalRow: ["1", "a"] + ) + + fixture.tabManager.selectedTabId = third.id + fixture.coordinator.handleTabChange(from: edited.id, to: third.id, tabs: fixture.tabManager.tabs) + + #expect((try #require(fixture.tabManager.tabs.first { $0.id == edited.id })).pendingChanges.hasChanges) + } + + @Test("A result installed on the selected tab clears its off-screen mark") + func resultOnTheSelectedTabClearsTheMark() throws { + let fixture = CoordinatorFixture() + let edited = fixture.addTab(QueryTab(title: "A", query: "SELECT * FROM users", tabType: .query)) + let other = fixture.addTab(QueryTab(title: "B", query: "SELECT 1", tabType: .query), select: false) + fixture.tabManager.selectedTabId = other.id + fixture.coordinator.dropParkedEdits(ofBackgroundTab: edited.id) + fixture.tabManager.selectedTabId = edited.id + + fixture.coordinator.queryExecutionCoordinator.presentMultiStatementResults( + tabId: edited.id, + timing: PluginQueryTiming(total: 0), + totalRowsAffected: 0, + newResultSets: [ResultSet(label: "users", tableRows: TableRows())], + sessionNotice: nil + ) + + #expect(fixture.coordinator.tabsGivenResultOffScreen.keys.contains(edited.id) == false) + } + + @Test("Edits come back when the tab still shows the result they were made on") + func switchingBackRestoresEditsOfTheSameResult() throws { + let fixture = CoordinatorFixture() + let edited = fixture.addTab(QueryTab(title: "A", query: "SELECT * FROM users", tabType: .query)) + let other = fixture.addTab(QueryTab(title: "B", query: "SELECT 1", tabType: .query), select: false) + let users = ResultSet(label: "users", tableRows: TableRows()) + fixture.tabManager.mutate(tabId: edited.id) { $0.display.replaceUnpinnedResults(with: [users]) } + fixture.changeManager.configureForTable( + tableName: "users", + columns: ["id", "name"], + primaryKeyColumns: ["id"], + databaseType: .mysql, + generatedColumns: [], + triggerReload: false + ) + fixture.changeManager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "a", newValue: "z", originalRow: ["1", "a"] + ) + + fixture.tabManager.selectedTabId = other.id + fixture.coordinator.handleTabChange(from: edited.id, to: other.id, tabs: fixture.tabManager.tabs) + #expect(!fixture.changeManager.hasChanges) + fixture.tabManager.selectedTabId = edited.id + fixture.coordinator.handleTabChange(from: other.id, to: edited.id, tabs: fixture.tabManager.tabs) + + #expect(fixture.changeManager.hasChanges) + } + + @Test("Only a background tab's parked edits are dropped") + func selectedTabKeepsItsSnapshot() throws { + let fixture = CoordinatorFixture() + let selected = fixture.addTab(QueryTab(title: "A", query: "SELECT * FROM orders", tabType: .query)) + fixture.tabManager.mutate(tabId: selected.id) { $0.pendingChanges = self.parkedEdits() } + + fixture.coordinator.dropParkedEdits(ofBackgroundTab: selected.id) + + let tab = try #require(fixture.tabManager.tabs.first { $0.id == selected.id }) + #expect(tab.pendingChanges.hasChanges) + } +} diff --git a/apple/TableProTests/Views/Main/RunDiscardGuardTests.swift b/apple/TableProTests/Views/Main/RunDiscardGuardTests.swift new file mode 100644 index 0000000000..662d476199 --- /dev/null +++ b/apple/TableProTests/Views/Main/RunDiscardGuardTests.swift @@ -0,0 +1,55 @@ +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +@MainActor +struct RunDiscardGuardTests { + // A query tab showing a result with staged edits. `isShowingConfirmAlert` stands in for the discard prompt, + // which answers false while it is up. + private func makeEditedResult() -> (CoordinatorFixture, UUID) { + let fixture = CoordinatorFixture() + fixture.coordinator.executionGate = AlwaysAllowGate() + let tab = fixture.addTab(QueryTab(title: "Query", query: "SELECT * FROM users", tabType: .query)) + let shown = ResultSet(label: "Result 1", tableRows: TableRows()) + fixture.tabManager.mutate(tabId: tab.id) { $0.display.replaceUnpinnedResults(with: [shown]) } + fixture.changeManager.hasChanges = true + fixture.coordinator.isShowingConfirmAlert = true + return (fixture, tab.id) + } + + @Test("Running a statement over staged edits asks before it replaces the result") + func runStatementAsksFirst() { + let (fixture, tabId) = makeEditedResult() + + #expect(!fixture.coordinator.runStatement("SELECT * FROM users")) + + #expect(fixture.changeManager.hasChanges) + #expect(!fixture.coordinator.isShowingSafeModePrompt) + #expect(!fixture.coordinator.tabExecution.isExecuting(tabId)) + } + + @Test("Run and Run All over staged edits ask before they replace the result") + func runAndRunAllAskFirst() { + let (fixture, tabId) = makeEditedResult() + + fixture.coordinator.runQuery(viewport: .firstRow) + fixture.coordinator.runAllStatements() + + #expect(fixture.changeManager.hasChanges) + #expect(!fixture.coordinator.isShowingSafeModePrompt) + #expect(!fixture.coordinator.tabExecution.isExecuting(tabId)) + } + + @Test("A tab with no result runs without asking about another tab's edits") + func tabWithoutAResultRunsAtOnce() { + let fixture = CoordinatorFixture() + fixture.coordinator.executionGate = AlwaysAllowGate() + fixture.addTab(QueryTab(title: "Query", query: "SELECT 1", tabType: .query)) + fixture.changeManager.hasChanges = true + fixture.coordinator.isShowingConfirmAlert = true + + #expect(fixture.coordinator.runStatement("SELECT 1")) + } +} diff --git a/apple/TableProTests/Views/Main/SaveAttributionTests.swift b/apple/TableProTests/Views/Main/SaveAttributionTests.swift new file mode 100644 index 0000000000..57aea277e9 --- /dev/null +++ b/apple/TableProTests/Views/Main/SaveAttributionTests.swift @@ -0,0 +1,131 @@ +import Foundation +import TableProPluginKit +import TableProTestSupport +import Testing + +@testable import TablePro + +@MainActor +private final class SaveAnswers { + var first: Bool? + var second: Bool? +} + +@MainActor +struct SaveAttributionTests { + // A table tab with one edited cell, and a gate that holds the save until the test answers. + private func makeEditedTable() -> (CoordinatorFixture, HeldExecutionGate, UUID) { + let fixture = CoordinatorFixture() + let gate = HeldExecutionGate() + fixture.coordinator.executionGate = gate + var tab = QueryTab(title: "users", query: "SELECT * FROM users", tabType: .table, tableName: "users") + tab.execution.lastExecutedAt = Date() + let rows = TableRows.from( + queryRows: [[.text("1"), .text("a")]], + columns: ["id", "name"], + columnTypes: [.text(rawType: nil), .text(rawType: nil)], + hasAuthoritativeSchema: true + ) + fixture.addTab(tab, rows: rows) + fixture.changeManager.configureForTable( + tableName: "users", + columns: ["id", "name"], + primaryKeyColumns: ["id"], + databaseType: .mysql, + generatedColumns: [] + ) + fixture.changeManager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "a", newValue: "z", originalRow: ["1", "a"] + ) + return (fixture, gate, tab.id) + } + + private func save(_ coordinator: MainContentCoordinator) { + var truncates: Set = [] + var deletes: Set = [] + var options: [DatabaseTreeTableRef: TableOperationOptions] = [:] + coordinator.saveChanges( + pendingTruncates: &truncates, + pendingDeletes: &deletes, + tableOperationOptions: &options + ) + } + + @Test("A save approved after a tab switch writes nothing and keeps the edits") + func saveApprovedAfterATabSwitchIsDropped() async throws { + let (fixture, gate, savingTabId) = makeEditedTable() + let other = fixture.addTab(QueryTab(title: "B", query: "SELECT 1", tabType: .query), select: false) + + save(fixture.coordinator) + try await waitUntil { gate.askedCount == 1 } + fixture.tabManager.selectedTabId = other.id + await gate.answer(allowing: true) + + try await waitUntil { + fixture.tabManager.tabs.first { $0.id == savingTabId }?.execution.errorMessage != nil + } + #expect(fixture.changeManager.hasChanges) + #expect(fixture.tabManager.tabs.first { $0.id == other.id }?.execution.errorMessage == nil) + // The save is over: a second one is not refused as already running. + #expect(fixture.coordinator.rowEditingCoordinator.beginSaveInFlight()) + fixture.coordinator.rowEditingCoordinator.endSaveInFlight() + } + + @Test("A value loaded during the approval prompt is not an edit, a new cell change is") + func loadedValueDoesNotChangeThePlannedEdits() { + let manager = DataChangeManager() + manager.configureForTable( + tableName: "users", + columns: ["id", "name", "bio"], + primaryKeyColumns: ["id"], + databaseType: .mysql, + generatedColumns: [] + ) + manager.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "a", newValue: "z", originalRow: ["1", "a", .null] + ) + let planned = PlannedEdits(manager.pending) + let before = manager.pending + + manager.adoptLoadedValues([.existing(0): [2: .text("a long value")]]) + #expect(manager.pending != before) + #expect(PlannedEdits(manager.pending) == planned) + + manager.recordCellChange( + rowID: .existing(0), columnIndex: 2, columnName: "bio", + oldValue: "a long value", newValue: "edited", originalRow: ["1", "a", "a long value"] + ) + #expect(PlannedEdits(manager.pending) != planned) + } + + @Test("A second save-then-close gets its own answer and leaves the first one waiting for its save") + func eachSaveResumesItsOwnWaiter() async throws { + let (fixture, gate, _) = makeEditedTable() + let coordinator = fixture.coordinator + let answers = SaveAnswers() + + Task { + answers.first = await withCheckedContinuation { continuation in + coordinator.saveCompletionContinuation = continuation + self.save(coordinator) + } + } + try await waitUntil { gate.askedCount == 1 } + Task { + answers.second = await withCheckedContinuation { continuation in + coordinator.saveCompletionContinuation = continuation + self.save(coordinator) + } + } + try await waitUntil { answers.second != nil } + #expect(answers.second == false) + #expect(answers.first == nil) + + await gate.answer(allowing: false) + try await waitUntil { answers.first != nil } + #expect(answers.first == false) + #expect(coordinator.saveCompletionContinuation == nil) + } +} diff --git a/apple/TableProTests/Views/Sidebar/FailedContainerSwitchCommandTests.swift b/apple/TableProTests/Views/Sidebar/FailedContainerSwitchCommandTests.swift new file mode 100644 index 0000000000..39c0195b2d --- /dev/null +++ b/apple/TableProTests/Views/Sidebar/FailedContainerSwitchCommandTests.swift @@ -0,0 +1,77 @@ +// +// FailedContainerSwitchCommandTests.swift +// TableProTests +// +// A command on a row in another database must not run where the session still is after a failed switch. +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +private final class CancelledSwitchDriverStub: PluginDriverStub, @unchecked Sendable { + func switchDatabase(to database: String) async throws { + throw CancellationError() + } +} + +@MainActor +struct FailedContainerSwitchCommandTests { + private func withCoordinator( + driver stub: any PluginDatabaseDriver, + _ body: (MainContentCoordinator, DatabaseConnection) async throws -> Void + ) async throws { + let connection = TestFixtures.makeConnection(database: "db_a", type: .mysql) + let driver = PluginDriverAdapter(connection: connection, pluginDriver: stub) + var session = ConnectionSession(connection: connection, driver: driver) + session.status = .connected + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let coordinator = MainContentCoordinator( + connection: connection, + tabManager: QueryTabManager(containerTarget: .database, visibleContainer: "db_a"), + changeManager: DataChangeManager(), + toolbarState: ConnectionToolbarState() + ) + defer { coordinator.teardown() } + + try await body(coordinator, connection) + } + + @Test("A cancelled database switch reports that the window did not get there") + func cancelledSwitchReportsFailure() async throws { + try await withCoordinator(driver: CancelledSwitchDriverStub()) { coordinator, connection in + let landed = await coordinator.switchContainers(database: "db_b", schema: nil) + + #expect(!landed) + #expect(DatabaseManager.shared.session(for: connection.id)?.resolvedBrowseDatabase == "db_a") + } + } + + @Test("A switch that lands, or was never needed, reports success") + func landedSwitchReportsSuccess() async throws { + try await withCoordinator(driver: SwitchingDriverStub()) { coordinator, _ in + let alreadyThere = await coordinator.switchContainers(database: "db_a", schema: nil) + let moved = await coordinator.switchContainers(database: "db_b", schema: nil) + + #expect(alreadyThere) + #expect(moved) + } + } + + @Test("A sidebar row's activation fails with the switch, so its command does not run") + func rowActivationFailsWithTheSwitch() async throws { + try await withCoordinator(driver: CancelledSwitchDriverStub()) { coordinator, _ in + let outline = DatabaseTreeOutlineCoordinator() + outline.mainCoordinator = coordinator + let ref = DatabaseTreeTableRef(database: "db_b", schema: nil, table: TestFixtures.makeTableInfo(name: "orders")) + + let activated = await outline.activate(ref) + + #expect(!activated) + } + } +} diff --git a/apple/TableProTests/Views/Structure/StructureRebuildOpenTransactionTests.swift b/apple/TableProTests/Views/Structure/StructureRebuildOpenTransactionTests.swift new file mode 100644 index 0000000000..c54fd582b5 --- /dev/null +++ b/apple/TableProTests/Views/Structure/StructureRebuildOpenTransactionTests.swift @@ -0,0 +1,88 @@ +// +// StructureRebuildOpenTransactionTests.swift +// TableProTests +// +// Inside an open transaction SQLite ignores PRAGMA foreign_keys = off, so a rebuild there cascades into child rows. +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class RebuildSessionDriver: PluginDriverStub, @unchecked Sendable { + private let state: PluginSessionTransactionState + private let lock = NSLock() + private var queries: [String] = [] + + init(state: PluginSessionTransactionState) { + self.state = state + } + + var executed: [String] { lock.withLock { queries } } + var supportsTransactions: Bool { true } + + func sessionTransactionState() async -> PluginSessionTransactionState { state } + + func execute(query: String) async throws -> PluginQueryResult { + lock.withLock { queries.append(query) } + return .empty + } + + func switchDatabase(to database: String) async throws {} +} + +@MainActor +struct StructureRebuildOpenTransactionTests { + private static let plan = PluginColumnReorderPlan( + statements: ["DROP TABLE parent"], + prologue: ["PRAGMA foreign_keys = off"], + epilogue: ["PRAGMA foreign_keys = on"], + isTransactional: true, + cost: .tableRebuild, + verifications: [] + ) + + private func run(state: PluginSessionTransactionState) async throws -> [String] { + let connection = TestFixtures.makeConnection(database: "app") + let driver = RebuildSessionDriver(state: state) + var session = ConnectionSession( + connection: connection, + driver: PluginDriverAdapter(connection: connection, pluginDriver: driver) + ) + session.status = .connected + session.browseDatabase = "app" + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + let scope = DatabaseScope(connectionId: connection.id, database: "app", schema: nil) + + do { + try await StructureRebuildPlanRunner.execute( + StructureRebuildPlanRunner.Prepared(plan: Self.plan, fingerprint: nil, scope: scope, tableName: "parent"), + databaseType: connection.type, + operationDescription: "Apply Schema Changes", + isConfirmationPreCleared: true, + gate: AlwaysAllowGate() + ) + } catch StructureRebuildPlanRunner.RunError.sessionTransactionOpen { + return driver.executed + ["refused"] + } + return driver.executed + } + + @Test("A rebuild that turns foreign keys off refuses to run inside the user's open transaction") + func refusesInsideAnOpenTransaction() async throws { + let executed = try await run(state: .inTransaction) + + #expect(executed == ["refused"]) + } + + @Test("With no transaction open the rebuild turns foreign keys off before its own BEGIN") + func runsOutsideATransaction() async throws { + let executed = try await run(state: .idle) + + #expect(executed == [ + "PRAGMA foreign_keys = off", "BEGIN", "DROP TABLE parent", "COMMIT", "PRAGMA foreign_keys = on" + ]) + } +} diff --git a/apple/scripts/probes/check-duckdb-streaming-results.sh b/apple/scripts/probes/check-duckdb-streaming-results.sh new file mode 100755 index 0000000000..3eac47247b --- /dev/null +++ b/apple/scripts/probes/check-duckdb-streaming-results.sh @@ -0,0 +1,221 @@ +#!/usr/bin/env bash +# +# Measures the libduckdb behavior DuckDBConnection.swift and DuckDBResultReader.swift read results by. +# +# A SELECT runs as a streaming result, so a result past PluginRowLimits.emergencyMax rows is exported whole and a +# capped read stops at its cap. A streaming result keeps its connection busy until it ends, so a second connection +# renders each chunk as text. Run this after bumping libduckdb. It checks that: +# - a streaming result hands back every row of a table larger than the old 5,000,000 row cap +# - the connection refuses another statement while the result is open, and takes one once it is destroyed +# - a table function on a second connection renders a chunk fetched from the first, re-binding on every execution +# - a time zone set on one connection does not reach the other +# - a statement that changes rows does not stream and reports its row count +# - dropping an unfinished SELECT that may write aborts the transaction it runs in, and draining it does not +# - an empty streaming result still reports its columns +# +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +HEADERS="$ROOT/Plugins/DuckDBDriverPlugin/CDuckDB/include" +LIB="$ROOT/Libs/libduckdb.a" + +if [ ! -f "$LIB" ]; then + echo "error: $LIB is missing. Run scripts/download-libs.sh first." >&2 + exit 3 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +cat > "$WORK/probe.c" <<'PROBE' +#include +#include +#include "duckdb.h" + +static int failures = 0; +static duckdb_result *source = NULL; +static duckdb_data_chunk pending = NULL; +static int binds = 0; + +static void check(int ok, const char *what) { + printf("%s: %s\n", ok ? "ok" : "FAIL", what); + if (!ok) failures++; +} + +static void reader_bind(duckdb_bind_info info) { + binds++; + if (!source) { duckdb_bind_set_error(info, "No result is being read"); return; } + for (idx_t i = 0; i < duckdb_column_count(source); i++) { + duckdb_logical_type type = duckdb_column_logical_type(source, i); + char name[16]; + snprintf(name, sizeof name, "c%llu", (unsigned long long)i + 1); + duckdb_bind_add_result_column(info, name, type); + duckdb_destroy_logical_type(&type); + } +} +static void reader_init(duckdb_init_info info) { duckdb_init_set_max_threads(info, 1); } +static void reader_scan(duckdb_function_info info, duckdb_data_chunk out) { + (void)info; + if (!pending) { duckdb_data_chunk_set_size(out, 0); return; } + for (idx_t i = 0; i < duckdb_data_chunk_get_column_count(pending); i++) { + duckdb_vector_reference_vector(duckdb_data_chunk_get_vector(out, i), duckdb_data_chunk_get_vector(pending, i)); + } + duckdb_data_chunk_set_size(out, duckdb_data_chunk_get_size(pending)); + pending = NULL; +} + +static int run(duckdb_connection c, const char *sql, duckdb_error_type *type) { + duckdb_result r; + int ok = duckdb_query(c, sql, &r) == DuckDBSuccess; + if (type) *type = ok ? DUCKDB_ERROR_INVALID : duckdb_result_error_type(&r); + duckdb_destroy_result(&r); + return ok; +} +static void text(duckdb_connection c, const char *sql, char *out, size_t size) { + duckdb_result r; + out[0] = '\0'; + if (duckdb_query(c, sql, &r) == DuckDBSuccess) { + char *v = duckdb_value_varchar(&r, 0, 0); + if (v) { snprintf(out, size, "%s", v); duckdb_free(v); } + } + duckdb_destroy_result(&r); +} + +int main(void) { + duckdb_database db; + duckdb_connection session, render; + if (duckdb_open(NULL, &db) != DuckDBSuccess) return 2; + if (duckdb_connect(db, &session) != DuckDBSuccess || duckdb_connect(db, &render) != DuckDBSuccess) return 2; + fprintf(stderr, "libduckdb %s\n", duckdb_library_version()); + + duckdb_table_function reader = duckdb_create_table_function(); + duckdb_table_function_set_name(reader, "_probe_reader"); + duckdb_table_function_set_bind(reader, reader_bind); + duckdb_table_function_set_init(reader, reader_init); + duckdb_table_function_set_function(reader, reader_scan); + check(duckdb_register_table_function(render, reader) == DuckDBSuccess, "the reader registers on the second connection"); + duckdb_destroy_table_function(&reader); + + run(session, "CREATE TABLE t AS SELECT i, DATE '2020-01-01' AS d, TIMESTAMPTZ '2026-08-14 23:59:00+00' AS tz " + "FROM range(5000100) r(i)", NULL); + run(session, "SET TimeZone = 'Asia/Tokyo'", NULL); + run(render, "SET TimeZone = 'UTC'", NULL); + char a[64], b[64]; + text(session, "SELECT CAST(TIMESTAMPTZ '2026-08-14 23:59:00+00' AS VARCHAR)", a, sizeof a); + text(render, "SELECT CAST(TIMESTAMPTZ '2026-08-14 23:59:00+00' AS VARCHAR)", b, sizeof b); + check(strcmp(a, b) != 0, "a time zone belongs to one connection"); + text(session, "SELECT value FROM duckdb_settings() WHERE name = 'TimeZone'", a, sizeof a); + check(strcmp(a, "Asia/Tokyo") == 0, "duckdb_settings() reports the session time zone"); + run(render, "SET TimeZone = 'Asia/Tokyo'", NULL); + + duckdb_prepared_statement select; + duckdb_result result; + duckdb_prepare(session, "SELECT * FROM t", &select); + check(duckdb_prepared_statement_type(select) == DUCKDB_STATEMENT_TYPE_SELECT, "a SELECT prepares as a SELECT"); + check(duckdb_prepared_statement_column_type(select, 2) == DUCKDB_TYPE_TIMESTAMP_TZ, "a prepared statement knows its column types"); + check(duckdb_execute_prepared_streaming(select, &result) == DuckDBSuccess && duckdb_result_is_streaming(result), "a SELECT streams"); + duckdb_destroy_prepare(&select); + + duckdb_error_type busy; + check(!run(session, "SELECT 1", &busy) && busy == DUCKDB_ERROR_RESOURCE_IN_USE, "an open streaming result keeps its connection busy"); + + source = &result; + duckdb_prepared_statement projection; + check(duckdb_prepare(render, "SELECT c1, CAST(c2 AS VARCHAR), CAST(c3 AS VARCHAR) FROM _probe_reader()", &projection) == DuckDBSuccess, + "the projection prepares against the open result"); + int bindsAfterPrepare = binds; + unsigned long long total = 0, rendered = 0; + int firstText = 0; + duckdb_data_chunk chunk; + while ((chunk = duckdb_fetch_chunk(result)) != NULL) { + total += duckdb_data_chunk_get_size(chunk); + pending = chunk; + duckdb_result out; + if (duckdb_execute_prepared(projection, &out) == DuckDBSuccess) { + duckdb_data_chunk part; + while ((part = duckdb_fetch_chunk(out)) != NULL) { + if (rendered == 0) { + duckdb_string_t *cells = (duckdb_string_t *)duckdb_vector_get_data(duckdb_data_chunk_get_vector(part, 2)); + firstText = duckdb_string_t_length(cells[0]) == 22 + && memcmp(duckdb_string_t_data(&cells[0]), "2026-08-15 08:59:00+09", 22) == 0; + } + rendered += duckdb_data_chunk_get_size(part); + duckdb_destroy_data_chunk(&part); + } + } + duckdb_destroy_result(&out); + duckdb_destroy_data_chunk(&chunk); + } + source = NULL; + check(duckdb_result_error(&result) == NULL && total == 5000100, "a streaming result hands back every row past 5,000,000"); + check(rendered == total, "the second connection renders every chunk"); + check(firstText, "the rendered text uses the aligned time zone"); + check(binds > bindsAfterPrepare + 1, "the table function binds again on every execution"); + duckdb_destroy_prepare(&projection); + duckdb_destroy_result(&result); + + duckdb_prepare(session, "SELECT * FROM t", &select); + duckdb_execute_prepared_streaming(select, &result); + chunk = duckdb_fetch_chunk(result); + duckdb_destroy_data_chunk(&chunk); + duckdb_destroy_result(&result); + check(run(session, "SELECT 1", NULL), "a destroyed unfinished result frees the connection"); + + run(session, "BEGIN", NULL); + run(session, "INSERT INTO t VALUES (-1, NULL, NULL)", NULL); + duckdb_execute_prepared_streaming(select, &result); + chunk = duckdb_fetch_chunk(result); + duckdb_destroy_data_chunk(&chunk); + duckdb_destroy_result(&result); + check(run(session, "SELECT 1", NULL), "dropping an unfinished read-only SELECT keeps the transaction"); + run(session, "ROLLBACK", NULL); + duckdb_destroy_prepare(&select); + + run(session, "CREATE SEQUENCE sq", NULL); + duckdb_prepared_statement writing; + duckdb_prepare(session, "SELECT nextval('sq') FROM t", &writing); + run(session, "BEGIN", NULL); + duckdb_execute_prepared_streaming(writing, &result); + chunk = duckdb_fetch_chunk(result); + duckdb_destroy_data_chunk(&chunk); + duckdb_destroy_result(&result); + duckdb_error_type aborted; + check(!run(session, "SELECT 1", &aborted) && aborted == DUCKDB_ERROR_TRANSACTION, "dropping an unfinished SELECT that may write aborts the transaction"); + run(session, "ROLLBACK", NULL); + + run(session, "BEGIN", NULL); + duckdb_execute_prepared_streaming(writing, &result); + while ((chunk = duckdb_fetch_chunk(result)) != NULL) duckdb_destroy_data_chunk(&chunk); + duckdb_destroy_result(&result); + check(run(session, "SELECT 1", NULL), "draining it keeps the transaction"); + run(session, "ROLLBACK", NULL); + duckdb_destroy_prepare(&writing); + + duckdb_prepared_statement insert; + duckdb_prepare(session, "INSERT INTO t SELECT * FROM t LIMIT 5", &insert); + duckdb_execute_prepared_streaming(insert, &result); + check(!duckdb_result_is_streaming(result) && duckdb_rows_changed(&result) == 5, "a statement that changes rows does not stream and reports its count"); + duckdb_destroy_result(&result); + duckdb_destroy_prepare(&insert); + + duckdb_prepare(session, "SELECT * FROM t WHERE i < -5", &select); + duckdb_execute_prepared_streaming(select, &result); + chunk = duckdb_fetch_chunk(result); + check(chunk == NULL && duckdb_result_error(&result) == NULL && duckdb_column_count(&result) == 3, "an empty streaming result reports its columns"); + duckdb_destroy_result(&result); + duckdb_destroy_prepare(&select); + + duckdb_disconnect(&render); + duckdb_disconnect(&session); + duckdb_close(&db); + return failures == 0 ? 0 : 1; +} +PROBE + +cc -O1 -I "$HEADERS" "$WORK/probe.c" -Wl,-force_load,"$LIB" -lc++ -o "$WORK/probe" +if "$WORK/probe"; then + echo "OK: libduckdb streams, renders and aborts the way the DuckDB driver reads results." + exit 0 +fi +echo "error: libduckdb no longer behaves the way DuckDBConnection.swift reads results." >&2 +exit 1 diff --git a/apple/scripts/probes/check-parquet-export-values.sh b/apple/scripts/probes/check-parquet-export-values.sh new file mode 100755 index 0000000000..966b46b5bd --- /dev/null +++ b/apple/scripts/probes/check-parquet-export-values.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# +# Check that the Parquet export writes the values it was given. +# +# The plugin links libduckdb.a, so no test bundle can hold its staging database. This compiles +# DuckDBStagingDatabase.swift against the shipped library, stages a table holding the values the +# export used to change (binary, a zero-length blob, bit(8), BIGINT UNSIGNED above 2^63, a TIME +# past 24 hours, an Oracle DATE with a time, text in an integer column), writes the file and reads +# it back. +# +# Usage: +# scripts/probes/check-parquet-export-values.sh +# +# Needs Libs/libduckdb.a (scripts/download-libs.sh) and a Debug build of TableProPluginKit in +# DerivedData (run verify.sh build first). Exits non-zero on a wrong value, 3 when a prerequisite +# is missing. Run it after touching the plugin's staging or type mapping, or bumping libduckdb. + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PLUGIN="$ROOT/Plugins/ParquetExportPlugin" +CDUCKDB="$ROOT/Plugins/DuckDBDriverPlugin/CDuckDB" +LIB="$ROOT/Libs/libduckdb.a" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +[ -f "$LIB" ] || { + echo "$LIB is missing; run scripts/download-libs.sh" >&2 + exit 3 +} + +if [ -z "${DEVELOPER_DIR:-}" ]; then + DEVELOPER_DIR="$(xcode-select -p)" + case "$DEVELOPER_DIR" in + *CommandLineTools*) DEVELOPER_DIR=/Applications/Xcode.app/Contents/Developer ;; + esac +fi +export DEVELOPER_DIR + +FRAMEWORK_DIR="" +for info in "$HOME"/Library/Developer/Xcode/DerivedData/TablePro-*/info.plist; do + [ -f "$info" ] || continue + workspace="$(/usr/libexec/PlistBuddy -c 'Print :WorkspacePath' "$info" 2> /dev/null)" + products="$(dirname "$info")/Build/Products/Debug" + if [ "$workspace" = "$ROOT/TablePro.xcodeproj" ] && [ -d "$products/TableProPluginKit.framework" ]; then + FRAMEWORK_DIR="$products" + fi +done +if [ -z "$FRAMEWORK_DIR" ]; then + FRAMEWORK_DIR="$(find "$HOME/Library/Developer/Xcode/DerivedData" -type d -path '*/Build/Products/Debug/TableProPluginKit.framework' -print 2> /dev/null \ + | while read -r path; do echo "$(stat -f %m "$path") $(dirname "$path")"; done \ + | sort -rn | head -1 | cut -d' ' -f2-)" +fi +[ -n "$FRAMEWORK_DIR" ] || { + echo "no Debug TableProPluginKit.framework in DerivedData; build the app first" >&2 + exit 3 +} + +cat > "$WORK/main.swift" << 'SWIFT' +import CDuckDB +import Foundation +import TableProPluginKit + +@main +enum ParquetValuesProbe { + static func main() throws { + let file = URL(fileURLWithPath: CommandLine.arguments[1]) + let declared: [(name: String, type: String, engine: String)] = [ + ("id", "bigint", "PostgreSQL"), + ("payload", "bytea", "PostgreSQL"), + ("flags", "bit(8)", "PostgreSQL"), + ("big", "bigint unsigned", "MySQL"), + ("elapsed", "time", "MySQL"), + ("seen", "DATE", "Oracle"), + ("count", "int", "SQLite"), + ("day", "date", "PostgreSQL"), + ("note", "text", "PostgreSQL") + ] + let types = declared.map { ParquetTypeMapper.duckDBType(forColumnType: $0.type, databaseTypeId: $0.engine) } + + let staging = try DuckDBStagingDatabase() + try staging.createTable(columns: declared.map(\.name), types: types) + try staging.insert(rows: [ + [ + .text("1"), .bytes(Data([0x41, 0x42, 0x43])), .text("10101010"), .text("18446744073709551615"), + .text("100:00:00"), .text("2024-01-01 12:30:00"), .text("7"), .text("2024-03-01"), .text("it's") + ], + [ + .text("2"), .bytes(Data()), .text("1"), .text("5"), + .text("10:00:00"), .text("2024-01-02 00:00:00"), .text("seven"), .text("2024-03-02"), .text("") + ] + ]) + try staging.insert(rows: [ + [.text("3"), .bytes(Data([0x00, 0xFF, 0x5C, 0x78])), .null, .null, .null, .null, .null, .null, .null] + ]) + let fallbacks = try staging.copyToParquet(fileURL: file, compression: "SNAPPY", rowGroupSize: 122_880) + + var failures = 0 + func expect(_ label: String, _ actual: String, _ expected: String) { + guard actual != expected else { return } + print("FAIL \(label): got \(actual), expected \(expected)") + failures += 1 + } + + let mapped: [(type: String, engine: String, expected: String)] = [ + ("bit", "PostgreSQL", "BOOLEAN"), ("bit(1)", "MySQL", "BOOLEAN"), ("bit", "SQL Server", "BOOLEAN"), + ("bit(8)", "MySQL", "VARCHAR"), ("bit varying(8)", "PostgreSQL", "VARCHAR"), + ("int unsigned", "MySQL", "BIGINT"), ("BIGINT(20) UNSIGNED", "MySQL", "UBIGINT"), + ("bigint", "MySQL", "BIGINT"), ("date", "MySQL", "DATE"), ("DATE", "Oracle", "TIMESTAMP") + ] + for entry in mapped { + expect( + "\(entry.engine) \(entry.type)", + ParquetTypeMapper.duckDBType(forColumnType: entry.type, databaseTypeId: entry.engine), + entry.expected) + } + expect("bytes in a text column", ParquetStaging.text(forBytes: Data([0x41, 0x00, 0x5C, 0xC3])), "A\\x00\\x5C\\xC3") + + expect( + "columns written as text", + fallbacks.map { "\($0.column):\($0.type):\($0.valueCount)" }.joined(separator: " "), + "elapsed:TIME:1 count:BIGINT:1") + + let reader = try Reader() + let source = "read_parquet('\(file.path)')" + let typeIds = try reader.columnTypes("SELECT * FROM \(source)") + let expectedTypes: [duckdb_type] = [ + DUCKDB_TYPE_BIGINT, DUCKDB_TYPE_BLOB, DUCKDB_TYPE_VARCHAR, DUCKDB_TYPE_UBIGINT, DUCKDB_TYPE_VARCHAR, + DUCKDB_TYPE_TIMESTAMP, DUCKDB_TYPE_VARCHAR, DUCKDB_TYPE_DATE, DUCKDB_TYPE_VARCHAR + ] + for (index, column) in declared.enumerated() { + expect("type of \(column.name)", "\(typeIds[index].rawValue)", "\(expectedTypes[index].rawValue)") + } + + func cell(_ expression: String, id: Int) throws -> String { + try reader.scalar("SELECT CAST(\(expression) AS VARCHAR) FROM \(source) WHERE id = \(id)") + } + expect("row count", try reader.scalar("SELECT CAST(count(*) AS VARCHAR) FROM \(source)"), "3") + expect("binary value", try cell("payload = '\\x41\\x42\\x43'::BLOB", id: 1), "true") + expect("zero-length blob", try cell("payload = ''::BLOB", id: 2), "true") + expect("binary with a NUL and a backslash", try cell("payload = '\\x00\\xFF\\x5C\\x78'::BLOB", id: 3), "true") + expect("bit(8)", try cell("flags", id: 1), "10101010") + expect("unsigned bigint", try cell("big", id: 1), "18446744073709551615") + expect("time past 24 hours", try cell("elapsed", id: 1), "100:00:00") + expect("Oracle date with a time", try cell("seen", id: 1), "2024-01-01 12:30:00") + expect("text in an integer column", try cell("count", id: 2), "seven") + expect("number beside it", try cell("count", id: 1), "7") + expect("date", try cell("day", id: 2), "2024-03-02") + expect("quoted text", try cell("note", id: 1), "it's") + expect("empty text", try cell("note = ''", id: 2), "true") + expect("null", try cell("note IS NULL AND big IS NULL AND seen IS NULL", id: 3), "true") + + if failures > 0 { exit(1) } + print("PASS: the Parquet file holds every staged value") + } +} + +final class Reader { + private var database: duckdb_database? + private var connection: duckdb_connection? + + init() throws { + guard duckdb_open(nil, &database) == DuckDBSuccess, + duckdb_connect(database, &connection) == DuckDBSuccess else { + throw ProbeError.failed("could not open the reader") + } + } + + deinit { + duckdb_disconnect(&connection) + duckdb_close(&database) + } + + func scalar(_ sql: String) throws -> String { + var result = duckdb_result() + defer { duckdb_destroy_result(&result) } + guard duckdb_query(connection, sql, &result) == DuckDBSuccess else { + throw ProbeError.failed(duckdb_result_error(&result).map { String(cString: $0) } ?? sql) + } + guard duckdb_row_count(&result) == 1, let value = duckdb_value_varchar(&result, 0, 0) else { + return "" + } + defer { duckdb_free(value) } + return String(cString: value) + } + + func columnTypes(_ sql: String) throws -> [duckdb_type] { + var result = duckdb_result() + defer { duckdb_destroy_result(&result) } + guard duckdb_query(connection, sql, &result) == DuckDBSuccess else { + throw ProbeError.failed(duckdb_result_error(&result).map { String(cString: $0) } ?? sql) + } + return (0 ..< duckdb_column_count(&result)).map { duckdb_column_type(&result, $0) } + } +} + +enum ProbeError: Error { + case failed(String) +} +SWIFT + +xcrun swiftc -swift-version 6 -parse-as-library -module-name TableProSQLQuoting -Onone \ + -emit-module -emit-module-path "$WORK/TableProSQLQuoting.swiftmodule" -emit-library -static \ + -o "$WORK/libTableProSQLQuoting.a" "$ROOT/Packages/TableProCore/Sources/TableProSQLQuoting"/*.swift \ + > "$WORK/quoting.log" 2>&1 || { + echo "TableProSQLQuoting failed to compile:" >&2 + grep -E 'error:' "$WORK/quoting.log" | sort -u | head -20 >&2 + exit 3 +} + +xcrun swiftc -swift-version 6 -parse-as-library -module-name ParquetValuesProbe -Onone \ + -I "$WORK" -Xlinker "$WORK/libTableProSQLQuoting.a" \ + -F "$FRAMEWORK_DIR" -framework TableProPluginKit -Xlinker -rpath -Xlinker "$FRAMEWORK_DIR" \ + -I "$CDUCKDB" -Xcc -I"$CDUCKDB/include" \ + -Xlinker -force_load -Xlinker "$LIB" -lc++ \ + "$PLUGIN/DuckDBStagingDatabase.swift" "$PLUGIN/ParquetStaging.swift" "$PLUGIN/ParquetTypeMapper.swift" \ + "$WORK/main.swift" -o "$WORK/parquet-values" > "$WORK/compile.log" 2>&1 || { + echo "harness failed to compile:" >&2 + grep -E 'error:' "$WORK/compile.log" | sort -u | head -20 >&2 + exit 3 +} + +echo "Checking the Parquet export against $(basename "$LIB")" +"$WORK/parquet-values" "$WORK/probe.parquet" From f7eed7a7c0e350fc4ed88b07d2c2d363590376ce Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Tue, 10 Nov 2026 13:46:24 +0700 Subject: [PATCH 2/3] docs(changelog): add the fragments for #4174 --- apple/changelog.d/4174.fixed.1.md | 1 + apple/changelog.d/4174.fixed.10.md | 1 + apple/changelog.d/4174.fixed.11.md | 1 + apple/changelog.d/4174.fixed.12.md | 1 + apple/changelog.d/4174.fixed.13.md | 1 + apple/changelog.d/4174.fixed.14.md | 1 + apple/changelog.d/4174.fixed.15.md | 1 + apple/changelog.d/4174.fixed.16.md | 1 + apple/changelog.d/4174.fixed.17.md | 1 + apple/changelog.d/4174.fixed.18.md | 1 + apple/changelog.d/4174.fixed.19.md | 1 + apple/changelog.d/4174.fixed.2.md | 1 + apple/changelog.d/4174.fixed.20.md | 1 + apple/changelog.d/4174.fixed.21.md | 1 + apple/changelog.d/4174.fixed.22.md | 1 + apple/changelog.d/4174.fixed.23.md | 1 + apple/changelog.d/4174.fixed.24.md | 1 + apple/changelog.d/4174.fixed.25.md | 1 + apple/changelog.d/4174.fixed.26.md | 1 + apple/changelog.d/4174.fixed.27.md | 1 + apple/changelog.d/4174.fixed.28.md | 1 + apple/changelog.d/4174.fixed.29.md | 1 + apple/changelog.d/4174.fixed.3.md | 1 + apple/changelog.d/4174.fixed.30.md | 1 + apple/changelog.d/4174.fixed.31.md | 1 + apple/changelog.d/4174.fixed.32.md | 1 + apple/changelog.d/4174.fixed.33.md | 1 + apple/changelog.d/4174.fixed.34.md | 1 + apple/changelog.d/4174.fixed.35.md | 1 + apple/changelog.d/4174.fixed.36.md | 1 + apple/changelog.d/4174.fixed.37.md | 1 + apple/changelog.d/4174.fixed.38.md | 1 + apple/changelog.d/4174.fixed.39.md | 1 + apple/changelog.d/4174.fixed.4.md | 1 + apple/changelog.d/4174.fixed.40.md | 1 + apple/changelog.d/4174.fixed.41.md | 1 + apple/changelog.d/4174.fixed.42.md | 1 + apple/changelog.d/4174.fixed.43.md | 1 + apple/changelog.d/4174.fixed.44.md | 1 + apple/changelog.d/4174.fixed.45.md | 1 + apple/changelog.d/4174.fixed.46.md | 1 + apple/changelog.d/4174.fixed.47.md | 1 + apple/changelog.d/4174.fixed.48.md | 1 + apple/changelog.d/4174.fixed.49.md | 1 + apple/changelog.d/4174.fixed.5.md | 1 + apple/changelog.d/4174.fixed.50.md | 1 + apple/changelog.d/4174.fixed.51.md | 1 + apple/changelog.d/4174.fixed.52.md | 1 + apple/changelog.d/4174.fixed.6.md | 1 + apple/changelog.d/4174.fixed.7.md | 1 + apple/changelog.d/4174.fixed.8.md | 1 + apple/changelog.d/4174.fixed.9.md | 1 + apple/changelog.d/4174.fixed.md | 1 + apple/changelog.d/4174.security.1.md | 1 + apple/changelog.d/4174.security.2.md | 1 + apple/changelog.d/4174.security.3.md | 1 + apple/changelog.d/4174.security.md | 1 + 57 files changed, 57 insertions(+) create mode 100644 apple/changelog.d/4174.fixed.1.md create mode 100644 apple/changelog.d/4174.fixed.10.md create mode 100644 apple/changelog.d/4174.fixed.11.md create mode 100644 apple/changelog.d/4174.fixed.12.md create mode 100644 apple/changelog.d/4174.fixed.13.md create mode 100644 apple/changelog.d/4174.fixed.14.md create mode 100644 apple/changelog.d/4174.fixed.15.md create mode 100644 apple/changelog.d/4174.fixed.16.md create mode 100644 apple/changelog.d/4174.fixed.17.md create mode 100644 apple/changelog.d/4174.fixed.18.md create mode 100644 apple/changelog.d/4174.fixed.19.md create mode 100644 apple/changelog.d/4174.fixed.2.md create mode 100644 apple/changelog.d/4174.fixed.20.md create mode 100644 apple/changelog.d/4174.fixed.21.md create mode 100644 apple/changelog.d/4174.fixed.22.md create mode 100644 apple/changelog.d/4174.fixed.23.md create mode 100644 apple/changelog.d/4174.fixed.24.md create mode 100644 apple/changelog.d/4174.fixed.25.md create mode 100644 apple/changelog.d/4174.fixed.26.md create mode 100644 apple/changelog.d/4174.fixed.27.md create mode 100644 apple/changelog.d/4174.fixed.28.md create mode 100644 apple/changelog.d/4174.fixed.29.md create mode 100644 apple/changelog.d/4174.fixed.3.md create mode 100644 apple/changelog.d/4174.fixed.30.md create mode 100644 apple/changelog.d/4174.fixed.31.md create mode 100644 apple/changelog.d/4174.fixed.32.md create mode 100644 apple/changelog.d/4174.fixed.33.md create mode 100644 apple/changelog.d/4174.fixed.34.md create mode 100644 apple/changelog.d/4174.fixed.35.md create mode 100644 apple/changelog.d/4174.fixed.36.md create mode 100644 apple/changelog.d/4174.fixed.37.md create mode 100644 apple/changelog.d/4174.fixed.38.md create mode 100644 apple/changelog.d/4174.fixed.39.md create mode 100644 apple/changelog.d/4174.fixed.4.md create mode 100644 apple/changelog.d/4174.fixed.40.md create mode 100644 apple/changelog.d/4174.fixed.41.md create mode 100644 apple/changelog.d/4174.fixed.42.md create mode 100644 apple/changelog.d/4174.fixed.43.md create mode 100644 apple/changelog.d/4174.fixed.44.md create mode 100644 apple/changelog.d/4174.fixed.45.md create mode 100644 apple/changelog.d/4174.fixed.46.md create mode 100644 apple/changelog.d/4174.fixed.47.md create mode 100644 apple/changelog.d/4174.fixed.48.md create mode 100644 apple/changelog.d/4174.fixed.49.md create mode 100644 apple/changelog.d/4174.fixed.5.md create mode 100644 apple/changelog.d/4174.fixed.50.md create mode 100644 apple/changelog.d/4174.fixed.51.md create mode 100644 apple/changelog.d/4174.fixed.52.md create mode 100644 apple/changelog.d/4174.fixed.6.md create mode 100644 apple/changelog.d/4174.fixed.7.md create mode 100644 apple/changelog.d/4174.fixed.8.md create mode 100644 apple/changelog.d/4174.fixed.9.md create mode 100644 apple/changelog.d/4174.fixed.md create mode 100644 apple/changelog.d/4174.security.1.md create mode 100644 apple/changelog.d/4174.security.2.md create mode 100644 apple/changelog.d/4174.security.3.md create mode 100644 apple/changelog.d/4174.security.md diff --git a/apple/changelog.d/4174.fixed.1.md b/apple/changelog.d/4174.fixed.1.md new file mode 100644 index 0000000000..7b32f9032a --- /dev/null +++ b/apple/changelog.d/4174.fixed.1.md @@ -0,0 +1 @@ +A failed trigger edit could leave the trigger dropped with only a syntax error shown. diff --git a/apple/changelog.d/4174.fixed.10.md b/apple/changelog.d/4174.fixed.10.md new file mode 100644 index 0000000000..a9b360e1ad --- /dev/null +++ b/apple/changelog.d/4174.fixed.10.md @@ -0,0 +1 @@ +Edits left on a background tab were reapplied to a different result that arrived while it was hidden. diff --git a/apple/changelog.d/4174.fixed.11.md b/apple/changelog.d/4174.fixed.11.md new file mode 100644 index 0000000000..70e04d39ff --- /dev/null +++ b/apple/changelog.d/4174.fixed.11.md @@ -0,0 +1 @@ +Reopening a closed connection from a deeplink or history replaced its saved tabs with one tab. diff --git a/apple/changelog.d/4174.fixed.12.md b/apple/changelog.d/4174.fixed.12.md new file mode 100644 index 0000000000..ebed9075ef --- /dev/null +++ b/apple/changelog.d/4174.fixed.12.md @@ -0,0 +1 @@ +Cmd+Return right after typing in a very large script ran the text from before the last keystrokes. diff --git a/apple/changelog.d/4174.fixed.13.md b/apple/changelog.d/4174.fixed.13.md new file mode 100644 index 0000000000..da6a8cb61d --- /dev/null +++ b/apple/changelog.d/4174.fixed.13.md @@ -0,0 +1 @@ +Deleting a row with unsaved edits could delete a different row, or none. diff --git a/apple/changelog.d/4174.fixed.14.md b/apple/changelog.d/4174.fixed.14.md new file mode 100644 index 0000000000..8690e00b44 --- /dev/null +++ b/apple/changelog.d/4174.fixed.14.md @@ -0,0 +1 @@ +Redoing a cell edit on a row with another unsaved edit showed the value but never saved it. diff --git a/apple/changelog.d/4174.fixed.15.md b/apple/changelog.d/4174.fixed.15.md new file mode 100644 index 0000000000..725a75f238 --- /dev/null +++ b/apple/changelog.d/4174.fixed.15.md @@ -0,0 +1 @@ +Pasting spreadsheet rows whose first cell is empty shifted every value one column left. diff --git a/apple/changelog.d/4174.fixed.16.md b/apple/changelog.d/4174.fixed.16.md new file mode 100644 index 0000000000..d7534ba4c2 --- /dev/null +++ b/apple/changelog.d/4174.fixed.16.md @@ -0,0 +1 @@ +A save could fail on a unique constraint because edits ran before earlier deletes. diff --git a/apple/changelog.d/4174.fixed.17.md b/apple/changelog.d/4174.fixed.17.md new file mode 100644 index 0000000000..b37af636c9 --- /dev/null +++ b/apple/changelog.d/4174.fixed.17.md @@ -0,0 +1 @@ +Pasting or bulk-editing tens of thousands of rows froze the window for many seconds. diff --git a/apple/changelog.d/4174.fixed.18.md b/apple/changelog.d/4174.fixed.18.md new file mode 100644 index 0000000000..01b0fcab57 --- /dev/null +++ b/apple/changelog.d/4174.fixed.18.md @@ -0,0 +1 @@ +Editing a MySQL or MariaDB BIT cell saved the wrong value: 5 was stored as 53. diff --git a/apple/changelog.d/4174.fixed.19.md b/apple/changelog.d/4174.fixed.19.md new file mode 100644 index 0000000000..498b95f1b1 --- /dev/null +++ b/apple/changelog.d/4174.fixed.19.md @@ -0,0 +1 @@ +A SQL dump of a MySQL BIT column restored different values than the table held. diff --git a/apple/changelog.d/4174.fixed.2.md b/apple/changelog.d/4174.fixed.2.md new file mode 100644 index 0000000000..39388a2f8d --- /dev/null +++ b/apple/changelog.d/4174.fixed.2.md @@ -0,0 +1 @@ +Rebuilding a SQLite table with a transaction open in the editor deleted its child rows. diff --git a/apple/changelog.d/4174.fixed.20.md b/apple/changelog.d/4174.fixed.20.md new file mode 100644 index 0000000000..1438f634be --- /dev/null +++ b/apple/changelog.d/4174.fixed.20.md @@ -0,0 +1 @@ +A transaction opened in a tab on another PostgreSQL database was silently rolled back after a few minutes. diff --git a/apple/changelog.d/4174.fixed.21.md b/apple/changelog.d/4174.fixed.21.md new file mode 100644 index 0000000000..69cafa86c0 --- /dev/null +++ b/apple/changelog.d/4174.fixed.21.md @@ -0,0 +1 @@ +After a dropped connection reconnected, statements ran outside the transaction that was lost, with no error. diff --git a/apple/changelog.d/4174.fixed.22.md b/apple/changelog.d/4174.fixed.22.md new file mode 100644 index 0000000000..d1db1cfffd --- /dev/null +++ b/apple/changelog.d/4174.fixed.22.md @@ -0,0 +1 @@ +Kafka topics and SurrealDB tables were browsed with a generic SQL SELECT instead of the driver's own query. diff --git a/apple/changelog.d/4174.fixed.23.md b/apple/changelog.d/4174.fixed.23.md new file mode 100644 index 0000000000..29f9c1c36c --- /dev/null +++ b/apple/changelog.d/4174.fixed.23.md @@ -0,0 +1 @@ +A DuckDB export or table copy of more than 5,000,000 rows stopped at 5,000,000 and reported success. diff --git a/apple/changelog.d/4174.fixed.24.md b/apple/changelog.d/4174.fixed.24.md new file mode 100644 index 0000000000..25747aa955 --- /dev/null +++ b/apple/changelog.d/4174.fixed.24.md @@ -0,0 +1 @@ +A DuckDB query over a very large result held every row in memory twice before showing the first page. diff --git a/apple/changelog.d/4174.fixed.25.md b/apple/changelog.d/4174.fixed.25.md new file mode 100644 index 0000000000..b4583e24fe --- /dev/null +++ b/apple/changelog.d/4174.fixed.25.md @@ -0,0 +1 @@ +Exporting an empty PostgreSQL table to CSV with field names wrote no header line. diff --git a/apple/changelog.d/4174.fixed.26.md b/apple/changelog.d/4174.fixed.26.md new file mode 100644 index 0000000000..93a96b2b07 --- /dev/null +++ b/apple/changelog.d/4174.fixed.26.md @@ -0,0 +1 @@ +Copy DDL and SQL export recreated a PostgreSQL partitioned or unlogged table as a plain table. diff --git a/apple/changelog.d/4174.fixed.27.md b/apple/changelog.d/4174.fixed.27.md new file mode 100644 index 0000000000..499934d407 --- /dev/null +++ b/apple/changelog.d/4174.fixed.27.md @@ -0,0 +1 @@ +Editing a field inside an Elasticsearch object left the old value in the document beside the new one. diff --git a/apple/changelog.d/4174.fixed.28.md b/apple/changelog.d/4174.fixed.28.md new file mode 100644 index 0000000000..1dda16bc44 --- /dev/null +++ b/apple/changelog.d/4174.fixed.28.md @@ -0,0 +1 @@ +Adding an Elasticsearch row with an _id that exists replaced the existing document. diff --git a/apple/changelog.d/4174.fixed.29.md b/apple/changelog.d/4174.fixed.29.md new file mode 100644 index 0000000000..152ffc9d29 --- /dev/null +++ b/apple/changelog.d/4174.fixed.29.md @@ -0,0 +1 @@ +Editing a Redis string value in the grid removed the key's TTL. diff --git a/apple/changelog.d/4174.fixed.3.md b/apple/changelog.d/4174.fixed.3.md new file mode 100644 index 0000000000..2f591471aa --- /dev/null +++ b/apple/changelog.d/4174.fixed.3.md @@ -0,0 +1 @@ +A structure save that failed part way on MySQL left edits staged that could not be saved again. diff --git a/apple/changelog.d/4174.fixed.30.md b/apple/changelog.d/4174.fixed.30.md new file mode 100644 index 0000000000..ddf1b6aa24 --- /dev/null +++ b/apple/changelog.d/4174.fixed.30.md @@ -0,0 +1 @@ +Renaming a Redis key or adding a string key overwrote an existing key of that name. diff --git a/apple/changelog.d/4174.fixed.31.md b/apple/changelog.d/4174.fixed.31.md new file mode 100644 index 0000000000..a8b1084c8d --- /dev/null +++ b/apple/changelog.d/4174.fixed.31.md @@ -0,0 +1 @@ +Editing a MongoDB string field to text that looks like a number or boolean changed its type. diff --git a/apple/changelog.d/4174.fixed.32.md b/apple/changelog.d/4174.fixed.32.md new file mode 100644 index 0000000000..68b3a5718b --- /dev/null +++ b/apple/changelog.d/4174.fixed.32.md @@ -0,0 +1 @@ +A long Snowflake statement was run a second time when the session expired while it ran. diff --git a/apple/changelog.d/4174.fixed.33.md b/apple/changelog.d/4174.fixed.33.md new file mode 100644 index 0000000000..a35b691db6 --- /dev/null +++ b/apple/changelog.d/4174.fixed.33.md @@ -0,0 +1 @@ +A Snowflake statement still running after 45 minutes was reported as failed but kept running on the server. diff --git a/apple/changelog.d/4174.fixed.34.md b/apple/changelog.d/4174.fixed.34.md new file mode 100644 index 0000000000..b7b907940f --- /dev/null +++ b/apple/changelog.d/4174.fixed.34.md @@ -0,0 +1 @@ +Editing a string in an untyped SurrealDB field to text that looks like a number changed its type. diff --git a/apple/changelog.d/4174.fixed.35.md b/apple/changelog.d/4174.fixed.35.md new file mode 100644 index 0000000000..cdce19e73a --- /dev/null +++ b/apple/changelog.d/4174.fixed.35.md @@ -0,0 +1 @@ +Renaming an etcd key or adding one overwrote an existing key, and a rename took two separate steps. diff --git a/apple/changelog.d/4174.fixed.36.md b/apple/changelog.d/4174.fixed.36.md new file mode 100644 index 0000000000..e3be20fbb0 --- /dev/null +++ b/apple/changelog.d/4174.fixed.36.md @@ -0,0 +1 @@ +Parquet export wrote wrong bytes for binary values and NULL for an empty one. diff --git a/apple/changelog.d/4174.fixed.37.md b/apple/changelog.d/4174.fixed.37.md new file mode 100644 index 0000000000..f8b5f1f1a5 --- /dev/null +++ b/apple/changelog.d/4174.fixed.37.md @@ -0,0 +1 @@ +Parquet export wrote NULL for values that did not fit the column type, with no warning. diff --git a/apple/changelog.d/4174.fixed.38.md b/apple/changelog.d/4174.fixed.38.md new file mode 100644 index 0000000000..3ccebda4fa --- /dev/null +++ b/apple/changelog.d/4174.fixed.38.md @@ -0,0 +1 @@ +XLSX export turned text such as 1E5 or Infinity into numbers and lost digits of long ids. diff --git a/apple/changelog.d/4174.fixed.39.md b/apple/changelog.d/4174.fixed.39.md new file mode 100644 index 0000000000..e3b4cefc90 --- /dev/null +++ b/apple/changelog.d/4174.fixed.39.md @@ -0,0 +1 @@ +Copying a PostgreSQL text array into a JSON column wrote invalid JSON or turned strings into numbers. diff --git a/apple/changelog.d/4174.fixed.4.md b/apple/changelog.d/4174.fixed.4.md new file mode 100644 index 0000000000..8106e3e0d2 --- /dev/null +++ b/apple/changelog.d/4174.fixed.4.md @@ -0,0 +1 @@ +A sidebar command on another database's table ran on the current database when the switch failed. diff --git a/apple/changelog.d/4174.fixed.40.md b/apple/changelog.d/4174.fixed.40.md new file mode 100644 index 0000000000..0575f765cc --- /dev/null +++ b/apple/changelog.d/4174.fixed.40.md @@ -0,0 +1 @@ +CSV import silently dropped extra fields. diff --git a/apple/changelog.d/4174.fixed.41.md b/apple/changelog.d/4174.fixed.41.md new file mode 100644 index 0000000000..8b9c8dcbf4 --- /dev/null +++ b/apple/changelog.d/4174.fixed.41.md @@ -0,0 +1 @@ +A write that outlasted the MCP or assistant timeout was reported as failed although it was applied. diff --git a/apple/changelog.d/4174.fixed.42.md b/apple/changelog.d/4174.fixed.42.md new file mode 100644 index 0000000000..968374dc11 --- /dev/null +++ b/apple/changelog.d/4174.fixed.42.md @@ -0,0 +1 @@ +Stopping the assistant during a tool call broke the conversation; every later message failed. diff --git a/apple/changelog.d/4174.fixed.43.md b/apple/changelog.d/4174.fixed.43.md new file mode 100644 index 0000000000..0c3173d455 --- /dev/null +++ b/apple/changelog.d/4174.fixed.43.md @@ -0,0 +1 @@ +An error sent mid-stream by an OpenAI-compatible or Gemini provider ended the reply silently. diff --git a/apple/changelog.d/4174.fixed.44.md b/apple/changelog.d/4174.fixed.44.md new file mode 100644 index 0000000000..e61fac71f0 --- /dev/null +++ b/apple/changelog.d/4174.fixed.44.md @@ -0,0 +1 @@ +An outside MCP server with long or dotted tool names made every chat request on its connections fail. diff --git a/apple/changelog.d/4174.fixed.45.md b/apple/changelog.d/4174.fixed.45.md new file mode 100644 index 0000000000..f8e760a269 --- /dev/null +++ b/apple/changelog.d/4174.fixed.45.md @@ -0,0 +1 @@ +One unreadable saved MCP token, or a failed keychain read, removed every paired client's token. diff --git a/apple/changelog.d/4174.fixed.46.md b/apple/changelog.d/4174.fixed.46.md new file mode 100644 index 0000000000..86592e895a --- /dev/null +++ b/apple/changelog.d/4174.fixed.46.md @@ -0,0 +1 @@ +Items deleted on another Mac while their sync category was off stayed here and could come back everywhere. diff --git a/apple/changelog.d/4174.fixed.47.md b/apple/changelog.d/4174.fixed.47.md new file mode 100644 index 0000000000..8e714b5ba6 --- /dev/null +++ b/apple/changelog.d/4174.fixed.47.md @@ -0,0 +1 @@ +Saving a password failed silently when the Keychain held both a synced and a local copy for the connection. diff --git a/apple/changelog.d/4174.fixed.48.md b/apple/changelog.d/4174.fixed.48.md new file mode 100644 index 0000000000..85ee0d29b1 --- /dev/null +++ b/apple/changelog.d/4174.fixed.48.md @@ -0,0 +1 @@ +Saving a connection deleted its API token, TOTP secret or key passphrase when the Keychain could not be read. diff --git a/apple/changelog.d/4174.fixed.49.md b/apple/changelog.d/4174.fixed.49.md new file mode 100644 index 0000000000..d73bb21e31 --- /dev/null +++ b/apple/changelog.d/4174.fixed.49.md @@ -0,0 +1 @@ +Format Query put spaces around the @ of a MySQL account name or quoted user variable. diff --git a/apple/changelog.d/4174.fixed.5.md b/apple/changelog.d/4174.fixed.5.md new file mode 100644 index 0000000000..828ba5f456 --- /dev/null +++ b/apple/changelog.d/4174.fixed.5.md @@ -0,0 +1 @@ +Shortcuts Add Rows inserted empty CSV cells as empty strings instead of NULL or the column default. diff --git a/apple/changelog.d/4174.fixed.50.md b/apple/changelog.d/4174.fixed.50.md new file mode 100644 index 0000000000..f9d67c7086 --- /dev/null +++ b/apple/changelog.d/4174.fixed.50.md @@ -0,0 +1 @@ +Format Query split PostgreSQL operators such as pgvector's <#> or pg_trgm's <%. diff --git a/apple/changelog.d/4174.fixed.51.md b/apple/changelog.d/4174.fixed.51.md new file mode 100644 index 0000000000..8f7d6e65bb --- /dev/null +++ b/apple/changelog.d/4174.fixed.51.md @@ -0,0 +1 @@ +Query parameters mistook Snowflake paths, array slices or ClickHouse typed slots for :name parameters. diff --git a/apple/changelog.d/4174.fixed.52.md b/apple/changelog.d/4174.fixed.52.md new file mode 100644 index 0000000000..293fec36c7 --- /dev/null +++ b/apple/changelog.d/4174.fixed.52.md @@ -0,0 +1 @@ +Keyword casing rewrote words inside string literals and in non-SQL editors such as MongoDB. diff --git a/apple/changelog.d/4174.fixed.6.md b/apple/changelog.d/4174.fixed.6.md new file mode 100644 index 0000000000..701e1306da --- /dev/null +++ b/apple/changelog.d/4174.fixed.6.md @@ -0,0 +1 @@ +A query approved in Safe Mode after switching tabs ran in the newly selected tab's database. diff --git a/apple/changelog.d/4174.fixed.7.md b/apple/changelog.d/4174.fixed.7.md new file mode 100644 index 0000000000..0d1740e6a6 --- /dev/null +++ b/apple/changelog.d/4174.fixed.7.md @@ -0,0 +1 @@ +Saving, switching tabs during the Safe Mode prompt, then approving wiped the other tab's edits. diff --git a/apple/changelog.d/4174.fixed.8.md b/apple/changelog.d/4174.fixed.8.md new file mode 100644 index 0000000000..c5a54bf554 --- /dev/null +++ b/apple/changelog.d/4174.fixed.8.md @@ -0,0 +1 @@ +Choosing Save on a second closing tab while the first was still saving left the first tab unclosable. diff --git a/apple/changelog.d/4174.fixed.9.md b/apple/changelog.d/4174.fixed.9.md new file mode 100644 index 0000000000..f4feb7b92b --- /dev/null +++ b/apple/changelog.d/4174.fixed.9.md @@ -0,0 +1 @@ +Running a query over an edited result discarded the unsaved edits without asking. diff --git a/apple/changelog.d/4174.fixed.md b/apple/changelog.d/4174.fixed.md new file mode 100644 index 0000000000..4f7fc4bd31 --- /dev/null +++ b/apple/changelog.d/4174.fixed.md @@ -0,0 +1 @@ +Editing a MySQL or MariaDB trigger dropped it and ran its body as a statement. diff --git a/apple/changelog.d/4174.security.1.md b/apple/changelog.d/4174.security.1.md new file mode 100644 index 0000000000..b17e025dc0 --- /dev/null +++ b/apple/changelog.d/4174.security.1.md @@ -0,0 +1 @@ +MCP argument completion listed a connection's databases and tables to a client that was not approved for it. diff --git a/apple/changelog.d/4174.security.2.md b/apple/changelog.d/4174.security.2.md new file mode 100644 index 0000000000..5bd754589b --- /dev/null +++ b/apple/changelog.d/4174.security.2.md @@ -0,0 +1 @@ +Deleting the connections.json tag file let an edited connection list run its password and tunnel commands. diff --git a/apple/changelog.d/4174.security.3.md b/apple/changelog.d/4174.security.3.md new file mode 100644 index 0000000000..8aae7af173 --- /dev/null +++ b/apple/changelog.d/4174.security.3.md @@ -0,0 +1 @@ +A Local only connection's newly saved passwords and keys still synced through iCloud Keychain. diff --git a/apple/changelog.d/4174.security.md b/apple/changelog.d/4174.security.md new file mode 100644 index 0000000000..b3644e2f60 --- /dev/null +++ b/apple/changelog.d/4174.security.md @@ -0,0 +1 @@ +Claude Agent could reach every saved connection through MCP instead of only the one the chat is on. From 6d0e854133092de16ff2c63ae13d2a341415348e Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Tue, 10 Nov 2026 17:54:19 +0700 Subject: [PATCH 3/3] fix(ios): add the CSV field count error to the iOS string catalog --- .../TableProMobile/Localizable.xcstrings | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/apple/TableProMobile/TableProMobile/Localizable.xcstrings b/apple/TableProMobile/TableProMobile/Localizable.xcstrings index 80019d348c..35d9ee1a79 100644 --- a/apple/TableProMobile/TableProMobile/Localizable.xcstrings +++ b/apple/TableProMobile/TableProMobile/Localizable.xcstrings @@ -18638,6 +18638,16 @@ } } }, + "Record %lld of the CSV data has %lld fields, but the header has %lld." : { + "localizations" : { + "en" : { + "stringUnit" : { + "state" : "new", + "value" : "Record %1$lld of the CSV data has %2$lld fields, but the header has %3$lld." + } + } + } + }, "Records" : { "extractionState" : "manual", "generatesSymbol" : false