From 5f6332f97d482caf234e2dc57eb2fa47902030cc Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 11 Nov 2026 14:39:10 +0700 Subject: [PATCH 1/7] fix(macos): keep T-SQL blocks and rewinds whole, stop CSV sanitizing by default, and fix Stop, defaults and gate cancels across drivers --- .../CassandraConditionalWrite.swift | 41 +++ .../RedisCommandSplitter.swift | 11 + .../SQLParenthesizedSemicolons.swift | 42 +++ .../SQLRoutineBodyTracker.swift | 124 +++++++-- .../SQLUnterminatedStatements.swift | 25 +- .../SqlBlockStructure.swift | 37 ++- .../Sources/TableProTeradataCore/Parcel.swift | 2 + .../TeradataConnection.swift | 204 +++++++++++++- .../TeradataMessages.swift | 8 + .../CassandraConditionalWriteTests.swift | 61 +++++ .../PluginParameterEscapingTests.swift | 11 + .../RedisCommandSplitterTests.swift | 12 + .../SQLParenthesizedSemicolonTests.swift | 77 ++++++ .../SQLStatementCursorIndexTests.swift | 13 +- .../TSQLControlFlowBlockTests.swift | 153 +++++++++++ .../TeradataStopTests.swift | 249 ++++++++++++++++++ .../TextViewController+EmphasizeBracket.swift | 17 +- .../Controller/TextViewControllerTests.swift | 18 ++ .../BeancountPluginDriver.swift | 6 +- .../BigtablePluginDriver+Execution.swift | 6 +- .../CSVExportPlugin/CSVExportModels.swift | 3 +- .../CSVExportOptionsView.swift | 6 +- .../CSVExportPlugin/CSVExportPlugin.swift | 95 +++++-- .../CSVImportPlugin/CSVImportPlugin.swift | 30 ++- .../CassandraPlugin.swift | 4 +- .../CassandraRowWriter.swift | 39 +-- .../DamengDriverPlugin/DamengPlugin.swift | 7 +- .../DynamoDBPluginDriver+Execution.swift | 4 +- .../EtcdDriverPlugin/EtcdPluginDriver.swift | 8 +- .../CSVExportFormulaAndDecimalTests.swift | 174 ++++++++++++ .../ExportImportRoundTripTests.swift | 17 +- .../FormatSettingsChangedMidRunTests.swift | 185 +++++++++++++ .../JSONExportPlugin/JSONExportPlugin.swift | 13 +- .../MarkdownExportPlugin.swift | 17 +- .../MongoDBConnection+SyncHelpers.swift | 5 +- .../MongoDBPluginDriver.swift | 7 +- .../MariaDBPluginConnection.swift | 36 ++- .../MariaDBStopThreadIdTests.swift | 38 +++ .../ParquetExportPlugin.swift | 9 +- .../RedisPluginDriver+Operations.swift | 5 +- .../RedisDriverPlugin/RedisPluginDriver.swift | 6 +- .../RedisConditionalWriteTests.swift | 19 ++ .../SQLImportPlugin/SQLImportPlugin.swift | 14 +- .../PluginDatabaseDriver.swift | 10 +- .../PluginExportUtilities.swift | 66 +++++ .../TableProPluginKit/PluginRowWriters.swift | 23 +- .../TeradataAsyncConnection.swift | 18 +- .../XLSXExportPlugin/XLSXExportPlugin.swift | 30 ++- .../Plugins/XLSXExportPlugin/XLSXWriter.swift | 63 +---- .../XLSXImportPlugin/XLSXImportPlugin.swift | 24 +- .../ChangeTracking/DataChangeModels.swift | 8 +- .../Core/ChangeTracking/DefaultedCells.swift | 38 +++ .../SQLStatementGenerator.swift | 7 +- .../Core/Compare/CompareSQLLiteral.swift | 22 +- .../Core/Compare/DataDiffEngine.swift | 26 +- .../Core/Compare/StreamingRowProvider.swift | 20 +- .../Coordinators/RowEditingCoordinator.swift | 6 +- .../DataWrite/Rewind/RewindExecutor.swift | 3 +- .../Core/DataWrite/Rewind/RewindPlanner.swift | 96 +++++-- .../DataWrite/RowChangeStatementFactory.swift | 111 +++++++- .../DataWrite/RowWriteOperationBuilder.swift | 9 +- .../DatabaseManager+ContainerDDL.swift | 36 +-- .../Database/DatabaseManager+Health.swift | 42 ++- .../Database/DatabaseManager+Principals.swift | 12 +- .../Database/DatabaseManager+Schema.swift | 14 +- .../Database/DatabaseManager+Sessions.swift | 5 +- .../Database/DatabaseManager+Startup.swift | 102 ++++++- .../Core/Database/DatabaseManager.swift | 4 + .../Core/Database/TriggerEditing.swift | 19 +- apple/TablePro/Core/LSP/LSPTransport.swift | 72 ++++- .../Core/ObjectCopy/ObjectCopyRowCopier.swift | 12 +- .../Core/ObjectCopy/ObjectCopyRunner.swift | 16 +- .../Plugins/ExportDataSourceAdapter.swift | 8 +- .../TablePro/Core/Plugins/ExportRowCut.swift | 11 + .../Execution/GatedOperationFailure.swift | 39 +++ .../Query/MetadataConnectionPool.swift | 18 +- .../Services/Query/RowOperationsManager.swift | 59 +++-- .../Core/Services/Query/RowParser.swift | 10 +- .../Query/SelectionSummaryCalculator.swift | 14 +- .../Shortcuts/Shared/PayloadRowPlanner.swift | 50 +++- .../Core/Shortcuts/Shared/RowPayload.swift | 20 +- .../Core/Shortcuts/ShortcutsRowWriter.swift | 5 +- .../UsersRoles/PrincipalChangeManager.swift | 17 +- .../JavaScript/QueryStatementModel.swift | 52 +++- .../Utilities/SQL/DisplayedResultReader.swift | 13 +- .../Core/Utilities/SQL/JsonRowConverter.swift | 2 +- .../Core/Utilities/SQL/QueryClassifier.swift | 13 +- .../Utilities/SQL/ResultJsonSerializer.swift | 2 + .../SQL/SQLRowToStatementConverter.swift | 14 +- .../Core/Vim/VimTextBufferAdapter.swift | 59 ----- .../PluginCellValue+DefaultMarker.swift | 19 +- .../ClickHouse/ClickHousePartInfo.swift | 4 +- .../Models/Query/SelectionSummary.swift | 10 +- .../Schema/ClickHousePartStatements.swift | 46 ++-- .../Models/UI/MultiRowEditState.swift | 8 +- .../TablePro/Resources/Localizable.xcstrings | 41 +++ .../ViewModels/SchemaEditorViewModel.swift | 2 +- .../ViewModels/UserTypeEditorModel.swift | 2 +- .../UsersRolesViewModel+Changes.swift | 2 +- .../Views/Compare/CompareRowDiffPane.swift | 15 ++ .../DataGrid/Cells/DataGridCellContent.swift | 3 +- .../Cells/DataGridCellPresenter.swift | 2 + .../Commands/DataGridCommands+Copy.swift | 1 + .../Commands/DataGridCommands+SetValue.swift | 4 +- .../DataGrid/DataGridViewController.swift | 3 +- .../CreateDatabaseSheet.swift | 2 +- .../Views/Editor/StatementRunController.swift | 50 +++- .../Views/Main/Child/DataTabSQLExporter.swift | 10 +- .../Main/Child/MainEditorContentView.swift | 1 + .../MainContentCoordinator+Navigation.swift | 11 +- ...ntentCoordinator+SafeModePromptRetry.swift | 28 ++ .../Main/MainContentCommandActions.swift | 67 +++-- .../Views/Main/MainContentCoordinator.swift | 9 +- .../ObjectCopy/CopyObjectsResultView.swift | 5 + .../UserType/CreateUserTypeSheet.swift | 9 +- .../Views/Results/ResultsJsonView.swift | 9 + .../Views/Sidebar/EditSequenceSheet.swift | 2 +- .../Views/Sidebar/ObjectCommentSheet.swift | 4 +- .../Views/Sidebar/SetSequenceValueSheet.swift | 2 +- .../Views/Structure/ClickHousePartsView.swift | 217 +++++++++------ .../Views/Structure/CreateTableDraft.swift | 15 +- .../StructureEditingSession+Apply.swift | 51 +--- ...tructureEditingSession+ColumnReorder.swift | 4 +- .../Views/Structure/TriggerDetailView.swift | 2 +- .../Views/Structure/TriggerEditorView.swift | 18 +- .../Completion/QueryCompletionCatalog.swift | 51 +++- .../QueryCompletionCatalogTests.swift | 29 ++ .../SQLStatementGeneratorBitColumnTests.swift | 4 +- .../SQLStatementGeneratorRowMatchTests.swift | 6 +- .../SQLStatementGeneratorTests.swift | 69 ++++- .../Compare/CompareDriverRowCutTests.swift | 90 +++++++ .../Core/Compare/CompareSQLLiteralTests.swift | 27 +- .../Compare/DataSyncScriptBuilderTests.swift | 2 +- .../RowEditingCoordinatorUUIDKeyTests.swift | 2 +- .../Core/DataWrite/RewindPlannerTests.swift | 81 ++++++ ...wChangeStatementFactoryCoverageTests.swift | 4 +- ...hangeStatementFactoryExpressionTests.swift | 84 +++++- .../RowWriteOperationBuilderTests.swift | 41 ++- .../Core/Database/DatabaseManagerTests.swift | 5 +- .../Database/SchemaWriteGateDenialTests.swift | 86 ++++++ .../Database/StartupCommandWarningTests.swift | 135 ++++++++++ .../Database/TriggerInfoMappingTests.swift | 86 ++++++ .../LSP/LSPTransportBrokenPipeTests.swift | 48 ++++ .../ObjectCopy/ObjectCopyRowCutTests.swift | 107 ++++++++ .../Query/MetadataConnectionPoolTests.swift | 21 +- .../SelectionSummaryCalculatorTests.swift | 17 +- .../RowOperationsFieldAbsenceTests.swift | 4 +- .../RowOperationsManagerPasteTests.swift | 18 +- .../Services/RowOperationsManagerTests.swift | 44 +++- .../RowOperationsManagerUUIDKeyTests.swift | 25 +- .../Services/RowPasteForeignTextTests.swift | 16 +- .../PayloadRowPlannerHeaderMatchTests.swift | 59 +++++ .../Shortcuts/RowPayloadFormatTests.swift | 60 +++++ .../PrincipalApplyOpenTransactionTests.swift | 44 ++++ .../PrincipalChangeManagerTests.swift | 40 +++ .../Utilities/JsonRowConverterTests.swift | 18 +- .../SQL/QueryClassifierTSQLBlockTests.swift | 39 +++ .../SQLRowToStatementConverterTests.swift | 31 ++- .../Vim/VimTextBufferAdapterPerfTests.swift | 103 ++------ .../Helpers/StubExecutionGates.swift | 9 +- .../Models/MultiRowEditStateTests.swift | 23 +- .../ClickHousePartStatementsTests.swift | 81 ++++-- .../Editor/StatementCaretTableTests.swift | 109 ++++++++ .../TableLoadAfterSafeModePromptTests.swift | 40 +++ .../WindowCloseSaveEveryConnectionTests.swift | 129 +++++++++ .../Views/Results/ResultsJsonViewTests.swift | 24 ++ .../SelectionSummaryTrackerTests.swift | 2 +- .../Structure/ClickHousePartsGridTests.swift | 101 ++++++- .../StructureListGridAccessibilityTests.swift | 3 +- .../StructureSaveConfirmationTests.swift | 10 +- .../TriggerEditorFontSizeTests.swift | 25 ++ core/crates/core/src/export/csv.rs | 55 ++-- core/crates/core/src/export/csv_options.rs | 4 +- core/crates/core/src/export/encoder/tests.rs | 6 +- core/crates/mcp/src/export_tools.rs | 10 +- .../storage/src/query_history/export.rs | 6 +- .../src/conformance/cases/large_table.rs | 9 +- .../src/conformance/cases/round_trip.rs | 8 +- .../crates/app/src/ui/export/encoding_sink.rs | 2 +- .../app/src/ui/export/export_ledger/tests.rs | 12 +- linux/crates/platform/tests/settings.rs | 2 +- .../data/app.tablepro.TablePro.gschema.xml.in | 2 +- linux/docs/storage.md | 2 +- 183 files changed, 4958 insertions(+), 1068 deletions(-) create mode 100644 apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift create mode 100644 apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLParenthesizedSemicolons.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLParenthesizedSemicolonTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift create mode 100644 apple/Plugins/FormatPluginTests/CSVExportFormulaAndDecimalTests.swift create mode 100644 apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift create mode 100644 apple/Plugins/MySQLDriverTests/MariaDBStopThreadIdTests.swift create mode 100644 apple/TablePro/Core/ChangeTracking/DefaultedCells.swift create mode 100644 apple/TablePro/Core/Services/Execution/GatedOperationFailure.swift create mode 100644 apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SafeModePromptRetry.swift create mode 100644 apple/TableProTests/Core/Compare/CompareDriverRowCutTests.swift create mode 100644 apple/TableProTests/Core/Database/SchemaWriteGateDenialTests.swift create mode 100644 apple/TableProTests/Core/Database/StartupCommandWarningTests.swift create mode 100644 apple/TableProTests/Core/LSP/LSPTransportBrokenPipeTests.swift create mode 100644 apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCutTests.swift create mode 100644 apple/TableProTests/Core/Shortcuts/PayloadRowPlannerHeaderMatchTests.swift create mode 100644 apple/TableProTests/Core/Shortcuts/RowPayloadFormatTests.swift create mode 100644 apple/TableProTests/Core/Utilities/SQL/QueryClassifierTSQLBlockTests.swift create mode 100644 apple/TableProTests/Views/Editor/StatementCaretTableTests.swift create mode 100644 apple/TableProTests/Views/Main/TableLoadAfterSafeModePromptTests.swift create mode 100644 apple/TableProTests/Views/Main/WindowCloseSaveEveryConnectionTests.swift create mode 100644 apple/TableProTests/Views/Structure/TriggerEditorFontSizeTests.swift diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift new file mode 100644 index 0000000000..a4772dda72 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift @@ -0,0 +1,41 @@ +// +// CassandraConditionalWrite.swift +// TableProCassandraCore +// + +import Foundation + +// A grid write names its row by key and is conditioned on it: an INSERT is an upsert, and an UPDATE writes its cells +// whether or not the row is there, so either would otherwise replace or recreate a row the grid never showed. +// A condition that fails answers one row whose `[applied]` is false, which the server reports as a success. +public enum CassandraConditionalWrite { + public static let appliedColumn = "[applied]" + public static let insertCondition = "IF NOT EXISTS" + public static let updateCondition = "IF EXISTS" + + public enum Refusal: Sendable, Equatable { + case keyTaken + case rowMissing + } + + public static func insertStatement(target: String, names: [String], values: [String]) -> String { + "INSERT INTO \(target) (\(names.joined(separator: ", "))) VALUES (\(values.joined(separator: ", "))) " + + insertCondition + } + + public static func updateStatement(target: String, assignments: [String], keyCondition: String) -> String { + "UPDATE \(target) SET \(assignments.joined(separator: ", ")) WHERE \(keyCondition) \(updateCondition)" + } + + public static func refusal(statement: String, firstColumn: String?, firstValue: String?) -> Refusal? { + guard firstColumn == appliedColumn, firstValue?.lowercased() == "false" else { return nil } + let normalized = statement.uppercased() + if normalized.hasPrefix("INSERT"), normalized.hasSuffix(" " + insertCondition) { + return .keyTaken + } + if normalized.hasPrefix("UPDATE"), normalized.hasSuffix(" " + updateCondition) { + return .rowMissing + } + return nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisCommandSplitter.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisCommandSplitter.swift index 8d42794c8d..d92e7bc4bc 100644 --- a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisCommandSplitter.swift +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisCommandSplitter.swift @@ -32,6 +32,17 @@ public enum RedisCommandSplitter { return nil } + // Every command with the last caret offset `command(at:in:)` gives it, for a caller that keeps them between + // caret moves. + public static func commandsWithCaretReach(in text: String) -> [(command: Command, reach: Int)] { + var lexer = RedisCommandLexer(text, endsCommands: true, decodesArguments: false) + var commands: [(command: Command, reach: Int)] = [] + while let found = lexer.next() { + commands.append((command: command(found, in: text), reach: found.reach)) + } + return commands + } + private static func command(_ found: RedisCommandLexer.Command, in text: String) -> Command { let range = NSRange(location: found.start, length: found.end - found.start) return Command(text: (text as NSString).substring(with: range), range: range) diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLParenthesizedSemicolons.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLParenthesizedSemicolons.swift new file mode 100644 index 0000000000..d5d1607083 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLParenthesizedSemicolons.swift @@ -0,0 +1,42 @@ +import Foundation + +// The scanner keeps the `;`s of a PostgreSQL rule's `DO ALSO (...; ...)` in its statement. A client that cuts at every +// `;` runs each piece, so a gate tiers them too: `DO ALSO (SELECT 1; DELETE FROM t)` by its `DELETE`. +enum SQLParenthesizedSemicolons { + static func pieces(in statement: String, grammar: SQLLexicalGrammar) -> [String] { + guard statement.utf16.contains(SqlLexer.semicolon) else { return [] } + let text = statement as NSString + let length = text.length + var cuts: [Int] = [] + var depth = 0 + var index = 0 + while index < length { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + index = max(span.end, index + 1) + continue + } + switch text.character(at: index) { + case SqlLexer.openParen: + depth += 1 + case SqlLexer.closeParen: + depth = max(0, depth - 1) + case SqlLexer.semicolon where depth > 0: + cuts.append(index) + default: + break + } + index += 1 + } + guard !cuts.isEmpty else { return [] } + var pieces: [String] = [] + var start = 0 + for cut in cuts + [length] { + let piece = StatementBlank.trimming(text.substring(with: NSRange(location: start, length: cut - start))) + if !piece.isEmpty { + pieces.append(piece) + } + start = cut + 1 + } + return pieces + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift index f1596e7ceb..d4abc7cc10 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift @@ -2,13 +2,26 @@ import Foundation /// Statement boundaries for the dialects whose only `;`-holding construct is a routine body written `BEGIN ... END`. /// -/// A `BEGIN` opens a body only inside a statement that defines a routine, for the safety reason recorded on +/// Outside T-SQL a `BEGIN` opens a body only inside a statement that defines a routine, for the safety reason recorded on /// ``SqlBlockStructure/opensRoutineDefinition(_:)``. The `;` belongs to the statement only after a T-SQL `MERGE`, which /// ``SQLLexicalGrammar/terminatedMergeStatements`` asks for: these engines accept every other statement without it. /// /// `BEGIN`, `CASE` and `END` count only where a keyword can stand: an engine that does not reserve them takes them as -/// names, and a block a name opened would never close. A text left inside one is refused by the external gate. +/// names, and a block a name opened would never close. T-SQL reserves all three, so there a block also opens outside +/// a routine, as in `IF @x = 1 BEGIN ... END` and `BEGIN TRY ... END TRY BEGIN CATCH ... END CATCH`. A text left +/// inside one is refused by the external gate. +/// +/// A `;` inside parentheses ends the statement, so a stray `(` cannot swallow what follows it, except in a PostgreSQL +/// rule's action list, `DO ALSO (INSERT ...; INSERT ...)`, which holds its commands together. public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { + // A PostgreSQL `CREATE RULE`: its header, the `DO [ALSO | INSTEAD]` before the action list, and the list itself. + private enum RuleAction: Equatable { + case none + case header + case afterDo + case list(depth: Int) + } + private enum Header { case opener /// After `CREATE`, `ALTER` or `REPLACE`. `accountWords` is what is left of an unquoted `DEFINER = user@host`. @@ -25,6 +38,11 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { private static let statementIntroducers: Set = ["THEN", "ELSE", "DO", "LOOP", "REPEAT"] private let grammar: SQLLexicalGrammar + private let readsBlocksEverywhere: Bool + // T-SQL outside a routine: where a block's `BEGIN` can stand, as the statement's first word, after `IF`, `WHILE` + // or `ELSE`, or after a block the statement already opened. Anywhere else `begin` is in a name's place. + private var followsControlFlow = false + private var pendingBeginMayOpen = false private var header = Header.opener private var definesRoutine = false private var openedBlockBody = false @@ -34,6 +52,8 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { /// whatever word follows it. private var openCaseExpressions = 0 private var parenDepth = 0 + private var ruleAction = RuleAction.none + private var ruleActionHeldSemicolon = false private var pendingBegin = false private var pendingEnd = false private var followsName = false @@ -50,12 +70,15 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public init(grammar: SQLLexicalGrammar) { self.grammar = grammar + self.readsBlocksEverywhere = grammar.contains(.unterminatedStatements) self.merge = grammar.contains(.terminatedMergeStatements) ? SQLMergeStatementTracker() : nil } public var needsWords: Bool { + // Inside parentheses no word opens or closes a block, and a `MERGE` there needs no `;`. + if readsBlocksEverywhere { return parenDepth == 0 } if case .read = header { - return definesRoutine || merge?.needsWords == true + return definesRoutine || ruleAction == .header || ruleAction == .afterDo || merge?.needsWords == true } return true } @@ -70,7 +93,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { // An `END` still waiting for the word after it closes a block when the text ends. public var holdsOpenBlock: Bool { - depth > (pendingEnd ? 1 : 0) + depth > (pendingEnd ? 1 : 0) || ruleActionHeldSemicolon } public mutating func observeWord(_ word: String) { @@ -87,10 +110,27 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { startsBodyStatement = false let statementPosition = atStatementPosition observeStatementPosition(after: word, wasAtStatementPosition: statementPosition) - guard !isMember, readHeader(word), definesRoutine, parenDepth == 0 else { return } + guard !isMember else { return } + var isFirstWord = false + if case .opener = header { + isFirstWord = true + } + let pastHeader = readHeader(word) + if pastHeader, parenDepth == 0 { + observeRuleWord(word) + } + guard parenDepth == 0, readsBlocksEverywhere || (pastHeader && definesRoutine) else { return } switch word { - case "BEGIN" where opensNestedBlock(atStatementPosition: statementPosition): + case "BEGIN" where readsBlocksEverywhere: + pendingBegin = true + pendingBeginMayOpen = opensNestedBlock(atStatementPosition: statementPosition, isFirstWord: isFirstWord) + case "BEGIN" where opensNestedBlock(atStatementPosition: statementPosition, isFirstWord: isFirstWord): pendingBegin = true + pendingBeginMayOpen = true + case "IF" where readsBlocksEverywhere, "WHILE" where readsBlocksEverywhere: + followsControlFlow = true + case "ELSE" where readsBlocksEverywhere && openCaseExpressions == 0: + followsControlFlow = true case "CASE" where !grammar.hasSingleBodyForm: depth += 1 openCaseExpressions += 1 @@ -103,6 +143,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeSymbol(_ symbol: UInt16) { merge?.observeSymbol(symbol) + if ruleAction == .afterDo, symbol != SqlLexer.openParen { + ruleAction = .none + } let opensBlock = symbol == SqlLexer.openParen && !grammar.contains(.atomicRoutineBodies) settlePendingBeforeNonWord(opensBlock: opensBlock) followsDot = followsName && symbol == Self.dot @@ -112,9 +155,16 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { switch symbol { case SqlLexer.openParen: parenDepth += 1 + if ruleAction == .afterDo { + ruleAction = .list(depth: parenDepth) + } endName() case SqlLexer.closeParen: parenDepth = max(0, parenDepth - 1) + if case .list(let listDepth) = ruleAction, parenDepth < listDepth { + ruleAction = .none + ruleActionHeldSemicolon = false + } default: break } @@ -122,6 +172,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeOpaqueToken() { merge?.observeOpaqueToken() + if ruleAction == .afterDo { + ruleAction = .none + } settlePendingBeforeNonWord(opensBlock: !grammar.contains(.atomicRoutineBodies)) followsName = true followsDot = false @@ -140,13 +193,27 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { pendingEnd = false closeBlock() } + if readsBlocksEverywhere { + // A T-SQL `CASE` is only an expression, which no `;` can stand inside, so one still open was never closed. + depth = max(0, depth - openCaseExpressions) + } openCaseExpressions = 0 - parenDepth = 0 followsName = false followsDot = false - let endsStatement = depth == 0 - startsBodyStatement = !endsStatement - atStatementPosition = !endsStatement + let heldByRuleAction: Bool + if case .list(let listDepth) = ruleAction { + heldByRuleAction = depth == 0 && parenDepth >= listDepth + } else { + heldByRuleAction = false + } + if heldByRuleAction { + ruleActionHeldSemicolon = true + } else { + parenDepth = 0 + } + let endsStatement = depth == 0 && !heldByRuleAction + startsBodyStatement = depth > 0 + atStatementPosition = depth > 0 labelMayFollow = false inDeclaration = false inHandlerDeclaration = false @@ -175,6 +242,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { } return false case .kind(let accountWords): + if word == "RULE", grammar.contains(.atomicRoutineBodies) { + ruleAction = .header + } if SqlBlockStructure.isRoutineKind(word, grammar: grammar) { definesRoutine = true header = .name @@ -207,11 +277,33 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { } } - private func opensNestedBlock(atStatementPosition: Bool) -> Bool { - guard depth > 0, !grammar.hasSingleBodyForm, !grammar.contains(.unterminatedStatements) else { return true } + private func opensNestedBlock(atStatementPosition: Bool, isFirstWord: Bool) -> Bool { + if readsBlocksEverywhere { + return depth > 0 || definesRoutine || isFirstWord || followsControlFlow || openedBlockBody + } + guard depth > 0, !grammar.hasSingleBodyForm else { return true } return atStatementPosition || inHandlerDeclaration } + private mutating func observeRuleWord(_ word: String) { + switch ruleAction { + case .header where word == "DO": + ruleAction = .afterDo + case .afterDo where word == "ALSO" || word == "INSTEAD": + break + case .afterDo: + ruleAction = .none + default: + break + } + } + + // `BEGIN TRY` and `BEGIN CATCH` are blocks wherever they stand, and nothing else follows a name that way. + private func beginOpensBlock(followedBy word: String) -> Bool { + if readsBlocksEverywhere, word == "TRY" || word == "CATCH" { return true } + return pendingBeginMayOpen && !SqlBlockStructure.beginStartsStatement(followedBy: word, grammar: grammar) + } + private mutating func endName() { if case .name = header { header = .read @@ -227,7 +319,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { openBlockBody() return true } - if !SqlBlockStructure.beginStartsTransaction(followedBy: word) { + if beginOpensBlock(followedBy: word) { openBlockBody() } } @@ -237,7 +329,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { closeBlock() return false } - switch SqlBlockStructure.endingFollowedBy(word, closesCaseExpression: openCaseExpressions > 0) { + switch SqlBlockStructure.endingFollowedBy(word, closesCaseExpression: openCaseExpressions > 0, grammar: grammar) { case .closesControlFlow: return true case .closesCaseStatement: @@ -254,7 +346,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { private mutating func settlePendingBeforeNonWord(opensBlock: Bool) { if pendingBegin { pendingBegin = false - if opensBlock { + // A T-SQL `begin(` outside a routine is a name, as in `CREATE TABLE begin(a int)`. + let opensAfterName = !readsBlocksEverywhere || definesRoutine || depth > 0 + if opensBlock, pendingBeginMayOpen, opensAfterName { openBlockBody() } } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift index 38f254c5bd..851741b05f 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLUnterminatedStatements.swift @@ -20,11 +20,13 @@ public enum SQLUnterminatedStatements { /// Every text a gate has to tier for `statement`: the statement itself, then, when `grammar` needs no terminator, /// the text of each statement that begins inside it, less the statements nested in that one, which come on their /// own. A grammar that takes a statement in place of a value or a table adds what ``SQLEmbeddedStatements`` finds, - /// and one with a pipe operator each statement ``SQLPipeOperatorChain`` finds. + /// and one with a pipe operator each statement ``SQLPipeOperatorChain`` finds. A grammar that needs a terminator + /// also adds the pieces a `;` inside the statement's parentheses separates, from ``SQLParenthesizedSemicolons``. public static func runnable(in statement: String, grammar: SQLLexicalGrammar) -> [String] { guard grammar.contains(.unterminatedStatements) else { return [statement] + SQLEmbeddedStatements.texts(in: statement, grammar: grammar) + SQLPipeOperatorChain.statements(in: statement, grammar: grammar) + + SQLParenthesizedSemicolons.pieces(in: statement, grammar: grammar) } let text = statement as NSString let tokens = TSQLTokens.read(text, grammar: grammar) @@ -32,6 +34,27 @@ public enum SQLUnterminatedStatements { return [statement] + ownTexts(of: statementSpans(in: tokens, length: text.length), in: text) } + // A scanner statement on T-SQL keeps a `;` outside parentheses only when a block holds it, so anything after one but + // the block's `END`, an `ELSE` or another `;` is a second statement the block runs. A stored routine runs nothing. + public static func blockHoldsSeveralStatements(_ statement: String, grammar: SQLLexicalGrammar) -> Bool { + guard grammar.contains(.unterminatedStatements) else { return false } + let tokens = TSQLTokens.read(statement as NSString, grammar: grammar) + guard !definesRoutine(tokens) else { return false } + return tokens.indices.dropLast().contains { index in + guard tokens[index].kind == .terminator, tokens[index].depth == 0 else { return false } + let next = tokens[index + 1] + guard next.kind != .terminator else { return false } + switch next.word { + case "ELSE": + return false + case "END": + return tokens.indices.contains(index + 2) && tokens[index + 2].word == "CONVERSATION" + default: + return true + } + } + } + /// The reserved keywords that begin a T-SQL statement. `ELSE` and `FETCH` are left out because they also continue /// a `CASE` and an `OFFSET` clause, and the statement after an `ELSE` begins with a keyword of its own. `WITH` is /// left out because a common table expression cannot follow a statement without a `;`, and the word also opens diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift index 0a11e84e6d..6968b11fef 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlBlockStructure.swift @@ -23,7 +23,8 @@ public enum SqlBlockStructure { /// What an `END` means, given the word after it. public enum EndFollower: Equatable, Sendable { - /// `END IF`, `END LOOP` and their kin close a construct that never opened a block, and the word is theirs. + /// `END IF`, `END LOOP` and their kin close a construct that never opened a block, and T-SQL's + /// `END CONVERSATION` is a statement that closes nothing. Either way the word is theirs. case closesControlFlow /// `END CASE` closes the `CASE` that opened a block, and the word is theirs. case closesCaseStatement @@ -48,6 +49,10 @@ public enum SqlBlockStructure { /// second opener. private static let controlFlowFollowers: Set = ["IF", "LOOP", "WHILE", "FOR", "REPEAT"] + // T-SQL's `BEGIN DIALOG`, `BEGIN CONVERSATION TIMER` and `END CONVERSATION` are statements, not a block's ends. + private static let serviceBrokerFollowers: Set = ["DIALOG", conversationKeyword] + private static let conversationKeyword = "CONVERSATION" + /// The keywords a statement has to open with before a `BEGIN` inside it is read as a routine body. /// /// Only ``SQLRoutineBodyTracker`` consults this, and the reason is safety rather than grammar. An anonymous @@ -56,7 +61,9 @@ public enum SqlBlockStructure { /// single statement whose first word is `BEGIN`, and `QueryClassifier` tiers a statement by its leading keyword, /// so a swallowed `DROP` would be tiered a plain write and skip the destructive confirmation. Folding cannot /// execute anything, so it takes every block. Oracle, whose anonymous blocks are the everyday form, has its own - /// grammar in ``PLSQLUnitTracker`` and a classifier that reads the whole block. + /// grammar in ``PLSQLUnitTracker`` and a classifier that reads the whole block. T-SQL reserves `BEGIN`, so its + /// blocks open outside a routine too, and the gate tiers every statement inside one through + /// ``SQLUnterminatedStatements``. private static let routineDefinitionOpeners: Set = ["CREATE", "ALTER", "REPLACE"] /// What such a statement has to define, for the same reason. `BEGIN` is not reserved on PostgreSQL, MySQL or @@ -111,6 +118,28 @@ public enum SqlBlockStructure { return transactionFollowers.contains(keyword) } + public static func beginStartsStatement(followedBy keyword: String?, grammar: SQLLexicalGrammar) -> Bool { + if beginStartsTransaction(followedBy: keyword) { return true } + return readsTSQLBlocks(grammar) && keyword.map(serviceBrokerFollowers.contains) == true + } + + // T-SQL closes every block with a bare `END`: in `END IF @x = 1 ...` the `IF` begins the next statement. + public static func endingFollowedBy( + _ keyword: String, + closesCaseExpression: Bool, + grammar: SQLLexicalGrammar + ) -> EndFollower { + guard readsTSQLBlocks(grammar) else { + return endingFollowedBy(keyword, closesCaseExpression: closesCaseExpression) + } + return !closesCaseExpression && keyword == conversationKeyword ? .closesControlFlow : .closesBlock + } + + // The grammar whose statements need no `;` between them is T-SQL's, which reserves `BEGIN`, `CASE` and `END`. + private static func readsTSQLBlocks(_ grammar: SQLLexicalGrammar) -> Bool { + grammar.contains(.unterminatedStatements) + } + /// What an `END` followed by `keyword` closes. /// /// - Parameter closesCaseExpression: whether a `CASE` is open that has held no `;`. Only the statement form can @@ -260,7 +289,7 @@ public enum SqlBlockStructure { let cursor = skipTrivia(from: offset, in: text, length: length, grammar: grammar) guard cursor < length else { return true } guard text.character(at: cursor) != SqlLexer.semicolon else { return true } - return beginStartsTransaction(followedBy: readKeyword(text, at: cursor, length: length).text) + return beginStartsStatement(followedBy: readKeyword(text, at: cursor, length: length).text, grammar: grammar) } /// Goes by the word after the `END` alone: only folding asks, and it keeps no record of which `CASE` is open. @@ -273,7 +302,7 @@ public enum SqlBlockStructure { let cursor = skipTrivia(from: offset, in: text, length: length, grammar: grammar) guard cursor < length else { return .closesBlock(resumeAt: offset) } let follower = readKeyword(text, at: cursor, length: length) - switch endingFollowedBy(follower.text, closesCaseExpression: false) { + switch endingFollowedBy(follower.text, closesCaseExpression: false, grammar: grammar) { case .closesControlFlow: return .none case .closesCaseStatement: diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift index e00a7551a5..cb810a30f5 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/Parcel.swift @@ -4,6 +4,8 @@ import TableProWireIO enum ParcelFlavor: UInt16 { case request = 1 case response = 4 + case abort = 6 + case cancel = 7 case success = 8 case failure = 9 case record = 10 diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift index ed5608fe9f..440de15bef 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift @@ -1,7 +1,8 @@ import Darwin import Foundation -public final class TeradataConnection { +// One queue drives it. Only `cancel()` comes from elsewhere, and what it reads and sends is under socketLock or wireLock. +public final class TeradataConnection: @unchecked Sendable { private let config: TeradataConnectionConfig private let socketLock = NSLock() private var _socket: (any TeradataTransport)? @@ -20,13 +21,56 @@ public final class TeradataConnection { private var gssSequence: UInt64 = 0 private var confidentialityBypassed = false + // Orders every post-logon send with the counters it consumes, since Stop sends its abort from another thread. + private let wireLock = NSLock() + // Under socketLock, which is never held across I/O. + private var _phase = RequestPhase.idle + private var _stopRequested = false + private var _operationActive = false + // The request a stop is waiting on the server for, so the stop's deadline closes nothing that came after it. + private var _stoppingRequest: UInt32? + + // 3110: "The transaction was aborted by the user." + static let userAbortCode = 3_110 + private static let abortLockWait: TimeInterval = 0.25 + // A server or network that stops answering would otherwise hold a stop for the whole 20-second receive timeout. + static let defaultStopGrace: TimeInterval = 5 + private let stopGrace: TimeInterval + private let stopDeadlineQueue = DispatchQueue(label: "com.TablePro.teradata.stop-deadline") + + private enum RequestPhase: Equatable { + case idle + case connecting + case awaitingResponse(UInt32) + case fetching(UInt32) + } + + private enum StopAction { + case nothing + case abort(UInt32) + case endOutput(UInt32) + case close + } + + private enum Exchange { + case answered([Parcel]) + case notSent + case outputEnded + } + public init(config: TeradataConnectionConfig) { self.config = config + stopGrace = Self.defaultStopGrace } - init(config: TeradataConnectionConfig, transport: any TeradataTransport) { + init( + config: TeradataConnectionConfig, + transport: any TeradataTransport, + stopGrace: TimeInterval = TeradataConnection.defaultStopGrace + ) { self.config = config self._socket = transport + self.stopGrace = stopGrace confidentialityBypassed = true } @@ -49,6 +93,7 @@ public final class TeradataConnection { socketLock.withLock { _socket = socket _hasLostConnection = false + _phase = .connecting } var established = false defer { @@ -57,6 +102,7 @@ public final class TeradataConnection { self.socket = nil connectDeadline = nil } + socketLock.withLock { _phase = .idle } } serverIP = Self.resolveAddress(config.host) ?? config.host _ = try deadline.remainingMilliseconds() @@ -91,28 +137,91 @@ public final class TeradataConnection { let body = TeradataMessages.optionsParcel(returnStatementInfo: false).encoded() + TeradataMessages.requestParcel(sql: sql).encoded() + TeradataMessages.responseParcel().encoded() - let parcels: [Parcel] + let outcome: Exchange do { - parcels = try exchange(body, requestNumber: requestNumber) + outcome = try exchange(body, requestNumber: requestNumber) } catch { + endRequest() // The reply still on its way would be read as the next request's. dropTransport() throw error } - return try TeradataResultParser.parse(parcels) + let stopped = endRequest() + guard case .answered(let parcels) = outcome else { throw TeradataWireError.cancelled } + do { + return try TeradataResultParser.parse(parcels) + } catch TeradataWireError.server(let code, _) where stopped && code == Self.userAbortCode { + // The server rolled back the transaction the request ran in; the host has no way yet to say so. + throw TeradataWireError.cancelled + } } - private func exchange(_ body: [UInt8], requestNumber: UInt32) throws -> [Parcel] { - try sendEncrypted(kind: .start, body: body, requestNumber: requestNumber) + private func exchange(_ body: [UInt8], requestNumber: UInt32) throws -> Exchange { + let sent = try wireLock.withLock { () -> Bool in + // A stop that landed before the request went out refuses it, and the session stays as it was. + let refused = socketLock.withLock { () -> Bool in + guard !_stopRequested else { return true } + _phase = .awaitingResponse(requestNumber) + return false + } + guard !refused else { return false } + try sendEncryptedLocked(kind: .start, body: body, requestNumber: requestNumber) + return true + } + guard sent else { return .notSent } var parcels: [Parcel] = [] while true { let batch = try readReply() parcels.append(contentsOf: batch) - if Self.responseIsComplete(batch) { return parcels } + if Self.responseIsComplete(batch) { return .answered(parcels) } + // Once the first reply is out an abort no longer takes effect, so a stop ends the output instead. + let stopRequested = socketLock.withLock { () -> Bool in + _phase = .fetching(requestNumber) + return _stopRequested + } + if stopRequested { + try endOutput(requestNumber: requestNumber) + return .outputEnded + } try sendContinue(requestNumber: requestNumber) } } + // The Cancel parcel in place of the Response parcel deletes the request's spool, and the server answers with + // End Request. One that does not throws, and the caller gives the transport up. + private func endOutput(requestNumber: UInt32) throws { + try sendEncrypted(kind: .cont, body: TeradataMessages.cancelParcel().encoded(), requestNumber: requestNumber) + guard Self.responseIsComplete(try readReply()) else { + throw TeradataWireError.malformed("the server kept answering a cancelled request") + } + } + + // A caller's operation, so a stop that lands before its request goes out still reaches it. + public func beginOperation() { + socketLock.withLock { + _operationActive = true + _stopRequested = false + } + } + + public func endOperation() { + socketLock.withLock { + _operationActive = false + _stopRequested = false + } + } + + @discardableResult + private func endRequest() -> Bool { + socketLock.withLock { + let stopped = _stopRequested + _phase = .idle + _stopRequested = false + _stoppingRequest = nil + return stopped + } + } + private func dropTransport() { let dropped = socketLock.withLock { let transport = _socket @@ -129,19 +238,80 @@ public final class TeradataConnection { return } if sessionNumber != 0 { - let message = LanMessage( - kind: .logoff, body: TeradataMessages.logoffParcel().encoded(), - sessionNumber: sessionNumber, byteVar: 0, - authentication: TeradataMessages.authBytes(nextMessage()), hostCharSet: charsetCode) - try? socket.send(message.encoded()) + wireLock.withLock { + let message = LanMessage( + kind: .logoff, body: TeradataMessages.logoffParcel().encoded(), + sessionNumber: sessionNumber, byteVar: 0, + authentication: TeradataMessages.authBytes(nextMessage()), hostCharSet: charsetCode) + try? socket.send(message.encoded()) + } } socket.close() self.socket = nil connectDeadline = nil } + // A request the server is still working on is aborted on the session, which rolls back its transaction, answers it + // with failure 3110 and stays logged on. Past the first reply the request's own loop ends the output instead, and a + // request not yet sent is refused. A connect has no request to stop, so it gives the transport up, as does an + // abort that cannot be sent or a server that has not ended the request within the grace. public func cancel() { - socket?.cancel() + let action: StopAction = socketLock.withLock { + switch _phase { + case .idle: + if _operationActive { _stopRequested = true } + return .nothing + case .connecting: + return .close + case .awaitingResponse(let request): + _stopRequested = true + _stoppingRequest = request + return .abort(request) + case .fetching(let request): + _stopRequested = true + _stoppingRequest = request + return .endOutput(request) + } + } + switch action { + case .nothing: + return + case .close: + socket?.cancel() + case .abort(let request): + guard sendAbort(for: request) else { + socket?.cancel() + return + } + closeUnlessEnded(request) + case .endOutput(let request): + closeUnlessEnded(request) + } + } + + private func closeUnlessEnded(_ request: UInt32) { + stopDeadlineQueue.asyncAfter(deadline: .now() + stopGrace) { [weak self] in + guard let self else { return } + // Shut down under the lock, so a request that ended meanwhile cannot hand the transport to a new one. + socketLock.withLock { + guard _stoppingRequest == request else { return } + _socket?.cancel() + } + } + } + + // A long start still going out holds the lock, and the caller may be the main thread. + private func sendAbort(for request: UInt32) -> Bool { + guard wireLock.lock(before: Date(timeIntervalSinceNow: Self.abortLockWait)) else { return false } + defer { wireLock.unlock() } + // Past the first reply the request's own loop ends the output; a request already answered needs nothing. + guard socketLock.withLock({ _phase == .awaitingResponse(request) }) else { return true } + do { + try sendEncryptedLocked(kind: .abort, body: TeradataMessages.abortParcel().encoded(), requestNumber: request) + return true + } catch { + return false + } } static func responseIsComplete(_ parcels: [Parcel]) -> Bool { @@ -252,6 +422,12 @@ public final class TeradataConnection { } private func sendEncrypted(kind: MessageKind, body: [UInt8], requestNumber: UInt32) throws { + try wireLock.withLock { + try sendEncryptedLocked(kind: kind, body: body, requestNumber: requestNumber) + } + } + + private func sendEncryptedLocked(kind: MessageKind, body: [UInt8], requestNumber: UInt32) throws { let message = LanMessage( kind: kind, body: body, sessionNumber: sessionNumber, requestNumber: requestNumber, byteVar: 0, authentication: TeradataMessages.authBytes(nextMessage()), diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataMessages.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataMessages.swift index 042633063c..d0ec9d359e 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataMessages.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataMessages.swift @@ -119,6 +119,14 @@ enum TeradataMessages { Parcel(.logoff) } + static func abortParcel() -> Parcel { + Parcel(.abort) + } + + static func cancelParcel() -> Parcel { + Parcel(.cancel) + } + static func charsetCode(fromConfigResponse body: [UInt8], name: String) -> UInt8? { let needle = Array(name.utf8) guard !needle.isEmpty else { return nil } diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift new file mode 100644 index 0000000000..b02a53181e --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift @@ -0,0 +1,61 @@ +// +// CassandraConditionalWriteTests.swift +// TableProCassandraCoreTests +// + +import TableProCassandraCore +import Testing + +struct CassandraConditionalWriteTests { + private static let update = CassandraConditionalWrite.updateStatement( + target: "\"shop\".\"orders\"", assignments: ["\"note\" = ?"], keyCondition: "\"id\" = ?" + ) + private static let insert = CassandraConditionalWrite.insertStatement( + target: "\"shop\".\"orders\"", names: ["\"id\"", "\"note\""], values: ["?", "?"] + ) + + // Without the condition CQL writes the cell anyway and recreates the row with every other column null. + @Test("A grid edit only writes a row that still exists") + func updateIsConditionedOnTheRow() { + #expect(Self.update == "UPDATE \"shop\".\"orders\" SET \"note\" = ? WHERE \"id\" = ? IF EXISTS") + #expect(Self.insert == "INSERT INTO \"shop\".\"orders\" (\"id\", \"note\") VALUES (?, ?) IF NOT EXISTS") + } + + @Test("An edit of a row deleted since it was read is refused, not reported as saved") + func updateOfAMissingRowIsRefused() { + #expect(CassandraConditionalWrite.refusal( + statement: Self.update, firstColumn: "[applied]", firstValue: "false" + ) == .rowMissing) + } + + @Test("A new row whose key is taken is refused") + func insertOfATakenKeyIsRefused() { + #expect(CassandraConditionalWrite.refusal( + statement: Self.insert, firstColumn: "[applied]", firstValue: "false" + ) == .keyTaken) + } + + @Test("A write the condition let through is no refusal") + func appliedWriteIsNotRefused() { + #expect(CassandraConditionalWrite.refusal( + statement: Self.update, firstColumn: "[applied]", firstValue: "true" + ) == nil) + #expect(CassandraConditionalWrite.refusal( + statement: Self.insert, firstColumn: "[applied]", firstValue: "true" + ) == nil) + } + + // A condition on a column answers with the row's current values, which the console shows as the server sent them. + @Test("An unapplied write with a condition of its own is left to the result") + func otherConditionsAreLeftToTheResult() { + #expect(CassandraConditionalWrite.refusal( + statement: "UPDATE t SET a = 1 WHERE k = 1 IF a = 0", firstColumn: "[applied]", firstValue: "false" + ) == nil) + #expect(CassandraConditionalWrite.refusal( + statement: "DELETE FROM t WHERE k = 1 IF EXISTS", firstColumn: "[applied]", firstValue: "false" + ) == nil) + #expect(CassandraConditionalWrite.refusal( + statement: "UPDATE t SET a = 1 WHERE k = 1", firstColumn: "a", firstValue: "false" + ) == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginParameterEscapingTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginParameterEscapingTests.swift index 4f2d558573..1f17a3ac4b 100644 --- a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginParameterEscapingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginParameterEscapingTests.swift @@ -107,6 +107,17 @@ struct IsNumericLiteralTests { #expect(!StubDriver.isNumericLiteral("12abc")) #expect(!StubDriver.isNumericLiteral("1.2.3")) } + + // Written bare, a zip or a product code is compared as a number and matches `123` and `0123` too. + @Test("Numeric-shaped text with padding zeros is bound as a string", arguments: ["00123", "-007", "0042", "00.5"]) + func paddedNumbersAreQuoted(value: String) { + #expect(StubDriver().escapedParameterValue(value).unicodeScalars.first == "'") + } + + @Test("A number that opens with a single zero is bound bare", arguments: ["0", "0.5", "-0.5", "0e3", "+0", "100"]) + func singleLeadingZeroStaysBare(value: String) { + #expect(StubDriver().escapedParameterValue(value) == value) + } } // MARK: - escapedParameterValue diff --git a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisCommandSplitterTests.swift b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisCommandSplitterTests.swift index 515d4828cd..5dee906b01 100644 --- a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisCommandSplitterTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisCommandSplitterTests.swift @@ -103,6 +103,18 @@ struct RedisCommandSplitterTests { } } + @Test("Each command's caret reach gives every caret the command command(at:) gives it", arguments: [ + "GET a; GET b\n\nGET c \n", "GET a\n", "GET a; ", "SET k \"a\nb\"\nGET k", " GET a ;", "SET 😀 1\r\nGET 😀", "", + ]) + func caretReachMatchesCommandAt(script: String) { + let reached = RedisCommandSplitter.commandsWithCaretReach(in: script) + #expect(reached.map { $0.command } == RedisCommandSplitter.commands(in: script)) + for caret in 0...((script as NSString).length + 1) { + let viaReach = reached.first { caret <= $0.reach }?.command + #expect(viaReach == RedisCommandSplitter.command(at: caret, in: script), "caret \(caret)") + } + } + @Test("A caret after the last command, or in text with none, has no command") func caretPastEveryCommand() { #expect(command(at: 6, in: "GET a\n") == nil) diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLParenthesizedSemicolonTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLParenthesizedSemicolonTests.swift new file mode 100644 index 0000000000..2f9cf1257c --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLParenthesizedSemicolonTests.swift @@ -0,0 +1,77 @@ +import Foundation +import TableProSQLGrammar +import Testing + +struct SQLParenthesizedSemicolonTests { + private static func grammar(_ typeId: String) -> SQLLexicalGrammar { + SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil).execution + } + + private func sent(_ sql: String, _ grammar: SQLLexicalGrammar) -> [String] { + SQLStatementScanner.executableStatements(in: sql, grammar: grammar).map(\.sql) + } + + @Test("A PostgreSQL rule's multi-command action list stays one statement", arguments: [ + "CREATE RULE r AS ON INSERT TO t DO ALSO (INSERT INTO a VALUES (1); INSERT INTO b VALUES (2))", + "CREATE OR REPLACE RULE r AS ON UPDATE TO t WHERE (NEW.a > 0) DO INSTEAD (UPDATE a SET n = 1; NOTIFY t)", + "CREATE RULE r AS ON DELETE TO t DO (DELETE FROM a; DELETE FROM b)", + ]) + func ruleActionStaysWhole(rule: String) { + let grammar = Self.grammar("PostgreSQL") + let sql = rule + ";\nSELECT 9;" + + #expect(sent(sql, grammar) == [rule, "SELECT 9"]) + #expect(SQLStatementScanner.navigableStatements(in: sql, grammar: grammar).count == 2) + #expect(!SQLStatementScanner.endsInsideOpenBlock(sql, grammar: grammar)) + } + + @Test("A rule whose action is one command ends at its ;") + func ruleWithoutAnActionList() { + let grammar = Self.grammar("PostgreSQL") + + #expect(sent("CREATE RULE r AS ON INSERT TO t DO ALSO INSERT INTO log VALUES (NEW.id);\nSELECT 2;", grammar).count == 2) + #expect(sent("CREATE RULE r AS ON INSERT TO t DO INSTEAD NOTHING;\nSELECT 2;", grammar).count == 2) + } + + @Test("A stray ( ends at its ; and leaves every statement after it on its own", arguments: [ + "PostgreSQL", "MySQL", "SQLite", "SQL Server", "DuckDB", "ClickHouse", + ]) + func strayParenthesisEndsAtItsSemicolon(typeId: String) { + let grammar = Self.grammar(typeId) + let sql = "SELECT 1;\nSELECT count(id FROM a;\nSELECT 2;\nSELECT 3;" + let caret = (sql as NSString).range(of: "SELECT 3").location + + #expect(sent(sql, grammar) == ["SELECT 1", "SELECT count(id FROM a", "SELECT 2", "SELECT 3"]) + #expect(SQLStatementScanner.statementAtCursor(in: sql, cursorPosition: caret, grammar: grammar) == "SELECT 3") + #expect(!SQLStatementScanner.endsInsideOpenBlock(sql, grammar: grammar)) + } + + @Test("A rule's action list left open holds the rest of the text, which the gate reads as more than one statement") + func openActionListLeavesTheTextInsideABlock() { + let grammar = Self.grammar("PostgreSQL") + let sql = "CREATE RULE r AS ON INSERT TO t DO ALSO (INSERT INTO a VALUES (1);\nDROP TABLE users;" + + #expect(sent(sql, grammar).count == 1) + #expect(SQLStatementScanner.endsInsideOpenBlock(sql, grammar: grammar)) + } + + @Test("The gate tiers each command a rule's action list holds") + func gateTiersThePieces() { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "PostgreSQL", declared: nil, session: nil) + + #expect(SQLStatementClassifier.deletesEveryRow( + "CREATE RULE r AS ON INSERT TO t DO ALSO (SELECT 1; DELETE FROM audit)", + databaseTypeId: "PostgreSQL", + readings: readings + )) + #expect(SQLStatementClassifier.classify( + "SELECT (1; DROP TABLE users; SELECT 1)", databaseTypeId: "PostgreSQL", readings: readings + ).tier == .destructive) + } + + @Test("Inside a routine body a parenthesis one statement leaves open does not hide the body's END") + func bodyStatementParenthesisDoesNotHideTheEnd() { + let sql = "CREATE PROCEDURE p()\nBEGIN\n SELECT (1;\n SELECT 2;\nEND;\nSELECT 3;" + #expect(sent(sql, Self.grammar("MySQL")).last == "SELECT 3") + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift index c63a3d7bbe..79bcaf44ed 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift @@ -20,6 +20,7 @@ private let cursorIndexFixtures = [ CREATE FUNCTION f() RETURNS int AS $$ BEGIN RETURN 1; END; $$ LANGUAGE plpgsql; /* c; */ SELECT E'x' 'y';; + CREATE RULE r AS ON INSERT TO t DO ALSO (INSERT INTO a VALUES (1); INSERT INTO b VALUES (2)); SELECT 2 """), CursorIndexFixture(typeId: "MySQL", text: """ @@ -44,6 +45,15 @@ private let cursorIndexFixtures = [ GO SELECT [a;b] GO + IF @x = 1 + BEGIN + SELECT 1; + SELECT CASE WHEN @x = 1 THEN 2 END; + END + ELSE BEGIN TRY SELECT 3; END TRY BEGIN CATCH SELECT 4; END CATCH; + SELECT (5; 6); + SET XACT_ABORT ON + BEGIN TRY SELECT 7; END TRY BEGIN CATCH THROW; END CATCH """), CursorIndexFixture(typeId: "Oracle", text: """ BEGIN @@ -70,7 +80,8 @@ private let cursorIndexFixtures = [ ] private let cursorIndexInsertions = [ - ";", "'", "\nGO\n", "\n/\n", " ", "x", "$$", "BEGIN ", "END;", "--", "/*", "*/", "\n", "GO", "/", + ";", "'", "\nGO\n", "\n/\n", " ", "x", "$$", "BEGIN ", "END;", "--", "/*", "*/", "\n", "GO", "/", "(", ")", + "CASE ", "END ", ] private let longScriptRow = "SELECT id, name FROM users WHERE active = true;\n" diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift new file mode 100644 index 0000000000..271bc67937 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift @@ -0,0 +1,153 @@ +import Foundation +import TableProSQLGrammar +import Testing + +// T-SQL reserves BEGIN, so a BEGIN ... END block holds its `;`s wherever it stands, not only in a routine's body. +struct TSQLControlFlowBlockTests { + private static let sqlServer = SQLLexicalReadings.resolve(databaseTypeId: "SQL Server", declared: nil, session: nil) + .execution + + private static let merge = "MERGE dbo.t AS t USING dbo.s AS s ON t.id = s.id WHEN MATCHED THEN DELETE" + + private func sent(_ sql: String) -> [String] { + SQLStatementScanner.executableStatements(in: sql, grammar: Self.sqlServer).map(\.sql) + } + + private func atCursor(_ sql: String, on marker: String) -> String { + let offset = (sql as NSString).range(of: marker).location + return SQLStatementScanner.statementAtCursor(in: sql, cursorPosition: offset, grammar: Self.sqlServer) + } + + @Test("A block an IF, a WHILE or a statement of its own opens stays one statement", arguments: [ + ("IF 1=1 BEGIN SELECT 1; SELECT 2; END; SELECT 3", ["IF 1=1 BEGIN SELECT 1; SELECT 2; END", "SELECT 3"]), + ( + "WHILE @i < 3\nBEGIN\n SET @i += 1;\n PRINT @i;\nEND;\nSELECT @i;", + ["WHILE @i < 3\nBEGIN\n SET @i += 1;\n PRINT @i;\nEND", "SELECT @i"] + ), + ("BEGIN\n SELECT 1;\n SELECT 2;\nEND;\nSELECT 3;", ["BEGIN\n SELECT 1;\n SELECT 2;\nEND", "SELECT 3"]), + ( + "IF @a = 1 BEGIN IF @b = 1 BEGIN SELECT 1; END ELSE BEGIN SELECT 2; END; SELECT 3; END; SELECT 4", + ["IF @a = 1 BEGIN IF @b = 1 BEGIN SELECT 1; END ELSE BEGIN SELECT 2; END; SELECT 3; END", "SELECT 4"] + ), + ( + "BEGIN TRY\n DELETE FROM a;\nEND TRY\nBEGIN CATCH\n SELECT ERROR_MESSAGE();\nEND CATCH;\nSELECT 9;", + ["BEGIN TRY\n DELETE FROM a;\nEND TRY\nBEGIN CATCH\n SELECT ERROR_MESSAGE();\nEND CATCH", "SELECT 9"] + ), + ]) + func blockHoldsItsSemicolons(sql: String, expected: [String]) { + #expect(sent(sql) == expected) + } + + @Test("Run Current anywhere in a conditional block runs the whole block") + func caretInsideTheBlockRunsTheBlock() { + let block = "IF @env = 'dev'\nBEGIN\n DELETE FROM a;\n DELETE FROM b;\nEND" + let sql = "SELECT 0;\n\(block);\nSELECT 9;" + + #expect(atCursor(sql, on: "IF @env") == block) + #expect(atCursor(sql, on: "DELETE FROM b") == block) + #expect(SQLStatementScanner.navigableStatements(in: sql, grammar: Self.sqlServer).count == 3) + } + + @Test("An END is never followed by more of its block, so the IF after it is the next statement") + func endBeforeTheNextStatementClosesTheBlock() { + let sql = "IF @a = 1 BEGIN SELECT 1; END\nIF @b = 1 BEGIN SELECT 2; END;\nSELECT 3;" + + #expect(sent(sql) == ["IF @a = 1 BEGIN SELECT 1; END\nIF @b = 1 BEGIN SELECT 2; END", "SELECT 3"]) + #expect(!SQLStatementScanner.endsInsideOpenBlock(sql, grammar: Self.sqlServer)) + } + + @Test("A transaction's BEGIN is a statement, not a block", arguments: ["TRAN", "TRANSACTION", "DISTRIBUTED TRANSACTION"]) + func transactionBeginIsAStatement(opener: String) { + #expect(sent("BEGIN \(opener);\nUPDATE t SET a = 1;\nCOMMIT;") == ["BEGIN \(opener)", "UPDATE t SET a = 1", "COMMIT"]) + } + + @Test("Service Broker's BEGIN DIALOG, BEGIN CONVERSATION TIMER and END CONVERSATION are statements") + func serviceBrokerStatementsAreNotBlocks() { + let script = """ + BEGIN DIALOG CONVERSATION @h FROM SERVICE [s] TO SERVICE 't' ON CONTRACT [c]; + BEGIN CONVERSATION TIMER (@h) TIMEOUT = 60; + END CONVERSATION @h; + SELECT 1; + """ + #expect(sent(script).count == 4) + + let inBlock = "IF @x = 1 BEGIN END CONVERSATION @h; SELECT 1; END; SELECT 2" + #expect(sent(inBlock) == ["IF @x = 1 BEGIN END CONVERSATION @h; SELECT 1; END", "SELECT 2"]) + } + + @Test("A CASE expression closes at its own END, and one left open at a ; does not hold the next statement") + func caseExpressions() { + #expect(sent("SELECT CASE WHEN a = 1 THEN 'x' END AS c FROM t;\nSELECT 2;").count == 2) + #expect(sent("IF @x = 1 BEGIN SELECT CASE WHEN @x = 1 THEN 1 END; SELECT 2; END; SELECT 3").count == 2) + #expect(sent("SELECT CASE WHEN a = 1 THEN 1;\nSELECT 2;") == ["SELECT CASE WHEN a = 1 THEN 1", "SELECT 2"]) + } + + @Test("A MERGE inside a block keeps its ; and the block's own ; is a separator") + func mergeInsideABlock() { + let block = "IF 1 = 1 BEGIN \(Self.merge); END" + #expect(sent(block + ";\nSELECT 1;") == [block, "SELECT 1"]) + } + + @Test("A block left open runs to its GO line and no further") + func goLineEndsAnOpenBlock() { + let sql = "IF 1 = 1 BEGIN SELECT 1;\nGO\nSELECT 2" + #expect(sent(sql) == ["IF 1 = 1 BEGIN SELECT 1", "SELECT 2"]) + #expect(SQLStatementScanner.endsInsideOpenBlock("IF 1 = 1 BEGIN SELECT 1;", grammar: Self.sqlServer)) + } + + @Test("The gate still tiers every statement a block holds") + func gateTiersEachStatementInsideTheBlock() { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "SQL Server", declared: nil, session: nil) + let sql = "IF @env = 'dev' BEGIN SELECT 1; DROP TABLE dbo.t; END" + + #expect(SQLStatementClassifier.classify(sql, databaseTypeId: "SQL Server", readings: readings).tier == .destructive) + #expect(SQLStatementClassifier.deletesEveryRow( + "IF 1 = 1 BEGIN DELETE FROM dbo.t; END", databaseTypeId: "SQL Server", readings: readings + )) + } + + @Test("A TRY block after a statement written without a ; is still one block") + func tryBlockAfterAnUnterminatedStatement() { + let script = "SET XACT_ABORT ON\nBEGIN TRY\n DELETE FROM a;\nEND TRY\nBEGIN CATCH\n THROW;\nEND CATCH;\nSELECT 1;" + #expect(sent(script).count == 2) + #expect(sent(script).last == "SELECT 1") + } + + @Test("A begin where a name stands opens no block", arguments: [ + ("CREATE TABLE begin(a int); DROP TABLE users", 2), + ("INSERT INTO begin VALUES (1); DROP TABLE users", 2), + ("DO BEGIN SELECT 1 FROM DUMMY; SELECT 2 FROM DUMMY; END;", 3), + ]) + func nameInBeginsPlaceOpensNothing(sql: String, count: Int) { + #expect(sent(sql).count == count) + } + + @Test("A block holding more than one statement is told apart from one statement", arguments: [ + ("IF x BEGIN UPDATE a SET c = 1; UPDATE b SET c = 1; END", true), + ("BEGIN MERGE dbo.t AS t USING dbo.s AS s ON t.id = s.id WHEN MATCHED THEN DELETE; SELECT 1; END", true), + ("IF x BEGIN SELECT 1; WITH c AS (SELECT id FROM t) DELETE FROM c; END", true), + ("IF x BEGIN SELECT 1; DISABLE TRIGGER trg ON t; END", true), + ("BEGIN TRY SELECT 1; THROW 50000, 'x', 1; END TRY BEGIN CATCH SELECT 2; END CATCH", true), + ("IF x BEGIN SELECT 1; done: SELECT 2; END", true), + ("IF x BEGIN SELECT 1; (SELECT 2); END", true), + ("IF x BEGIN SELECT 1; END CONVERSATION @h; END", true), + ("IF x SELECT 1", false), + ("IF x BEGIN SELECT 1; END", false), + ("IF x BEGIN SELECT 1;; END", false), + ("BEGIN IF x SELECT 1; ELSE SELECT 2; END", false), + ("BEGIN TRY SELECT 1; END TRY BEGIN CATCH THROW; END CATCH", false), + ("CREATE PROCEDURE p AS BEGIN SELECT 1; SELECT 2; END", false), + ]) + func blockHoldingSeveralStatements(sql: String, holdsSeveral: Bool) { + #expect(SQLUnterminatedStatements.blockHoldsSeveralStatements(sql, grammar: Self.sqlServer) == holdsSeveral) + let postgreSQL = SQLLexicalReadings.resolve(databaseTypeId: "PostgreSQL", declared: nil, session: nil).execution + #expect(!SQLUnterminatedStatements.blockHoldsSeveralStatements(sql, grammar: postgreSQL)) + } + + @Test("Other engines still read BEGIN outside a routine as a word", arguments: ["PostgreSQL", "MySQL", "SQLite"]) + func otherEnginesAreUnchanged(typeId: String) { + let grammar = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil).execution + let statements = SQLStatementScanner.executableStatements(in: "BEGIN\nDELETE FROM t;\nSELECT 1;", grammar: grammar) + #expect(statements.map { $0.sql } == ["BEGIN\nDELETE FROM t", "SELECT 1"]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift new file mode 100644 index 0000000000..4f19e1b292 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift @@ -0,0 +1,249 @@ +import Foundation +@testable import TableProTeradataCore +import Testing + +// Replies are read in order; a hook keyed by a reply's index runs just before that reply is read, which is where a +// request is waiting on the server when Stop is pressed. +private final class StoppableTransport: TeradataTransport { + private var replies: [[UInt8]] + private var stream: [UInt8] = [] + private var repliesRead = 0 + var beforeReply: [Int: () -> Void] = [:] + private(set) var sent: [LanMessage] = [] + private(set) var shutDown = false + private(set) var closed = false + + var sentKinds: [UInt8] { + sent.map { $0.kind } + } + + init(replies: [[UInt8]]) { + self.replies = replies + } + + func send(_ bytes: [UInt8]) throws { + if shutDown || closed { throw TeradataWireError.cancelled } + sent.append(LanMessage.decode( + header: Array(bytes.prefix(LanMessage.headerLength)), + body: Array(bytes.dropFirst(LanMessage.headerLength)) + )) + } + + func receive(_ count: Int) throws -> [UInt8] { + if stream.isEmpty { + beforeReply.removeValue(forKey: repliesRead)?() + if shutDown || closed { throw TeradataWireError.cancelled } + guard !replies.isEmpty else { throw TeradataWireError.truncated("no reply") } + stream = replies.removeFirst() + repliesRead += 1 + } + guard stream.count >= count else { throw TeradataWireError.truncated("closed") } + defer { stream.removeFirst(count) } + return Array(stream.prefix(count)) + } + + func finishConnecting() {} + + func cancel() { + shutDown = true + } + + func close() { + closed = true + } +} + +// A server that never answers: a receive waits until the transport is shut down, which the stop's deadline does from +// its own queue. +private final class SilentTransport: TeradataTransport, @unchecked Sendable { + private let lock = NSLock() + private let shutDown = DispatchSemaphore(value: 0) + private var sent: [UInt8] = [] + private var shut = false + private var received = false + var onFirstReceive: (() -> Void)? + + var sentKinds: [UInt8] { lock.withLock { sent } } + var wasShutDown: Bool { lock.withLock { shut } } + + func send(_ bytes: [UInt8]) throws { + try lock.withLock { + if shut { throw TeradataWireError.cancelled } + sent.append(bytes[2]) + } + } + + func receive(_ count: Int) throws -> [UInt8] { + let first = lock.withLock { () -> Bool in + defer { received = true } + return !received + } + if first { onFirstReceive?() } + _ = shutDown.wait(timeout: .now() + 10) + throw TeradataWireError.cancelled + } + + func finishConnecting() {} + + func cancel() { + lock.withLock { shut = true } + shutDown.signal() + } + + func close() { + cancel() + } +} + +struct TeradataStopTests { + private static let config = TeradataConnectionConfig(host: "td.example", username: "dbc", password: "dbc") + private static let start = MessageKind.start.rawValue + private static let cont = MessageKind.cont.rawValue + private static let abort = MessageKind.abort.rawValue + + private static func reply(_ parcels: [Parcel]) -> [UInt8] { + LanMessage(kind: .start, body: parcels.flatMap { $0.encoded() }).encoded() + } + + private static func failure(code: Int, _ message: String) -> Parcel { + let text = Array(message.utf8) + let body = [UInt8](repeating: 0, count: 8) + + [UInt8(code >> 8), UInt8(code & 0xFF), UInt8(text.count >> 8), UInt8(text.count & 0xFF)] + text + return Parcel(.failure, body: body) + } + + private static let finished = reply([Parcel(.success), Parcel(.endRequest)]) + + private static func thrown(_ body: () throws -> Void) -> Error? { + do { + try body() + return nil + } catch { + return error + } + } + + private static func isCancelled(_ error: Error?) -> Bool { + guard let wire = error as? TeradataWireError, case .cancelled = wire else { return false } + return true + } + + @Test("Stop while the server works on a request aborts it on the session, which stays logged on") + func stopAbortsTheRequestAndKeepsTheSession() throws { + let aborted = Self.reply([Self.failure(code: 3_110, "The transaction was aborted by the user.")]) + let transport = StoppableTransport(replies: [aborted, Self.finished]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[0] = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders CROSS JOIN sales.items") } + + #expect(Self.isCancelled(error)) + #expect(transport.sentKinds == [Self.start, Self.abort]) + #expect(transport.sent.last?.requestNumber == transport.sent.first?.requestNumber) + #expect(transport.sent.last?.body == TeradataMessages.abortParcel().encoded()) + #expect(!transport.shutDown) + #expect(!transport.closed) + #expect(connection.isConnected) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.abort, Self.start]) + } + + @Test("A request that failed on its own before the abort landed keeps its own error") + func failureOtherThanTheAbortIsReported() throws { + let missing = Self.reply([Self.failure(code: 3_807, "Object 'sales.orderz' does not exist.")]) + let transport = StoppableTransport(replies: [missing]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[0] = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orderz") } + + guard let wire = error as? TeradataWireError, case .server(let code, _) = wire else { + Issue.record("expected the server's error, got \(String(describing: error))") + return + } + #expect(code == 3_807) + #expect(connection.isConnected) + } + + // An export Stop lands here: the abort does not take effect once the server has started answering. + @Test("Stop while a long answer is being fetched ends the output and keeps the session") + func stopDuringFetchEndsTheOutput() throws { + let batch = Self.reply([Parcel(.success)]) + let ended = Self.reply([Parcel(.endRequest)]) + let transport = StoppableTransport(replies: [batch, batch, ended, Self.finished]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[1] = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders") } + + #expect(Self.isCancelled(error)) + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont]) + #expect(transport.sent.last?.body == TeradataMessages.cancelParcel().encoded()) + #expect(!transport.shutDown) + #expect(!transport.closed) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont, Self.start]) + } + + @Test("An output the server keeps sending after the Cancel parcel gives the session up") + func unendedOutputGivesTheSessionUp() throws { + let batch = Self.reply([Parcel(.success)]) + let transport = StoppableTransport(replies: [batch, batch, batch]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[1] = { connection.cancel() } + + #expect(Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders") } != nil) + #expect(transport.closed) + #expect(connection.hasLostConnection) + } + + @Test("Stop in a running call before its request goes out refuses the request and keeps the session") + func stopBeforeTheRequestIsSentRefusesIt() throws { + let transport = StoppableTransport(replies: [Self.finished]) + let connection = TeradataConnection(config: Self.config, transport: transport) + + connection.beginOperation() + connection.cancel() + let error = Self.thrown { _ = try connection.execute("DELETE FROM sales.orders") } + connection.endOperation() + + #expect(Self.isCancelled(error)) + #expect(transport.sent.isEmpty) + #expect(connection.isConnected) + + connection.beginOperation() + _ = try connection.execute("SELECT 1") + connection.endOperation() + #expect(transport.sentKinds == [Self.start]) + } + + @Test("A stopped request the server never answers gives the session up after the grace", .timeLimit(.minutes(1))) + func unansweredStopGivesTheSessionUp() throws { + let transport = SilentTransport() + let connection = TeradataConnection(config: Self.config, transport: transport, stopGrace: 0.05) + transport.onFirstReceive = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders CROSS JOIN sales.items") } + + #expect(Self.isCancelled(error)) + #expect(transport.sentKinds == [Self.start, Self.abort]) + #expect(transport.wasShutDown) + #expect(connection.hasLostConnection) + } + + @Test("Stop with no request running sends nothing and keeps the session") + func stopWhileIdleDoesNothing() throws { + let transport = StoppableTransport(replies: [Self.finished]) + let connection = TeradataConnection(config: Self.config, transport: transport) + + connection.cancel() + _ = try connection.execute("SELECT 1") + + #expect(transport.sentKinds == [Self.start]) + #expect(!transport.shutDown) + } +} diff --git a/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+EmphasizeBracket.swift b/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+EmphasizeBracket.swift index 0d390a3be1..04429cdc55 100644 --- a/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+EmphasizeBracket.swift +++ b/apple/Packages/TableProEditor/Sources/TableProEditorKit/Controller/TextViewController+EmphasizeBracket.swift @@ -37,7 +37,7 @@ extension TextViewController { pair.0, pair.1, from: range.location, - limit: min((textView.visibleTextRange ?? .zero).max + 4_096, textView.documentRange.max), + limit: min(max(range.location, (textView.visibleTextRange ?? .zero).max) + 4_096, textView.documentRange.max), reverse: false ) { emphasizeCharacter(characterIndex) @@ -52,7 +52,10 @@ extension TextViewController { pair.1, pair.0, from: range.location - 1, - limit: max((textView.visibleTextRange?.location ?? 0) - 4_096, textView.documentRange.location), + limit: max( + min(range.location - 1, textView.visibleTextRange?.location ?? 0) - 4_096, + textView.documentRange.location + ), reverse: true ) { emphasizeCharacter(characterIndex) @@ -91,12 +94,16 @@ extension TextViewController { if reverse { options = options.union(.reverse) } + // A caret outside the visible range can put the limit on the wrong side of `from`, and NSString raises + // an exception nothing can catch for a range past the text. + let length = textView.textStorage.length + let lower = max(0, min(reverse ? limit : from, length)) + let upper = max(0, min(reverse ? from : limit, length)) + guard upper > lower else { return nil } var closeCount = 0 var index: Int? textView.textStorage.mutableString.enumerateSubstrings( - in: reverse ? - NSRange(location: limit, length: from - limit) : - NSRange(location: from, length: limit - from), + in: NSRange(location: lower, length: upper - lower), options: options, using: { substring, range, _, stop in if substring == close { diff --git a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/TextViewControllerTests.swift b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/TextViewControllerTests.swift index ff714958bf..8542ff6869 100644 --- a/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/TextViewControllerTests.swift +++ b/apple/Packages/TableProEditor/Tests/TableProEditorKitTests/Controller/TextViewControllerTests.swift @@ -330,6 +330,24 @@ final class TextViewControllerTests: XCTestCase { ) } + func test_findClosingPairWithALimitOnTheWrongSideOrPastTheText() { + controller.textView.string = "{ Lorem }" + + XCTAssertNil(controller.findClosingPair("{", "}", from: 5, limit: 2, reverse: false)) + XCTAssertNil(controller.findClosingPair("}", "{", from: 2, limit: 5, reverse: true)) + XCTAssertEqual(controller.findClosingPair("{", "}", from: 1, limit: 10_000, reverse: false), 8) + } + + func test_bracketEmphasisFarPastTheVisibleTextFindsThePair() { + controller.configuration.appearance.bracketPairEmphasis = .flash + controller.setText(String(repeating: "x", count: 60_000) + "{ a }") + + controller.setCursorPositions([CursorPosition(range: NSRange(location: 60_001, length: 0))]) + + XCTAssertEqual(controller.cursorPositions.first?.range.location, 60_001) + XCTAssertEqual(controller.findClosingPair("{", "}", from: 60_001, limit: 60_005, reverse: false), 60_004) + } + // MARK: Set Text func test_setText() { diff --git a/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift b/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift index 4d776c917f..661f20f618 100644 --- a/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift +++ b/apple/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift @@ -460,11 +460,13 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } return try executeSQLite(query: query, parameters: []) } - continuation.yield(.header(PluginStreamHeader( + var header = PluginStreamHeader( columns: result.columns, columnTypeNames: result.columnTypeNames, estimatedRowCount: result.rows.count - ))) + ) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) continuation.yield(.rows(result.rows)) continuation.finish() } catch { diff --git a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift index 1b3c890c81..e362dfd30d 100644 --- a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift +++ b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Execution.swift @@ -156,9 +156,11 @@ extension BigtablePluginDriver { try await self.export(spec, continuation: continuation) } else { let result = try await self.execute(query: query) - continuation.yield(.header(PluginStreamHeader( + var header = PluginStreamHeader( columns: result.columns, columnTypeNames: result.columnTypeNames, estimatedRowCount: nil - ))) + ) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } } continuation.finish() diff --git a/apple/Plugins/CSVExportPlugin/CSVExportModels.swift b/apple/Plugins/CSVExportPlugin/CSVExportModels.swift index 2fdbd570a2..cd7942fb4b 100644 --- a/apple/Plugins/CSVExportPlugin/CSVExportModels.swift +++ b/apple/Plugins/CSVExportPlugin/CSVExportModels.swift @@ -79,7 +79,8 @@ public struct CSVExportOptions: Equatable, Codable, Sendable { public var quoteHandling: CSVQuoteHandling = .asNeeded public var lineBreak: CSVLineBreak = .lf public var decimalFormat: CSVDecimalFormat = .period - public var sanitizeFormulas: Bool = true + // Off: RFC 4180 carries a field as written, and OWASP scopes the quote prefix to files meant for a spreadsheet. + public var sanitizeFormulas: Bool = false /// What the file's bytes are written in. UTF-8 keeps every existing export byte for byte. public var encoding: PluginTextEncoding = .utf8 diff --git a/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift b/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift index a35bd63f80..f38791eced 100644 --- a/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift +++ b/apple/Plugins/CSVExportPlugin/CSVExportOptionsView.swift @@ -23,7 +23,11 @@ struct CSVExportOptionsView: View { Toggle("Sanitize formula-like values", isOn: $plugin.settings.sanitizeFormulas) .toggleStyle(.checkbox) - .help("Prevent CSV formula injection by prefixing values starting with =, +, -, @ with a single quote") + .help(""" + For a file opened in a spreadsheet. Values that would run as formulas, such as those starting \ + with =, +, - or @, get a leading single quote, which stays in the value for anything that \ + imports the file. + """) } Divider() diff --git a/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift b/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift index 6f11d384b2..f4f2bf17b3 100644 --- a/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift +++ b/apple/Plugins/CSVExportPlugin/CSVExportPlugin.swift @@ -59,6 +59,7 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi let lineBreak = options.lineBreak.value let encoding = options.encoding var report = CSVEncodingReport() + var neutralisedCount = 0 if options.writesByteOrderMark, !encoding.byteOrderMark.isEmpty { try fileHandle.write(contentsOf: Data(encoding.byteOrderMark)) @@ -80,7 +81,7 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi } var isFirstBatch = true - var columns: [String] = [] + var columnNumbers: [PluginColumnNumbers] = [] let stream = dataSource.streamRows(for: table) for try await element in stream { @@ -88,17 +89,27 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi switch element { case .header(let header): - columns = header.columns + columnNumbers = header.columnTypeNames.map { PluginColumnNumbers(typeName: $0) } if isFirstBatch && options.includeFieldNames { - let headerLine = columns - .map { escapeCSVField($0, options: options) } + if options.sanitizeFormulas { + neutralisedCount += header.columns.filter(PluginRowWriters.isFormulaLike).count + } + let headerLine = header.columns + .map { escapeCSVField($0, options: options, sanitizes: options.sanitizeFormulas) } .joined(separator: options.delimiter.actualValue) try write(headerLine + lineBreak, to: fileHandle, encoding: encoding, report: &report) } isFirstBatch = false case .rows(let rows): for row in rows { - try writeCSVRow(row, options: options, to: fileHandle, report: &report) + try writeCSVRow( + row, + columnNumbers: columnNumbers, + options: options, + to: fileHandle, + report: &report, + neutralisedCount: &neutralisedCount + ) progress.incrementRow() } } @@ -114,7 +125,22 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi try PluginExportUtilities.commitAtomicWrite(from: tempURL, to: destination) committed = true progress.finalizeTable() - return ExportFormatResult(warnings: report.warnings(for: encoding)) + return ExportFormatResult(warnings: report.warnings(for: encoding) + Self.neutralisedWarnings(neutralisedCount)) + } + + // The quote is part of the value for anything that imports the file, so the export says how many it changed. + static func neutralisedWarnings(_ count: Int) -> [String] { + guard count > 0 else { return [] } + return [String( + format: String( + localized: """ + Values a spreadsheet would read as formulas, written with a leading single quote: %lld. \ + Turn off Sanitize formula-like values to export them unchanged. + """, + bundle: .main + ), + Int64(count) + )] } // MARK: - Private @@ -143,26 +169,31 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi try fileHandle.write(contentsOf: encoded.data) } + // A text column holding "1.10" is a version or a code; a column with no usable type keeps the decimal comma. private func writeCSVRow( _ row: [PluginCellValue], + columnNumbers: [PluginColumnNumbers], options: CSVExportOptions, to fileHandle: FileHandle, - report: inout CSVEncodingReport + report: inout CSVEncodingReport, + neutralisedCount: inout Int ) throws { - let delimiter = options.delimiter.actualValue - let lineBreak = options.lineBreak.value + var fields: [String] = [] + fields.reserveCapacity(row.count) - let rowLine = row.map { cell -> String in + for (index, cell) in row.enumerated() { let val: String switch cell { case .null: - return options.convertNullToEmpty ? "" : "NULL" + fields.append(options.convertNullToEmpty ? "" : "NULL") + continue case .text(let s): val = s case .bytes(let d): val = "0x" + d.map { String(format: "%02X", $0) }.joined() } + let takesDecimalComma = index >= columnNumbers.count || columnNumbers[index] != .text var processed = val // `\r\n` is one Character, so only a scalar test sees a Windows line break. let hadLineBreaks = val.unicodeScalars.contains { $0 == "\n" || $0 == "\r" } @@ -174,40 +205,54 @@ final class CSVExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlugi .replacingOccurrences(of: "\n", with: " ", options: .literal) } - if options.decimalFormat == .comma { + var isCommaDecimal = false + if options.decimalFormat == .comma, takesDecimalComma { let range = NSRange(processed.startIndex..., in: processed) if Self.decimalFormatRegex.firstMatch(in: processed, range: range) != nil { processed = processed.replacingOccurrences(of: ".", with: ",") + isCommaDecimal = true } } - return escapeCSVField(processed, options: options, originalHadLineBreaks: hadLineBreaks) - }.joined(separator: delimiter) + // `-1,5` is still a number, and the quote would turn it into text. + let sanitizes = options.sanitizeFormulas && !isCommaDecimal + if sanitizes, PluginRowWriters.isFormulaLike(processed) { + neutralisedCount += 1 + } + fields.append(escapeCSVField( + processed, options: options, sanitizes: sanitizes, originalHadLineBreaks: hadLineBreaks + )) + } - try write(rowLine + lineBreak, to: fileHandle, encoding: options.encoding, report: &report) + try write( + fields.joined(separator: options.delimiter.actualValue) + options.lineBreak.value, + to: fileHandle, + encoding: options.encoding, + report: &report + ) } - /// Escaping and quoting live in `PluginRowWriters`, so this format, the other export formats - /// and the MCP tool spell a value the same way. Only the option mapping is this plugin's own. - /// - /// `originalHadLineBreaks` says the value's breaks were already replaced with spaces upstream, - /// and it still forces quoting: the source text spanned lines, and a reader that splits on the - /// delimiter has no way to know the space it now sees was one. - private func escapeCSVField(_ field: String, options: CSVExportOptions, originalHadLineBreaks: Bool = false) -> String { - let escaped = PluginRowWriters.csvField(field, options: writeOptions(options)) + // `originalHadLineBreaks` still forces quoting after the breaks became spaces: the source text spanned lines. + private func escapeCSVField( + _ field: String, + options: CSVExportOptions, + sanitizes: Bool, + originalHadLineBreaks: Bool = false + ) -> String { + let escaped = PluginRowWriters.csvField(field, options: writeOptions(options, sanitizes: sanitizes)) guard originalHadLineBreaks, options.quoteHandling == .asNeeded, escaped.unicodeScalars.first != "\"" else { return escaped } return "\"\(escaped.replacingOccurrences(of: "\"", with: "\"\"", options: .literal))\"" } - private func writeOptions(_ options: CSVExportOptions) -> PluginCsvWriteOptions { + private func writeOptions(_ options: CSVExportOptions, sanitizes: Bool) -> PluginCsvWriteOptions { PluginCsvWriteOptions( delimiter: options.delimiter.actualValue, quoteHandling: quoteHandling(options.quoteHandling), lineEnding: options.lineBreak.value, nullAsEmpty: true, - sanitizesFormulas: options.sanitizeFormulas, + sanitizesFormulas: sanitizes, flattensLineBreaks: false ) } diff --git a/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift b/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift index 0a3b2d9ae3..c9716c5229 100644 --- a/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift +++ b/apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift @@ -29,13 +29,13 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi var fieldDetectionSignature: String { settings.detectionSignature } - var rowImportConfiguration: RowImportRunner.Configuration? { runConfiguration } + var rowImportConfiguration: RowImportRunner.Configuration? { Self.runConfiguration(settings) } - private var runConfiguration: RowImportRunner.Configuration { + private static func runConfiguration(_ options: CSVImportOptions) -> RowImportRunner.Configuration { RowImportRunner.Configuration( - errorHandling: settings.errorHandling, - wrapInTransaction: settings.wrapInTransaction, - deleteExistingRows: settings.deleteExistingRows + errorHandling: options.errorHandling, + wrapInTransaction: options.wrapInTransaction, + deleteExistingRows: options.deleteExistingRows ) } @@ -58,10 +58,12 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi ) async throws -> PluginImportResult { let startTime = Date() let url = source.fileURL() + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-import. + let options = settings let text: CSVImportText do { - text = try CSVImportText.contents(of: url, encoding: settings.encoding, isCancelled: { progress.isCancelled }) + text = try CSVImportText.contents(of: url, encoding: options.encoding, isCancelled: { progress.isCancelled }) } catch is TabularCancellation { throw PluginImportCancellationError() } catch { @@ -80,9 +82,9 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi } let data = text.data - let dialect = CSVImportParsing.resolveDialect(in: data, options: settings) + let dialect = CSVImportParsing.resolveDialect(in: data, options: options) let parser = CSVStreamingParser(dialect: dialect) - let hasHeader = settings.hasHeaderRow + let hasHeader = options.hasHeaderRow let (dataRanges, columnNames) = indexRowsAndNames(in: data, parser: parser, hasHeader: hasHeader) guard !columnNames.isEmpty else { @@ -93,7 +95,7 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi let lineOffset = hasHeader ? 2 : 1 var cursor = 0 - let configuration = runConfiguration + let configuration = Self.runConfiguration(options) var pendingFailure: PluginImportError? var malformedCount = 0 var malformedErrors: [PluginImportResult.ImportStatementError] = [] @@ -111,7 +113,8 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi ranges: dataRanges[cursor..>, startIndex: Int, lineOffset: Int, - columnNames: [String] + columnNames: [String], + options: CSVImportOptions ) -> (rows: [RowImportRunner.Entry], malformed: [MalformedRow]) { data.withUnsafeBytes { raw -> (rows: [RowImportRunner.Entry], malformed: [MalformedRow]) in guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return ([], []) } @@ -185,12 +189,12 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi if CSVImportParsing.isBlank(fields) { continue } let line = startIndex + offset + lineOffset if let error = CSVImportParsing.shapeError( - fields: fields, columnCount: columnNames.count, options: settings + fields: fields, columnCount: columnNames.count, options: options ) { malformed.append((line, error)) continue } - out.append((line, CSVImportParsing.row(fields: fields, columnNames: columnNames, options: settings))) + out.append((line, CSVImportParsing.row(fields: fields, columnNames: columnNames, options: options))) } return (out, malformed) } diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift index f59b0a903d..ff8717faec 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift @@ -324,7 +324,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen let rawResult = try await connectionActor.executeQuery( query, consistency: consistency, cancellation: cancellation ) - if let refusal = CassandraRowWriter.unappliedInsertRefusal( + if let refusal = CassandraRowWriter.unappliedWriteRefusal( statement: query, columns: rawResult.columns, rows: rawResult.rows ) { throw CassandraPluginError.queryFailed(refusal) @@ -371,7 +371,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen let rawResult = try await connectionActor.executePrepared( query, parameters: parameters, consistency: consistency ) - if let refusal = CassandraRowWriter.unappliedInsertRefusal( + if let refusal = CassandraRowWriter.unappliedWriteRefusal( statement: query, columns: rawResult.columns, rows: rawResult.rows ) { throw CassandraPluginError.queryFailed(refusal) diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift b/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift index 0c7ce00b73..ca4edf9125 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift @@ -4,16 +4,15 @@ // import Foundation +import TableProCassandraCore import TableProPluginKit -/// The CQL a grid save writes: one statement per row, each naming the row by its whole primary key. -/// -/// CQL has no `OR` and no parenthesised groups, so several rows cannot share one `DELETE`, and an `INSERT` is an -/// upsert, so a new row whose key is taken would silently replace the row already there. A new row is therefore -/// written `IF NOT EXISTS`, and anything CQL cannot do to a row is refused by name rather than sent. +// One statement per row, each naming the row by its whole primary key: CQL has no `OR` and no parenthesised groups. +// A new row is written `IF NOT EXISTS` and an edit `IF EXISTS` (CassandraConditionalWrite), and anything CQL cannot +// do to a row is refused by name rather than sent. enum CassandraRowWriter { static let defaultSentinel = "__DEFAULT__" - static let unappliedMarker = "[applied]" + static let unappliedMarker = CassandraConditionalWrite.appliedColumn static func rowWrites( keyspace: String?, @@ -54,15 +53,18 @@ enum CassandraRowWriter { } } - /// An `INSERT … IF NOT EXISTS` that found its key taken answers with `[applied] = false` and the row already - /// there, which the server reports as a success. - static func unappliedInsertRefusal(statement: String, columns: [String], rows: [[PluginCellValue]]) -> String? { - let normalized = statement.uppercased() - guard normalized.hasPrefix("INSERT"), normalized.hasSuffix("IF NOT EXISTS"), - columns.first == unappliedMarker, - rows.first?.first?.asText?.lowercased() == "false" - else { return nil } - return String(localized: "A row with this primary key already exists. Edit that row instead.") + static func unappliedWriteRefusal(statement: String, columns: [String], rows: [[PluginCellValue]]) -> String? { + let refusal = CassandraConditionalWrite.refusal( + statement: statement, firstColumn: columns.first, firstValue: rows.first?.first?.asText + ) + switch refusal { + case .keyTaken: + return String(localized: "A row with this primary key already exists. Edit that row instead.") + case .rowMissing: + return String(localized: "No row has this primary key, so nothing was updated. Another client may have deleted it.") + case nil: + return nil + } } private struct Writer { @@ -100,8 +102,7 @@ enum CassandraRowWriter { )) } } - let statement = "INSERT INTO \(target) (\(names.joined(separator: ", "))) " - + "VALUES (\(placeholders.joined(separator: ", "))) IF NOT EXISTS" + let statement = CassandraConditionalWrite.insertStatement(target: target, names: names, values: placeholders) return PluginRowWrite(statement: statement, parameters: parameters, rowIndices: [change.rowIndex]) } @@ -133,7 +134,9 @@ enum CassandraRowWriter { } } let key = try keyCondition(for: change) - let statement = "UPDATE \(target) SET \(assignments.joined(separator: ", ")) WHERE \(key.cql)" + let statement = CassandraConditionalWrite.updateStatement( + target: target, assignments: assignments, keyCondition: key.cql + ) return PluginRowWrite(statement: statement, parameters: parameters + key.values, rowIndices: [change.rowIndex]) } diff --git a/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift b/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift index cc80a87298..70b89896ec 100644 --- a/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift +++ b/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift @@ -323,10 +323,9 @@ final class DamengPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let task = Task { do { let result = try await self.execute(query: query, context: context) - continuation.yield(.header(PluginStreamHeader( - columns: result.columns, - columnTypeNames: result.columnTypeNames - ))) + var header = PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } diff --git a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift index ee013d3fe9..72389cdf12 100644 --- a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift +++ b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver+Execution.swift @@ -373,7 +373,9 @@ extension DynamoDBPluginDriver { source = try await spoolSource(for: statement, session: session) } catch DynamoDBError.invalidStatement(let message) where message == Self.notAReadMarker { let result = try await runStatement(text, parameters: [], rowCap: PluginRowLimits.emergencyMax) - continuation.yield(.header(PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames))) + var header = PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } return } diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift index e8d3c50f61..038ed7dbcc 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift @@ -242,11 +242,11 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation: AsyncThrowingStream.Continuation ) async throws { let matcher = try filter.matcher() - continuation.yield(.header(PluginStreamHeader( + var header = PluginStreamHeader( columns: Self.columns, columnTypeNames: Self.columnTypeNames, estimatedRowCount: nil - ))) + ) let range = Self.prefixRange(prefix) guard sortAscending else { @@ -256,6 +256,9 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { req.sortTarget = "KEY" let response = try await client.rangeRequest(req) let rows = (response.kvs ?? []).map(Self.filterRow).filter(matcher.matches).map(\.cells) + // `more` says the range held keys past the limit. + if response.more == true { header.rowLimit = rows.count } + continuation.yield(.header(header)) if !rows.isEmpty { continuation.yield(.rows(rows)) } @@ -263,6 +266,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return } + continuation.yield(.header(header)) do { try await scanKeyValues(in: range, client: client) { kvs in let rows = kvs.map(Self.filterRow).filter(matcher.matches).map(\.cells) diff --git a/apple/Plugins/FormatPluginTests/CSVExportFormulaAndDecimalTests.swift b/apple/Plugins/FormatPluginTests/CSVExportFormulaAndDecimalTests.swift new file mode 100644 index 0000000000..db59df3638 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/CSVExportFormulaAndDecimalTests.swift @@ -0,0 +1,174 @@ +// +// CSVExportFormulaAndDecimalTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +private final class TypedCSVSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId = "PostgreSQL" + private let header: PluginStreamHeader + private let rows: [PluginRow] + + init(columns: [String], columnTypeNames: [String], rows: [PluginRow]) { + header = PluginStreamHeader(columns: columns, columnTypeNames: columnTypeNames) + self.rows = rows + } + + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + let header = header + let rows = rows + return AsyncThrowingStream { continuation in + continuation.yield(.header(header)) + continuation.yield(.rows(rows)) + continuation.finish() + } + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { "" } + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { "\"\(identifier)\"" } + + func escapeStringLiteral(_ value: String) -> String { value } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { nil } +} + +struct CSVExportFormulaAndDecimalTests { + private func export( + _ options: CSVExportOptions, + columns: [String], + types: [String], + rows: [PluginRow] + ) async throws -> (text: String, warnings: [String]) { + let output = try await CSVExportHarness.shared.bytes( + options: options, + tables: [PluginExportTable(name: "users", databaseName: "", tableType: "table", schema: nil, kind: .table)], + dataSource: TypedCSVSource(columns: columns, columnTypeNames: types, rows: rows) + ) + return (String(decoding: output.data, as: UTF8.self), output.result.warnings) + } + + // A database export is read back by importers, which keep the quote as part of the value. + @Test("The stock export writes a value that starts like a formula as it is") + func stockExportKeepsValues() async throws { + let output = try await export( + CSVExportOptions(), + columns: ["handle"], + types: ["text"], + rows: [["@handle"], ["+84 90 123 4567"], ["-pending"]] + ) + #expect(output.text == "handle\n@handle\n+84 90 123 4567\n-pending\n") + #expect(output.warnings.isEmpty) + } + + @Test("Sanitizing says how many values it gave a leading quote") + func sanitizingCountsWhatItChanged() async throws { + var options = CSVExportOptions() + options.sanitizeFormulas = true + let output = try await export( + options, + columns: ["handle"], + types: ["text"], + rows: [["@handle"], ["plain"], ["=1+1"], ["-5"], ["-pending"]] + ) + #expect(output.text == "handle\n\"'@handle\"\nplain\n\"'=1+1\"\n-5\n\"'-pending\"\n") + #expect(output.warnings == CSVExportPlugin.neutralisedWarnings(3)) + } + + @Test("Nothing changed means no warning about changes") + func noCountWithoutChanges() { + #expect(CSVExportPlugin.neutralisedWarnings(0).isEmpty) + } + + // A text column holding "1.10" is a version or a code, not a number. + @Test("A decimal comma leaves text columns alone and keeps columns with no usable type") + func decimalCommaSkipsTextColumns() async throws { + var options = CSVExportOptions() + options.decimalFormat = .comma + options.delimiter = .semicolon + let output = try await export( + options, + columns: ["version", "amount", "untyped"], + types: ["varchar(16)", "numeric(10,2)", ""], + rows: [["1.10", "1.10", "1.10"]] + ) + #expect(output.text == "version;amount;untyped\n1.10;1,10;1,10\n") + } + + @Test("A decimal comma reaches every engine's numeric type", arguments: [ + "Nullable(Float64)", "Nullable(Decimal(10, 2))", "money", "smallmoney", "BINARY_DOUBLE", "BINARY_FLOAT", + "BIGNUMERIC", "double precision", "NUMBER(10,2)" + ]) + func decimalCommaReachesNumericTypes(typeName: String) async throws { + var options = CSVExportOptions() + options.decimalFormat = .comma + options.delimiter = .semicolon + let output = try await export(options, columns: ["n"], types: [typeName], rows: [["1.5"]]) + #expect(output.text == "n\n1,5\n") + } + + @Test("A decimal comma never reaches a text type", arguments: [ + "varchar(16)", "String", "LowCardinality(String)", "nvarchar(10)", "TEXT", "jsonb" + ]) + func decimalCommaSkipsTextTypes(typeName: String) async throws { + var options = CSVExportOptions() + options.decimalFormat = .comma + options.delimiter = .semicolon + let output = try await export(options, columns: ["n"], types: [typeName], rows: [["1.5"]]) + #expect(output.text == "n\n1.5\n") + } + + @Test("A header name sanitizing changes is counted with the values") + func headerNamesAreCounted() async throws { + var options = CSVExportOptions() + options.sanitizeFormulas = true + let output = try await export(options, columns: ["=total"], types: ["text"], rows: [["=1"]]) + #expect(output.text == "\"'=total\"\n\"'=1\"\n") + #expect(output.warnings == CSVExportPlugin.neutralisedWarnings(2)) + } + + // `'-1,5` is text to a spreadsheet, so a negative number written with a decimal comma lost its value. + @Test("A number written with a decimal comma is never given a quote") + func commaDecimalsAreNotSanitized() async throws { + var options = CSVExportOptions() + options.sanitizeFormulas = true + options.decimalFormat = .comma + options.delimiter = .semicolon + let output = try await export( + options, + columns: ["amount", "note"], + types: ["numeric", "text"], + rows: [["-1.5", "-1.5x"]] + ) + #expect(output.text == "amount;note\n-1,5;\"'-1.5x\"\n") + #expect(output.warnings == CSVExportPlugin.neutralisedWarnings(1)) + } +} + +struct PluginRowWritersFormulaLeadTests { + // OWASP lists a leading line feed and the full-width operators, which a Japanese locale reads as formulas. + @Test("A leading line feed or a full-width operator counts as a formula lead") + func owaspLeadsAreNeutralised() { + let options = PluginCsvWriteOptions.default + #expect(PluginRowWriters.csvField("\n=1+1", options: options).hasPrefix("\"'")) + for lead in ["\u{FF1D}", "\u{FF0B}", "\u{FF0D}", "\u{FF20}"] { + #expect(PluginRowWriters.csvField("\(lead)SUM(A1)", options: options) == "\"'\(lead)SUM(A1)\"") + } + } + + @Test("Only a value that would be prefixed is formula-like") + func formulaLikeMatchesTheWriter() { + #expect(PluginRowWriters.isFormulaLike("=1+1")) + #expect(PluginRowWriters.isFormulaLike("\t=1+1")) + #expect(!PluginRowWriters.isFormulaLike("-5")) + #expect(!PluginRowWriters.isFormulaLike("plain")) + #expect(!PluginRowWriters.isFormulaLike("")) + } +} diff --git a/apple/Plugins/FormatPluginTests/ExportImportRoundTripTests.swift b/apple/Plugins/FormatPluginTests/ExportImportRoundTripTests.swift index 29503ea388..ed0cae7764 100644 --- a/apple/Plugins/FormatPluginTests/ExportImportRoundTripTests.swift +++ b/apple/Plugins/FormatPluginTests/ExportImportRoundTripTests.swift @@ -85,6 +85,14 @@ struct ExportImportRoundTripTests { try RoundTrip.expectSame(target, RoundTripFixture.table, FormatLimits.csv.carried(RoundTripFixture.rows)) } + @Test("A CSV file sanitized for a spreadsheet imports back with the quote it gave formula-like values") + func csvSanitizedImportsBackWithItsQuotes() async throws { + var options = CSVExportOptions() + options.sanitizeFormulas = true + let target = try await RoundTrip.csv(options) + try RoundTrip.expectSame(target, RoundTripFixture.table, FormatLimits.csvSanitized.carried(RoundTripFixture.rows)) + } + @Test("A CSV file written with a byte order mark imports back") func csvWithByteOrderMarkImportsBack() async throws { var options = CSVExportOptions() @@ -166,9 +174,14 @@ struct ExportImportRoundTripTests { extension FormatLimits { // CSV has no NULL and no binary: both NULL and '' are written as an empty field and read as - // NULL, and a binary value is written as hex text. A value a spreadsheet would run as a - // formula gains a leading quote. + // NULL, and a binary value is written as hex text. static let csv = FormatLimits( + emptyTextIsNull: true, + binary: { .text(FormatLimits.hex($0)) } + ) + + // Sanitizing for a spreadsheet gives a value that would run as a formula a leading quote, which the import keeps. + static let csvSanitized = FormatLimits( emptyTextIsNull: true, binary: { .text(FormatLimits.hex($0)) }, text: { FormatLimits.csvNeutralised[$0] ?? $0 } diff --git a/apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift b/apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift new file mode 100644 index 0000000000..f995008f40 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/FormatSettingsChangedMidRunTests.swift @@ -0,0 +1,185 @@ +// +// FormatSettingsChangedMidRunTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +// One plugin instance serves every window, so a dialog open in another window changes `settings` while a run reads. +// Each source here changes them between its first and second batch, the way that dialog would. +private final class TwoBatchSource: PluginExportDataSource, @unchecked Sendable { + let databaseTypeId = "SQLite" + private let header: PluginStreamHeader + private let batches: [[PluginRow]] + private let betweenBatches: @Sendable () -> Void + + init(columns: [String], batches: [[PluginRow]], betweenBatches: @escaping @Sendable () -> Void) { + header = PluginStreamHeader(columns: columns, columnTypeNames: columns.map { _ in "TEXT" }) + self.batches = batches + self.betweenBatches = betweenBatches + } + + // Pulled, so the change lands only after the plugin has written the first batch. + func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { + let header = header + let batches = batches + let betweenBatches = betweenBatches + let pulls = Counter() + return AsyncThrowingStream(unfolding: { + let pull = pulls.next() + if pull == 0 { return .header(header) } + guard pull <= batches.count else { return nil } + if pull == 2 { betweenBatches() } + return .rows(batches[pull - 1]) + }) + } + + func fetchTableDDL(table: String, databaseName: String) async throws -> String { "" } + + func execute(query: String) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ identifier: String) -> String { "\"\(identifier)\"" } + + func escapeStringLiteral(_ value: String) -> String { value } + + func fetchApproximateRowCount(table: String, databaseName: String) async throws -> Int? { nil } +} + +private final class Counter: @unchecked Sendable { + private let lock = NSLock() + private var value = 0 + + func next() -> Int { + lock.lock() + defer { lock.unlock() } + defer { value += 1 } + return value + } +} + +private final class ForeignKeyRecordingSink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "SQLite" + var onExecute: () -> Void = {} + private(set) var disabled = 0 + private(set) var enabled = 0 + + func execute(statement: String) async throws { onExecute() } + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} + func disableForeignKeyChecks() async throws { disabled += 1 } + func enableForeignKeyChecks() async throws { enabled += 1 } +} + +private final class OneStatementSource: PluginImportSource, @unchecked Sendable { + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + AsyncThrowingStream { continuation in + continuation.yield((statement: "INSERT INTO t VALUES (1);", lineNumber: 1)) + continuation.finish() + } + } + + func fileURL() -> URL { URL(fileURLWithPath: "/dev/null") } + + func fileSizeBytes() -> Int64 { 0 } +} + +@Suite(.serialized) +struct FormatSettingsChangedMidRunTests { + private let table = PluginExportTable(name: "t", databaseName: "", tableType: "table", schema: nil, kind: .table) + private let rowWithNull: PluginRow = [.text("1"), .null] + + private func export( + _ plugin: any ExportFormatPlugin, + from source: TwoBatchSource, + fileExtension: String + ) async throws -> Data { + let destination = FileManager.default.temporaryDirectory + .appendingPathComponent("\(UUID().uuidString).\(fileExtension)") + defer { try? FileManager.default.removeItem(at: destination) } + _ = try await plugin.export( + tables: [table], + dataSource: source, + destination: destination, + progress: PluginExportProgress(progress: Progress(totalUnitCount: 1)) + ) + return try Data(contentsOf: destination) + } + + @Test("JSON keeps writing null keys after another window turns Include NULL values off") + func jsonKeepsItsNullChoice() async throws { + let plugin = JSONExportPlugin() + let stored = plugin.settings + defer { plugin.settings = stored } + var options = JSONExportOptions() + options.layout = .newlineDelimited + options.includeNullValues = true + plugin.settings = options + + let source = TwoBatchSource(columns: ["id", "note"], batches: [[rowWithNull], [rowWithNull]]) { + plugin.settings.includeNullValues = false + } + let text = String(decoding: try await export(plugin, from: source, fileExtension: "ndjson"), as: UTF8.self) + #expect(text == "{\"id\": \"1\", \"note\": null}\n{\"id\": \"1\", \"note\": null}\n") + } + + @Test("Markdown keeps its NULL placeholder after another window changes it") + func markdownKeepsItsPlaceholder() async throws { + let plugin = MarkdownExportPlugin() + let stored = plugin.settings + defer { plugin.settings = stored } + var options = MarkdownExportOptions() + options.alignsColumns = false + options.includesTableNames = false + options.nullPlaceholder = "NULL" + plugin.settings = options + + let source = TwoBatchSource(columns: ["id", "note"], batches: [[rowWithNull], [rowWithNull]]) { + plugin.settings.nullPlaceholder = "(none)" + } + let text = String(decoding: try await export(plugin, from: source, fileExtension: "md"), as: UTF8.self) + #expect(!text.contains("(none)")) + #expect(text.components(separatedBy: "| 1 | NULL |").count == 3) + } + + @Test("XLSX keeps writing NULL as an empty cell after another window turns the option off") + func xlsxKeepsItsNullChoice() async throws { + let plugin = XLSXExportPlugin() + let stored = plugin.settings + defer { plugin.settings = stored } + var options = XLSXExportOptions() + options.convertNullToEmpty = true + plugin.settings = options + + let source = TwoBatchSource(columns: ["id", "note"], batches: [[rowWithNull], [rowWithNull]]) { + plugin.settings.convertNullToEmpty = false + } + let written = try await export(plugin, from: source, fileExtension: "xlsx") + let text = try #require(String(bytes: written, encoding: .isoLatin1)) + #expect(!text.contains("NULL")) + } + + @Test("A SQL import turns foreign key checks back on after another window unticks the option") + func sqlImportRestoresTheChecksItTurnedOff() async throws { + let plugin = SQLImportPlugin() + let stored = plugin.settings + defer { plugin.settings = stored } + var options = SQLImportOptions() + options.disableForeignKeyChecks = true + plugin.settings = options + + let sink = ForeignKeyRecordingSink() + sink.onExecute = { plugin.settings.disableForeignKeyChecks = false } + _ = try await plugin.performImport( + source: OneStatementSource(), + sink: sink, + progress: PluginImportProgress(progress: Progress()) + ) + #expect(sink.disabled == 1) + #expect(sink.enabled == 1) + } +} diff --git a/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift b/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift index cf23a29b92..60adecb627 100644 --- a/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift +++ b/apple/Plugins/JSONExportPlugin/JSONExportPlugin.swift @@ -53,10 +53,11 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug } } - /// NDJSON is one row per line by definition, so the pretty-print setting cannot apply to it - /// and the wrapping object has no place to go. - let isNewlineDelimited = settings.layout == .newlineDelimited - let prettyPrint = settings.prettyPrint && !isNewlineDelimited + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-export. + let options = settings + // NDJSON is one row per line by definition, so pretty-printing and the wrapping object do not apply. + let isNewlineDelimited = options.layout == .newlineDelimited + let prettyPrint = options.prettyPrint && !isNewlineDelimited let indent = prettyPrint ? " " : "" let newline = prettyPrint ? "\n" : "" @@ -104,7 +105,7 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug for (colIndex, column) in columns.enumerated() { if colIndex < row.count { let value = row[colIndex] - if settings.includeNullValues || !value.isNull { + if options.includeNullValues || !value.isNull { if !isFirstField { rowString += ", " } @@ -117,7 +118,7 @@ final class JSONExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug let jsonValue = formatJSONValue( value, columnTypeName: colTypeName, - preserveAsString: settings.preserveAllAsStrings + preserveAsString: options.preserveAllAsStrings ) rowString += "\"\(escapedKey)\": \(jsonValue)" } diff --git a/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift b/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift index 2ee2f17664..8fc08e0fd3 100644 --- a/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift +++ b/apple/Plugins/MarkdownExportPlugin/MarkdownExportPlugin.swift @@ -54,13 +54,15 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable if !committed { PluginExportUtilities.rollbackAtomicWrite(at: tempURL) } } + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-export. + let options = settings for (index, table) in tables.enumerated() { try progress.checkCancellation() progress.setCurrentTable(table.qualifiedName, index: index + 1) - if settings.includesTableNames { + if options.includesTableNames { try fileHandle.write(contentsOf: "## \(table.qualifiedName)\n\n".toUTF8Data()) } - try await writeTable(table, dataSource: dataSource, to: fileHandle, progress: progress) + try await writeTable(table, options: options, dataSource: dataSource, to: fileHandle, progress: progress) try fileHandle.write(contentsOf: "\n".toUTF8Data()) } @@ -73,6 +75,7 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable private func writeTable( _ table: PluginExportTable, + options: MarkdownExportOptions, dataSource: any PluginExportDataSource, to fileHandle: FileHandle, progress: PluginExportProgress @@ -89,9 +92,9 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable columns = header.columns case .rows(let rows): for row in rows { - let cells = row.map { cellText($0) } + let cells = row.map { cellText($0, nullPlaceholder: options.nullPlaceholder) } progress.incrementRow() - guard settings.alignsColumns, !wroteHeader else { + guard options.alignsColumns, !wroteHeader else { if !wroteHeader { try writeHeader(columns, widths: nil, to: fileHandle) wroteHeader = true @@ -115,7 +118,7 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable } if !wroteHeader { - widths = settings.alignsColumns + widths = options.alignsColumns ? MarkdownTableRenderer.widths(header: columns, sample: buffered) : nil try writeHeader(columns, widths: widths, to: fileHandle) @@ -138,10 +141,10 @@ final class MarkdownExportPlugin: ObservableObject, ExportFormatPlugin, Settable ).toUTF8Data()) } - private func cellText(_ value: PluginCellValue) -> String { + private func cellText(_ value: PluginCellValue, nullPlaceholder: String) -> String { switch value { case .null: - return MarkdownTableRenderer.cell(settings.nullPlaceholder) + return MarkdownTableRenderer.cell(nullPlaceholder) case .bytes(let data): return MarkdownTableRenderer.cell("0x\(data.map { String(format: "%02X", $0) }.joined())") case .text(let text): diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift index 3be35d7dd1..55c8df7418 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBConnection+SyncHelpers.swift @@ -518,7 +518,10 @@ nonisolated extension MongoDBConnection { let documents = try MongoExportDocument.read( MongoReadDocuments(dictionaries: sample, texts: texts), representation: uuidRepresentation ) - continuation.yield(.header(projection.header)) + var header = projection.header + // iterateCursorStreaming stops there, so a stream that reaches it was cut. + header.rowLimit = PluginRowLimits.emergencyMax + continuation.yield(.header(header)) if !sample.isEmpty { let rows = BsonDocumentFlattener.flatten( documents: sample, columns: projection.columns, kinds: projection.kinds, diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 5e10afbea4..05bfefbd69 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -1031,10 +1031,9 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se } ) if !result.columns.isEmpty { - continuation.yield(.header(PluginStreamHeader( - columns: result.columns, - columnTypeNames: result.columnTypeNames - ))) + var header = PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) } if !result.rows.isEmpty { continuation.yield(.rows(result.rows, documents: result.exportDocuments)) diff --git a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift index edea147522..ba69a63dc3 100644 --- a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift +++ b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift @@ -88,7 +88,19 @@ struct MariaDBPluginQueryResult { // MARK: - Connection Class final class MariaDBPluginConnection: @unchecked Sendable { - private var mysql: UnsafeMutablePointer? + // Under stateLock, which disconnect() clears it under. Only `queue` may call into the handle: the close is queued there. + private var _mysql: UnsafeMutablePointer? + private var _serverThreadId: UInt = 0 + + private var mysql: UnsafeMutablePointer? { + stateLock.withLock { _mysql } + } + + // Read from the snapshot taken at connect: Stop runs off `queue`, where the handle may already be freed. + private var connectedThreadId: UInt? { + stateLock.withLock { _mysql == nil ? nil : _serverThreadId } + } + private let queue = DispatchQueue(label: "com.TablePro.mariadb.plugin", qos: .userInitiated) /// Serial, and separate from `queue`, which the read being cancelled is sitting on. Serial so a @@ -342,9 +354,9 @@ final class MariaDBPluginConnection: @unchecked Sendable { if let cachePath = kerberos?.cachePath { KerberosTicket.discard(cachePath) } - let handle = mysql + let handle = _mysql let cleanupQueue = queue - mysql = nil + _mysql = nil if let handle = handle { cleanupQueue.async { mysql_close(handle) @@ -390,8 +402,10 @@ final class MariaDBPluginConnection: @unchecked Sendable { self.baselineSelectLimit = nil self.hasCapturedBaselineSelectLimit = false + let threadId = mysql_thread_id(handle) self.stateLock.lock() - self.mysql = handle + self._mysql = handle + self._serverThreadId = threadId self._isConnected = true self._errorLanguage = session.errorLanguage self._sessionEndedByKill = false @@ -600,10 +614,10 @@ final class MariaDBPluginConnection: @unchecked Sendable { KerberosTicket.discard(cachePath) } - let handle = mysql - mysql = nil - stateLock.lock() + let handle = _mysql + _mysql = nil + _serverThreadId = 0 _isConnected = false stateLock.unlock() @@ -635,7 +649,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { let target = killTarget guard mayEndSession || !target.endsSession else { return } - guard let mysql = mysql, let statement = target.statement(threadId: mysql_thread_id(mysql)) else { return } + guard let threadId = connectedThreadId, let statement = target.statement(threadId: threadId) else { return } cancelQueue.async { [self] in guard deliverKill(statement) else { return } if target.endsSession { @@ -657,11 +671,9 @@ final class MariaDBPluginConnection: @unchecked Sendable { recordKillDelivered(generation: generation) } - /// The server thread this connection is on, read before a statement goes out so a kill still - /// has somewhere to go once the handle itself is unusable. + // Read before a statement goes out, so a kill still has somewhere to go once the handle itself is unusable. internal var currentThreadId: UInt { - guard let mysql = self.mysql else { return 0 } - return mysql_thread_id(mysql) + connectedThreadId ?? 0 } /// The kill has to reach the server the query is running on, which means repeating the transport diff --git a/apple/Plugins/MySQLDriverTests/MariaDBStopThreadIdTests.swift b/apple/Plugins/MySQLDriverTests/MariaDBStopThreadIdTests.swift new file mode 100644 index 0000000000..52bb160f9a --- /dev/null +++ b/apple/Plugins/MySQLDriverTests/MariaDBStopThreadIdTests.swift @@ -0,0 +1,38 @@ +// +// MariaDBStopThreadIdTests.swift +// MySQLDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +// Stop runs off the statement queue, where a disconnect's queued close can free the handle under it, so the kill's +// thread id is the one the handshake gave, held beside the handle and cleared with it. +@Suite(.timeLimit(.minutes(1))) +struct MariaDBStopThreadIdTests { + @Test("The kill's thread id is the session's own, and none is left once the session is closed") + func threadIdFollowsTheSession() async throws { + let server = try PlaintextMySQLServer(asksForCleartext: false) + defer { server.close() } + async let session = server.serveOneSession() + let connection = MariaDBPluginConnection( + transport: .tcp(host: "127.0.0.1", port: UInt32(server.port)), + user: "app", + password: "app-password", + database: "", + sslConfig: SSLConfiguration(mode: .disabled), + enableCleartextPlugin: false + ) + + try await connection.connect() + let signedIn = connection.currentThreadId + connection.disconnect() + let closed = connection.currentThreadId + _ = await session + + // The fake server's handshake names connection 1. + #expect(signedIn == 1) + #expect(closed == 0) + } +} diff --git a/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift b/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift index 90eeda01e8..2495d396c6 100644 --- a/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift +++ b/apple/Plugins/ParquetExportPlugin/ParquetExportPlugin.swift @@ -59,10 +59,12 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP progress: PluginExportProgress ) async throws -> ExportFormatResult { guard !tables.isEmpty else { return ExportFormatResult() } + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-export. + let options = settings var warnings: [String] = [] try await ParquetTableFiles.write(tables, destination: destination, progress: progress) { table, fileURL in - warnings += try await writeTable(table, dataSource: dataSource, to: fileURL, progress: progress) + warnings += try await writeTable(table, options: options, dataSource: dataSource, to: fileURL, progress: progress) } if tables.count > 1 { @@ -76,6 +78,7 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP private func writeTable( _ table: PluginExportTable, + options: ParquetExportOptions, dataSource: any PluginExportDataSource, to fileURL: URL, progress: PluginExportProgress @@ -123,8 +126,8 @@ final class ParquetExportPlugin: ObservableObject, ExportFormatPlugin, SettableP } let fallbacks = try staging.copyToParquet( fileURL: fileURL, - compression: settings.compression.duckDBValue, - rowGroupSize: settings.rowGroupSize + compression: options.compression.duckDBValue, + rowGroupSize: options.rowGroupSize ) return fallbacks.map { fallback in String( diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift index c2f93088de..6ac131744d 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift @@ -141,8 +141,9 @@ extension RedisPluginDriver { if opts.xx { args.append("XX".redisArgument) } if opts.keepTTL { args.append("KEEPTTL".redisArgument) } } - try await conn.run(args) - return buildStatusResult("OK", startTime: startTime) + // NX or XX that stops the write answers nil, not an error. + let reply = try await conn.run(args) + return buildStatusResult(reply.displayText, startTime: startTime) case .del(let keys): let args = ["DEL"] + keys diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift index 43d70ee2b9..4d4ffcb41b 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift @@ -561,11 +561,13 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { default: let startTime = Date() let result = try await executeOperation(operation, connection: conn, startTime: startTime) - continuation.yield(.header(PluginStreamHeader( + var header = PluginStreamHeader( columns: result.columns, columnTypeNames: result.columnTypeNames, estimatedRowCount: nil - ))) + ) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } diff --git a/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift b/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift index 72e4aa3bd3..4402cf7e01 100644 --- a/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisConditionalWriteTests.swift @@ -180,6 +180,25 @@ struct RedisConditionalWriteTests { #expect(result.rows.first?.first?.asText == "OK") } + @Test("A SET that NX or XX kept from writing shows nil, as the server answered") + func skippedSetShowsNil() async throws { + let channel = StubRedisChannel([.null]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("SET lock:job 1 NX"), connection: channel, startTime: Date() + ) + #expect(result.rows.first?.first?.asText == "(nil)") + #expect(channel.sentCommands == [["SET", "lock:job", "1", "NX"]]) + } + + @Test("A SET that wrote the key shows OK") + func appliedSetShowsOK() async throws { + let channel = StubRedisChannel([.status("OK")]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("SET lock:job 1 XX EX 60"), connection: channel, startTime: Date() + ) + #expect(result.rows.first?.first?.asText == "OK") + } + @Test("A conditional write the server refused outright keeps the server's message") func execKeepsServerErrorForConditionalWrite() { let reply = RedisReply.array([.error("ERR no such key")]) diff --git a/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift b/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift index 91b6a9c205..137d0a7ac3 100644 --- a/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift +++ b/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift @@ -51,15 +51,17 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi var errors: [PluginImportResult.ImportStatementError] = [] let maxErrors = 1_000 - let errorMode = settings.errorHandling - let useTransaction = settings.wrapInTransaction && errorMode != .skipAndContinue + // One plugin instance serves every window: a checkbox unticked elsewhere mid-import left the checks off. + let options = settings + let errorMode = options.errorHandling + let useTransaction = options.wrapInTransaction && errorMode != .skipAndContinue let fileSizeBytes = source.fileSizeBytes() let estimatedTotal = max(1, Int(fileSizeBytes / 500)) progress.setEstimatedTotal(estimatedTotal) do { - if settings.disableForeignKeyChecks { + if options.disableForeignKeyChecks { try await sink.disableForeignKeyChecks() } @@ -99,7 +101,7 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi Self.logger.warning("Failed to commit partial import") } } - if settings.disableForeignKeyChecks { + if options.disableForeignKeyChecks { do { try await sink.enableForeignKeyChecks() } catch { @@ -134,7 +136,7 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi didCommit = true } - if settings.disableForeignKeyChecks { + if options.disableForeignKeyChecks { do { try await sink.enableForeignKeyChecks() } catch { @@ -156,7 +158,7 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi } } - if settings.disableForeignKeyChecks { + if options.disableForeignKeyChecks { do { try await sink.enableForeignKeyChecks() } catch { diff --git a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index dc8a667c2c..53d2af6929 100644 --- a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -1567,7 +1567,8 @@ public extension PluginDatabaseDriver { } func escapedParameterValue(_ value: String) -> String { - if Self.isNumericLiteral(value) { + // No numeric column reads as `00123` or `-007`, so that text is a code or a zip and stays a string. + if Self.isNumericLiteral(value), !opensWithPaddingZero(value) { return value } return PluginSQLQuoting.stringLiteral(value, escapes: PluginSQLQuoting.Escapes(features: literalFeatures)) @@ -1611,3 +1612,10 @@ extension PluginDatabaseDriver { return backslash ? [.backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes] : [] } } + +// `007`, `-0042`: an integer part that opens with 0 and goes on with another digit. +private func opensWithPaddingZero(_ value: String) -> Bool { + let unsigned = value.unicodeScalars.drop { $0 == "-" || $0 == "+" } + guard unsigned.first == "0", let next = unsigned.dropFirst().first else { return false } + return ("0"..."9").contains(next) +} diff --git a/apple/Plugins/TableProPluginKit/PluginExportUtilities.swift b/apple/Plugins/TableProPluginKit/PluginExportUtilities.swift index 726b45de69..a89463cd38 100644 --- a/apple/Plugins/TableProPluginKit/PluginExportUtilities.swift +++ b/apple/Plugins/TableProPluginKit/PluginExportUtilities.swift @@ -109,6 +109,72 @@ public enum PluginExportUtilities { } } +// Shared so XLSX number cells and the CSV decimal comma read a column's declared type the same way. +public enum PluginColumnNumbers: Equatable, Sendable { + case text + case exact + case approximate + case untyped + + public init(typeName: String, bitsAreStrings: Bool = false) { + let base = Self.baseName(typeName) + if base.isEmpty { + self = .untyped + } else if Self.bitTypes.contains(base) { + self = bitsAreStrings ? .text : .exact + } else if Self.approximateTypes.contains(base) { + self = .approximate + } else if Self.exactTypes.contains(base) { + self = .exact + } else if Self.textTypes.contains(base) || Self.textMarkers.contains(where: { base.contains($0) }) { + self = .text + } else { + self = .untyped + } + } + + public var isNumeric: Bool { + self == .exact || self == .approximate + } + + // `Nullable(UInt64)` and `LowCardinality(Nullable(Int32))` name the type inside. The base is the + // first word without its width: `int4`, `Int64` and `INT UNSIGNED` are all `int`. + private static func baseName(_ typeName: String) -> String { + var name = typeName.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() + while name.hasSuffix(")"), let open = name.firstIndex(of: "("), + ["nullable", "lowcardinality"].contains(String(name[.. = [ + "int", "uint", "integer", "bigint", "smallint", "tinyint", "mediumint", "hugeint", + "uinteger", "ubigint", "usmallint", "utinyint", "uhugeint", "decimal", "dec", "numeric", + "number", "bignumeric", "serial", "bigserial", "smallserial", "money", "smallmoney", "year", "oid" + ] + + private static let approximateTypes: Set = [ + "float", "double", "real", "binary_float", "binary_double" + ] + + private static let textTypes: Set = ["enum", "set", "json", "jsonb", "name"] + + private static let bitTypes: Set = ["bit", "varbit"] + + // PostgreSQL's `bit(n)` is a string of 0 and 1. SQL Server's is a flag and MySQL's reads as a number. + public static func bitsAreStrings(databaseTypeId: String) -> Bool { + enginesWithBitStrings.contains(databaseTypeId) + } + + private static let enginesWithBitStrings: Set = [ + "PostgreSQL", "Redshift", "CockroachDB", "PGlite", "AlloyDB", "Citus", "Greenplum", "KingbaseES" + ] + + private static let textMarkers = ["char", "text", "string", "clob"] +} + public extension String { func toUTF8Data() throws -> Data { guard let data = self.data(using: .utf8) else { diff --git a/apple/Plugins/TableProPluginKit/PluginRowWriters.swift b/apple/Plugins/TableProPluginKit/PluginRowWriters.swift index e92617afb2..d3c0be5caa 100644 --- a/apple/Plugins/TableProPluginKit/PluginRowWriters.swift +++ b/apple/Plugins/TableProPluginKit/PluginRowWriters.swift @@ -18,9 +18,7 @@ public struct PluginCsvWriteOptions: Sendable, Equatable { public let lineEnding: String public let nullAsEmpty: Bool - /// Prefixes a value opening with `=`, `+`, `-` or `@` with a quote. A spreadsheet treats a cell - /// starting with one of those as a formula, and the values in an export come from the database - /// rather than from the person opening the file. + // The quote stays in the value for any reader but a spreadsheet, so it suits files meant for one. public let sanitizesFormulas: Bool /// Replaces a line break inside a value with a space. Off means the value keeps its break and is @@ -73,9 +71,7 @@ public enum PluginRowWriters { .replacingOccurrences(of: "\r", with: " ", options: .literal) } var wasNeutralised = false - // Scalars, not Characters: `\r\n` is one Character, so a Character test misses a Windows line break. - if options.sanitizesFormulas, let first = processed.unicodeScalars.first, - isFormulaLead(Character(first)), !isPlainNumber(processed) { + if options.sanitizesFormulas, isFormulaLike(processed) { processed = "'" + processed wasNeutralised = true } @@ -95,12 +91,19 @@ public enum PluginRowWriters { private static let quotedScalars: Set = ["\"", "\n", "\r", "\t"] - /// A leading tab or carriage return is treated as a formula lead too: Excel strips it before - /// parsing the cell, so `\t=1+1` reaches the formula engine exactly as `=1+1` would. - private static func isFormulaLead(_ character: Character) -> Bool { - formulaPrefixes.contains(character) || character == "\t" || character == "\r" + // Public so a format plugin can count the values sanitizing changes. + public static func isFormulaLike(_ value: String) -> Bool { + // Scalars, not Characters: `\r\n` is one Character, so a Character test misses a Windows line break. + guard let first = value.unicodeScalars.first, formulaLeads.contains(first) else { return false } + return !isPlainNumber(value) } + // OWASP's list: the four operators, the whitespace Excel strips before parsing (so `\t=1+1` runs as `=1+1`), + // and the full-width operators a Japanese locale reads as formulas. The same set as the shared core's `FORMULA_LEADS`. + private static let formulaLeads: Set = [ + "=", "+", "-", "@", "\t", "\r", "\n", "\u{FF1D}", "\u{FF0B}", "\u{FF0D}", "\u{FF20}" + ] + // A number is never a formula, and `'-5` is text to every reader, a spreadsheet included. // The same rule as the Linux app's `is_plain_number`. private static func isPlainNumber(_ value: String) -> Bool { diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift index ef3166e4c1..ea42a44623 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift @@ -4,6 +4,8 @@ import TableProTeradataCore final class TeradataAsyncConnection: @unchecked Sendable { private let connection: TeradataConnection private let queue = DispatchQueue(label: "com.TablePro.teradata.connection") + private let runningLock = NSLock() + private var running: TeradataAsyncCancellation? init(config: TeradataConnectionConfig) { connection = TeradataConnection(config: config) @@ -41,6 +43,13 @@ final class TeradataAsyncConnection: @unchecked Sendable { return try await withTaskCancellationHandler { try await withCheckedThrowingContinuation { continuation in queue.async { + // Begun before the call is marked running, so a Stop either side of the mark reaches it. + self.connection.beginOperation() + self.runningLock.withLock { self.running = cancellation } + defer { + self.runningLock.withLock { self.running = nil } + self.connection.endOperation() + } do { try cancellation.throwIfCancelled() let value = try body(self.connection) @@ -53,9 +62,12 @@ final class TeradataAsyncConnection: @unchecked Sendable { } } onCancel: { cancellation.cancel() - // TeradataConnection snapshots its transport under a lock; the transport's cancel - // path uses shutdown rather than disconnecting concurrently with protocol cleanup. - connection.cancel() + // A call still queued never starts; stopping the connection would stop the request of the one running. + // Held across the stop, so the call cannot end and hand the session to the next one meanwhile. + runningLock.withLock { + guard running === cancellation else { return } + connection.cancel() + } } } } diff --git a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift index 63a467ef35..f773973f4a 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift @@ -43,6 +43,8 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug destination: URL, progress: PluginExportProgress ) async throws -> ExportFormatResult { + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-export. + let options = settings let writer = XLSXWriter(databaseTypeId: dataSource.databaseTypeId) var didSplitSheets = false @@ -56,7 +58,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug var currentSheetRowCount = 0 var columns: [String] = [] var columnTypeNames: [String] = [] - let headerRowCount = settings.includeHeaderRow ? 1 : 0 + let headerRowCount = options.includeHeaderRow ? 1 : 0 let stream = dataSource.streamRows(for: table) for try await element in stream { @@ -70,8 +72,8 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug name: table.name, columns: columns, columnTypeNames: columnTypeNames, - includeHeader: settings.includeHeaderRow, - convertNullToEmpty: settings.convertNullToEmpty + includeHeader: options.includeHeaderRow, + convertNullToEmpty: options.convertNullToEmpty ) currentSheetRowCount = headerRowCount isFirstBatch = false @@ -83,7 +85,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug let remaining = Self.maxRowsPerSheet - currentSheetRowCount if rowBatch.count <= remaining { try autoreleasepool { - try writer.addRows(rowBatch, convertNullToEmpty: settings.convertNullToEmpty) + try writer.addRows(rowBatch, convertNullToEmpty: options.convertNullToEmpty) } currentSheetRowCount += rowBatch.count } else { @@ -91,7 +93,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug let overflow = Array(rowBatch.dropFirst(remaining)) if !fitting.isEmpty { try autoreleasepool { - try writer.addRows(fitting, convertNullToEmpty: settings.convertNullToEmpty) + try writer.addRows(fitting, convertNullToEmpty: options.convertNullToEmpty) } currentSheetRowCount += fitting.count } @@ -99,14 +101,14 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug baseName: table.name, columns: columns, columnTypeNames: columnTypeNames, - includeHeader: settings.includeHeaderRow, - convertNullToEmpty: settings.convertNullToEmpty + includeHeader: options.includeHeaderRow, + convertNullToEmpty: options.convertNullToEmpty ) didSplitSheets = true currentSheetRowCount = headerRowCount if !overflow.isEmpty { try autoreleasepool { - try writer.addRows(overflow, convertNullToEmpty: settings.convertNullToEmpty) + try writer.addRows(overflow, convertNullToEmpty: options.convertNullToEmpty) } currentSheetRowCount += overflow.count } @@ -124,7 +126,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug let remaining = Self.maxRowsPerSheet - currentSheetRowCount if rowBatch.count <= remaining { try autoreleasepool { - try writer.addRows(rowBatch, convertNullToEmpty: settings.convertNullToEmpty) + try writer.addRows(rowBatch, convertNullToEmpty: options.convertNullToEmpty) } currentSheetRowCount += rowBatch.count } else { @@ -132,21 +134,21 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug let overflow = Array(rowBatch.dropFirst(remaining)) if !fitting.isEmpty { try autoreleasepool { - try writer.addRows(fitting, convertNullToEmpty: settings.convertNullToEmpty) + try writer.addRows(fitting, convertNullToEmpty: options.convertNullToEmpty) } } try writer.continueSheet( baseName: table.name, columns: columns, columnTypeNames: columnTypeNames, - includeHeader: settings.includeHeaderRow, - convertNullToEmpty: settings.convertNullToEmpty + includeHeader: options.includeHeaderRow, + convertNullToEmpty: options.convertNullToEmpty ) didSplitSheets = true currentSheetRowCount = headerRowCount if !overflow.isEmpty { try autoreleasepool { - try writer.addRows(overflow, convertNullToEmpty: settings.convertNullToEmpty) + try writer.addRows(overflow, convertNullToEmpty: options.convertNullToEmpty) } currentSheetRowCount += overflow.count } @@ -163,7 +165,7 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug name: table.name, columns: [], includeHeader: false, - convertNullToEmpty: settings.convertNullToEmpty + convertNullToEmpty: options.convertNullToEmpty ) try writer.finishSheet() } diff --git a/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift b/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift index c404199eac..cdb19eb612 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXWriter.swift @@ -46,66 +46,7 @@ final class XLSXWriter { case empty } - enum ColumnNumbers: Equatable { - case text - case exact - case approximate - case untyped - - init(typeName: String, bitsAreStrings: Bool = false) { - let base = Self.baseName(typeName) - if base.isEmpty { - self = .untyped - } else if Self.bitTypes.contains(base) { - self = bitsAreStrings ? .text : .exact - } else if Self.approximateTypes.contains(base) { - self = .approximate - } else if Self.exactTypes.contains(base) { - self = .exact - } else if Self.textTypes.contains(base) || Self.textMarkers.contains(where: { base.contains($0) }) { - self = .text - } else { - self = .untyped - } - } - - // `Nullable(UInt64)` and `LowCardinality(Nullable(Int32))` name the type inside. The base is the - // first word without its width: `int4`, `Int64` and `INT UNSIGNED` are all `int`. - private static func baseName(_ typeName: String) -> String { - var name = typeName.trimmingCharacters(in: .whitespacesAndNewlines).lowercased() - while name.hasSuffix(")"), let open = name.firstIndex(of: "("), - ["nullable", "lowcardinality"].contains(String(name[.. = [ - "int", "uint", "integer", "bigint", "smallint", "tinyint", "mediumint", "hugeint", - "uinteger", "ubigint", "usmallint", "utinyint", "uhugeint", "decimal", "dec", "numeric", - "number", "bignumeric", "serial", "bigserial", "smallserial", "money", "smallmoney", "year", "oid" - ] - - private static let approximateTypes: Set = [ - "float", "double", "real", "binary_float", "binary_double" - ] - - private static let textTypes: Set = ["enum", "set", "json", "jsonb", "name"] - - private static let bitTypes: Set = ["bit", "varbit"] - - // PostgreSQL's `bit(n)` is a string of 0 and 1. SQL Server's is a flag and MySQL's reads as a number. - static func bitsAreStrings(databaseTypeId: String) -> Bool { - enginesWithBitStrings.contains(databaseTypeId) - } - - private static let enginesWithBitStrings: Set = [ - "PostgreSQL", "Redshift", "CockroachDB", "PGlite", "AlloyDB", "Citus", "Greenplum", "KingbaseES" - ] - - private static let textMarkers = ["char", "text", "string", "clob"] - } + typealias ColumnNumbers = PluginColumnNumbers // Excel keeps 15 significant digits of a number. private static let maximumExactDigits = 15 @@ -225,6 +166,8 @@ final class XLSXWriter { let isPlain = literal == text && !literal.contains("e") let digitCount = literal.filter { $0.isNumber }.count return isPlain && digitCount <= maximumExactDigits ? .number(literal) : .string(text) + @unknown default: + return .string(text) } } diff --git a/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift b/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift index 6f7592ab3a..d74bafbfba 100644 --- a/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift +++ b/apple/Plugins/XLSXImportPlugin/XLSXImportPlugin.swift @@ -39,13 +39,13 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug var fieldDetectionSignature: String { settings.detectionSignature } - var rowImportConfiguration: RowImportRunner.Configuration? { runConfiguration } + var rowImportConfiguration: RowImportRunner.Configuration? { Self.runConfiguration(settings) } - private var runConfiguration: RowImportRunner.Configuration { + private static func runConfiguration(_ options: XLSXImportOptions) -> RowImportRunner.Configuration { RowImportRunner.Configuration( - errorHandling: settings.errorHandling, - wrapInTransaction: settings.wrapInTransaction, - deleteExistingRows: settings.deleteExistingRows + errorHandling: options.errorHandling, + wrapInTransaction: options.wrapInTransaction, + deleteExistingRows: options.deleteExistingRows ) } @@ -78,11 +78,13 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug progress: PluginImportProgress ) async throws -> PluginImportResult { let started = Date() + // One plugin instance serves every window, so a dialog in another window can change `settings` mid-import. + let options = settings let sheet: XLSXImportSheet do { sheet = try XLSXImportSheet.read( at: source.fileURL(), - hasHeaderRow: settings.hasHeaderRow, + hasHeaderRow: options.hasHeaderRow, isCancelled: { progress.isCancelled } ) } catch is TabularCancellation { @@ -96,7 +98,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug progress.setEstimatedTotal(max(1, sheet.rowCount)) var cursor = 0 let outcome = try await RowImportRunner.run( - configuration: runConfiguration, + configuration: Self.runConfiguration(options), sink: sink, progress: progress ) { @@ -108,7 +110,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug if filled.count < rows.count { progress.incrementStatement(by: rows.count - filled.count) } - return filled.map { (line: $0.line, row: self.row($0.values, header: header)) } + return filled.map { (line: $0.line, row: Self.row($0.values, header: header, options: options)) } } return PluginImportResult( @@ -121,7 +123,7 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug // MARK: - Private - private func row(_ values: [String?], header: [String]) -> [String: PluginCellValue] { + private static func row(_ values: [String?], header: [String], options: XLSXImportOptions) -> [String: PluginCellValue] { var mapped: [String: PluginCellValue] = [:] for (index, column) in header.enumerated() { let raw = values[safe: index] ?? nil @@ -129,10 +131,10 @@ final class XLSXImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlug mapped[column] = .null continue } - let trimmed = settings.trimWhitespace + let trimmed = options.trimWhitespace ? raw.trimmingCharacters(in: .whitespacesAndNewlines) : raw - if settings.emptyAsNull, trimmed.isEmpty { + if options.emptyAsNull, trimmed.isEmpty { mapped[column] = .null } else { mapped[column] = .text(trimmed) diff --git a/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift b/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift index 7a795ff87d..f86e46a25c 100644 --- a/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift +++ b/apple/TablePro/Core/ChangeTracking/DataChangeModels.swift @@ -49,8 +49,8 @@ struct CellChange: Identifiable, Equatable { let oldIsAbsent: Bool let newIsAbsent: Bool let oldIsExpression: Bool - /// The new value is `.text` holding SQL from `CellExpressionVocabulary`, written unquoted. Never set with - /// `newIsAbsent`. + /// The new value is `.text` holding SQL from `CellExpressionVocabulary`, or `PluginCellValue.columnDefault`, + /// written unquoted. Never set with `newIsAbsent`. let newIsExpression: Bool init( @@ -92,8 +92,8 @@ struct RowChange: Identifiable, Equatable { /// stands for an insert. Empty on every engine that cannot tell a missing field from NULL. var absentColumns: Set - /// An insert's cells that hold a SQL expression, as the row stands. An update keeps its own on each - /// `CellChange`. + /// An insert's cells that hold a SQL expression or the column default, as the row stands. An update keeps its own + /// on each `CellChange`. var expressionColumns: Set /// The order the user made this change in. diff --git a/apple/TablePro/Core/ChangeTracking/DefaultedCells.swift b/apple/TablePro/Core/ChangeTracking/DefaultedCells.swift new file mode 100644 index 0000000000..bdbb48fc6b --- /dev/null +++ b/apple/TablePro/Core/ChangeTracking/DefaultedCells.swift @@ -0,0 +1,38 @@ +// +// DefaultedCells.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +// The cells a pending insert or edit set to the column's default, by row and data index. Only those rows can hold one. +enum DefaultedCells { + static func marked( + in rows: TableRows, + rowIDs: Set, + expressionColumns: (RowID) -> Set + ) -> [RowID: Set] { + var cells: [RowID: Set] = [:] + for rowID in rowIDs { + guard let row = rows.row(withID: rowID) else { continue } + let defaulted = expressionColumns(rowID).filter { + row.values.indices.contains($0) && row.values[$0].isColumnDefault(isExpression: true) + } + if !defaulted.isEmpty { cells[rowID] = defaulted } + } + return cells + } +} + +extension ChangeManaging { + func defaultedCells(in rows: TableRows) -> [RowID: Set] { + DefaultedCells.marked(in: rows, rowIDs: insertedRowIDs.union(modifiedCells.keys)) { expressionColumns(forRow: $0) } + } +} + +extension AnyChangeManager { + func defaultedCells(in rows: TableRows) -> [RowID: Set] { + DefaultedCells.marked(in: rows, rowIDs: insertedRowIDs.union(modifiedCells.keys)) { expressionColumns(forRow: $0) } + } +} diff --git a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift index 17f1871fa9..21df0ea117 100644 --- a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift +++ b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift @@ -257,7 +257,8 @@ struct SQLStatementGenerator { var bindParameters: [Any?] = [] for (index, value) in values.enumerated() { - if case .text(let s) = value, s == "__DEFAULT__" { continue } + // Left out rather than written as DEFAULT, which SQLite does not take in VALUES. + if value.isColumnDefault(isExpression: expressionColumns.contains(index)) { continue } guard index < columns.count else { continue } let columnName = columns[index] @@ -383,7 +384,7 @@ struct SQLStatementGenerator { guard !change.cellChanges.isEmpty else { return nil } let nonDefaultChanges = change.cellChanges.filter { - $0.newValue != .text("__DEFAULT__") && !generatedColumns.contains($0.columnName) + !$0.newValue.isColumnDefault(isExpression: $0.newIsExpression) && !generatedColumns.contains($0.columnName) } guard !nonDefaultChanges.isEmpty else { return nil } @@ -419,8 +420,6 @@ struct SQLStatementGenerator { var parameters: [Any?] = [] let setClauses = writableChanges.map { cellChange -> String in switch cellChange.newValue { - case .text(let s) where s == "__DEFAULT__": - return "\(quoteIdentifierFn(cellChange.columnName)) = DEFAULT" case .text(let expression) where cellChange.newIsExpression: return "\(quoteIdentifierFn(cellChange.columnName)) = \(expression)" default: diff --git a/apple/TablePro/Core/Compare/CompareSQLLiteral.swift b/apple/TablePro/Core/Compare/CompareSQLLiteral.swift index 24cc1a74a8..40fc624f42 100644 --- a/apple/TablePro/Core/Compare/CompareSQLLiteral.swift +++ b/apple/TablePro/Core/Compare/CompareSQLLiteral.swift @@ -77,17 +77,20 @@ internal enum CompareSQLLiteral { return SQLStringLiteralPrefix.forDatabaseType(databaseType) + literal } + // Shared by Compare & Sync and Copy as SQL. `SQLExportBinaryLiteral` in the SQL export plugin holds the + // measurements behind each spelling; the plugin cannot link app code, so it keeps its own copy. internal static func binaryLiteral(for data: Data, databaseType: DatabaseType) -> String? { - let hex = data.map { String(format: "%02X", $0) }.joined() + let hex = data.uppercaseHex() switch binaryStyle(for: databaseType) { case .bitString: return "X'\(hex)'" case .postgresBytea: - return "'\\x\(hex.lowercased())'::bytea" + return postgresFamilyBinaryLiteral(hex: hex, databaseType: databaseType) case .zeroX: return "0x\(hex)" case .hexToRaw: - return "HEXTORAW('\(hex)')" + // `HEXTORAW('')` stores NULL, which a `BLOB NOT NULL` refuses. + return data.isEmpty ? "EMPTY_BLOB()" : "HEXTORAW('\(hex)')" case .unhexFunction: return "unhex('\(hex)')" case .unknown: @@ -95,6 +98,19 @@ internal enum CompareSQLLiteral { } } + // `'\x…'::bytea` follows `standard_conforming_strings`: with it off the server stores the text's bytes. KingbaseES + // documents `decode` only as Oracle's DECODE, so it keeps the bytea literal; Redshift has no bytea and reads VARBYTE. + private static func postgresFamilyBinaryLiteral(hex: String, databaseType: DatabaseType) -> String { + switch databaseType { + case .kingbasees: + return "'\\x\(hex)'::bytea" + case .redshift: + return "from_hex('\(hex)')" + default: + return "decode('\(hex)', 'hex')" + } + } + internal enum BinaryStyle { case bitString case postgresBytea diff --git a/apple/TablePro/Core/Compare/DataDiffEngine.swift b/apple/TablePro/Core/Compare/DataDiffEngine.swift index 882c746b62..df96ff5b92 100644 --- a/apple/TablePro/Core/Compare/DataDiffEngine.swift +++ b/apple/TablePro/Core/Compare/DataDiffEngine.swift @@ -30,10 +30,16 @@ internal struct DataRow: Hashable, Sendable { internal protocol DataRowProviding: AnyObject { var endedAtRowLimit: Bool { get } + // The limit the driver stopped this side's stream at, once it has ended there. + var driverRowCut: Int? { get } func nextRow() async throws -> DataRow? func drain() async throws } +internal extension DataRowProviding { + var driverRowCut: Int? { nil } +} + internal protocol OneSidedRowResolving: AnyObject { func rows(matching keys: [[PluginCellValue]], on side: ComparisonSide) async throws -> [DataRow] } @@ -111,6 +117,10 @@ internal struct DataDiffSummary: Hashable, Sendable { /// as `id > ` resumes from without skipping a key or comparing one twice. internal let resumeKey: [PluginCellValue]? + // The limit a driver stopped a side's stream at, so the answer covers only the rows read before it. + internal let sourceRowCut: Int? + internal let targetRowCut: Int? + /// One comparison's own identity. Two runs that produce the same counts and the same digest are /// still two answers, and a view that caches on the answer has to be able to tell them apart: /// matching rows are counted rather than digested, so both can move while the digest stands. @@ -130,6 +140,8 @@ internal struct DataDiffSummary: Hashable, Sendable { stoppedAtRowLimit: Bool = false, differenceDigest: String = "", resumeKey: [PluginCellValue]? = nil, + sourceRowCut: Int? = nil, + targetRowCut: Int? = nil, runIdentity: UUID = UUID() ) { self.insertCount = insertCount @@ -145,6 +157,8 @@ internal struct DataDiffSummary: Hashable, Sendable { self.stoppedAtRowLimit = stoppedAtRowLimit self.differenceDigest = differenceDigest self.resumeKey = resumeKey + self.sourceRowCut = sourceRowCut + self.targetRowCut = targetRowCut self.runIdentity = runIdentity } @@ -359,7 +373,9 @@ internal struct DataDiffEngine { return accumulator.summary( comparedKeyCount: positions, stoppedAtRowLimit: stoppedAtRowLimit, - resumeKey: stoppedAtRowLimit ? lastClassifiedKey : nil + resumeKey: stoppedAtRowLimit ? lastClassifiedKey : nil, + sourceRowCut: source.driverRowCut, + targetRowCut: target.driverRowCut ) } @@ -536,7 +552,9 @@ internal extension DataDiffEngine { func summary( comparedKeyCount: Int, stoppedAtRowLimit: Bool, - resumeKey: [PluginCellValue]? = nil + resumeKey: [PluginCellValue]? = nil, + sourceRowCut: Int? = nil, + targetRowCut: Int? = nil ) -> DataDiffSummary { DataDiffSummary( insertCount: counts[.insert] ?? 0, @@ -551,7 +569,9 @@ internal extension DataDiffEngine { comparedKeyCount: comparedKeyCount, stoppedAtRowLimit: stoppedAtRowLimit, differenceDigest: digest.finalize().map { String(format: "%02x", $0) }.joined(), - resumeKey: resumeKey + resumeKey: resumeKey, + sourceRowCut: sourceRowCut, + targetRowCut: targetRowCut ) } diff --git a/apple/TablePro/Core/Compare/StreamingRowProvider.swift b/apple/TablePro/Core/Compare/StreamingRowProvider.swift index 979cabe6fb..54a4ac107f 100644 --- a/apple/TablePro/Core/Compare/StreamingRowProvider.swift +++ b/apple/TablePro/Core/Compare/StreamingRowProvider.swift @@ -13,6 +13,7 @@ // import Foundation +import os import TableProPluginKit internal struct RowReadContext: Sendable { @@ -55,6 +56,7 @@ internal final class StreamingRowProvider: DataRowProviding { private var deliveredCount = 0 private var isFinished = false private var sawRowPastTheLimit = false + private let rowCut: OSAllocatedUnfairLock internal init( stream: AsyncThrowingStream, @@ -62,17 +64,25 @@ internal final class StreamingRowProvider: DataRowProviding { rowLimit: Int? = nil, context: RowReadContext? = nil ) { - self.iterator = stream.makeAsyncIterator() + let rowCut = OSAllocatedUnfairLock(initialState: nil) + self.rowCut = rowCut + self.iterator = ExportRowCut.reporting(stream, engineCeiling: nil) { limit in + rowCut.withLock { $0 = limit } + }.makeAsyncIterator() self.columns = columns self.rowLimit = rowLimit self.context = context } - /// The read asks for one row more than the limit and never hands that row out, so this is the - /// server's own answer to whether anything was left rather than a guess from the row count: a - /// table holding exactly the limit is not a truncated read. + // The read asks for one row past the limit, so a table holding exactly the limit is not a cut read. A stream the + // driver stopped at a limit it declares ends here too: a key only the other side holds was never read on this one. internal var endedAtRowLimit: Bool { - sawRowPastTheLimit + sawRowPastTheLimit || driverRowCut != nil + } + + internal var driverRowCut: Int? { + guard isFinished else { return nil } + return rowCut.withLock { $0 } } internal func nextRow() async throws -> DataRow? { diff --git a/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift b/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift index ec3657620b..9cf2f91330 100644 --- a/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift +++ b/apple/TablePro/Core/Coordinators/RowEditingCoordinator.swift @@ -218,7 +218,7 @@ final class RowEditingCoordinator: ObservableObject { private static func sampleText(atColumn index: Int, in tableRows: TableRows) -> String? { for row in tableRows.rows where row.values.indices.contains(index) { - if case .text(let text) = row.values[index], text != "__DEFAULT__" { return text } + if case .text(let text) = row.values[index], row.values[index] != .columnDefault { return text } } return nil } @@ -430,10 +430,12 @@ final class RowEditingCoordinator: ObservableObject { self?.copyRowsAsJson(rowIDs) }) else { return } let model = parent.gridModel(forTab: tab.id) + let tableRows = parent.tabSessionRegistry.tableRows(for: tab.id) let output = ResultJsonSerializer.serialize( - tableRows: parent.tabSessionRegistry.tableRows(for: tab.id), + tableRows: tableRows, displayIDs: model.displayedRowIDs, selectedDisplayIndices: Set(rowIDs.compactMap { model.displayIndex(of: $0) }), + defaultedCells: parent.changeManager.defaultedCells(in: tableRows), columns: VisibleColumnProjection(indices: presentedColumnIndices(tabId: tab.id)) ) guard output.rowCount > 0 else { return } diff --git a/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift b/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift index a4c255ea3b..3686d2ca50 100644 --- a/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift +++ b/apple/TablePro/Core/DataWrite/Rewind/RewindExecutor.swift @@ -134,8 +134,9 @@ struct RewindExecutor { ) } + // In the order the restore ran them, since undoing the restore runs its record backwards. static func inverseOperations(of plan: RewindPlan) -> [RowWriteOperation] { - plan.rows.filter { $0.outcome.restores }.map { row -> RowWriteOperation in + plan.rows.filter { $0.outcome.restores }.reversed().map { row -> RowWriteOperation in let source = row.operation let kind = source.kind.inverted // The same stored refusal a save through a view writes, so a build without the view flag refuses too. diff --git a/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift b/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift index 0438224191..361fb6bc40 100644 --- a/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift +++ b/apple/TablePro/Core/DataWrite/Rewind/RewindPlanner.swift @@ -82,6 +82,7 @@ struct RewindPlanner { currentRow = byLocator[Array(locator.utf8)] } else { currentRow = primaryKey(of: operation).flatMap { current[$0] } + ?? rowRestoredUnderKeyBefore(operation, in: current) } let outcome = classify(operation, current: currentRow) rows.append( @@ -135,7 +136,7 @@ struct RewindPlanner { } } - /// Whether the row holds what the image says on every column the save wrote: the same value, + /// Whether the row holds what the image says on every compared column: the same value, /// and the field there or missing as it was. Set NULL on a missing field and Remove Field on a /// NULL one leave the value alone, so comparing values alone reads either as already restored. /// @@ -147,7 +148,7 @@ struct RewindPlanner { absent expectedAbsent: Set?, on operation: RowWriteOperation ) -> Bool { - operation.writtenColumns.allSatisfy { column in + comparedColumns(of: operation).allSatisfy { column in guard let index = operation.columns.firstIndex(of: column), index < current.values.count, index < expected.count, current.values[index] == expected[index] @@ -157,6 +158,14 @@ struct RewindPlanner { } } + // A row found by a key the save wrote holds that key whatever row it is, so a moved row is compared on every + // column the server does not compute; a trigger's stamp then reads as changed, which leaves the row alone. + private func comparedColumns(of operation: RowWriteOperation) -> [String] { + guard keyBeforeMove(of: operation) != nil else { return operation.writtenColumns } + let generated = Set(record.generatedColumns) + return operation.columns.filter { !generated.contains($0) } + } + // MARK: - Reading private func indexByKey(_ rows: [RewindCurrentRow]) -> [RewindRowKey: RewindCurrentRow] { @@ -194,9 +203,11 @@ struct RewindPlanner { guard !keyColumns.isEmpty else { return [] } if keyColumns.count == 1, let keyColumn = keyColumns.first { - let filters = operations.compactMap { operation -> TableFilter? in - guard let value = keyValue(of: operation, column: keyColumn) else { return nil } - return TableFilter(columnName: keyColumn, filterOperator: .equal, value: value) + let filters = operations.flatMap { operation in + readImages(of: operation).compactMap { image -> TableFilter? in + guard let value = keyValue(in: image, of: operation, column: keyColumn) else { return nil } + return TableFilter(columnName: keyColumn, filterOperator: .equal, value: value) + } } guard !filters.isEmpty, let query = queryBuilder.buildFilteredQuery( @@ -211,21 +222,23 @@ struct RewindPlanner { return [query] } - return operations.compactMap { operation in - let filters = keyColumns.compactMap { column -> TableFilter? in - guard let value = keyValue(of: operation, column: column) else { return nil } - return TableFilter(columnName: column, filterOperator: .equal, value: value) + return operations.flatMap { operation in + readImages(of: operation).compactMap { image -> String? in + let filters = keyColumns.compactMap { column -> TableFilter? in + guard let value = keyValue(in: image, of: operation, column: column) else { return nil } + return TableFilter(columnName: column, filterOperator: .equal, value: value) + } + guard filters.count == keyColumns.count else { return nil } + return queryBuilder.buildFilteredQuery( + tableName: record.target.table, + schemaName: record.target.schema, + filters: filters, + logicMode: .and, + columns: columns, + selectColumns: columns, + limit: 1 + ) } - guard filters.count == keyColumns.count else { return nil } - return queryBuilder.buildFilteredQuery( - tableName: record.target.table, - schemaName: record.target.schema, - filters: filters, - logicMode: .and, - columns: columns, - selectColumns: columns, - limit: 1 - ) } } @@ -293,12 +306,40 @@ struct RewindPlanner { // MARK: - Keys + // Where the save left the row, or for a delete where it was: an edit that rewrote the key moved it. + private func savedImage(of operation: RowWriteOperation) -> [PluginCellValue]? { + operation.postImage ?? operation.preImage + } + private func primaryKey(of operation: RowWriteOperation) -> RewindRowKey? { - let source = operation.preImage ?? operation.postImage - guard let source else { return nil } + guard let source = savedImage(of: operation) else { return nil } return primaryKey(of: source, columns: operation.columns, keyColumns: operation.primaryKeyColumns) } + private func keyBeforeMove(of operation: RowWriteOperation) -> RewindRowKey? { + guard operation.kind == .update, let preImage = operation.preImage else { return nil } + let before = primaryKey(of: preImage, columns: operation.columns, keyColumns: operation.primaryKeyColumns) + return before == primaryKey(of: operation) ? nil : before + } + + // A rewind that already ran put a moved row back under its old key, where it reads as restored. + private func readImages(of operation: RowWriteOperation) -> [[PluginCellValue]] { + guard let saved = savedImage(of: operation) else { return [] } + guard keyBeforeMove(of: operation) != nil, let preImage = operation.preImage else { return [saved] } + return [saved, preImage] + } + + // Only a row holding the old values, so another row that took the old key since is not mistaken for this one. + private func rowRestoredUnderKeyBefore( + _ operation: RowWriteOperation, + in current: [RewindRowKey: RewindCurrentRow] + ) -> RewindCurrentRow? { + guard let key = keyBeforeMove(of: operation), let row = current[key], let preImage = operation.preImage, + matches(row, preImage, absent: operation.preImageAbsentColumns, on: operation) + else { return nil } + return row + } + private func primaryKey( of row: [PluginCellValue], columns: [String], @@ -319,16 +360,17 @@ struct RewindPlanner { /// compare against, and a delete would be reported as restorable with no conflict check at all. private func keyIsComparable(_ operation: RowWriteOperation) -> Bool { !operation.primaryKeyColumns.isEmpty - && operation.primaryKeyColumns.allSatisfy { keyValue(of: operation, column: $0) != nil } + && operation.primaryKeyColumns.allSatisfy { + keyValue(in: savedImage(of: operation), of: operation, column: $0) != nil + } } - private func keyValue(of operation: RowWriteOperation, column: String) -> String? { - let source = operation.preImage ?? operation.postImage - guard let source, + private func keyValue(in image: [PluginCellValue]?, of operation: RowWriteOperation, column: String) -> String? { + guard let image, let index = operation.columns.firstIndex(of: column), - index < source.count + index < image.count else { return nil } - return source[index].asText + return image[index].asText } } diff --git a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift index 968b67c8df..19517845a3 100644 --- a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift +++ b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift @@ -146,7 +146,7 @@ struct RowChangeStatementFactory { try refuseUnwritableExpressions(in: changes, insertedRowData: insertedRowData) try refuseMisdimensionedVectors(in: changes, insertedRowData: insertedRowData) let insertedRowData = insertedRowData.mapValues(leavingServerOwnedColumnsToTheServer) - let changes = changes.map(droppingServerOwnedExpressions) + let changes = changes.map { markingServerOwnedDefaults($0, in: insertedRowData) } if let driverStatements = try pluginRowWrites( for: changes, insertedRowData: insertedRowData, @@ -190,7 +190,9 @@ struct RowChangeStatementFactory { private func refuseDefaultOnUpdate(in changes: [RowChange]) throws { guard !CellExpressionVocabulary.updateTakesDefault(databaseType) else { return } for change in changes where change.type == .update { - guard let defaulted = change.cellChanges.first(where: { $0.newValue.isDefaultMarker }) else { continue } + guard let defaulted = change.cellChanges.first(where: { + $0.newValue.isColumnDefault(isExpression: $0.newIsExpression) + }) else { continue } throw DataWriteError.changeRefused( table: tableName, kind: .update, @@ -209,9 +211,10 @@ struct RowChangeStatementFactory { in changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]] ) throws { + // A default reaches a driver in the spelling it reads, so only SQL proper has to be SQL this engine runs. let marked = changes.flatMap { change in Self.markedCells(of: change, insertedRowData: insertedRowData, columns: columns) - } + }.filter { $0.value != .columnDefault } guard !marked.isEmpty else { return } for cell in marked { guard case .text(let sql) = cell.value, CellExpressionVocabulary.isCanonical(sql, for: databaseType) @@ -309,21 +312,58 @@ struct RowChangeStatementFactory { guard !generatedColumns.isEmpty else { return values } return values.enumerated().map { index, value in guard columns.indices.contains(index), generatedColumns.contains(columns[index]) else { return value } - return Self.defaultMarker + return .columnDefault } } - /// The cell now holds the default marker, so it is no longer SQL to write. - private func droppingServerOwnedExpressions(_ change: RowChange) -> RowChange { - guard change.type == .insert, !change.expressionColumns.isEmpty, !generatedColumns.isEmpty else { return change } - var kept = change - kept.expressionColumns = change.expressionColumns.filter { - !(columns.indices.contains($0) && generatedColumns.contains(columns[$0])) + // Whatever SQL the cell held, it now holds the default, which only the mark makes one. + private func markingServerOwnedDefaults(_ change: RowChange, in insertedRowData: [RowID: [PluginCellValue]]) -> RowChange { + guard change.type == .insert, !generatedColumns.isEmpty else { return change } + let owned = Set(columns.indices.filter { generatedColumns.contains(columns[$0]) }) + var marked = change + marked.expressionColumns.subtract(owned) + if let values = insertedRowData[change.rowID] { + marked.expressionColumns.formUnion(owned.filter { values.indices.contains($0) }) } - return kept + return marked } - private static let defaultMarker = PluginCellValue.text("__DEFAULT__") + // A driver reads its own spelling for a default, not the mark, so a default reaches it spelled that way and unmarked. + private static func spellingDefaultsForDrivers( + _ changes: [RowChange], + insertedRowData: [RowID: [PluginCellValue]] + ) -> (changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]]) { + var rows = insertedRowData + let spelledChanges = changes.map { change -> RowChange in + var spelled = change + if change.type == .insert, var values = rows[change.rowID] { + for index in change.expressionColumns where values.indices.contains(index) && values[index] == .columnDefault { + values[index] = .pluginDefaultMarker + spelled.expressionColumns.remove(index) + } + rows[change.rowID] = values + } + let defaulted = change.cellChanges.filter { $0.newValue.isColumnDefault(isExpression: $0.newIsExpression) } + guard !defaulted.isEmpty else { return spelled } + spelled.cellChanges = change.cellChanges.map { cell in + guard cell.newValue.isColumnDefault(isExpression: cell.newIsExpression) else { return cell } + return CellChange( + columnIndex: cell.columnIndex, + columnName: cell.columnName, + oldValue: cell.oldValue, + newValue: .pluginDefaultMarker, + oldIsAbsent: cell.oldIsAbsent, + newIsAbsent: cell.newIsAbsent, + oldIsExpression: cell.oldIsExpression + ) + } + if change.type == .insert, rows[change.rowID] == nil { + spelled.expressionColumns.subtract(defaulted.map(\.columnIndex)) + } + return spelled + } + return (spelledChanges, rows) + } private func rowWriteContext(for keyed: PluginKeyedChanges) -> PluginRowWriteContext { var context = PluginRowWriteContext() @@ -386,6 +426,7 @@ struct RowChangeStatementFactory { }, context: restoreContext(rowCount: rows.count) ) { + try refuseRestoredTextDriversReadAsDefault(rows, restorable: restorable) try refuseUnfilteredWrites(restored.map { $0.statement }) return RestoreStatements(statements: restored.map { ParameterizedStatement(sql: $0.statement, parameters: $0.parameters.map(\.asAny)) @@ -489,9 +530,10 @@ struct RowChangeStatementFactory { insertedRowIDs: Set ) throws -> [DriverWrittenStatement]? { guard let pluginDriver else { return nil } + let spelled = Self.spellingDefaultsForDrivers(changes, insertedRowData: insertedRowData) let keyed = PluginKeyedChanges( - changes: changes, - insertedRowData: insertedRowData, + changes: spelled.changes, + insertedRowData: spelled.insertedRowData, deletedRowIDs: deletedRowIDs, insertedRowIDs: insertedRowIDs ) @@ -524,6 +566,7 @@ struct RowChangeStatementFactory { throw DataWriteError.changeRefused(table: tableName, kind: nil, reason: error.localizedDescription) } + try refuseTextDriversReadAsDefault(in: changes, insertedRowData: insertedRowData) try refuseUnfilteredWrites(writes.map(\.statement)) let unwritten = RowWriteCoverage.unwrittenChanges( changes, @@ -547,6 +590,46 @@ struct RowChangeStatementFactory { } } + // A driver may read the text `__DEFAULT__` as the column default, and none can say whether it does until PluginKit + // carries the mark beside the value, so the text is never handed to one as a value. + private func refuseTextDriversReadAsDefault( + in changes: [RowChange], + insertedRowData: [RowID: [PluginCellValue]] + ) throws { + for change in changes where change.type != .delete { + var written = change.cellChanges.filter { !$0.newIsExpression }.map { (column: $0.columnName, value: $0.newValue) } + if change.type == .insert, let stored = insertedRowData[change.rowID] { + written = zip(columns, stored).enumerated() + .filter { !change.expressionColumns.contains($0.offset) } + .map { (column: $0.element.0, value: $0.element.1) } + } + guard let literal = written.first(where: { $0.value == .pluginDefaultMarker }) else { continue } + throw textReadAsDefault(in: literal.column, kind: change.type == .insert ? .insert : .update) + } + } + + // A rewind puts a deleted row back with the values it held, which can be that text. + private func refuseRestoredTextDriversReadAsDefault(_ rows: [[PluginCellValue]], restorable: [Int]) throws { + for row in rows { + guard let index = restorable.first(where: { row.indices.contains($0) && row[$0] == .pluginDefaultMarker }) + else { continue } + throw textReadAsDefault(in: columns[index], kind: .insert) + } + } + + private func textReadAsDefault(in column: String, kind: RowWriteKind) -> DataWriteError { + .changeRefused( + table: tableName, + kind: kind, + reason: String( + format: String( + localized: "This database's driver may read the text __DEFAULT__ in %@ as the column default, so it cannot be saved as text." + ), + column + ) + ) + } + // The host's own generator never writes one, so only a driver's statements are read. private func refuseUnfilteredWrites(_ statements: [String]) throws { guard PluginManager.shared.editorLanguage(for: databaseType) == .sql else { return } diff --git a/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift b/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift index 457424223b..11cafebc8f 100644 --- a/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift +++ b/apple/TablePro/Core/DataWrite/RowWriteOperationBuilder.swift @@ -34,7 +34,8 @@ enum RowWriteOperationBuilder { writesThroughView: Bool = false, rowLocators: [RowID: String] = [:] ) -> [RowWriteOperation] { - changes.compactMap { change in + // A rewind undoes them newest first, and `changes` stops being in edit order once a change is cancelled. + changes.sorted { $0.sequence < $1.sequence }.compactMap { change in switch change.type { case .update: var operation = update( @@ -111,7 +112,7 @@ enum RowWriteOperationBuilder { ) -> RewindRefusal? { if containsTableOperation { return .destructiveTableOperation } if primaryKeyColumns.isEmpty || preImage == nil { return .noPrimaryKey } - if change.cellChanges.contains(where: { $0.newIsExpression || $0.newValue.isDefaultMarker }) { + if change.cellChanges.contains(where: \.newIsExpression) { return .serverComputedValue } if change.cellChanges.contains(where: { generatedColumns.contains($0.columnName) }) { @@ -172,11 +173,11 @@ enum RowWriteOperationBuilder { return .serverAssignedKey } let value = values[index] - if value.isNull || value.isDefaultMarker || change.expressionColumns.contains(index) { + if value.isNull || change.expressionColumns.contains(index) { return .serverAssignedKey } } - if values.contains(where: \.isDefaultMarker) || !change.expressionColumns.isEmpty { + if !change.expressionColumns.isEmpty { return .serverComputedValue } if values.contains(where: exceedsCapture) { return .valueTooLarge } diff --git a/apple/TablePro/Core/Database/DatabaseManager+ContainerDDL.swift b/apple/TablePro/Core/Database/DatabaseManager+ContainerDDL.swift index 147c37dfaf..f4550da094 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+ContainerDDL.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+ContainerDDL.swift @@ -59,10 +59,7 @@ extension DatabaseManager { } } - /// Runs a container operation the driver performs itself, with no statement to show. - /// - /// `kind` is the caller's to state because there is no text to classify: a drop is - /// `.destructiveQuery` and everything else is `.schemaMutation`. + // No statement to classify, so the caller states `kind`: a drop is destructive, anything else a schema mutation. func runContainerOperation( description: String, kind: OperationKind, @@ -70,6 +67,7 @@ extension DatabaseManager { databaseType: DatabaseType, event: CatalogEvent?, isConfirmationPreCleared: Bool = false, + gate: any ExecutionGate = ExecutionGateProvider.shared, perform: @Sendable @escaping (any DatabaseDriver) async throws -> Void ) async throws { try await authorizeContainerOperation( @@ -78,7 +76,8 @@ extension DatabaseManager { connectionId: scope.connectionId, databaseType: databaseType, description: description, - isConfirmationPreCleared: isConfirmationPreCleared + isConfirmationPreCleared: isConfirmationPreCleared, + gate: gate ) try await withScopedDriver( scope: scope, @@ -90,16 +89,14 @@ extension DatabaseManager { if let event { CatalogChangeService.post(event) } } - /// Runs a container operation the driver expressed as statements. - /// - /// The kind comes from the statements rather than from the action's name, so an edit that - /// revokes a privilege is authorized at the destructive tier the same way a drop is. + // The kind comes from the statements, so an edit that revokes a privilege is authorized like a drop. func runContainerStatements( _ plan: ContainerDDLPlan, description: String, scope: DatabaseScope, databaseType: DatabaseType, - event: CatalogEvent? + event: CatalogEvent?, + gate: any ExecutionGate = ExecutionGateProvider.shared ) async throws { let statements = plan.statements guard !statements.isEmpty else { return } @@ -113,7 +110,8 @@ extension DatabaseManager { connectionId: scope.connectionId, databaseType: databaseType, description: description, - isConfirmationPreCleared: false + isConfirmationPreCleared: false, + gate: gate ) let route = schemaChangeRoute(for: scope) @@ -139,21 +137,19 @@ extension DatabaseManager { if let event { CatalogChangeService.post(event) } } - /// `isConfirmationPreCleared` is for a caller that has already shown the user a destructive - /// confirmation of its own. Without it the sidebar's Drop dialog is followed by the gate's, - /// asking twice for one gesture and once per target for a batch. It clears the confirmation - /// only: Touch ID, the read-only refusal and the audit record all still apply. + // Pre-cleared skips only a confirmation the caller already showed; Touch ID, Read-Only and the audit still apply. private func authorizeContainerOperation( sql: String?, kind: OperationKind, connectionId: UUID, databaseType: DatabaseType, description: String, - isConfirmationPreCleared: Bool + isConfirmationPreCleared: Bool, + gate: any ExecutionGate ) async throws { var capabilities = CallerCapabilities.interactiveUser if isConfirmationPreCleared { capabilities.insert(.confirmationPreCleared) } - let decision = await ExecutionGateProvider.shared.authorize( + let decision = await gate.authorize( OperationRequest( connectionId: connectionId, databaseType: databaseType, @@ -164,11 +160,7 @@ extension DatabaseManager { operationDescription: description ) ) - guard case .authorized = decision else { - throw DatabaseError.queryFailed( - decision.deniedReason ?? String(localized: "This change was not authorized") - ) - } + if let denial = decision.denialError { throw denial } } private static func execute( diff --git a/apple/TablePro/Core/Database/DatabaseManager+Health.swift b/apple/TablePro/Core/Database/DatabaseManager+Health.swift index b42ed9eae7..3f9fc5feb2 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Health.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Health.swift @@ -149,10 +149,17 @@ extension DatabaseManager { result.driver.disconnect() return .abort } + let warning = openingWarning( + driver: result.driver, + startupFailures: result.startupFailures, + restoreWarning: result.restoreWarning, + for: connectionId + ) updateSession(connectionId) { session in session.driver = result.driver session.effectiveConnection = result.effectiveConnection session.status = .connected + session.connectionWarning = warning if let schemaDriver = result.driver as? SchemaSwitchable { session.browseSchema = schemaDriver.currentSchema } @@ -233,6 +240,8 @@ extension DatabaseManager { let driver: DatabaseDriver let effectiveConnection: DatabaseConnection let cachedPassword: String? + let startupFailures: [StartupCommandFailure] + let restoreWarning: String? } // Rebuilds the tunnel for a tunneled session before connecting the new driver. Tunnels are keyed by @@ -282,13 +291,13 @@ extension DatabaseManager { throw CancellationError() } - await applyTimeoutAndStartupCommands( + let startupFailures = await applyTimeoutAndStartupCommands( on: driver, queryTimeoutSeconds: session.effectiveQueryTimeoutSeconds, startupCommands: session.connection.startupCommands, connectionName: session.connection.name ) - await restoreSchemaAndDatabase( + let restoreWarning = await restoreSchemaAndDatabase( on: driver, savedSchema: session.browseSchema, savedDatabase: databaseSwitchRequiresReconnect(session.connection) ? nil : session.browseDatabase @@ -297,7 +306,9 @@ extension DatabaseManager { return ReconnectResult( driver: driver, effectiveConnection: connectionForDriver, - cachedPassword: connectResult.cachedPassword + cachedPassword: connectResult.cachedPassword, + startupFailures: startupFailures, + restoreWarning: restoreWarning ) } @@ -306,7 +317,7 @@ extension DatabaseManager { queryTimeoutSeconds: Int, startupCommands: String?, connectionName: String - ) async { + ) async -> [StartupCommandFailure] { do { try await driver.applyQueryTimeout(queryTimeoutSeconds) } catch { @@ -315,7 +326,7 @@ extension DatabaseManager { ) } - await executeStartupCommands(startupCommands, on: driver, connectionName: connectionName) + return await executeStartupCommands(startupCommands, on: driver, connectionName: connectionName) } private func databaseSwitchRequiresReconnect(_ connection: DatabaseConnection) -> Bool { @@ -323,17 +334,23 @@ extension DatabaseManager { .capabilities.requiresReconnectForDatabaseSwitch ?? false } + // The reconnected session stays on the server's default when a restore fails, which the window would not show. func restoreSchemaAndDatabase( on driver: DatabaseDriver, savedSchema: String?, savedDatabase: String? - ) async { + ) async -> String? { + var warnings: [String?] = [] // The schema lives inside the database, and a database switch can leave the driver on none. if let savedDatabase, let adapter = driver as? PluginDriverAdapter { do { try await adapter.switchDatabase(to: savedDatabase) } catch { Self.logger.warning("Failed to restore database '\(savedDatabase)' on reconnect: \(error.localizedDescription)") + warnings.append(String( + format: String(localized: "Reconnecting could not reopen the database %1$@: %2$@"), + savedDatabase, error.localizedDescription + )) } } @@ -342,8 +359,13 @@ extension DatabaseManager { try await schemaDriver.switchSchema(to: savedSchema) } catch { Self.logger.warning("Failed to restore schema '\(savedSchema)' on reconnect: \(error.localizedDescription)") + warnings.append(String( + format: String(localized: "Reconnecting could not reopen the schema %1$@: %2$@"), + savedSchema, error.localizedDescription + )) } } + return Self.joinedWarning(warnings) } internal func stopHealthMonitor(for connectionId: UUID) async { @@ -440,13 +462,13 @@ extension DatabaseManager { } let driver = connectResult.driver - await applyTimeoutAndStartupCommands( + let startupFailures = await applyTimeoutAndStartupCommands( on: driver, queryTimeoutSeconds: session.effectiveQueryTimeoutSeconds, startupCommands: session.connection.startupCommands, connectionName: session.connection.name ) - await restoreSchemaAndDatabase( + let restoreWarning = await restoreSchemaAndDatabase( on: driver, savedSchema: activeSessions[sessionId]?.browseSchema, savedDatabase: databaseSwitchRequiresReconnect(session.connection) ? nil : activeSessions[sessionId]?.browseDatabase @@ -456,10 +478,14 @@ extension DatabaseManager { throw CancellationError() } + let warning = openingWarning( + driver: driver, startupFailures: startupFailures, restoreWarning: restoreWarning, for: sessionId + ) updateSession(sessionId) { session in session.driver = driver session.status = .connected session.effectiveConnection = effectiveConnection + session.connectionWarning = warning if let schemaDriver = driver as? SchemaSwitchable { session.browseSchema = schemaDriver.currentSchema } diff --git a/apple/TablePro/Core/Database/DatabaseManager+Principals.swift b/apple/TablePro/Core/Database/DatabaseManager+Principals.swift index 7cf505ddcd..12680f123e 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Principals.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Principals.swift @@ -14,7 +14,8 @@ extension DatabaseManager { func executePrincipalChanges( changes: [PrincipalChange], databaseType: DatabaseType, - connectionId: UUID + connectionId: UUID, + gate: any ExecutionGate = ExecutionGateProvider.shared ) async throws { guard let driver = driver(for: connectionId) else { throw DatabaseError.notConnected @@ -39,7 +40,7 @@ extension DatabaseManager { ? .destructiveQuery : .schemaMutation - let authorization = await ExecutionGateProvider.shared.authorize( + let authorization = await gate.authorize( OperationRequest( connectionId: connectionId, databaseType: databaseType, @@ -50,12 +51,7 @@ extension DatabaseManager { operationDescription: String(localized: "Apply User and Role Changes") ) ) - guard case .authorized = authorization else { - throw DatabaseError.queryFailed( - authorization.deniedReason - ?? String(localized: "User and role changes were not authorized") - ) - } + if let denial = authorization.denialError { throw denial } /// Reported whether or not every statement ran: an engine that cannot roll principal /// statements back keeps the ones before a failure, and `DROP OWNED` among them drops objects. diff --git a/apple/TablePro/Core/Database/DatabaseManager+Schema.swift b/apple/TablePro/Core/Database/DatabaseManager+Schema.swift index fd4b884352..01ab285459 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Schema.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Schema.swift @@ -226,13 +226,7 @@ extension DatabaseManager { } } - /// Run a Create Table draft's statements, on the same isolated route and in the same shape as - /// `executeSchemaChanges`. - /// - /// The route is what matters. The view used to reach for `driver(for:)`, which is the session - /// driver, so a `BEGIN` opened here joined whatever transaction a query tab had left open and - /// the matching `COMMIT` took that tab's uncommitted work with it. `schemaChangeRoute` exists - /// for exactly that, and the app's own DDL has no business on the user's connection. + // On the schema change route: a BEGIN on the session driver would join a query tab's open transaction. func executeCreateTable( statements: [String], databaseType: DatabaseType, @@ -254,11 +248,7 @@ extension DatabaseManager { operationDescription: String(localized: "Create Table") ) ) - guard case .authorized = authorization else { - throw DatabaseError.queryFailed( - authorization.deniedReason ?? String(localized: "Operation not permitted") - ) - } + if let denial = authorization.denialError { throw denial } let executionTimes: [TimeInterval] do { diff --git a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift index 2e0bfa525e..23c6f022e6 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift @@ -136,7 +136,7 @@ extension DatabaseManager { attempt: attempt, for: connection.id ) - await applyTimeoutAndStartupCommands( + let startupFailures = await applyTimeoutAndStartupCommands( on: driver, queryTimeoutSeconds: effectiveQueryTimeoutSeconds, startupCommands: resolvedConnection.startupCommands, @@ -167,7 +167,7 @@ extension DatabaseManager { session.driver = driver session.status = driver.status session.effectiveConnection = effectiveConnection - session.connectionWarning = driver.connectionWarning + session.connectionWarning = openingWarning(driver: driver, startupFailures: startupFailures, for: connection.id) // A connect answers whatever went wrong before it, including a reconnect that gave up on this entry. session.liveness = .live session.recovery = nil @@ -1172,6 +1172,7 @@ extension DatabaseManager { monitorPingedDrivers.removeValue(forKey: connectionId) sessionDriverGate.drain(connectionId: connectionId) connectionStatusVersions.removeValue(forKey: connectionId) + reportedStartupFailures.removeValue(forKey: connectionId) forgetVerification(for: connectionId) aiAccessApprovals.revoke(connectionId) AppEvents.shared.connectionStatusChanged.send( diff --git a/apple/TablePro/Core/Database/DatabaseManager+Startup.swift b/apple/TablePro/Core/Database/DatabaseManager+Startup.swift index 5f68e9c7a7..3804867891 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Startup.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Startup.swift @@ -61,26 +61,116 @@ extension DatabaseManager { return statements } - @discardableResult nonisolated internal func executeStartupCommands( _ commands: String?, on driver: DatabaseDriver, connectionName: String - ) async -> [(statement: String, error: String)] { - var failures: [(statement: String, error: String)] = [] + ) async -> [StartupCommandFailure] { + var failures: [StartupCommandFailure] = [] let databaseType = driver.connection.type let grammar = SQLLexicalResolver.readings(for: databaseType, session: driver.sessionLexicalState).execution for statement in Self.startupStatements(from: commands, databaseType: databaseType, grammar: grammar) { do { _ = try await driver.execute(query: statement) Self.startupLogger.info( - "Startup command succeeded for '\(connectionName)': \(statement)" + "Startup command succeeded for '\(connectionName, privacy: .private)': \(statement, privacy: .private)" ) } catch { Self.startupLogger.warning( - "Startup command failed for '\(connectionName)': \(statement) — \(error.localizedDescription)" + "Startup command failed for '\(connectionName, privacy: .private)': \(statement, privacy: .private): \(error.localizedDescription, privacy: .private)" ) - failures.append((statement: statement, error: error.localizedDescription)) + failures.append(StartupCommandFailure(statement: statement, error: error)) } } return failures } + + // A startup command can be a safety rail, `SET ROLE app_ro` or a read-only default, so a session that opened + // without one says so on the connection's warning rather than only in the log. + nonisolated internal static func startupWarning(for failures: [StartupCommandFailure]) -> String? { + guard let first = failures.first else { return nil } + guard failures.count > 1 else { + guard let code = first.code else { + return String( + format: String(localized: "The startup command %@ failed, so this session runs without it."), + shownStatement(first.statement) + ) + } + return String( + format: String(localized: "The startup command %1$@ failed (%2$@), so this session runs without it."), + shownStatement(first.statement), code + ) + } + let lines = failures.map { failure in + failure.code.map { "\(shownStatement(failure.statement)) (\($0))" } ?? shownStatement(failure.statement) + } + return String( + format: String(localized: "Startup commands failed, so this session runs without them:\n%@"), + lines.joined(separator: "\n") + ) + } + + // A startup command can carry a secret, a password after IDENTIFIED BY or a key in a literal, so the banner shows + // it only up to its first quote or credential clause. + nonisolated internal static func shownStatement(_ statement: String) -> String { + let quote = statement.firstIndex(where: { $0 == "'" || $0 == "\"" || $0 == "$" }) + let stops = [quote] + ["IDENTIFIED", "PASSWORD"].map { statement.range(of: $0, options: .caseInsensitive)?.lowerBound } + let cut = stops.compactMap { $0 }.min() ?? statement.endIndex + let kept = statement[.. limit else { return kept } + return String(kept.prefix(limit)) + "…" + } + + nonisolated internal static func joinedWarning(_ parts: [String?]) -> String? { + let present = parts.compactMap { $0 }.filter { !$0.isEmpty } + return present.isEmpty ? nil : present.joined(separator: "\n\n") + } + + // The warning a session opens with: the login's own, then what its setup could not do. It replaces the previous + // one, so a reconnect that ran every startup command drops a banner about the session before it, and the record + // of named failures starts over. + internal func openingWarning( + driver: DatabaseDriver, + startupFailures: [StartupCommandFailure], + restoreWarning: String? = nil, + for connectionId: UUID + ) -> String? { + reportedStartupFailures[connectionId] = Set(startupFailures.map { $0.statement }) + return Self.joinedWarning([driver.connectionWarning, Self.startupWarning(for: startupFailures), restoreWarning]) + } + + // A pool entry or a database lane opens a session of its own, which can fail a command the window's session ran. + // Each failing statement is named once per connect: the pool keeps opening entries while the window is open. + internal func reportStartupFailures(_ failures: [StartupCommandFailure], for connectionId: UUID) { + guard activeSessions[connectionId] != nil else { return } + var reported = reportedStartupFailures[connectionId] ?? [] + let unreported = failures.filter { reported.insert($0.statement).inserted } + reportedStartupFailures[connectionId] = reported + guard let warning = Self.startupWarning(for: unreported) else { return } + updateSession(connectionId) { session in + session.connectionWarning = Self.joinedWarning([session.connectionWarning, warning]) + } + } +} + +// What the banner may show of a failed startup command: the statement and the server's code, never its message, which +// can echo the values the statement carried. The message goes to the log, private. +struct StartupCommandFailure: Sendable, Equatable { + let statement: String + let code: String? + + init(statement: String, code: String?) { + self.statement = statement + self.code = code + } + + init(statement: String, error: Error) { + self.init(statement: statement, code: Self.code(of: error)) + } + + static func code(of error: Error) -> String? { + guard let driverError = error as? any PluginDriverError else { return nil } + if let state = driverError.pluginSqlState, !state.isEmpty { return state } + if let code = driverError.pluginErrorCode, code != 0 { return String(code) } + return nil + } } diff --git a/apple/TablePro/Core/Database/DatabaseManager.swift b/apple/TablePro/Core/Database/DatabaseManager.swift index 15072aca99..2dec865754 100644 --- a/apple/TablePro/Core/Database/DatabaseManager.swift +++ b/apple/TablePro/Core/Database/DatabaseManager.swift @@ -126,6 +126,10 @@ final class DatabaseManager: ObservableObject { /// for the same reason every other shared write here is generation-checked. internal var connectionStages: [UUID: ConnectionStage] = [:] + // The startup statements a connection's warning has already named since its connect, so a pool entry or lane + // opened later names only a failure the window was not told about. + internal var reportedStartupFailures: [UUID: Set] = [:] + /// Orders operations that move the shared driver, so two windows cannot interleave /// their pins and each run against the other's database. internal let sessionDriverGate = SessionDriverGate() diff --git a/apple/TablePro/Core/Database/TriggerEditing.swift b/apple/TablePro/Core/Database/TriggerEditing.swift index c3744ad7e4..8a427c1bfd 100644 --- a/apple/TablePro/Core/Database/TriggerEditing.swift +++ b/apple/TablePro/Core/Database/TriggerEditing.swift @@ -14,7 +14,6 @@ import TableProSQLGrammar enum TriggerEditingError: LocalizedError { case notConnected - case denied(String) case dropUnavailable case notCreateTrigger case definitionUnavailable @@ -23,7 +22,6 @@ enum TriggerEditingError: LocalizedError { var errorDescription: String? { switch self { case .notConnected: String(localized: "Not connected to database") - case let .denied(reason): reason case .dropUnavailable: String(localized: "This database cannot drop triggers") case .notCreateTrigger: String( @@ -121,9 +119,7 @@ enum TriggerApplyStrategy: Equatable { enum TriggerEditing { nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "TriggerEditing") - /// Runs on the schema change route, never on the session driver: the trigger's own BEGIN - /// would join a transaction a query tab left open, and its COMMIT or ROLLBACK would take - /// that tab's uncommitted work with it. + // On the schema change route: a BEGIN on the session driver would join a query tab's open transaction. static func apply( scope: DatabaseScope, connection: DatabaseConnection, @@ -162,9 +158,7 @@ enum TriggerEditing { statements: plan.gatedStatements(for: sql) ) ) - guard case .authorized = decision else { - throw TriggerEditingError.denied(decision.deniedReason ?? String(localized: "Operation not permitted")) - } + if let denial = decision.denialError { throw denial } let startedAt = Date() defer { @@ -230,10 +224,7 @@ enum TriggerEditing { name: String, gate: any ExecutionGate = ExecutionGateProvider.shared ) async throws { - /// Built on a driver in the scope that will run it, the way `apply` builds its own inside - /// the lease. The session driver is wherever the sidebar last went, and an engine that - /// qualifies its DDL with the connection's current database writes that name into a - /// statement the pooled connection then runs somewhere else. + // Built in the scope that runs it: the session driver may qualify the DDL with another database. let generated = try await DatabaseManager.shared.withMetadataDriver(scope: scope) { driver in driver.generateDropTriggerSQL(name: name, table: tableName) } @@ -252,9 +243,7 @@ enum TriggerEditing { operationDescription: String(localized: "Drop Trigger") ) ) - guard case .authorized = decision else { - throw TriggerEditingError.denied(decision.deniedReason ?? String(localized: "Operation not permitted")) - } + if let denial = decision.denialError { throw denial } let startedAt = Date() try await withSchemaChangeDriver(scope: scope) { driver in diff --git a/apple/TablePro/Core/LSP/LSPTransport.swift b/apple/TablePro/Core/LSP/LSPTransport.swift index 47e274cc6a..6677bb3748 100644 --- a/apple/TablePro/Core/LSP/LSPTransport.swift +++ b/apple/TablePro/Core/LSP/LSPTransport.swift @@ -32,6 +32,45 @@ enum LSPTransportError: Error, LocalizedError { } } +// MARK: - LSPMessageWriter + +// Writes on its own queue, so a server that stops reading blocks the queue and never the transport. +final class LSPMessageWriter: @unchecked Sendable { + private let handle: FileHandle + private let queue = DispatchQueue(label: "com.TablePro.LSPTransport.writer") + private var isClosed = false + + init(_ handle: FileHandle) throws { + // Without it, a write to a server that has exited raises SIGPIPE and ends the app. + guard fcntl(handle.fileDescriptor, F_SETNOSIGPIPE, 1) == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + self.handle = handle + } + + func write(_ frame: Data, onFailure: @escaping @Sendable (any Error) -> Void) { + queue.async { [self] in + guard !isClosed else { + onFailure(LSPTransportError.processNotRunning) + return + } + do { + try handle.write(contentsOf: frame) + } catch { + onFailure(error) + } + } + } + + func close() { + queue.async { [self] in + guard !isClosed else { return } + isClosed = true + try? handle.close() + } + } +} + // MARK: - LSPTransport actor LSPTransport { @@ -39,6 +78,7 @@ actor LSPTransport { private var process: Process? private var stdinPipe: Pipe? + private var writer: LSPMessageWriter? private var stdoutPipe: Pipe? private var stderrPipe: Pipe? private var stderrReader: PipeReader? @@ -66,6 +106,7 @@ actor LSPTransport { proc.standardOutput = stdout proc.standardError = stderr + self.writer = try LSPMessageWriter(stdin.fileHandleForWriting) self.stdinPipe = stdin self.stdoutPipe = stdout self.stderrPipe = stderr @@ -105,9 +146,8 @@ actor LSPTransport { readerTask?.cancel() readerTask = nil - if let stdinHandle = stdinPipe?.fileHandleForWriting { - try? stdinHandle.close() - } + writer?.close() + writer = nil if let stdoutHandle = stdoutPipe?.fileHandleForReading { try? stdoutHandle.close() } @@ -144,7 +184,7 @@ actor LSPTransport { return try await withCheckedThrowingContinuation { continuation in pendingRequests[requestID] = continuation do { - try writeMessage(data) + try writeMessage(data, failingRequest: requestID) } catch { pendingRequests.removeValue(forKey: requestID) continuation.resume(throwing: error) @@ -212,19 +252,22 @@ actor LSPTransport { // MARK: - Private - private func writeMessage(_ data: Data) throws { - guard let stdinPipe else { + private func writeMessage(_ data: Data, failingRequest requestID: Int? = nil) throws { + guard let writer else { throw LSPTransportError.processNotRunning } - - let header = "Content-Length: \(data.count)\r\n\r\n" - guard let headerData = header.data(using: .utf8) else { - throw LSPTransportError.invalidResponse + let frame = Data("Content-Length: \(data.count)\r\n\r\n".utf8) + data + writer.write(frame) { [weak self] error in + Task { [weak self] in + await self?.writeFailed(error, requestID: requestID) + } } + } - let handle = stdinPipe.fileHandleForWriting - handle.write(headerData) - handle.write(data) + private func writeFailed(_ error: any Error, requestID: Int?) { + Self.logger.warning("LSP write failed: \(error.localizedDescription)") + guard let requestID, let continuation = pendingRequests.removeValue(forKey: requestID) else { return } + continuation.resume(throwing: error) } private func runReadLoop(handle: FileHandle) async { @@ -341,6 +384,9 @@ actor LSPTransport { private func handleProcessExit(code: Int32) { Self.logger.info("LSP process exited with code \(code)") + writer?.close() + writer = nil + stdinPipe = nil let pending = pendingRequests pendingRequests.removeAll() for (_, continuation) in pending { diff --git a/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift b/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift index 164f6c3613..5b6028255f 100644 --- a/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift +++ b/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift @@ -38,6 +38,8 @@ internal struct ObjectCopyRowCopier: Sendable { internal struct Outcome: Sendable { internal let inserted: Int internal let cancelled: Bool + // The limit the source's stream stopped at, so the table holds only its leading rows. + internal var rowCut: Int? } /// A table that failed after some of its batches were written. Outside a transaction those stay @@ -94,7 +96,13 @@ internal struct ObjectCopyRowCopier: Sendable { let generator = try makeGenerator(targetDriver: targetDriver) let budget = SQLWriteBatchBudget(columnCount: step.columns.count, generator: generator) var filler = SQLWriteBatchFiller<[PluginCellValue]>(budget: budget) - var stream = sourceDriver.streamRows(query: step.sourceQuery).makeAsyncIterator() + let cut = OSAllocatedUnfairLock(initialState: nil) + // The copy's SELECT states no engine ceiling, so only a limit the driver declares applies, as for its own export query. + var stream = ExportRowCut.reporting( + sourceDriver.streamRows(query: step.sourceQuery), engineCeiling: nil + ) { limit in + cut.withLock { $0 = limit } + }.makeAsyncIterator() var inserted = 0 @@ -121,7 +129,7 @@ internal struct ObjectCopyRowCopier: Sendable { inserted += try await write(batch, generator: generator, driver: targetDriver) onProgress(inserted) } - return Outcome(inserted: inserted, cancelled: Task.isCancelled) + return Outcome(inserted: inserted, cancelled: Task.isCancelled, rowCut: cut.withLock { $0 }) } catch is CancellationError { return Outcome(inserted: inserted, cancelled: true) } catch where inserted > 0 { diff --git a/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift b/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift index 6cef58ed51..f8c04fd656 100644 --- a/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift +++ b/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift @@ -22,6 +22,7 @@ internal struct ObjectCopyObjectOutcome: Identifiable, Sendable { internal let selection: ObjectCopySelection internal let rowsCopied: Int internal let error: String? + internal var warning: String? internal var id: String { selection.id } internal var succeeded: Bool { error == nil } @@ -67,6 +68,8 @@ internal struct ObjectCopyRunResult: Sendable { internal var firstError: String? { outcomes.compactMap(\.error).first } + internal var warnings: [String] { outcomes.compactMap(\.warning) } + /// Every failure, each with an identity of its own. /// /// One object fails once per phase it reaches, so a table whose `CREATE` failed under Skip and @@ -507,9 +510,14 @@ internal struct ObjectCopyRunner { /// would send a retry over them. var rowsCopied = 0 - mutating func recordCopy(of step: ObjectCopyTableStep, inserted: Int) { + mutating func recordCopy(of step: ObjectCopyTableStep, inserted: Int, rowCut: Int? = nil) { rowsCopied += inserted - outcomes.append(ObjectCopyObjectOutcome(selection: step.selection, rowsCopied: inserted, error: nil)) + outcomes.append(ObjectCopyObjectOutcome( + selection: step.selection, + rowsCopied: inserted, + error: nil, + warning: rowCut.map { StreamRowCutWarning.text(table: step.selection.qualifiedName, limit: $0) } + )) } /// Stop reaching the driver mid-stream is the user stopping, not the copy failing, and the @@ -702,7 +710,7 @@ internal struct ObjectCopyRunner { result.recordStop(of: step, inserted: outcome.inserted, progress: progress) break } - result.recordCopy(of: step, inserted: outcome.inserted) + result.recordCopy(of: step, inserted: outcome.inserted, rowCut: outcome.rowCut) } catch is CancellationError { result.recordStop(of: step, inserted: 0, progress: progress) break @@ -748,7 +756,7 @@ internal struct ObjectCopyRunner { result.recordStop(of: step, inserted: outcome.inserted, progress: progress) break } - result.recordCopy(of: step, inserted: outcome.inserted) + result.recordCopy(of: step, inserted: outcome.inserted, rowCut: outcome.rowCut) } catch is CancellationError { result.recordStop(of: step, inserted: 0, progress: progress) break diff --git a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift index 4c67711e74..5878dcda37 100644 --- a/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift +++ b/apple/TablePro/Core/Plugins/ExportDataSourceAdapter.swift @@ -61,13 +61,7 @@ final class ExportDataSourceAdapter: PluginExportDataSource, @unchecked Sendable // One line per table whose rows stopped at a ceiling, so a partial copy is never reported as the whole table. var cappedTableWarnings: [String] { - cappedTables.withLock { $0 }.map { capped in - String( - format: String(localized: "%1$@: only the first %2$lld rows were read, the most this database returns from one query."), - capped.table, - capped.limit - ) - } + cappedTables.withLock { $0 }.map { StreamRowCutWarning.text(table: $0.table, limit: $0.limit) } } func streamRows(table: String, databaseName: String) -> AsyncThrowingStream { diff --git a/apple/TablePro/Core/Plugins/ExportRowCut.swift b/apple/TablePro/Core/Plugins/ExportRowCut.swift index 04efb07dc6..a3be667953 100644 --- a/apple/TablePro/Core/Plugins/ExportRowCut.swift +++ b/apple/TablePro/Core/Plugins/ExportRowCut.swift @@ -58,6 +58,17 @@ final class ExportRowCut: @unchecked Sendable { } } +// One wording for every reader of a cut stream: export, Table Transfer, Object Copy and Data Compare. +enum StreamRowCutWarning { + static func text(table: String, limit: Int) -> String { + String( + format: String(localized: "%1$@: only the first %2$lld rows were read, the most this database returns from one query."), + table, + limit + ) + } +} + extension ExportRowCut where Element == PluginStreamElement { static func reporting( _ upstream: AsyncThrowingStream, diff --git a/apple/TablePro/Core/Services/Execution/GatedOperationFailure.swift b/apple/TablePro/Core/Services/Execution/GatedOperationFailure.swift new file mode 100644 index 0000000000..f0e6bdaeae --- /dev/null +++ b/apple/TablePro/Core/Services/Execution/GatedOperationFailure.swift @@ -0,0 +1,39 @@ +// +// GatedOperationFailure.swift +// TablePro +// + +import Foundation + +// Cancel at the gate's confirmation is the user's own choice: it shows nothing and the sheet keeps its edits. +internal enum GatedOperationFailure: Equatable { + case cancelledByUser + case refused(String) + case failed(String) + + internal init(_ error: any Error) { + guard let gateError = error as? ExecutionGateError else { + self = .failed(error.localizedDescription) + return + } + switch gateError { + case .cancelledByUser: + self = .cancelledByUser + case .denied(let reason): + self = .refused(reason) + } + } + + internal var message: String? { + switch self { + case .cancelledByUser: nil + case .refused(let reason), .failed(let reason): reason + } + } + + // A refusal and a Cancel both stop before the first statement: nothing to report or reload. + internal var reportsFailure: Bool { + if case .failed = self { return true } + return false + } +} diff --git a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift index 2d85197c26..c7a274f88f 100644 --- a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift +++ b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift @@ -591,13 +591,14 @@ final class MetadataConnectionPool { ) do { try await driver.connect() - try await Self.prepareSession( + let startupFailures = try await Self.prepareSession( driver, queryTimeoutSeconds: queryTimeout, startupCommands: session.connection.startupCommands, connectionName: session.connection.name, timeoutSeconds: operationTimeoutSeconds ) + DatabaseManager.shared.reportStartupFailures(startupFailures, for: scope.connectionId) if let database = plan.switchDatabase { try await Self.switchDatabase(driver, to: database, timeoutSeconds: operationTimeoutSeconds) } @@ -680,14 +681,14 @@ final class MetadataConnectionPool { startupCommands: String?, connectionName: String, timeoutSeconds: Double? - ) async throws { + ) async throws -> [StartupCommandFailure] { try await bounded( driver: driver, timeoutSeconds: timeoutSeconds, timeoutMessage: String(localized: "Preparing the metadata connection timed out.") ) { try? await driver.applyQueryTimeout(queryTimeoutSeconds) - await DatabaseManager.shared.executeStartupCommands( + return await DatabaseManager.shared.executeStartupCommands( startupCommands, on: driver, connectionName: connectionName ) } @@ -695,18 +696,17 @@ final class MetadataConnectionPool { /// Disconnects the driver when the deadline fires so a driver call that /// ignores task cancellation still completes and the timeout can propagate. - private static func bounded( + private static func bounded( driver: DatabaseDriver, timeoutSeconds: Double?, timeoutMessage: String, - _ operation: @escaping @Sendable () async throws -> Void - ) async throws { + _ operation: @escaping @Sendable () async throws -> T + ) async throws -> T { guard let timeoutSeconds, timeoutSeconds > 0 else { - try await operation() - return + return try await operation() } do { - try await withTimeout( + return try await withTimeout( seconds: timeoutSeconds, onTimeout: { driver.disconnect() }, operation: operation diff --git a/apple/TablePro/Core/Services/Query/RowOperationsManager.swift b/apple/TablePro/Core/Services/Query/RowOperationsManager.swift index 66f5ebf5e4..2a33425139 100644 --- a/apple/TablePro/Core/Services/Query/RowOperationsManager.swift +++ b/apple/TablePro/Core/Services/Query/RowOperationsManager.swift @@ -49,10 +49,12 @@ final class RowOperationsManager { func addNewRow(tableRows: inout TableRows, uuidKeyForms: [Int: UUIDCellForm] = [:]) -> AddNewRowResult? { var newRowValues: [PluginCellValue] = [] var absentColumns: Set = [] + var defaultColumns: Set = [] var prefilled: Set = [] for (index, column) in tableRows.columns.enumerated() { if tableRows.generatedColumns.contains(column) || tableRows.serverAssignsValue(forColumn: column) { - newRowValues.append(.text("__DEFAULT__")) + newRowValues.append(.columnDefault) + defaultColumns.insert(index) } else if let form = uuidKeyForms[index] { newRowValues.append(Self.freshUUID(form)) prefilled.insert(index) @@ -62,7 +64,11 @@ final class RowOperationsManager { } } return appendInsertedRow( - values: newRowValues, absentColumns: absentColumns, prefilledColumns: prefilled, to: &tableRows + values: newRowValues, + absentColumns: absentColumns, + expressionColumns: defaultColumns, + prefilledColumns: prefilled, + to: &tableRows ) } @@ -86,14 +92,15 @@ final class RowOperationsManager { + Array(tableRows.columnIdentity.keys) for resetColumn in resetColumns { if let index = tableRows.columns.firstIndex(of: resetColumn), index < newValues.count { + absentColumns.remove(index) if let form = uuidKeyForms[index] { newValues[index] = Self.freshUUID(form) prefilled.insert(index) + expressionColumns.remove(index) } else { - newValues[index] = .text("__DEFAULT__") + newValues[index] = .columnDefault + expressionColumns.insert(index) } - absentColumns.remove(index) - expressionColumns.remove(index) } } return appendInsertedRow( @@ -406,9 +413,9 @@ final class RowOperationsManager { } if let pkIndex = schema.primaryKeyIndex, pkIndex < values.count { - values[pkIndex] = .text("__DEFAULT__") + values[pkIndex] = .columnDefault absentColumns.remove(pkIndex) - expressionColumns.remove(pkIndex) + expressionColumns.insert(pkIndex) } return ParsedRow( @@ -438,8 +445,8 @@ final class RowOperationsManager { return .entered(text, columnType: tableRows.editingColumnType(at: column)) } if let pkIndex = schema.primaryKeyIndex, pkIndex < values.count { - values[pkIndex] = .text("__DEFAULT__") - expressionColumns.remove(pkIndex) + values[pkIndex] = .columnDefault + expressionColumns.insert(pkIndex) } return ParsedRow(values: values, sourceLineNumber: index + 1, expressionColumns: expressionColumns) } @@ -447,28 +454,39 @@ final class RowOperationsManager { // A blank cell of pasted text was left unfilled, so it takes what a new row's cell does: the server's value where // the column has a default, else NULL. A NOT NULL text column with no default takes the empty string it can hold. - private static func unfilledCellValues(columns: [String], tableRows: TableRows) -> [PluginCellValue] { - columns.map { name in + private static func unfilledCellValues( + columns: [String], + tableRows: TableRows + ) -> (values: [PluginCellValue], defaults: Set) { + var defaults: Set = [] + let values = columns.enumerated().map { position, name -> PluginCellValue in guard let index = tableRows.columns.firstIndex(of: name) else { return .null } if tableRows.generatedColumns.contains(name) || tableRows.serverAssignsValue(forColumn: name) { - return .text("__DEFAULT__") + defaults.insert(position) + return .columnDefault } if tableRows.refusesNull(columnAt: index), tableRows.editingColumnType(at: index)?.hasEmptyValue == true { return .text("") } return .null } + return (values, defaults) } - private static func fillingBlankCells(_ row: ParsedRow, with unfilled: [PluginCellValue]) -> ParsedRow { + private static func fillingBlankCells( + _ row: ParsedRow, + with unfilled: (values: [PluginCellValue], defaults: Set) + ) -> ParsedRow { guard !row.blankColumns.isEmpty else { return row } var values = row.values - for index in row.blankColumns where index < values.count && index < unfilled.count { - values[index] = unfilled[index] + var expressionColumns = row.expressionColumns + for index in row.blankColumns where index < values.count && index < unfilled.values.count { + values[index] = unfilled.values[index] + if unfilled.defaults.contains(index) { expressionColumns.insert(index) } } var filled = ParsedRow(values: values, sourceLineNumber: row.sourceLineNumber) filled.absentColumns = row.absentColumns - filled.expressionColumns = row.expressionColumns + filled.expressionColumns = expressionColumns return filled } @@ -509,7 +527,7 @@ final class RowOperationsManager { return containsComma ? CSVRowParser() : TSVRowParser() } - // Every paste path resets the key to __DEFAULT__ first; an unfilled UUID key then takes a fresh UUID per row. + // Every paste path resets the key to its default first; an unfilled UUID key then takes a fresh UUID per row. private func insertParsedRows( _ parsedRows: [ParsedRow], uuidKeyForms: [Int: UUIDCellForm], @@ -531,11 +549,12 @@ final class RowOperationsManager { var absentColumns = parsedRow.absentColumns var expressionColumns = parsedRow.expressionColumns.filter { $0 < rowValues.count } for index in serverOwned where index < rowValues.count { - rowValues[index] = .text("__DEFAULT__") + rowValues[index] = .columnDefault absentColumns.remove(index) - expressionColumns.remove(index) + expressionColumns.insert(index) } - for (index, form) in uuidKeyForms where index < rowValues.count && rowValues[index] == .text("__DEFAULT__") { + for (index, form) in uuidKeyForms where index < rowValues.count + && rowValues[index].isColumnDefault(isExpression: expressionColumns.contains(index)) { rowValues[index] = Self.freshUUID(form) absentColumns.remove(index) expressionColumns.remove(index) diff --git a/apple/TablePro/Core/Services/Query/RowParser.swift b/apple/TablePro/Core/Services/Query/RowParser.swift index cdd13c09a6..64179f3039 100644 --- a/apple/TablePro/Core/Services/Query/RowParser.swift +++ b/apple/TablePro/Core/Services/Query/RowParser.swift @@ -37,13 +37,17 @@ private func fittedRow( blankColumns.insert(values.count) values.append(nil) } - if let pkIndex = schema.primaryKeyIndex, pkIndex < values.count { - values[pkIndex] = "__DEFAULT__" + var cells = values.map(PluginCellValue.fromOptional) + var defaultColumns: Set = [] + if let pkIndex = schema.primaryKeyIndex, pkIndex < cells.count { + cells[pkIndex] = .columnDefault + defaultColumns.insert(pkIndex) blankColumns.remove(pkIndex) } return ParsedRow( - values: values.map(PluginCellValue.fromOptional), + values: cells, sourceLineNumber: lineNumber, + expressionColumns: defaultColumns, blankColumns: blankColumns ) } diff --git a/apple/TablePro/Core/Services/Query/SelectionSummaryCalculator.swift b/apple/TablePro/Core/Services/Query/SelectionSummaryCalculator.swift index 7470c95bb8..4d031fd612 100644 --- a/apple/TablePro/Core/Services/Query/SelectionSummaryCalculator.swift +++ b/apple/TablePro/Core/Services/Query/SelectionSummaryCalculator.swift @@ -61,11 +61,8 @@ actor SelectionSummaryCalculator { } private struct Tally { - private static let defaultMarkerLength = PluginCellValue.defaultMarkerText.utf8.count - let emptyTextIsEmpty: Bool - let insertedRowIDs: Set - let modifiedCells: [RowID: Set] + let defaultedCells: [RowID: Set] var valueCount = 0 var emptyCount = 0 var notANumberCount = 0 @@ -73,8 +70,7 @@ private struct Tally { init(input: SelectionSummaryInput) { emptyTextIsEmpty = input.policy.emptyTextIsEmpty - insertedRowIDs = input.insertedRowIDs - modifiedCells = input.modifiedCells + defaultedCells = input.defaultedCells } mutating func add(_ value: PluginCellValue, column: Int, rule: SelectionSummaryColumnRule, rowID: RowID) { @@ -108,13 +104,9 @@ private struct Tally { return numbers.add(text.trimmingCharacters(in: .whitespaces)) } - // The default marker is the grid's placeholder only where an edit put it; elsewhere it is text the table holds. private func isEmpty(_ text: String, column: Int, rowID: RowID) -> Bool { if text.isEmpty { return emptyTextIsEmpty } - guard text.utf8.count == Self.defaultMarkerLength, text == PluginCellValue.defaultMarkerText else { - return false - } - return insertedRowIDs.contains(rowID) || modifiedCells[rowID]?.contains(column) == true + return defaultedCells[rowID]?.contains(column) == true } func summary(coversWholeColumn: Bool) -> SelectionSummary { diff --git a/apple/TablePro/Core/Shortcuts/Shared/PayloadRowPlanner.swift b/apple/TablePro/Core/Shortcuts/Shared/PayloadRowPlanner.swift index 47d4e2da17..0065a4e3d0 100644 --- a/apple/TablePro/Core/Shortcuts/Shared/PayloadRowPlanner.swift +++ b/apple/TablePro/Core/Shortcuts/Shared/PayloadRowPlanner.swift @@ -16,17 +16,17 @@ nonisolated enum PayloadRowPlanner { dropsEmptyPrimaryKey: Bool = true ) throws -> [[(column: String, value: String?)]] { guard !columns.isEmpty else { throw IntentDataError.noColumns(table) } - let columnNames = Set(columns.map(\.name)) + let columnNames = columns.map(\.name) let primaryKeys = Set(columns.filter(\.isPrimaryKey).map(\.name)) return try rows.compactMap { row in - let unknown = row.keys.filter { !columnNames.contains($0) } - guard unknown.isEmpty else { throw IntentDataError.unknownColumns(unknown.sorted(), table) } + let matched = keysByColumn(row.keys, columns: columnNames) + guard matched.unknown.isEmpty else { throw IntentDataError.unknownColumns(matched.unknown.sorted(), table) } var values: [(column: String, value: String?)] = [] for column in columns { guard !column.isServerOwned else { continue } - guard let value = row.value(for: column.name) else { continue } + guard let key = matched.byColumn[column.name], let value = row.value(for: key) else { continue } if dropsEmptyPrimaryKey, primaryKeys.contains(column.name), value.isEmptyOrNull { continue } values.append((column: column.name, value: value.sqlValue)) } @@ -34,4 +34,46 @@ nonisolated enum PayloadRowPlanner { return values } } + + // As the import sheet matches a header: its exact spelling claims a column first, then a case-insensitive match. + // There is no sheet to map a header by hand, so SQL Server's `[name]` also finds `name`. + static func keysByColumn(_ keys: [String], columns: [String]) -> (byColumn: [String: String], unknown: [String]) { + var byColumn: [String: String] = [:] + var loose: [String] = [] + let spelled = Set(columns) + for key in keys.sorted() { + if spelled.contains(key) { + byColumn[key] = key + } else { + loose.append(key) + } + } + var unknown: [String] = [] + for key in loose { + let claimed = byColumn + guard let column = matchingColumn(for: key, in: columns, unclaimed: { claimed[$0] == nil }) else { + unknown.append(key) + continue + } + byColumn[column] = key + } + return (byColumn, unknown) + } + + private static func matchingColumn(for key: String, in columns: [String], unclaimed: (String) -> Bool) -> String? { + for name in [key, unbracketed(key)].compactMap({ $0 }) { + if columns.contains(name), unclaimed(name) { return name } + let folded = columns.filter { $0.caseInsensitiveCompare(name) == .orderedSame && unclaimed($0) } + if folded.count == 1 { return folded[0] } + } + return nil + } + + // `]` inside the brackets is doubled, so a lone one means the key is not a bracketed name. + private static func unbracketed(_ key: String) -> String? { + guard key.count > 2, key.hasPrefix("["), key.hasSuffix("]") else { return nil } + let inner = String(key.dropFirst().dropLast()) + guard !inner.replacingOccurrences(of: "]]", with: "", options: .literal).contains("]") else { return nil } + return inner.replacingOccurrences(of: "]]", with: "]", options: .literal) + } } diff --git a/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift b/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift index 99fd7d2c56..95736a027e 100644 --- a/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift +++ b/apple/TablePro/Core/Shortcuts/Shared/RowPayload.swift @@ -11,15 +11,29 @@ nonisolated enum RowPayload { let trimmed = raw.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { throw IntentDataError.emptyPayload } - let rows = trimmed.hasPrefix("{") || trimmed.hasPrefix("[") - ? try parseJSON(trimmed) - : try parseCSV(raw) + let rows = try sniffedRows(raw: raw, trimmed: trimmed) guard !rows.isEmpty else { throw IntentDataError.emptyPayload } guard rows.count <= maxRows else { throw IntentDataError.tooManyRows(maxRows) } return rows } + // A CSV header can open with `[` or `{`, as SQL Server's `[id],[name]` does, so only text that parses as JSON is + // JSON. Text that is neither keeps the JSON error, which names what broke in a payload meant as JSON. So does JSON + // that still splits into even CSV records, one object per line with a trailing comma: its headers hold `":`. + private static func sniffedRows(raw: String, trimmed: String) throws -> [PayloadRow] { + guard trimmed.hasPrefix("{") || trimmed.hasPrefix("[") else { return try parseCSV(raw) } + do { + return try parseJSON(trimmed) + } catch IntentDataError.malformedPayload(let reason) { + guard let rows = try? parseCSV(raw), !rows.isEmpty, + !rows.contains(where: { row in row.keys.contains { $0.contains("\":") } }) else { + throw IntentDataError.malformedPayload(reason) + } + return rows + } + } + static func parseSingle(data: String?, file: IntentFile?) async throws -> [PayloadRow] { let rows = try await parse(data: data, file: file) guard rows.count == 1 else { throw IntentDataError.expectedSingleRow } diff --git a/apple/TablePro/Core/Shortcuts/ShortcutsRowWriter.swift b/apple/TablePro/Core/Shortcuts/ShortcutsRowWriter.swift index 1baee5a518..4336e6c51f 100644 --- a/apple/TablePro/Core/Shortcuts/ShortcutsRowWriter.swift +++ b/apple/TablePro/Core/Shortcuts/ShortcutsRowWriter.swift @@ -157,10 +157,11 @@ internal enum ShortcutsRowWriter { let rowID = RowID.inserted(UUID()) let byColumn = Dictionary(values.map { ($0.column, $0.value) }, uniquingKeysWith: { first, _ in first }) insertedRowData[rowID] = names.map { name in - guard let value = byColumn[name] else { return .text(PluginCellValue.defaultMarkerText) } + guard let value = byColumn[name] else { return .columnDefault } return value.map(PluginCellValue.text) ?? .null } - changes.append(RowChange(rowID: rowID, type: .insert, sequence: index)) + let defaulted = Set(names.indices.filter { byColumn[names[$0]] == nil }) + changes.append(RowChange(rowID: rowID, type: .insert, expressionColumns: defaulted, sequence: index)) } let factory = RowChangeStatementFactory( diff --git a/apple/TablePro/Core/UsersRoles/PrincipalChangeManager.swift b/apple/TablePro/Core/UsersRoles/PrincipalChangeManager.swift index 1a7be6130b..578cfc531f 100644 --- a/apple/TablePro/Core/UsersRoles/PrincipalChangeManager.swift +++ b/apple/TablePro/Core/UsersRoles/PrincipalChangeManager.swift @@ -432,14 +432,20 @@ final class PrincipalChangeManager: ObservableObject { // MARK: - Output + // A staged drop keeps the principal's other edits so undoing the drop brings them back, but only the DROP runs. func pendingChanges() -> [PrincipalChange] { var changes: [PrincipalChange] = pendingCreates.map { .create($0) } changes += pendingAlters.compactMap { ref, definition in - guard let original = principals.first(where: { $0.ref == ref }) else { return nil } + guard pendingDrops[ref] == nil, + let original = principals.first(where: { $0.ref == ref }) + else { return nil } return .alter(old: Self.definition(from: original), new: definition) } - changes += pendingPasswords.map { .setPassword(ref: $0.key, password: $0.value) } + changes += pendingPasswords.compactMap { ref, password in + guard pendingDrops[ref] == nil else { return nil } + return .setPassword(ref: ref, password: password) + } changes += grantChangeSets().map { .modifyGrants($0) } changes += pendingDrops.map { .drop(ref: $0.key, options: $0.value) } return changes @@ -531,11 +537,12 @@ final class PrincipalChangeManager: ObservableObject { } private func recomputeChangeCount() { - let grantChanges = grantDeltas.values.reduce(0) { $0 + $1.count } + let isLive = { (ref: PluginPrincipalRef) in self.pendingDrops[ref] == nil } + let grantChanges = grantDeltas.reduce(0) { isLive($1.key) ? $0 + $1.value.count : $0 } changeCount = pendingCreates.count + pendingDrops.count - + pendingPasswords.count - + pendingAlters.count + + pendingPasswords.keys.count(where: isLive) + + pendingAlters.keys.count(where: isLive) + grantChanges } diff --git a/apple/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift b/apple/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift index 5161eb1e38..a805e5be70 100644 --- a/apple/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift +++ b/apple/TablePro/Core/Utilities/JavaScript/QueryStatementModel.swift @@ -189,11 +189,11 @@ enum QueryStatementScanner { } } - private static func located(_ command: RedisCommandSplitter.Command) -> SQLStatementScanner.LocatedStatement { + fileprivate static func located(_ command: RedisCommandSplitter.Command) -> SQLStatementScanner.LocatedStatement { SQLStatementScanner.LocatedStatement(wholeText: command.text, offset: command.range.location) } - private static func located( + fileprivate static func located( _ statement: JavaScriptStatementScanner.Statement ) -> SQLStatementScanner.LocatedStatement { SQLStatementScanner.LocatedStatement( @@ -203,3 +203,51 @@ enum QueryStatementScanner { ) } } + +// A JavaScript or Redis document's statements, read once per text, so a caret move finds its statement by search and +// answers what `QueryStatementScanner.locatedStatementAtCursor` would. A SQL document uses `SQLStatementCursorIndex`. +struct QueryStatementCaretTable { + private let model: QueryStatementModel + private let statements: [SQLStatementScanner.LocatedStatement] + // The last caret offset each statement owns, in document order. + private let reaches: [Int] + + init?(text: String, model: QueryStatementModel) { + self.model = model + switch model { + case .sql: + return nil + case .javascript: + let found = JavaScriptStatementScanner.executableStatements(in: text) + statements = found.map(QueryStatementScanner.located) + reaches = found.map { NSMaxRange($0.range) } + case .redis: + let found = RedisCommandSplitter.commandsWithCaretReach(in: text) + statements = found.map { QueryStatementScanner.located($0.command) } + reaches = found.map { $0.reach } + } + } + + func statement(atCursor cursor: Int) -> SQLStatementScanner.LocatedStatement { + var low = 0 + var high = reaches.count + while low < high { + let middle = (low + high) / 2 + if reaches[middle] < cursor { + low = middle + 1 + } else { + high = middle + } + } + let nothing = SQLStatementScanner.LocatedStatement(sql: "", offset: 0, hasContent: false) + switch model { + case .redis: + return low < statements.count ? statements[low] : nothing + case .javascript, .sql: + // Off every statement, the caret takes the one before it, or the first. + guard !statements.isEmpty else { return nothing } + if low < statements.count, statements[low].offset <= cursor { return statements[low] } + return statements[max(0, low - 1)] + } + } +} diff --git a/apple/TablePro/Core/Utilities/SQL/DisplayedResultReader.swift b/apple/TablePro/Core/Utilities/SQL/DisplayedResultReader.swift index 869817eb65..a9463a1cb4 100644 --- a/apple/TablePro/Core/Utilities/SQL/DisplayedResultReader.swift +++ b/apple/TablePro/Core/Utilities/SQL/DisplayedResultReader.swift @@ -15,12 +15,14 @@ internal enum DisplayedResultReader { internal let skippedDeletedCount: Int } - // Empty selectedDisplayIndices means every displayed row. + // Empty selectedDisplayIndices means every displayed row. A cell in defaultedCells has no value until the server + // gives it one, so it reads as NULL. internal static func read( tableRows: TableRows, displayIDs: [RowID]?, selectedDisplayIndices: Set, deletedRowIDs: Set = [], + defaultedCells: [RowID: Set] = [:], columns projection: VisibleColumnProjection ) -> Output { let positions: [Int] @@ -38,7 +40,7 @@ internal enum DisplayedResultReader { skippedDeleted += 1 return nil } - return projection.values(Self.values(of: row)) + return projection.values(Self.values(of: row, defaulted: defaultedCells[row.id] ?? [])) } return Output( @@ -50,12 +52,15 @@ internal enum DisplayedResultReader { } // A value the page left on the server says so, rather than reading as NULL. - private static func values(of row: Row) -> [PluginCellValue] { - guard row.hasUnloadedValues else { return Array(row.values) } + private static func values(of row: Row, defaulted: Set) -> [PluginCellValue] { + guard row.hasUnloadedValues || !defaulted.isEmpty else { return Array(row.values) } var values = Array(row.values) for (column, byteCount) in row.unloadedColumns where values.indices.contains(column) { values[column] = .text(UnloadedValueText.notLoaded(byteCount: byteCount)) } + for column in defaulted where values.indices.contains(column) { + values[column] = .null + } return values } } diff --git a/apple/TablePro/Core/Utilities/SQL/JsonRowConverter.swift b/apple/TablePro/Core/Utilities/SQL/JsonRowConverter.swift index efc41e9320..5cb6637e19 100644 --- a/apple/TablePro/Core/Utilities/SQL/JsonRowConverter.swift +++ b/apple/TablePro/Core/Utilities/SQL/JsonRowConverter.swift @@ -40,7 +40,7 @@ internal struct JsonRowConverter { } let cell = row[colIdx] - if cell.isNull || cell.isDefaultMarker { + if cell.isNull { result.append("null") appendPropertySuffix(to: &result, colIdx: colIdx) continue diff --git a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift index c326a505df..19d3dc08ff 100644 --- a/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift +++ b/apple/TablePro/Core/Utilities/SQL/QueryClassifier.swift @@ -102,16 +102,21 @@ enum QueryClassifier { isMultiStatement(sql, databaseType: databaseType, readings: databaseType.lexicalReadings) } - /// Whether `sql` runs more than one statement: it holds several, a `GO n` line runs its one statement n times, or it - /// ends inside a block it opened, which an engine that reads the opener as a name runs as several statements. A - /// caller that sends one statement once cannot honour the count, and dropping it runs the statement once. + /// Whether `sql` runs more than one statement: it holds several, a `GO n` line runs its one statement n times, it + /// ends inside a block it opened, which an engine that reads the opener as a name runs as several statements, or a + /// T-SQL block in it holds several. A caller that sends one statement once cannot honour the count, and dropping it + /// runs the statement once. static func isMultiStatement(_ sql: String, databaseType: DatabaseType, readings: SQLLexicalReadings) -> Bool { let model = QueryStatementModel.forDatabaseType(databaseType) guard let lexings = readings.lexings(of: sql) else { return true } return lexings.contains { lexing in let batches = QueryBatchPlanner.batches(in: lexing.text, model: model, grammar: lexing.grammar) - return batches.flatMap(\.statements).count > 1 || batches.contains { $0.repeatCount > 1 } + let statements = batches.flatMap(\.statements) + return statements.count > 1 || batches.contains { $0.repeatCount > 1 } || SQLStatementScanner.endsInsideOpenBlock(lexing.text, grammar: lexing.grammar) + || statements.contains { statement in + SQLUnterminatedStatements.blockHoldsSeveralStatements(statement.sql, grammar: lexing.grammar) + } } } diff --git a/apple/TablePro/Core/Utilities/SQL/ResultJsonSerializer.swift b/apple/TablePro/Core/Utilities/SQL/ResultJsonSerializer.swift index c2f9ead605..1bb897caff 100644 --- a/apple/TablePro/Core/Utilities/SQL/ResultJsonSerializer.swift +++ b/apple/TablePro/Core/Utilities/SQL/ResultJsonSerializer.swift @@ -36,6 +36,7 @@ internal enum ResultJsonSerializer { displayIDs: [RowID]?, selectedDisplayIndices: Set, deletedRowIDs: Set = [], + defaultedCells: [RowID: Set] = [:], columns projection: VisibleColumnProjection ) -> Output { let read = DisplayedResultReader.read( @@ -43,6 +44,7 @@ internal enum ResultJsonSerializer { displayIDs: displayIDs, selectedDisplayIndices: selectedDisplayIndices, deletedRowIDs: deletedRowIDs, + defaultedCells: defaultedCells, columns: projection ) diff --git a/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift b/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift index 4eac9054f4..3c763effc6 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift @@ -228,18 +228,6 @@ internal struct SQLRowToStatementConverter { } private func formatBinaryLiteral(_ data: Data) -> String { - var hex = "" - hex.reserveCapacity(data.count * 2) - for byte in data { - hex += String(format: "%02X", byte) - } - switch databaseType { - case .postgresql, .redshift, .cockroachdb, .pglite, .kingbasees: - return "'\\x\(hex)'::bytea" - case .mssql: - return "0x\(hex)" - default: - return "X'\(hex)'" - } + CompareSQLLiteral.binaryLiteral(for: data, databaseType: databaseType) ?? "X'\(data.uppercaseHex())'" } } diff --git a/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift b/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift index 4d012d8074..9d8bde2d77 100644 --- a/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift +++ b/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift @@ -51,65 +51,6 @@ final class VimTextBufferAdapter: VimTextBuffer { cachedLineCount = nil } - /// Incrementally update the cached line count based on text change delta. - /// Avoids a full O(n) recount on every keystroke. - func textDidChange(in range: NSRange, replacementLength: Int) { - guard let textView else { - cachedLineCount = nil - return - } - - guard let cached = cachedLineCount else { return } - - let nsString = textView.string as NSString - - // Pure insertion: count newlines in the new text and apply delta - if range.length == 0 { - var addedNewlines = 0 - let end = range.location + replacementLength - if replacementLength > 0 && end <= nsString.length { - for i in range.location.. 0 && range.location + range.length <= oldNs.length { - let end = range.location + range.length - for i in range.location.. 0 && replacementEnd <= nsString.length { - for i in range.location.. NSRange { guard let textView else { return NSRange(location: 0, length: 0) } let nsString = textView.string as NSString diff --git a/apple/TablePro/Extensions/PluginCellValue+DefaultMarker.swift b/apple/TablePro/Extensions/PluginCellValue+DefaultMarker.swift index 74b11c7a5b..63f9e24d11 100644 --- a/apple/TablePro/Extensions/PluginCellValue+DefaultMarker.swift +++ b/apple/TablePro/Extensions/PluginCellValue+DefaultMarker.swift @@ -7,16 +7,15 @@ import Foundation import TableProPluginKit internal extension PluginCellValue { - /// The sentinel a pending inserted row carries for a column the server fills in, written by - /// `RowOperationsManager` and stripped again by `SQLStatementGenerator`. - /// - /// It is an internal marker, never a value, so every view that renders a cell has to recognise - /// it. The grid shows it as a placeholder; anything that renders the same rows some other way - /// has to agree, or the marker leaks out as literal text. - static let defaultMarkerText = "__DEFAULT__" + // A cell left to the column's default is the SQL keyword, marked as SQL beside the value like any expression. The + // mark decides, never the text: a stored or typed value can read the same. + static let columnDefaultKeyword = "DEFAULT" + static let columnDefault = PluginCellValue.text(columnDefaultKeyword) - var isDefaultMarker: Bool { - guard case .text(let value) = self else { return false } - return value == Self.defaultMarkerText + // The spelling driver plugins read as the default in the rows the host hands them. + static let pluginDefaultMarker = PluginCellValue.text("__DEFAULT__") + + func isColumnDefault(isExpression: Bool) -> Bool { + isExpression && self == Self.columnDefault } } diff --git a/apple/TablePro/Models/ClickHouse/ClickHousePartInfo.swift b/apple/TablePro/Models/ClickHouse/ClickHousePartInfo.swift index baa347a335..afe2544e04 100644 --- a/apple/TablePro/Models/ClickHouse/ClickHousePartInfo.swift +++ b/apple/TablePro/Models/ClickHouse/ClickHousePartInfo.swift @@ -2,15 +2,13 @@ // ClickHousePartInfo.swift // TablePro // -// Model for ClickHouse partition/part information from system.parts. -// import Foundation -/// Represents a single part from the ClickHouse system.parts table struct ClickHousePartInfo: Identifiable { let id = UUID() let partition: String + let partitionId: String let name: String let rows: UInt64 let bytesOnDisk: UInt64 diff --git a/apple/TablePro/Models/Query/SelectionSummary.swift b/apple/TablePro/Models/Query/SelectionSummary.swift index af5da02cb1..923812aa44 100644 --- a/apple/TablePro/Models/Query/SelectionSummary.swift +++ b/apple/TablePro/Models/Query/SelectionSummary.swift @@ -32,8 +32,8 @@ struct SelectionSummaryInput: Sendable { let dataColumnsByDisplayPosition: [Int] let policy: SelectionSummaryColumnPolicy let deletedRowIDs: Set - let insertedRowIDs: Set - let modifiedCells: [RowID: Set] + // The cells a pending edit set to the column's default, which hold the keyword as text. + let defaultedCells: [RowID: Set] init( selection: GridSelection, @@ -42,8 +42,7 @@ struct SelectionSummaryInput: Sendable { dataColumnsByDisplayPosition: [Int], policy: SelectionSummaryColumnPolicy, deletedRowIDs: Set = [], - insertedRowIDs: Set = [], - modifiedCells: [RowID: Set] = [:] + defaultedCells: [RowID: Set] = [:] ) { self.selection = selection self.tableRows = tableRows @@ -51,8 +50,7 @@ struct SelectionSummaryInput: Sendable { self.dataColumnsByDisplayPosition = dataColumnsByDisplayPosition self.policy = policy self.deletedRowIDs = deletedRowIDs - self.insertedRowIDs = insertedRowIDs - self.modifiedCells = modifiedCells + self.defaultedCells = defaultedCells } } diff --git a/apple/TablePro/Models/Schema/ClickHousePartStatements.swift b/apple/TablePro/Models/Schema/ClickHousePartStatements.swift index 5436cfced7..c11617c1c6 100644 --- a/apple/TablePro/Models/Schema/ClickHousePartStatements.swift +++ b/apple/TablePro/Models/Schema/ClickHousePartStatements.swift @@ -2,15 +2,10 @@ // ClickHousePartStatements.swift // TablePro // -// The statements the Parts tab issues, built from the tab's own container. -// import Foundation -/// ClickHouse carries the database as a request parameter rather than in the session, and the app -/// moves that parameter whenever any tab runs somewhere else. An unqualified name therefore names -/// whichever database the connection was last pinned to, which for `DROP PARTITION` is data loss in -/// a database the user was not looking at. +// ClickHouse takes the database from a request parameter any tab can move, so every statement names its own. internal enum ClickHousePartStatements { internal static func qualifiedName( database: String, @@ -32,42 +27,49 @@ internal enum ClickHousePartStatements { internal static func dropPartition( database: String, table: String, - partition: String, - quote: (String) -> String, - escape: (String) -> String - ) -> String { + partitionId: String, + quote: (String) -> String + ) -> String? { + guard isValidPartitionId(partitionId) else { return nil } let name = qualifiedName(database: database, table: table, quote: quote) - return "ALTER TABLE \(name) DROP PARTITION '\(escape(partition))'" + return "ALTER TABLE \(name) DROP PARTITION ID '\(partitionId)'" } internal static func detachPartition( database: String, table: String, - partition: String, - quote: (String) -> String, - escape: (String) -> String - ) -> String { + partitionId: String, + quote: (String) -> String + ) -> String? { + guard isValidPartitionId(partitionId) else { return nil } let name = qualifiedName(database: database, table: table, quote: quote) - return "ALTER TABLE \(name) DETACH PARTITION '\(escape(partition))'" + return "ALTER TABLE \(name) DETACH PARTITION ID '\(partitionId)'" + } + + // The server's own rule (MergeTreePartInfo::validatePartitionID); it also keeps quotes out of the literal. + internal static func isValidPartitionId(_ id: String) -> Bool { + !id.isEmpty && id.unicodeScalars.allSatisfy { scalar in + switch scalar { + case "0"..."9", "a"..."z", "A"..."Z", "-": true + default: false + } + } } - /// A connection saved with no database of its own has none to name, and ClickHouse then resolves - /// an unqualified request against its own configured default, which `currentDatabase()` reports. - /// Comparing against an empty string instead matches nothing and empties the tab. + // With no database of its own, ClickHouse resolves a request against its configured default. private static func databasePredicate(_ database: String, escape: (String) -> String) -> String { guard !database.isEmpty else { return "currentDatabase()" } return "'\(escape(database))'" } - /// Filtered on the named database rather than `currentDatabase()`, which answers with the - /// request parameter and so listed another database's parts under this table's name. + // `partition` renders a tuple key as `(202401,'eu')` and no key as `tuple()`, so statements name `partition_id`. internal static func parts( database: String, table: String, escape: (String) -> String ) -> String { """ - SELECT partition, name, rows, bytes_on_disk, + SELECT partition, partition_id, name, rows, bytes_on_disk, toString(modification_time) AS mod_time, active FROM system.parts WHERE database = \(databasePredicate(database, escape: escape)) AND table = '\(escape(table))' diff --git a/apple/TablePro/Models/UI/MultiRowEditState.swift b/apple/TablePro/Models/UI/MultiRowEditState.swift index 26c98ce2f2..a9c0f4ae65 100644 --- a/apple/TablePro/Models/UI/MultiRowEditState.swift +++ b/apple/TablePro/Models/UI/MultiRowEditState.swift @@ -70,7 +70,7 @@ struct FieldEditState: Identifiable { var effectiveValue: String? { if isPendingDefault { - return "__DEFAULT__" + return PluginCellValue.columnDefaultKeyword } else if isPendingNull { return nil } else { @@ -244,6 +244,8 @@ final class MultiRowEditState: ObservableObject { hasCommittedEdit = true } else if originalValue == nil { isPendingNull = true + } else if expression.first == true, originalValue == PluginCellValue.columnDefaultKeyword { + isPendingDefault = true } else { pendingValue = originalValue isPendingExpression = expression.first == true @@ -508,7 +510,7 @@ final class MultiRowEditState: ObservableObject { guard index < fields.count else { return } resetPending(at: index) fields[index].isPendingDefault = true - onFieldChanged?(index, .text("__DEFAULT__"), .discrete) + onFieldExpressionChanged?(index, PluginCellValue.columnDefaultKeyword, .discrete) } func setFieldToExpression(at index: Int, expression: CellExpression) { @@ -619,7 +621,7 @@ final class MultiRowEditState: ObservableObject { columnName: field.columnName, newValue: field.effectiveValue, removesField: field.isPendingRemoval, - isExpression: field.isPendingExpression + isExpression: field.isPendingExpression || field.isPendingDefault ) } } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 4ba0fefbde..245084252f 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -55039,6 +55039,9 @@ } } } + }, + "Couldn't Detach Partition" : { + }, "Couldn't Discard Changes" : { "localizations" : { @@ -55052,6 +55055,9 @@ }, "Couldn't download cloud-sql-proxy. The server answered with HTTP status %lld." : { + }, + "Couldn't Drop Partition" : { + }, "Couldn't Duplicate the Theme" : { @@ -55119,6 +55125,9 @@ }, "Couldn't Move the Theme to the Trash" : { + }, + "Couldn't Optimize Table" : { + }, "Couldn't publish to the team catalog" : { "localizations" : { @@ -55254,6 +55263,9 @@ }, "Couldn't Read the DDL" : { + }, + "Couldn't Read the Parts" : { + }, "Couldn't read the referenced table" : { "localizations" : { @@ -120449,6 +120461,10 @@ } } }, + "No row has this primary key, so nothing was updated. Another client may have deleted it." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "No row holds the values it was loaded with. Another client may have changed or deleted it." : { }, @@ -145185,6 +145201,12 @@ "Reconnecting (%lld of %lld)" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Reconnecting could not reopen the database %1$@: %2$@" : { + + }, + "Reconnecting could not reopen the schema %1$@: %2$@" : { + }, "Reconnecting failed after %d attempts." : { "localizations" : { @@ -174217,6 +174239,9 @@ } } } + }, + "Startup commands failed, so this session runs without them:\n%@" : { + }, "Startup SQL" : { "comment" : "A section header for the startup SQL commands of a connection.", @@ -193166,6 +193191,9 @@ } } } + }, + "The server reported a partition ID that cannot be used in a statement, so nothing was run." : { + }, "The server reported an error." : { "localizations" : { @@ -194368,6 +194396,12 @@ } } } + }, + "The startup command %@ failed, so this session runs without it." : { + + }, + "The startup command %1$@ failed (%2$@), so this session runs without it." : { + }, "The statement before it stays applied." : { "localizations" : { @@ -198452,6 +198486,9 @@ }, "This database's driver cannot write SQL into %@. Update the driver." : { + }, + "This database's driver may read the text __DEFAULT__ in %@ as the column default, so it cannot be saved as text." : { + }, "This deletes only the queries listed here, across every connection. Queries the source, date, outcome and search filters are hiding stay. You cannot undo this." : { "comment" : "Message for the \"Delete this connection's query history\" button.", @@ -213159,6 +213196,10 @@ } } }, + "Values a spreadsheet would read as formulas, written with a leading single quote: %lld. Turn off Sanitize formula-like values to export them unchanged." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Values from %d loaded rows" : { "extractionState" : "stale", "localizations" : { diff --git a/apple/TablePro/ViewModels/SchemaEditorViewModel.swift b/apple/TablePro/ViewModels/SchemaEditorViewModel.swift index 3d86e3b1fe..5c20599902 100644 --- a/apple/TablePro/ViewModels/SchemaEditorViewModel.swift +++ b/apple/TablePro/ViewModels/SchemaEditorViewModel.swift @@ -330,7 +330,7 @@ final class SchemaEditorViewModel: ObservableObject { renameCommitted: plan.renamesFirst && committedFirst ) } catch { - failure = error.localizedDescription + failure = GatedOperationFailure(error).message return ApplyOutcome( succeeded: false, committedName: originalName ?? "", renameCommitted: false ) diff --git a/apple/TablePro/ViewModels/UserTypeEditorModel.swift b/apple/TablePro/ViewModels/UserTypeEditorModel.swift index 3d6bc764f8..ae428d77f5 100644 --- a/apple/TablePro/ViewModels/UserTypeEditorModel.swift +++ b/apple/TablePro/ViewModels/UserTypeEditorModel.swift @@ -302,7 +302,7 @@ final class UserTypeEditorModel: ObservableObject { do { outcome = try await UserTypeChangeRunner.run(plan, in: scope, connection: connection, gate: gate) } catch { - let failure = StructureApplyFailure(error) + let failure = GatedOperationFailure(error) if let message = failure.message { AlertHelper.showErrorSheet( title: String(localized: "Error Applying Changes"), diff --git a/apple/TablePro/ViewModels/UsersRolesViewModel+Changes.swift b/apple/TablePro/ViewModels/UsersRolesViewModel+Changes.swift index d54a6fc8bc..162da203eb 100644 --- a/apple/TablePro/ViewModels/UsersRolesViewModel+Changes.swift +++ b/apple/TablePro/ViewModels/UsersRolesViewModel+Changes.swift @@ -290,7 +290,7 @@ extension UsersRolesViewModel { .joined(separator: "\n") await load(forceReload: true) } catch { - applyFailure = error.localizedDescription + applyFailure = GatedOperationFailure(error).message } } diff --git a/apple/TablePro/Views/Compare/CompareRowDiffPane.swift b/apple/TablePro/Views/Compare/CompareRowDiffPane.swift index 542c2d52e9..f80cb6f1c3 100644 --- a/apple/TablePro/Views/Compare/CompareRowDiffPane.swift +++ b/apple/TablePro/Views/Compare/CompareRowDiffPane.swift @@ -208,12 +208,27 @@ internal struct CompareRowDiffPane: View { ) } + // Named by side, because the two tables usually share a name and only one database cut its stream. + private func rowCutNotices(for summary: DataDiffSummary, plan: DataComparePlan) -> [String] { + let sides: [(endpoint: DatabaseEndpoint?, cut: Int?)] = [ + (session.source, summary.sourceRowCut), (session.target, summary.targetRowCut) + ] + return sides.compactMap { side in + guard let cut = side.cut else { return nil } + let table = ([side.endpoint?.qualifiedDescription].compactMap { $0 } + [plan.table]).joined(separator: " / ") + return StreamRowCutWarning.text(table: table, limit: cut) + } + } + @ViewBuilder private func notices(for plan: DataComparePlan) -> some View { if let failure = plan.comparisonFailure { notice(failure, systemImage: "exclamationmark.octagon.fill") } if let summary = plan.summary { + ForEach(rowCutNotices(for: summary, plan: plan), id: \.self) { text in + notice(text, systemImage: "exclamationmark.triangle.fill") + } if summary.stoppedAtRowLimit { notice(rowLimitNotice(for: summary), systemImage: "info.circle.fill") } diff --git a/apple/TablePro/Views/DataGrid/Cells/DataGridCellContent.swift b/apple/TablePro/Views/DataGrid/Cells/DataGridCellContent.swift index b32be5c107..1f918e38b5 100644 --- a/apple/TablePro/Views/DataGrid/Cells/DataGridCellContent.swift +++ b/apple/TablePro/Views/DataGrid/Cells/DataGridCellContent.swift @@ -23,16 +23,17 @@ struct DataGridCellContent { static func placeholder( for rawValue: PluginCellValue, isAbsent: Bool = false, + isExpression: Bool = false, unloadedByteCount: Int? = nil, marksEmptyText: Bool = true ) -> DataGridCellPlaceholder? { if let unloadedByteCount { return .unloaded(byteCount: unloadedByteCount) } guard !isAbsent else { return .absent } + if rawValue.isColumnDefault(isExpression: isExpression) { return .defaultMarker } switch rawValue { case .null: return .null case .text(let value): - if value == PluginCellValue.defaultMarkerText { return .defaultMarker } return value.isEmpty && marksEmptyText ? .empty : nil case .bytes: return nil diff --git a/apple/TablePro/Views/DataGrid/Cells/DataGridCellPresenter.swift b/apple/TablePro/Views/DataGrid/Cells/DataGridCellPresenter.swift index 8f7745aac0..651da4002b 100644 --- a/apple/TablePro/Views/DataGrid/Cells/DataGridCellPresenter.swift +++ b/apple/TablePro/Views/DataGrid/Cells/DataGridCellPresenter.swift @@ -250,6 +250,7 @@ final class DataGridCellPresenter { placeholder: DataGridCellContent.placeholder( for: rawValue, isAbsent: row.isAbsent(dataIndex), + isExpression: visualState.isExpression(columnIndex: dataIndex), unloadedByteCount: row.unloadedByteCount(dataIndex), marksEmptyText: pass.marksEmptyText ) @@ -297,6 +298,7 @@ final class DataGridCellPresenter { switch DataGridCellContent.placeholder( for: rawValue, isAbsent: row.isAbsent(dataIndex), + isExpression: model.changeState?.isExpression(rowID: row.id, columnIndex: dataIndex) ?? false, unloadedByteCount: row.unloadedByteCount(dataIndex), marksEmptyText: model.configuration.marksEmptyText ) { diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift index e27584da1e..fd99df077b 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift @@ -133,6 +133,7 @@ extension DataGridCommands { tableRows: model.rows, displayIDs: model.order.filteredIDs, selectedDisplayIndices: displayIndices, + defaultedCells: model.changeState?.defaultedCells(in: model.rows) ?? [:], columns: projection ) guard output.rowCount > 0 else { return } diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+SetValue.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+SetValue.swift index cc2d571948..cdfa6e7fa8 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+SetValue.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+SetValue.swift @@ -78,7 +78,7 @@ extension DataGridCommands { private static func sampleText(ofColumn dataIndex: Int, in rows: TableRows) -> String? { for row in rows.rows where dataIndex < row.values.count { guard let text = row.values[dataIndex].asText, - !text.isEmpty, text != PluginCellValue.defaultMarkerText else { continue } + !text.isEmpty, row.values[dataIndex] != .columnDefault else { continue } return text } return nil @@ -91,7 +91,7 @@ extension DataGridCommands { case .null: return (value: .null, isExpression: false) case .defaultValue: - return (value: .text(PluginCellValue.defaultMarkerText), isExpression: false) + return (value: .columnDefault, isExpression: true) case .expression(let expression): return (value: .text(expression.sql), isExpression: true) case .uuid(let version): diff --git a/apple/TablePro/Views/DataGrid/DataGridViewController.swift b/apple/TablePro/Views/DataGrid/DataGridViewController.swift index dd7176f7af..a3b65c5666 100644 --- a/apple/TablePro/Views/DataGrid/DataGridViewController.swift +++ b/apple/TablePro/Views/DataGrid/DataGridViewController.swift @@ -388,8 +388,7 @@ final class DataGridViewController: NSViewController, DataGridMountedView { dataColumnsByDisplayPosition: model.columns.columns.map(\.dataIndex), policy: policy, deletedRowIDs: changeState?.deletedRowIDs ?? [], - insertedRowIDs: changeState?.insertedRowIDs ?? [], - modifiedCells: changeState?.modifiedCells ?? [:] + defaultedCells: changeState?.defaultedCells(in: rows) ?? [:] ) } } diff --git a/apple/TablePro/Views/DatabaseSwitcher/CreateDatabaseSheet.swift b/apple/TablePro/Views/DatabaseSwitcher/CreateDatabaseSheet.swift index 7cdc8a564d..1b6314b1ba 100644 --- a/apple/TablePro/Views/DatabaseSwitcher/CreateDatabaseSheet.swift +++ b/apple/TablePro/Views/DatabaseSwitcher/CreateDatabaseSheet.swift @@ -176,7 +176,7 @@ struct CreateDatabaseSheet: View { onCreated?(name) dismiss() } catch { - errorMessage = error.localizedDescription + errorMessage = GatedOperationFailure(error).message isCreating = false } } diff --git a/apple/TablePro/Views/Editor/StatementRunController.swift b/apple/TablePro/Views/Editor/StatementRunController.swift index 7909dcd81a..b7b0acef3c 100644 --- a/apple/TablePro/Views/Editor/StatementRunController.swift +++ b/apple/TablePro/Views/Editor/StatementRunController.swift @@ -17,7 +17,8 @@ enum StatementNavigationDirection { // The gutter's run controls and the caret's statement band both come from the scanner execution uses, so a control // never offers to run something other than what it is drawn beside. The band runs on every caret move, so a SQL -// document resolves it through `SQLStatementCursorIndex`, which rescans only from the last boundary before the caret. +// document resolves it through `SQLStatementCursorIndex`, which rescans only from the last boundary before the caret, +// and a JavaScript or Redis document through a `QueryStatementCaretTable` read once per text. // The controls need every statement, a full pass of 31ms at the size limit, so that refresh is debounced. @MainActor final class StatementRunController { @@ -44,11 +45,17 @@ final class StatementRunController { /// How the document divides into statements. JavaScript engines do not end a statement at every /// semicolon, so the gutter controls and the caret band have to ask the same splitter execution does. var statementModel: QueryStatementModel = .sql { - didSet { if statementModel != oldValue { caretIndex = nil } } + didSet { + if statementModel != oldValue { + caretIndex = nil + caretTable = nil + } + } } - // The band asks on every caret move, so a SQL document keeps its statement boundaries between edits. + // The band asks on every caret move, so a document keeps its statement boundaries between edits. private var caretIndex: SQLStatementCursorIndex? + private var caretTable: QueryStatementCaretTable? private weak var caretIndexStorage: NSTextStorage? private var caretIndexEdits: StorageEditObservation? @@ -56,6 +63,8 @@ final class StatementRunController { caretIndex?.unitsScanned ?? 0 } + private(set) var caretTableReads = 0 + /// Called with the SQL the pressed control stands for, read from the document at the moment of the press, and /// where that SQL starts in the document. /// @@ -213,6 +222,8 @@ final class StatementRunController { pendingRefresh?.cancel() pendingRefresh = nil caretIndex = nil + caretTable = nil + caretIndexStorage = nil caretIndexEdits = nil controller?.runnableStatements = [] controller?.highlightedStatementRange = nil @@ -267,7 +278,7 @@ final class StatementRunController { guard (text as NSString).length <= sizeLimit else { return nil } let statement: SQLStatementScanner.LocatedStatement - if statementModel == .sql, let storage = textView.textStorage { + if let storage = textView.textStorage { statement = indexedStatement(at: selection.range.location, in: text, storage: storage) } else { statement = QueryStatementScanner.locatedStatementAtCursor( @@ -286,19 +297,38 @@ final class StatementRunController { in text: String, storage: NSTextStorage ) -> SQLStatementScanner.LocatedStatement { - if caretIndex == nil || caretIndexStorage !== storage { - startCaretIndex(on: storage) + if caretIndexStorage !== storage { + observeEdits(on: storage) } // Inside an editing batch the text has changed but the storage has not posted its notification yet. if storage.editedMask.contains(.editedCharacters) { - caretIndex?.textDidChange(at: storage.editedRange.location) + textDidChange(at: storage.editedRange.location) + } + guard statementModel == .sql else { + if caretTable == nil { + caretTable = QueryStatementCaretTable(text: text, model: statementModel) + caretTableReads += 1 + } + return caretTable?.statement(atCursor: cursor) + ?? QueryStatementScanner.locatedStatementAtCursor( + in: text, cursorPosition: cursor, model: statementModel, grammar: grammar + ) + } + if caretIndex == nil { + caretIndex = SQLStatementCursorIndex(grammar: grammar) } return caretIndex?.statementAtCursor(cursor, in: text) ?? SQLStatementScanner.locatedStatementAtCursor(in: text, cursorPosition: cursor, grammar: grammar) } - private func startCaretIndex(on storage: NSTextStorage) { - caretIndex = SQLStatementCursorIndex(grammar: grammar) + private func textDidChange(at location: Int) { + caretIndex?.textDidChange(at: location) + caretTable = nil + } + + private func observeEdits(on storage: NSTextStorage) { + caretIndex = nil + caretTable = nil caretIndexStorage = storage // Every way the text changes passes through the storage, including the ones that skip the editor's delegate. let token = NotificationCenter.default.addObserver( @@ -313,7 +343,7 @@ final class StatementRunController { private func caretIndexStorageDidProcessEditing() { guard let storage = caretIndexStorage, storage.editedMask.contains(.editedCharacters) else { return } - caretIndex?.textDidChange(at: storage.editedRange.location) + textDidChange(at: storage.editedRange.location) } } diff --git a/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift b/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift index f9f0e6a53f..7e4e542b48 100644 --- a/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift +++ b/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift @@ -155,14 +155,10 @@ final class DataTabSQLExporter: DataGridSQLExporting { ) } - // Every marker in a row not saved yet, and in a saved row the cells a pending edit set to Default. + // The cells a pending insert or edit marked as the default. private func stagedDefaults(of row: Row, target: Target) -> Set { - let marker = PluginCellValue.text("__DEFAULT__") - if target.changeManager.insertedRowIDs.contains(row.id) { - return Set(row.values.indices.filter { row.values[$0] == marker }) - } - let pendingUpdate = target.changeManager.pendingUpdate(forRow: row.id) - return Set(pendingUpdate?.cellChanges.filter { $0.newValue == marker }.map(\.columnIndex) ?? []) + let marked = target.changeManager.expressionColumns(forRow: row.id) + return Set(marked.filter { row.values.indices.contains($0) && row.values[$0] == .columnDefault }) } private func nonEmpty(_ statements: String) -> String? { diff --git a/apple/TablePro/Views/Main/Child/MainEditorContentView.swift b/apple/TablePro/Views/Main/Child/MainEditorContentView.swift index 81b48e98af..a246fc2a70 100644 --- a/apple/TablePro/Views/Main/Child/MainEditorContentView.swift +++ b/apple/TablePro/Views/Main/Child/MainEditorContentView.swift @@ -734,6 +734,7 @@ struct MainEditorContentView: View { selectedRowIDs: Set(model.selectedRowIDs), displayIDs: model.displayedRowIDs, deletedRowIDs: changeManager.deletedRowIDs, + defaultedCells: changeManager.defaultedCells(in: rows), valueFilter: tab.valueFilter, dataRevision: coordinator.tabSessionRegistry.session(for: tab.id)?.dataRevision ?? 0, displayRevision: coordinator.gridDisplayRevision, diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift index d7eab33615..ecd1426d4c 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+Navigation.swift @@ -677,10 +677,13 @@ extension MainContentCoordinator { try await dropContainer(target) services.catalogChangeService.record(.containerDropped(target, connectionId: connectionId)) } catch { - navigationLogger.error( - "Failed to drop \(target.id, privacy: .public): \(error.publicLogShape, privacy: .public)" - ) - failures.append((target.name, error.localizedDescription)) + let failure = GatedOperationFailure(error) + if failure.reportsFailure { + navigationLogger.error( + "Failed to drop \(target.id, privacy: .public): \(error.publicLogShape, privacy: .public)" + ) + } + if let message = failure.message { failures.append((target.name, message)) } } } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SafeModePromptRetry.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SafeModePromptRetry.swift new file mode 100644 index 0000000000..92846198f7 --- /dev/null +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+SafeModePromptRetry.swift @@ -0,0 +1,28 @@ +// +// MainContentCoordinator+SafeModePromptRetry.swift +// TablePro +// + +import Foundation + +extension MainContentCoordinator { + // The prompt is one per window, so a first load that meets another run's prompt waits for it to close. + func turnAwayTableLoad(tabId: UUID, trigger: TableLoadTrigger) { + traceNavigationAbandoned(tabId: tabId, outcome: .safeModePromptAlreadyOpen) + tableLoadTurnedAwayByPrompt = ParkedTableLoad(tabId: tabId, trigger: trigger) + declineTableLoad(for: tabId) + } + + func safeModePromptDidChange(wasShowing: Bool) { + guard wasShowing, !isShowingSafeModePrompt, tableLoadTurnedAwayByPrompt != nil else { return } + // Once the run that lowered it has started, so a load it starts itself is not asked for twice. + Task { @MainActor [weak self] in self?.retryTableLoadTurnedAwayByPrompt() } + } + + private func retryTableLoadTurnedAwayByPrompt() { + guard !isShowingSafeModePrompt, !isTearingDown, let turnedAway = tableLoadTurnedAwayByPrompt else { return } + tableLoadTurnedAwayByPrompt = nil + guard turnedAway.tabId == tabManager.selectedTabId else { return } + lazyLoadCurrentTabIfNeeded(trigger: turnedAway.trigger) + } +} diff --git a/apple/TablePro/Views/Main/MainContentCommandActions.swift b/apple/TablePro/Views/Main/MainContentCommandActions.swift index ec02fa09d7..0decc6483f 100644 --- a/apple/TablePro/Views/Main/MainContentCommandActions.swift +++ b/apple/TablePro/Views/Main/MainContentCommandActions.swift @@ -66,6 +66,9 @@ final class MainContentCommandActions: ObservableObject { await RunningWorkApproval.confirm(prompt, window: window) } + // What a window close waits on while another connection's pane mounts. + var paneWaitClock: any Clock = ContinuousClock() + private var notificationTasks: [Task] = [] private var eventCancellables: Set = [] @@ -556,12 +559,7 @@ final class MainContentCommandActions: ObservableObject { // Every connection in the window: closing it closes them all, and skipping a background one is silent data loss. internal var hasUnsavedWorkInWindow: Bool { - guard let host = window?.contentViewController as? MainSplitViewController else { - return coordinator?.hasAnyUnsavedWork() ?? false - } - return host.workspaces.workspaces.contains { workspace in - workspace.sessionState?.coordinator.hasAnyUnsavedWork() == true - } + coordinatorsInWindow.contains { $0.hasAnyUnsavedWork() } } // This connection only: closing its tabs leaves another connection's pending work alone. @@ -650,13 +648,16 @@ final class MainContentCommandActions: ObservableObject { return .closed } // Work is stopped only once the close is certain: a cancelled or failed save keeps it running. - let result = await AlertHelper.confirmSaveChanges( - message: RunningWorkPrompt.saveMessage( - String(localized: "Your changes will be lost if you don't save them."), + let keepsTabsOpen = coordinatorsInWindow.contains { hosted in + hosted.tabManager.tabs.contains { hosted.savability(of: $0) == .mountedOnly } + } + let result = await confirmSaveChanges( + RunningWorkPrompt.saveMessage( + Self.unsavedWorkMessage(keepsTabsOpen: keepsTabsOpen), action: .close, alsoStopping: work ), - window: closeAnchorWindow + closeAnchorWindow ) switch result { @@ -775,13 +776,17 @@ final class MainContentCommandActions: ObservableObject { return true } + // The prompt asked about every connection in the window, so Save reaches every tab of each; one it cannot save keeps the window. private func saveAndClose(asBatchSurvivor: Bool?) async -> Bool { - guard let coordinator else { - finish(asBatchSurvivor: asBatchSurvivor) - return true + let host = window?.contentViewController as? MainSplitViewController + let shown = host?.workspaces.selectedConnectionId + for hosted in coordinatorsInWindow where hosted === coordinator || hosted.hasAnyUnsavedWork() { + // A save can raise a sheet, which has to sit over the connection it is about. + host?.workspaces.select(hosted.connectionId) + guard let actions = await shownActions(of: hosted) else { return false } + guard await actions.saveEveryTabsWork() else { return false } } - guard await applyStagedStructureEdits(in: coordinator.tabManager.tabs) else { return false } - guard await saveSelectedTabWork() else { return false } + if let shown { host?.workspaces.select(shown) } finish(asBatchSurvivor: asBatchSurvivor) return true } @@ -1407,6 +1412,38 @@ final class MainContentCommandActions: ObservableObject { } extension MainContentCommandActions { + // This connection's first. + private var coordinatorsInWindow: [MainContentCoordinator] { + guard let host = window?.contentViewController as? MainSplitViewController else { + return [coordinator].compactMap { $0 } + } + let hosted = host.workspaces.workspaces.compactMap { $0.sessionState?.coordinator } + guard let coordinator else { return hosted } + return [coordinator] + hosted.filter { $0 !== coordinator } + } + + // A connection never shown has no actions until its pane mounts, a few run loop turns after it is selected. + private func shownActions(of hosted: MainContentCoordinator) async -> MainContentCommandActions? { + if hosted === coordinator { return self } + return await Self.actions(of: hosted, waitingOn: paneWaitClock) + } + + // A cancelled close stops waiting at once rather than polling out the rest of the deadline. + private static func actions>( + of hosted: MainContentCoordinator, + waitingOn clock: C + ) async -> MainContentCommandActions? { + let deadline = clock.now.advanced(by: .seconds(2)) + while hosted.commandActions == nil, clock.now < deadline { + do { + try await clock.sleep(for: .milliseconds(20)) + } catch { + return nil + } + } + return hosted.commandActions + } + // Copy with the grid not focused takes the selected rows in the row copy format; a focused grid answers `copy:` itself. func copySelectedRowsInCopyFormat() { guard dataGridOwnsSelection, let model = coordinator?.activeGridModel else { diff --git a/apple/TablePro/Views/Main/MainContentCoordinator.swift b/apple/TablePro/Views/Main/MainContentCoordinator.swift index b338583e42..2c2c39815a 100644 --- a/apple/TablePro/Views/Main/MainContentCoordinator.swift +++ b/apple/TablePro/Views/Main/MainContentCoordinator.swift @@ -371,7 +371,12 @@ final class MainContentCoordinator: ObservableObject { // Guards re-entrant confirm dialogs (nested run loop during runModal). internal var isShowingConfirmAlert = false - internal var isShowingSafeModePrompt = false + // Lowered by whichever run raised it, so a table load it turned away is retried whatever path asked. + internal var isShowingSafeModePrompt = false { + didSet { safeModePromptDidChange(wasShowing: oldValue) } + } + + var tableLoadTurnedAwayByPrompt: ParkedTableLoad? @Published internal var rewindPlan: RewindPlan? @@ -1110,7 +1115,7 @@ final class MainContentCoordinator: ObservableObject { tab.execution.lastExecutedAt == nil { guard !isShowingSafeModePrompt else { - traceNavigationAbandoned(tabId: tab.id, outcome: .safeModePromptAlreadyOpen) + turnAwayTableLoad(tabId: tab.id, trigger: trigger) return nil } isShowingSafeModePrompt = true diff --git a/apple/TablePro/Views/ObjectCopy/CopyObjectsResultView.swift b/apple/TablePro/Views/ObjectCopy/CopyObjectsResultView.swift index 3bd997547b..38c7cf6698 100644 --- a/apple/TablePro/Views/ObjectCopy/CopyObjectsResultView.swift +++ b/apple/TablePro/Views/ObjectCopy/CopyObjectsResultView.swift @@ -41,6 +41,11 @@ internal struct CopyObjectsResultView: View { .font(.callout) .foregroundStyle(.secondary) } + ForEach(Array(result.warnings.enumerated()), id: \.offset) { _, warning in + NoticeLabel(warning, severity: .warning) + .font(.callout) + .textSelection(.enabled) + } outcomes(result) } .frame(maxWidth: .infinity, alignment: .leading) diff --git a/apple/TablePro/Views/ObjectSource/UserType/CreateUserTypeSheet.swift b/apple/TablePro/Views/ObjectSource/UserType/CreateUserTypeSheet.swift index 31526e51b4..8c0baa020c 100644 --- a/apple/TablePro/Views/ObjectSource/UserType/CreateUserTypeSheet.swift +++ b/apple/TablePro/Views/ObjectSource/UserType/CreateUserTypeSheet.swift @@ -245,11 +245,12 @@ struct CreateUserTypeSheet: View { try await UserTypeCreation.create(definition, in: scope, connection: connection) dismiss() onCreated(definition) - } catch ExecutionGateError.cancelledByUser { - return } catch { - Self.logger.error("Create Type failed: \(error.publicLogShape, privacy: .public)") - failure = error.localizedDescription + let outcome = GatedOperationFailure(error) + if outcome.reportsFailure { + Self.logger.error("Create Type failed: \(error.publicLogShape, privacy: .public)") + } + failure = outcome.message } } } diff --git a/apple/TablePro/Views/Results/ResultsJsonView.swift b/apple/TablePro/Views/Results/ResultsJsonView.swift index d8cc4d3395..6a04df3448 100644 --- a/apple/TablePro/Views/Results/ResultsJsonView.swift +++ b/apple/TablePro/Views/Results/ResultsJsonView.swift @@ -11,6 +11,7 @@ internal struct ResultsJsonView: View { let selectedRowIDs: Set let displayIDs: [RowID]? let deletedRowIDs: Set + let defaultedCells: [RowID: Set] let valueFilter: GridValueFilterState let dataRevision: Int let displayRevision: Int @@ -32,6 +33,7 @@ internal struct ResultsJsonView: View { selectedRowIDs: Set, displayIDs: [RowID]?, deletedRowIDs: Set, + defaultedCells: [RowID: Set] = [:], valueFilter: GridValueFilterState, dataRevision: Int, displayRevision: Int, @@ -41,6 +43,7 @@ internal struct ResultsJsonView: View { self.selectedRowIDs = selectedRowIDs self.displayIDs = displayIDs self.deletedRowIDs = deletedRowIDs + self.defaultedCells = defaultedCells self.valueFilter = valueFilter self.dataRevision = dataRevision self.displayRevision = displayRevision @@ -58,6 +61,7 @@ internal struct ResultsJsonView: View { let displayRevision: Int let selectedRowIDs: Set let deletedRowIDs: Set + let defaultedCells: [RowID: Set] let valueFilter: GridValueFilterState let hiddenColumns: Set let columnOrder: [String]? @@ -69,6 +73,7 @@ internal struct ResultsJsonView: View { displayRevision: displayRevision, selectedRowIDs: selectedRowIDs, deletedRowIDs: deletedRowIDs, + defaultedCells: defaultedCells, valueFilter: valueFilter, hiddenColumns: columnLayout.hiddenColumns, columnOrder: columnLayout.columnOrder @@ -202,6 +207,7 @@ internal struct ResultsJsonView: View { let ids = displayIDs let selectedIndices = Set(DataGridRowOrder(filteredIDs: ids, revision: 0).displayIndexes(of: selectedRowIDs, in: snapshot)) let deletedIDs = deletedRowIDs + let defaulted = defaultedCells let layout = columnLayout let result = await Task.detached(priority: .userInitiated) { @@ -210,6 +216,7 @@ internal struct ResultsJsonView: View { displayIDs: ids, selectedIndices: selectedIndices, deletedRowIDs: deletedIDs, + defaultedCells: defaulted, columnLayout: layout ) }.value @@ -245,6 +252,7 @@ internal struct ResultsJsonView: View { displayIDs: [RowID]?, selectedIndices: Set, deletedRowIDs: Set = [], + defaultedCells: [RowID: Set] = [:], columnLayout: ColumnLayoutState ) -> RenderedJson { let output = ResultJsonSerializer.serialize( @@ -252,6 +260,7 @@ internal struct ResultsJsonView: View { displayIDs: displayIDs, selectedDisplayIndices: selectedIndices, deletedRowIDs: deletedRowIDs, + defaultedCells: defaultedCells, columns: .fromColumnLayout(columnLayout, columns: tableRows.columns) ) return RenderedJson( diff --git a/apple/TablePro/Views/Sidebar/EditSequenceSheet.swift b/apple/TablePro/Views/Sidebar/EditSequenceSheet.swift index 11ae7629f4..c72f6f12ea 100644 --- a/apple/TablePro/Views/Sidebar/EditSequenceSheet.swift +++ b/apple/TablePro/Views/Sidebar/EditSequenceSheet.swift @@ -188,7 +188,7 @@ struct EditSequenceSheet: View { try await SequenceEditing.alter(details, to: settings, target: target, connection: connection) dismiss() } catch { - saveError = error.localizedDescription + saveError = GatedOperationFailure(error).message phase = .editing(details) } } diff --git a/apple/TablePro/Views/Sidebar/ObjectCommentSheet.swift b/apple/TablePro/Views/Sidebar/ObjectCommentSheet.swift index bdc4397a00..b73407bedf 100644 --- a/apple/TablePro/Views/Sidebar/ObjectCommentSheet.swift +++ b/apple/TablePro/Views/Sidebar/ObjectCommentSheet.swift @@ -144,10 +144,8 @@ struct ObjectCommentSheet: View { do { try await ObjectCommentEditing.setComment(draft.commentToSave, on: target, connection: connection) dismiss() - } catch ExecutionGateError.cancelledByUser { - phase = .editing } catch { - saveError = error.localizedDescription + saveError = GatedOperationFailure(error).message phase = .editing } } diff --git a/apple/TablePro/Views/Sidebar/SetSequenceValueSheet.swift b/apple/TablePro/Views/Sidebar/SetSequenceValueSheet.swift index e63e765d73..d970311881 100644 --- a/apple/TablePro/Views/Sidebar/SetSequenceValueSheet.swift +++ b/apple/TablePro/Views/Sidebar/SetSequenceValueSheet.swift @@ -176,7 +176,7 @@ struct SetSequenceValueSheet: View { try await SequenceEditing.setNextValue(nextValue, of: details, target: target, connection: connection) dismiss() } catch { - saveError = error.localizedDescription + saveError = GatedOperationFailure(error).message phase = .editing(details) } } diff --git a/apple/TablePro/Views/Structure/ClickHousePartsView.swift b/apple/TablePro/Views/Structure/ClickHousePartsView.swift index 6f37691102..f95d50739e 100644 --- a/apple/TablePro/Views/Structure/ClickHousePartsView.swift +++ b/apple/TablePro/Views/Structure/ClickHousePartsView.swift @@ -8,6 +8,56 @@ import SwiftUI import TableProPluginKit import TableProSQLGrammar +internal struct ClickHousePartsAlert: Equatable { + let title: String + let message: String +} + +// A failed or refused action never replaces the list it acted on; only a load with nothing shown does. +internal struct ClickHousePartsPaneState { + internal enum Display: Equatable { + case loading + case loadFailed(String) + case empty + case parts + } + + private(set) var parts: [ClickHousePartInfo] = [] + private(set) var isLoading = true + private(set) var loadError: String? + var alert: ClickHousePartsAlert? + + var display: Display { + guard parts.isEmpty else { return .parts } + if let loadError { return .loadFailed(loadError) } + return isLoading ? .loading : .empty + } + + mutating func beginLoad() { + isLoading = true + loadError = nil + } + + mutating func finishLoad(_ parts: [ClickHousePartInfo]) { + self.parts = parts + isLoading = false + } + + mutating func failLoad(_ message: String) { + isLoading = false + guard parts.isEmpty else { + alert = ClickHousePartsAlert(title: String(localized: "Couldn't Read the Parts"), message: message) + return + } + loadError = message + } + + mutating func actionEnded(_ failure: GatedOperationFailure, title: String) { + guard let message = failure.message else { return } + alert = ClickHousePartsAlert(title: title, message: message) + } +} + struct ClickHousePartsView: View { private static let logger = Logger(subsystem: "com.TablePro", category: "ClickHousePartsView") @@ -17,31 +67,31 @@ struct ClickHousePartsView: View { let connection: DatabaseConnection let reloadToken: Int - @State private var parts: [ClickHousePartInfo] = [] - @State private var isLoading = true - @State private var errorMessage: String? + @State private var pane = ClickHousePartsPaneState() @State private var loadedToken: Int? @StateObject private var grid = StructureListGridModel(accessibilityLabel: StructurePane.parts.displayName) @StateObject private var selection = StructureListSelection() var body: some View { VStack(spacing: 0) { - if isLoading { + switch pane.display { + case .loading: ProgressView() .frame(maxWidth: .infinity, maxHeight: .infinity) .gridStandInBackground() - } else if let error = errorMessage { - VStack(spacing: 8) { - Image(systemName: "exclamationmark.triangle") - .font(.largeTitle) - .foregroundStyle(.orange) - .accessibilityHidden(true) - RevealedTextView(error) - .foregroundStyle(.secondary) + case .loadFailed(let message): + UnavailableStateView { + Label(String(localized: "Couldn't Read the Parts"), systemImage: "exclamationmark.triangle") + } description: { + RevealedTextView(message) + } actions: { + Button(String(localized: "Try Again")) { + Task { await loadParts() } + } } .frame(maxWidth: .infinity, maxHeight: .infinity) .gridStandInBackground() - } else if parts.isEmpty { + case .empty: VStack(spacing: 8) { Image(systemName: "tray") .font(.largeTitle) @@ -52,11 +102,20 @@ struct ClickHousePartsView: View { } .frame(maxWidth: .infinity, maxHeight: .infinity) .gridStandInBackground() - } else { + case .parts: partsToolbar partsTable } } + .alert( + pane.alert?.title ?? "", + isPresented: Binding(get: { pane.alert != nil }, set: { if !$0 { pane.alert = nil } }), + presenting: pane.alert + ) { _ in + Button(String(localized: "OK"), role: .cancel) {} + } message: { alert in + Text(verbatim: RevealedText(alert.message).plainText) + } .task(id: reloadToken) { await loadPartsIfNeeded() } } @@ -92,7 +151,7 @@ struct ClickHousePartsView: View { columns: Self.columnTitles, defaultWidths: Self.columnWidths, minimumWidths: Self.columnMinimumWidths, - rows: parts.map(Self.gridRow(for:)), + rows: pane.parts.map(Self.gridRow(for:)), databaseType: connection.type, selectedIDs: selection.ids, select: { selection.ids = $0 }, @@ -130,6 +189,20 @@ struct ClickHousePartsView: View { ) } + // Column order is ClickHousePartStatements.parts. + internal static func part(from row: [PluginCellValue]) -> ClickHousePartInfo? { + guard let name = row[safe: 2]?.asText else { return nil } + return ClickHousePartInfo( + partition: row[safe: 0]?.asText ?? "", + partitionId: row[safe: 1]?.asText ?? "", + name: name, + rows: row[safe: 3]?.asText.flatMap { UInt64($0) } ?? 0, + bytesOnDisk: row[safe: 4]?.asText.flatMap { UInt64($0) } ?? 0, + modificationTime: row[safe: 5]?.asText ?? "", + active: row[safe: 6]?.asText == "1" + ) + } + // MARK: - Actions private func optimizeTable() { @@ -140,19 +213,20 @@ struct ClickHousePartsView: View { database: scope.database, table: tableName, quote: driver.quoteIdentifier ), description: String(localized: "Optimize Table"), + failureTitle: String(localized: "Couldn't Optimize Table"), kind: .maintenance ) } } private func dropSelectedPartition() { - guard let partitionValue = selectedPartitionValue() else { return } + guard let part = selectedPart() else { return } Task { @MainActor in let confirmed = await AlertHelper.confirmDestructive( title: String(localized: "Drop Partition?"), message: String( format: String(localized: "This will permanently delete all data in partition '%@'."), - partitionValue + part.partition ), confirmButton: String(localized: "Drop"), cancelButton: String(localized: "Cancel") @@ -160,27 +234,31 @@ struct ClickHousePartsView: View { guard confirmed else { return } guard let driver = DatabaseManager.shared.driver(for: scope.connectionId) else { return } - let sql = ClickHousePartStatements.dropPartition( + let failureTitle = String(localized: "Couldn't Drop Partition") + guard let sql = ClickHousePartStatements.dropPartition( database: scope.database, table: tableName, - partition: partitionValue, - quote: driver.quoteIdentifier, - escape: driver.escapeStringLiteral - ) - guard await run(sql, description: String(localized: "Drop Partition"), kind: .destructiveQuery) - else { return } + partitionId: part.partitionId, + quote: driver.quoteIdentifier + ) else { + pane.alert = ClickHousePartsAlert(title: failureTitle, message: Self.unusablePartitionIdMessage) + return + } + guard await run( + sql, description: String(localized: "Drop Partition"), failureTitle: failureTitle, kind: .destructiveQuery + ) else { return } selection.ids.removeAll() } } private func detachSelectedPartition() { - guard let partitionValue = selectedPartitionValue() else { return } + guard let part = selectedPart() else { return } Task { @MainActor in let confirmed = await AlertHelper.confirmDestructive( title: String(localized: "Detach Partition?"), message: String( format: String(localized: "This will detach partition '%@'. Data will be preserved but inaccessible until re-attached."), - partitionValue + part.partition ), confirmButton: String(localized: "Detach"), cancelButton: String(localized: "Cancel") @@ -188,22 +266,30 @@ struct ClickHousePartsView: View { guard confirmed else { return } guard let driver = DatabaseManager.shared.driver(for: scope.connectionId) else { return } - let sql = ClickHousePartStatements.detachPartition( + let failureTitle = String(localized: "Couldn't Detach Partition") + guard let sql = ClickHousePartStatements.detachPartition( database: scope.database, table: tableName, - partition: partitionValue, - quote: driver.quoteIdentifier, - escape: driver.escapeStringLiteral - ) - guard await run(sql, description: String(localized: "Detach Partition"), kind: .destructiveQuery) - else { return } + partitionId: part.partitionId, + quote: driver.quoteIdentifier + ) else { + pane.alert = ClickHousePartsAlert(title: failureTitle, message: Self.unusablePartitionIdMessage) + return + } + guard await run( + sql, description: String(localized: "Detach Partition"), failureTitle: failureTitle, kind: .destructiveQuery + ) else { return } selection.ids.removeAll() } } + private static var unusablePartitionIdMessage: String { + String(localized: "The server reported a partition ID that cannot be used in a statement, so nothing was run.") + } + @MainActor @discardableResult - private func run(_ sql: String, description: String, kind: OperationKind) async -> Bool { + private func run(_ sql: String, description: String, failureTitle: String, kind: OperationKind) async -> Bool { let scope = scope do { let decision = await ExecutionGateProvider.shared.authorize( @@ -217,10 +303,7 @@ struct ClickHousePartsView: View { operationDescription: description ) ) - guard case .authorized = decision else { - errorMessage = decision.deniedReason - return false - } + if let denial = decision.denialError { throw denial } _ = try await DatabaseManager.shared.withScopedDriver( scope: scope, route: DatabaseManager.shared.schemaChangeRoute(for: scope), @@ -228,30 +311,30 @@ struct ClickHousePartsView: View { ) { driver in try await driver.execute(query: sql) } - if kind != .maintenance { - CatalogChangeService.post( - .changed(CatalogChange(connectionId: scope.connectionId, database: scope.database, kinds: .tables)) - ) - } - await loadParts() - return true } catch { - Self.logger.error("\(description, privacy: .public) failed: \(error.publicLogShape, privacy: .public)") - errorMessage = error.localizedDescription + let failure = GatedOperationFailure(error) + if failure.reportsFailure { + Self.logger.error("\(description, privacy: .public) failed: \(error.publicLogShape, privacy: .public)") + } + pane.actionEnded(failure, title: failureTitle) return false } + if kind != .maintenance { + CatalogChangeService.post( + .changed(CatalogChange(connectionId: scope.connectionId, database: scope.database, kinds: .tables)) + ) + } + await loadParts() + return true } - private func selectedPartitionValue() -> String? { - Self.partition(of: selection.ids, in: parts) + private func selectedPart() -> ClickHousePartInfo? { + Self.selectedPart(of: selection.ids, in: pane.parts) } - internal static func partition(of selectedIDs: Set, in parts: [ClickHousePartInfo]) -> String? { - guard selectedIDs.count == 1, - let selectedId = selectedIDs.first, - let part = parts.first(where: { $0.id == selectedId }) - else { return nil } - return part.partition + internal static func selectedPart(of selectedIDs: Set, in parts: [ClickHousePartInfo]) -> ClickHousePartInfo? { + guard selectedIDs.count == 1, let selectedId = selectedIDs.first else { return nil } + return parts.first { $0.id == selectedId } } // MARK: - Data Loading @@ -263,8 +346,7 @@ struct ClickHousePartsView: View { private func loadParts() async { let token = reloadToken - isLoading = true - errorMessage = nil + pane.beginLoad() let scope = scope let tableName = tableName @@ -276,29 +358,12 @@ struct ClickHousePartsView: View { } guard !Task.isCancelled else { return } selection.ids = [] - parts = result.rows.compactMap { row -> ClickHousePartInfo? in - guard let name = row[safe: 1]?.asText else { return nil } - let partition = row[safe: 0]?.asText ?? "" - let rows = row[safe: 2]?.asText.flatMap { UInt64($0) } ?? 0 - let bytesOnDisk = row[safe: 3]?.asText.flatMap { UInt64($0) } ?? 0 - let modTime = row[safe: 4]?.asText ?? "" - let active = row[safe: 5]?.asText == "1" - return ClickHousePartInfo( - partition: partition, - name: name, - rows: rows, - bytesOnDisk: bytesOnDisk, - modificationTime: modTime, - active: active - ) - } + pane.finishLoad(result.rows.compactMap(Self.part(from:))) loadedToken = token } catch { guard !Task.isCancelled else { return } Self.logger.error("Failed to load parts: \(error.publicLogShape, privacy: .public)") - errorMessage = error.localizedDescription + pane.failLoad(error.localizedDescription) } - - isLoading = false } } diff --git a/apple/TablePro/Views/Structure/CreateTableDraft.swift b/apple/TablePro/Views/Structure/CreateTableDraft.swift index bffd4f2653..854b5fea41 100644 --- a/apple/TablePro/Views/Structure/CreateTableDraft.swift +++ b/apple/TablePro/Views/Structure/CreateTableDraft.swift @@ -469,8 +469,7 @@ internal final class CreateTableDraft: ObservableObject { coordinator: coordinator ) } catch { - Self.logger.error("Create table failed: \(error.publicLogShape, privacy: .public)") - reportFailure(error.localizedDescription, on: coordinator) + reportFailure(error: error, on: coordinator) } } } @@ -508,8 +507,7 @@ internal final class CreateTableDraft: ObservableObject { } catch let formError as PluginCreateTableFormError { form?.recordSubmissionError(formError) } catch { - Self.logger.error("Create table failed: \(error.publicLogShape, privacy: .public)") - reportFailure(error.localizedDescription, on: coordinator) + reportFailure(error: error, on: coordinator) } } } @@ -560,6 +558,15 @@ internal final class CreateTableDraft: ObservableObject { ) } + private func reportFailure(error: any Error, on coordinator: MainContentCoordinator) { + let failure = GatedOperationFailure(error) + if failure.reportsFailure { + Self.logger.error("Create table failed: \(error.publicLogShape, privacy: .public)") + } + guard let message = failure.message else { return } + reportFailure(message, on: coordinator) + } + private func reportFailure(_ message: String, on coordinator: MainContentCoordinator) { AlertHelper.showErrorSheet( title: String(localized: "Create Table Failed"), diff --git a/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift b/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift index b760819838..15048571f6 100644 --- a/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift +++ b/apple/TablePro/Views/Structure/StructureEditingSession+Apply.swift @@ -31,52 +31,13 @@ internal enum StructureSaveOutcome: Equatable { } } -/// How a save that did not apply ends, read from the error alone so it can be tested without -/// presenting anything. -/// -/// The gate's sheet is the one confirmation a save gets, so Cancel there is the ordinary way to -/// back out, and it ends quietly the way closing any confirmation does. Answering it with an -/// "Error Applying Changes" sheet put a second dialog in front of the user for the choice they had -/// just made. -internal enum StructureApplyFailure: Equatable { - case cancelledByUser - case refused(String) - case failed(String) - - internal init(_ error: any Error) { - guard let gateError = error as? ExecutionGateError else { - self = .failed(error.localizedDescription) - return - } - switch gateError { - case .cancelledByUser: - self = .cancelledByUser - case .denied(let reason): - self = .refused(reason) - } - } - - internal var outcome: StructureSaveOutcome { +internal extension GatedOperationFailure { + var outcome: StructureSaveOutcome { switch self { case .cancelledByUser, .refused: .refused case .failed(let message): .failed(message) } } - - /// What the error sheet says. Nil for a Cancel, which shows nothing. - internal var message: String? { - switch self { - case .cancelledByUser: nil - case .refused(let reason), .failed(let reason): reason - } - } - - /// A refusal and a Cancel both stop at the gate, before the first statement, so neither is a - /// failed operation to report or a reason to reload the catalog. - internal var reportsFailure: Bool { - if case .failed = self { return true } - return false - } } internal extension StructureEditingSession { @@ -159,7 +120,7 @@ internal extension StructureEditingSession { // A failed save that may have written; its error waits for the table to be read again. private struct AlterFailure { - let failure: StructureApplyFailure + let failure: GatedOperationFailure let startedAt: ContinuousClock.Instant } @@ -240,7 +201,7 @@ internal extension StructureEditingSession { report(.succeeded(OperationSummary()), startedAt: operationStart, coordinator: coordinator) return (.applied, nil) } catch { - let failure = StructureApplyFailure(error) + let failure = GatedOperationFailure(error) // A gate refusal and a driver's refusal both stop before the first statement. guard failure.reportsFailure, !(error is SchemaOperationRefusedError) else { present(failure, startedAt: operationStart, coordinator: coordinator) @@ -251,7 +212,7 @@ internal extension StructureEditingSession { } private func present( - _ failure: StructureApplyFailure, + _ failure: GatedOperationFailure, note: String? = nil, startedAt: ContinuousClock.Instant, coordinator: MainContentCoordinator? @@ -340,7 +301,7 @@ internal extension StructureEditingSession { } catch { changeManager.releaseHold(hold, written: false) isApplying = false - let failure = StructureApplyFailure(error) + let failure = GatedOperationFailure(error) if failure.reportsFailure { CatalogChangeService.post( .changed(CatalogChange(connectionId: connection.id, database: prepared.scope.database, kinds: .tables)) diff --git a/apple/TablePro/Views/Structure/StructureEditingSession+ColumnReorder.swift b/apple/TablePro/Views/Structure/StructureEditingSession+ColumnReorder.swift index 30c1ff93cc..47f269457e 100644 --- a/apple/TablePro/Views/Structure/StructureEditingSession+ColumnReorder.swift +++ b/apple/TablePro/Views/Structure/StructureEditingSession+ColumnReorder.swift @@ -65,7 +65,7 @@ internal extension StructureEditingSession { ) await finishColumnReorder(prepared, clearTarget: clearTarget) } catch { - if StructureApplyFailure(error).reportsFailure { + if GatedOperationFailure(error).reportsFailure { CatalogChangeService.post( .changed( CatalogChange( @@ -115,7 +115,7 @@ internal extension StructureEditingSession { } private func reportColumnReorderFailure(_ error: any Error) { - guard let message = StructureApplyFailure(error).message else { return } + guard let message = GatedOperationFailure(error).message else { return } AlertHelper.showErrorSheet( title: String(localized: "Column Reorder Failed"), message: message, diff --git a/apple/TablePro/Views/Structure/TriggerDetailView.swift b/apple/TablePro/Views/Structure/TriggerDetailView.swift index a0885aac43..34a9be327b 100644 --- a/apple/TablePro/Views/Structure/TriggerDetailView.swift +++ b/apple/TablePro/Views/Structure/TriggerDetailView.swift @@ -277,7 +277,7 @@ struct TriggerDetailView: View { name: trigger.name ) } catch { - actionError = error.localizedDescription + actionError = GatedOperationFailure(error).message } } } diff --git a/apple/TablePro/Views/Structure/TriggerEditorView.swift b/apple/TablePro/Views/Structure/TriggerEditorView.swift index 6dee7b596e..ef9b441168 100644 --- a/apple/TablePro/Views/Structure/TriggerEditorView.swift +++ b/apple/TablePro/Views/Structure/TriggerEditorView.swift @@ -29,7 +29,12 @@ struct TriggerEditorView: View { @State private var isApplying = false @State private var errorMessage: String? @State private var droppedDefinition: String? - @AppStorage("structureCodeFontSize", store: AppStorageEnvironment.shared.defaults) private var fontSize: Double = 13 + @AppStorage(TriggerEditorView.fontSizeKey, store: AppStorageEnvironment.shared.defaults) + private var fontSize = TriggerEditorView.defaultFontSize + + // The Structure tab's DDL pane writes its zoom here too. + static let fontSizeKey = "structureCodeFontSize" + static let defaultFontSize = 13.0 init( scope: DatabaseScope, @@ -45,7 +50,14 @@ struct TriggerEditorView: View { self.mode = mode self.onClose = onClose self._sql = State(wrappedValue: initialSQL) - self._editorConfiguration = State(wrappedValue: Self.makeConfiguration(fontSize: 13)) + self._editorConfiguration = State(wrappedValue: Self.initialConfiguration()) + } + + // The State is seeded once, here, so it reads the store; `fontSize` only follows later zooms. + static func initialConfiguration( + in defaults: UserDefaults = AppStorageEnvironment.shared.defaults + ) -> SourceEditorConfiguration { + makeConfiguration(fontSize: defaults.object(forKey: fontSizeKey) as? Double ?? defaultFontSize) } var body: some View { @@ -145,7 +157,7 @@ struct TriggerEditorView: View { if let dropped = (error as? TriggerEditingError)?.droppedDefinition { droppedDefinition = dropped } - errorMessage = error.localizedDescription + errorMessage = GatedOperationFailure(error).message } } } diff --git a/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift b/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift index e7c5b14de0..9b87a2bce5 100644 --- a/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift +++ b/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift @@ -21,22 +21,39 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { let columns: [ColumnInfo] } - private var tables: [TableInfo] + private var tables: [TableInfo] { + didSet { + cachedTableItems = nil + cachedTableColumnItems = nil + } + } + // Every table-slot request reads all of them, so they are built once per table list. + private var cachedTableItems: [SQLCompletionItem]? private let databaseType: DatabaseType private let schema: String? private let readColumns: QueryColumnReader private let readCatalogColumns: QueryColumnReader? private var columnReadsAllowed = true private var columnGeneration = 0 - private var columns: [ColumnKey: CachedColumns] = [:] + private var columns: [ColumnKey: CachedColumns] = [:] { + didSet { cachedTableColumnItems = nil } + } + // Every column request before a FROM reads all of them, so they are built once per column cache, table list and + // namespace list. + private var cachedTableColumnItems: [SQLCompletionItem]? + private(set) var tableColumnItemsBuildCount = 0 private var reads: [ColumnKey: Task<[ColumnInfo]?, any Error>] = [:] private var sessionReads: [ColumnKey: Task<[ColumnInfo]?, any Error>] = [:] private let readSchemaTables: QuerySchemaTableReader? private let readCatalogSchemaTables: QuerySchemaTableReader? private var sessionSchemaReads: [String: Task<[TableInfo]?, any Error>] = [:] private var sessionLocalTables: Set = [] - private var schemas: [String] - private var databases: [String] + private var schemas: [String] { + didSet { cachedTableColumnItems = nil } + } + private var databases: [String] { + didSet { cachedTableColumnItems = nil } + } private var schemaTables: [String: (generation: Int, tables: [TableInfo])] = [:] private var schemaTableReads: [String: Task<[TableInfo]?, any Error>] = [:] @@ -123,19 +140,21 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { } func tableCompletionItems() -> [SQLCompletionItem] { - tables.map { table in - let isView = table.type == .view || table.type == .materializedView - return SQLCompletionItem(label: table.name, kind: isView ? .view : .table, insertText: quoted(table.name)) - } + if let cachedTableItems { return cachedTableItems } + let items = tables.map { tableItem($0) } + cachedTableItems = items + return items + } + + private func tableItem(_ table: TableInfo) -> SQLCompletionItem { + let isView = table.type == .view || table.type == .materializedView + return SQLCompletionItem(label: table.name, kind: isView ? .view : .table, insertText: quoted(table.name)) } func tableCompletionItems(inSchema schema: String) async -> [SQLCompletionItem] { guard isKnownSchema(schema) else { return [] } guard !isActiveSchema(schema) else { return tableCompletionItems() } - return await tables(inSchema: schema).map { table in - let isView = table.type == .view || table.type == .materializedView - return SQLCompletionItem(label: table.name, kind: isView ? .view : .table, insertText: quoted(table.name)) - } + return await tables(inSchema: schema).map { tableItem($0) } } func schemaCompletionItems() -> [SQLCompletionItem] { @@ -317,6 +336,14 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { } func allColumnsFromCachedTables() -> [SQLCompletionItem] { + if let cachedTableColumnItems { return cachedTableColumnItems } + let items = buildTableColumnItems() + tableColumnItemsBuildCount += 1 + cachedTableColumnItems = items + return items + } + + private func buildTableColumnItems() -> [SQLCompletionItem] { // Sorted, because the ranker keeps emission order between equal scores and a dictionary's order is per process. let cached = columns.sorted { lhs, rhs in (lhs.key.schema ?? "", lhs.key.table) < (rhs.key.schema ?? "", rhs.key.table) diff --git a/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift b/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift index f5322a5f2e..5dd5104935 100644 --- a/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift @@ -400,6 +400,35 @@ struct QueryCompletionCatalogTests { #expect(await source.reads == ["orders", "customers"]) } + @Test("cached column suggestions are built once, and again only after a read, a table list or a namespace list") + func cachedColumnsAreBuiltOnce() async { + let source = CatalogColumnSource(["orders": [column("id")], "customers": [column("name")]]) + let catalog = catalog(source, tables: [table("orders"), table("customers")]) + _ = await catalog.columnCompletionItems(for: "orders", schema: nil) + + let first = await catalog.allColumnsFromCachedTables() + let second = await catalog.allColumnsFromCachedTables() + #expect(labels(second) == labels(first)) + #expect(await catalog.tableColumnItemsBuildCount == 1) + + _ = await catalog.columnCompletionItems(for: "customers", schema: nil) + let afterRead = await catalog.allColumnsFromCachedTables() + #expect(labels(afterRead) == ["name", "id"]) + #expect(await catalog.tableColumnItemsBuildCount == 2) + + let listedBefore = await catalog.tableCompletionItems() + #expect(labels(listedBefore) == ["orders", "customers"]) + await catalog.replaceTables([table("orders"), table("customers"), table("refunds")]) + _ = await catalog.allColumnsFromCachedTables() + #expect(await catalog.tableColumnItemsBuildCount == 3) + let listedAfter = await catalog.tableCompletionItems() + #expect(labels(listedAfter) == ["orders", "customers", "refunds"]) + + await catalog.replaceNamespaces(schemas: ["main"], databases: []) + _ = await catalog.allColumnsFromCachedTables() + #expect(await catalog.tableColumnItemsBuildCount == 4) + } + @Test("the catalog connection reads first and is not closed by a run; the session reads only without one") func catalogConnectionFirst() async { let catalogSource = CatalogColumnSource(["orders": [column("id")]]) diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift index d3d4c2d300..aed8b0f01c 100644 --- a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorBitColumnTests.swift @@ -92,8 +92,8 @@ struct SQLStatementGeneratorBitColumnTests { let rowID = RowID.inserted(UUID()) let statements = generator.generateStatements( - from: [RowChange(rowID: rowID, type: .insert)], - insertedRowData: [rowID: ["__DEFAULT__", "5", "Ann"]], + from: [RowChange(rowID: rowID, type: .insert, expressionColumns: [0])], + insertedRowData: [rowID: [.columnDefault, "5", "Ann"]], deletedRowIDs: [], insertedRowIDs: [rowID] ) diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorRowMatchTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorRowMatchTests.swift index 44d6a863ef..ea1bbb0f87 100644 --- a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorRowMatchTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorRowMatchTests.swift @@ -62,10 +62,12 @@ struct SQLStatementGeneratorRowMatchTests { @Test("A Databend row left entirely on defaults names one column with DEFAULT") func databendAllDefaultsInsert() throws { - let change = RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil) + let change = RowChange( + rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil, expressionColumns: [0, 1, 2, 3] + ) let statements = try generator(excluding: []).generateStatements( from: [change], - insertedRowData: [.existing(0): [.text("__DEFAULT__"), .text("__DEFAULT__"), .text("__DEFAULT__"), .text("__DEFAULT__")]], + insertedRowData: [.existing(0): [.columnDefault, .columnDefault, .columnDefault, .columnDefault]], deletedRowIDs: [], insertedRowIDs: [.existing(0)] ) diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorTests.swift index edeb219dea..731615a286 100644 --- a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorTests.swift +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorTests.swift @@ -89,14 +89,14 @@ struct SQLStatementGeneratorTests { #expect(statements[0].parameters[2] == nil) } - @Test("Insert skips __DEFAULT__ columns") + @Test("Insert leaves out a column marked as its default") func testInsertSkipsDefaultColumns() throws { let generator = try makeGenerator() let insertedRowData: [RowID: [PluginCellValue]] = [ - .existing(0): ["__DEFAULT__", "John", "john@example.com"] + .existing(0): [.columnDefault, "John", "john@example.com"] ] let changes: [RowChange] = [ - RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil) + RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil, expressionColumns: [0]) ] let statements = generator.generateStatements( @@ -114,11 +114,35 @@ struct SQLStatementGeneratorTests { #expect(stmt.parameters.count == 2) } + @Test("Insert binds text that only reads like the default, unmarked", arguments: ["__DEFAULT__", "DEFAULT"]) + func testInsertBindsUnmarkedDefaultText(text: String) throws { + let generator = try makeGenerator() + let insertedRowData: [RowID: [PluginCellValue]] = [ + .existing(0): [.columnDefault, .text(text), "john@example.com"] + ] + let changes: [RowChange] = [ + RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil, expressionColumns: [0]) + ] + + let statements = generator.generateStatements( + from: changes, + insertedRowData: insertedRowData, + deletedRowIDs: [], + insertedRowIDs: [.existing(0)] + ) + + #expect(statements.count == 1) + let stmt = statements[0] + #expect(!stmt.sql.contains("`id`")) + #expect(stmt.sql.contains("`name`")) + #expect(stmt.parameters.compactMap { $0 as? String } == [text, "john@example.com"]) + } + /// This used to expect no statement at all, which dropped the row from the batch while the rest /// of the save committed and reported success. A table of nothing but an identity column and /// defaults is exactly the shape that produces it. @Test( - "Insert with all __DEFAULT__ names no column", + "Insert with every column marked as its default names no column", arguments: [ (DatabaseType.mysql, "() VALUES ()"), (DatabaseType.postgresql, "DEFAULT VALUES"), @@ -128,10 +152,10 @@ struct SQLStatementGeneratorTests { func testInsertAllDefaultNamesNoColumn(databaseType: DatabaseType, expected: String) throws { let generator = try makeGenerator(databaseType: databaseType) let insertedRowData: [RowID: [PluginCellValue]] = [ - .existing(0): ["__DEFAULT__", "__DEFAULT__", "__DEFAULT__"] + .existing(0): [.columnDefault, .columnDefault, .columnDefault] ] let changes: [RowChange] = [ - RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil) + RowChange(rowID: .existing(0), type: .insert, cellChanges: [], originalRow: nil, expressionColumns: [0, 1, 2]) ] let statements = generator.generateStatements( @@ -378,7 +402,7 @@ struct SQLStatementGeneratorTests { #expect(statements[0].parameters[0] == nil) } - @Test("Update with __DEFAULT__ value") + @Test("Update with a cell marked as its default") func testUpdateWithDefaultValue() throws { let generator = try makeGenerator() let changes: [RowChange] = [ @@ -386,7 +410,10 @@ struct SQLStatementGeneratorTests { rowID: .existing(0), type: .update, cellChanges: [ - CellChange(columnIndex: 1, columnName: "name", oldValue: "John", newValue: "__DEFAULT__") + CellChange( + columnIndex: 1, columnName: "name", oldValue: "John", newValue: .columnDefault, + newIsExpression: true + ) ], originalRow: ["1", "John", "john@example.com"] ) @@ -405,6 +432,32 @@ struct SQLStatementGeneratorTests { #expect(stmt.parameters.count == 1) } + @Test("Update binds typed text that only reads like the default", arguments: ["__DEFAULT__", "DEFAULT"]) + func testUpdateBindsUnmarkedDefaultText(text: String) throws { + let generator = try makeGenerator() + let changes: [RowChange] = [ + RowChange( + rowID: .existing(0), + type: .update, + cellChanges: [CellChange(columnIndex: 1, columnName: "name", oldValue: "John", newValue: .text(text))], + originalRow: ["1", "John", "john@example.com"] + ) + ] + + let statements = generator.generateStatements( + from: changes, + insertedRowData: [:], + deletedRowIDs: [], + insertedRowIDs: [] + ) + + #expect(statements.count == 1) + let stmt = statements[0] + #expect(stmt.sql.contains("`name` = ?")) + #expect(!stmt.sql.contains("DEFAULT")) + #expect(stmt.parameters.compactMap { $0 as? String } == [text, "1"]) + } + @Test("Update with a cell marked as SQL inlines it") func testUpdateWithSQLFunction() throws { let generator = try makeGenerator() diff --git a/apple/TableProTests/Core/Compare/CompareDriverRowCutTests.swift b/apple/TableProTests/Core/Compare/CompareDriverRowCutTests.swift new file mode 100644 index 0000000000..1d8668db4f --- /dev/null +++ b/apple/TableProTests/Core/Compare/CompareDriverRowCutTests.swift @@ -0,0 +1,90 @@ +// +// CompareDriverRowCutTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@testable import TablePro + +struct CompareDriverRowCutTests { + private func stream(rowLimit: Int?, ids: [Int]) -> AsyncThrowingStream { + var header = PluginStreamHeader(columns: ["id", "name"], columnTypeNames: ["integer", "text"]) + header.rowLimit = rowLimit + let rows: [PluginRow] = ids.map { [.text(String($0)), .text("n")] } + return AsyncThrowingStream { continuation in + continuation.yield(.header(header)) + continuation.yield(.rows(rows)) + continuation.finish() + } + } + + private func targetRows(_ ids: [Int]) -> [DataRow] { + ids.map { DataRow(values: ["id": .text(String($0)), "name": .text("n")]) } + } + + private func engine() -> DataDiffEngine { + DataDiffEngine( + options: DataCompareOptions(), + shape: DataComparisonShape( + keyColumns: ["id"], + keyOrders: [.numeric], + comparedColumns: ["name"], + digestColumns: ["id", "name"] + ) + ) + } + + @Test("A stream that ends at the limit its driver declares reads as cut there") + func streamEndingAtTheDeclaredLimitIsCut() async throws { + let provider = StreamingRowProvider(stream: stream(rowLimit: 2, ids: [1, 2])) + while try await provider.nextRow() != nil {} + + #expect(provider.endedAtRowLimit) + #expect(provider.driverRowCut == 2) + } + + @Test("A stream that ends short of its declared limit read every row") + func streamShortOfTheDeclaredLimitIsWhole() async throws { + let provider = StreamingRowProvider(stream: stream(rowLimit: 5, ids: [1, 2])) + while try await provider.nextRow() != nil {} + + #expect(!provider.endedAtRowLimit) + #expect(provider.driverRowCut == nil) + } + + // A HANA source stops at 5,000,000 rows: every target row past them read as a delete, and a sync removed them. + @Test("A source the driver cut short reports no deletes past the cut") + func cutSourceProducesNoDeletesPastIt() async throws { + let summary = try await engine().compare( + source: StreamingRowProvider(stream: stream(rowLimit: 2, ids: [1, 2])), + target: ArrayRowProvider(rows: targetRows([1, 2, 3, 4])) + ) + + #expect(summary.deleteCount == 0) + #expect(summary.identicalCount == 2) + #expect(summary.stoppedAtRowLimit) + #expect(summary.sourceRowCut == 2) + #expect(summary.targetRowCut == nil) + #expect(summary.resumeKey == [.text("2")]) + } + + @Test("A target the driver cut short reports no inserts past the cut") + func cutTargetProducesNoInsertsPastIt() async throws { + let summary = try await engine().compare( + source: ArrayRowProvider(rows: targetRows([1, 2, 3, 4])), + target: StreamingRowProvider(stream: stream(rowLimit: 2, ids: [1, 2])) + ) + + #expect(summary.insertCount == 0) + #expect(summary.targetRowCut == 2) + #expect(summary.stoppedAtRowLimit) + } + + @Test("The cut is named in the wording the export uses") + func cutWordingMatchesTheExport() { + #expect(StreamRowCutWarning.text(table: "orders", limit: 5_000_000).hasPrefix("orders: ")) + } +} diff --git a/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift b/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift index d6ba71ac85..04baec1f63 100644 --- a/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift +++ b/apple/TableProTests/Core/Compare/CompareSQLLiteralTests.swift @@ -99,14 +99,27 @@ struct CompareSQLLiteralTests { #expect(CompareSQLLiteral.prefixed("0x89504E47", databaseType: .mssql) == "0x89504E47") } + // `'\x…'::bytea` reads its backslash by `standard_conforming_strings`, and with that off a sync stored other bytes. @Test - func postgresFamilyUsesAByteaCast() throws { - for type in [DatabaseType.postgresql, .cockroachdb, .redshift, .pglite, .kingbasees] { + func postgresFamilyDecodesHex() throws { + for type in [DatabaseType.postgresql, .cockroachdb, .pglite] { let literal = try #require(CompareSQLLiteral.binaryLiteral(for: png, databaseType: type)) - #expect(literal == "'\\x89504e47'::bytea", "\(type.rawValue) uses a bytea literal") + #expect(literal == "decode('89504E47', 'hex')", "\(type.rawValue) decodes hex") } } + // KingbaseES documents `decode` only as Oracle's DECODE; its bytea decoder is not the PostgreSQL one. + @Test + func kingbaseKeepsTheByteaLiteral() { + #expect(CompareSQLLiteral.binaryLiteral(for: png, databaseType: .kingbasees) == "'\\x89504E47'::bytea") + } + + // Redshift has no bytea, and its DECODE is the CASE-like one; a binary column there is VARBYTE. + @Test + func redshiftReadsHexIntoVarbyte() { + #expect(CompareSQLLiteral.binaryLiteral(for: png, databaseType: .redshift) == "from_hex('89504E47')") + } + @Test func sQLServerUsesZeroX() throws { let literal = try #require(CompareSQLLiteral.binaryLiteral(for: png, databaseType: .mssql)) @@ -121,6 +134,12 @@ struct CompareSQLLiteralTests { #expect(literal == "HEXTORAW('89504E47')") } + // `HEXTORAW('')` stores NULL, and a `BLOB NOT NULL` refuses it. + @Test + func oracleWritesAnEmptyValueAsAnEmptyBlob() { + #expect(CompareSQLLiteral.binaryLiteral(for: Data(), databaseType: .oracle) == "EMPTY_BLOB()") + } + /// An engine this build does not name falls back to the driver's own spelling rather than /// guessing at one, which is why the helper returns nil instead of a default. @Test @@ -132,6 +151,6 @@ struct CompareSQLLiteralTests { func emptyDataStillProducesAValidLiteral() throws { let literal = try #require(CompareSQLLiteral.binaryLiteral(for: Data(), databaseType: .postgresql)) - #expect(literal == "'\\x'::bytea") + #expect(literal == "decode('', 'hex')") } } diff --git a/apple/TableProTests/Core/Compare/DataSyncScriptBuilderTests.swift b/apple/TableProTests/Core/Compare/DataSyncScriptBuilderTests.swift index 72c7c861e1..517a89829e 100644 --- a/apple/TableProTests/Core/Compare/DataSyncScriptBuilderTests.swift +++ b/apple/TableProTests/Core/Compare/DataSyncScriptBuilderTests.swift @@ -795,7 +795,7 @@ final class DataSyncScriptBuilderColumnTests: XCTestCase { XCTAssertEqual( statements.map(\.sql), - [#"INSERT INTO "public"."files" ("id", "blob") VALUES (1, '\x8950'::bytea)"#] + [#"INSERT INTO "public"."files" ("id", "blob") VALUES (1, decode('8950', 'hex'))"#] ) } diff --git a/apple/TableProTests/Core/Coordinators/RowEditingCoordinatorUUIDKeyTests.swift b/apple/TableProTests/Core/Coordinators/RowEditingCoordinatorUUIDKeyTests.swift index 71a31a2c8f..1386171d49 100644 --- a/apple/TableProTests/Core/Coordinators/RowEditingCoordinatorUUIDKeyTests.swift +++ b/apple/TableProTests/Core/Coordinators/RowEditingCoordinatorUUIDKeyTests.swift @@ -71,7 +71,7 @@ struct RowEditingCoordinatorUUIDKeyTests { coordinator.addNewRow() - #expect(try #require(lastRow(of: coordinator)).values[0] == .text("__DEFAULT__")) + #expect(try #require(lastRow(of: coordinator)).values[0] == .columnDefault) } @Test("Add Row on an integer key leaves it as before") diff --git a/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift b/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift index fb7a319753..10d189f39a 100644 --- a/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift +++ b/apple/TableProTests/Core/DataWrite/RewindPlannerTests.swift @@ -277,6 +277,87 @@ struct RewindPlannerTests { #expect(queries.first?.contains("SELECT *") == false) } + @Test("An edit that changed the key is read and restored under the key the save gave it") + func movedKeyIsFoundUnderItsNewKey() throws { + let moved = operation(kind: .update, preImage: ["k5", "Ada"], postImage: ["k6", "Ada"], writtenColumns: ["id"]) + let rewind = planner(operations: [moved]) + + let query = try #require(rewind.readQueries().first) + #expect(query.contains("k6")) + #expect(query.contains("k5")) + + let plan = try rewind.plan(currentRows: [RewindCurrentRow(values: ["k6", "Ada"])]) + #expect(plan.rows.map { $0.outcome } == [.willRestore]) + #expect(plan.statements.first?.sql.hasPrefix("UPDATE") == true) + #expect(plan.statements.first?.parameters.compactMap { $0 as? String } == ["k5", "k6"]) + } + + @Test("A moved row a rewind already put back reads as restored, and another row on its old key does not") + func movedKeyAlreadyRestored() throws { + let moved = operation( + kind: .update, preImage: ["k5", "Ada"], postImage: ["k6", "Grace"], writtenColumns: ["id", "name"] + ) + + let restored = try planner(operations: [moved]).plan(currentRows: [RewindCurrentRow(values: ["k5", "Ada"])]) + #expect(restored.rows.map { $0.outcome } == [.alreadyRestored]) + #expect(restored.statements.isEmpty) + + let taken = try planner(operations: [moved]).plan(currentRows: [RewindCurrentRow(values: ["k5", "Bo"])]) + #expect(taken.rows.map { $0.outcome } == [.rowMissing]) + #expect(taken.statements.isEmpty) + } + + @Test("A row that took the key a save moved to is left alone, though the save wrote only the key") + func rowOnTheMovedKeyIsNotRewritten() throws { + let moved = operation(kind: .update, preImage: ["k5", "Ada"], postImage: ["k6", "Ada"], writtenColumns: ["id"]) + + let plan = try planner(operations: [moved]).plan(currentRows: [RewindCurrentRow(values: ["k6", "Bo"])]) + + #expect(plan.rows.map { $0.outcome } == [.changedSinceSave]) + #expect(plan.statements.isEmpty) + } + + @Test("A moved row is not held to a column the server computes") + func movedKeyIgnoresGeneratedColumns() throws { + let columns = ["id", "name", "updated_at"] + let moved = RowWriteOperation( + kind: .update, target: target, columns: columns, primaryKeyColumns: ["id"], + preImage: ["k5", "Ada", "2026-01-01"], postImage: ["k6", "Ada", "2026-01-01"], + writtenColumns: ["id"], refusal: nil + ) + let record = RewindRecord( + id: UUID(), historyId: nil, connectionId: UUID(), databaseType: .sqlite, + target: target, capturedAt: Date(timeIntervalSince1970: 0), + generatedColumns: ["updated_at"], identityColumns: [], operations: [moved] + ) + let planner = RewindPlanner( + record: record, + factory: RowChangeStatementFactory( + tableName: target.table, schemaName: nil, columns: columns, primaryKeyColumns: ["id"], + generatedColumns: ["updated_at"], databaseType: .sqlite, pluginDriver: nil + ), + queryBuilder: sqliteQueryBuilder + ) + + let plan = try planner.plan(currentRows: [RewindCurrentRow(values: ["k6", "Ada", "2026-06-30"])]) + + #expect(plan.rows.map { $0.outcome } == [.willRestore]) + } + + @Test("Undoing a restore takes back what the restore did last first") + func inverseRecordRunsBackwards() throws { + let deleted = operation(kind: .delete, preImage: ["7", "Ada"], postImage: nil, writtenColumns: columns) + let inserted = operation(kind: .insert, preImage: nil, postImage: ["8", "Ada"], writtenColumns: columns) + let plan = try planner(operations: [deleted, inserted]) + .plan(currentRows: [RewindCurrentRow(values: ["8", "Ada"])]) + + let inverse = RewindExecutor.inverseOperations(of: plan) + #expect(inverse.map { $0.kind } == [.delete, .insert]) + + let undo = try planner(operations: inverse).plan(currentRows: [RewindCurrentRow(values: ["7", "Ada"])]) + #expect(undo.statements.map { String($0.sql.prefix(6)) } == ["DELETE", "INSERT"]) + } + @Test("The inverse runs newest first, so a value one row freed is not taken before the other gives it up") func inverseRunsInReverseOrder() throws { let first = operation(kind: .delete, preImage: ["7", "Ada"], postImage: nil, writtenColumns: columns) diff --git a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift index e688dff4ff..570e20367b 100644 --- a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryCoverageTests.swift @@ -255,8 +255,8 @@ struct RowChangeStatementFactoryCoverageTests { let engine = DatabaseType(rawValue: "ProbeEngine") let hostOnly = LegacyStatementStubDriver { _, _, _, _ in nil } let inserted = RowID.inserted(UUID()) - let allDefaults: [PluginCellValue] = [.text("__DEFAULT__"), .text("__DEFAULT__")] - let changes = [nameEdit(), RowChange(rowID: inserted, type: .insert)] + let allDefaults: [PluginCellValue] = [.columnDefault, .columnDefault] + let changes = [nameEdit(), RowChange(rowID: inserted, type: .insert, expressionColumns: [0, 1])] let generator = try SQLStatementGenerator( tableName: "t", columns: columns, primaryKeyColumns: ["_id"], databaseType: engine, diff --git a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift index 93c24b8d32..0109aeb1ec 100644 --- a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift @@ -36,6 +36,21 @@ private final class ExpressionRecordingDriver: PluginDriverStub, @unchecked Send func quoteIdentifier(_ name: String) -> String { "[\(name)]" } } +private final class RestoringDriver: PluginDriverStub, @unchecked Sendable { + func generateIdentityPreservingInsert( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + rows: [[PluginCellValue]], + absentCells: [Int: Set] + ) -> [(statement: String, parameters: [PluginCellValue])]? { + rows.map { (statement: "INSERT INTO [events] VALUES (?, ?, ?)", parameters: $0) } + } + + func quoteIdentifier(_ name: String) -> String { "[\(name)]" } +} + @MainActor struct RowChangeStatementFactoryExpressionTests { private let columns = ["id", "name", "stamped_at"] @@ -192,7 +207,7 @@ struct RowChangeStatementFactoryExpressionTests { @Test("SQLite refuses DEFAULT on a saved row, naming the column") func sqliteDefaultUpdateIsRefused() { - let change = edit(to: .text(PluginCellValue.defaultMarkerText), isExpression: false, column: 1) + let change = edit(to: .columnDefault, isExpression: true, column: 1) let refused = refusal { _ = try factory(databaseType: .sqlite).statements(for: [change]) } #expect(refused?.kind == .update) #expect(refused?.reason.contains("name") == true) @@ -202,8 +217,8 @@ struct RowChangeStatementFactoryExpressionTests { func sqliteDefaultInsertIsWritten() throws { let rowID = RowID.inserted(UUID()) let statements = try factory(databaseType: .sqlite).statements( - for: [RowChange(rowID: rowID, type: .insert)], - insertedRowData: [rowID: ["1", .text(PluginCellValue.defaultMarkerText), "2026-01-01"]], + for: [RowChange(rowID: rowID, type: .insert, expressionColumns: [1])], + insertedRowData: [rowID: ["1", .columnDefault, "2026-01-01"]], insertedRowIDs: [rowID] ) #expect(statements.count == 1) @@ -212,8 +227,69 @@ struct RowChangeStatementFactoryExpressionTests { @Test("PostgreSQL takes DEFAULT on a saved row") func postgresDefaultUpdate() throws { - let change = edit(to: .text(PluginCellValue.defaultMarkerText), isExpression: false, column: 1) + let change = edit(to: .columnDefault, isExpression: true, column: 1) let statements = try factory().statements(for: [change]) #expect(statements.first?.sql.contains("\"name\" = DEFAULT") == true) } + + @Test("Text a row holds or a user typed that reads like the default is written as that text", arguments: [ + "__DEFAULT__", "DEFAULT", + ]) + func unmarkedDefaultTextIsAValue(text: String) throws { + let saved = try factory(databaseType: .sqlite).statements(for: [edit(to: .text(text), isExpression: false, column: 1)]) + #expect(saved.first?.sql.contains("DEFAULT") == false) + #expect(saved.first?.parameters.first as? String == text) + + let rowID = RowID.inserted(UUID()) + let inserted = try factory(databaseType: .sqlite).statements( + for: [RowChange(rowID: rowID, type: .insert)], + insertedRowData: [rowID: ["1", .text(text), "2026-01-01"]], + insertedRowIDs: [rowID] + ) + #expect(inserted.first?.parameters.compactMap { $0 as? String } == ["1", text, "2026-01-01"]) + } + + @Test("A driver is handed a default in the spelling it reads, with no SQL mark, whatever it declares") + func defaultReachesADriverInItsSpelling() throws { + let driver = ExpressionRecordingDriver(capabilities: []) + _ = try factory(databaseType: .mssql, driver: driver).statements( + for: [edit(to: .columnDefault, isExpression: true, column: 1)] + ) + #expect(driver.changes.first?.cellChanges.first?.newValue == .pluginDefaultMarker) + #expect(driver.changes.first?.expressionColumns == nil) + } + + // A driver may write DEFAULT for it, and none can say it will not, so every one that writes its own rows is refused. + @Test("Text spelled the way drivers read a default is refused by a driver that writes its own rows", arguments: [ + DatabaseType.mssql, DatabaseType.redis, + ]) + func textADriverMayReadAsDefaultIsRefused(databaseType: DatabaseType) { + let driver = ExpressionRecordingDriver(capabilities: []) + let refused = refusal { + _ = try factory(databaseType: databaseType, driver: driver).statements( + for: [edit(to: "__DEFAULT__", isExpression: false, column: 1)] + ) + } + #expect(refused?.kind == .update) + #expect(refused?.reason.contains("name") == true) + } + + @Test("A deleted row a driver puts back is refused when it holds that text", arguments: [ + DatabaseType.dameng, DatabaseType.redis, + ]) + func restoredTextADriverMayReadAsDefaultIsRefused(databaseType: DatabaseType) { + let refused = refusal { + _ = try factory(databaseType: databaseType, driver: RestoringDriver()) + .restoreStatements(rows: [["1", "__DEFAULT__", "2026-01-01"]]) + } + #expect(refused?.kind == .insert) + #expect(refused?.reason.contains("name") == true) + } + + @Test("A deleted row a driver puts back with no such text is restored") + func restoredRowWithoutThatTextIsWritten() throws { + let restored = try factory(databaseType: .dameng, driver: RestoringDriver()) + .restoreStatements(rows: [["1", "Ada", "2026-01-01"]]) + #expect(restored.statements.first?.parameters.compactMap { $0 as? String } == ["1", "Ada", "2026-01-01"]) + } } diff --git a/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift b/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift index d4c40a9899..fdf82b33af 100644 --- a/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowWriteOperationBuilderTests.swift @@ -78,14 +78,14 @@ struct RowWriteOperationBuilderTests { #expect(now.first?.refusal == .serverComputedValue) let marker = operations( - changes: [ - cellEdit( - column: "updated_at", index: 2, from: "2026-01-01", - to: .text(PluginCellValue.defaultMarkerText) - ), - ] + changes: [cellEdit(column: "updated_at", index: 2, from: "2026-01-01", to: .columnDefault, isExpression: true)] ) #expect(marker.first?.refusal == .serverComputedValue) + + let text = operations( + changes: [cellEdit(column: "updated_at", index: 2, from: "2026-01-01", to: "__DEFAULT__")] + ) + #expect(text.first?.refusal == nil) } @Test("A generated column cannot be written back") @@ -136,8 +136,8 @@ struct RowWriteOperationBuilderTests { func serverAssignedKeyIsRefused() { let change = RowChange(rowID: .existing(0), type: .insert) let withMarker = operations( - changes: [change], - insertedRowData: [.existing(0): [.text(PluginCellValue.defaultMarkerText), "Ada", "2026-01-01"]], + changes: [RowChange(rowID: .existing(0), type: .insert, expressionColumns: [0])], + insertedRowData: [.existing(0): [.columnDefault, "Ada", "2026-01-01"]], insertedRowIDs: [.existing(0)] ) #expect(withMarker.first?.refusal == .serverAssignedKey) @@ -184,6 +184,31 @@ struct RowWriteOperationBuilderTests { #expect(literal.first?.refusal == nil) } + @Test("The record keeps the order the rows were changed in after an earlier change is cancelled") + func recordFollowsEditOrder() { + var pending = PendingChanges() + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", + oldValue: "Ada", newValue: "Grace", originalRow: ["7", "Ada", "2026-01-01"] + ) + pending.recordRowDeletion(rowID: .existing(1), originalRow: ["8", "Bo", "2026-01-01"]) + let inserted = RowID.inserted(UUID()) + pending.recordRowInsertion(rowID: inserted, values: ["9", "Bo", "2026-01-01"]) + pending.recordCellChange( + rowID: .existing(0), columnIndex: 1, columnName: "name", oldValue: "Grace", newValue: "Ada" + ) + #expect(pending.changes.map { $0.type } == [.insert, .delete]) + + let result = operations( + changes: pending.changes, + insertedRowData: pending.insertedRowData, + deletedRowIDs: pending.deletedRowIDs, + insertedRowIDs: pending.insertedRowIDs + ) + + #expect(result.map { $0.kind } == [.delete, .insert]) + } + @Test("An insert carrying its own key is reversible") func userSuppliedKeyIsReversible() { let result = operations( diff --git a/apple/TableProTests/Core/Database/DatabaseManagerTests.swift b/apple/TableProTests/Core/Database/DatabaseManagerTests.swift index 1d05998bfb..10989dcb28 100644 --- a/apple/TableProTests/Core/Database/DatabaseManagerTests.swift +++ b/apple/TableProTests/Core/Database/DatabaseManagerTests.swift @@ -412,8 +412,11 @@ struct DatabaseManagerDatabaseSwitchTests { let pluginDriver = DatabaseSwitchingDriver() let adapter = PluginDriverAdapter(connection: connection, pluginDriver: pluginDriver) - await DatabaseManager.shared.restoreSchemaAndDatabase(on: adapter, savedSchema: "idb", savedDatabase: "ice") + let warning = await DatabaseManager.shared.restoreSchemaAndDatabase( + on: adapter, savedSchema: "idb", savedDatabase: "ice" + ) #expect(pluginDriver.switches == ["database ice", "schema idb"]) + #expect(warning == nil) } } diff --git a/apple/TableProTests/Core/Database/SchemaWriteGateDenialTests.swift b/apple/TableProTests/Core/Database/SchemaWriteGateDenialTests.swift new file mode 100644 index 0000000000..5af1cd879d --- /dev/null +++ b/apple/TableProTests/Core/Database/SchemaWriteGateDenialTests.swift @@ -0,0 +1,86 @@ +// +// SchemaWriteGateDenialTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +// A denial came back as a failed statement, so Create Database, the schema editor and Create Table +// showed the user's own Cancel at the gate as an error. +@MainActor +struct SchemaWriteGateDenialTests { + private let scope = DatabaseScope(connectionId: UUID(), database: "app", schema: nil) + + private func gateFailure(of body: () async throws -> Void) async -> GatedOperationFailure? { + do { + try await body() + return nil + } catch { + return GatedOperationFailure(error) + } + } + + private func runCreateDatabase(gate: any ExecutionGate) async -> GatedOperationFailure? { + await gateFailure { + try await DatabaseManager.shared.runContainerOperation( + description: "Create database \"sales\"", + kind: .schemaMutation, + scope: scope, + databaseType: .postgresql, + event: nil, + gate: gate + ) { _ in + Issue.record("The driver was asked after the gate said no") + } + } + } + + @Test("A Cancel at the gate ends a database operation quietly, before the driver is asked") + func containerOperationCancelIsQuiet() async { + let failure = await runCreateDatabase(gate: AlwaysDenyGate.cancelling) + + #expect(failure == .cancelledByUser) + #expect(failure?.message == nil) + } + + @Test("A refused database operation says why and is not a failed statement") + func containerOperationRefusalExplainsItself() async { + let failure = await runCreateDatabase(gate: AlwaysDenyGate(reason: "Safe Mode is read-only")) + + #expect(failure == .refused("Safe Mode is read-only")) + #expect(failure?.reportsFailure == false) + } + + @Test("A Cancel at the gate ends a schema editor apply quietly") + func containerStatementsCancelIsQuiet() async { + let failure = await gateFailure { + try await DatabaseManager.shared.runContainerStatements( + .statements([SchemaStatement(sql: "CREATE SCHEMA sales", description: "Create", isDestructive: false)]), + description: "Create Schema", + scope: scope, + databaseType: .postgresql, + event: nil, + gate: AlwaysDenyGate.cancelling + ) + } + + #expect(failure == .cancelledByUser) + } + + @Test("A Cancel at the gate ends a Create Table quietly") + func createTableCancelIsQuiet() async { + let failure = await gateFailure { + try await DatabaseManager.shared.executeCreateTable( + statements: ["CREATE TABLE invoices (id INT)"], + databaseType: .postgresql, + scope: scope, + gate: AlwaysDenyGate.cancelling + ) + } + + #expect(failure == .cancelledByUser) + #expect(failure?.message == nil) + } +} diff --git a/apple/TableProTests/Core/Database/StartupCommandWarningTests.swift b/apple/TableProTests/Core/Database/StartupCommandWarningTests.swift new file mode 100644 index 0000000000..11fbac5867 --- /dev/null +++ b/apple/TableProTests/Core/Database/StartupCommandWarningTests.swift @@ -0,0 +1,135 @@ +// +// StartupCommandWarningTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct ServerRefusal: PluginDriverError { + let pluginErrorMessage: String + let pluginSqlState: String? +} + +// A startup command can be a safety rail, so a session that opened without one says so on the connection's warning. +@MainActor +struct StartupCommandWarningTests { + @Test("A startup command the server refused is named on the session's warning, the ones that ran are not") + func refusedStartupCommandIsNamed() async { + let driver = MockDatabaseDriver() + driver.failsStatementContaining = ("SET ROLE", DatabaseError.queryFailed("role \"app_ro\" does not exist")) + + let failures = await DatabaseManager.shared.applyTimeoutAndStartupCommands( + on: driver, + queryTimeoutSeconds: 0, + startupCommands: "SET ROLE app_ro\nSET search_path = app", + connectionName: "Reporting" + ) + let warning = DatabaseManager.startupWarning(for: failures) + + #expect(failures.map { $0.statement } == ["SET ROLE app_ro"]) + #expect(warning?.contains("SET ROLE app_ro") == true) + #expect(warning?.contains("SET search_path") == false) + } + + // PostgreSQL can quote the statement's values back in its message, so only the code reaches the screen. + @Test("The warning names the server's code, never its message") + func warningShowsTheCodeNotTheMessage() { + let refusal = ServerRefusal( + pluginErrorMessage: "invalid value for parameter \"app.api_key\": \"sk-live-123\"", pluginSqlState: "22023" + ) + let failure = StartupCommandFailure(statement: "SET app.api_key = 42", error: refusal) + let warning = DatabaseManager.startupWarning(for: [failure]) + + #expect(failure.code == "22023") + #expect(warning?.contains("SET app.api_key = 42") == true) + #expect(warning?.contains("22023") == true) + #expect(warning?.contains("sk-live-123") == false) + } + + @Test("Startup commands that all ran leave no warning") + func successfulStartupCommandsLeaveNoWarning() async { + let driver = MockDatabaseDriver() + + let failures = await DatabaseManager.shared.applyTimeoutAndStartupCommands( + on: driver, queryTimeoutSeconds: 0, startupCommands: "SET search_path = app", connectionName: "Reporting" + ) + + #expect(DatabaseManager.startupWarning(for: failures) == nil) + } + + @Test("Several failed startup commands are each named") + func severalFailuresAreEachNamed() { + let warning = DatabaseManager.startupWarning(for: [ + StartupCommandFailure(statement: "SET ROLE app_ro", code: "42704"), + StartupCommandFailure(statement: "SET default_transaction_read_only = on", code: nil), + ]) + + #expect(warning?.contains("SET ROLE app_ro (42704)") == true) + #expect(warning?.contains("SET default_transaction_read_only = on") == true) + } + + // A startup command can carry a password or a key, and the banner is on screen, screen sharing included. + @Test("A failed startup command is shown only up to its first literal or credential clause") + func shownStatementStopsBeforeSecrets() { + #expect(DatabaseManager.shownStatement("SELECT set_config('app.api_key', 'sk-live-123', false)") + == "SELECT set_config(…") + #expect(DatabaseManager.shownStatement("SET ROLE reporting IDENTIFIED BY hunter2") == "SET ROLE reporting…") + #expect(DatabaseManager.shownStatement("ALTER USER app WITH PASSWORD 'x'") == "ALTER USER app WITH…") + #expect(DatabaseManager.shownStatement("SET search_path = app") == "SET search_path = app") + + let warning = DatabaseManager.startupWarning(for: [ + StartupCommandFailure(statement: "SET ROLE reporting IDENTIFIED BY hunter2", code: "28000"), + ]) + #expect(warning?.contains("hunter2") == false) + } + + @Test("A reconnect whose startup commands all ran drops the banner about the session before it") + func cleanReconnectClearsTheStartupWarning() { + let connectionId = UUID() + let manager = DatabaseManager.shared + defer { manager.reportedStartupFailures.removeValue(forKey: connectionId) } + + let failed = manager.openingWarning( + driver: MockDatabaseDriver(), + startupFailures: [StartupCommandFailure(statement: "SET ROLE app_ro", code: "42704")], + for: connectionId + ) + let clean = manager.openingWarning(driver: MockDatabaseDriver(), startupFailures: [], for: connectionId) + + #expect(failed?.contains("SET ROLE app_ro") == true) + #expect(clean == nil) + #expect(manager.reportedStartupFailures[connectionId]?.isEmpty == true) + } + + @Test("The login's warning and the startup warning are both kept") + func warningsAreJoined() { + #expect(DatabaseManager.joinedWarning(["ORA-28002", nil, "startup"]) == "ORA-28002\n\nstartup") + #expect(DatabaseManager.joinedWarning([nil, ""]) == nil) + } + + // The pool opens entries for as long as the window is open, so a failure it was already told about stays dismissed. + @Test("A pool or lane session's failed startup command reaches the window once") + func poolFailureIsReportedOnce() { + let connection = TestFixtures.makeConnection(name: "Startup rail") + let manager = DatabaseManager.shared + manager.injectSession(ConnectionSession(connection: connection), for: connection.id) + defer { manager.removeSession(for: connection.id) } + let role = StartupCommandFailure(statement: "SET ROLE app_ro", code: "42704") + let path = StartupCommandFailure(statement: "SET search_path = app", code: "3F000") + + manager.reportStartupFailures([role], for: connection.id) + #expect(manager.session(for: connection.id)?.connectionWarning?.contains("SET ROLE app_ro") == true) + + manager.dismissConnectionWarning(for: connection.id) + manager.reportStartupFailures([role], for: connection.id) + #expect(manager.session(for: connection.id)?.connectionWarning == nil) + + manager.reportStartupFailures([role, path], for: connection.id) + let warning = manager.session(for: connection.id)?.connectionWarning + #expect(warning?.contains("SET search_path = app") == true) + #expect(warning?.contains("SET ROLE") == false) + } +} diff --git a/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift b/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift index e67a5599bd..0b19025ea9 100644 --- a/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift +++ b/apple/TableProTests/Core/Database/TriggerInfoMappingTests.swift @@ -437,3 +437,89 @@ struct TriggerEditSafetyTests { #expect(requests.isEmpty) } } + +// A Cancel in the gate's confirmation reached the trigger editor as an error row under the SQL. +@MainActor +struct TriggerGateDenialTests { + private func install() async throws -> (connection: DatabaseConnection, scope: DatabaseScope, stub: StubTriggerDriver) { + let connection = TestFixtures.makeConnection(database: "app", type: .postgresql) + let stub = StubTriggerDriver() + stub.dropToReturn = "DROP TRIGGER t" + let adapter = PluginDriverAdapter(connection: connection, pluginDriver: stub) + try await adapter.connect() + DatabaseManager.shared.injectSession(ConnectionSession(connection: connection, driver: adapter), for: connection.id) + let scope = DatabaseScope(connectionId: connection.id, database: "app", schema: nil) + MetadataConnectionPool.shared.injectEntry(adapter, scope: scope) + return (connection, scope, stub) + } + + private func uninstall(_ connection: DatabaseConnection) { + MetadataConnectionPool.shared.closeAll(connectionId: connection.id) + DatabaseManager.shared.removeSession(for: connection.id) + } + + private func failure(of body: () async throws -> Void) async -> GatedOperationFailure? { + do { + try await body() + return nil + } catch { + return GatedOperationFailure(error) + } + } + + @Test("A Cancel at the gate ends a trigger save and a trigger drop quietly, with nothing run") + func cancelIsQuiet() async throws { + let installed = try await install() + defer { uninstall(installed.connection) } + + let save = await failure { + try await TriggerEditing.apply( + scope: installed.scope, + connection: installed.connection, + tableName: "orders", + sql: "CREATE TRIGGER t", + isEdit: false, + originalName: nil, + originalDefinition: nil, + gate: AlwaysDenyGate.cancelling + ) + } + let drop = await failure { + try await TriggerEditing.drop( + scope: installed.scope, + connection: installed.connection, + tableName: "orders", + name: "t", + gate: AlwaysDenyGate.cancelling + ) + } + + #expect(save == .cancelledByUser) + #expect(drop == .cancelledByUser) + #expect(save?.message == nil) + #expect(installed.stub.executedQueries.isEmpty) + } + + @Test("A refused trigger save says why and is not a failed statement") + func refusalExplainsItself() async throws { + let installed = try await install() + defer { uninstall(installed.connection) } + + let save = await failure { + try await TriggerEditing.apply( + scope: installed.scope, + connection: installed.connection, + tableName: "orders", + sql: "CREATE TRIGGER t", + isEdit: false, + originalName: nil, + originalDefinition: nil, + gate: AlwaysDenyGate(reason: "Safe Mode is read-only") + ) + } + + #expect(save == .refused("Safe Mode is read-only")) + #expect(save?.reportsFailure == false) + #expect(installed.stub.executedQueries.isEmpty) + } +} diff --git a/apple/TableProTests/Core/LSP/LSPTransportBrokenPipeTests.swift b/apple/TableProTests/Core/LSP/LSPTransportBrokenPipeTests.swift new file mode 100644 index 0000000000..7af8c30567 --- /dev/null +++ b/apple/TableProTests/Core/LSP/LSPTransportBrokenPipeTests.swift @@ -0,0 +1,48 @@ +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +struct LSPTransportBrokenPipeTests { + static let perl = "/usr/bin/perl" + + static var hasPerl: Bool { + FileManager.default.isExecutableFile(atPath: perl) + } + + // A server on its way out: it stops reading, asks the client something, then says it has stopped. + private static let closingServer = #""" + close STDIN; $| = 1; + for my $body ('{"jsonrpc":"2.0","id":7,"method":"workspace/configuration","params":{}}', + '{"jsonrpc":"2.0","method":"test/inputClosed"}') { + print "Content-Length: " . length($body) . "\r\n\r\n" . $body; + } + sleep 30; + """# + + @Test( + "A server that stopped reading fails the request instead of ending the app", + .enabled(if: LSPTransportBrokenPipeTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func closedInputFailsTheRequest() async throws { + let transport = LSPTransport() + let inputClosed = AsyncGate() + await transport.onNotification(method: "test/inputClosed") { _ in + Task { await inputClosed.open() } + } + try await transport.start(executablePath: Self.perl, arguments: ["-e", Self.closingServer]) + await inputClosed.wait() + + let error = await #expect(throws: (any Error).self) { + _ = try await transport.sendRequest(method: "test/afterClose", params: EmptyLSPParams()) + } + await transport.stop() + + let failure = error.map { $0 as NSError } + let posix = failure?.domain == NSPOSIXErrorDomain + ? failure + : failure?.underlyingErrors.first.map { $0 as NSError } + #expect(posix?.code == Int(EPIPE)) + } +} diff --git a/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCutTests.swift b/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCutTests.swift new file mode 100644 index 0000000000..c9e9c4e532 --- /dev/null +++ b/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCutTests.swift @@ -0,0 +1,107 @@ +// +// ObjectCopyRowCutTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private final class CappedSourceDriver: PluginDriverStub, @unchecked Sendable { + let declaredLimit: Int? + let rowCount: Int + + init(declaredLimit: Int?, rowCount: Int) { + self.declaredLimit = declaredLimit + self.rowCount = rowCount + } + + func streamRows(query: String) -> AsyncThrowingStream { + var header = PluginStreamHeader(columns: ["id", "name"], columnTypeNames: []) + header.rowLimit = declaredLimit + let rows: [PluginRow] = (0 ..< rowCount).map { [.text(String($0)), .text("row\($0)")] } + return AsyncThrowingStream { continuation in + continuation.yield(.header(header)) + continuation.yield(.rows(rows)) + continuation.finish() + } + } +} + +private final class WritingTargetDriver: PluginDriverStub, @unchecked Sendable { + var parameterStyle: ParameterStyle { .questionMark } + + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + + func quoteIdentifier(_ name: String) -> String { "`\(name)`" } +} + +struct ObjectCopyRowCutTests { + private func step() -> ObjectCopyTableStep { + ObjectCopyTableStep( + selection: ObjectCopySelection(kind: .table, name: "orders", schema: "public"), + dropStatements: [], + sequenceStatements: [], + createStatements: [], + truncateStatements: [], + columns: ["id", "name"], + targetColumnTypeNames: [:], + primaryKeyColumns: ["id"], + sourceQuery: "SELECT `id`, `name` FROM `public`.`orders`", + targetTable: "orders", + targetSchema: "public", + estimatedRows: nil, + copiesData: true, + copiesIdentityColumn: false, + coercer: nil, + serverSideInsert: nil, + note: nil + ) + } + + private func copy(from source: CappedSourceDriver) async throws -> ObjectCopyRowCopier.Outcome { + try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) + .copy(from: source, to: WritingTargetDriver()) { _ in } + } + + // A 6,000,000-row SAP HANA table arrived as 5,000,000 rows and the copy reported success. + @Test("A copy whose source stream stopped at the driver's limit says where") + func cutStreamIsReported() async throws { + let outcome = try await copy(from: CappedSourceDriver(declaredLimit: 3, rowCount: 3)) + + #expect(outcome.inserted == 3) + #expect(outcome.rowCut == 3) + } + + @Test("A copy that read fewer rows than the driver's limit is whole") + func shortStreamIsWhole() async throws { + let outcome = try await copy(from: CappedSourceDriver(declaredLimit: 10, rowCount: 3)) + + #expect(outcome.inserted == 3) + #expect(outcome.rowCut == nil) + } + + @Test("A cut table's outcome carries the export's wording") + func cutOutcomeCarriesTheWarning() { + let selection = ObjectCopySelection(kind: .table, name: "orders", schema: "public") + let result = ObjectCopyRunResult( + outcomes: [ + ObjectCopyObjectOutcome( + selection: selection, + rowsCopied: 3, + error: nil, + warning: StreamRowCutWarning.text(table: selection.qualifiedName, limit: 3) + ) + ], + rowsCopied: 3, + cancelled: false, + createdDatabase: nil + ) + + #expect(result.warnings == [StreamRowCutWarning.text(table: selection.qualifiedName, limit: 3)]) + #expect(result.failedCount == 0) + } +} diff --git a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolTests.swift b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolTests.swift index 0a184ad353..04b349fbe9 100644 --- a/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolTests.swift +++ b/apple/TableProTests/Core/Services/Query/MetadataConnectionPoolTests.swift @@ -84,7 +84,7 @@ struct MetadataConnectionPoolTests { func prepareSessionPassesThrough() async throws { let driver = MockDatabaseDriver() - try await MetadataConnectionPool.prepareSession( + let failures = try await MetadataConnectionPool.prepareSession( driver, queryTimeoutSeconds: 30, startupCommands: nil, @@ -93,6 +93,25 @@ struct MetadataConnectionPoolTests { ) #expect(driver.applyQueryTimeoutValues == [30]) + #expect(failures.isEmpty) + } + + // A pool entry or a lane whose `SET ROLE` failed would otherwise run with the login's own rights unannounced. + @Test("session preparation hands back the startup commands that failed") + func prepareSessionReturnsFailedStartupCommands() async throws { + let driver = MockDatabaseDriver() + driver.failsStatementContaining = ("SET ROLE", DatabaseError.queryFailed("role \"app_ro\" does not exist")) + + let failures = try await MetadataConnectionPool.prepareSession( + driver, + queryTimeoutSeconds: 0, + startupCommands: "SET ROLE app_ro\nSET search_path = app", + connectionName: "test", + timeoutSeconds: 1 + ) + + #expect(failures.map { $0.statement } == ["SET ROLE app_ro"]) + #expect(driver.executedQueries == ["SET ROLE app_ro", "SET search_path = app"]) } @Test("database switch rejects a driver that cannot switch") diff --git a/apple/TableProTests/Core/Services/Query/SelectionSummaryCalculatorTests.swift b/apple/TableProTests/Core/Services/Query/SelectionSummaryCalculatorTests.swift index 2f7e00976f..953b41e5c4 100644 --- a/apple/TableProTests/Core/Services/Query/SelectionSummaryCalculatorTests.swift +++ b/apple/TableProTests/Core/Services/Query/SelectionSummaryCalculatorTests.swift @@ -25,8 +25,7 @@ struct SelectionSummaryCalculatorTests { dataColumns: [Int]? = nil, policy: SelectionSummaryColumnPolicy? = nil, deleted: Set = [], - inserted: Set = [], - modified: [RowID: Set] = [:] + defaulted: [RowID: Set] = [:] ) -> SelectionSummaryInput { SelectionSummaryInput( selection: selection(rects), @@ -39,8 +38,7 @@ struct SelectionSummaryCalculatorTests { columnCount: tableRows.columns.count ), deletedRowIDs: deleted, - insertedRowIDs: inserted, - modifiedCells: modified + defaultedCells: defaulted ) } @@ -113,25 +111,24 @@ struct SelectionSummaryCalculatorTests { #expect(summary.numbers?.sum.decimal == 1) } - @Test("NULL is empty, empty text is a value, and the default marker is empty only where an edit put it") + @Test("NULL is empty, empty text is a value, and DEFAULT is empty only where an edit marked it") func emptyCellRules() async throws { let insertedID = RowID.inserted(UUID()) var table = Self.tableRows( [ [.null, .text("")], - [.text(PluginCellValue.defaultMarkerText), .text(PluginCellValue.defaultMarkerText)], - [.text("7"), .text(PluginCellValue.defaultMarkerText)] + [.text("DEFAULT"), .text("DEFAULT")], + [.text("7"), .columnDefault] ], types: [Self.integer, Self.untyped] ) - table.appendInsertedRow(id: insertedID, values: [.text(PluginCellValue.defaultMarkerText), .text("5")]) + table.appendInsertedRow(id: insertedID, values: [.columnDefault, .text("5")]) let summary = try await Self.summarize( Self.input( table, selecting: [GridRect(rows: 0...3, columns: 0...1)], - inserted: [insertedID], - modified: [.existing(2): [1]] + defaulted: [.existing(2): [1], insertedID: [0]] ) ) diff --git a/apple/TableProTests/Core/Services/RowOperationsFieldAbsenceTests.swift b/apple/TableProTests/Core/Services/RowOperationsFieldAbsenceTests.swift index a0e0ecf3b2..2823bea961 100644 --- a/apple/TableProTests/Core/Services/RowOperationsFieldAbsenceTests.swift +++ b/apple/TableProTests/Core/Services/RowOperationsFieldAbsenceTests.swift @@ -82,7 +82,7 @@ extension GlobalStateSuites { #expect(tableRows.row(withID: nullKept.rowID)?.absentColumns.isEmpty == true) #expect(tableRows.row(withID: missingKept.rowID)?.absentColumns == [2]) #expect(changes.pending.insertedAbsentColumns(forRow: missingKept.rowID) == [2]) - #expect(missingKept.values[0] == .text("__DEFAULT__")) + #expect(missingKept.values[0] == .columnDefault) } @Test("Copying rows records which fields each lacked, in the copied column order") @@ -123,7 +123,7 @@ extension GlobalStateSuites { let (tableRows, result) = paste(clipboard, into: manager) - #expect(result.pastedRows.first?.values == [.text("__DEFAULT__"), "Ada", .null]) + #expect(result.pastedRows.first?.values == [.columnDefault, "Ada", .null]) #expect(tableRows.rows[0].absentColumns.isEmpty) } diff --git a/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift b/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift index 80819290f7..51e4d37d67 100644 --- a/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift +++ b/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift @@ -51,7 +51,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) #expect(values.count == 3) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("Smith, John")) #expect(values[2] == .text("a@test.com")) } @@ -120,7 +120,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("Jo")) #expect(values[2] == .text("e@x.com")) } @@ -136,7 +136,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) #expect(values.count == 3) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("Jo")) #expect(values[2] == .text("e@x.com")) } @@ -151,7 +151,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("2")) #expect(values[2] == .text("3")) } @@ -170,8 +170,8 @@ struct RowOperationsManagerPasteTests { let rows = pasteRows(clipboard, manager: makeManager()) #expect(rows.count == 2) - #expect(rows[0] == [.text("__DEFAULT__"), .text("Alice"), .text("a@test.com")]) - #expect(rows[1] == [.text("__DEFAULT__"), .text("Bob"), .null]) + #expect(rows[0] == [.columnDefault, .text("Alice"), .text("a@test.com")]) + #expect(rows[1] == [.columnDefault, .text("Bob"), .null]) } @Test("Plain-text TSV with commas inside a value parses as TSV, not CSV") @@ -182,7 +182,7 @@ struct RowOperationsManagerPasteTests { let values = pasteValues(clipboard, manager: makeManager()) #expect(values.count == 3) - #expect(values[0] == .text("__DEFAULT__")) + #expect(values[0] == .columnDefault) #expect(values[1] == .text("Mozilla/5.0 (KHTML, like Gecko) Safari")) #expect(values[2] == .text("a@test.com")) } @@ -257,7 +257,7 @@ struct RowOperationsManagerPasteServerOwnedTests { func resetsNonKeyIdentity() { let values = paste(payload, columnIdentity: ["code": .always]) - #expect(values[1] == .text("__DEFAULT__")) + #expect(values[1] == .columnDefault) #expect(values[2] == .text("Ada")) } @@ -265,7 +265,7 @@ struct RowOperationsManagerPasteServerOwnedTests { func resetsGeneratedColumn() { let values = paste(payload, generatedColumns: ["code"]) - #expect(values[1] == .text("__DEFAULT__")) + #expect(values[1] == .columnDefault) #expect(values[2] == .text("Ada")) } diff --git a/apple/TableProTests/Core/Services/RowOperationsManagerTests.swift b/apple/TableProTests/Core/Services/RowOperationsManagerTests.swift index 7358b7b600..c56df2acad 100644 --- a/apple/TableProTests/Core/Services/RowOperationsManagerTests.swift +++ b/apple/TableProTests/Core/Services/RowOperationsManagerTests.swift @@ -118,8 +118,8 @@ struct RowOperationsManagerTests { let result = manager.addNewRow(tableRows: &tableRows) #expect(result != nil) - #expect(result?.values[0] == "__DEFAULT__") - #expect(result?.values[2] == "__DEFAULT__") + #expect(result?.values[0] == .columnDefault) + #expect(result?.values[2] == .columnDefault) } // PostgreSQL leaves `column_default` null for an identity column, so a new row sent an explicit NULL it refuses. @@ -133,7 +133,7 @@ struct RowOperationsManagerTests { let result = manager.addNewRow(tableRows: &tableRows) - #expect(result?.values[0] == "__DEFAULT__") + #expect(result?.values[0] == .columnDefault) #expect(result?.values[1] == .null) } @@ -146,7 +146,7 @@ struct RowOperationsManagerTests { let result = manager.addNewRow(tableRows: &tableRows) - #expect(result?.values[2] == "__DEFAULT__") + #expect(result?.values[2] == .columnDefault) #expect(result?.values[1] == .null) } @@ -159,7 +159,7 @@ struct RowOperationsManagerTests { let result = manager.addNewRow(tableRows: &tableRows) #expect(result?.values[1] == .null) - #expect(result?.values[2] == "__DEFAULT__") + #expect(result?.values[2] == .columnDefault) #expect(tableRows.serverAssignsValue(forColumn: "name") == false) } @@ -246,7 +246,7 @@ struct RowOperationsManagerTests { ) #expect(result != nil) - #expect(result?.values[0] == "__DEFAULT__") + #expect(result?.values[0] == .columnDefault) if case .rowsInserted(let indices) = result?.delta { #expect(indices == IndexSet(integer: 3)) } else { @@ -263,10 +263,40 @@ struct RowOperationsManagerTests { let result = manager.duplicateRow(sourceRowIndex: 1, tableRows: &tableRows) - #expect(result?.values[2] == "__DEFAULT__") + #expect(result?.values[2] == .columnDefault) #expect(result?.values[1] == sourceValues[1]) } + @Test("The change manager names the cells a new row leaves to their default, for the JSON view and Copy as JSON") + func defaultedCellsOfANewRow() throws { + let (manager, changeManager) = makeManager() + var tableRows = emptyTableRows(columnDefaults: ["id": "auto_increment", "email": "'x'"]) + + let added = manager.addNewRow(tableRows: &tableRows) + let result = try #require(added) + + #expect(changeManager.defaultedCells(in: tableRows) == [result.rowID: [0, 2]]) + } + + @Test("A duplicate writes stored text that reads like the default as text, and leaves only its key to the server") + func duplicateRowKeepsDefaultLookingText() throws { + let (manager, changeManager) = makeManager() + var tableRows = TableRows.from( + queryRows: [["7", "__DEFAULT__", "DEFAULT"]], + columns: Self.testColumns, + columnTypes: Self.testColumnTypes + ) + + let duplicated = manager.duplicateRow(sourceRowIndex: 0, tableRows: &tableRows) + let result = try #require(duplicated) + + #expect(changeManager.expressionColumns(forRow: result.rowID) == [0]) + let statements = try changeManager.generateSQL() + #expect(statements.count == 1) + #expect(statements.first?.sql.contains("`id`") == false) + #expect(statements.first?.parameters.compactMap { $0 as? String } == ["__DEFAULT__", "DEFAULT"]) + } + @Test("duplicateRow returns nil for invalid source index") func duplicateRowReturnsNilForInvalidIndex() { let (manager, _) = makeManager() diff --git a/apple/TableProTests/Core/Services/RowOperationsManagerUUIDKeyTests.swift b/apple/TableProTests/Core/Services/RowOperationsManagerUUIDKeyTests.swift index 5601c64f35..4122e13d78 100644 --- a/apple/TableProTests/Core/Services/RowOperationsManagerUUIDKeyTests.swift +++ b/apple/TableProTests/Core/Services/RowOperationsManagerUUIDKeyTests.swift @@ -80,7 +80,7 @@ struct RowOperationsManagerUUIDKeyTests { let result = try #require(manager.addNewRow(tableRows: &rows, uuidKeyForms: Self.uuidKey)) - #expect(result.values[0] == .text("__DEFAULT__")) + #expect(result.values[0] == .columnDefault) #expect(result.prefilledColumns.isEmpty) } @@ -99,7 +99,7 @@ struct RowOperationsManagerUUIDKeyTests { #expect(copyKey != sourceKey) #expect(copy.values[1] == .text("a")) #expect(copy.prefilledColumns == [0]) - #expect(plain.values[0] == .text("__DEFAULT__")) + #expect(plain.values[0] == .columnDefault) } @Test("Duplicating a pending row keeps the SQL staged in it, except in the columns the copy resets") @@ -118,14 +118,16 @@ struct RowOperationsManagerUUIDKeyTests { #expect(uuidText(copy.values[0]) != nil) } - @Test("Duplicating a saved row stages no SQL") + @Test("Duplicating a saved row stages no SQL but the default its reset key takes") func duplicateOfASavedRowHasNoMarks() throws { let (manager, changeManager) = makeManager() var rows = makeRows([[.text("0b1e7c9a-58f4-4b8e-9c55-1f1c86f0a001"), .text("now()"), .null]]) let copy = try #require(manager.duplicateRow(sourceRowIndex: 0, tableRows: &rows)) - #expect(changeManager.expressionColumns(forRow: copy.rowID).isEmpty) + #expect(changeManager.expressionColumns(forRow: copy.rowID) == [0]) + #expect(copy.values[0] == .columnDefault) + #expect(copy.values[1] == .text("now()")) } // MARK: - Paste rows @@ -188,7 +190,7 @@ struct RowOperationsManagerUUIDKeyTests { clipboard: clipboard ) - #expect(result.pastedRows.first?.values[0] == .text("__DEFAULT__")) + #expect(result.pastedRows.first?.values[0] == .columnDefault) } } @@ -249,8 +251,8 @@ extension GlobalStateSuites { let pasted = result.pastedRows.map(\.rowID) #expect(pasted.count == 2) - #expect(changeManager.expressionColumns(forRow: pasted[0]).isEmpty) - #expect(changeManager.expressionColumns(forRow: pasted[1]) == [2]) + #expect(changeManager.expressionColumns(forRow: pasted[0]) == [0]) + #expect(changeManager.expressionColumns(forRow: pasted[1]) == [0, 2]) #expect(result.pastedRows[0].values[2] == .text("CURRENT_TIMESTAMP")) #expect(result.pastedRows[1].values[2] == .text("now()")) } @@ -268,8 +270,9 @@ extension GlobalStateSuites { let result = try paste(manager, into: &rows) - let rowID = try #require(result.pastedRows.first?.rowID) - #expect(changeManager.expressionColumns(forRow: rowID) == [2]) + let pastedRow = try #require(result.pastedRows.first) + #expect(pastedRow.values[0] == .columnDefault) + #expect(changeManager.expressionColumns(forRow: pastedRow.rowID) == [0, 2]) } @Test("Cells the grid copied as wide as the table paste as rows with their marks") @@ -287,8 +290,8 @@ extension GlobalStateSuites { let pasted = result.pastedRows.map(\.rowID) #expect(pasted.count == 2) - #expect(changeManager.expressionColumns(forRow: pasted[0]) == [2]) - #expect(changeManager.expressionColumns(forRow: pasted[1]).isEmpty) + #expect(changeManager.expressionColumns(forRow: pasted[0]) == [0, 2]) + #expect(changeManager.expressionColumns(forRow: pasted[1]) == [0]) } } } diff --git a/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift b/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift index 9c1a9b1dd4..e886b4cf83 100644 --- a/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift +++ b/apple/TableProTests/Core/Services/RowPasteForeignTextTests.swift @@ -22,8 +22,8 @@ struct RowPasteForeignTextTests { let rows = try parse("\tAnn\tann@x.com\n\tBob\tbob@x.com") #expect(rows == [ - [.text("__DEFAULT__"), .text("Ann"), .text("ann@x.com")], - [.text("__DEFAULT__"), .text("Bob"), .text("bob@x.com")] + [.columnDefault, .text("Ann"), .text("ann@x.com")], + [.columnDefault, .text("Bob"), .text("bob@x.com")] ]) } @@ -74,7 +74,8 @@ struct RowPasteForeignTextTests { columns: Self.columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard ) - #expect(result.pastedRows.first?.values == [.text("__DEFAULT__"), .text("Ann"), .text("ann@x.com")]) + #expect(result.pastedRows.first?.values == [.columnDefault, .text("Ann"), .text("ann@x.com")]) + #expect(result.pastedRows.first.map { changeManager.expressionColumns(forRow: $0.rowID) } == [0]) } @Test("A blank pasted cell takes the column's default, an empty string where NULL is refused, else NULL") @@ -107,11 +108,12 @@ struct RowPasteForeignTextTests { columns: columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard ) - let unfilled: [PluginCellValue] = [.text("__DEFAULT__"), .text(""), .null, .null] + let unfilled: [PluginCellValue] = [.columnDefault, .text(""), .null, .null] #expect(result.pastedRows.map { $0.values } == [ - [.text("__DEFAULT__"), .text("Ann")] + unfilled, - [.text("__DEFAULT__"), .text("Bob")] + unfilled, + [.columnDefault, .text("Ann")] + unfilled, + [.columnDefault, .text("Bob")] + unfilled, ]) + #expect(result.pastedRows.map { changeManager.expressionColumns(forRow: $0.rowID) } == [[0, 2], [0, 2]]) } @Test("A pasted NULL stays NULL in a column with a default") @@ -136,7 +138,7 @@ struct RowPasteForeignTextTests { columns: columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard ) - #expect(result.pastedRows.first?.values == [.text("__DEFAULT__"), .null]) + #expect(result.pastedRows.first?.values == [.columnDefault, .null]) } } diff --git a/apple/TableProTests/Core/Shortcuts/PayloadRowPlannerHeaderMatchTests.swift b/apple/TableProTests/Core/Shortcuts/PayloadRowPlannerHeaderMatchTests.swift new file mode 100644 index 0000000000..a34eb3e553 --- /dev/null +++ b/apple/TableProTests/Core/Shortcuts/PayloadRowPlannerHeaderMatchTests.swift @@ -0,0 +1,59 @@ +// +// PayloadRowPlannerHeaderMatchTests.swift +// TableProTests +// + +import Foundation +import Testing + +@testable import TablePro + +struct PayloadRowPlannerHeaderMatchTests { + private func columns(_ names: [String]) -> [PayloadColumn] { + names.map { PayloadColumn(name: $0, isPrimaryKey: false, isServerOwned: false) } + } + + private func planned(_ rows: [PayloadRow], into names: [String]) throws -> [[String]] { + try PayloadRowPlanner.values(table: "people", columns: columns(names), rows: rows) + .map { row in row.map { "\($0.column)=\($0.value ?? "NULL")" } } + } + + // SQL Server exports its headers bracketed, and Shortcuts has no sheet to map them by hand. + @Test("A CSV with bracketed headers inserts into the columns they name") + func bracketedHeadersFindTheirColumns() async throws { + let rows = try await RowPayload.parse(data: "[id],[name]\n1,Bob\n", file: nil) + #expect(try planned(rows, into: ["id", "name"]) == [["id=1", "name=Bob"]]) + } + + @Test("A doubled closing bracket inside the brackets is one bracket of the name") + func doubledBracketIsUnescaped() throws { + let row = PayloadRow(values: ["[a]]b]": .text("x")]) + #expect(try planned([row], into: ["a]b"]) == [["a]b=x"]]) + } + + @Test("A lone closing bracket inside means the key is not a bracketed name") + func loneBracketIsNotUnwrapped() { + #expect(throws: IntentDataError.unknownColumns(["[a]b]"], "people")) { + _ = try planned([PayloadRow(values: ["[a]b]": .text("x")])], into: ["a]b"]) + } + } + + @Test("A key differing only in case finds its column, as the import sheet matches headers") + func caseInsensitiveMatch() throws { + #expect(try planned([PayloadRow(values: ["Name": .text("Ada")])], into: ["name"]) == [["name=Ada"]]) + } + + @Test("A case match two columns share is refused rather than guessed") + func ambiguousCaseMatchIsUnknown() { + #expect(throws: IntentDataError.unknownColumns(["NAME"], "people")) { + _ = try planned([PayloadRow(values: ["NAME": .text("Ada")])], into: ["Name", "name"]) + } + } + + @Test("A key spelled exactly claims its column before a bracketed one can") + func exactSpellingClaimsFirst() { + #expect(throws: IntentDataError.unknownColumns(["[id]"], "people")) { + _ = try planned([PayloadRow(values: ["id": .text("1"), "[id]": .text("2")])], into: ["id"]) + } + } +} diff --git a/apple/TableProTests/Core/Shortcuts/RowPayloadFormatTests.swift b/apple/TableProTests/Core/Shortcuts/RowPayloadFormatTests.swift new file mode 100644 index 0000000000..f5f9743a48 --- /dev/null +++ b/apple/TableProTests/Core/Shortcuts/RowPayloadFormatTests.swift @@ -0,0 +1,60 @@ +// +// RowPayloadFormatTests.swift +// TableProTests +// + +import Foundation +import Testing + +@testable import TablePro + +struct RowPayloadFormatTests { + // SQL Server exports its headers bracketed, so a valid CSV opens with `[`. + @Test("CSV with bracketed headers is read as CSV") + func bracketedHeaderIsCSV() async throws { + let rows = try await RowPayload.parse(data: "[id],[name]\n1,Bob\n", file: nil) + #expect(rows == [PayloadRow(values: ["[id]": .text("1"), "[name]": .text("Bob")])]) + } + + @Test("CSV whose first header opens with a brace is read as CSV") + func braceHeaderIsCSV() async throws { + let rows = try await RowPayload.parse(data: "{key},value\na,1\n", file: nil) + #expect(rows == [PayloadRow(values: ["{key}": .text("a"), "value": .text("1")])]) + } + + @Test("A JSON object and a JSON array of objects are still read as JSON") + func jsonStaysJSON() async throws { + let object = try await RowPayload.parse(data: #"{"name":"Ada"}"#, file: nil) + #expect(object == [PayloadRow(values: ["name": .text("Ada")])]) + let array = try await RowPayload.parse(data: #"[{"name":"Ada"},{"name":"Grace"}]"#, file: nil) + #expect(array.count == 2) + } + + @Test("Malformed JSON that is no CSV either reports the JSON error") + func malformedJSONKeepsItsError() async { + await #expect { + _ = try await RowPayload.parse(data: #"{"name": tru}"#, file: nil) + } throws: { error in + guard case .malformedPayload = error as? IntentDataError else { return false } + return true + } + } + + // Each line splits into the same number of CSV fields, so the text read as CSV with `{"id": 1` for a header. + @Test("JSON with a trailing comma after each line's object keeps the JSON error") + func jsonThatSplitsEvenlyKeepsItsError() async { + await #expect { + _ = try await RowPayload.parse(data: "{\"id\": 1, \"name\": \"a\"},\n{\"id\": 2, \"name\": \"b\"},\n", file: nil) + } throws: { error in + guard case .malformedPayload = error as? IntentDataError else { return false } + return true + } + } + + @Test("A JSON array of something other than objects keeps its own error") + func jsonArrayOfValuesIsNotRetriedAsCSV() async { + await #expect(throws: IntentDataError.jsonArrayContainsNonObject) { + _ = try await RowPayload.parse(data: "[1,2]", file: nil) + } + } +} diff --git a/apple/TableProTests/Core/UsersRoles/PrincipalApplyOpenTransactionTests.swift b/apple/TableProTests/Core/UsersRoles/PrincipalApplyOpenTransactionTests.swift index 4cb996f0e4..0ee582a754 100644 --- a/apple/TableProTests/Core/UsersRoles/PrincipalApplyOpenTransactionTests.swift +++ b/apple/TableProTests/Core/UsersRoles/PrincipalApplyOpenTransactionTests.swift @@ -62,3 +62,47 @@ struct PrincipalApplyOpenTransactionTests { #expect(driver.executedQueries.isEmpty) } } + +// The review sheet showed a Cancel at the gate as a failed apply. +@MainActor +struct PrincipalApplyGateDenialTests { + private func apply(gate: any ExecutionGate) async -> (failure: GatedOperationFailure?, executed: [String]) { + let driver = TransactionHoldingPrincipalDriver(rollsBack: true) + let connection = TestFixtures.makeConnection(type: .postgresql) + DatabaseManager.shared.injectSession( + ConnectionSession(connection: connection, driver: PluginDriverAdapter(connection: connection, pluginDriver: driver)), + for: connection.id + ) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let change = PrincipalChange.create(PluginPrincipalDefinition(ref: PluginPrincipalRef(name: "X"), password: "p")) + do { + try await DatabaseManager.shared.executePrincipalChanges( + changes: [change], + databaseType: .postgresql, + connectionId: connection.id, + gate: gate + ) + return (nil, driver.executedQueries) + } catch { + return (GatedOperationFailure(error), driver.executedQueries) + } + } + + @Test("A Cancel at the gate ends the apply quietly, with nothing run") + func cancelIsQuiet() async { + let result = await apply(gate: AlwaysDenyGate.cancelling) + + #expect(result.failure == .cancelledByUser) + #expect(result.failure?.message == nil) + #expect(result.executed.isEmpty) + } + + @Test("A refused apply says why and is not a failed statement") + func refusalExplainsItself() async { + let result = await apply(gate: AlwaysDenyGate(reason: "Safe Mode is read-only")) + + #expect(result.failure == .refused("Safe Mode is read-only")) + #expect(result.executed.isEmpty) + } +} diff --git a/apple/TableProTests/Core/UsersRoles/PrincipalChangeManagerTests.swift b/apple/TableProTests/Core/UsersRoles/PrincipalChangeManagerTests.swift index fcdd2c5053..d8c628c9eb 100644 --- a/apple/TableProTests/Core/UsersRoles/PrincipalChangeManagerTests.swift +++ b/apple/TableProTests/Core/UsersRoles/PrincipalChangeManagerTests.swift @@ -329,6 +329,46 @@ struct PrincipalChangeManagerTests { #expect(manager.changeCount == 1) } + @Test("A staged drop runs only the DROP and counts as one change") + func dropSupersedesOtherEdits() { + let manager = makeManager() + manager.stageAlter(PluginPrincipalDefinition(ref: alice, canLogin: false), for: alice) + manager.stageSetPassword("s3cret", for: alice) + manager.setGranted(true, privilege: "CREATE", scope: app, for: alice) + #expect(manager.changeCount == 3) + + manager.stageDrop(alice, options: PluginPrincipalDropOptions()) + + let changes = manager.pendingChanges() + #expect(changes.count == 1) + guard case .drop(let ref, _) = changes.first else { + Issue.record("Expected only the DROP, got \(changes)") + return + } + #expect(ref == alice) + #expect(manager.changeCount == 1) + } + + @Test("Undoing a staged drop brings the principal's other edits back") + func unstagingDropRestoresEdits() { + let manager = makeManager() + manager.stageAlter(PluginPrincipalDefinition(ref: alice, canLogin: false), for: alice) + manager.stageSetPassword("s3cret", for: alice) + manager.setGranted(true, privilege: "CREATE", scope: app, for: alice) + manager.stageDrop(alice, options: PluginPrincipalDropOptions()) + + manager.unstageDrop(alice) + + let changes = manager.pendingChanges() + let dropsAnything = changes.contains { change in + if case .drop = change { return true } + return false + } + #expect(changes.count == 3) + #expect(!dropsAnything) + #expect(manager.changeCount == 3) + } + @Test("Dropping the connected account is reported as self-impact") func reportsSelfImpact() { let manager = makeManager() diff --git a/apple/TableProTests/Core/Utilities/JsonRowConverterTests.swift b/apple/TableProTests/Core/Utilities/JsonRowConverterTests.swift index 759614a413..0e3f59c5d3 100644 --- a/apple/TableProTests/Core/Utilities/JsonRowConverterTests.swift +++ b/apple/TableProTests/Core/Utilities/JsonRowConverterTests.swift @@ -384,20 +384,16 @@ struct JsonRowConverterTests { // MARK: - Default-value marker - /// A pending inserted row carries the default marker for columns the server fills in. The grid - /// draws it as a placeholder, so JSON has to agree instead of printing the internal token. - @Test("The default-value marker renders as null, not as literal text") - func defaultMarkerRendersAsNull() { + // The default is marked beside the value, so text a table holds that spells the old marker is a value. + @Test("Stored text that spells the old default marker renders as that text") + func oldMarkerTextIsAValue() { let converter = makeConverter( - columns: ["id", "name"], - columnTypes: [.integer(rawType: nil), .text(rawType: nil)] - ) - let result = converter.generateJson( - rows: [[.text(PluginCellValue.defaultMarkerText), .text("Ada")]] + columns: ["status", "name"], + columnTypes: [.text(rawType: nil), .text(rawType: nil)] ) + let result = converter.generateJson(rows: [[.text("__DEFAULT__"), .text("Ada")]]) - #expect(!result.contains(PluginCellValue.defaultMarkerText)) - #expect(result.contains("\"id\": null")) + #expect(result.contains("\"status\": \"__DEFAULT__\"")) #expect(result.contains("\"Ada\"")) } diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierTSQLBlockTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierTSQLBlockTests.swift new file mode 100644 index 0000000000..86fc85825a --- /dev/null +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierTSQLBlockTests.swift @@ -0,0 +1,39 @@ +import Foundation +@testable import TablePro +import TableProSQLGrammar +import Testing + +// A T-SQL block runs as one statement, and a caller allowed one statement at a time is still refused one holding more. +struct QueryClassifierTSQLBlockTests { + @Test("A block holding more than one statement is multi-statement", arguments: [ + "IF @x = 1 BEGIN UPDATE a SET c = 1; UPDATE b SET c = 1; END", + "BEGIN TRY DELETE FROM a WHERE id = 1; DELETE FROM b WHERE id = 1; END TRY BEGIN CATCH SELECT 1; END CATCH", + "WHILE @i < 3 BEGIN SET @i += 1; PRINT @i; END", + "IF @x = 1 BEGIN SELECT 1; WITH c AS (SELECT id FROM t) DELETE FROM c; END", + "IF @x = 1 BEGIN SELECT 1; DISABLE TRIGGER trg ON t; END", + "BEGIN TRY SELECT 1; THROW 50000, 'x', 1; END TRY BEGIN CATCH SELECT 2; END CATCH", + "IF @x = 1 BEGIN SELECT 1; done: SELECT 2; END", + "IF @x = 1 BEGIN SELECT 1; (SELECT 2); END", + ]) + func blockOfSeveralIsMulti(sql: String) { + #expect(QueryClassifier.isMultiStatement(sql, databaseType: .mssql)) + } + + @Test("A conditional statement, or a block holding one, is a single statement", arguments: [ + "IF @x = 1 SELECT 1", + "IF @x = 1 BEGIN SELECT 1; END", + "BEGIN SELECT 1 END", + "BEGIN IF @x = 1 SELECT 1; ELSE SELECT 2; END", + "BEGIN TRY SELECT 1; END TRY BEGIN CATCH THROW; END CATCH", + ]) + func singleStatementIsNotMulti(sql: String) { + #expect(!QueryClassifier.isMultiStatement(sql, databaseType: .mssql)) + } + + @Test("A stored routine's body runs nothing, so it is one statement however many it holds") + func routineBodyIsNotMulti() { + #expect(!QueryClassifier.isMultiStatement( + "CREATE PROCEDURE dbo.p AS BEGIN UPDATE a SET c = 1; UPDATE b SET c = 1; END", databaseType: .mssql + )) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQLRowToStatementConverterTests.swift b/apple/TableProTests/Core/Utilities/SQLRowToStatementConverterTests.swift index b5df30bfd0..8fc75fa0d1 100644 --- a/apple/TableProTests/Core/Utilities/SQLRowToStatementConverterTests.swift +++ b/apple/TableProTests/Core/Utilities/SQLRowToStatementConverterTests.swift @@ -384,8 +384,9 @@ struct SQLRowToStatementConverterTests { #expect(lines.count == 50_000) } - @Test("PostgreSQL: binary cell renders as bytea hex literal in INSERT") - func postgresBinaryInsertEmitsByteaLiteral() throws { + // `'\x…'::bytea` reads its backslash by `standard_conforming_strings`; `decode` means the same bytes either way. + @Test("PostgreSQL: binary cell decodes hex in INSERT") + func postgresBinaryInsertDecodesHex() throws { let converter = try SQLRowToStatementConverter( tableName: "documents", columns: ["id", "payload"], @@ -396,10 +397,28 @@ struct SQLRowToStatementConverterTests { ) let bytes = Data([0xD3, 0x8C, 0xE5, 0x66]) let result = converter.generateInserts(rows: plain([[.text("1"), .bytes(bytes)]])) - #expect(result.contains("'\\xD38CE566'::bytea")) + #expect(result.contains("decode('D38CE566', 'hex')")) #expect(!result.contains("NULL")) } + // DuckDB reads `X'…'` as the text `x…` and Oracle has no such literal. + @Test("DuckDB and Oracle binary cells use their own hex functions in INSERT", arguments: [ + (DatabaseType.duckdb, "unhex('CAFE')"), (DatabaseType.oracle, "HEXTORAW('CAFE')"), + ]) + func binaryInsertUsesTheEnginesHexFunction(databaseType: DatabaseType, literal: String) throws { + let converter = try SQLRowToStatementConverter( + tableName: "documents", + columns: ["id", "payload"], + primaryKeyColumns: ["id"], + databaseType: databaseType, + quoteIdentifier: { "\"\($0)\"" }, + escapeStringLiteral: { $0.replacingOccurrences(of: "'", with: "''") } + ) + let result = converter.generateInserts(rows: plain([[.text("1"), .bytes(Data([0xCA, 0xFE]))]])) + #expect(result.contains(literal)) + #expect(!result.contains("X'CAFE'")) + } + @Test("MySQL: binary cell renders as X'...' literal in INSERT") func mysqlBinaryInsertEmitsXLiteral() throws { let converter = try makeConverter( @@ -441,7 +460,7 @@ struct SQLRowToStatementConverterTests { ) let bytes = Data([0xAB, 0xCD]) let result = converter.generateUpdates(rows: plain([[.text("42"), .bytes(bytes)]])) - #expect(result.contains("\"payload\" = '\\xABCD'::bytea")) + #expect(result.contains("\"payload\" = decode('ABCD', 'hex')")) #expect(result.contains("WHERE \"id\" = '42'")) } @@ -656,8 +675,8 @@ struct SQLRowToStatementConverterTests { databaseType: .pglite, dialect: Self.postgresDialect ) let rows = plain([[.text("1"), .bytes(Data([0xDE, 0xAD]))]]) - #expect(converter.generateInserts(rows: rows).contains("'\\xDEAD'::bytea")) - #expect(converter.generateUpdates(rows: rows).contains("\"payload\" = '\\xDEAD'::bytea")) + #expect(converter.generateInserts(rows: rows).contains("decode('DEAD', 'hex')")) + #expect(converter.generateUpdates(rows: rows).contains("\"payload\" = decode('DEAD', 'hex')")) #expect(!converter.generateInserts(rows: rows).contains("X'DEAD'")) } } diff --git a/apple/TableProTests/Core/Vim/VimTextBufferAdapterPerfTests.swift b/apple/TableProTests/Core/Vim/VimTextBufferAdapterPerfTests.swift index 6fadfaf773..48c97aac20 100644 --- a/apple/TableProTests/Core/Vim/VimTextBufferAdapterPerfTests.swift +++ b/apple/TableProTests/Core/Vim/VimTextBufferAdapterPerfTests.swift @@ -2,8 +2,7 @@ // VimTextBufferAdapterPerfTests.swift // TableProTests // -// Regression tests for VimTextBufferAdapter incremental lineCount -// and setSelectedRange guard +// Regression tests for VimTextBufferAdapter lineCount and setSelectedRange guard // import AppKit @@ -65,83 +64,29 @@ struct VimTextBufferAdapterPerfTests { #expect(adapter.lineCount == 2) } - // MARK: - textDidChange incremental (pure insertion) - - @Test("textDidChange with pure insertion updates line count incrementally") - func insertionUpdatesLineCount() { - let (adapter, textView) = makeAdapter(string: "hello") - - // Prime the cache - let initial = adapter.lineCount - #expect(initial == 1) - - // Simulate inserting "\nworld" at offset 5 - textView.replaceCharacters(in: NSRange(location: 5, length: 0), with: "\nworld") - adapter.textDidChange(in: NSRange(location: 5, length: 0), replacementLength: 6) - - #expect(adapter.lineCount == 2) - } - - @Test("textDidChange with multi-newline insertion") - func multiNewlineInsertion() { - let (adapter, textView) = makeAdapter(string: "hello") - - _ = adapter.lineCount // prime cache - - textView.replaceCharacters(in: NSRange(location: 5, length: 0), with: "\n\n\n") - adapter.textDidChange(in: NSRange(location: 5, length: 0), replacementLength: 3) - - #expect(adapter.lineCount == 4) - } - - // MARK: - textDidChange fallback on deletion - - @Test("textDidChange with deletion invalidates cache") - func deletionInvalidatesCache() { - let (adapter, textView) = makeAdapter(string: "a\nb\nc") - - let initial = adapter.lineCount - #expect(initial == 3) - - // Simulate deleting "\nb" (range.length > 0 means it's not pure insertion) - textView.replaceCharacters(in: NSRange(location: 1, length: 2), with: "") - adapter.textDidChange(in: NSRange(location: 1, length: 2), replacementLength: 0) - - // Cache should be invalidated; next access does a full recount - #expect(adapter.lineCount == 2) - } - - // MARK: - textDidChange(oldText:...) incremental replacement - - @Test("textDidChange with oldText correctly computes delta for replacement") - func oldTextReplacementDelta() { - let originalText = "a\nb\nc" - let (adapter, textView) = makeAdapter(string: originalText) - - let initial = adapter.lineCount - #expect(initial == 3) - - // Replace "b\nc" (range 2..4, contains 1 newline) with "x" (0 newlines) - textView.replaceCharacters(in: NSRange(location: 2, length: 3), with: "x") - adapter.textDidChange(oldText: originalText, in: NSRange(location: 2, length: 3), replacementLength: 1) - - // 3 original - 1 removed newline + 0 added = 2 - #expect(adapter.lineCount == 2) - } - - @Test("textDidChange with oldText adding newlines") - func oldTextAddingNewlines() { - let originalText = "abc" - let (adapter, textView) = makeAdapter(string: originalText) - - _ = adapter.lineCount // prime: 1 - - // Replace "b" with "x\ny\nz" (adding 2 newlines) - textView.replaceCharacters(in: NSRange(location: 1, length: 1), with: "x\ny\nz") - adapter.textDidChange(oldText: originalText, in: NSRange(location: 1, length: 1), replacementLength: 5) - - // 1 original - 0 removed + 2 added = 3 - #expect(adapter.lineCount == 3) + // MARK: - Line count after an edit + + // The editor drops the cache on every edit, so the next count reads the text as it is now. + @Test("After an edit the line count matches a fresh count of the new text", arguments: [ + (NSRange(location: 5, length: 0), "\nworld", "hello", 2), + (NSRange(location: 5, length: 0), "\n\n\n", "hello", 3), + (NSRange(location: 1, length: 2), "", "a\nb\nc", 2), + (NSRange(location: 2, length: 3), "x", "a\nb\nc", 2), + (NSRange(location: 1, length: 1), "x\ny\nz", "abc", 3), + (NSRange(location: 1, length: 0), "\r", "ab", 2), + ]) + func lineCountAfterAnEdit(range: NSRange, replacement: String, original: String, expected: Int) { + let (adapter, textView) = makeAdapter(string: original) + _ = adapter.lineCount + + textView.replaceCharacters(in: range, with: replacement) + adapter.invalidateLineCache() + + let counted = withExtendedLifetime(textView) { adapter.lineCount } + let (fresh, freshTextView) = makeAdapter(string: textView.string) + let freshCount = withExtendedLifetime(freshTextView) { fresh.lineCount } + #expect(counted == expected) + #expect(counted == freshCount) } // MARK: - setSelectedRange guard diff --git a/apple/TableProTests/Helpers/StubExecutionGates.swift b/apple/TableProTests/Helpers/StubExecutionGates.swift index 83dce282ed..afcfa85d48 100644 --- a/apple/TableProTests/Helpers/StubExecutionGates.swift +++ b/apple/TableProTests/Helpers/StubExecutionGates.swift @@ -27,13 +27,18 @@ internal struct AlwaysAllowGate: ExecutionGate { internal struct AlwaysDenyGate: ExecutionGate { internal let reason: String + internal let cause: OperationDenialCause - internal init(reason: String = "Read-Only connection") { + internal init(reason: String = "Read-Only connection", cause: OperationDenialCause = .policy) { self.reason = reason + self.cause = cause } + // The user answering Cancel in the gate's confirmation. + internal static let cancelling = AlwaysDenyGate(reason: "Operation cancelled by user", cause: .cancelledByUser) + internal func authorize(_ request: OperationRequest) async -> OperationDecision { - .denied(reason: reason) + .denied(reason: reason, cause: cause) } } diff --git a/apple/TableProTests/Models/MultiRowEditStateTests.swift b/apple/TableProTests/Models/MultiRowEditStateTests.swift index ce162e198a..1cdb06f345 100644 --- a/apple/TableProTests/Models/MultiRowEditStateTests.swift +++ b/apple/TableProTests/Models/MultiRowEditStateTests.swift @@ -95,14 +95,14 @@ struct MultiRowEditStateTests { #expect(field.effectiveValue == nil) } - @Test("effectiveValue returns __DEFAULT__ when isPendingDefault") + @Test("effectiveValue returns the DEFAULT keyword when isPendingDefault") func effectiveValueReturnsDefaultWhenPendingDefault() { let field = FieldEditState( columnIndex: 0, columnName: "id", columnTypeEnum: .text(rawType: nil), isLongText: false, isJson: false, originalValue: "1", hasMultipleValues: false, pendingValue: nil, isPendingNull: false, isPendingDefault: true ) - #expect(field.effectiveValue == "__DEFAULT__") + #expect(field.effectiveValue == "DEFAULT") } @Test("effectiveValue returns nil when no edit is pending") @@ -644,13 +644,14 @@ struct MultiRowEditStateTests { #expect(edited[0].newValue == nil) } - @Test("Returns __DEFAULT__ for default edit") + @Test("Returns the DEFAULT keyword marked as SQL for a default edit") func returnsDefaultForDefaultEdit() { let sut = makeSUT() sut.setFieldToDefault(at: 0) let edited = sut.getEditedFields() #expect(edited.count == 1) - #expect(edited[0].newValue == "__DEFAULT__") + #expect(edited[0].newValue == "DEFAULT") + #expect(edited[0].isExpression) } @Test("Returns empty array when no edits") @@ -774,18 +775,22 @@ struct MultiRowEditStateTests { #expect(callbackCalls[0].value == nil) } - @Test("setFieldToDefault fires callback with __DEFAULT__") + // Through `onFieldChanged` the keyword would be a literal, which a stored value can also be. + @Test("setFieldToDefault sends the DEFAULT keyword through the expression callback") func setFieldToDefaultFiresCallback() { let sut = makeSUT() - var callbackCalls: [(index: Int, value: String?)] = [] - sut.onFieldChanged = { index, value, _ in - callbackCalls.append((index, value.asText)) + var literals: [PluginCellValue] = [] + var callbackCalls: [(index: Int, sql: String)] = [] + sut.onFieldChanged = { _, value, _ in literals.append(value) } + sut.onFieldExpressionChanged = { index, sql, _ in + callbackCalls.append((index, sql)) } sut.setFieldToDefault(at: 0) + #expect(literals.isEmpty) #expect(callbackCalls.count == 1) #expect(callbackCalls[0].index == 0) - #expect(callbackCalls[0].value == "__DEFAULT__") + #expect(callbackCalls[0].sql == "DEFAULT") } // `onFieldChanged` carries a literal, so an expression sent through it would be saved as quoted text. diff --git a/apple/TableProTests/Models/Schema/ClickHousePartStatementsTests.swift b/apple/TableProTests/Models/Schema/ClickHousePartStatementsTests.swift index cc67c4e094..cdee26b95c 100644 --- a/apple/TableProTests/Models/Schema/ClickHousePartStatementsTests.swift +++ b/apple/TableProTests/Models/Schema/ClickHousePartStatementsTests.swift @@ -2,9 +2,6 @@ // ClickHousePartStatementsTests.swift // TableProTests // -// The Parts tab's statements name the tab's own database, because ClickHouse resolves an -// unqualified one from a request parameter the app moves whenever any tab runs elsewhere. -// import Foundation @testable import TablePro @@ -14,8 +11,7 @@ struct ClickHousePartStatementsTests { private let quote: (String) -> String = { "`\($0.replacingOccurrences(of: "`", with: "``"))`" } private let escape: (String) -> String = { $0.replacingOccurrences(of: "'", with: "\\'") } - /// The defect: an unqualified DROP PARTITION deleted data from whichever database the shared - /// driver was last pinned to, while the tab reported it was acting on its own. + // An unqualified DROP PARTITION deleted data from whichever database the shared driver was last pinned to. @Test("Every statement names the database it was given") func statementsCarryTheDatabase() { #expect( @@ -24,20 +20,59 @@ struct ClickHousePartStatementsTests { ) #expect( ClickHousePartStatements.dropPartition( - database: "analytics", table: "events", partition: "202401", - quote: quote, escape: escape - ) == "ALTER TABLE `analytics`.`events` DROP PARTITION '202401'" + database: "analytics", table: "events", partitionId: "202401", quote: quote + ) == "ALTER TABLE `analytics`.`events` DROP PARTITION ID '202401'" ) #expect( ClickHousePartStatements.detachPartition( - database: "analytics", table: "events", partition: "202401", - quote: quote, escape: escape - ) == "ALTER TABLE `analytics`.`events` DETACH PARTITION '202401'" + database: "analytics", table: "events", partitionId: "202401", quote: quote + ) == "ALTER TABLE `analytics`.`events` DETACH PARTITION ID '202401'" ) } - /// `currentDatabase()` answers with the request parameter, so the list showed another database's - /// parts under this table's name. + // A tuple key's rendered value `(202401,'eu')` and an unpartitioned table's `tuple()` were each + // wrapped as one string literal, which names no partition. + @Test("Drop and Detach name the partition by its ID for every key shape") + func everyKeyShapeUsesThePartitionId() { + let shapes = [ + "202401-7", + "all", + "4f8b2b7c9a1d3e5f60718293a4b5c6d7", + "1--5", + ] + for id in shapes { + #expect( + ClickHousePartStatements.dropPartition( + database: "analytics", table: "events", partitionId: id, quote: quote + ) == "ALTER TABLE `analytics`.`events` DROP PARTITION ID '\(id)'" + ) + #expect( + ClickHousePartStatements.detachPartition( + database: "analytics", table: "events", partitionId: id, quote: quote + ) == "ALTER TABLE `analytics`.`events` DETACH PARTITION ID '\(id)'" + ) + } + } + + @Test("An ID the server would refuse is never spliced into a statement") + func unusableIdBuildsNothing() { + let unusable = ["", "it's", "a\\'b", "(202401,'eu')", "tuple()", "2024 01", "202401_1", "é"] + for id in unusable { + #expect(!ClickHousePartStatements.isValidPartitionId(id)) + #expect( + ClickHousePartStatements.dropPartition( + database: "analytics", table: "events", partitionId: id, quote: quote + ) == nil + ) + #expect( + ClickHousePartStatements.detachPartition( + database: "analytics", table: "events", partitionId: id, quote: quote + ) == nil + ) + } + } + + // `currentDatabase()` answers with the request parameter, so the list showed another database's parts. @Test("The parts read filters on the named database, never the connection's current one") func partsReadNamesTheDatabase() { let sql = ClickHousePartStatements.parts( @@ -48,9 +83,13 @@ struct ClickHousePartStatementsTests { #expect(!sql.contains("currentDatabase()")) } - /// A connection saved with no database of its own has nothing to qualify against, and an empty - /// qualifier is a statement the server refuses. The parts read has to fall back too: comparing - /// against '' matches nothing, where ClickHouse's own default database is what the user sees. + @Test("The parts read selects the partition ID beside the displayed partition value") + func partsReadSelectsThePartitionId() { + let sql = ClickHousePartStatements.parts(database: "analytics", table: "events", escape: escape) + #expect(sql.contains("SELECT partition, partition_id, name, rows, bytes_on_disk,")) + } + + // With no database the qualifier is refused by the server, and comparing against '' matches nothing. @Test("With no database the name stays unqualified and the read asks the server") func emptyDatabaseLeavesTheNameBare() { #expect( @@ -66,13 +105,11 @@ struct ClickHousePartStatementsTests { #expect(!sql.contains("database = ''")) } - @Test("A quote in a partition value is escaped, and a backtick in a name is quoted") - func valuesAreEscaped() { + @Test("A backtick in a database or table name is quoted") + func namesAreQuoted() { let sql = ClickHousePartStatements.dropPartition( - database: "we`ird", table: "ev`ents", partition: "it's", - quote: quote, escape: escape + database: "we`ird", table: "ev`ents", partitionId: "202401", quote: quote ) - #expect(sql.contains("`we``ird`.`ev``ents`")) - #expect(sql.contains("'it\\'s'")) + #expect(sql == "ALTER TABLE `we``ird`.`ev``ents` DROP PARTITION ID '202401'") } } diff --git a/apple/TableProTests/Views/Editor/StatementCaretTableTests.swift b/apple/TableProTests/Views/Editor/StatementCaretTableTests.swift new file mode 100644 index 0000000000..ba5ea78102 --- /dev/null +++ b/apple/TableProTests/Views/Editor/StatementCaretTableTests.swift @@ -0,0 +1,109 @@ +import AppKit +import Foundation +@testable import TablePro +import TableProEditorKit +import TableProGrammars +import TableProSQLGrammar +import TableProTextEngine +import Testing + +private let caretTableMongoScript = """ +db.users.find({ name: "a;b" }); +// note + +for (var i = 0; i < 3; i++) { + print(i); +} +db.orders.insertOne({ total: 1 }) +db.orders.find() +// trailing +""" + +private let caretTableRedisScript = "GET a; GET b\n\nSET k \"a\nb\"\nGET c \n " + +@MainActor +struct StatementCaretTableTests { + private struct Resolved: Equatable { + let offset: Int + let sql: String + let hasContent: Bool + let contentRange: NSRange + + init(_ statement: SQLStatementScanner.LocatedStatement) { + offset = statement.offset + sql = statement.sql + hasContent = statement.hasContent + contentRange = statement.contentRange + } + } + + @Test("Every caret resolves to what a scan of the whole document resolves to", arguments: [ + (QueryStatementModel.javascript, caretTableMongoScript), + (QueryStatementModel.redis, caretTableRedisScript), + (QueryStatementModel.javascript, ""), + (QueryStatementModel.redis, ""), + ]) + func everyCaretMatchesAFullScan(model: QueryStatementModel, text: String) throws { + let table = try #require(QueryStatementCaretTable(text: text, model: model)) + for caret in -1...((text as NSString).length + 1) { + let fresh = QueryStatementScanner.locatedStatementAtCursor( + in: text, cursorPosition: caret, model: model, grammar: .ansi + ) + #expect(Resolved(table.statement(atCursor: caret)) == Resolved(fresh), "caret \(caret)") + } + } + + @Test("A SQL document has no table, since it resumes through its own index") + func sqlHasNoTable() { + #expect(QueryStatementCaretTable(text: "SELECT 1;", model: .sql) == nil) + } + + private func makeController(text: String) -> TextViewController { + let configuration = SourceEditorConfiguration( + appearance: .init( + theme: ThemeEngine.shared.makeEditorTheme(), + font: .monospacedSystemFont(ofSize: 12, weight: .regular), + wrapLines: false + ), + peripherals: EditorPeripherals.editor(lineNumbers: true, folding: true, statementRunControls: true) + ) + let controller = TextViewController(string: text, language: .sql, configuration: configuration, cursorPositions: []) + controller.loadView() + return controller + } + + private func band(at caret: Int, in controller: TextViewController, subject: StatementRunController) -> NSRange? { + controller.setCursorPositions([CursorPosition(range: NSRange(location: caret, length: 0))]) + subject.refreshHighlight(in: controller) + return controller.highlightedStatementRange + } + + private func freshBand(at caret: Int, in controller: TextViewController) -> NSRange? { + let statement = QueryStatementScanner.locatedStatementAtCursor( + in: controller.textView.string, cursorPosition: caret, model: .javascript, grammar: .ansi + ) + guard statement.hasContent, statement.contentRange.length > 0 else { return nil } + return statement.contentRange + } + + @Test("Caret moves through a MongoDB script read it once, and an edit reads it again") + func caretMovesReadTheScriptOnce() { + let script = String(repeating: "db.orders.find({ total: 1 });\n", count: 2_000) + let controller = makeController(text: script) + let subject = StatementRunController() + subject.statementModel = .javascript + subject.install(on: controller) + let readsAtInstall = subject.caretTableReads + + for caret in stride(from: 59_000, to: 58_000, by: -7) { + #expect(band(at: caret, in: controller, subject: subject) == freshBand(at: caret, in: controller)) + } + #expect(subject.caretTableReads <= readsAtInstall + 1) + + controller.textView.replaceCharacters(in: NSRange(location: 0, length: 0), with: "db.a.find();\n") + let afterEdit = subject.caretTableReads + #expect(band(at: 100, in: controller, subject: subject) == freshBand(at: 100, in: controller)) + #expect(band(at: 58_013, in: controller, subject: subject) == freshBand(at: 58_013, in: controller)) + #expect(subject.caretTableReads == afterEdit + 1) + } +} diff --git a/apple/TableProTests/Views/Main/TableLoadAfterSafeModePromptTests.swift b/apple/TableProTests/Views/Main/TableLoadAfterSafeModePromptTests.swift new file mode 100644 index 0000000000..920544485f --- /dev/null +++ b/apple/TableProTests/Views/Main/TableLoadAfterSafeModePromptTests.swift @@ -0,0 +1,40 @@ +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +@MainActor +struct TableLoadAfterSafeModePromptTests { + @Test("A table load turned away while a query waits for approval runs once that approval closes") + func turnedAwayLoadRunsAfterThePrompt() async throws { + let connection = DatabaseConnection( + name: "Confirm", database: "app", type: .postgresql, safeModeLevel: .alertFull + ) + let fixture = CoordinatorFixture( + connection: connection, + driver: ScriptAnsweringDriver(connection: connection, sendsBatchesWhole: false) + ) + fixture.coordinator.toolbarState.safeModeLevel = .alertFull + let gate = HeldExecutionGate() + fixture.coordinator.executionGate = gate + defer { fixture.coordinator.cancelAllQueryTasks() } + + fixture.addTab(QueryTab(title: "Query", query: "SELECT 1", tabType: .query)) + #expect(fixture.coordinator.runStatement("SELECT 1")) + try await waitUntil { gate.askedCount == 1 } + + var table = QueryTab(title: "users", query: "SELECT * FROM users LIMIT 1000", tabType: .table, tableName: "users") + table.pagination.isLoading = true + fixture.addTab(table) + #expect(fixture.coordinator.executeTableTabQueryDirectly(viewport: .firstRow) == nil) + #expect(fixture.tabManager.tabs.first { $0.id == table.id }?.pagination.isLoading == false) + + await gate.answer(allowing: true) + try await waitUntil { gate.askedCount == 2 } + await gate.answer(allowing: false) + try await waitUntil { + fixture.tabManager.tabs.first { $0.id == table.id }?.execution.errorMessage == "Refused by the test" + } + } +} diff --git a/apple/TableProTests/Views/Main/WindowCloseSaveEveryConnectionTests.swift b/apple/TableProTests/Views/Main/WindowCloseSaveEveryConnectionTests.swift new file mode 100644 index 0000000000..bb0a30c859 --- /dev/null +++ b/apple/TableProTests/Views/Main/WindowCloseSaveEveryConnectionTests.swift @@ -0,0 +1,129 @@ +import AppKit +import Foundation +import SwiftUI +@testable import TablePro +import TableProPluginKit +import TableProTestSupport +import Testing + +@MainActor +struct WindowCloseSaveEveryConnectionTests { + private func actions( + for coordinator: MainContentCoordinator, + in window: NSWindow, + attach: Bool = true + ) -> MainContentCommandActions { + var selectedTables: Set = [] + var pendingTruncates: Set = [] + var pendingDeletes: Set = [] + var tableOperationOptions: [DatabaseTreeTableRef: TableOperationOptions] = [:] + let actions = MainContentCommandActions( + coordinator: coordinator, + connection: coordinator.connection, + selectedTables: Binding(get: { selectedTables }, set: { selectedTables = $0 }), + pendingTruncates: Binding(get: { pendingTruncates }, set: { pendingTruncates = $0 }), + pendingDeletes: Binding(get: { pendingDeletes }, set: { pendingDeletes = $0 }), + tableOperationOptions: Binding( + get: { tableOperationOptions }, + set: { tableOperationOptions = $0 } + ), + trailingPaneState: TrailingPaneState() + ) + actions.window = window + if attach { coordinator.commandActions = actions } + return actions + } + + @Test("Save on a window close keeps the window while another connection holds edits only their tab can save") + func saveKeepsTheWindowForAnotherConnectionsEdits() async throws { + let harness = TwoConnectionWindow() + defer { harness.tearDown() } + let front = try harness.coordinator(of: harness.foreground) + let back = try harness.coordinator(of: harness.background) + let frontActions = actions(for: front, in: harness.window) + let backActions = actions(for: back, in: harness.window) + try harness.addSelectedTab(to: harness.foreground) + try harness.addSelectedTab(to: harness.background) + + var edited = QueryTab(title: "users", query: "SELECT * FROM users", tabType: .table, tableName: "users") + edited.pendingChanges.insertedRowIDs = [.inserted(UUID())] + edited.pendingChanges.columns = ["id", "name"] + edited.pendingChanges.primaryKeyColumns = ["id"] + back.tabManager.tabs.append(edited) + frontActions.confirmSaveChanges = { _, _ in .save } + + let outcome = await frontActions.closeWindowAwaiting() + + #expect(outcome == .cancelled) + #expect(back.tabManager.tabs.contains { $0.id == edited.id }) + withExtendedLifetime(backActions) {} + } + + // Its actions arrive only once its pane mounts, after Save has shown it. + @Test("Save waits for a connection that has not shown its pane yet, then saves it and closes") + func saveReachesAConnectionNotShownYet() async throws { + let harness = TwoConnectionWindow() + defer { harness.tearDown() } + let front = try harness.coordinator(of: harness.foreground) + let back = try harness.coordinator(of: harness.background) + front.contentWindow = harness.window + let frontActions = actions(for: front, in: harness.window) + let backActions = actions(for: back, in: harness.window, attach: false) + back.commandActions = nil + try harness.addSelectedTab(to: harness.foreground) + back.toolbarState.hasCreateTablePending = true + frontActions.confirmSaveChanges = { _, _ in + Task { @MainActor in back.commandActions = backActions } + return .save + } + + let outcome = await frontActions.closeWindowAwaiting() + + #expect(outcome == .closed) + } + + @Test("A close cancelled while it waits for a connection's pane stops after that one wait and keeps the window") + func cancelledCloseStopsWaiting() async throws { + let harness = TwoConnectionWindow() + defer { harness.tearDown() } + let front = try harness.coordinator(of: harness.foreground) + let back = try harness.coordinator(of: harness.background) + front.contentWindow = harness.window + let frontActions = actions(for: front, in: harness.window) + back.commandActions = nil + try harness.addSelectedTab(to: harness.foreground) + back.toolbarState.hasCreateTablePending = true + let clock = CountingClock() + frontActions.paneWaitClock = clock + frontActions.confirmSaveChanges = { _, _ in .save } + + let close = Task { await frontActions.closeWindowAwaiting() } + await clock.firstSleep.wait() + close.cancel() + let outcome = await close.value + + #expect(outcome == .cancelled) + #expect(clock.sleepCount == 1) + } +} + +// Time stands still, so a sleep ends only by cancellation; the first one opens a gate the test waits on. +private final class CountingClock: Clock, @unchecked Sendable { + let firstSleep = AsyncGate() + private let base = ManualClock() + private let lock = NSLock() + private var count = 0 + + var sleepCount: Int { lock.withLock { count } } + var now: ManualClock.Instant { base.now } + var minimumResolution: Duration { base.minimumResolution } + + func sleep(until deadline: ManualClock.Instant, tolerance: Duration?) async throws { + let isFirst = lock.withLock { + count += 1 + return count == 1 + } + if isFirst { await firstSleep.open() } + try await base.sleep(until: deadline, tolerance: tolerance) + } +} diff --git a/apple/TableProTests/Views/Results/ResultsJsonViewTests.swift b/apple/TableProTests/Views/Results/ResultsJsonViewTests.swift index d64d2975d0..167773b1fa 100644 --- a/apple/TableProTests/Views/Results/ResultsJsonViewTests.swift +++ b/apple/TableProTests/Views/Results/ResultsJsonViewTests.swift @@ -128,6 +128,30 @@ struct ResultsJsonViewTests { } } + @Test("A cell a pending edit set to the default renders as null, and stored text that reads DEFAULT stays text") + func pendingDefaultRendersAsNull() { + let inserted = RowID.inserted(UUID()) + let tableRows = TableRows( + rows: [ + Row(id: .existing(0), values: [.text("DEFAULT"), .text("a")]), + Row(id: inserted, values: [.columnDefault, .text("b")]), + ], + columns: ["status", "name"], + columnTypes: [.text(rawType: nil), .text(rawType: nil)] + ) + + let result = ResultsJsonView.computeJson( + tableRows: tableRows, + displayIDs: nil, + selectedIndices: [], + defaultedCells: [inserted: [0]], + columnLayout: ColumnLayoutState() + ) + + #expect(result.json.contains("\"status\": \"DEFAULT\"")) + #expect(result.json.contains("\"status\": null")) + } + @Test("an empty result renders an empty array") func emptyResultRendersEmptyArray() { let result = ResultsJsonView.computeJson( diff --git a/apple/TableProTests/Views/Results/SelectionSummaryTrackerTests.swift b/apple/TableProTests/Views/Results/SelectionSummaryTrackerTests.swift index 26110af586..926bf1f31f 100644 --- a/apple/TableProTests/Views/Results/SelectionSummaryTrackerTests.swift +++ b/apple/TableProTests/Views/Results/SelectionSummaryTrackerTests.swift @@ -176,7 +176,7 @@ struct SelectionSummaryTrackerTests { defer { grid.close() } grid.model.setSelection(Self.block(rows: 0...1, columns: 0...0)) - grid.edit(row: 1, to: .value(.text(PluginCellValue.defaultMarkerText), isExpression: false)) + grid.edit(row: 1, to: .value(.columnDefault, isExpression: true)) await grid.tracker.pendingTask?.value #expect(state.summary?.emptyCount == 1) diff --git a/apple/TableProTests/Views/Structure/ClickHousePartsGridTests.swift b/apple/TableProTests/Views/Structure/ClickHousePartsGridTests.swift index 41044fa125..fde04c7c82 100644 --- a/apple/TableProTests/Views/Structure/ClickHousePartsGridTests.swift +++ b/apple/TableProTests/Views/Structure/ClickHousePartsGridTests.swift @@ -5,12 +5,14 @@ import Foundation @testable import TablePro +import TableProPluginKit import Testing @MainActor struct ClickHousePartsGridTests { private let part = ClickHousePartInfo( partition: "202601", + partitionId: "202601", name: "202601_1_4_1", rows: 1_234_567, bytesOnDisk: 5_242_880, @@ -35,7 +37,7 @@ struct ClickHousePartsGridTests { @Test("An inactive part reads NO") func inactivePart() { let inactive = ClickHousePartInfo( - partition: "", name: "p", rows: 0, bytesOnDisk: 0, modificationTime: "", active: false + partition: "", partitionId: "", name: "p", rows: 0, bytesOnDisk: 0, modificationTime: "", active: false ) #expect(ClickHousePartsView.gridRow(for: inactive).values.last == "NO") } @@ -43,7 +45,7 @@ struct ClickHousePartsGridTests { @Test("Active reads aloud as Active or Inactive rather than YES or NO") func activeIsSpokenByName() { let inactive = ClickHousePartInfo( - partition: "", name: "p", rows: 0, bytesOnDisk: 0, modificationTime: "", active: false + partition: "", partitionId: "", name: "p", rows: 0, bytesOnDisk: 0, modificationTime: "", active: false ) let active = ClickHousePartsView.activeColumn @@ -59,23 +61,25 @@ struct ClickHousePartsGridTests { #expect(zip(ClickHousePartsView.columnMinimumWidths, ClickHousePartsView.columnWidths).allSatisfy { $0 <= $1 }) } - @Test("Drop and Detach name the one selected part's partition, and none while several are selected") + @Test("Drop and Detach act on the one selected part, and on none while several are selected") func selectedPartition() { let other = ClickHousePartInfo( - partition: "202602", name: "202602_1_1_0", rows: 1, bytesOnDisk: 1, modificationTime: "", active: true + partition: "202602", partitionId: "202602", name: "202602_1_1_0", rows: 1, bytesOnDisk: 1, + modificationTime: "", active: true ) let parts = [part, other] - #expect(ClickHousePartsView.partition(of: [other.id], in: parts) == "202602") - #expect(ClickHousePartsView.partition(of: [part.id, other.id], in: parts) == nil) - #expect(ClickHousePartsView.partition(of: [], in: parts) == nil) - #expect(ClickHousePartsView.partition(of: [UUID()], in: parts) == nil) + #expect(ClickHousePartsView.selectedPart(of: [other.id], in: parts)?.id == other.id) + #expect(ClickHousePartsView.selectedPart(of: [part.id, other.id], in: parts)?.id == nil) + #expect(ClickHousePartsView.selectedPart(of: [], in: parts)?.id == nil) + #expect(ClickHousePartsView.selectedPart(of: [UUID()], in: parts)?.id == nil) } @Test("A selected grid row names its own part") func selectedRowResolvesToItsPart() throws { let other = ClickHousePartInfo( - partition: "202602", name: "202602_1_1_0", rows: 1, bytesOnDisk: 1, modificationTime: "", active: true + partition: "202602", partitionId: "202602", name: "202602_1_1_0", rows: 1, bytesOnDisk: 1, + modificationTime: "", active: true ) let parts = [part, other] let grid = StructureListGridModel() @@ -95,7 +99,84 @@ struct ClickHousePartsGridTests { #expect(grid.id(for: second) == other.id) #expect(selected == [other.id]) - #expect(ClickHousePartsView.partition(of: selected, in: parts) == "202602") + #expect(ClickHousePartsView.selectedPart(of: selected, in: parts)?.partition == "202602") #expect(grid.model.rowID(at: 2) == nil) } + + @Test("A parts row keeps the partition ID a statement names beside the value the grid shows") + func rowCarriesThePartitionId() throws { + let row: [PluginCellValue] = [ + "(202401,'eu')", + "c1f8e6e2a7b04d4c9e1f0a2b3c4d5e6f", + "c1f8e6e2a7b04d4c9e1f0a2b3c4d5e6f_1_1_0", + "12", + "4096", + "2026-01-05 10:00:00", + "1", + ] + + let parsed = try #require(ClickHousePartsView.part(from: row)) + + #expect(parsed.partition == "(202401,'eu')") + #expect(parsed.partitionId == "c1f8e6e2a7b04d4c9e1f0a2b3c4d5e6f") + #expect(parsed.name == "c1f8e6e2a7b04d4c9e1f0a2b3c4d5e6f_1_1_0") + #expect(parsed.rows == 12) + #expect(parsed.bytesOnDisk == 4096) + #expect(parsed.modificationTime == "2026-01-05 10:00:00") + #expect(parsed.active) + #expect(ClickHousePartsView.gridRow(for: parsed).values[0] == "(202401,'eu')") + } + + // The pane used to swap the list and its toolbar for the error text until the tab was refreshed. + @Test("A refused or failed action keeps the parts list and its toolbar on screen") + func actionFailureKeepsTheList() { + var pane = ClickHousePartsPaneState() + pane.beginLoad() + pane.finishLoad([part]) + + pane.actionEnded(.refused("Safe Mode is read-only"), title: "Couldn't Drop Partition") + #expect(pane.display == .parts) + #expect(pane.alert == ClickHousePartsAlert(title: "Couldn't Drop Partition", message: "Safe Mode is read-only")) + + pane.alert = nil + pane.actionEnded(.failed("Wrong partition value"), title: "Couldn't Detach Partition") + #expect(pane.display == .parts) + #expect(pane.alert == ClickHousePartsAlert(title: "Couldn't Detach Partition", message: "Wrong partition value")) + } + + @Test("A Cancel at the gate's confirmation shows nothing") + func cancelIsQuiet() { + var pane = ClickHousePartsPaneState() + pane.finishLoad([part]) + + pane.actionEnded( + GatedOperationFailure(ExecutionGateError.cancelledByUser("Operation cancelled by user")), + title: "Couldn't Drop Partition" + ) + + #expect(pane.alert == nil) + #expect(pane.display == .parts) + } + + @Test("A load that fails with nothing shown offers Try Again, and a reload that fails keeps the list") + func loadFailure() { + var pane = ClickHousePartsPaneState() + #expect(pane.display == .loading) + + pane.failLoad("Connection refused") + #expect(pane.display == .loadFailed("Connection refused")) + #expect(pane.alert == nil) + + pane.beginLoad() + #expect(pane.display == .loading) + pane.finishLoad([]) + #expect(pane.display == .empty) + + pane.finishLoad([part]) + pane.beginLoad() + #expect(pane.display == .parts) + pane.failLoad("Timeout exceeded") + #expect(pane.display == .parts) + #expect(pane.alert?.message == "Timeout exceeded") + } } diff --git a/apple/TableProTests/Views/Structure/StructureListGridAccessibilityTests.swift b/apple/TableProTests/Views/Structure/StructureListGridAccessibilityTests.swift index b4e8440113..7910b8ae10 100644 --- a/apple/TableProTests/Views/Structure/StructureListGridAccessibilityTests.swift +++ b/apple/TableProTests/Views/Structure/StructureListGridAccessibilityTests.swift @@ -109,7 +109,8 @@ struct StructureListGridAccessibilityTests { defer { DataGridAccessibility.isActive = false } let parts = [true, false].map { active in ClickHousePartInfo( - partition: "202601", name: "p", rows: 1, bytesOnDisk: 1, modificationTime: "", active: active + partition: "202601", partitionId: "202601", name: "p", rows: 1, bytesOnDisk: 1, modificationTime: "", + active: active ) } let rows = parts.map { part in diff --git a/apple/TableProTests/Views/Structure/StructureSaveConfirmationTests.swift b/apple/TableProTests/Views/Structure/StructureSaveConfirmationTests.swift index 348093f635..fcb0310781 100644 --- a/apple/TableProTests/Views/Structure/StructureSaveConfirmationTests.swift +++ b/apple/TableProTests/Views/Structure/StructureSaveConfirmationTests.swift @@ -235,7 +235,7 @@ struct StructureSaveConfirmationTests { ).authorize(request) let error = try #require(decision.denialError) - #expect(StructureApplyFailure(error) == .cancelledByUser) + #expect(GatedOperationFailure(error) == .cancelledByUser) } @Test("A declined Touch ID is a refusal that says why, not a quiet Cancel") @@ -252,7 +252,7 @@ struct StructureSaveConfirmationTests { ).authorize(request) let error = try #require(decision.denialError) - let failure = StructureApplyFailure(error) + let failure = GatedOperationFailure(error) #expect(failure.outcome == .refused) #expect(failure.message != nil) #expect(!failure.reportsFailure) @@ -404,7 +404,7 @@ struct StructureSaveConfirmationTests { @Test("A Cancel leaves the edits staged and shows nothing") func cancelIsQuiet() { - let failure = StructureApplyFailure(ExecutionGateError.cancelledByUser("Operation cancelled by user")) + let failure = GatedOperationFailure(ExecutionGateError.cancelledByUser("Operation cancelled by user")) #expect(failure == .cancelledByUser) #expect(failure.outcome == .refused) @@ -414,7 +414,7 @@ struct StructureSaveConfirmationTests { @Test("A Safe Mode refusal leaves the edits staged and says why, and is not a failed operation") func refusalExplainsItself() { - let failure = StructureApplyFailure(ExecutionGateError.denied("Safe Mode is read-only")) + let failure = GatedOperationFailure(ExecutionGateError.denied("Safe Mode is read-only")) #expect(failure.outcome == .refused) #expect(failure.message == "Safe Mode is read-only") @@ -423,7 +423,7 @@ struct StructureSaveConfirmationTests { @Test("A statement the server rejected is a failed save") func serverRejectionIsAFailure() { - let failure = StructureApplyFailure(ServerRejection()) + let failure = GatedOperationFailure(ServerRejection()) #expect(failure.outcome == .failed("Duplicate column name 'notes'")) #expect(failure.message == "Duplicate column name 'notes'") diff --git a/apple/TableProTests/Views/Structure/TriggerEditorFontSizeTests.swift b/apple/TableProTests/Views/Structure/TriggerEditorFontSizeTests.swift new file mode 100644 index 0000000000..3745115a46 --- /dev/null +++ b/apple/TableProTests/Views/Structure/TriggerEditorFontSizeTests.swift @@ -0,0 +1,25 @@ +import AppKit +@testable import TablePro +import TableProEditorKit +import Testing + +@MainActor +struct TriggerEditorFontSizeTests { + @Test("New and Edit Trigger open at the size the DDL pane is zoomed to") + func opensAtTheStoredSize() throws { + let zoomed = try EphemeralDefaults() + zoomed.defaults.set(18.0, forKey: TriggerEditorView.fontSizeKey) + + #expect(TriggerEditorView.initialConfiguration(in: zoomed.defaults).appearance.font.pointSize == 18) + } + + @Test("Without a stored size the sheet opens at the default") + func opensAtTheDefaultSize() throws { + let fresh = try EphemeralDefaults() + + #expect( + TriggerEditorView.initialConfiguration(in: fresh.defaults).appearance.font.pointSize + == CGFloat(TriggerEditorView.defaultFontSize) + ) + } +} diff --git a/core/crates/core/src/export/csv.rs b/core/crates/core/src/export/csv.rs index f3ff38e741..55ad66485b 100644 --- a/core/crates/core/src/export/csv.rs +++ b/core/crates/core/src/export/csv.rs @@ -184,11 +184,30 @@ mod tests { } } + fn sanitized() -> CsvOptions { + CsvOptions { + sanitize_formulas: true, + ..Default::default() + } + } + + // An importer reads the quote a sanitized field gains as part of the value. #[test] - fn csv_defaults_quote_every_field_when_sanitising() { + fn csv_defaults_write_values_as_they_are() { + let columns = cols(&["id", "handle"]); + let rows = vec![ + vec![Value::Int(1), Value::Text("@handle".into())], + vec![Value::Int(2), Value::Text("-pending".into())], + ]; + let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + assert_eq!(out, "id,handle\n1,@handle\n2,-pending\n"); + } + + #[test] + fn csv_sanitising_quotes_every_field() { let columns = cols(&["id", "name"]); let rows = vec![vec![Value::Int(1), Value::Text("Alice".into())]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out, "\"id\",\"name\"\n\"1\",\"Alice\"\n"); } @@ -266,7 +285,7 @@ mod tests { let rows = vec![vec![Value::Text("=cmd".into())]]; let opts = CsvOptions { quote: CsvQuote::Never, - ..Default::default() + ..sanitized() }; let out = render_csv(&columns, &rows, &opts).unwrap(); assert_eq!(out, "\"a\"\n\"'=cmd\"\n"); @@ -277,7 +296,7 @@ mod tests { let columns = cols(&["a"]); for ch in ['=', '+', '-', '@'] { let rows = vec![vec![Value::Text(format!("{ch}cmd"))]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out, format!("\"a\"\n\"'{ch}cmd\"\n"), "prefix {ch} should be sanitized"); } } @@ -287,7 +306,7 @@ mod tests { let columns = cols(&["a"]); for lead in ['\t', '\r', '\n'] { let rows = vec![vec![Value::Text(format!("{lead}=cmd|'/C calc'!A0"))]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); let expected = format!("\"a\"\n\"'{lead}=cmd|'/C calc'!A0\"\n"); assert_eq!(out, expected, "lead {lead:?} should be sanitized and quoted"); } @@ -298,7 +317,7 @@ mod tests { let columns = cols(&["a"]); for lead in ['\u{FF1D}', '\u{FF0B}', '\u{FF0D}', '\u{FF20}'] { let rows = vec![vec![Value::Text(format!("{lead}SUM(A1)"))]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!( out, format!("\"a\"\n\"'{lead}SUM(A1)\"\n"), @@ -319,7 +338,7 @@ mod tests { fn csv_does_not_sanitize_non_formula_prefixes() { let columns = cols(&["a"]); let rows = vec![vec![Value::Text("plain text".into())]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out, "\"a\"\n\"plain text\"\n"); } @@ -347,7 +366,7 @@ mod tests { #[test] fn csv_keeps_negative_int() { let columns = cols(&["a"]); - let out = render_csv(&columns, &[vec![Value::Int(-5)]], &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &[vec![Value::Int(-5)]], &sanitized()).unwrap(); assert_eq!(out, "\"a\"\n\"-5\"\n"); } @@ -355,14 +374,14 @@ mod tests { fn csv_keeps_negative_decimal() { let columns = cols(&["a"]); let rows = vec![vec![Value::Decimal("-12.30".parse().expect("a decimal"))]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out, "\"a\"\n\"-12.30\"\n"); } #[test] fn csv_keeps_negative_float() { let columns = cols(&["a"]); - let out = render_csv(&columns, &[vec![Value::Float64(-1.5)]], &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &[vec![Value::Float64(-1.5)]], &sanitized()).unwrap(); assert_eq!(out, "\"a\"\n\"-1.5\"\n"); } @@ -371,7 +390,7 @@ mod tests { let columns = cols(&["a"]); for text in ["-12", "+1.5e3", "-.5", "1.", "-7E-3"] { let rows = vec![vec![Value::Text(text.into())]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out, format!("\"a\"\n\"{text}\"\n"), "{text} is a number, not a formula"); } } @@ -381,7 +400,7 @@ mod tests { let columns = cols(&["a"]); for text in ["-cmd", "=SUM(A1)", "+1+cmd", "-", "-.", "-1e"] { let rows = vec![vec![Value::Text(text.into())]]; - let out = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out, format!("\"a\"\n\"'{text}\"\n"), "{text} must be neutralised"); } } @@ -401,7 +420,7 @@ mod tests { other("-2+cmd"), ]; for value in neutralised { - let out = render_csv(&columns, &[vec![value.clone()]], &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &[vec![value.clone()]], &sanitized()).unwrap(); assert!(out.contains("\"'"), "{value:?} reached the sheet as a formula: {out}"); } let kept = [ @@ -412,7 +431,7 @@ mod tests { (other("+1.5e3"), "+1.5e3"), ]; for (value, text) in kept { - let out = render_csv(&columns, &[vec![value]], &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &[vec![value]], &sanitized()).unwrap(); assert_eq!(out, format!("\"a\"\n\"{text}\"\n"), "{text} is data, not a formula"); } } @@ -420,14 +439,14 @@ mod tests { #[test] fn csv_neutralises_formula_header() { let columns = cols(&["=HYPERLINK(\"x\")"]); - let out = render_csv(&columns, &[], &CsvOptions::default()).unwrap(); + let out = render_csv(&columns, &[], &sanitized()).unwrap(); assert_eq!(out, "\"'=HYPERLINK(\"\"x\"\")\"\n"); } #[test] fn text_csv_treats_every_field_as_free_text() { let records = vec![vec!["-cmd".to_string(), "12".to_string(), String::new()]]; - let out = render_text_csv(&["q", "n", "e"], &records, &CsvOptions::default()).unwrap(); + let out = render_text_csv(&["q", "n", "e"], &records, &sanitized()).unwrap(); assert_eq!(out, "\"q\",\"n\",\"e\"\n\"'-cmd\",\"12\",\"\"\n"); } @@ -449,11 +468,11 @@ mod tests { fn csv_null_modes() { let columns = cols(&["a"]); let rows = vec![vec![Value::Null]]; - let out_empty = render_csv(&columns, &rows, &CsvOptions::default()).unwrap(); + let out_empty = render_csv(&columns, &rows, &sanitized()).unwrap(); assert_eq!(out_empty, "\"a\"\n\"\"\n"); let opts = CsvOptions { null_to_empty: false, - ..Default::default() + ..sanitized() }; let out_null = render_csv(&columns, &rows, &opts).unwrap(); assert_eq!(out_null, "\"a\"\n\"NULL\"\n"); diff --git a/core/crates/core/src/export/csv_options.rs b/core/crates/core/src/export/csv_options.rs index 3ff8f7adaf..409842ab0d 100644 --- a/core/crates/core/src/export/csv_options.rs +++ b/core/crates/core/src/export/csv_options.rs @@ -68,7 +68,9 @@ impl Default for CsvOptions { null_to_empty: true, line_break_to_space: false, header_row: true, - sanitize_formulas: true, + // RFC 4180 carries a field as written, and OWASP scopes the quote prefix to files meant + // for a spreadsheet: an importer keeps the quote as part of the value. + sanitize_formulas: false, delimiter: CsvDelimiter::Comma, quote: CsvQuote::IfNeeded, line_break: CsvLineBreak::Lf, diff --git a/core/crates/core/src/export/encoder/tests.rs b/core/crates/core/src/export/encoder/tests.rs index 1e65cb7aa8..6484071e73 100644 --- a/core/crates/core/src/export/encoder/tests.rs +++ b/core/crates/core/src/export/encoder/tests.rs @@ -377,7 +377,11 @@ fn csv_neutralises_text_not_negative_numbers_through_encoder() { vec![Value::Text("-12".into())], ]; - let out = streamed(CsvEncoder::new(columns, CsvOptions::default()), &rows); + let options = CsvOptions { + sanitize_formulas: true, + ..CsvOptions::default() + }; + let out = streamed(CsvEncoder::new(columns, options), &rows); assert_eq!( out, diff --git a/core/crates/mcp/src/export_tools.rs b/core/crates/mcp/src/export_tools.rs index 971b6b73a6..bdcf434f47 100644 --- a/core/crates/mcp/src/export_tools.rs +++ b/core/crates/mcp/src/export_tools.rs @@ -158,7 +158,15 @@ fn render( rows: &[Vec], ) -> Result { let encoded = match format { - ExportFormat::Csv => render_csv(columns, rows, &CsvOptions::default()), + // A tool result has no option to turn it on, so it keeps the neutralising the macOS tool result has. + ExportFormat::Csv => render_csv( + columns, + rows, + &CsvOptions { + sanitize_formulas: true, + ..CsvOptions::default() + }, + ), ExportFormat::Json => render_json(columns, rows), ExportFormat::Sql => { let dialect = session diff --git a/core/crates/storage/src/query_history/export.rs b/core/crates/storage/src/query_history/export.rs index b9d9bacdcc..a4753f5251 100644 --- a/core/crates/storage/src/query_history/export.rs +++ b/core/crates/storage/src/query_history/export.rs @@ -17,7 +17,11 @@ const HISTORY_CSV_HEADER: [&str; 10] = [ pub(super) fn history_csv(entries: &[Entry]) -> Result { let records: Vec> = entries.iter().map(history_record).collect(); - let options = tablepro_core::export::CsvOptions::default(); + // The history export has no option of its own, so it keeps neutralising rather than follow the export default. + let options = tablepro_core::export::CsvOptions { + sanitize_formulas: true, + ..tablepro_core::export::CsvOptions::default() + }; Ok(tablepro_core::export::render_text_csv( &HISTORY_CSV_HEADER, &records, diff --git a/core/crates/test-support/src/conformance/cases/large_table.rs b/core/crates/test-support/src/conformance/cases/large_table.rs index 4b77f07bfb..5dc295e763 100644 --- a/core/crates/test-support/src/conformance/cases/large_table.rs +++ b/core/crates/test-support/src/conformance/cases/large_table.rs @@ -6,7 +6,7 @@ use std::time::{Duration, Instant}; use async_trait::async_trait; use tablepro_core::browse::{ExportRequest, PageRead, PageRequest, PagingMode, build_export, read_page}; -use tablepro_core::export::{CsvEncoder, CsvOptions, RowEncoder}; +use tablepro_core::export::{CsvEncoder, CsvOptions, CsvQuote, RowEncoder}; use tablepro_core::meta::{PagePosition, RowIdentity, TableRef}; use tablepro_core::{ ColumnInfo, Connection, FilterSet, ReadStatement, ResultEvent, ResultSink, SinkFlow, UserSql, Value, WriteKind, @@ -538,7 +538,12 @@ impl ResultSink for CsvFileSink { match event { ResultEvent::ResultSetStarted { columns } => { lock(&self.tally).result_sets += 1; - let mut encoder = CsvEncoder::new(columns, CsvOptions::default()); + // `check_file` reads every field quoted. + let options = CsvOptions { + quote: CsvQuote::Always, + ..CsvOptions::default() + }; + let mut encoder = CsvEncoder::new(columns, options); encoder.begin(&mut self.buffer); self.encoder = Some(encoder); SinkFlow::Continue diff --git a/core/crates/test-support/src/conformance/cases/round_trip.rs b/core/crates/test-support/src/conformance/cases/round_trip.rs index 9e17db85c6..94f0a82d79 100644 --- a/core/crates/test-support/src/conformance/cases/round_trip.rs +++ b/core/crates/test-support/src/conformance/cases/round_trip.rs @@ -47,7 +47,7 @@ const IMPORT_DEADLINE: Duration = Duration::from_secs(120); /// that hangs. const EXPORT_DEADLINE: Duration = Duration::from_secs(120); -/// Under the export defaults these rows of 200-character labels make a +/// With every field quoted these rows of 200-character labels make a /// file just under `MAX_CSV_FILE_BYTES`. 200 characters is the widest /// label every engine's scratch table takes. const NEAR_LIMIT_ROWS: u64 = 315_000; @@ -428,7 +428,11 @@ pub async fn csv_export_near_the_import_limit_imports_back( FILL_DEADLINE, ) .await?; - let file = export_file(connection.as_ref(), &source, csv_encoder(CsvOptions::default())).await?; + let quoted = CsvOptions { + quote: CsvQuote::Always, + ..CsvOptions::default() + }; + let file = export_file(connection.as_ref(), &source, csv_encoder(quoted)).await?; let size = file.len(); expect( NEAR_LIMIT_CASE, diff --git a/linux/crates/app/src/ui/export/encoding_sink.rs b/linux/crates/app/src/ui/export/encoding_sink.rs index 947fcb8fa8..bb1b3bb509 100644 --- a/linux/crates/app/src/ui/export/encoding_sink.rs +++ b/linux/crates/app/src/ui/export/encoding_sink.rs @@ -239,7 +239,7 @@ mod tests { assert_eq!(finished, Some(1)); let text: Vec = chunks.into_iter().flat_map(|(bytes, _)| bytes).collect(); - assert_eq!(String::from_utf8(text).expect("utf-8"), "\"x\",\"y\"\n\"1\",\"2\"\n"); + assert_eq!(String::from_utf8(text).expect("utf-8"), "x,y\n1,2\n"); } #[tokio::test(start_paused = true)] diff --git a/linux/crates/app/src/ui/export/export_ledger/tests.rs b/linux/crates/app/src/ui/export/export_ledger/tests.rs index 97e830ca17..63fdf02bb8 100644 --- a/linux/crates/app/src/ui/export/export_ledger/tests.rs +++ b/linux/crates/app/src/ui/export/export_ledger/tests.rs @@ -250,8 +250,8 @@ fn entire_table_export_streams_every_row() { let text = std::fs::read_to_string(&path).expect("the export"); let lines: Vec<&str> = text.lines().collect(); assert_eq!(lines.len(), 25_001); - assert_eq!(lines[0], "\"id\",\"name\""); - assert_eq!(lines[25_000], "\"24999\",\"user 24999\""); + assert_eq!(lines[0], "id,name"); + assert_eq!(lines[25_000], "24999,user 24999"); assert_eq!( fake.connection.observed_deadlines("stream_read"), [None], @@ -298,7 +298,7 @@ fn entire_table_export_has_no_identity_columns() { "{sql}" ); let text = std::fs::read_to_string(&path).expect("the export"); - assert_eq!(text.lines().next(), Some("\"id\",\"name\"")); + assert_eq!(text.lines().next(), Some("id,name")); } #[gtk4::test] @@ -510,11 +510,7 @@ fn snapshot_export_of_10k_rows_keeps_main_loop_responsive() { assert!(ticks.get() > 0, "the main loop never ran while the export did"); let text = std::fs::read_to_string(&path).expect("the export"); assert_eq!(text.lines().count(), 10_001); - assert!( - text.starts_with("\"id\",\"name\",\"note\"\n\"0\",\"name 0\""), - "{}", - &text[..40] - ); + assert!(text.starts_with("id,name,note\n0,name 0"), "{}", &text[..40]); assert_eq!(harness.ledger.in_flight(), 0); assert_eq!(harness.hold.live(), 0, "the export kept the process held"); } diff --git a/linux/crates/platform/tests/settings.rs b/linux/crates/platform/tests/settings.rs index e95371763f..6939de58a3 100644 --- a/linux/crates/platform/tests/settings.rs +++ b/linux/crates/platform/tests/settings.rs @@ -213,7 +213,7 @@ async fn csv_options_round_trip() -> TestResult { null_to_empty: false, line_break_to_space: true, header_row: false, - sanitize_formulas: false, + sanitize_formulas: true, delimiter: CsvDelimiter::Pipe, quote: CsvQuote::Always, line_break: CsvLineBreak::CrLf, diff --git a/linux/data/app.tablepro.TablePro.gschema.xml.in b/linux/data/app.tablepro.TablePro.gschema.xml.in index 2044a4b947..af9c9f021a 100644 --- a/linux/data/app.tablepro.TablePro.gschema.xml.in +++ b/linux/data/app.tablepro.TablePro.gschema.xml.in @@ -324,7 +324,7 @@ - true + false Neutralise spreadsheet formulas Whether a CSV export prefixes values starting with =, +, - or @ so a spreadsheet does not run them. diff --git a/linux/docs/storage.md b/linux/docs/storage.md index f190f695be..ce39a839d1 100644 --- a/linux/docs/storage.md +++ b/linux/docs/storage.md @@ -378,7 +378,7 @@ The schema id is the application id and the path follows it: `app.tablepro.Table | `csv-null-to-empty` | `b` | `true` | | | `csv-line-break-to-space` | `b` | `false` | | | `csv-header-row` | `b` | `true` | | -| `csv-sanitize-formulas` | `b` | `true` | | +| `csv-sanitize-formulas` | `b` | `false` | | | `csv-delimiter` | enum | `comma` | comma, semicolon, tab, pipe | | `csv-quote` | enum | `if-needed` | always, if-needed, never | | `csv-line-break` | enum | `lf` | lf, crlf, cr | From 5b9d13a6df2e055f899b5c0d7d08d67602e7d308 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 11 Nov 2026 14:52:25 +0700 Subject: [PATCH 2/7] docs(macos): add the changelog fragments for #4185 --- apple/changelog.d/4185.fixed.1.md | 1 + apple/changelog.d/4185.fixed.10.md | 1 + apple/changelog.d/4185.fixed.11.md | 1 + apple/changelog.d/4185.fixed.12.md | 1 + apple/changelog.d/4185.fixed.13.md | 1 + apple/changelog.d/4185.fixed.14.md | 1 + apple/changelog.d/4185.fixed.15.md | 1 + apple/changelog.d/4185.fixed.16.md | 1 + apple/changelog.d/4185.fixed.17.md | 1 + apple/changelog.d/4185.fixed.18.md | 1 + apple/changelog.d/4185.fixed.19.md | 1 + apple/changelog.d/4185.fixed.2.md | 1 + apple/changelog.d/4185.fixed.20.md | 1 + apple/changelog.d/4185.fixed.21.md | 1 + apple/changelog.d/4185.fixed.22.md | 1 + apple/changelog.d/4185.fixed.23.md | 1 + apple/changelog.d/4185.fixed.24.md | 1 + apple/changelog.d/4185.fixed.25.md | 1 + apple/changelog.d/4185.fixed.26.md | 1 + apple/changelog.d/4185.fixed.27.md | 1 + apple/changelog.d/4185.fixed.28.md | 1 + apple/changelog.d/4185.fixed.29.md | 1 + apple/changelog.d/4185.fixed.3.md | 1 + apple/changelog.d/4185.fixed.30.md | 1 + apple/changelog.d/4185.fixed.31.md | 1 + apple/changelog.d/4185.fixed.32.md | 1 + apple/changelog.d/4185.fixed.4.md | 1 + apple/changelog.d/4185.fixed.5.md | 1 + apple/changelog.d/4185.fixed.6.md | 1 + apple/changelog.d/4185.fixed.7.md | 1 + apple/changelog.d/4185.fixed.8.md | 1 + apple/changelog.d/4185.fixed.9.md | 1 + apple/changelog.d/4185.fixed.md | 1 + linux/changelog.d/4185.fixed.md | 1 + 34 files changed, 34 insertions(+) create mode 100644 apple/changelog.d/4185.fixed.1.md create mode 100644 apple/changelog.d/4185.fixed.10.md create mode 100644 apple/changelog.d/4185.fixed.11.md create mode 100644 apple/changelog.d/4185.fixed.12.md create mode 100644 apple/changelog.d/4185.fixed.13.md create mode 100644 apple/changelog.d/4185.fixed.14.md create mode 100644 apple/changelog.d/4185.fixed.15.md create mode 100644 apple/changelog.d/4185.fixed.16.md create mode 100644 apple/changelog.d/4185.fixed.17.md create mode 100644 apple/changelog.d/4185.fixed.18.md create mode 100644 apple/changelog.d/4185.fixed.19.md create mode 100644 apple/changelog.d/4185.fixed.2.md create mode 100644 apple/changelog.d/4185.fixed.20.md create mode 100644 apple/changelog.d/4185.fixed.21.md create mode 100644 apple/changelog.d/4185.fixed.22.md create mode 100644 apple/changelog.d/4185.fixed.23.md create mode 100644 apple/changelog.d/4185.fixed.24.md create mode 100644 apple/changelog.d/4185.fixed.25.md create mode 100644 apple/changelog.d/4185.fixed.26.md create mode 100644 apple/changelog.d/4185.fixed.27.md create mode 100644 apple/changelog.d/4185.fixed.28.md create mode 100644 apple/changelog.d/4185.fixed.29.md create mode 100644 apple/changelog.d/4185.fixed.3.md create mode 100644 apple/changelog.d/4185.fixed.30.md create mode 100644 apple/changelog.d/4185.fixed.31.md create mode 100644 apple/changelog.d/4185.fixed.32.md create mode 100644 apple/changelog.d/4185.fixed.4.md create mode 100644 apple/changelog.d/4185.fixed.5.md create mode 100644 apple/changelog.d/4185.fixed.6.md create mode 100644 apple/changelog.d/4185.fixed.7.md create mode 100644 apple/changelog.d/4185.fixed.8.md create mode 100644 apple/changelog.d/4185.fixed.9.md create mode 100644 apple/changelog.d/4185.fixed.md create mode 100644 linux/changelog.d/4185.fixed.md diff --git a/apple/changelog.d/4185.fixed.1.md b/apple/changelog.d/4185.fixed.1.md new file mode 100644 index 0000000000..625f6be5b1 --- /dev/null +++ b/apple/changelog.d/4185.fixed.1.md @@ -0,0 +1 @@ +CSV export with a decimal comma rewrote text values that looked like decimals, such as a version 1.10. diff --git a/apple/changelog.d/4185.fixed.10.md b/apple/changelog.d/4185.fixed.10.md new file mode 100644 index 0000000000..c4812cbb1b --- /dev/null +++ b/apple/changelog.d/4185.fixed.10.md @@ -0,0 +1 @@ +Rewinding a save could fail on a unique key when an earlier edit in it had been undone before saving. diff --git a/apple/changelog.d/4185.fixed.11.md b/apple/changelog.d/4185.fixed.11.md new file mode 100644 index 0000000000..03913855f0 --- /dev/null +++ b/apple/changelog.d/4185.fixed.11.md @@ -0,0 +1 @@ +Rewind reported a row as missing when the save had changed its primary key. diff --git a/apple/changelog.d/4185.fixed.12.md b/apple/changelog.d/4185.fixed.12.md new file mode 100644 index 0000000000..427ed680e8 --- /dev/null +++ b/apple/changelog.d/4185.fixed.12.md @@ -0,0 +1 @@ +A value of __DEFAULT__ was saved as the column default when duplicating a row or typing it into a cell. diff --git a/apple/changelog.d/4185.fixed.13.md b/apple/changelog.d/4185.fixed.13.md new file mode 100644 index 0000000000..f3f1e23763 --- /dev/null +++ b/apple/changelog.d/4185.fixed.13.md @@ -0,0 +1 @@ +A table opened while another query waited for Safe Mode approval stayed empty with its loading indicator. diff --git a/apple/changelog.d/4185.fixed.14.md b/apple/changelog.d/4185.fixed.14.md new file mode 100644 index 0000000000..bfaa1727f7 --- /dev/null +++ b/apple/changelog.d/4185.fixed.14.md @@ -0,0 +1 @@ +Saving while closing a window could close it over unsaved edits in another connection or tab. diff --git a/apple/changelog.d/4185.fixed.15.md b/apple/changelog.d/4185.fixed.15.md new file mode 100644 index 0000000000..157fe467ec --- /dev/null +++ b/apple/changelog.d/4185.fixed.15.md @@ -0,0 +1 @@ +Saving a Cassandra row that another client deleted recreated it with every other column empty. diff --git a/apple/changelog.d/4185.fixed.16.md b/apple/changelog.d/4185.fixed.16.md new file mode 100644 index 0000000000..9b9cdee113 --- /dev/null +++ b/apple/changelog.d/4185.fixed.16.md @@ -0,0 +1 @@ +Running SET with NX or XX in the Redis console showed OK when the key was not written. diff --git a/apple/changelog.d/4185.fixed.17.md b/apple/changelog.d/4185.fixed.17.md new file mode 100644 index 0000000000..27e55c9db4 --- /dev/null +++ b/apple/changelog.d/4185.fixed.17.md @@ -0,0 +1 @@ +Pressing Stop as a MySQL tab closed could crash the app. diff --git a/apple/changelog.d/4185.fixed.18.md b/apple/changelog.d/4185.fixed.18.md new file mode 100644 index 0000000000..18a4d89177 --- /dev/null +++ b/apple/changelog.d/4185.fixed.18.md @@ -0,0 +1 @@ +A startup command that failed, such as SET ROLE, left the session running without it and nothing said so. diff --git a/apple/changelog.d/4185.fixed.19.md b/apple/changelog.d/4185.fixed.19.md new file mode 100644 index 0000000000..e052ba407c --- /dev/null +++ b/apple/changelog.d/4185.fixed.19.md @@ -0,0 +1 @@ +Exporting more than 5,000,000 MongoDB documents wrote the first 5,000,000 and reported success. diff --git a/apple/changelog.d/4185.fixed.2.md b/apple/changelog.d/4185.fixed.2.md new file mode 100644 index 0000000000..51fe78d8f7 --- /dev/null +++ b/apple/changelog.d/4185.fixed.2.md @@ -0,0 +1 @@ +Changing export or import options in another window changed a running export or import, or left foreign keys off. diff --git a/apple/changelog.d/4185.fixed.20.md b/apple/changelog.d/4185.fixed.20.md new file mode 100644 index 0000000000..a4aa8a48a5 --- /dev/null +++ b/apple/changelog.d/4185.fixed.20.md @@ -0,0 +1 @@ +Stopping a Teradata query ended the whole session, so volatile tables were lost and the next query failed. diff --git a/apple/changelog.d/4185.fixed.21.md b/apple/changelog.d/4185.fixed.21.md new file mode 100644 index 0000000000..21106f7c75 --- /dev/null +++ b/apple/changelog.d/4185.fixed.21.md @@ -0,0 +1 @@ +A query parameter such as zip code 00123 was sent to Trino as the number 123. diff --git a/apple/changelog.d/4185.fixed.22.md b/apple/changelog.d/4185.fixed.22.md new file mode 100644 index 0000000000..05b1cf291d --- /dev/null +++ b/apple/changelog.d/4185.fixed.22.md @@ -0,0 +1 @@ +TablePro could quit when the Copilot language server exited while TablePro was sending it a message. diff --git a/apple/changelog.d/4185.fixed.23.md b/apple/changelog.d/4185.fixed.23.md new file mode 100644 index 0000000000..244113e6d1 --- /dev/null +++ b/apple/changelog.d/4185.fixed.23.md @@ -0,0 +1 @@ +Run Current and the gutter cut a SQL Server IF, WHILE or TRY...CATCH block at each semicolon inside it. diff --git a/apple/changelog.d/4185.fixed.24.md b/apple/changelog.d/4185.fixed.24.md new file mode 100644 index 0000000000..d34f707ee7 --- /dev/null +++ b/apple/changelog.d/4185.fixed.24.md @@ -0,0 +1 @@ +Run Current split a PostgreSQL rule with several commands in parentheses at each semicolon. diff --git a/apple/changelog.d/4185.fixed.25.md b/apple/changelog.d/4185.fixed.25.md new file mode 100644 index 0000000000..4ff326bcbd --- /dev/null +++ b/apple/changelog.d/4185.fixed.25.md @@ -0,0 +1 @@ +New Trigger and Edit Trigger opened at 13 pt instead of the size the Structure tab's DDL pane was zoomed to. diff --git a/apple/changelog.d/4185.fixed.26.md b/apple/changelog.d/4185.fixed.26.md new file mode 100644 index 0000000000..65af010304 --- /dev/null +++ b/apple/changelog.d/4185.fixed.26.md @@ -0,0 +1 @@ +Moving the caret in a long MongoDB or Redis query re-read the whole document every time. diff --git a/apple/changelog.d/4185.fixed.27.md b/apple/changelog.d/4185.fixed.27.md new file mode 100644 index 0000000000..f0afd7c7e2 --- /dev/null +++ b/apple/changelog.d/4185.fixed.27.md @@ -0,0 +1 @@ +Column and table suggestions on iPhone and iPad were rebuilt on every keystroke. diff --git a/apple/changelog.d/4185.fixed.28.md b/apple/changelog.d/4185.fixed.28.md new file mode 100644 index 0000000000..27178d8fa0 --- /dev/null +++ b/apple/changelog.d/4185.fixed.28.md @@ -0,0 +1 @@ +The ClickHouse Parts tab hid its list and toolbar after a cancelled or failed partition action. diff --git a/apple/changelog.d/4185.fixed.29.md b/apple/changelog.d/4185.fixed.29.md new file mode 100644 index 0000000000..0ac47956a5 --- /dev/null +++ b/apple/changelog.d/4185.fixed.29.md @@ -0,0 +1 @@ +Dropping or detaching a ClickHouse partition failed on tables partitioned by several columns or not at all. diff --git a/apple/changelog.d/4185.fixed.3.md b/apple/changelog.d/4185.fixed.3.md new file mode 100644 index 0000000000..8fa5c94f07 --- /dev/null +++ b/apple/changelog.d/4185.fixed.3.md @@ -0,0 +1 @@ +Data sync and Copy as SQL wrote PostgreSQL binary values that stored other bytes with standard_conforming_strings off. diff --git a/apple/changelog.d/4185.fixed.30.md b/apple/changelog.d/4185.fixed.30.md new file mode 100644 index 0000000000..6be6044706 --- /dev/null +++ b/apple/changelog.d/4185.fixed.30.md @@ -0,0 +1 @@ +Cancelling the Safe Mode confirmation in an object editor or sheet showed an error. diff --git a/apple/changelog.d/4185.fixed.31.md b/apple/changelog.d/4185.fixed.31.md new file mode 100644 index 0000000000..c6d66b0771 --- /dev/null +++ b/apple/changelog.d/4185.fixed.31.md @@ -0,0 +1 @@ +Applying user changes ran ALTER and password statements for a user staged for drop, and overcounted pending changes. diff --git a/apple/changelog.d/4185.fixed.32.md b/apple/changelog.d/4185.fixed.32.md new file mode 100644 index 0000000000..98e47dea15 --- /dev/null +++ b/apple/changelog.d/4185.fixed.32.md @@ -0,0 +1 @@ +Moving the caret next to a bracket far below the visible text could crash the editor. diff --git a/apple/changelog.d/4185.fixed.4.md b/apple/changelog.d/4185.fixed.4.md new file mode 100644 index 0000000000..a1b1cd6aa7 --- /dev/null +++ b/apple/changelog.d/4185.fixed.4.md @@ -0,0 +1 @@ +Copy as SQL wrote DuckDB and Oracle binary values that DuckDB stored as text and Oracle rejected. diff --git a/apple/changelog.d/4185.fixed.5.md b/apple/changelog.d/4185.fixed.5.md new file mode 100644 index 0000000000..2aea41ab0b --- /dev/null +++ b/apple/changelog.d/4185.fixed.5.md @@ -0,0 +1 @@ +Data sync wrote an empty Oracle BLOB as NULL, which a NOT NULL column refused. diff --git a/apple/changelog.d/4185.fixed.6.md b/apple/changelog.d/4185.fixed.6.md new file mode 100644 index 0000000000..77fcfcfd1f --- /dev/null +++ b/apple/changelog.d/4185.fixed.6.md @@ -0,0 +1 @@ +Data sync and Copy as SQL wrote Redshift binary values in a form Redshift refused. diff --git a/apple/changelog.d/4185.fixed.7.md b/apple/changelog.d/4185.fixed.7.md new file mode 100644 index 0000000000..2b5c350f28 --- /dev/null +++ b/apple/changelog.d/4185.fixed.7.md @@ -0,0 +1 @@ +Add Rows in Shortcuts refused CSV with bracketed headers such as [id], as SQL Server exports them. diff --git a/apple/changelog.d/4185.fixed.8.md b/apple/changelog.d/4185.fixed.8.md new file mode 100644 index 0000000000..88dc60ec1c --- /dev/null +++ b/apple/changelog.d/4185.fixed.8.md @@ -0,0 +1 @@ +Add Rows in Shortcuts refused a header that differed from its column's name only in case. diff --git a/apple/changelog.d/4185.fixed.9.md b/apple/changelog.d/4185.fixed.9.md new file mode 100644 index 0000000000..081e98b9ba --- /dev/null +++ b/apple/changelog.d/4185.fixed.9.md @@ -0,0 +1 @@ +Copy to another database and Data Compare missed rows past a database's row limit without saying so. diff --git a/apple/changelog.d/4185.fixed.md b/apple/changelog.d/4185.fixed.md new file mode 100644 index 0000000000..a90c14e4ae --- /dev/null +++ b/apple/changelog.d/4185.fixed.md @@ -0,0 +1 @@ +CSV export put a single quote before values starting with =, +, - or @ by default, which then stayed in imports. diff --git a/linux/changelog.d/4185.fixed.md b/linux/changelog.d/4185.fixed.md new file mode 100644 index 0000000000..a90c14e4ae --- /dev/null +++ b/linux/changelog.d/4185.fixed.md @@ -0,0 +1 @@ +CSV export put a single quote before values starting with =, +, - or @ by default, which then stayed in imports. From 4bd9bc7dc60094b452cd050f80a2a5d907a6ae24 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 11 Nov 2026 17:26:04 +0700 Subject: [PATCH 3/7] build(pluginkit): raise the kit to 44 for the additive API --- apple/Configs/PluginKit.xcconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apple/Configs/PluginKit.xcconfig b/apple/Configs/PluginKit.xcconfig index fef1b5c3ce..207ba58516 100644 --- a/apple/Configs/PluginKit.xcconfig +++ b/apple/Configs/PluginKit.xcconfig @@ -1,3 +1,3 @@ // When to raise each, and how the build stamps them: .claude/rules/plugin-system.md, ABI. -TABLEPRO_PLUGINKIT_VERSION = 43 +TABLEPRO_PLUGINKIT_VERSION = 44 TABLEPRO_PLUGINKIT_MINIMUM_VERSION = 19 From e0003b0d9ed668ff77770b5caada10097b198340 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 11 Nov 2026 20:24:01 +0700 Subject: [PATCH 4/7] fix(plugins): bind parameters by type, carry the default mark, stream large reads, and read Oracle region zones --- .github/workflows/ci-lint.yml | 1 + .../Native/HanaBridge/internal/hana/bridge.go | 43 ++ .../Native/HanaBridge/internal/hana/cursor.go | 191 ++++++++ .../HanaBridge/internal/hana/cursor_test.go | 412 +++++++++++++++++ .../HanaBridge/internal/hana/envelope.go | 74 ++- .../internal/hana/fakedriver_test.go | 212 +++++++++ .../Native/HanaBridge/internal/hana/opslot.go | 10 + .../HanaBridge/internal/hana/session.go | 19 +- .../HanaBridge/internal/hana/statement.go | 91 +++- apple/Native/HanaBridge/server.go | 40 +- apple/Native/HanaBridge/server_test.go | 143 ++++++ .../TableProInfluxDBCore/InfluxCSV.swift | 53 ++- .../InfluxGridQuery.swift | 7 + .../InfluxGridStream.swift | 166 +++++++ .../TableProInfluxDBCore/InfluxRequest.swift | 11 +- .../InfluxResponseDecoder.swift | 44 +- .../MemcachedKeyListing.swift | 5 +- .../TableProQdrantCore/QdrantRowWrites.swift | 5 +- .../SQL/SpannerRowEditSQL.swift | 16 +- .../TeradataConnection.swift | 107 ++++- .../TeradataResultParser.swift | 86 +++- .../TableProTeradataCore/TeradataSocket.swift | 3 + .../InfluxGridStreamTests.swift | 224 +++++++++ .../InfluxRequestTests.swift | 32 ++ .../MemcachedKeyListingTests.swift | 8 + .../PluginTypedParameterBindingTests.swift | 189 ++++++++ .../QdrantRowWriteTests.swift | 12 +- .../TeradataReplyDecodingTests.swift | 10 +- .../TeradataStopTests.swift | 31 +- .../TeradataStreamTests.swift | 287 ++++++++++++ .../OracleCellFormatting.swift | 34 +- .../OracleCoreConnection.swift | 81 +++- .../OracleTimeZoneRegions.swift | 63 +++ .../OracleCellDecodingTests.swift | 15 +- .../OracleCellFormattingTests.swift | 57 ++- .../OracleTimeZoneRegionsTests.swift | 77 ++++ .../BigQueryPluginDriver.swift | 1 + .../BigQueryStatementGenerator.swift | 10 +- .../BigtablePluginDriver+Editing.swift | 13 +- .../BigtablePluginDriver.swift | 2 +- .../CassandraPlugin.swift | 1 + .../CassandraRowWriter.swift | 13 +- .../ClickHousePlugin.swift | 1 + .../ClickHouseRowWrites.swift | 8 +- .../ClickHouseDMLStatementTests.swift | 15 +- .../ColumnTypeCorpus/Corpus/oracle.json | 3 +- .../DamengDriverPlugin/DamengPlugin.swift | 2 +- .../DamengPluginDriver+Editing.swift | 18 +- .../DynamoDBPluginDriver.swift | 2 +- .../DynamoDBWriteStatements.swift | 16 +- .../ElasticsearchPluginDriver.swift | 30 +- .../EtcdDriverPlugin/EtcdPluginDriver.swift | 3 +- .../FirebasePluginDriver+Editing.swift | 1 + .../FirebasePluginDriver.swift | 2 +- .../HanaDriverPlugin/HanaBridgeContract.swift | 36 ++ .../HanaDriverPlugin/HanaConnection.swift | 82 ++++ .../HanaDriverPlugin/HanaHelperBridge.swift | 12 + .../HanaDriverPlugin/HanaHelperFrame.swift | 5 +- .../HanaDriverPlugin/HanaNativeBridge.swift | 3 + .../HanaDriverPlugin/HanaPluginDriver.swift | 73 +++ .../HanaDriverPlugin/HanaResultMapping.swift | 76 ++- .../HanaDriverPlugin/HanaSession.swift | 6 + apple/Plugins/HanaDriverPlugin/README.md | 3 +- .../InfluxDBConnection.swift | 54 ++- .../InfluxDBHTTPChunks.swift | 109 +++++ .../InfluxDBPluginDriver+Stream.swift | 106 +++++ .../InfluxDBPluginDriver.swift | 81 ++-- .../MSSQLDriverPlugin/MSSQLPlugin.swift | 1 + .../MSSQLStatementGenerator.swift | 13 +- .../MemcachedConnection.swift | 12 +- .../MemcachedPluginDriver.swift | 132 ++++-- .../MongoDBPluginDriver.swift | 2 +- .../MongoDBStatementGenerator.swift | 17 +- .../OracleDriverPlugin/OraclePlugin.swift | 1 + .../OracleDriverPlugin/OracleRowWriter.swift | 31 +- .../QdrantPluginDriver+Metadata.swift | 9 +- .../QdrantPluginDriver.swift | 2 +- .../RedisDriverPlugin/RedisPluginDriver.swift | 3 +- .../RedisColumnDefaultTests.swift | 50 ++ .../SnowflakePluginDriver.swift | 1 + .../SnowflakeStatementGenerator.swift | 11 +- .../SpannerPluginDriver+Editing.swift | 7 +- .../SpannerPluginDriver.swift | 5 +- .../SurrealDBPluginDriver.swift | 2 +- .../SurrealStatementGenerator.swift | 22 +- .../PluginBoundedStream.swift | 6 + .../PluginCapabilities.swift | 3 + .../PluginDatabaseDriver.swift | 43 +- .../TableProPluginKit/PluginDriverError.swift | 14 + .../PluginParameterBinder.swift | 15 + .../TableProPluginKit/PluginRowWrite.swift | 9 + .../PluginStatementContext.swift | 4 + .../TeradataAsyncConnection.swift | 17 + .../TeradataDriverPlugin/TeradataPlugin.swift | 80 +++- .../TeradataPluginDriver+Editing.swift | 6 +- .../TeradataRowEditSQL.swift | 12 +- .../TrinoDriverPlugin/TrinoPluginDriver.swift | 17 +- .../TrinoRowEditValues.swift | 9 +- .../TypesensePluginDriver.swift | 3 +- ...yExecutionCoordinator+MultiStatement.swift | 2 +- ...QueryExecutionCoordinator+Parameters.swift | 10 +- .../DataWrite/RowChangeStatementFactory.swift | 36 +- .../Core/Plugins/PluginDriverAdapter.swift | 8 +- .../Utilities/SQL/SQLParameterExtractor.swift | 4 +- .../SQL/SQLRowToStatementConverter.swift | 4 +- .../Models/Query/QueryParameter.swift | 48 ++ .../TablePro/Models/Query/QueryTabState.swift | 4 +- apple/TablePro/Models/Query/ResultSet.swift | 2 +- .../TablePro/Resources/Localizable.xcstrings | 12 + .../MainContentCoordinator+QueryHelpers.swift | 9 + .../Drivers/PluginRowWritingDriver.swift | 10 +- .../Drivers/PluginBackedDriverTests.swift | 28 ++ .../Mocks/StubPluginDriver.swift | 3 + ...hangeStatementFactoryExpressionTests.swift | 66 +++ ...inDriverAdapterStatementContextTests.swift | 39 ++ .../SQLParameterExtractorTests.swift | 16 +- .../Models/PaginationStateTests.swift | 9 +- .../MainContentCoordinatorSortTests.swift | 2 +- .../Views/Main/StopRollbackNoticeTests.swift | 40 ++ .../probes/check-oracle-time-zone-regions.sh | 434 ++++++++++++++++++ 120 files changed, 4568 insertions(+), 423 deletions(-) create mode 100644 apple/Native/HanaBridge/internal/hana/cursor.go create mode 100644 apple/Native/HanaBridge/internal/hana/cursor_test.go create mode 100644 apple/Native/HanaBridge/internal/hana/fakedriver_test.go create mode 100644 apple/Native/HanaBridge/server_test.go create mode 100644 apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridStream.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxGridStreamTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginTypedParameterBindingTests.swift create mode 100644 apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStreamTests.swift create mode 100644 apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTimeZoneRegions.swift create mode 100644 apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleTimeZoneRegionsTests.swift create mode 100644 apple/Plugins/InfluxDBDriverPlugin/InfluxDBHTTPChunks.swift create mode 100644 apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift create mode 100644 apple/Plugins/RedisDriverTests/RedisColumnDefaultTests.swift create mode 100644 apple/TableProTests/Views/Main/StopRollbackNoticeTests.swift create mode 100755 apple/scripts/probes/check-oracle-time-zone-regions.sh diff --git a/.github/workflows/ci-lint.yml b/.github/workflows/ci-lint.yml index e742097778..16789ed7a3 100644 --- a/.github/workflows/ci-lint.yml +++ b/.github/workflows/ci-lint.yml @@ -208,6 +208,7 @@ jobs: go mod verify go mod tidy -diff GOOS=darwin go vet ./... + go test ./... # go tool would build staticcheck for GOOS too, and Linux cannot run a darwin binary. go build -modfile=tools.mod -o "$RUNNER_TEMP/staticcheck" honnef.co/go/tools/cmd/staticcheck GOOS=darwin "$RUNNER_TEMP/staticcheck" ./... diff --git a/apple/Native/HanaBridge/internal/hana/bridge.go b/apple/Native/HanaBridge/internal/hana/bridge.go index 64ba0bbf4d..43a6002574 100644 --- a/apple/Native/HanaBridge/internal/hana/bridge.go +++ b/apple/Native/HanaBridge/internal/hana/bridge.go @@ -28,6 +28,21 @@ func (b *Bridge) Explain(sessionID uint64, operationID uint64, requestJSON []byt return encoded, encodedFailure(bridgeFailure) } +func (b *Bridge) OpenCursor(sessionID uint64, operationID uint64, requestJSON []byte) (result []byte, failure []byte) { + encoded, bridgeFailure := b.openCursorOnSession(sessionID, operationID, requestJSON) + return encoded, encodedFailure(bridgeFailure) +} + +func (b *Bridge) Fetch(sessionID uint64, operationID uint64) (result []byte, failure []byte) { + encoded, bridgeFailure := b.fetchOnSession(sessionID, operationID) + return encoded, encodedFailure(bridgeFailure) +} + +func (b *Bridge) CloseCursor(sessionID uint64, operationID uint64) (result []byte, failure []byte) { + encoded, bridgeFailure := b.closeCursorOnSession(sessionID, operationID) + return encoded, encodedFailure(bridgeFailure) +} + func (b *Bridge) Ping(sessionID uint64, operationID uint64) (failure []byte) { return encodedFailure(b.pingSession(sessionID, operationID)) } @@ -101,6 +116,34 @@ func (b *Bridge) explainOnSession(sessionID uint64, operationID uint64, requestJ return entry.explain(operationID, request) } +func (b *Bridge) openCursorOnSession(sessionID uint64, operationID uint64, requestJSON []byte) ([]byte, *bridgeError) { + entry, failure := b.sessions.lookup(sessionID) + if failure != nil { + return nil, failure + } + request, failure := decodeRequest[cursorRequest](requestJSON) + if failure != nil { + return nil, failure + } + return entry.openCursor(operationID, request) +} + +func (b *Bridge) fetchOnSession(sessionID uint64, operationID uint64) ([]byte, *bridgeError) { + entry, failure := b.sessions.lookup(sessionID) + if failure != nil { + return nil, failure + } + return entry.fetch(operationID) +} + +func (b *Bridge) closeCursorOnSession(sessionID uint64, operationID uint64) ([]byte, *bridgeError) { + entry, failure := b.sessions.lookup(sessionID) + if failure != nil { + return nil, failure + } + return entry.closeCursor(operationID) +} + func (b *Bridge) pingSession(sessionID uint64, operationID uint64) *bridgeError { entry, failure := b.sessions.lookup(sessionID) if failure != nil { diff --git a/apple/Native/HanaBridge/internal/hana/cursor.go b/apple/Native/HanaBridge/internal/hana/cursor.go new file mode 100644 index 0000000000..949f2b9667 --- /dev/null +++ b/apple/Native/HanaBridge/internal/hana/cursor.go @@ -0,0 +1,191 @@ +package hana + +import ( + "context" + "database/sql" + "sync/atomic" +) + +const ( + defaultCursorBatchRows = 1_000 + maxCursorBatchRows = 100_000 + cursorBatchBytes = 4 << 20 + noCursorMessage = "no cursor is open for this operation" + cursorParametersMessage = "a cursor takes no parameters" +) + +type cursorRequest struct { + executeRequest + BatchRows int `json:"batchRows"` +} + +func (r cursorRequest) batchRows() int { + if r.BatchRows <= 0 { + return defaultCursorBatchRows + } + return min(r.BatchRows, maxCursorBatchRows) +} + +type cursor struct { + op *operation + rows *sql.Rows + reader *resultReader + maxRows int + maxBytes int +} + +func (c *cursor) nextBatch() (*rowBatch, error) { + batch := &rowBatch{maxRows: c.maxRows, maxBytes: c.maxBytes} + ended, err := c.reader.read(batch) + batch.done = ended + return batch, err +} + +type openedStatement struct { + header *resultEnvelope + rows *sql.Rows + columns []columnInfo +} + +func (s *session) openCursor(operationID uint64, request cursorRequest) ([]byte, *bridgeError) { + if request.hasParameters() { + return nil, internalError(cursorParametersMessage) + } + // Once the result exists a stop sends no CANCEL SESSION: it would mostly reach an idle session, and a refused + // cancel severs the session. The row loop polls the stop flag, and the forced sever covers an unanswered fetch. + var resultReady atomic.Bool + op, conn, failure := s.beginOperation(operationID, func(reason stopReason) { + if !resultReady.Load() { + s.interruptStatement(reason) + } + }) + if failure != nil { + return nil, failure + } + op.stopAfter(secondsDuration(request.TimeoutSeconds)) + opened, err := openStatement(conn, op, request.SQL) + if err != nil { + return nil, s.endOperation(op, err) + } + if opened.rows == nil { + s.slot.finish(op) + opened.header.sessionLost = s.currentState() == sessionLost + return appendCursorOpenedJSON(nil, opened.header, &rowBatch{done: true}), nil + } + resultReady.Store(true) + // The query timeout covers producing the result. Moving its rows can take far longer. + op.disarmDeadline() + current := &cursor{ + op: op, + rows: opened.rows, + reader: &resultReader{rows: opened.rows, columns: opened.columns, op: op}, + maxRows: request.batchRows(), + maxBytes: cursorBatchBytes, + } + batch, failure := s.advance(current) + if failure != nil { + return nil, failure + } + opened.header.sessionLost = s.currentState() == sessionLost + return appendCursorOpenedJSON(nil, opened.header, batch), nil +} + +func (s *session) fetch(operationID uint64) ([]byte, *bridgeError) { + current, failure := s.takeCursor(operationID) + if failure != nil { + return nil, failure + } + batch, failure := s.advance(current) + if failure != nil { + return nil, failure + } + return appendCursorBatchJSON(nil, batch, s.currentState() == sessionLost), nil +} + +func (s *session) closeCursor(operationID uint64) ([]byte, *bridgeError) { + current, failure := s.takeCursor(operationID) + if failure != nil { + return nil, failure + } + _, err := runUnderWatchdog(cancelDeadline, s.loseConnection, current.rows.Close) + if isConnectionFailure(err) { + s.loseConnection() + } + s.slot.finish(current.op) + return appendCursorClosedJSON(nil, s.currentState() == sessionLost), nil +} + +func (s *session) advance(current *cursor) (*rowBatch, *bridgeError) { + batch, err := current.nextBatch() + if err == nil && !batch.done { + if s.park(current) { + return batch, nil + } + err = errSlotClosed + } + err = joinCleanupFailure(err, current.rows.Close()) + if failure := s.endOperation(current.op, err); failure != nil { + return nil, failure + } + return batch, nil +} + +func (s *session) endOperation(op *operation, err error) *bridgeError { + op.settle() + var failure *bridgeError + if err != nil { + failure = s.failure(op, err) + } + s.slot.finish(op) + return failure +} + +func (s *session) takeCursor(operationID uint64) (*cursor, *bridgeError) { + s.mu.Lock() + defer s.mu.Unlock() + if s.state == sessionClosed { + return nil, closedError() + } + current := s.parkedCursor + if current == nil || current.op.id != operationID { + return nil, internalError(noCursorMessage) + } + s.parkedCursor = nil + return current, nil +} + +func (s *session) park(current *cursor) bool { + s.mu.Lock() + defer s.mu.Unlock() + if s.state == sessionClosed { + return false + } + s.parkedCursor = current + return true +} + +func openStatement(conn *sql.Conn, op *operation, statement string) (openedStatement, error) { + if op.stopped() { + return openedStatement{}, errOperationStopped + } + if routesToExec(statement) { + envelope, err := runCountingStatement(conn, statement) + return openedStatement{header: envelope}, err + } + rows, err := conn.QueryContext(context.Background(), statement) + if err != nil { + return openedStatement{}, err + } + found, err := moveToFirstResultSet(rows) + var columns []columnInfo + if err == nil && found { + columns, err = describeColumns(rows) + } + if err != nil || !found { + if err = joinCleanupFailure(err, rows.Close()); err != nil { + return openedStatement{}, err + } + return openedStatement{header: affectedRowsEnvelope(0)}, nil + } + return openedStatement{header: tabularEnvelope(columns), rows: rows, columns: columns}, nil +} diff --git a/apple/Native/HanaBridge/internal/hana/cursor_test.go b/apple/Native/HanaBridge/internal/hana/cursor_test.go new file mode 100644 index 0000000000..caea056889 --- /dev/null +++ b/apple/Native/HanaBridge/internal/hana/cursor_test.go @@ -0,0 +1,412 @@ +package hana + +import ( + "encoding/json" + "slices" + "strconv" + "strings" + "testing" + "time" +) + +const selectAll = "SELECT ID, NAME FROM T" + +type cursorReply struct { + Columns []string `json:"columns"` + ColumnTypeNames []string `json:"columnTypeNames"` + RowsAffected int64 `json:"rowsAffected"` + HasResultSet bool `json:"hasResultSet"` + Rows [][]*string `json:"rows"` + TruncatedLobCount int `json:"truncatedLobCount"` + Done bool `json:"done"` + SessionLost bool `json:"sessionLost"` + IsTruncated bool `json:"isTruncated"` +} + +func decodeReply(t *testing.T, encoded []byte, failure *bridgeError) cursorReply { + t.Helper() + if failure != nil { + t.Fatalf("the helper answered with a failure: %v", failure) + } + var reply cursorReply + if err := json.Unmarshal(encoded, &reply); err != nil { + t.Fatalf("the reply is not JSON: %v\n%s", err, encoded) + } + return reply +} + +func requireRefused(t *testing.T, encoded []byte, failure *bridgeError, kind string) *bridgeError { + t.Helper() + if failure == nil { + t.Fatalf("the helper answered %s, want a %s failure", encoded, kind) + } + if failure.Kind != kind { + t.Fatalf("the failure is %v, want kind %s", failure, kind) + } + return failure +} + +func cursorOver(sql string, batchRows int) cursorRequest { + return cursorRequest{executeRequest: executeRequest{SQL: sql}, BatchRows: batchRows} +} + +func mustOpen(t *testing.T, entry *session, operationID uint64, request cursorRequest) cursorReply { + t.Helper() + encoded, failure := entry.openCursor(operationID, request) + return decodeReply(t, encoded, failure) +} + +func mustFetch(t *testing.T, entry *session, operationID uint64) cursorReply { + t.Helper() + encoded, failure := entry.fetch(operationID) + return decodeReply(t, encoded, failure) +} + +func mustExecute(t *testing.T, entry *session, operationID uint64, request executeRequest) cursorReply { + t.Helper() + encoded, failure := entry.execute(operationID, request) + return decodeReply(t, encoded, failure) +} + +func requireFetchRefused(t *testing.T, entry *session, operationID uint64) *bridgeError { + t.Helper() + encoded, failure := entry.fetch(operationID) + return requireRefused(t, encoded, failure, kindInternal) +} + +func parkedOperation(entry *session) *operation { + entry.mu.Lock() + defer entry.mu.Unlock() + return entry.parkedCursor.op +} + +func firstColumn(rows [][]*string) []string { + values := make([]string, 0, len(rows)) + for _, row := range rows { + if row[0] == nil { + values = append(values, "NULL") + continue + } + values = append(values, *row[0]) + } + return values +} + +func numbered(count int) []string { + values := make([]string, count) + for index := range values { + values[index] = strconv.Itoa(index) + } + return values +} + +func drain(t *testing.T, entry *session, operationID uint64, opened cursorReply) (batchSizes []int, ids []string) { + t.Helper() + reply := opened + batchSizes = append(batchSizes, len(reply.Rows)) + ids = append(ids, firstColumn(reply.Rows)...) + for !reply.Done { + reply = mustFetch(t, entry, operationID) + batchSizes = append(batchSizes, len(reply.Rows)) + ids = append(ids, firstColumn(reply.Rows)...) + } + return batchSizes, ids +} + +func TestCursorReadsEveryRowInBatchesAndMarksTheLastDone(t *testing.T) { + server := newFakeServer(table(10)) + entry := connectedSession(t, server) + + opened := mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + + if !slices.Equal(opened.Columns, []string{"ID", "NAME"}) { + t.Fatalf("columns are %v", opened.Columns) + } + if !slices.Equal(opened.ColumnTypeNames, []string{"INTEGER", "NVARCHAR"}) { + t.Fatalf("column types are %v", opened.ColumnTypeNames) + } + if !opened.HasResultSet || opened.Done { + t.Fatalf("the opened cursor reads hasResultSet %v, done %v", opened.HasResultSet, opened.Done) + } + sizes, ids := drain(t, entry, 1, opened) + if !slices.Equal(sizes, []int{3, 3, 3, 1}) { + t.Fatalf("batch sizes are %v, want [3 3 3 1]", sizes) + } + if !slices.Equal(ids, numbered(10)) { + t.Fatalf("rows arrived as %v", ids) + } + if !server.lastRows().closed.Load() { + t.Fatal("the rows stayed open after the last batch") + } + requireFetchRefused(t, entry, 1) + requireSlotFree(t, entry, 2) +} + +func TestCursorMarksAnExactMultipleDoneOnItsLastBatch(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(6))) + + opened := mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + if opened.Done { + t.Fatal("the first of two full batches reads done") + } + last := mustFetch(t, entry, 1) + + if len(last.Rows) != 3 || !last.Done { + t.Fatalf("the last batch has %d rows and done %v, want 3 and true", len(last.Rows), last.Done) + } + requireSlotFree(t, entry, 2) +} + +func TestCursorOverNoRowsSendsTheHeaderAndDone(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(0))) + + opened := mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + + if len(opened.Rows) != 0 || !opened.Done || !opened.HasResultSet { + t.Fatalf("an empty result opened with %d rows, done %v, hasResultSet %v", len(opened.Rows), opened.Done, opened.HasResultSet) + } + if !slices.Equal(opened.Columns, []string{"ID", "NAME"}) { + t.Fatalf("columns are %v", opened.Columns) + } + requireSlotFree(t, entry, 2) +} + +func TestCursorEndsABatchAtItsByteBudget(t *testing.T) { + wide := table(10) + wide.name = strings.Repeat("x", 1<<20) + entry := connectedSession(t, newFakeServer(wide)) + + opened := mustOpen(t, entry, 1, cursorOver(selectAll, 1_000)) + sizes, ids := drain(t, entry, 1, opened) + + if !slices.Equal(sizes, []int{4, 4, 2}) { + t.Fatalf("batch sizes are %v, want [4 4 2] for 1 MiB rows under a 4 MiB budget", sizes) + } + if !slices.Equal(ids, numbered(10)) { + t.Fatalf("rows arrived as %v", ids) + } +} + +func TestStatementWithoutAResultSetOpensNoCursor(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + + opened := mustOpen(t, entry, 1, cursorOver("INSERT INTO T VALUES (1, 'a')", 3)) + + if opened.HasResultSet || !opened.Done || opened.RowsAffected != 1 || len(opened.Rows) != 0 { + t.Fatalf("an insert opened as %+v", opened) + } + requireFetchRefused(t, entry, 1) + requireSlotFree(t, entry, 2) +} + +func TestClosingACursorEarlyClosesItsRowsAndFreesTheSlot(t *testing.T) { + server := newFakeServer(table(10)) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + + encoded, failure := entry.closeCursor(1) + closed := decodeReply(t, encoded, failure) + + if closed.SessionLost { + t.Fatal("closing a cursor lost the session") + } + if !rows.closed.Load() { + t.Fatal("closing the cursor left its rows open") + } + if served := rows.served.Load(); served != 4 { + t.Fatalf("the server handed out %d rows, want the 3 sent and the 1 read ahead", served) + } + requireFetchRefused(t, entry, 1) + requireSlotFree(t, entry, 2) +} + +func TestStopBetweenFetchesFailsTheNextFetchWithoutCancellingOnTheServer(t *testing.T) { + server := newFakeServer(table(10)) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + op := parkedOperation(entry) + + entry.cancel(1) + op.inFlight.Wait() + encoded, failure := entry.fetch(1) + + requireRefused(t, encoded, failure, kindCancelled) + if !rows.closed.Load() { + t.Fatal("a stopped cursor left its rows open") + } + for _, statement := range server.executed() { + if strings.HasPrefix(statement, "ALTER SYSTEM CANCEL SESSION") { + t.Fatalf("a stop between fetches sent %q, though no statement was running", statement) + } + } + requireSlotFree(t, entry, 2) +} + +func TestStopDuringAFetchEndsItAtTheNextRowWithoutCancellingOnTheServer(t *testing.T) { + held := table(10) + held.hold = holdRow(4) + server := newFakeServer(held) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + op := parkedOperation(entry) + type outcome struct { + encoded []byte + failure *bridgeError + } + fetched := make(chan outcome, 1) + go func() { + encoded, failure := entry.fetch(1) + fetched <- outcome{encoded: encoded, failure: failure} + }() + select { + case <-held.hold.reached: + case <-time.After(5 * time.Second): + t.Fatal("the fetch never reached the held row") + } + + entry.cancel(1) + op.inFlight.Wait() + close(held.hold.released) + + var result outcome + select { + case result = <-fetched: + case <-time.After(5 * time.Second): + t.Fatal("the stopped fetch never answered") + } + requireRefused(t, result.encoded, result.failure, kindCancelled) + if served := rows.served.Load(); served != 5 { + t.Fatalf("the server handed out %d rows, want the fetch to stop at the row it was reading", served) + } + if !rows.closed.Load() { + t.Fatal("a fetch stopped in flight left the rows open") + } + for _, statement := range server.executed() { + if strings.HasPrefix(statement, "ALTER SYSTEM CANCEL SESSION") { + t.Fatalf("a stop after the result existed sent %q", statement) + } + } + requireSlotFree(t, entry, 2) +} + +func TestFetchThatFailsClosesTheCursor(t *testing.T) { + failing := table(10) + failing.failAt = 5 + server := newFakeServer(failing) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + + encoded, failure := entry.fetch(1) + + refused := requireRefused(t, encoded, failure, kindInternal) + if !strings.Contains(refused.Message, errFakeRead.Error()) { + t.Fatalf("the failure reads %q, want the server's read error", refused.Message) + } + if !rows.closed.Load() { + t.Fatal("a failed fetch left the rows open") + } + requireFetchRefused(t, entry, 1) + // The next statement reads a table that no longer fails, so only a held slot can fail it. + server.mu.Lock() + server.table.failAt = -1 + server.mu.Unlock() + requireSlotFree(t, entry, 2) +} + +func TestFetchWithNoOpenCursorIsRefused(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + + refused := requireFetchRefused(t, entry, 1) + if refused.Message != noCursorMessage { + t.Fatalf("the refusal reads %q", refused.Message) + } + encoded, failure := entry.closeCursor(1) + requireRefused(t, encoded, failure, kindInternal) + requireSlotFree(t, entry, 2) +} + +func TestFetchForAnotherOperationLeavesTheCursorOpen(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + mustOpen(t, entry, 5, cursorOver(selectAll, 3)) + + requireFetchRefused(t, entry, 6) + + next := mustFetch(t, entry, 5) + if !slices.Equal(firstColumn(next.Rows), []string{"3", "4", "5"}) { + t.Fatalf("the cursor resumed at %v", firstColumn(next.Rows)) + } +} + +func TestCursorRefusesParameters(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + request := cursorOver(selectAll+" WHERE ID = ?", 3) + request.Parameters = &[]cell{textCell("1")} + + encoded, failure := entry.openCursor(1, request) + + requireRefused(t, encoded, failure, kindInternal) + requireSlotFree(t, entry, 2) +} + +func TestQueryTimeoutStopsCoveringTheCursorOnceItIsOpen(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + request := cursorOver(selectAll, 3) + request.TimeoutSeconds = 0.2 + + mustOpen(t, entry, 1, request) + + op := parkedOperation(entry) + op.mu.Lock() + armed := op.deadline != nil + op.mu.Unlock() + if armed { + t.Fatal("the query timeout is still armed after the cursor opened") + } + // Twice the timeout: a deadline left armed would have stopped the cursor by now. + time.Sleep(400 * time.Millisecond) + if op.stopped() { + t.Fatal("the query timeout stopped an open cursor") + } + if next := mustFetch(t, entry, 1); len(next.Rows) != 3 { + t.Fatalf("a fetch past the timeout read %d rows", len(next.Rows)) + } +} + +func TestClosingTheSessionClosesAParkedCursor(t *testing.T) { + server := newFakeServer(table(10)) + entry := connectedSession(t, server) + mustOpen(t, entry, 1, cursorOver(selectAll, 3)) + rows := server.lastRows() + + entry.close() + + deadline := time.Now().Add(5 * time.Second) + for !rows.closed.Load() { + if time.Now().After(deadline) { + t.Fatal("closing the session left the parked cursor's rows open") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestCollectedExecuteStillCutsAtTheRowCap(t *testing.T) { + entry := connectedSession(t, newFakeServer(table(10))) + + cut := mustExecute(t, entry, 1, executeRequest{SQL: selectAll, RowCap: 4}) + whole := mustExecute(t, entry, 2, executeRequest{SQL: selectAll, RowCap: 10}) + uncapped := mustExecute(t, entry, 3, executeRequest{SQL: selectAll}) + + if !slices.Equal(firstColumn(cut.Rows), numbered(4)) || !cut.IsTruncated { + t.Fatalf("a cap of 4 read %v, truncated %v", firstColumn(cut.Rows), cut.IsTruncated) + } + if len(whole.Rows) != 10 || whole.IsTruncated { + t.Fatalf("a cap equal to the row count read %d rows, truncated %v", len(whole.Rows), whole.IsTruncated) + } + if len(uncapped.Rows) != 10 || uncapped.IsTruncated { + t.Fatalf("no cap read %d rows, truncated %v", len(uncapped.Rows), uncapped.IsTruncated) + } +} diff --git a/apple/Native/HanaBridge/internal/hana/envelope.go b/apple/Native/HanaBridge/internal/hana/envelope.go index 5b5d3a743b..39210549fd 100644 --- a/apple/Native/HanaBridge/internal/hana/envelope.go +++ b/apple/Native/HanaBridge/internal/hana/envelope.go @@ -154,12 +154,8 @@ func tabularEnvelope(columns []columnInfo) *resultEnvelope { } func (e *resultEnvelope) appendJSON(buffer []byte) []byte { - buffer = append(buffer, `{"columns":`...) - buffer = appendStringArray(buffer, e.columns) - buffer = append(buffer, `,"columnTypeNames":`...) - buffer = appendStringArray(buffer, e.columnTypeNames) - buffer = append(buffer, `,"columnClassifications":`...) - buffer = appendOptionalStringArray(buffer, e.columnClassifications) + buffer = append(buffer, '{') + buffer = e.appendColumnFields(buffer) buffer = append(buffer, `,"rows":[`...) for index, row := range e.rows { if index > 0 { @@ -182,6 +178,72 @@ func (e *resultEnvelope) appendJSON(buffer []byte) []byte { return append(buffer, '}') } +func (e *resultEnvelope) appendColumnFields(buffer []byte) []byte { + buffer = append(buffer, `"columns":`...) + buffer = appendStringArray(buffer, e.columns) + buffer = append(buffer, `,"columnTypeNames":`...) + buffer = appendStringArray(buffer, e.columnTypeNames) + buffer = append(buffer, `,"columnClassifications":`...) + return appendOptionalStringArray(buffer, e.columnClassifications) +} + +type rowBatch struct { + maxRows int + maxBytes int + encodedRows []byte + count int + truncatedLobs int + done bool +} + +func (b *rowBatch) full() bool { + return b.count >= b.maxRows || len(b.encodedRows) >= b.maxBytes +} + +func (b *rowBatch) add(row []cell, truncatedLobs int) { + if b.count > 0 { + b.encodedRows = append(b.encodedRows, ',') + } + b.encodedRows = appendRow(b.encodedRows, row) + b.count++ + b.truncatedLobs += truncatedLobs +} + +func appendCursorOpenedJSON(buffer []byte, header *resultEnvelope, batch *rowBatch) []byte { + buffer = append(buffer, '{') + buffer = header.appendColumnFields(buffer) + buffer = append(buffer, `,"rowsAffected":`...) + buffer = strconv.AppendInt(buffer, header.rowsAffected, 10) + buffer = append(buffer, `,"hasResultSet":`...) + buffer = strconv.AppendBool(buffer, header.hasResultSet) + buffer = append(buffer, ',') + buffer = appendBatchFields(buffer, batch, header.sessionLost) + return append(buffer, '}') +} + +func appendCursorBatchJSON(buffer []byte, batch *rowBatch, sessionLost bool) []byte { + buffer = append(buffer, '{') + buffer = appendBatchFields(buffer, batch, sessionLost) + return append(buffer, '}') +} + +func appendBatchFields(buffer []byte, batch *rowBatch, sessionLost bool) []byte { + buffer = append(buffer, `"rows":[`...) + buffer = append(buffer, batch.encodedRows...) + buffer = append(buffer, `],"truncatedLobCount":`...) + buffer = strconv.AppendInt(buffer, int64(batch.truncatedLobs), 10) + buffer = append(buffer, `,"done":`...) + buffer = strconv.AppendBool(buffer, batch.done) + buffer = append(buffer, `,"sessionLost":`...) + return strconv.AppendBool(buffer, sessionLost) +} + +func appendCursorClosedJSON(buffer []byte, sessionLost bool) []byte { + buffer = append(buffer, `{"sessionLost":`...) + buffer = strconv.AppendBool(buffer, sessionLost) + return append(buffer, '}') +} + func appendRow(buffer []byte, row []cell) []byte { buffer = append(buffer, '[') for index, value := range row { diff --git a/apple/Native/HanaBridge/internal/hana/fakedriver_test.go b/apple/Native/HanaBridge/internal/hana/fakedriver_test.go new file mode 100644 index 0000000000..3ce08ef0e0 --- /dev/null +++ b/apple/Native/HanaBridge/internal/hana/fakedriver_test.go @@ -0,0 +1,212 @@ +package hana + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "io" + "reflect" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" +) + +var errFakeRead = errors.New("the fake server failed to read a row") + +type fakeTable struct { + rowCount int + // The row whose read fails, or -1. + failAt int + name string + hold *rowHold +} + +// Holds the read of one row until the test releases it, like a fetch waiting on the server. +type rowHold struct { + at int + reached chan struct{} + released chan struct{} +} + +func holdRow(at int) *rowHold { + return &rowHold{at: at, reached: make(chan struct{}, 1), released: make(chan struct{})} +} + +func table(rowCount int) fakeTable { + return fakeTable{rowCount: rowCount, failAt: -1} +} + +func (t fakeTable) nameOf(index int) string { + if t.name != "" { + return t.name + } + return "row " + strconv.Itoa(index) +} + +type fakeServer struct { + table fakeTable + + mu sync.Mutex + opened []*fakeRows + execs []string +} + +func newFakeServer(table fakeTable) *fakeServer { + return &fakeServer{table: table} +} + +func (f *fakeServer) Connect(context.Context) (driver.Conn, error) { + return &fakeConn{server: f}, nil +} + +func (f *fakeServer) Driver() driver.Driver { + return fakeDriver{} +} + +func (f *fakeServer) open() *fakeRows { + f.mu.Lock() + defer f.mu.Unlock() + rows := &fakeRows{table: f.table} + f.opened = append(f.opened, rows) + return rows +} + +func (f *fakeServer) lastRows() *fakeRows { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.opened) == 0 { + return nil + } + return f.opened[len(f.opened)-1] +} + +func (f *fakeServer) recordExec(statement string) { + f.mu.Lock() + defer f.mu.Unlock() + f.execs = append(f.execs, statement) +} + +func (f *fakeServer) executed() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.execs...) +} + +type fakeDriver struct{} + +func (fakeDriver) Open(string) (driver.Conn, error) { + return nil, errors.New("the fake server opens connections through its connector") +} + +type fakeConn struct { + server *fakeServer +} + +func (c *fakeConn) Prepare(string) (driver.Stmt, error) { + return nil, errors.New("the fake server prepares no statements") +} + +func (c *fakeConn) Close() error { + return nil +} + +func (c *fakeConn) Begin() (driver.Tx, error) { + return nil, errors.New("the fake server has no transactions") +} + +func (c *fakeConn) QueryContext(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + return c.server.open(), nil +} + +func (c *fakeConn) ExecContext(_ context.Context, statement string, _ []driver.NamedValue) (driver.Result, error) { + c.server.recordExec(statement) + return driver.RowsAffected(1), nil +} + +type fakeRows struct { + table fakeTable + served atomic.Int64 + closed atomic.Bool +} + +func (r *fakeRows) Columns() []string { + return []string{"ID", "NAME"} +} + +func (r *fakeRows) Close() error { + r.closed.Store(true) + return nil +} + +func (r *fakeRows) Next(dest []driver.Value) error { + index := int(r.served.Load()) + if hold := r.table.hold; hold != nil && index == hold.at { + hold.reached <- struct{}{} + <-hold.released + } + if index == r.table.failAt { + return errFakeRead + } + if index >= r.table.rowCount { + return io.EOF + } + dest[0] = int64(index) + dest[1] = r.table.nameOf(index) + r.served.Add(1) + return nil +} + +func (r *fakeRows) ColumnTypeDatabaseTypeName(index int) string { + if index == 0 { + return "INTEGER" + } + return "NVARCHAR" +} + +func (r *fakeRows) ColumnTypeScanType(index int) reflect.Type { + if index == 0 { + return reflect.TypeFor[int64]() + } + return reflect.TypeFor[string]() +} + +func connectedSession(t *testing.T, server *fakeServer) *session { + t.Helper() + db := sql.OpenDB(server) + conn, err := db.Conn(context.Background()) + if err != nil { + t.Fatalf("the fake server refused a connection: %v", err) + } + entry := &session{ + dialer: newTrackingDialer(nil, nil), + db: db, + controlDB: sql.OpenDB(server), + state: sessionConnected, + conn: conn, + connectionID: 1, + } + entry.slot = newOperationSlot(ForcedSeverGrace, entry.loseConnection) + t.Cleanup(entry.close) + return entry +} + +// A slot the cursor still held would block this statement forever. +func requireSlotFree(t *testing.T, entry *session, operationID uint64) { + t.Helper() + finished := make(chan *bridgeError, 1) + go func() { + _, failure := entry.execute(operationID, executeRequest{SQL: "SELECT ID, NAME FROM T"}) + finished <- failure + }() + select { + case failure := <-finished: + if failure != nil { + t.Fatalf("a statement after the cursor failed: %v", failure) + } + case <-time.After(5 * time.Second): + t.Fatal("the cursor still holds the operation slot") + } +} diff --git a/apple/Native/HanaBridge/internal/hana/opslot.go b/apple/Native/HanaBridge/internal/hana/opslot.go index 5afce97c6a..174fbfee15 100644 --- a/apple/Native/HanaBridge/internal/hana/opslot.go +++ b/apple/Native/HanaBridge/internal/hana/opslot.go @@ -154,6 +154,16 @@ func (o *operation) stopAfter(timeout time.Duration) { o.deadline = time.AfterFunc(timeout, func() { o.requestStop(stopTimedOut) }) } +// Unlike settle, the operation stays stoppable: a cursor still answers Stop after its timeout is gone. +func (o *operation) disarmDeadline() { + o.mu.Lock() + defer o.mu.Unlock() + if o.deadline != nil { + o.deadline.Stop() + o.deadline = nil + } +} + func (o *operation) settle() { o.mu.Lock() o.settled = true diff --git a/apple/Native/HanaBridge/internal/hana/session.go b/apple/Native/HanaBridge/internal/hana/session.go index 4e900bff2b..4e66ff3639 100644 --- a/apple/Native/HanaBridge/internal/hana/session.go +++ b/apple/Native/HanaBridge/internal/hana/session.go @@ -64,6 +64,8 @@ type session struct { state sessionState conn *sql.Conn connectionID int64 + // Set only between fetches; a fetch takes it out while it reads. + parkedCursor *cursor controlMu sync.Mutex control *sql.Conn @@ -267,7 +269,11 @@ func (s *session) adopt(conn *sql.Conn, connectionID int64) *bridgeError { } func (s *session) beginStatement(operationID uint64) (*operation, *sql.Conn, *bridgeError) { - op, err := s.slot.begin(operationID, s.interruptStatement) + return s.beginOperation(operationID, s.interruptStatement) +} + +func (s *session) beginOperation(operationID uint64, interrupt interruptFunc) (*operation, *sql.Conn, *bridgeError) { + op, err := s.slot.begin(operationID, interrupt) if err != nil { return nil, nil, s.failure(nil, err) } @@ -380,16 +386,23 @@ func (s *session) close() { s.state = sessionClosed conn := s.conn s.conn = nil + parked := s.parkedCursor + s.parkedCursor = nil s.mu.Unlock() s.slot.close() s.dialer.sever() - go s.release(conn) + go s.release(conn, parked) } -func (s *session) release(conn *sql.Conn) { +func (s *session) release(conn *sql.Conn, parked *cursor) { defer func() { _ = recover() }() + // Open rows hold the connection, so it cannot close before them. + if parked != nil { + _ = parked.rows.Close() + s.slot.finish(parked.op) + } if conn != nil { _ = conn.Close() } diff --git a/apple/Native/HanaBridge/internal/hana/statement.go b/apple/Native/HanaBridge/internal/hana/statement.go index a43e53deeb..5265d9176f 100644 --- a/apple/Native/HanaBridge/internal/hana/statement.go +++ b/apple/Native/HanaBridge/internal/hana/statement.go @@ -152,19 +152,27 @@ func rowsAffected(result sql.Result) int64 { } func readFirstResultSet(rows *sql.Rows, op *operation, rowLimit int) (*resultEnvelope, error) { + found, err := moveToFirstResultSet(rows) + if err != nil { + return nil, err + } + if !found { + return affectedRowsEnvelope(0), nil + } + return readResultSet(rows, op, rowLimit) +} + +func moveToFirstResultSet(rows *sql.Rows) (bool, error) { for { columns, err := rows.Columns() if err != nil { - return nil, err + return false, err } if len(columns) > 0 { - return readResultSet(rows, op, rowLimit) + return true, nil } if !rows.NextResultSet() { - if err := rows.Err(); err != nil { - return nil, err - } - return affectedRowsEnvelope(0), nil + return false, rows.Err() } } } @@ -175,25 +183,68 @@ func readResultSet(rows *sql.Rows, op *operation, rowLimit int) (*resultEnvelope return nil, err } envelope := tabularEnvelope(columns) - for rows.Next() { - if op.stopped() { - return nil, errOperationStopped + reader := &resultReader{rows: rows, columns: columns, op: op} + collected := &collectedRows{limit: rowLimit} + ended, err := reader.read(collected) + if err != nil { + return nil, err + } + envelope.rows = collected.rows + envelope.isTruncated = !ended + envelope.truncatedLobCount = collected.truncatedLobs + return envelope, nil +} + +type rowSink interface { + full() bool + add(row []cell, truncatedLobs int) +} + +type resultReader struct { + rows *sql.Rows + columns []columnInfo + op *operation + // Next has moved to a row that no sink has taken yet, so the next read starts with it. + pending bool +} + +// The bool is true once the result set has no rows left. +func (r *resultReader) read(sink rowSink) (bool, error) { + for { + if !r.pending { + if !r.rows.Next() { + return true, r.rows.Err() + } + r.pending = true } - if rowLimit > 0 && len(envelope.rows) >= rowLimit { - envelope.isTruncated = true - break + if r.op.stopped() { + return false, errOperationStopped } - row, truncatedLobs, err := scanRow(rows, columns, op) + if sink.full() { + return false, nil + } + row, truncatedLobs, err := scanRow(r.rows, r.columns, r.op) if err != nil { - return nil, err + return false, err } - envelope.rows = append(envelope.rows, row) - envelope.truncatedLobCount += truncatedLobs - } - if err := rows.Err(); err != nil { - return nil, err + r.pending = false + sink.add(row, truncatedLobs) } - return envelope, nil +} + +type collectedRows struct { + limit int + rows [][]cell + truncatedLobs int +} + +func (c *collectedRows) full() bool { + return c.limit > 0 && len(c.rows) >= c.limit +} + +func (c *collectedRows) add(row []cell, truncatedLobs int) { + c.rows = append(c.rows, row) + c.truncatedLobs += truncatedLobs } func describeColumns(rows *sql.Rows) (columns []columnInfo, err error) { diff --git a/apple/Native/HanaBridge/server.go b/apple/Native/HanaBridge/server.go index 511a302c9c..0645971207 100644 --- a/apple/Native/HanaBridge/server.go +++ b/apple/Native/HanaBridge/server.go @@ -13,16 +13,19 @@ import ( "github.com/TableProApp/TablePro/Native/HanaBridge/internal/hana" ) -const protocolVersion = 1 +const protocolVersion = 2 const ( - opcodeOpen byte = 1 - opcodeConnect byte = 2 - opcodeExecute byte = 3 - opcodeExplain byte = 4 - opcodePing byte = 5 - opcodeCancel byte = 6 - opcodeClose byte = 7 + opcodeOpen byte = 1 + opcodeConnect byte = 2 + opcodeExecute byte = 3 + opcodeExplain byte = 4 + opcodePing byte = 5 + opcodeCancel byte = 6 + opcodeClose byte = 7 + opcodeOpenCursor byte = 8 + opcodeFetch byte = 9 + opcodeCloseCursor byte = 10 ) const ( @@ -46,6 +49,9 @@ type backend interface { Connect(sessionID uint64, operationID uint64) (result []byte, failure []byte) Execute(sessionID uint64, operationID uint64, requestJSON []byte) (result []byte, failure []byte) Explain(sessionID uint64, operationID uint64, requestJSON []byte) (result []byte, failure []byte) + OpenCursor(sessionID uint64, operationID uint64, requestJSON []byte) (result []byte, failure []byte) + Fetch(sessionID uint64, operationID uint64) (result []byte, failure []byte) + CloseCursor(sessionID uint64, operationID uint64) (result []byte, failure []byte) Ping(sessionID uint64, operationID uint64) (failure []byte) Cancel(sessionID uint64, operationID uint64) Close(sessionID uint64) @@ -119,6 +125,12 @@ func (s *server) dispatch(request frame.Frame) { s.cancelOperation(request) case opcodeClose: s.closeSession(request) + case opcodeOpenCursor: + s.start(request, s.openCursor) + case opcodeFetch: + s.start(request, s.fetch) + case opcodeCloseCursor: + s.start(request, s.closeCursor) default: s.reply(request.ID, nil, hana.InternalFailure(fmt.Sprintf("unknown opcode %d", request.Code))) } @@ -157,6 +169,18 @@ func (s *server) explain(body requestBody) ([]byte, []byte) { return s.backend.Explain(body.Session, body.Operation, body.Request) } +func (s *server) openCursor(body requestBody) ([]byte, []byte) { + return s.backend.OpenCursor(body.Session, body.Operation, body.Request) +} + +func (s *server) fetch(body requestBody) ([]byte, []byte) { + return s.backend.Fetch(body.Session, body.Operation) +} + +func (s *server) closeCursor(body requestBody) ([]byte, []byte) { + return s.backend.CloseCursor(body.Session, body.Operation) +} + func (s *server) ping(body requestBody) ([]byte, []byte) { if failure := s.backend.Ping(body.Session, body.Operation); failure != nil { return nil, failure diff --git a/apple/Native/HanaBridge/server_test.go b/apple/Native/HanaBridge/server_test.go new file mode 100644 index 0000000000..4eab00f27c --- /dev/null +++ b/apple/Native/HanaBridge/server_test.go @@ -0,0 +1,143 @@ +package main + +import ( + "encoding/json" + "io" + "slices" + "sync" + "testing" + + "github.com/TableProApp/TablePro/Native/HanaBridge/internal/frame" +) + +type recordedCall struct { + name string + session uint64 + operation uint64 + request string +} + +type recordingBackend struct { + mu sync.Mutex + calls []recordedCall +} + +func (b *recordingBackend) record(call recordedCall) { + b.mu.Lock() + defer b.mu.Unlock() + b.calls = append(b.calls, call) +} + +func (b *recordingBackend) recorded() []recordedCall { + b.mu.Lock() + defer b.mu.Unlock() + return append([]recordedCall(nil), b.calls...) +} + +func (b *recordingBackend) Open([]byte) (uint64, []byte) { + return 1, nil +} + +func (b *recordingBackend) Connect(uint64, uint64) ([]byte, []byte) { + return []byte("{}"), nil +} + +func (b *recordingBackend) Execute(uint64, uint64, []byte) ([]byte, []byte) { + return []byte("{}"), nil +} + +func (b *recordingBackend) Explain(uint64, uint64, []byte) ([]byte, []byte) { + return []byte("{}"), nil +} + +func (b *recordingBackend) OpenCursor(sessionID uint64, operationID uint64, requestJSON []byte) ([]byte, []byte) { + b.record(recordedCall{name: "openCursor", session: sessionID, operation: operationID, request: string(requestJSON)}) + return []byte(`{"done":false}`), nil +} + +func (b *recordingBackend) Fetch(sessionID uint64, operationID uint64) ([]byte, []byte) { + b.record(recordedCall{name: "fetch", session: sessionID, operation: operationID}) + return []byte(`{"done":true}`), nil +} + +func (b *recordingBackend) CloseCursor(sessionID uint64, operationID uint64) ([]byte, []byte) { + b.record(recordedCall{name: "closeCursor", session: sessionID, operation: operationID}) + return nil, []byte(`{"kind":"internal","message":"no cursor"}`) +} + +func (b *recordingBackend) Ping(uint64, uint64) []byte { + return nil +} + +func (b *recordingBackend) Cancel(uint64, uint64) {} + +func (b *recordingBackend) Close(uint64) {} + +func (b *recordingBackend) CloseAll() {} + +func readReply(t *testing.T, replies io.Reader) frame.Frame { + t.Helper() + reply, err := frame.Read(replies, frame.MaxBodyLength) + if err != nil { + t.Fatalf("reading a reply failed: %v", err) + } + return reply +} + +func exchange(t *testing.T, requests io.Writer, replies io.Reader, request frame.Frame) frame.Frame { + t.Helper() + if err := frame.Write(requests, request); err != nil { + t.Fatalf("writing frame %d failed: %v", request.ID, err) + } + reply := readReply(t, replies) + if reply.ID != request.ID { + t.Fatalf("the reply answers frame %d, want %d", reply.ID, request.ID) + } + return reply +} + +func TestServerRoutesCursorOperationsAndAnnouncesProtocolTwo(t *testing.T) { + requestReader, requests := io.Pipe() + replies, replyWriter := io.Pipe() + backend := &recordingBackend{} + exited := make(chan int, 1) + go func() { + exited <- run(requestReader, replyWriter, backend) + _ = replyWriter.Close() + }() + + var greeting struct { + Protocol int `json:"protocol"` + } + if err := json.Unmarshal(readReply(t, replies).Body, &greeting); err != nil || greeting.Protocol != 2 { + t.Fatalf("the greeting names protocol %d (%v), want 2", greeting.Protocol, err) + } + + opened := exchange(t, requests, replies, frame.Frame{ID: 1, Code: opcodeOpenCursor, Body: []byte( + `{"session":4,"operation":9,"request":{"sql":"SELECT 1","batchRows":5}}`, + )}) + fetched := exchange(t, requests, replies, frame.Frame{ID: 2, Code: opcodeFetch, Body: []byte(`{"session":4,"operation":9}`)}) + closed := exchange(t, requests, replies, frame.Frame{ID: 3, Code: opcodeCloseCursor, Body: []byte(`{"session":4,"operation":9}`)}) + _ = requests.Close() + + if opened.Code != statusOK || string(opened.Body) != `{"done":false}` { + t.Fatalf("open answered %d %s", opened.Code, opened.Body) + } + if fetched.Code != statusOK || string(fetched.Body) != `{"done":true}` { + t.Fatalf("fetch answered %d %s", fetched.Code, fetched.Body) + } + if closed.Code != statusError { + t.Fatalf("a refused close answered status %d", closed.Code) + } + if code := <-exited; code != exitInputClosed { + t.Fatalf("the server exited with %d", code) + } + want := []recordedCall{ + {name: "openCursor", session: 4, operation: 9, request: `{"sql":"SELECT 1","batchRows":5}`}, + {name: "fetch", session: 4, operation: 9}, + {name: "closeCursor", session: 4, operation: 9}, + } + if got := backend.recorded(); !slices.Equal(got, want) { + t.Fatalf("the backend saw %+v, want %+v", got, want) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift index 2d96cde84c..482f9d9b15 100644 --- a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift @@ -6,43 +6,55 @@ import Foundation enum InfluxCSV { static func records(_ data: Data) -> [[String]] { var reader = Reader() - data.withUnsafeBytes { reader.read($0) } + reader.read(data) return reader.finish() } - private struct Reader { - var records: [[String]] = [] - var record: [String] = [] - var field: [UInt8] = [] - var inQuotes = false - var lineHasContent = false + // Holds its state between reads, so a body can be split anywhere: inside a quoted field, between the two quotes of + // an escaped one, or between the CR and LF of one line end. + struct Reader { + private var records: [[String]] = [] + private var record: [String] = [] + private var field: [UInt8] = [] + private var inQuotes = false + // A quote right after the one that closed a quoted field is an escaped quote inside it. + private var closedQuote = false + // An LF right after the CR that ended a record is the same line end. + private var endedOnCarriageReturn = false + private var lineHasContent = false + + mutating func read(_ data: Data) { + data.withUnsafeBytes { read($0) } + } mutating func read(_ bytes: UnsafeRawBufferPointer) { - var index = 0 - while index < bytes.count { - let byte = bytes[index] - index += 1 + for byte in bytes { + if endedOnCarriageReturn { + endedOnCarriageReturn = false + if byte == Self.lineFeed { continue } + } if inQuotes { - if byte == Self.quote, index < bytes.count, bytes[index] == Self.quote { - field.append(Self.quote) - index += 1 - } else if byte == Self.quote { + if byte == Self.quote { inQuotes = false + closedQuote = true } else { field.append(byte) } continue } + let afterClosingQuote = closedQuote + closedQuote = false switch byte { case Self.quote: + if afterClosingQuote { field.append(Self.quote) } inQuotes = true lineHasContent = true case Self.comma: endField() lineHasContent = true case Self.carriageReturn: - if index < bytes.count, bytes[index] == Self.lineFeed { index += 1 } endRecord() + endedOnCarriageReturn = true case Self.lineFeed: endRecord() default: @@ -52,9 +64,16 @@ enum InfluxCSV { } } + // The records completed so far. One still open stays for the next read. + mutating func drain() -> [[String]] { + let completed = records + records = [] + return completed + } + mutating func finish() -> [[String]] { if lineHasContent || !record.isEmpty { endRecord() } - return records + return drain() } private mutating func endField() { diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridQuery.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridQuery.swift index cdfa81e9cd..d874a4634b 100644 --- a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridQuery.swift +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridQuery.swift @@ -104,6 +104,13 @@ public struct InfluxGridQuery: Sendable, Equatable, Hashable { return marked ? InfluxStatement.marked(statement, language: language) : statement } + // A read that matched nothing comes back with no columns at all, so the grid still gets them from the schema. + public func emptyResult(schema: [InfluxColumn]) -> InfluxResultTable { + let names = projection.isEmpty ? schema.map(\.name) : projection + let types = Dictionary(schema.map { ($0.name, $0.typeName) }, uniquingKeysWith: { first, _ in first }) + return InfluxResultTable(columns: names, typeNames: names.map { types[$0] ?? InfluxTypeName.string }, rows: []) + } + // InfluxQL answers `COUNT(*)` with one `count_` per field, the point count only when there is one field. public static func rowCount(from table: InfluxResultTable) -> Int? { guard let row = table.rows.first else { return 0 } diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridStream.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridStream.swift new file mode 100644 index 0000000000..c03223406d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxGridStream.swift @@ -0,0 +1,166 @@ +import Foundation + +// A grid page or export read as its body arrives. Each batch holds the rows the collected read gives for the same bytes, +// under one header that the measurement's schema types, as `typed(by:)` types the collected table. +public struct InfluxGridStream { + public enum Body: Sendable, Equatable { + // 3.x SQL and InfluxQL: CSV, with the header record on the first record batch only. + case csv + // 2.x InfluxQL with `chunked=true`: one JSON object per line, each in the v1 shape. + case influxQLChunks + } + + static let batchRows = 5_000 + // A wide header shrinks the batch, so one holds about a million cells at any width. + static let batchCells = 1 << 20 + + private let body: Body + private let schema: [InfluxColumn] + private let emptyResult: InfluxResultTable + private var csv = InfluxCSV.Reader() + private var lines = InfluxJSONLines() + private var layout: InfluxResponseDecoder.V3Layout? + private var header: InfluxResultTable? + private var pending: [[String?]] = [] + private var bytesRead = 0 + private var rowsRead = 0 + + public init(body: Body, grid: InfluxGridQuery, schema: [InfluxColumn]) { + self.body = body + self.schema = schema + emptyResult = grid.emptyResult(schema: schema) + } + + // Every batch carries the same columns and types, and the first one's are the stream's header. + public mutating func consume(_ chunk: Data) throws -> [InfluxResultTable] { + switch body { + case .csv: + bytesRead += chunk.count + csv.read(chunk) + return try csvBatches(csv.drain()) + case .influxQLChunks: + var batches: [InfluxResultTable] = [] + for line in lines.consume(chunk) { + if let batch = try influxQLBatch(line) { batches.append(batch) } + } + return batches + } + } + + // What is left once the body ends. A stream that has sent nothing gets its header here: from the CSV header record, + // or from the schema when the body named no columns. + public mutating func finish() throws -> [InfluxResultTable] { + var batches: [InfluxResultTable] = [] + switch body { + case .csv: + batches = try csvBatches(csv.finish()) + if let layout, !pending.isEmpty || header == nil { + batches.append(batch(pending, layout: layout)) + pending = [] + } + case .influxQLChunks: + if let line = lines.finish(), let batch = try influxQLBatch(line) { + batches.append(batch) + } + } + if header == nil { + batches.append(fixedHeader(emptyResult)) + } + return batches + } + + private mutating func csvBatches(_ records: [[String]]) throws -> [InfluxResultTable] { + var records = records[...] + if layout == nil, let columns = records.popFirst() { + layout = InfluxResponseDecoder.V3Layout(columns: columns) + } + guard let layout, !records.isEmpty else { return [] } + rowsRead += records.count + try InfluxResponseDecoder.CellBudget.perByte(bytes: bytesRead).check(rows: rowsRead, columns: layout.columns.count) + let size = max(1, min(Self.batchRows, Self.batchCells / max(layout.columns.count, 1))) + var batches: [InfluxResultTable] = [] + for record in records { + pending.append(layout.row(record)) + if pending.count >= size { + batches.append(batch(pending, layout: layout)) + pending.removeAll(keepingCapacity: true) + } + } + return batches + } + + // The first batch fixes the header. A column the schema lacks is typed from that batch's values, where the collected + // read infers it from every row. + private mutating func batch(_ rows: [[String?]], layout: InfluxResponseDecoder.V3Layout) -> InfluxResultTable { + let header = self.header ?? fixedHeader(layout.table(rows).typed(by: schema)) + return InfluxResultTable(columns: header.columns, typeNames: header.typeNames, rows: rows) + } + + private mutating func influxQLBatch(_ line: Data) throws -> InfluxResultTable? { + let table = try InfluxResponseDecoder.decodeV1JSON(line) + guard !table.columns.isEmpty else { return nil } + guard let header else { + let typed = table.typed(by: schema) + _ = fixedHeader(typed) + return typed + } + if table.columns == header.columns { + return InfluxResultTable(columns: header.columns, typeNames: header.typeNames, rows: table.rows) + } + let position = Dictionary(header.columns.enumerated().map { ($1, $0) }, uniquingKeysWith: { first, _ in first }) + let targets = try table.columns.map { column -> Int in + guard let index = position[column] else { + throw InfluxError.malformedResponse( + String(format: String(localized: "column %@ first appears in a later chunk"), column) + ) + } + return index + } + let rows = table.rows.map { row -> [String?] in + var mapped = [String?](repeating: nil, count: header.columns.count) + for (index, target) in targets.enumerated() where index < row.count { + mapped[target] = row[index] + } + return mapped + } + return InfluxResultTable(columns: header.columns, typeNames: header.typeNames, rows: rows) + } + + private mutating func fixedHeader(_ table: InfluxResultTable) -> InfluxResultTable { + if let header { return header } + let fixed = InfluxResultTable(columns: table.columns, typeNames: table.typeNames, rows: []) + header = fixed + return fixed + } +} + +// The lines of a body that arrives in pieces, each line whole however the pieces split it. +struct InfluxJSONLines { + private var partial = Data() + + mutating func consume(_ chunk: Data) -> [Data] { + var lines: [Data] = [] + var start = chunk.startIndex + while let end = chunk[start...].firstIndex(of: Self.lineFeed) { + var line = Data() + swap(&line, &partial) + line.append(chunk[start ..< end]) + if !Self.isBlank(line) { lines.append(line) } + start = chunk.index(after: end) + } + partial.append(chunk[start...]) + return lines + } + + mutating func finish() -> Data? { + var line = Data() + swap(&line, &partial) + return Self.isBlank(line) ? nil : line + } + + private static func isBlank(_ line: Data) -> Bool { + line.allSatisfy { $0 == 0x20 || $0 == 0x09 || $0 == 0x0D || $0 == 0x0A } + } + + private static let lineFeed = UInt8(ascii: "\n") +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift index e70ee42d85..e254fb5ec0 100644 --- a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift @@ -76,6 +76,8 @@ public enum InfluxCredentials: Sendable, Equatable { public enum InfluxRequests { public static let bucketPageSize = 100 + // 2.x's own default for `chunked=true`. + public static let influxQLChunkSize = 10_000 // 3.x with auth on refuses an anonymous `/ping`. Before the version is known the token goes out as `Token`. public static func ping(credentials: InfluxCredentials, generation: InfluxGeneration? = nil) -> InfluxRequest { @@ -148,13 +150,16 @@ public enum InfluxRequests { InfluxRequest(method: "GET", path: "/api/v2/orgs", headers: authHeaders(credentials, generation: .v2)) } + // `streamed` asks 2.x InfluxQL for chunks: without it the server gathers the whole result in its memory before the + // first byte. 3.x writes CSV per record batch either way. public static func query( _ statement: InfluxStatement, database: String, retentionPolicy: String? = nil, organization: String, generation: InfluxGeneration, - credentials: InfluxCredentials + credentials: InfluxCredentials, + streamed: Bool = false ) throws -> InfluxRequest { var headers = authHeaders(credentials, generation: generation) switch (generation, statement.language) { @@ -190,6 +195,10 @@ public enum InfluxRequests { var form = [InfluxQueryItem("q", statement.text)] if !database.isEmpty { form.append(InfluxQueryItem("db", database)) } if let retentionPolicy, !retentionPolicy.isEmpty { form.append(InfluxQueryItem("rp", retentionPolicy)) } + if streamed { + form.append(InfluxQueryItem("chunked", "true")) + form.append(InfluxQueryItem("chunk_size", String(influxQLChunkSize))) + } return InfluxRequest( method: "POST", path: "/query", diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift index 4d1eb53d11..e9d6c7d518 100644 --- a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift @@ -6,24 +6,38 @@ public enum InfluxResponseDecoder { var records = InfluxCSV.records(data)[...] guard let columns = records.popFirst() else { return .empty } try CellBudget.perByte(of: data).check(rows: records.count, columns: columns.count) - let rows: [[String?]] = records.map { record in - columns.indices.map { index in + let layout = V3Layout(columns: columns) + return layout.table(records.map(layout.row)) + } + + // A 3.x CSV record as a row under its header. The stream maps records with this too, so a cell reads the same. + struct V3Layout { + let columns: [String] + // Only `time` is inferred as a timestamp, so it is the one column whose zone-less values get a `Z`. + private let timeColumns: [Int] + + init(columns: [String]) { + self.columns = columns + timeColumns = columns.indices.filter { columns[$0] == InfluxTypeName.timeColumn } + } + + func row(_ record: [String]) -> [String?] { + var row: [String?] = columns.indices.map { index in guard index < record.count, !record[index].isEmpty else { return nil } return record[index] } - } - let typeNames = columns.indices.map { index in - InfluxTypeName.inferred(column: columns[index], values: rows.map { $0[index] }) - } - let timestampColumns = typeNames.indices.filter { typeNames[$0] == InfluxTypeName.timestamp } - let normalized = timestampColumns.isEmpty ? rows : rows.map { row in - var row = row - for index in timestampColumns { + for index in timeColumns { row[index] = row[index].map(InfluxTypeName.normalizedTimestamp) } return row } - return InfluxResultTable(columns: columns, typeNames: typeNames, rows: normalized) + + func table(_ rows: [[String?]]) -> InfluxResultTable { + let typeNames = columns.indices.map { index in + InfluxTypeName.inferred(column: columns[index], values: rows.map { $0[index] }) + } + return InfluxResultTable(columns: columns, typeNames: typeNames, rows: rows) + } } // InfluxQL over `/query` (1.x and the 2.x compatibility API). Every series of every statement lands in one grid: a @@ -159,12 +173,16 @@ public enum InfluxResponseDecoder { } // Rows are as wide as every header together, so a short record costs more cells than bytes. - private struct CellBudget { + struct CellBudget { let limit: Int // Every v3 CSV row carries every column, a byte per cell at least: 16 cells per byte, or a million at any size. static func perByte(of data: Data) -> CellBudget { - let (scaled, overflow) = data.count.multipliedReportingOverflow(by: 16) + perByte(bytes: data.count) + } + + static func perByte(bytes: Int) -> CellBudget { + let (scaled, overflow) = bytes.multipliedReportingOverflow(by: 16) return CellBudget(limit: overflow ? .max : max(scaled, 1 << 20)) } diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift index 21fb9221ce..b34e744280 100644 --- a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift @@ -52,7 +52,10 @@ public enum MemcachedKeyListing { guard let equals = part.firstIndex(of: "=") else { continue } fields[part[.. + // The columns left to their default, by name like `expressionColumns`. Every other cell is a value. + public let defaultColumns: Set public init( rowIndex: Int, kind: Kind, cellChanges: [CellChange] = [], originalRow: [SpannerCell]? = nil, - expressionColumns: Set = [] + expressionColumns: Set = [], + defaultColumns: Set = [] ) { self.rowIndex = rowIndex self.kind = kind self.cellChanges = cellChanges self.originalRow = originalRow self.expressionColumns = expressionColumns + self.defaultColumns = defaultColumns } } public enum SpannerRowEditSQL { - public static let defaultSentinel = "__DEFAULT__" - private static let logger = Logger(subsystem: "com.TablePro", category: "SpannerRowEditSQL") public static func statements( @@ -115,7 +117,7 @@ internal struct SpannerRowEditTarget { func insert(_ change: SpannerRowChange, row: [SpannerCell]?) -> SpannerRenderedStatement? { let cells = row.map { zip(columns, $0).map { SpannerRowChange.CellChange(column: $0.0, value: $0.1) } } ?? Self.lastValuePerColumn(change.cellChanges) - let written = cells.filter { !Self.isDefaultSentinel($0.value) } + let written = cells.filter { !change.defaultColumns.contains($0.column) } guard written.isEmpty else { return Self.insertStatement( table: table, cells: written, dialect: dialect, expressionColumns: change.expressionColumns @@ -132,7 +134,7 @@ internal struct SpannerRowEditTarget { var parameters: [SpannerCell] = [] let assignments = Self.lastValuePerColumn(change.cellChanges).map { cell in let column = dialect.quoteIdentifier(cell.column) - guard !Self.isDefaultSentinel(cell.value) else { return "\(column) = DEFAULT" } + guard !change.defaultColumns.contains(cell.column) else { return "\(column) = DEFAULT" } if let sql = Self.expression(cell, in: change.expressionColumns) { return "\(column) = \(sql)" } parameters.append(cell.value) return "\(column) = ?" @@ -191,10 +193,6 @@ internal struct SpannerRowEditTarget { return (terms.joined(separator: " AND "), parameters) } - private static func isDefaultSentinel(_ cell: SpannerCell) -> Bool { - cell == .text(SpannerRowEditSQL.defaultSentinel) - } - private static func lastValuePerColumn(_ cells: [SpannerRowChange.CellChange]) -> [SpannerRowChange.CellChange] { var order: [String] = [] var values: [String: SpannerCell] = [:] diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift index 440de15bef..8f8d3c83fd 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift @@ -31,7 +31,7 @@ public final class TeradataConnection: @unchecked Sendable { private var _stoppingRequest: UInt32? // 3110: "The transaction was aborted by the user." - static let userAbortCode = 3_110 + public static let userAbortCode = 3_110 private static let abortLockWait: TimeInterval = 0.25 // A server or network that stops answering would otherwise hold a stop for the whole 20-second receive timeout. static let defaultStopGrace: TimeInterval = 5 @@ -53,9 +53,11 @@ public final class TeradataConnection: @unchecked Sendable { } private enum Exchange { - case answered([Parcel]) + case answered case notSent case outputEnded + // The consumer threw on a reply, and the request has ended on the wire. + case refused(any Error) } public init(config: TeradataConnectionConfig) { @@ -120,8 +122,7 @@ public final class TeradataConnection: @unchecked Sendable { established = true } - /// Keeps the transport's connect deadline active through the driver's required VERSION probe. - /// Only then may ordinary query I/O use the session's 20-second socket timeout. + // The connect deadline holds through the driver's VERSION probe; only then does query I/O get the 20-second timeout. public func finishConnecting() throws { guard let socket, let connectDeadline else { throw TeradataWireError.connectionFailed("not connecting") @@ -132,14 +133,62 @@ public final class TeradataConnection: @unchecked Sendable { } public func execute(_ sql: String) throws -> TeradataResultSet { + var columns: [TeradataColumn] = [] + var rows: [[TeradataValue]] = [] + let activityCount = try perform(sql, onColumns: { columns = $0 }, onRows: { batch in + rows.append(contentsOf: batch) + return true + }) + return TeradataResultSet(columns: columns, rows: rows, activityCount: activityCount) + } + + // The reply that passes the cap ends the output on the server, so the rows past it are never fetched. + public func execute(_ sql: String, rowCap: Int?) throws -> TeradataCappedResult { + guard let rowCap, rowCap > 0 else { + return TeradataCappedResult(resultSet: try execute(sql), isTruncated: false) + } + var columns: [TeradataColumn] = [] + var rows: [[TeradataValue]] = [] + var isTruncated = false + let activityCount = try perform(sql, onColumns: { columns = $0 }, onRows: { batch in + guard rows.count + batch.count > rowCap else { + rows.append(contentsOf: batch) + return true + } + rows.append(contentsOf: batch.prefix(rowCap - rows.count)) + isTruncated = true + return false + }) + return TeradataCappedResult( + resultSet: TeradataResultSet(columns: columns, rows: rows, activityCount: activityCount), + isTruncated: isTruncated) + } + + // `onColumns` runs once before any rows, with none for a request that has no result set; false from `onRows` ends the output. + public func stream( + _ sql: String, + onColumns: ([TeradataColumn]) throws -> Void, + onRows: ([[TeradataValue]]) throws -> Bool + ) throws { + _ = try perform(sql, onColumns: onColumns, onRows: onRows) + } + + private func perform( + _ sql: String, + onColumns: ([TeradataColumn]) throws -> Void, + onRows: ([[TeradataValue]]) throws -> Bool + ) throws -> Int { guard socket != nil else { throw TeradataWireError.connectionFailed("not connected") } let requestNumber = nextRequest() let body = TeradataMessages.optionsParcel(returnStatementInfo: false).encoded() + TeradataMessages.requestParcel(sql: sql).encoded() + TeradataMessages.responseParcel().encoded() + var reader = TeradataResultParser.Reader() let outcome: Exchange do { - outcome = try exchange(body, requestNumber: requestNumber) + outcome = try exchange(body, requestNumber: requestNumber) { parcels in + try reader.read(parcels, onColumns: onColumns, onRows: onRows) + } } catch { endRequest() // The reply still on its way would be read as the next request's. @@ -147,16 +196,28 @@ public final class TeradataConnection: @unchecked Sendable { throw error } let stopped = endRequest() - guard case .answered(let parcels) = outcome else { throw TeradataWireError.cancelled } - do { - return try TeradataResultParser.parse(parcels) - } catch TeradataWireError.server(let code, _) where stopped && code == Self.userAbortCode { - // The server rolled back the transaction the request ran in; the host has no way yet to say so. + switch outcome { + case .notSent: + throw TeradataWireError.cancelled + case .refused(let error): + if stopped, case .server(let code, _)? = error as? TeradataWireError, code == Self.userAbortCode { + // In Teradata mode a request outside BT is its own transaction, so the abort undid that request alone. + throw config.transactionMode == .ansi ? TeradataWireError.rolledBackByStop : TeradataWireError.cancelled + } + throw error + case .outputEnded where stopped: throw TeradataWireError.cancelled + case .answered, .outputEnded: + try reader.finish(onColumns: onColumns) + return reader.activityCount } } - private func exchange(_ body: [UInt8], requestNumber: UInt32) throws -> Exchange { + private func exchange( + _ body: [UInt8], + requestNumber: UInt32, + consume: ([Parcel]) throws -> Bool + ) throws -> Exchange { let sent = try wireLock.withLock { () -> Bool in // A stop that landed before the request went out refuses it, and the session stays as it was. let refused = socketLock.withLock { () -> Bool in @@ -169,17 +230,21 @@ public final class TeradataConnection: @unchecked Sendable { return true } guard sent else { return .notSent } - var parcels: [Parcel] = [] while true { let batch = try readReply() - parcels.append(contentsOf: batch) - if Self.responseIsComplete(batch) { return .answered(parcels) } + let complete = Self.responseIsComplete(batch) // Once the first reply is out an abort no longer takes effect, so a stop ends the output instead. - let stopRequested = socketLock.withLock { () -> Bool in - _phase = .fetching(requestNumber) - return _stopRequested + socketLock.withLock { _phase = .fetching(requestNumber) } + let wantsMore: Bool + do { + wantsMore = try consume(batch) + } catch { + // Ending the output keeps the session; a failed end leaves the rest to be read as the next request's. + if !complete, (try? endOutput(requestNumber: requestNumber)) == nil { throw error } + return .refused(error) } - if stopRequested { + if complete { return .answered } + if !wantsMore || socketLock.withLock({ _stopRequested }) { try endOutput(requestNumber: requestNumber) return .outputEnded } @@ -516,3 +581,9 @@ public final class TeradataConnection: @unchecked Sendable { return String(cString: buffer) } } + +public struct TeradataCappedResult: Sendable { + public let resultSet: TeradataResultSet + // The answer held more rows than the cap, and its output was ended there. + public let isTruncated: Bool +} diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift index b8c409d28f..13073c15a2 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataResultParser.swift @@ -2,36 +2,62 @@ import Foundation import TableProWireIO enum TeradataResultParser { - static func parse(_ parcels: [Parcel]) throws -> TeradataResultSet { - for parcel in parcels where parcel.flavor == ParcelFlavor.failure.rawValue - || parcel.flavor == ParcelFlavor.error.rawValue - || parcel.flavor == ParcelFlavor.statementError.rawValue { - let (code, message) = errorDetail(parcel) - throw TeradataWireError.server(code: code, message: message) - } + // The answer is its first result set; a later statement's rows join only when they decode the same way. + struct Reader { + private var resultColumns: [ColumnMeta]? + private var statementColumns: [ColumnMeta] = [] + private var statementJoins = false + private var firstActivityCount: Int? - let activityCount = parcels - .first { $0.flavor == ParcelFlavor.success.rawValue } - .map { readActivityCount(fromSuccess: $0.body) } ?? 0 + var activityCount: Int { firstActivityCount ?? 0 } - var columns: [ColumnMeta] = [] - if let prepInfoX = parcels.first(where: { $0.flavor == ParcelFlavor.prepInfoX.rawValue }) { - columns = try parsePrepInfo(prepInfoX.body, extended: true) - } else if let prepInfo = parcels.first(where: { $0.flavor == ParcelFlavor.prepInfo.rawValue }) { - columns = try parsePrepInfo(prepInfo.body, extended: false) - } - - var rows: [[TeradataValue]] = [] - if !columns.isEmpty { - for record in parcels where record.flavor == ParcelFlavor.record.rawValue { - rows.append(try RecordDecoder.decode(recordBody: record.body, columns: columns)) + // False when the consumer wants no more rows. + mutating func read( + _ parcels: [Parcel], + onColumns: ([TeradataColumn]) throws -> Void, + onRows: ([[TeradataValue]]) throws -> Bool + ) throws -> Bool { + if let failure = parcels.first(where: TeradataResultParser.isError) { + let (code, message) = TeradataResultParser.errorDetail(failure) + throw TeradataWireError.server(code: code, message: message) + } + var rows: [[TeradataValue]] = [] + for parcel in parcels { + switch parcel.knownFlavor { + case .success?: + if firstActivityCount == nil { + firstActivityCount = TeradataResultParser.readActivityCount(fromSuccess: parcel.body) + } + case .prepInfo?, .prepInfoX?: + let columns = try TeradataResultParser.parsePrepInfo( + parcel.body, extended: parcel.flavor == ParcelFlavor.prepInfoX.rawValue) + if let resultColumns { + statementJoins = !resultColumns.isEmpty + && TeradataResultParser.decodeAlike(columns, resultColumns) + } else { + resultColumns = columns + statementJoins = !columns.isEmpty + try onColumns(columns.map { + TeradataColumn(name: $0.name, typeCode: $0.typeCode, dataLength: $0.dataLength) + }) + } + statementColumns = columns + case .record? where statementJoins: + rows.append(try RecordDecoder.decode(recordBody: parcel.body, columns: statementColumns)) + default: + break + } } + guard !rows.isEmpty else { return true } + return try onRows(rows) } - let resultColumns = columns.map { - TeradataColumn(name: $0.name, typeCode: $0.typeCode, dataLength: $0.dataLength) + // A request that described no result set still answers its consumer, with no columns. + mutating func finish(onColumns: ([TeradataColumn]) throws -> Void) throws { + guard resultColumns == nil else { return } + resultColumns = [] + try onColumns([]) } - return TeradataResultSet(columns: resultColumns, rows: rows, activityCount: activityCount) } static func errorDetail(_ parcel: Parcel) -> (code: Int, message: String) { @@ -45,6 +71,18 @@ enum TeradataResultParser { return (code, message.isEmpty ? decodedText(body) : message) } + private static func isError(_ parcel: Parcel) -> Bool { + parcel.flavor == ParcelFlavor.failure.rawValue + || parcel.flavor == ParcelFlavor.error.rawValue + || parcel.flavor == ParcelFlavor.statementError.rawValue + } + + // A record decodes by each field's base type and declared length, so two descriptions alike in those read it alike. + private static func decodeAlike(_ lhs: [ColumnMeta], _ rhs: [ColumnMeta]) -> Bool { + lhs.count == rhs.count + && zip(lhs, rhs).allSatisfy { $0.baseCode == $1.baseCode && $0.dataLength == $1.dataLength } + } + private static func decodedText(_ bytes: some Collection) -> String { String(bytes: bytes, encoding: .utf8) ?? String(bytes: bytes, encoding: .isoLatin1) ?? "" } diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift index a82563f33b..af9f40875e 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift @@ -8,6 +8,8 @@ public enum TeradataWireError: Error, CustomStringConvertible, LocalizedError { case server(code: Int, message: String) case unsupported(String) case cancelled + // Stop aborted the request, and the server rolled back the transaction it ran in. + case rolledBackByStop public var description: String { switch self { @@ -17,6 +19,7 @@ public enum TeradataWireError: Error, CustomStringConvertible, LocalizedError { case .server(let code, let message): return "\(message) (\(code))" case .unsupported(let what): return "unsupported: \(what)" case .cancelled: return "cancelled" + case .rolledBackByStop: return "stopped, and the server rolled back the transaction (\(TeradataConnection.userAbortCode))" } } diff --git a/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxGridStreamTests.swift b/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxGridStreamTests.swift new file mode 100644 index 0000000000..d5ea9e7298 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxGridStreamTests.swift @@ -0,0 +1,224 @@ +import Foundation +@testable import TableProInfluxDBCore +import Testing + +private func bytes(_ text: String) -> Data { + Data(text.utf8) +} + +private func pieces(_ data: Data, size: Int) -> [Data] { + stride(from: 0, to: data.count, by: size).map { data.subdata(in: $0 ..< min($0 + size, data.count)) } +} + +private func records(_ chunks: [Data]) -> [[String]] { + var reader = InfluxCSV.Reader() + var records: [[String]] = [] + for chunk in chunks { + reader.read(chunk) + records += reader.drain() + } + return records + reader.finish() +} + +private func streamed( + _ body: InfluxGridStream.Body, + _ chunks: [Data], + grid: InfluxGridQuery = cpuExport, + schema: [InfluxColumn] = cpuSchema +) throws -> [InfluxResultTable] { + var stream = InfluxGridStream(body: body, grid: grid, schema: schema) + var batches: [InfluxResultTable] = [] + for chunk in chunks { + batches += try stream.consume(chunk) + } + let rest = try stream.finish() + return batches + rest +} + +// The batches under the header the first one carries, which every other one repeats. +private func joined(_ batches: [InfluxResultTable]) throws -> InfluxResultTable { + let first = try #require(batches.first) + #expect(batches.allSatisfy { $0.columns == first.columns && $0.typeNames == first.typeNames }) + return InfluxResultTable(columns: first.columns, typeNames: first.typeNames, rows: batches.flatMap { $0.rows }) +} + +private let cpuExport = InfluxGridQuery(measurement: "cpu", shape: .all) + +private let cpuSchema = [ + InfluxColumn(name: "time", typeName: InfluxTypeName.timestamp, role: .time), + InfluxColumn(name: "host", typeName: InfluxTypeName.tag, role: .tag), + InfluxColumn(name: "idle", typeName: InfluxTypeName.integer, role: .field), + InfluxColumn(name: "note", typeName: InfluxTypeName.string, role: .field), + InfluxColumn(name: "usage", typeName: InfluxTypeName.float, role: .field), +] + +private let v3Body = bytes( + "host,time,usage,note,idle\r\n" + + "a,2024-10-27T03:33:20,12.5,\"hot, \"\"quoted\"\"\",10\r\n" + + "b,2024-10-27T03:34:20.500Z,,,\r\n" + + ",2024-10-27T03:35:20,7,\"two\r\nlines é\",40\r\n" +) + +private let v2Rows = [ + #"["2024-10-27T03:35:20Z","c",10,"a;b",7]"#, + #"["2024-10-27T03:34:20Z","b",40,"hot, \"quoted\"",50.1]"#, + #"["2024-10-27T03:33:20Z","a",null,null,12.5]"#, +] + +private func v2Chunk(_ rows: ArraySlice, partial: Bool) -> String { + let flag = partial ? #","partial":true"# : "" + return #"{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","host","idle","note","usage"],"values":["# + + rows.joined(separator: ",") + "]" + flag + "}]" + flag + "}]}" +} + +private let tooWide = InfluxError.malformedResponse( + String(localized: "the result has far more columns than its rows fill") +) + +struct InfluxGridStreamTests { + @Test func aCREndingOneChunkAndTheLFStartingTheNextAreOneLineEnd() { + #expect(records([bytes("a,b\r"), bytes("\nc,d\r\n")]) == [["a", "b"], ["c", "d"]]) + } + + @Test func anEscapedQuoteSplitBetweenItsTwoQuotesIsOneQuote() { + #expect(records([bytes("\"x\""), bytes("\"y\",z\n")]) == [["x\"y", "z"]]) + } + + @Test func aQuotedLineEndSplitAcrossChunksStaysInTheField() { + #expect(records([bytes("\"a\r"), bytes("\nb\",c\n"), bytes("\"d\n"), bytes("e\"\n")]) == [["a\r\nb", "c"], ["d\ne"]]) + } + + @Test("Every chunk size reads the records one whole body gives", arguments: [1, 2, 3, 7, 64]) + func chunkSizeDoesNotChangeTheRecords(size: Int) { + var body = bytes("host,note\r\na,\"x, \"\"y\"\"\"\r\n\r\nb,\"two\nlines\"\nc,日本\r\"\"\"\",é\r\n") + body.append(contentsOf: [0x61, 0xFF, 0x62, 0x2C, 0x7A]) + #expect(InfluxCSV.records(body) == [ + ["host", "note"], + ["a", "x, \"y\""], + [""], + ["b", "two\nlines"], + ["c", "日本"], + ["\"", "é"], + ["a\u{FFFD}b", "z"], + ]) + #expect(records(pieces(body, size: size)) == InfluxCSV.records(body)) + } + + @Test("A v3 stream reads the rows and types the collected read gives", arguments: [1, 3, 64, 100_000]) + func aV3StreamReadsLikeTheCollectedRead(size: Int) throws { + let collected = try InfluxResponseDecoder.decodeV3CSV(v3Body).typed(by: cpuSchema) + let table = try joined(streamed(.csv, pieces(v3Body, size: size))) + #expect(table == collected) + #expect(table.typeNames == [ + InfluxTypeName.tag, InfluxTypeName.timestamp, InfluxTypeName.float, InfluxTypeName.string, InfluxTypeName.integer, + ]) + #expect(table.column(named: "time") == ["2024-10-27T03:33:20Z", "2024-10-27T03:34:20.500Z", "2024-10-27T03:35:20Z"]) + } + + @Test func aV3StreamSendsBatchesOfFiveThousandRows() throws { + let body = bytes("host,usage\n" + String(repeating: "a,1\n", count: 12_001)) + let batches = try streamed(.csv, [body]) + #expect(batches.map { $0.rows.count } == [5_000, 5_000, 2_001]) + #expect(try joined(batches).typeNames == [InfluxTypeName.tag, InfluxTypeName.float]) + } + + @Test func aWideV3HeaderShrinksTheBatchToAboutAMillionCells() throws { + let columns = (0 ..< 4_096).map { "c\($0)" } + let record = Array(repeating: "1", count: columns.count).joined(separator: ",") + let body = bytes(columns.joined(separator: ",") + "\n" + String(repeating: record + "\n", count: 257)) + let batches = try streamed(.csv, [body], schema: []) + #expect(batches.map { $0.rows.count } == [256, 1]) + } + + @Test("A v3 header of 40,000 columns over 1,000 empty records is refused, as the collected read refuses it") + func shortV3RecordsUnderAWideHeaderAreRefused() { + let header = (0 ..< 40_000).map { "c\($0)" }.joined(separator: ",") + let body = bytes(header + "\n" + String(repeating: "\n", count: 1_000)) + #expect(throws: tooWide) { try streamed(.csv, [body], schema: []) } + } + + @Test func aV3HeaderRecordWithNoRowsKeepsItsColumns() throws { + let body = bytes("usage,time,extra\n") + let batches = try streamed(.csv, [body]) + #expect(batches == [InfluxResultTable( + columns: ["usage", "time", "extra"], + typeNames: [InfluxTypeName.float, InfluxTypeName.timestamp, InfluxTypeName.string], + rows: [] + )]) + let collected = try InfluxResponseDecoder.decodeV3CSV(body).typed(by: cpuSchema) + #expect(batches == [collected]) + } + + @Test func aV3ColumnTheSchemaLacksIsTypedFromTheFirstBatch() throws { + let table = try joined(streamed(.csv, [bytes("time,extra\n2024-10-27T03:33:20Z,1\n2024-10-27T03:34:20Z,2\n")])) + #expect(table.typeNames == [InfluxTypeName.timestamp, InfluxTypeName.integer]) + } + + @Test("An empty body gets the header from the schema", arguments: [InfluxGridStream.Body.csv, .influxQLChunks]) + func anEmptyBodyGetsTheHeaderFromTheSchema(body: InfluxGridStream.Body) throws { + let empty = body == .csv ? Data() : bytes(#"{"results":[{"statement_id":0}]}"# + "\n") + #expect(try streamed(body, [empty]) == [InfluxResultTable( + columns: ["time", "host", "idle", "note", "usage"], + typeNames: [ + InfluxTypeName.timestamp, InfluxTypeName.tag, InfluxTypeName.integer, InfluxTypeName.string, InfluxTypeName.float, + ], + rows: [] + )]) + let page = InfluxGridQuery(measurement: "cpu", projection: ["usage", "time", "gone"], shape: .page(limit: 50, offset: 0)) + #expect(try streamed(body, [empty], grid: page) == [InfluxResultTable( + columns: ["usage", "time", "gone"], + typeNames: [InfluxTypeName.float, InfluxTypeName.timestamp, InfluxTypeName.string], + rows: [] + )]) + } + + @Test("v2 chunks read the rows and types one collected body gives", arguments: [1, 5, 64, 100_000]) + func v2ChunksReadLikeTheCollectedBody(size: Int) throws { + let collected = try InfluxResponseDecoder.decodeV1JSON(bytes(v2Chunk(v2Rows[...], partial: false))) + .typed(by: cpuSchema) + let body = bytes(v2Chunk(v2Rows[..<2], partial: true) + "\n" + v2Chunk(v2Rows[2...], partial: false) + "\n") + let batches = try streamed(.influxQLChunks, pieces(body, size: size)) + #expect(batches.map { $0.rows.count } == [2, 1]) + #expect(try joined(batches) == collected) + #expect(collected.typeNames == [ + InfluxTypeName.timestamp, InfluxTypeName.tag, InfluxTypeName.integer, InfluxTypeName.string, InfluxTypeName.float, + ]) + } + + @Test func aLastV2ChunkWithoutItsNewlineIsStillRead() throws { + let batches = try streamed(.influxQLChunks, [bytes(v2Chunk(v2Rows[...], partial: false))]) + #expect(batches.map { $0.rows.count } == [3]) + } + + @Test( + "An error object after some rows throws", + arguments: [ + #"{"results":[{"statement_id":0,"error":"max-select-point limit exceeded"}]}"#, + #"{"error":"max-select-point limit exceeded"}"#, + ] + ) + func anErrorObjectAfterSomeRowsThrows(errorLine: String) throws { + var stream = InfluxGridStream(body: .influxQLChunks, grid: cpuExport, schema: cpuSchema) + let first = try stream.consume(bytes(v2Chunk(v2Rows[..<2], partial: true) + "\n")) + #expect(first.map { $0.rows.count } == [2]) + #expect(throws: InfluxError.query("max-select-point limit exceeded")) { + try stream.consume(bytes(errorLine + "\n")) + } + } + + @Test func aLaterV2ChunkWithItsColumnsReorderedMapsThemByName() throws { + let later = #"{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["usage","time"],"values":[[1.5,"2024-10-27T03:30:00Z"]]}]}]}"# + let body = bytes(v2Chunk(v2Rows[..<1], partial: true) + "\n" + later + "\n") + let table = try joined(streamed(.influxQLChunks, [body])) + #expect(table.rows.last == ["2024-10-27T03:30:00Z", nil, nil, nil, "1.5"]) + } + + @Test func aLaterV2ChunkWithAColumnTheFirstLacksThrows() { + let later = #"{"results":[{"statement_id":0,"series":[{"name":"cpu","columns":["time","extra"],"values":[["2024-10-27T03:30:00Z",1]]}]}]}"# + let body = bytes(v2Chunk(v2Rows[..<1], partial: true) + "\n" + later + "\n") + let refusal = InfluxError.malformedResponse( + String(format: String(localized: "column %@ first appears in a later chunk"), "extra") + ) + #expect(throws: refusal) { try streamed(.influxQLChunks, [body]) } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxRequestTests.swift b/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxRequestTests.swift index 6a032a6746..8aae352353 100644 --- a/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxRequestTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProInfluxDBCoreTests/InfluxRequestTests.swift @@ -224,6 +224,38 @@ struct InfluxRequestTests { #expect(try InfluxCatalogDecoder.v3Databases(body) == ["_internal", "metrics"]) } + @Test func aStreamedV2InfluxQLQueryAsksForChunksOfTenThousand() throws { + let statement = InfluxStatement(language: .influxQL, text: #"SELECT * FROM "cpu""#) + let streamed = try InfluxRequests.query( + statement, database: "metrics", organization: "", generation: .v2, credentials: token, streamed: true + ) + #expect(queryItems(ofForm: streamed.body) == [ + URLQueryItem(name: "q", value: #"SELECT * FROM "cpu""#), + URLQueryItem(name: "db", value: "metrics"), + URLQueryItem(name: "chunked", value: "true"), + URLQueryItem(name: "chunk_size", value: "10000"), + ]) + let collected = try InfluxRequests.query( + statement, database: "metrics", organization: "", generation: .v2, credentials: token + ) + #expect(queryItems(ofForm: collected.body) == [ + URLQueryItem(name: "q", value: #"SELECT * FROM "cpu""#), + URLQueryItem(name: "db", value: "metrics"), + ]) + } + + @Test(arguments: [InfluxQueryLanguage.sql, .influxQL]) + func aStreamedV3QueryIsTheCollectedRequest(language: InfluxQueryLanguage) throws { + let statement = InfluxStatement(language: language, text: #"SELECT * FROM "cpu""#) + let streamed = try InfluxRequests.query( + statement, database: "metrics", organization: "", generation: .v3, credentials: token, streamed: true + ) + let collected = try InfluxRequests.query( + statement, database: "metrics", organization: "", generation: .v3, credentials: token + ) + #expect(streamed == collected) + } + @Test func anInfluxQLFormBodyCarriesTheRetentionPolicy() throws { let request = try InfluxRequests.query( InfluxStatement(language: .influxQL, text: "SHOW MEASUREMENTS"), diff --git a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift index 62c2c55cf8..6ee0c64ead 100644 --- a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedKeyListingTests.swift @@ -12,6 +12,14 @@ struct MemcachedKeyListingTests { #expect(entry?.secondsLeft(at: 1_793_499_867) == 300) } + @Test("A listed key a text command line cannot carry is left out", arguments: [ + "key=x%0D%0Aflush_all exp=-1 la=1 cas=1 fetch=no cls=1 size=1", + "key=two%20words exp=-1 la=1 cas=1 fetch=no cls=1 size=1", + ]) + func unsendableKeyIsLeftOut(line: String) { + #expect(MemcachedKeyListing.metadumpEntry(line) == nil) + } + @Test("exp=-1 means the item never expires") func metadumpNeverExpires() { let entry = MemcachedKeyListing.metadumpEntry("key=%22sp%22 exp=-1 la=1 cas=4 fetch=no cls=1 size=64 flags=0") diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginTypedParameterBindingTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginTypedParameterBindingTests.swift new file mode 100644 index 0000000000..2d29b5378b --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginTypedParameterBindingTests.swift @@ -0,0 +1,189 @@ +import Foundation +import TableProPluginKit +import Testing + +struct PluginTypedParameterBindingTests { + private func bound( + _ values: [String], + kinds: [PluginColumnKind?]?, + query: String = "SELECT * FROM t WHERE a = ?" + ) async throws -> String { + let driver = RecordingDriver() + var context = PluginStatementContext() + context.parameterKinds = kinds + _ = try await driver.executeParameterized(query: query, parameters: values.map { .text($0) }, context: context) + return try #require(driver.executed.first) + } + + @Test("A String parameter is quoted whatever it looks like", arguments: ["1E5", "42", "00123", "-7"]) + func stringIsQuoted(value: String) async throws { + #expect(try await bound([value], kinds: [.text]) == "SELECT * FROM t WHERE a = '\(value)'") + } + + @Test("An Integer or Decimal parameter that reads as that number is bare") + func numbersAreBare() async throws { + #expect(try await bound(["42"], kinds: [.integer]) == "SELECT * FROM t WHERE a = 42") + #expect(try await bound(["1E5"], kinds: [.decimal]) == "SELECT * FROM t WHERE a = 1E5") + } + + @Test("Text that is not the declared number is sent as a string for the server to refuse") + func notANumberIsQuoted() async throws { + #expect(try await bound(["1.5"], kinds: [.integer]) == "SELECT * FROM t WHERE a = '1.5'") + #expect(try await bound(["1; DROP TABLE t"], kinds: [.decimal]) == "SELECT * FROM t WHERE a = '1; DROP TABLE t'") + } + + @Test("A Boolean parameter is TRUE or FALSE", arguments: [("true", "TRUE"), ("0", "FALSE"), ("Yes", "TRUE")]) + func booleans(value: String, literal: String) async throws { + #expect(try await bound([value], kinds: [.boolean]) == "SELECT * FROM t WHERE a = \(literal)") + } + + @Test("A Boolean parameter that is no boolean is a string") + func notABooleanIsQuoted() async throws { + #expect(try await bound(["maybe"], kinds: [.boolean]) == "SELECT * FROM t WHERE a = 'maybe'") + } + + @Test("A Date parameter is a quoted string") + func dateIsQuoted() async throws { + #expect(try await bound(["2026-01-01"], kinds: [.other]) == "SELECT * FROM t WHERE a = '2026-01-01'") + } + + @Test("A value with no type keeps the reading by shape") + func untypedKeepsTheShape() async throws { + #expect(try await bound(["42"], kinds: nil) == "SELECT * FROM t WHERE a = 42") + #expect(try await bound(["42", "x"], kinds: [.text, nil], query: "SELECT ?, ?") == "SELECT '42', 'x'") + #expect(try await bound(["42"], kinds: [nil]) == "SELECT * FROM t WHERE a = 42") + } + + @Test("Each parameter takes its own type by position") + func typesFollowPosition() async throws { + let sql = try await bound(["7", "7"], kinds: [.integer, .text], query: "SELECT ?, ?") + #expect(sql == "SELECT 7, '7'") + } + + @Test("A driver that forwards its context overload keeps the types through withHostContext") + func forwardingDriverKeepsTheTypes() async throws { + let driver = ForwardingDriver() + var context = PluginStatementContext() + context.parameterKinds = [.text] + _ = try await driver.executeUserQuery( + query: "SELECT ?", rowCap: nil, parameters: [.text("5")], context: context + ) + #expect(driver.inner.executed == ["SELECT '5'"]) + } +} + +struct PluginRowChangeDefaultMarkTests { + @Test("Only a marked column is the default") + func onlyMarkedColumnsAreDefaults() { + var change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + #expect(!change.isColumnDefault(0)) + change.defaultColumns = [1] + #expect(change.isColumnDefault(1)) + #expect(!change.isColumnDefault(0)) + } + + @Test("An engine with no defaults refuses the first change that leaves a cell to one, naming its row") + func defaultsUnsupportedNamesTheRow() { + let plain = PluginRowChange(rowIndex: 3, type: .insert, cellChanges: [], originalRow: nil) + var defaulted = PluginRowChange(rowIndex: 7, type: .update, cellChanges: [], originalRow: nil) + defaulted.defaultColumns = [0] + #expect(PluginRowWriteRefusal.columnDefaultsUnsupported(in: [plain]) == nil) + #expect(PluginRowWriteRefusal.columnDefaultsUnsupported(in: [plain, defaulted])?.rowIndex == 7) + } +} + +struct PluginBoundedStreamDeclaredLimitTests { + private func stream(rows: Int, rowLimit: Int?) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + var header = PluginStreamHeader(columns: ["id"], columnTypeNames: ["INT"]) + header.rowLimit = rowLimit + continuation.yield(.header(header)) + continuation.yield(.rows((0.. PluginQueryResult { + lock.withLock { statements.append(query) } + return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) + } + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} + +// The shape of a driver that overrides the context overload to set its own state, then runs the context-free one. +private final class ForwardingDriver: PluginDatabaseDriver, @unchecked Sendable { + let inner = RecordingDriver() + let supportsSchemas = false + let supportsTransactions = false + let currentSchema: String? = nil + let serverVersion: String? = nil + + func connect() async throws {} + func disconnect() {} + func ping() async throws {} + func execute(query: String) async throws -> PluginQueryResult { + try await inner.execute(query: query) + } + func executeUserQuery( + query: String, + rowCap: Int?, + parameters: [PluginCellValue]?, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + try await PluginParameterBinder.withHostContext(context) { + try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) + } + } + func fetchTables(schema: String?) async throws -> [PluginTableInfo] { [] } + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { [] } + func fetchIndexes(table: String, schema: String?) async throws -> [PluginIndexInfo] { [] } + func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { [] } + func fetchTableDDL(table: String, schema: String?) async throws -> String { "" } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { "" } + func fetchTableMetadata(table: String, schema: String?) async throws -> PluginTableMetadata { + PluginTableMetadata(tableName: table) + } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> PluginDatabaseMetadata { + PluginDatabaseMetadata(name: database) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift index cd0bbbdaab..279cc22ae3 100644 --- a/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQdrantCoreTests/QdrantRowWriteTests.swift @@ -98,14 +98,22 @@ struct QdrantRowWriteTests { #expect(try written(#"["a"]"#) == prefix + #"{"payload":{"mixed":["a"]},"points":[7]}"#) } - @Test("A duplicated row's default marker asks for a new id instead of failing as one") + @Test("A duplicated row's defaulted id asks for a new id instead of failing as one") func duplicatedRowGetsNewID() throws { let change = QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [ - QdrantCell(column: "id", text: "__DEFAULT__"), QdrantCell(column: "city", text: "Hanoi") + QdrantCell(column: "id", text: nil), QdrantCell(column: "city", text: "Hanoi") ]) #expect(try writes([change]).first?.contains(#""id":"00000000-0000-0000-0000-000000000001""#) == true) } + @Test("The text __DEFAULT__ in a payload field is stored as that text") + func defaultLookingTextIsAValue() throws { + let change = QdrantRowChange(kind: .insert, rowIndex: 0, id: nil, cells: [ + QdrantCell(column: "id", text: "7"), QdrantCell(column: "city", text: "__DEFAULT__") + ]) + #expect(try writes([change]).first?.contains(#""city":"__DEFAULT__""#) == true) + } + @Test("A server that would overwrite a taken id gets no grid insert") func insertNeedsConditionalUpsert() { let change = QdrantRowChange(kind: .insert, rowIndex: 6, id: nil, cells: [QdrantCell(column: "city", text: "Hanoi")]) diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift index 76bf57dbd6..46f6d66fc2 100644 --- a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataReplyDecodingTests.swift @@ -19,6 +19,12 @@ struct TeradataReplyDecodingTests { return bytes } + private static func execute(_ reply: [UInt8]) throws -> TeradataResultSet { + let transport = StoppableTransport(replies: [LanMessage(kind: .start, body: reply).encoded()]) + let config = TeradataConnectionConfig(host: "td.example", username: "dbc", password: "dbc") + return try TeradataConnection(config: config, transport: transport).execute("SELECT c FROM t") + } + @Test("A result description whose CHAR column has length -1 fails the reply instead of crashing") func negativeColumnLengthFailsTheReply() throws { let reply = Self.reply(charLength: [UInt8](repeating: 0xFF, count: 8), record: [0x00, 0x78]) @@ -26,13 +32,13 @@ struct TeradataReplyDecodingTests { let parcels = try Parcel.decodeAll(reply) let flavors = parcels.map(\.flavor) #expect(flavors == [ParcelFlavor.prepInfoX.rawValue, ParcelFlavor.record.rawValue, ParcelFlavor.endRequest.rawValue]) - #expect(throws: WireIOError.lengthOutOfRange("-1", limit: 1)) { try TeradataResultParser.parse(parcels) } + #expect(throws: WireIOError.lengthOutOfRange("-1", limit: 1)) { try Self.execute(reply) } } @Test("A CHAR column of the declared length still decodes") func charColumnDecodes() throws { let reply = Self.reply(charLength: [0, 0, 0, 0, 0, 0, 0, 3], record: [0x00, 0x61, 0x62, 0x63]) - let result = try TeradataResultParser.parse(try Parcel.decodeAll(reply)) + let result = try Self.execute(reply) #expect(result.rows == [[TeradataValue.text("abc")]]) } diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift index 4f19e1b292..76202549d8 100644 --- a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStopTests.swift @@ -4,7 +4,7 @@ import Testing // Replies are read in order; a hook keyed by a reply's index runs just before that reply is read, which is where a // request is waiting on the server when Stop is pressed. -private final class StoppableTransport: TeradataTransport { +final class StoppableTransport: TeradataTransport { private var replies: [[UInt8]] private var stream: [UInt8] = [] private var repliesRead = 0 @@ -97,6 +97,9 @@ private final class SilentTransport: TeradataTransport, @unchecked Sendable { struct TeradataStopTests { private static let config = TeradataConnectionConfig(host: "td.example", username: "dbc", password: "dbc") + private static let ansiConfig = TeradataConnectionConfig( + host: "td.example", username: "dbc", password: "dbc", transactionMode: .ansi + ) private static let start = MessageKind.start.rawValue private static let cont = MessageKind.cont.rawValue private static let abort = MessageKind.abort.rawValue @@ -128,16 +131,36 @@ struct TeradataStopTests { return true } - @Test("Stop while the server works on a request aborts it on the session, which stays logged on") + private static func isRolledBack(_ error: Error?) -> Bool { + guard let wire = error as? TeradataWireError, case .rolledBackByStop = wire else { return false } + return true + } + + @Test("A Stop outside a session set to ANSI reads as a cancel") + func stopOutsideANSIModeIsACancel() throws { + let aborted = Self.reply([Self.failure(code: 3_110, "The transaction was aborted by the user.")]) + let transport = StoppableTransport(replies: [aborted]) + let connection = TeradataConnection(config: Self.config, transport: transport) + transport.beforeReply[0] = { connection.cancel() } + + let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders") } + + guard let wire = error as? TeradataWireError, case .cancelled = wire else { + Issue.record("Expected cancelled, got \(String(describing: error))") + return + } + } + + @Test("Stop while the server works on a request aborts it on the session, which stays logged on and reports the rollback") func stopAbortsTheRequestAndKeepsTheSession() throws { let aborted = Self.reply([Self.failure(code: 3_110, "The transaction was aborted by the user.")]) let transport = StoppableTransport(replies: [aborted, Self.finished]) - let connection = TeradataConnection(config: Self.config, transport: transport) + let connection = TeradataConnection(config: Self.ansiConfig, transport: transport) transport.beforeReply[0] = { connection.cancel() } let error = Self.thrown { _ = try connection.execute("SELECT * FROM sales.orders CROSS JOIN sales.items") } - #expect(Self.isCancelled(error)) + #expect(Self.isRolledBack(error)) #expect(transport.sentKinds == [Self.start, Self.abort]) #expect(transport.sent.last?.requestNumber == transport.sent.first?.requestNumber) #expect(transport.sent.last?.body == TeradataMessages.abortParcel().encoded()) diff --git a/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStreamTests.swift b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStreamTests.swift new file mode 100644 index 0000000000..96e3f43823 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProTeradataCoreTests/TeradataStreamTests.swift @@ -0,0 +1,287 @@ +@testable import TableProTeradataCore +import Testing + +struct TeradataStreamTests { + private static let start = MessageKind.start.rawValue + private static let cont = MessageKind.cont.rawValue + private static let abort = MessageKind.abort.rawValue + private static let integer: UInt16 = 496 + private static let char: UInt16 = 452 + + private static let numbers = prepInfoX([(integer, 4, "n")]) + private static let finished = reply([Parcel(.success), Parcel(.endRequest)]) + + private static func reply(_ parcels: [Parcel]) -> [UInt8] { + LanMessage(kind: .start, body: parcels.flatMap { $0.encoded() }).encoded() + } + + private static func u16(_ value: Int) -> [UInt8] { + [UInt8((value >> 8) & 0xFF), UInt8(value & 0xFF)] + } + + // A PrepInfoX parcel laid out as in TeradataReplyDecodingTests. + private static func prepInfoX(_ columns: [(type: UInt16, length: Int, name: String)]) -> Parcel { + var body = [UInt8](repeating: 0, count: 8) + u16(0) + u16(columns.count) + for column in columns { + let name = Array(column.name.utf8) + body += u16(Int(column.type)) + body += (0..<8).map { UInt8(truncatingIfNeeded: column.length >> (8 * (7 - $0))) } + body += [0, 0] + body += u16(name.count) + name + body += u16(0) + u16(0) + } + return Parcel(.prepInfoX, body: body) + } + + private static func number(_ value: Int32) -> Parcel { + let bits = UInt32(bitPattern: value) + return Parcel(.record, body: [0x00] + (0..<4).map { UInt8(truncatingIfNeeded: bits >> (8 * (3 - $0))) }) + } + + private static func code(_ text: String) -> Parcel { + Parcel(.record, body: [0x00] + Array(text.utf8)) + } + + private static func failure(code: Int, _ message: String) -> Parcel { + let text = Array(message.utf8) + return Parcel(.failure, body: [UInt8](repeating: 0, count: 8) + u16(code) + u16(text.count) + text) + } + + private static func thrown(_ body: () throws -> Void) -> Error? { + do { + try body() + return nil + } catch { + return error + } + } + + private static func isCancelled(_ error: Error?) -> Bool { + guard let wire = error as? TeradataWireError, case .cancelled = wire else { return false } + return true + } + + private static func connection( + _ replies: [[UInt8]], + transactionMode: TeradataTransactionMode = .default + ) -> (TeradataConnection, StoppableTransport) { + let transport = StoppableTransport(replies: replies) + let config = TeradataConnectionConfig( + host: "td.example", username: "dbc", password: "dbc", transactionMode: transactionMode + ) + return (TeradataConnection(config: config, transport: transport), transport) + } + + @Test("Each reply's rows reach the consumer before the next Continue goes out") + func rowsArriveReplyByReply() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1), Self.number(2)]), + Self.reply([Self.number(3)]), + Self.reply([Self.number(4), Parcel(.endRequest)]), + ]) + var headers: [[String]] = [] + var batches: [[[TeradataValue]]] = [] + var sentBeforeEachBatch: [Int] = [] + + try connection.stream("SELECT n FROM sales.orders", onColumns: { headers.append($0.map { $0.name }) }, onRows: { rows in + batches.append(rows) + sentBeforeEachBatch.append(transport.sent.count) + return true + }) + + #expect(headers == [["n"]]) + #expect(batches == [[[.integer(1)], [.integer(2)]], [[.integer(3)]], [[.integer(4)]]]) + #expect(sentBeforeEachBatch == [1, 2, 3]) + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont]) + #expect(transport.sent.dropFirst().allSatisfy { $0.body == TeradataMessages.responseParcel().encoded() }) + } + + @Test("A SELECT that matches no row still describes its columns") + func emptyResultStillHasColumns() throws { + let (connection, _) = Self.connection([ + Self.reply([Parcel(.success), Self.numbers, Parcel(.endStatement), Parcel(.endRequest)]), + ]) + var headers: [[String]] = [] + var batches = 0 + + try connection.stream("SELECT n FROM sales.orders WHERE 1 = 0", onColumns: { headers.append($0.map { $0.name }) }, onRows: { _ in + batches += 1 + return true + }) + + #expect(headers == [["n"]]) + #expect(batches == 0) + } + + @Test("A statement without a result set still answers, with no columns") + func statementWithoutResultHasNoColumns() throws { + let (connection, _) = Self.connection([ + Self.reply([Parcel(.success), Parcel(.endStatement), Parcel(.endRequest)]), + ]) + var headers: [[String]] = [] + + try connection.stream("DELETE FROM sales.orders", onColumns: { headers.append($0.map { $0.name }) }, onRows: { _ in true }) + + #expect(headers == [[]]) + } + + @Test("A consumer that wants no more rows ends the output and keeps the session") + func consumerEndsTheOutput() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1)]), + Self.reply([Parcel(.endRequest)]), + Self.finished, + ]) + var batches = 0 + + try connection.stream("SELECT n FROM sales.orders", onColumns: { _ in }, onRows: { _ in + batches += 1 + return false + }) + + #expect(batches == 1) + #expect(transport.sentKinds == [Self.start, Self.cont]) + #expect(transport.sent.last?.body == TeradataMessages.cancelParcel().encoded()) + #expect(!transport.shutDown) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.start]) + } + + @Test("A failure after some rows throws the server's error and keeps the session") + func failureAfterRowsThrows() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1)]), + Self.reply([Self.failure(code: 2_646, "No more spool space in dbc.")]), + Self.finished, + ]) + var batches: [[[TeradataValue]]] = [] + + let error = Self.thrown { + try connection.stream("SELECT n FROM sales.orders", onColumns: { _ in }, onRows: { rows in + batches.append(rows) + return true + }) + } + + var code: Int? + if case .server(let failed, _)? = error as? TeradataWireError { code = failed } + #expect(code == 2_646) + #expect(batches == [[[.integer(1)]]]) + #expect(connection.isConnected) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.start]) + } + + @Test("Stop between replies ends the output, and the stream ends as cancelled") + func stopBetweenRepliesEndsTheStream() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1)]), + Self.reply([Parcel(.endRequest)]), + Self.finished, + ]) + var batches = 0 + + let error = Self.thrown { + try connection.stream("SELECT n FROM sales.orders", onColumns: { _ in }, onRows: { _ in + batches += 1 + connection.cancel() + return true + }) + } + + #expect(Self.isCancelled(error)) + #expect(batches == 1) + #expect(transport.sentKinds == [Self.start, Self.cont]) + #expect(transport.sent.last?.body == TeradataMessages.cancelParcel().encoded()) + #expect(!transport.shutDown) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.start]) + } + + @Test("A later statement's records are never read against the first statement's columns") + func laterStatementKeepsItsOwnColumns() throws { + let answer = Self.reply([ + Self.numbers, Self.number(1), Parcel(.endStatement), + Self.prepInfoX([(Self.char, 3, "code")]), Self.code("abc"), Parcel(.endStatement), + Self.numbers, Self.number(2), Parcel(.endStatement), + Parcel(.endRequest), + ]) + let sql = "SELECT n FROM sales.a; SELECT code FROM sales.b; SELECT n FROM sales.c" + + let result = try Self.connection([answer]).0.execute(sql) + let names = result.columns.map { $0.name } + #expect(names == ["n"]) + #expect(result.rows == [[.integer(1)], [.integer(2)]]) + + var headers: [[String]] = [] + var streamed: [[TeradataValue]] = [] + try Self.connection([answer]).0.stream(sql, onColumns: { headers.append($0.map { $0.name }) }, onRows: { rows in + streamed += rows + return true + }) + #expect(headers == [["n"]]) + #expect(streamed == result.rows) + } + + @Test("Stop before the first reply aborts an ANSI stream's request, and the stream reports the rollback") + func abortedStreamReportsTheRollback() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.failure(code: 3_110, "The transaction was aborted by the user.")]), + Self.finished, + ], transactionMode: .ansi) + transport.beforeReply[0] = { connection.cancel() } + var headers = 0 + + let error = Self.thrown { + try connection.stream("SELECT n FROM sales.orders", onColumns: { _ in headers += 1 }, onRows: { _ in true }) + } + + var rolledBack = false + if case .rolledBackByStop? = error as? TeradataWireError { rolledBack = true } + #expect(rolledBack) + #expect(headers == 0) + #expect(transport.sentKinds == [Self.start, Self.abort]) + #expect(connection.isConnected) + #expect(!connection.hasLostConnection) + } + + @Test("A row cap ends the output at the reply that passes it and marks the result cut") + func rowCapEndsTheOutput() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1), Self.number(2)]), + Self.reply([Self.number(3), Self.number(4)]), + Self.reply([Parcel(.endRequest)]), + Self.finished, + ]) + + let capped = try connection.execute("SELECT n FROM sales.orders", rowCap: 3) + + #expect(capped.isTruncated) + #expect(capped.resultSet.rows == [[.integer(1)], [.integer(2)], [.integer(3)]]) + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont]) + #expect(transport.sent.last?.body == TeradataMessages.cancelParcel().encoded()) + #expect(!connection.hasLostConnection) + + _ = try connection.execute("SELECT 1") + #expect(transport.sentKinds == [Self.start, Self.cont, Self.cont, Self.start]) + } + + @Test("A result with exactly as many rows as the cap is not cut") + func resultAtTheCapIsComplete() throws { + let (connection, transport) = Self.connection([ + Self.reply([Self.numbers, Self.number(1), Self.number(2), Parcel(.endRequest)]), + ]) + + let capped = try connection.execute("SELECT n FROM sales.orders", rowCap: 2) + + #expect(!capped.isTruncated) + #expect(capped.resultSet.rows == [[.integer(1)], [.integer(2)]]) + #expect(transport.sentKinds == [Self.start]) + } +} diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCellFormatting.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCellFormatting.swift index c40818145b..eb1c3c1459 100644 --- a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCellFormatting.swift +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCellFormatting.swift @@ -11,9 +11,6 @@ import OSLog public enum OracleCellFormatting { public static let maxHexBytes = 4_096 - /// A region-named zone is sent as an id into Oracle's time zone file, which the client does not have. - public static let timeZoneRegionPlaceholder = unsupportedPlaceholder(typeName: "time zone region") - // MARK: - Datetimes /// A zone's hour and minute are each sent above a bias, so a zone west of UTC sends bytes below it. @@ -32,16 +29,25 @@ public enum OracleCellFormatting { return fields.text(fractionDigits: fractionDigits) } - /// `TIMESTAMP(p) WITH TIME ZONE` as the wall clock at the value's own offset, followed by that offset - /// with no space. The date and time bytes are UTC. - public static func timestampWithTimeZoneText(wire bytes: [UInt8], fractionDigits: Int) -> String? { + // The date and time bytes are UTC, whether the zone is an offset or a region. + public static func timestampWithTimeZoneText( + wire bytes: [UInt8], + fractionDigits: Int, + regions: OracleTimeZoneRegions = .none + ) -> String? { guard var fields = OracleWireDateTime(bytes) else { return nil } + if let id = OracleTimeZoneRegions.regionID(wire: bytes) { + guard let region = regions.region(forID: id), let zone = region.zone else { + return fields.text(fractionDigits: fractionDigits) + offsetText(minutes: 0) + } + let instant = Date(timeIntervalSince1970: TimeInterval(fields.secondsTreatingFieldsAsUTC)) + fields.shift(bySeconds: zone.secondsFromGMT(for: instant)) + // A space before the name, as Oracle's TZR spells it. + return fields.text(fractionDigits: fractionDigits) + " " + region.name + } guard bytes.count >= 13, bytes[11] != 0, bytes[12] != 0 else { return fields.text(fractionDigits: fractionDigits) + "+00:00" } - if bytes[11] & 0x80 != 0 { - return timeZoneRegionPlaceholder - } let offsetMinutes = (Int(bytes[11]) - zoneHourBias) * 60 + Int(bytes[12]) - zoneMinuteBias fields.shift(bySeconds: offsetMinutes * 60) return fields.text(fractionDigits: fractionDigits) + offsetText(minutes: offsetMinutes) @@ -324,11 +330,12 @@ struct OracleResultColumn: Sendable, Equatable { enum OracleCellDecoding { private static let logger = Logger(subsystem: "com.TablePro", category: "OracleCellDecoding") - /// `unsupported` hears the type name of a cell rendered as a placeholder, so the caller can log it once. + // `unsupported` hears the type name of a cell rendered as a placeholder, so the caller can log it once. static func decode( _ cell: OracleCell, column: OracleResultColumn, zones: OracleSessionTimeZones, + regions: OracleTimeZoneRegions = .none, unsupported: (String) -> Void ) -> OracleRawCell { guard let bytes = cell.bytes else { return .null } @@ -339,7 +346,7 @@ enum OracleCellDecoding { do { guard let text = try text(of: cell, wire: Array(bytes.readableBytesView), column: column, zones: zones, - unsupported: unsupported) + regions: regions, unsupported: unsupported) else { return .null } return .string(text) } catch { @@ -355,6 +362,7 @@ enum OracleCellDecoding { wire: [UInt8], column: OracleResultColumn, zones: OracleSessionTimeZones, + regions: OracleTimeZoneRegions, unsupported: (String) -> Void ) throws -> String? { switch cell.dataType { @@ -380,7 +388,9 @@ enum OracleCellDecoding { return OracleCellFormatting.timestampText(wire: wire, fractionDigits: column.fractionDigits) case .timestampTZ: - return OracleCellFormatting.timestampWithTimeZoneText(wire: wire, fractionDigits: column.fractionDigits) + return OracleCellFormatting.timestampWithTimeZoneText( + wire: wire, fractionDigits: column.fractionDigits, regions: regions + ) case .timestampLTZ: return OracleCellFormatting.localTimestampText(wire: wire, fractionDigits: column.fractionDigits, zones: zones) diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift index b7b8433292..15758e58f6 100644 --- a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleCoreConnection.swift @@ -109,9 +109,10 @@ public final class OracleCoreConnection: @unchecked Sendable { var close = OracleCloseRecord() var serverRelease: OracleServerRelease? var loginWarning: String? - /// Read before the first statement of each session, and again before the statement after one that can - /// change them, so no result ever waits on them mid-stream. + // Read before a session's first statement and again after one that can change them, never mid-stream. var sessionZones: OracleSessionTimeZones? + // The server's time zone file fixes these, so only a new session clears them. + var timeZoneRegions: OracleTimeZoneRegions? var isReservedForDebugging = false } @@ -218,6 +219,7 @@ public final class OracleCoreConnection: @unchecked Sendable { current.serverRelease = release current.loginWarning = connection.loginWarning?.message current.sessionZones = nil + current.timeZoneRegions = nil current.close.clearOnConnect() return true } @@ -846,8 +848,13 @@ public final class OracleCoreConnection: @unchecked Sendable { let admitted = try admit(role) let result = try await withQueryDeadline { [self] in let zones = await sessionZones(on: connection) + let regions = await timeZoneRegions(on: connection) return try await collectRows( - statement, options: OracleRowLimits.collecting(admitted.options), zones: zones, on: connection + statement, + options: OracleRowLimits.collecting(admitted.options), + zones: zones, + regions: regions, + on: connection ) } await recordSuccess(of: role, on: admitted.session) @@ -936,7 +943,7 @@ public final class OracleCoreConnection: @unchecked Sendable { } } - /// The app's own session statements, none of which reads a `TIMESTAMP WITH LOCAL TIME ZONE`. + // The app's own session statements, none of which reads a zoned timestamp. private func collectRows( _ query: String, options: StatementOptions, @@ -946,6 +953,7 @@ public final class OracleCoreConnection: @unchecked Sendable { OracleStatement(unsafeSQL: query), options: options, zones: .driverDefault, + regions: OracleTimeZoneRegions.none, on: connection ) } @@ -954,12 +962,13 @@ public final class OracleCoreConnection: @unchecked Sendable { _ statement: OracleStatement, options: StatementOptions, zones: OracleSessionTimeZones, + regions: OracleTimeZoneRegions, on connection: OracleNIO.OracleConnection ) async throws -> OracleRawResult { let stream = try await connection.execute(statement, options: options, logger: nioLogger) let columns = stream.columns.map(OracleResultColumn.init) - var decoder = CellDecoder(columns: columns, zones: zones) + var decoder = CellDecoder(columns: columns, zones: zones, regions: regions) var allRows: [[OracleRawCell]] = [] var truncated = false @@ -1005,10 +1014,12 @@ public final class OracleCoreConnection: @unchecked Sendable { let admitted = try admit(role) try await withQueryDeadline { [self] in let zones = await sessionZones(on: connection) + let regions = await timeZoneRegions(on: connection) try await streamRows( query, options: admitted.options, zones: zones, + regions: regions, batching: batching, on: connection, continuation: continuation @@ -1031,6 +1042,7 @@ public final class OracleCoreConnection: @unchecked Sendable { _ query: String, options: StatementOptions, zones: OracleSessionTimeZones, + regions: OracleTimeZoneRegions, batching: OracleRowBatcher, on connection: OracleNIO.OracleConnection, continuation: AsyncThrowingStream.Continuation @@ -1039,7 +1051,7 @@ public final class OracleCoreConnection: @unchecked Sendable { let stream = try await connection.execute(statement, options: options, logger: nioLogger) let columns = stream.columns.map(OracleResultColumn.init) - var decoder = CellDecoder(columns: columns, zones: zones) + var decoder = CellDecoder(columns: columns, zones: zones, regions: regions) continuation.yield(.header(columns: columns.map(\.descriptor))) var batcher = batching @@ -1104,17 +1116,68 @@ public final class OracleCoreConnection: @unchecked Sendable { state.withLock { $0.sessionZones = nil } } + // MARK: - Time Zone Regions + + // A failed read keeps an empty map for the session, so a region value reads as its UTC wall clock and the + // read is not retried before every statement. + private func timeZoneRegions(on connection: OracleNIO.OracleConnection) async -> OracleTimeZoneRegions { + if let regions = state.withLock({ $0.timeZoneRegions }) { + return regions + } + do { + let regions = try await readTimeZoneRegions(on: connection) + state.withLock { $0.timeZoneRegions = regions } + return regions + } catch is CancellationError { + return OracleTimeZoneRegions.none + } catch { + osLogger.error( + "Reading the Oracle time zone regions failed: \(String(describing: type(of: error)), privacy: .public)" + ) + state.withLock { $0.timeZoneRegions = OracleTimeZoneRegions.none } + return OracleTimeZoneRegions.none + } + } + + // Read straight from the stream: the cell decode would turn the region column back into the UTC fallback. + private func readTimeZoneRegions(on connection: OracleNIO.OracleConnection) async throws -> OracleTimeZoneRegions { + let stream = try await connection.execute( + OracleStatement(unsafeSQL: OracleTimeZoneRegions.query), + options: OracleRowLimits.collecting(Self.sessionSetupOptions), + logger: nioLogger + ) + var rows: [(name: String, wire: [UInt8])] = [] + for try await row in stream { + let cells = Array(row) + guard cells.count == 2, let name = try? cells[0].decode(String.self), let wire = cells[1].bytes else { + continue + } + rows.append((name: name, wire: Array(wire.readableBytesView))) + } + let regions = OracleTimeZoneRegions(rows: rows) + let repeated = regions.duplicateIDs + let unknown = regions.unknownNames + if repeated > 0 || unknown > 0 { + osLogger.notice( + "Oracle time zone regions: \(regions.count, privacy: .public), repeated ids \(repeated, privacy: .public), unknown names \(unknown, privacy: .public)" + ) + } + return regions + } + // MARK: - Cell Decoding - /// Decodes the cells of one result, collecting the names of the types it had to render as placeholders. + // Collects the names of the types it had to render as placeholders, so each is logged once. private struct CellDecoder { let columns: [OracleResultColumn] let zones: OracleSessionTimeZones + let regions: OracleTimeZoneRegions private(set) var unsupportedTypeNames: Set = [] - init(columns: [OracleResultColumn], zones: OracleSessionTimeZones) { + init(columns: [OracleResultColumn], zones: OracleSessionTimeZones, regions: OracleTimeZoneRegions) { self.columns = columns self.zones = zones + self.regions = regions } mutating func decode(_ cells: some Sequence) -> [OracleRawCell] { @@ -1124,7 +1187,7 @@ public final class OracleCoreConnection: @unchecked Sendable { let column = columns.indices.contains(index) ? columns[index] : OracleResultColumn(name: cell.columnName, dataType: cell.dataType, scale: 9) - values.append(OracleCellDecoding.decode(cell, column: column, zones: zones) { name in + values.append(OracleCellDecoding.decode(cell, column: column, zones: zones, regions: regions) { name in unsupportedTypeNames.insert(name) }) } diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTimeZoneRegions.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTimeZoneRegions.swift new file mode 100644 index 0000000000..34ec98233f --- /dev/null +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTimeZoneRegions.swift @@ -0,0 +1,63 @@ +import Foundation + +// A TIMESTAMP WITH TIME ZONE stored with a region name carries an id into the server's time zone file, which a +// thin client does not have, so the names are read from the server once per connection. +public struct OracleTimeZoneRegions: Sendable, Equatable { + public struct Region: Sendable, Equatable { + public let name: String + public let zone: TimeZone? + } + + public static let none = OracleTimeZoneRegions(regions: [:], duplicateIDs: 0) + + // AT TIME ZONE converts an instant, so no name can fail on a wall clock its zone skips. + public static let query = "SELECT TZNAME, TIMESTAMP '2000-01-01 00:00:00 UTC' AT TIME ZONE TZNAME FROM " + + "(SELECT DISTINCT TZNAME FROM \(OracleDictionary.performanceView("V$TIMEZONE_NAMES"))) ORDER BY TZNAME" + + private static let regionFlag: UInt8 = 0x80 + + let regions: [Int: Region] + let duplicateIDs: Int + + private init(regions: [Int: Region], duplicateIDs: Int) { + self.regions = regions + self.duplicateIDs = duplicateIDs + } + + public init(rows: [(name: String, wire: [UInt8])]) { + var regions: [Int: Region] = [:] + var duplicateIDs = 0 + for row in rows { + guard let id = Self.regionID(wire: row.wire) else { continue } + guard regions[id] == nil else { + duplicateIDs += 1 + continue + } + regions[id] = Region(name: row.name, zone: TimeZone(identifier: row.name)) + } + self.init(regions: regions, duplicateIDs: duplicateIDs) + } + + public var isEmpty: Bool { + regions.isEmpty + } + + public var count: Int { + regions.count + } + + var unknownNames: Int { + regions.values.filter { $0.zone == nil }.count + } + + func region(forID id: Int) -> Region? { + regions[id] + } + + // An offset's hour byte is at most 20 + 14, so the flag alone marks a region; the low two bits of byte 12 are not + // part of the id. + public static func regionID(wire bytes: [UInt8]) -> Int? { + guard bytes.count >= 13, bytes[11] & regionFlag != 0 else { return nil } + return (Int(bytes[11] & 0x7F) << 6) | (Int(bytes[12] & 0xFC) >> 2) + } +} diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellDecodingTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellDecodingTests.swift index dbc3ed24d6..5e5f2c8306 100644 --- a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellDecodingTests.swift +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellDecodingTests.swift @@ -15,12 +15,15 @@ struct OracleCellDecodingTests { _ bytes: [UInt8]?, as type: OracleDataType, declared: OracleDataType? = nil, - scale: Int = 0 + scale: Int = 0, + regions: OracleTimeZoneRegions = .none ) -> (OracleRawCell, Set) { let cell = OracleCell(bytes: bytes.map { ByteBuffer(bytes: $0) }, dataType: type, columnName: "C", columnIndex: 0) let column = OracleResultColumn(name: "C", dataType: declared ?? type, scale: scale) var unsupported: Set = [] - let value = OracleCellDecoding.decode(cell, column: column, zones: Self.utcPlusSeven) { unsupported.insert($0) } + let value = OracleCellDecoding.decode(cell, column: column, zones: Self.utcPlusSeven, regions: regions) { + unsupported.insert($0) + } return (value, unsupported) } @@ -73,6 +76,14 @@ struct OracleCellDecodingTests { #expect(decode(Self.hex("787c0102020506000003e8"), as: .timestampLTZ, scale: 6).0 == .string("2024-01-02 08:04:05.000001+07:00")) } + @Test func testRegionZoneDecodesThroughTheRegionMap() { + let paris = Self.hex("787c01020305061dcd650085f8") + let (named, unsupported) = decode(paris, as: .timestampTZ, scale: 6, regions: OracleTimeZoneRegionFixtures.paris) + #expect(named == .string("2024-01-02 03:04:05.500000 Europe/Paris")) + #expect(unsupported.isEmpty) + #expect(decode(paris, as: .timestampTZ, scale: 6).0 == .string("2024-01-02 02:04:05.500000+00:00")) + } + @Test func testUniversalRowIDReadsAsOracleSpellsIt() { #expect(decode(Array("*BAYAAesCwQL+".utf8), as: .uRowID).0 == .string("*BAYAAesCwQL+")) } diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellFormattingTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellFormattingTests.swift index 96687599e5..a766acd508 100644 --- a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellFormattingTests.swift +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleCellFormattingTests.swift @@ -54,10 +54,58 @@ struct OracleCellFormattingTests { #expect(OracleCellFormatting.timestampWithTimeZoneText(wire: utc, fractionDigits: 0) == "2024-01-01 00:30:00+05:00") } - /// A zone stored by region name is an id into Oracle's time zone file. It used to read ``. - @Test func testRegionNamedZoneIsAPlaceholder() { - let paris: [UInt8] = [0x78, 0x7c, 0x01, 0x02, 0x03, 0x05, 0x06, 0x1d, 0xcd, 0x65, 0x00, 0x85, 0xf8] - #expect(OracleCellFormatting.timestampWithTimeZoneText(wire: paris, fractionDigits: 6) == "") + private static let parisInJanuary: [UInt8] = [0x78, 0x7c, 0x01, 0x02, 0x03, 0x05, 0x06, 0x1d, 0xcd, 0x65, 0x00, 0x85, 0xf8] + + private static func regionText(_ wire: [UInt8], fractionDigits: Int = 6, regions: OracleTimeZoneRegions) -> String? { + OracleCellFormatting.timestampWithTimeZoneText(wire: wire, fractionDigits: fractionDigits, regions: regions) + } + + // A zone stored by region name used to read ``, and export wrote that text. + @Test func testRegionZoneIsTheWallClockAndOraclesName() { + #expect(Self.regionText(Self.parisInJanuary, regions: OracleTimeZoneRegionFixtures.paris) == "2024-01-02 03:04:05.500000 Europe/Paris") + } + + @Test func testRegionZoneFollowsDaylightSaving() { + let july: [UInt8] = [0x78, 0x7c, 0x07, 0x01, 0x0d, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x85, 0xf8] + #expect(Self.regionText(july, fractionDigits: 0, regions: OracleTimeZoneRegionFixtures.paris) == "2024-07-01 14:00:00 Europe/Paris") + } + + @Test func testRegionZoneCanCrossAYearBoundary() { + let lastHalfHourOf2024: [UInt8] = [0x78, 0x7c, 0x0c, 0x1f, 0x18, 0x1f, 0x01, 0x00, 0x00, 0x00, 0x00, 0x85, 0xf8] + let paris = Self.regionText(lastHalfHourOf2024, fractionDigits: 0, regions: OracleTimeZoneRegionFixtures.paris) + #expect(paris == "2025-01-01 00:30:00 Europe/Paris") + + // Id 65 is made up; only the map decides which name it stands for. + let losAngeles = OracleTimeZoneRegions(rows: [OracleTimeZoneRegionFixtures.row("America/Los_Angeles", zone: (0x81, 0x04))]) + let threeAMOn2025: [UInt8] = [0x78, 0x7d, 0x01, 0x01, 0x04, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x81, 0x04] + #expect(Self.regionText(threeAMOn2025, fractionDigits: 0, regions: losAngeles) == "2024-12-31 19:00:00 America/Los_Angeles") + } + + @Test func testRegionZoneIgnoresTheLowBitsOfByte12() { + var lowBitsSet = Self.parisInJanuary + lowBitsSet[12] = 0xf9 + #expect(Self.regionText(lowBitsSet, regions: OracleTimeZoneRegionFixtures.paris) == "2024-01-02 03:04:05.500000 Europe/Paris") + } + + // Without a name the text keeps the instant, as the UTC wall clock in the offset spelling the grid reads. + @Test func testRegionMissingFromTheMapReadsAsUTC() { + var otherID = Self.parisInJanuary + otherID[12] = 0xfc + #expect(Self.regionText(otherID, regions: OracleTimeZoneRegionFixtures.paris) == "2024-01-02 02:04:05.500000+00:00") + #expect(Self.regionText(Self.parisInJanuary, regions: .none) == "2024-01-02 02:04:05.500000+00:00") + #expect(OracleCellFormatting.timestampWithTimeZoneText(wire: Self.parisInJanuary, fractionDigits: 6) == "2024-01-02 02:04:05.500000+00:00") + } + + @Test func testRegionFoundationDoesNotKnowReadsAsUTC() { + let unknown = OracleTimeZoneRegions(rows: [ + OracleTimeZoneRegionFixtures.row("Not/AZone", zone: OracleTimeZoneRegionFixtures.parisZone), + ]) + #expect(Self.regionText(Self.parisInJanuary, regions: unknown) == "2024-01-02 02:04:05.500000+00:00") + } + + @Test func testRegionsLeaveAnOffsetValueAlone() { + let minusFiveThirty: [UInt8] = [0x78, 0x7c, 0x01, 0x02, 0x09, 0x23, 0x06, 0x07, 0x5b, 0xca, 0x00, 0x0f, 0x1e] + #expect(Self.regionText(minusFiveThirty, regions: OracleTimeZoneRegionFixtures.paris) == "2024-01-02 03:04:05.123456-05:30") } /// The bytes are the value in the database's zone; the text is the session's wall clock and the session's @@ -113,6 +161,7 @@ struct OracleCellFormattingTests { "2024-01-02 03:04:05.123456-05:30", "2024-01-02 08:04:05.000001+07:00", "2024-07-01 14:00:00+02:00", + "2024-01-02 02:04:05.500000+00:00", ] for spelling in spellings { let range = NSRange(spelling.startIndex..., in: spelling) diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleTimeZoneRegionsTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleTimeZoneRegionsTests.swift new file mode 100644 index 0000000000..da3f079302 --- /dev/null +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleTimeZoneRegionsTests.swift @@ -0,0 +1,77 @@ +import Foundation +@testable import TableProOracleCore +import Testing + +enum OracleTimeZoneRegionFixtures { + // Bytes 11 and 12 of a Europe/Paris value measured on 23ai: region id 382. + static let parisZone: (UInt8, UInt8) = (0x85, 0xf8) + + // A row as the map query returns it: 2000-01-01 00:00:00 UTC at the named zone. + static func row(_ name: String, zone: (UInt8, UInt8)) -> (name: String, wire: [UInt8]) { + (name, [0x78, 0x64, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, zone.0, zone.1]) + } + + static let paris = OracleTimeZoneRegions(rows: [row("Europe/Paris", zone: parisZone)]) +} + +struct OracleTimeZoneRegionsTests { + private static func wire(zone: (UInt8, UInt8)) -> [UInt8] { + OracleTimeZoneRegionFixtures.row("", zone: zone).wire + } + + @Test func testRegionIDSpansBytes11And12() { + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: OracleTimeZoneRegionFixtures.parisZone)) == 382) + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0xff, 0xfc))) == 8_191) + } + + @Test func testRegionIDIgnoresTheLowBitsOfByte12() { + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x85, 0xf9))) == 382) + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x85, 0xfb))) == 382) + } + + // The flag alone marks a region, so an id whose low six bits are zero is not read as a zero offset. + @Test func testRegionIDWithAZeroByte12IsStillARegion() { + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x85, 0x00))) == 320) + } + + @Test func testOffsetZonesHaveNoRegionID() { + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x14, 0x3c))) == nil) + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x0f, 0x1e))) == nil) + #expect(OracleTimeZoneRegions.regionID(wire: Self.wire(zone: (0x00, 0x00))) == nil) + #expect(OracleTimeZoneRegions.regionID(wire: Array(Self.wire(zone: (0x85, 0xf8)).prefix(11))) == nil) + } + + @Test func testMapDropsOffsetRowsAndKeepsTheFirstOfARepeatedID() throws { + let regions = OracleTimeZoneRegions(rows: [ + OracleTimeZoneRegionFixtures.row("Europe/Paris", zone: OracleTimeZoneRegionFixtures.parisZone), + OracleTimeZoneRegionFixtures.row("Europe/Monaco", zone: OracleTimeZoneRegionFixtures.parisZone), + OracleTimeZoneRegionFixtures.row("UTC", zone: (0x14, 0x3c)), + ]) + #expect(regions.count == 1) + #expect(regions.duplicateIDs == 1) + let paris = try #require(regions.region(forID: 382)) + #expect(paris.name == "Europe/Paris") + #expect(paris.zone?.identifier == "Europe/Paris") + } + + @Test func testMapKeepsANameFoundationDoesNotKnowWithoutAZone() throws { + let regions = OracleTimeZoneRegions(rows: [ + OracleTimeZoneRegionFixtures.row("Not/AZone", zone: OracleTimeZoneRegionFixtures.parisZone), + ]) + let region = try #require(regions.region(forID: 382)) + #expect(region.name == "Not/AZone") + #expect(region.zone == nil) + #expect(regions.unknownNames == 1) + } + + @Test func testNoneHoldsNoRegions() { + #expect(OracleTimeZoneRegions.none.isEmpty) + #expect(OracleTimeZoneRegions(rows: []) == OracleTimeZoneRegions.none) + } + + // A schema can hold its own V$TIMEZONE_NAMES, which an unqualified name resolves to first. + @Test func testQueryReadsTheQualifiedPerformanceView() { + #expect(OracleTimeZoneRegions.query.contains("SELECT DISTINCT TZNAME FROM SYS.V_$TIMEZONE_NAMES)")) + #expect(OracleTimeZoneRegions.query.hasSuffix("ORDER BY TZNAME")) + } +} diff --git a/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift b/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift index 5bbd4314cd..103e191ecc 100644 --- a/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift +++ b/apple/Plugins/BigQueryDriverPlugin/BigQueryPluginDriver.swift @@ -71,6 +71,7 @@ internal final class BigQueryPluginDriver: PluginDatabaseDriver, @unchecked Send .cellExpressions, .uuidCellText, .rowWrites, + .columnDefaultMarks, ] } diff --git a/apple/Plugins/BigQueryDriverPlugin/BigQueryStatementGenerator.swift b/apple/Plugins/BigQueryDriverPlugin/BigQueryStatementGenerator.swift index 0ee163ee61..c75a935967 100644 --- a/apple/Plugins/BigQueryDriverPlugin/BigQueryStatementGenerator.swift +++ b/apple/Plugins/BigQueryDriverPlugin/BigQueryStatementGenerator.swift @@ -6,8 +6,6 @@ import TableProPluginKit internal struct BigQueryStatementGenerator { typealias Statement = (statement: String, parameters: [PluginCellValue]) - static let defaultSentinel = "__DEFAULT__" - private static let logger = Logger(subsystem: "com.TablePro", category: "BigQueryStatementGenerator") private static let placeholder = "?" @@ -81,7 +79,7 @@ internal struct BigQueryStatementGenerator { var placeholders: [String] = [] var parameters: [PluginCellValue] = [] for (index, column) in columns.enumerated() { - guard let value = values[column], !Self.isDefaultSentinel(value) else { continue } + guard let value = values[column], !change.isColumnDefault(index) else { continue } names.append(GoogleSQLLiteral.quotedIdentifier(column)) if let sql = Self.expression(value, isMarked: expressionColumns.contains(index)) { placeholders.append(sql) @@ -124,7 +122,7 @@ internal struct BigQueryStatementGenerator { var parameters: [PluginCellValue] = [] let assignments = change.cellChanges.map { cellChange -> String in let column = GoogleSQLLiteral.quotedIdentifier(cellChange.columnName) - guard !Self.isDefaultSentinel(cellChange.newValue) else { return "\(column) = DEFAULT" } + guard !change.isColumnDefault(cellChange.columnIndex) else { return "\(column) = DEFAULT" } if let sql = Self.expression( cellChange.newValue, isMarked: expressionColumns.contains(cellChange.columnIndex) @@ -185,10 +183,6 @@ internal struct BigQueryStatementGenerator { ) } - private static func isDefaultSentinel(_ value: PluginCellValue) -> Bool { - value.asText == defaultSentinel - } - // The host checked a marked cell's text against the engine's vocabulary before handing it over. private static func expression(_ value: PluginCellValue, isMarked: Bool) -> String? { guard isMarked, case .text(let sql) = value else { return nil } diff --git a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift index 6f92b8c0a4..78becc1c1e 100644 --- a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift +++ b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift @@ -3,8 +3,6 @@ import TableProBigtableCore import TableProPluginKit extension BigtablePluginDriver { - private static let defaultMarker = "__DEFAULT__" - func generateRowWrites( table: String, schema: String?, @@ -50,11 +48,13 @@ extension BigtablePluginDriver { switch change.type { case .insert: return try BigtableRowWrites.insert( - table: table, columns: columns, values: cells(insertedRowData[change.rowIndex] ?? [], absent: absent) + table: table, + columns: columns, + values: cells(insertedRowData[change.rowIndex] ?? [], absent: absent.union(change.defaultColumns ?? [])) ) case .update: let changes = try change.cellChanges.map { cell in - guard cell.newValue.asText != Self.defaultMarker else { + guard !change.isColumnDefault(cell.columnIndex) else { throw BigtableError.unsupported(String(localized: "Bigtable has no default values. Delete the cell, or enter a value.")) } return (column: cell.columnName, value: absent.contains(cell.columnIndex) ? BigtableRowSet.Cell.absent : Self.cell(cell.newValue)) @@ -74,11 +74,10 @@ extension BigtablePluginDriver { return key } - /// A new row holds the host's Default marker where it has no value yet. Bigtable generates - /// nothing, so that cell is left out, and a `_key` left that way is refused. + // Bigtable generates nothing, so a cell left to the default is left out, and a `_key` left that way is refused. private func cells(_ row: [PluginCellValue], absent: Set) -> [BigtableRowSet.Cell] { row.enumerated().map { index, value in - absent.contains(index) || value.asText == Self.defaultMarker ? .absent : Self.cell(value) + absent.contains(index) ? .absent : Self.cell(value) } } diff --git a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift index 6a6b3de9c0..c67626f445 100644 --- a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift +++ b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver.swift @@ -22,7 +22,7 @@ internal final class BigtablePluginDriver: PluginDatabaseDriver, @unchecked Send } var serverVersion: String? { "Google Cloud Bigtable" } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } var supportsTransactions: Bool { false } var parameterStyle: ParameterStyle { .questionMark } diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift index ff8717faec..64b527cbf6 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift @@ -172,6 +172,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen .alterTableDDL, .cellExpressions, .uuidCellText, + .columnDefaultMarks, ] } diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift b/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift index ca4edf9125..0f9731a695 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift @@ -11,7 +11,6 @@ import TableProPluginKit // A new row is written `IF NOT EXISTS` and an edit `IF EXISTS` (CassandraConditionalWrite), and anything CQL cannot // do to a row is refused by name rather than sent. enum CassandraRowWriter { - static let defaultSentinel = "__DEFAULT__" static let unappliedMarker = CassandraConditionalWrite.appliedColumn static func rowWrites( @@ -79,12 +78,16 @@ enum CassandraRowWriter { )) } let row = values ?? rowFromCellChanges(change) + // A new row built from its edits leaves every column it never set to the server. + let leftOut = (change.defaultColumns ?? []).union( + values == nil ? Set(columns.indices).subtracting(change.cellChanges.map(\.columnIndex)) : [] + ) let expressionColumns = change.expressionColumns ?? [] var names: [String] = [] var placeholders: [String] = [] var parameters: [PluginCellValue] = [] for (index, value) in row.enumerated() where index < columns.count { - if value.asText == defaultSentinel { continue } + if leftOut.contains(index) { continue } names.append(CassandraBrowseRenderer.quote(columns[index])) if let call = expression(value, isMarked: expressionColumns.contains(index)) { placeholders.append(call) @@ -95,7 +98,7 @@ enum CassandraRowWriter { } for keyColumn in primaryKeyColumns { guard let index = columns.firstIndex(of: keyColumn), index < row.count, !row[index].isNull, - row[index].asText != defaultSentinel + !leftOut.contains(index) else { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( format: String(localized: "Enter a value for the primary key column %@."), keyColumn @@ -120,7 +123,7 @@ enum CassandraRowWriter { cell.columnName )) } - if cell.newValue.asText == defaultSentinel { + if change.isColumnDefault(cell.columnIndex) { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( localized: "Cassandra columns have no default value." )) @@ -168,7 +171,7 @@ enum CassandraRowWriter { } private func rowFromCellChanges(_ change: PluginRowChange) -> [PluginCellValue] { - var row = [PluginCellValue](repeating: .text(defaultSentinel), count: columns.count) + var row = [PluginCellValue](repeating: .null, count: columns.count) for cell in change.cellChanges where cell.columnIndex < row.count { row[cell.columnIndex] = cell.newValue } diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift index b0ea856105..5166531fd1 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHousePlugin.swift @@ -223,6 +223,7 @@ nonisolated final class ClickHousePluginDriver: PluginDatabaseDriver, @unchecked .uuidCellText, .repeatingPrimaryKey, .readOnlyStatements, + .columnDefaultMarks, ] } diff --git a/apple/Plugins/ClickHouseDriverPlugin/ClickHouseRowWrites.swift b/apple/Plugins/ClickHouseDriverPlugin/ClickHouseRowWrites.swift index 94d7513dfc..961f6f6185 100644 --- a/apple/Plugins/ClickHouseDriverPlugin/ClickHouseRowWrites.swift +++ b/apple/Plugins/ClickHouseDriverPlugin/ClickHouseRowWrites.swift @@ -17,8 +17,6 @@ nonisolated enum ClickHouseRowWrites { let delete: (_ table: String, _ predicate: String) -> String } - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - static func writes( table: String, schema: String?, @@ -52,7 +50,8 @@ nonisolated enum ClickHouseRowWrites { let values = insertedRowData[change.rowIndex] else { continue } writes.append(insert( into: target, columns: columns, values: values, expressions: change.expressionColumns ?? [], - serverOwned: context.serverOwnedColumns, rowIndex: change.rowIndex + defaults: change.defaultColumns ?? [], serverOwned: context.serverOwnedColumns, + rowIndex: change.rowIndex )) case .update: guard !change.cellChanges.isEmpty else { continue } @@ -78,13 +77,14 @@ nonisolated enum ClickHouseRowWrites { columns: [String], values: [PluginCellValue], expressions: Set, + defaults: Set = [], serverOwned: Set = [], rowIndex: Int ) -> PluginRowWrite { var names: [String] = [] var placeholders: [String] = [] var parameters: [PluginCellValue] = [] - for (index, value) in values.enumerated() where index < columns.count && value != defaultMarker { + for (index, value) in values.enumerated() where index < columns.count && !defaults.contains(index) { names.append(clickHouseQuotedIdentifier(columns[index])) if expressions.contains(index), case .text(let sql) = value { placeholders.append(sql) diff --git a/apple/Plugins/ClickHouseDriverTests/ClickHouseDMLStatementTests.swift b/apple/Plugins/ClickHouseDriverTests/ClickHouseDMLStatementTests.swift index 6c16be41f6..d1ef570299 100644 --- a/apple/Plugins/ClickHouseDriverTests/ClickHouseDMLStatementTests.swift +++ b/apple/Plugins/ClickHouseDriverTests/ClickHouseDMLStatementTests.swift @@ -248,6 +248,7 @@ struct ClickHouseDMLStatementTests { func insertStaysPlain() throws { var change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) change.expressionColumns = [1] + change.defaultColumns = [2] let write = try #require(try writes( [change], columns: ["id", "at", "note"], inserted: [0: ["7", "now()", "__DEFAULT__"]] )?.first) @@ -257,9 +258,21 @@ struct ClickHouseDMLStatementTests { #expect(write.lookup == nil) } + @Test("The text __DEFAULT__ in a column with no default mark is written as text") + func unmarkedDefaultTextIsAValue() throws { + let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + let write = try #require(try writes( + [change], columns: ["id", "note"], inserted: [0: ["7", "__DEFAULT__"]] + )?.first) + + #expect(write.statement == "INSERT INTO `users` (`id`, `note`) VALUES (?, ?)") + #expect(write.parameters == ["7", "__DEFAULT__"]) + } + @Test("A new row that sets nothing takes every default, named through a column the server does not fill") func emptyInsertTakesDefaults() throws { - let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + var change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + change.defaultColumns = [0, 1] var context = PluginRowWriteContext() context.serverOwnedColumns = ["id"] let write = try #require(try waitingDriver.generateRowWrites( diff --git a/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json b/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json index 89595dcb46..2c58ed400f 100644 --- a/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json +++ b/apple/Plugins/ColumnTypeCorpus/Corpus/oracle.json @@ -146,7 +146,8 @@ { "type": "TIMESTAMP WITH TIME ZONE", "values": [ - {"sql": "TO_TIMESTAMP_TZ('2026-10-06 15:22:20.000001 -05:30', 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')", "cell": "2026-10-06 15:22:20.000001-05:30"} + {"sql": "TO_TIMESTAMP_TZ('2026-10-06 15:22:20.000001 -05:30', 'YYYY-MM-DD HH24:MI:SS.FF TZH:TZM')", "cell": "2026-10-06 15:22:20.000001-05:30"}, + {"sql": "TIMESTAMP '2026-01-01 00:00:00 Europe/Paris'", "cell": "2026-01-01 00:00:00.000000 Europe/Paris"} ], "edit": {"text": "2000-01-01 00:00:00.000000+00:00", "cell": "2000-01-01 00:00:00.000000+00:00"}, "known": { diff --git a/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift b/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift index 70b89896ec..46a2450be2 100644 --- a/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift +++ b/apple/Plugins/DamengDriverPlugin/DamengPlugin.swift @@ -160,7 +160,7 @@ final class DamengPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var capabilities: PluginCapabilities { [ .parameterizedQueries, .transactions, .alterTableDDL, .multiSchema, .schemaCompare, .dataCompare, - .rowWrites, .cellExpressions, + .rowWrites, .cellExpressions, .columnDefaultMarks, ] } diff --git a/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Editing.swift b/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Editing.swift index cb50bfa039..f06963b60f 100644 --- a/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Editing.swift +++ b/apple/Plugins/DamengDriverPlugin/DamengPluginDriver+Editing.swift @@ -19,7 +19,10 @@ extension DamengPluginDriver { switch change.type { case .insert: guard insertedRowIndices.contains(change.rowIndex), let row = insertedRowData[change.rowIndex], - let statement = insertStatement(table: qualifiedTable, columns: columns, values: row, expressions: expressions) + let statement = insertStatement( + table: qualifiedTable, columns: columns, values: row, expressions: expressions, + defaults: change.defaultColumns ?? [] + ) else { continue } writes.append(PluginRowWrite(statement: statement.statement, parameters: statement.parameters, rowIndices: [change.rowIndex])) case .update: @@ -28,7 +31,7 @@ extension DamengPluginDriver { var parameters: [PluginCellValue] = [] let assignments = change.cellChanges.map { cell -> String in let column = quoteIdentifier(cell.columnName) - if cell.newValue == Self.defaultMarker { return "\(column) = DEFAULT" } + if change.isColumnDefault(cell.columnIndex) { return "\(column) = DEFAULT" } if expressions.contains(cell.columnIndex), case .text(let sql) = cell.newValue { return "\(column) = \(sql)" } parameters.append(cell.newValue) return "\(column) = ?" @@ -61,11 +64,11 @@ extension DamengPluginDriver { rows: [[PluginCellValue]] ) -> [(statement: String, parameters: [PluginCellValue])]? { let qualifiedTable = qualifiedName(schema: schema, object: table) - return rows.compactMap { insertStatement(table: qualifiedTable, columns: columns, values: $0, expressions: []) } + return rows.compactMap { + insertStatement(table: qualifiedTable, columns: columns, values: $0, expressions: [], defaults: []) + } } - private static let defaultMarker = PluginCellValue.text("__DEFAULT__") - func generateCreateTableSQL(definition: PluginCreateTableDefinition) -> String? { guard !definition.columns.isEmpty else { return nil } let table = qualifiedName(schema: nil, object: definition.tableName) @@ -200,14 +203,15 @@ extension DamengPluginDriver { table: String, columns: [String], values: [PluginCellValue], - expressions: Set + expressions: Set, + defaults: Set ) -> (statement: String, parameters: [PluginCellValue])? { var insertColumns: [String] = [] var placeholders: [String] = [] var parameters: [PluginCellValue] = [] for (index, value) in values.enumerated() where index < columns.count { insertColumns.append(quoteIdentifier(columns[index])) - if value == Self.defaultMarker { + if defaults.contains(index) { placeholders.append("DEFAULT") } else if expressions.contains(index), case .text(let sql) = value { placeholders.append(sql) diff --git a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift index 41e0c101de..e02600add0 100644 --- a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift +++ b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBPluginDriver.swift @@ -34,7 +34,7 @@ final class DynamoDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return client.endpoint.isLocal ? "DynamoDB Local" : "DynamoDB \(client.endpoint.signingRegion)" } - var capabilities: PluginCapabilities { [.parameterizedQueries, .cancelQuery] } + var capabilities: PluginCapabilities { [.parameterizedQueries, .cancelQuery, .columnDefaultMarks] } var supportsTransactions: Bool { false } var parameterStyle: ParameterStyle { .questionMark } diff --git a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift index 85756fc6c9..07585c2818 100644 --- a/apple/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift +++ b/apple/Plugins/DynamoDBDriverPlugin/DynamoDBWriteStatements.swift @@ -10,8 +10,6 @@ import TableProSQLQuoting /// the grid loaded, so it neither recreates an item someone deleted nor overwrites one someone /// changed. struct DynamoDBWriteStatements { - static let defaultSentinel = "__DEFAULT__" - let table: String let columns: [String] let keyColumns: [String] @@ -55,13 +53,12 @@ struct DynamoDBWriteStatements { return writes.map { (statement: $0.statement, parameters: $0.parameters) } } - func insert(row: [PluginCellValue]) -> (statement: String, parameters: [PluginCellValue]) { + func insert(row: [PluginCellValue], defaults: Set = []) -> (statement: String, parameters: [PluginCellValue]) { var names: [String] = [] var parameters: [PluginCellValue] = [] - for (index, column) in columns.enumerated() where index < row.count { + for (index, column) in columns.enumerated() where index < row.count && !defaults.contains(index) { let value = row[index] if case .null = value { continue } - if case .text(let text) = value, text == Self.defaultSentinel { continue } names.append(column) parameters.append(value) } @@ -73,14 +70,15 @@ struct DynamoDBWriteStatements { _ change: PluginRowChange, rowData: [PluginCellValue]? ) -> (statement: String, parameters: [PluginCellValue]) { + let defaults = change.defaultColumns ?? [] if let rowData { - return insert(row: rowData) + return insert(row: rowData, defaults: defaults) } var row = [PluginCellValue](repeating: .null, count: columns.count) for cell in change.cellChanges where cell.columnIndex < row.count { row[cell.columnIndex] = cell.newValue } - return insert(row: row) + return insert(row: row, defaults: defaults) } private func update(_ change: PluginRowChange) throws -> (statement: String, parameters: [PluginCellValue])? { @@ -92,7 +90,7 @@ struct DynamoDBWriteStatements { var guardValues: [PluginCellValue] = [] for cell in change.cellChanges { - if case .text(let text) = cell.newValue, text == Self.defaultSentinel { + if change.isColumnDefault(cell.columnIndex) { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( format: String(localized: "DynamoDB has no default values, so %@ cannot be set to DEFAULT."), cell.columnName @@ -239,7 +237,7 @@ struct DynamoDBParameterBinder { } return .binary(data) case .text(let text): - guard !text.isEmpty, text != DynamoDBWriteStatements.defaultSentinel else { + guard !text.isEmpty else { throw DynamoDBError.invalidValue( attribute: attribute, reason: String(localized: "A key attribute needs a value") ) diff --git a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift index 2d51700557..bb48629513 100644 --- a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift +++ b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift @@ -35,7 +35,7 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked var supportsTransactions: Bool { false } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } init(config: DriverConnectionConfig) { self.config = config @@ -248,6 +248,11 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes.filter { $0.type != .insert }) { + throw refusal + } + var insertedRowData = insertedRowData + let changes = changes.map { Self.leavingDefaultsOut($0, insertedRowData: &insertedRowData) } let typeNames = columnTypeNames(for: columns, index: table) let generator = ElasticsearchStatementGenerator( index: table, @@ -263,6 +268,29 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked ) } + // A new document leaves out a field left to its default, and one whose `_id` is left to it gets an id from the server. + private static func leavingDefaultsOut( + _ change: PluginRowChange, + insertedRowData: inout [Int: [PluginCellValue]] + ) -> PluginRowChange { + guard change.type == .insert, let defaults = change.defaultColumns, !defaults.isEmpty else { return change } + if var row = insertedRowData[change.rowIndex] { + for index in defaults where row.indices.contains(index) { row[index] = .null } + insertedRowData[change.rowIndex] = row + } + var cleared = PluginRowChange( + rowIndex: change.rowIndex, + type: change.type, + cellChanges: change.cellChanges.map { cell in + defaults.contains(cell.columnIndex) ? (cell.columnIndex, cell.columnName, cell.oldValue, .null) : cell + }, + originalRow: change.originalRow + ) + cleared.absentColumns = change.absentColumns + cleared.expressionColumns = change.expressionColumns + return cleared + } + // MARK: - Caching func cachedMappingColumns(_ index: String) async throws -> [ElasticsearchColumn] { diff --git a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift index 038ed7dbcc..9544d1e405 100644 --- a/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift +++ b/apple/Plugins/EtcdDriverPlugin/EtcdPluginDriver.swift @@ -51,7 +51,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var supportsTransactions: Bool { false } var capabilities: PluginCapabilities { - [.cancelQuery, .statementPolicy] + [.cancelQuery, .statementPolicy, .columnDefaultMarks] } // etcd has no transactions. @@ -449,6 +449,7 @@ final class EtcdPluginDriver: PluginDatabaseDriver, @unchecked Sendable { deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let generator = EtcdStatementGenerator( prefix: keyspace.prefix(forTable: table), columns: columns diff --git a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Editing.swift b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Editing.swift index 3e89356275..0dfd9c9ccc 100644 --- a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Editing.swift +++ b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver+Editing.swift @@ -35,6 +35,7 @@ extension FirebasePluginDriver { insertedRowIndices: Set, context: PluginRowWriteContext ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let types = writeColumnTypes(table, host: context.columnTypeNames) return try changes.compactMap { change in do { diff --git a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift index abad5c8f93..daa0134f66 100644 --- a/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift +++ b/apple/Plugins/FirebaseDriverPlugin/FirebasePluginDriver.swift @@ -34,7 +34,7 @@ internal final class FirebasePluginDriver: PluginDatabaseDriver, @unchecked Send variant == .firestore ? "Cloud Firestore" : "Firebase Realtime Database" } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } var supportsTransactions: Bool { false } var parameterStyle: ParameterStyle { .questionMark } diff --git a/apple/Plugins/HanaDriverPlugin/HanaBridgeContract.swift b/apple/Plugins/HanaDriverPlugin/HanaBridgeContract.swift index 962b4b563c..c1a4784456 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaBridgeContract.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaBridgeContract.swift @@ -121,6 +121,42 @@ struct HanaExplainRequest: Encodable, Sendable { let timeoutSeconds: Int } +struct HanaCursorRequest: Encodable, Sendable { + let sql: String + let timeoutSeconds: Int + let batchRows: Int +} + +struct HanaCursorHeader: Decodable, Equatable, Sendable { + let columns: [String] + let columnTypeNames: [String] + let columnClassifications: [String?] + let rowsAffected: Int64 + let hasResultSet: Bool +} + +struct HanaCursorBatch: Decodable, Equatable, Sendable { + let rows: [[HanaBridgeCell]] + let truncatedLobCount: Int + let done: Bool + let sessionLost: Bool +} + +struct HanaCursorOpening: Decodable, Sendable { + let header: HanaCursorHeader + let batch: HanaCursorBatch + + // The helper sends the header and the first batch as one flat object. + init(from decoder: any Decoder) throws { + header = try HanaCursorHeader(from: decoder) + batch = try HanaCursorBatch(from: decoder) + } +} + +struct HanaCursorClosing: Decodable, Sendable { + let sessionLost: Bool +} + struct HanaConnectResult: Decodable, Equatable, Sendable { let serverVersion: String let currentSchema: String diff --git a/apple/Plugins/HanaDriverPlugin/HanaConnection.swift b/apple/Plugins/HanaDriverPlugin/HanaConnection.swift index f129f13637..e8cfcba775 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaConnection.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaConnection.swift @@ -21,6 +21,7 @@ final class HanaConnection: HanaSession, @unchecked Sendable { var hasLostConnection = false } + private static let cursorBatchRows = 1_000 private static let logger = Logger(subsystem: "com.TablePro", category: "HanaConnection") private let bridge: any HanaNativeBridge @@ -105,6 +106,28 @@ final class HanaConnection: HanaSession, @unchecked Sendable { } } + func stream( + sql: String, + cancellation: HanaOperationSlot, + onHeader: @escaping @Sendable (HanaCursorHeader) -> Void, + onBatch: @escaping @Sendable (HanaCursorBatch) -> Void + ) async throws { + let request = HanaCursorRequest(sql: sql, timeoutSeconds: queryTimeoutSeconds, batchRows: Self.cursorBatchRows) + let requestJSON = try JSONEncoder().encode(request) + // Every fetch stays in this one queue item. The cursor holds the helper's operation slot, so a read queued + // between two fetches would wait on that slot while the next fetch waited behind it on this queue. + try await perform(.connected, cancellation: cancellation) { bridge, ticket in + try self.drainCursor( + bridge, + ticket: ticket, + request: requestJSON, + cancellation: cancellation, + onHeader: onHeader, + onBatch: onBatch + ) + } + } + func cancel(_ cancellation: HanaOperationSlot) { guard let ticket = cancellation.cancel() else { return } bridge.cancel(ticket) @@ -195,6 +218,65 @@ final class HanaConnection: HanaSession, @unchecked Sendable { } } + private func drainCursor( + _ bridge: any HanaNativeBridge, + ticket: HanaOperationTicket, + request: Data, + cancellation: HanaOperationSlot, + onHeader: (HanaCursorHeader) -> Void, + onBatch: (HanaCursorBatch) -> Void + ) throws { + let opened = try bridge.openCursor(ticket, request: request) + // A failure reply means the helper already closed the cursor. Any other exit leaves it open until closed here. + var holdsCursor = true + defer { + if holdsCursor { + closeCursor(bridge, ticket: ticket) + } + } + let opening = try Self.decode(HanaCursorOpening.self, from: opened) + onHeader(opening.header) + var batch = opening.batch + while true { + if batch.sessionLost { + Self.logger.error("SAP HANA session was lost while its cursor was open") + markLost(ticket.session) + } + if !batch.rows.isEmpty { + onBatch(batch) + } + if batch.done { + holdsCursor = false + return + } + guard !cancellation.isCancelled else { + throw HanaBridgeFailure(kind: .cancelled) + } + let fetched: Data + do { + fetched = try bridge.fetch(ticket) + } catch { + holdsCursor = false + throw error + } + batch = try Self.decode(HanaCursorBatch.self, from: fetched) + } + } + + private func closeCursor(_ bridge: any HanaNativeBridge, ticket: HanaOperationTicket) { + do { + let closing = try JSONDecoder().decode(HanaCursorClosing.self, from: bridge.closeCursor(ticket)) + if closing.sessionLost { + markLost(ticket.session) + } + } catch { + if (error as? HanaBridgeFailure)?.kind == .connectionLost { + markLost(ticket.session) + } + Self.logger.error("SAP HANA cursor did not close: \(error.localizedDescription, privacy: .private)") + } + } + private func perform( _ target: Target, cancellation slot: HanaOperationSlot, diff --git a/apple/Plugins/HanaDriverPlugin/HanaHelperBridge.swift b/apple/Plugins/HanaDriverPlugin/HanaHelperBridge.swift index 93afd8b101..337323aff8 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaHelperBridge.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaHelperBridge.swift @@ -152,6 +152,18 @@ final class HanaHelperBridge: HanaNativeBridge, @unchecked Sendable { try call(.explain, ticket: ticket) { HanaHelperMessage.statement($0, request: request) } } + func openCursor(_ ticket: HanaOperationTicket, request: Data) throws -> Data { + try call(.openCursor, ticket: ticket) { HanaHelperMessage.statement($0, request: request) } + } + + func fetch(_ ticket: HanaOperationTicket) throws -> Data { + try call(.fetch, ticket: ticket) { HanaHelperMessage.operation($0) } + } + + func closeCursor(_ ticket: HanaOperationTicket) throws -> Data { + try call(.closeCursor, ticket: ticket) { HanaHelperMessage.operation($0) } + } + func ping(_ ticket: HanaOperationTicket) throws { _ = try call(.ping, ticket: ticket) { HanaHelperMessage.operation($0) } } diff --git a/apple/Plugins/HanaDriverPlugin/HanaHelperFrame.swift b/apple/Plugins/HanaDriverPlugin/HanaHelperFrame.swift index 74f17e800d..764e5d2605 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaHelperFrame.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaHelperFrame.swift @@ -8,6 +8,9 @@ enum HanaHelperOpcode: UInt8, Sendable { case ping = 5 case cancel = 6 case close = 7 + case openCursor = 8 + case fetch = 9 + case closeCursor = 10 } enum HanaHelperFrameError: Error, Equatable, Sendable { @@ -135,7 +138,7 @@ struct HanaHelperGreeting: Equatable, Sendable { } enum HanaHelperHandshake { - static let protocolVersion = 1 + static let protocolVersion = 2 static let frameID: UInt64 = 0 static let maximumBodyLength = 4_096 static let acceptedForcedSeverGraceSeconds = 1...3_600 diff --git a/apple/Plugins/HanaDriverPlugin/HanaNativeBridge.swift b/apple/Plugins/HanaDriverPlugin/HanaNativeBridge.swift index 7e72d072b1..ec789006e5 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaNativeBridge.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaNativeBridge.swift @@ -5,6 +5,9 @@ protocol HanaNativeBridge: Sendable { func connect(_ ticket: HanaOperationTicket) throws -> Data func execute(_ ticket: HanaOperationTicket, request: Data) throws -> Data func explain(_ ticket: HanaOperationTicket, request: Data) throws -> Data + func openCursor(_ ticket: HanaOperationTicket, request: Data) throws -> Data + func fetch(_ ticket: HanaOperationTicket) throws -> Data + func closeCursor(_ ticket: HanaOperationTicket) throws -> Data func ping(_ ticket: HanaOperationTicket) throws func cancel(_ ticket: HanaOperationTicket) func close(session: UInt64) diff --git a/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift b/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift index 6bc959cc7c..571d6669e2 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift @@ -100,6 +100,22 @@ final class HanaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { try await run(query, parameters: parameters, rowCap: nil) } + // Collected here rather than through boundedQueryFromStream, which would drop the column hints and the + // large object warning that execute returns. + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + let report = HanaStreamReport() + let collected = try await PluginBoundedStream.collect( + stream(query, report: report), + rowCap: rowCap, + startedAt: Date() + ) + return report.completing(collected) + } + + func streamRows(query: String) -> AsyncThrowingStream { + stream(query, report: HanaStreamReport()) + } + func switchSchema(to schema: String) async throws { guard !schema.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { throw HanaError(kind: .configuration, message: String(localized: "The SAP HANA schema name cannot be empty.")) @@ -159,6 +175,63 @@ final class HanaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } + private func stream(_ sql: String, report: HanaStreamReport) -> AsyncThrowingStream { + PluginRowStream.make { continuation, abort in + let driver = self + let producer = Task { + do { + try await driver.produce(sql, into: continuation, report: report) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + report.logTruncatedLargeObjects() + } + // The cancel takes the Stop path: the helper ends a fetch in flight, or the cursor loop closes the cursor. + abort.onAbort { producer.cancel() } + } + } + + private func produce( + _ sql: String, + into continuation: AsyncThrowingStream.Continuation, + report: HanaStreamReport + ) async throws { + if HanaExplainStatement.explainedStatement(in: sql) != nil { + let result = try await run(sql, parameters: nil, rowCap: nil) + continuation.yield(.header(PluginStreamHeader( + columns: result.columns, + columnTypeNames: result.columnTypeNames + ))) + if !result.rows.isEmpty { + continuation.yield(.rows(result.rows)) + } + return + } + let query = beginQuery() + defer { endQuery(query) } + do { + try await session.stream( + sql: sql, + cancellation: query.cancellation, + onHeader: { header in + report.record(header) + continuation.yield(.header(PluginStreamHeader( + columns: header.columns, + columnTypeNames: header.columnTypeNames + ))) + }, + onBatch: { batch in + report.record(truncatedLargeObjects: batch.truncatedLobCount) + continuation.yield(.rows(HanaResultMapping.pluginRows(batch.rows))) + } + ) + } catch let failure as HanaBridgeFailure { + let requested = Task.isCancelled || cancellationGate.isCancelled(query.generation) + throw HanaFailureMapping.error(for: failure, cancellationRequested: requested) + } + } + private func beginQuery() -> RunningQuery { let cancellation = HanaOperationSlot() return stateLock.withLock { diff --git a/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift b/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift index 2da12083d7..0b2131cc46 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift @@ -1,4 +1,5 @@ import Foundation +import os import TableProPluginKit enum HanaResultMapping { @@ -6,15 +7,23 @@ enum HanaResultMapping { PluginQueryResult( columns: envelope.columns, columnTypeNames: envelope.columnTypeNames, - rows: envelope.rows.map { row in row.map(\.pluginValue) }, + rows: pluginRows(envelope.rows), rowsAffected: Int(clamping: envelope.rowsAffected), timing: PluginQueryTiming(total: envelope.executionTime), isTruncated: envelope.isTruncated, statusMessage: statusMessage(truncatedLobCount: envelope.truncatedLobCount), - columnMeta: columnMeta(for: envelope) + columnMeta: columnMeta( + columns: envelope.columns, + columnTypeNames: envelope.columnTypeNames, + classifications: envelope.columnClassifications + ) ) } + static func pluginRows(_ rows: [[HanaBridgeCell]]) -> [PluginRow] { + rows.map { row in row.map(\.pluginValue) } + } + static func statusMessage(truncatedLobCount: Int) -> String? { guard truncatedLobCount > 0 else { return nil } return String( @@ -23,17 +32,20 @@ enum HanaResultMapping { ) } - static func columnMeta(for envelope: HanaResultEnvelope) -> [PluginColumnInfo]? { - let hints = envelope.columnClassifications + static func columnMeta( + columns: [String], + columnTypeNames: [String], + classifications hints: [String?] + ) -> [PluginColumnInfo]? { guard hints.contains(where: { $0 != nil }), - hints.count == envelope.columns.count, - envelope.columnTypeNames.count == envelope.columns.count else { + hints.count == columns.count, + columnTypeNames.count == columns.count else { return nil } - return envelope.columns.indices.map { index in + return columns.indices.map { index in PluginColumnInfo( - name: envelope.columns[index], - dataType: envelope.columnTypeNames[index], + name: columns[index], + dataType: columnTypeNames[index], isNullable: true, isPrimaryKey: false, defaultValue: nil, @@ -55,3 +67,49 @@ enum HanaResultMapping { } } } + +final class HanaStreamReport: @unchecked Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "HanaPluginDriver") + + private let lock = NSLock() + private var header: HanaCursorHeader? + private var truncatedLargeObjects = 0 + + func record(_ header: HanaCursorHeader) { + lock.withLock { self.header = header } + } + + func record(truncatedLargeObjects count: Int) { + guard count > 0 else { return } + lock.withLock { truncatedLargeObjects += count } + } + + // A stream header has no field for the warning a result carries in statusMessage, so an export only logs it. + func logTruncatedLargeObjects() { + let count = lock.withLock { truncatedLargeObjects } + guard count > 0 else { return } + Self.logger.warning("SAP HANA cut \(count) large object values in a streamed result at 64 MiB") + } + + func completing(_ collected: PluginQueryResult) -> PluginQueryResult { + let (header, count) = lock.withLock { (self.header, self.truncatedLargeObjects) } + var result = PluginQueryResult( + columns: collected.columns, + columnTypeNames: collected.columnTypeNames, + rows: collected.rows, + rowsAffected: collected.rowsAffected, + timing: collected.timing, + isTruncated: collected.isTruncated, + statusMessage: HanaResultMapping.statusMessage(truncatedLobCount: count), + columnMeta: header.flatMap { + HanaResultMapping.columnMeta( + columns: $0.columns, + columnTypeNames: $0.columnTypeNames, + classifications: $0.columnClassifications + ) + } + ) + result.columnOrigins = collected.columnOrigins + return result + } +} diff --git a/apple/Plugins/HanaDriverPlugin/HanaSession.swift b/apple/Plugins/HanaDriverPlugin/HanaSession.swift index 3efa43e337..7405b5be75 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaSession.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaSession.swift @@ -13,6 +13,12 @@ protocol HanaSession: AnyObject, Sendable { cancellation: HanaOperationSlot ) async throws -> HanaResultEnvelope func explain(sql: String, cancellation: HanaOperationSlot) async throws -> HanaResultEnvelope + func stream( + sql: String, + cancellation: HanaOperationSlot, + onHeader: @escaping @Sendable (HanaCursorHeader) -> Void, + onBatch: @escaping @Sendable (HanaCursorBatch) -> Void + ) async throws func cancel(_ cancellation: HanaOperationSlot) func applyQueryTimeout(seconds: Int) } diff --git a/apple/Plugins/HanaDriverPlugin/README.md b/apple/Plugins/HanaDriverPlugin/README.md index 0a7efd020c..7834573d17 100644 --- a/apple/Plugins/HanaDriverPlugin/README.md +++ b/apple/Plugins/HanaDriverPlugin/README.md @@ -11,7 +11,8 @@ What the plugin does: - User name and password login, with TLS in every TablePro mode. Preferred and Required encrypt without checking the certificate, Verify CA checks the chain only, and Verify Identity checks the chain and the host name, against the system roots unless a CA file is set. A client certificate and key can be added for mutual TLS. The TLS Server Name field overrides the name checked by Verify Identity when the certificate names another host. - Schema browsing, columns with full types, primary keys, identity and generated columns, indexes, foreign keys, row counts, table DDL and view definitions. - SQL execution with bound parameters, so grid edits, inserts and deletes work. +- Exports and capped editor reads stream. The helper keeps the statement's result open as a cursor and sends it in batches of up to 1,000 rows or 4 MiB, so the row count has no limit. The query timeout covers running the statement and stops once its result is ready. A stream that ends early, or Stop after the result is ready, ends the fetch at its next row and closes the result without `ALTER SYSTEM CANCEL SESSION`. - Stop and the query timeout. Both send `ALTER SYSTEM CANCEL SESSION` for the connection's own session, which keeps the session open. The bridge closes the session instead, and TablePro reconnects, when the server refuses the cancel or does not answer it within 10 seconds, or when the statement is still running `hana.ForcedSeverGrace` (30 seconds) later. After Stop, and not after a timeout, the plugin also stops the helper when the operation is still pending 10 seconds past that grace, which the helper announces as `forcedSeverGraceSeconds` in its first frame. - `EXPLAIN PLAN FOR` runs as one step that saves the plan, reads it and deletes it again. Reading plans needs the `OPTIMIZER ADMIN` privilege on recent HANA versions. -Not supported: structure editing, transactions, LDAP, JWT and SSO logins. Large object values longer than 64 MiB are cut short, and the result says so. A reply frame holds at most 2 GiB - 1 bytes, so a larger result comes back as an internal error, "the result is larger than 2 GiB". +Not supported: structure editing, transactions, LDAP, JWT and SSO logins. Large object values longer than 64 MiB are cut short. A query result says so; an export only logs it, since a stream has no field for the warning. A reply frame holds at most 2 GiB - 1 bytes, so a statement that is not streamed and returns more comes back as an internal error, "the result is larger than 2 GiB". diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift index 1f73cebab5..9e670008b7 100644 --- a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBConnection.swift @@ -63,17 +63,7 @@ internal final class InfluxDBConnection: @unchecked Sendable { } func send(_ request: InfluxRequest, timeout: TimeInterval? = nil) async throws -> InfluxDBResponse { - guard let url = request.url(base: baseURL) else { - throw InfluxError.invalidURL(baseURL.absoluteString + request.path) - } - var urlRequest = URLRequest(url: url) - urlRequest.httpMethod = request.method - urlRequest.httpBody = request.body - urlRequest.timeoutInterval = timeout ?? queryTimeout.requestTimeoutInterval - for (name, value) in request.headers { - urlRequest.setValue(value, forHTTPHeaderField: name) - } - + let urlRequest = try makeURLRequest(request, timeout: timeout) let taskBox = PluginURLSessionTaskBox() let key = ObjectIdentifier(taskBox) lock.withLock { inFlight[key] = taskBox } @@ -115,6 +105,48 @@ internal final class InfluxDBConnection: @unchecked Sendable { task.resume() } } + + // The body chunk by chunk, the next one read only once `body` returns. A status outside 2xx throws what `send`'s + // caller would, and `cancelAll` stops it like any other request. + func stream(_ request: InfluxRequest, body: (Data) async throws -> Void) async throws { + let urlRequest = try makeURLRequest(request, timeout: nil) + let taskBox = PluginURLSessionTaskBox() + let key = ObjectIdentifier(taskBox) + lock.withLock { inFlight[key] = taskBox } + defer { lock.withLock { _ = inFlight.removeValue(forKey: key) } } + // Made under the lock `close()` invalidates with, as in `perform`. + guard let task = lock.withLock({ session?.dataTask(with: urlRequest) }) else { + throw InfluxError.notConnected + } + let chunks = InfluxDBHTTPChunks(task: task) + taskBox.set(task) + task.resume() + try await withTaskCancellationHandler { + do { + try await chunks.forEachChunk(body) + } catch let error as URLError where error.code == .cancelled { + throw CancellationError() + } + // A cancelled reader ends the chunk sequence without an error. + try Task.checkCancellation() + } onCancel: { + taskBox.cancel() + } + } + + private func makeURLRequest(_ request: InfluxRequest, timeout: TimeInterval?) throws -> URLRequest { + guard let url = request.url(base: baseURL) else { + throw InfluxError.invalidURL(baseURL.absoluteString + request.path) + } + var urlRequest = URLRequest(url: url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + urlRequest.timeoutInterval = timeout ?? queryTimeout.requestTimeoutInterval + for (name, value) in request.headers { + urlRequest.setValue(value, forHTTPHeaderField: name) + } + return urlRequest + } } private final class InfluxDBTrustingDelegate: PluginRedirectRefusingDelegate, @unchecked Sendable { diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBHTTPChunks.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBHTTPChunks.swift new file mode 100644 index 0000000000..b492c1455f --- /dev/null +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBHTTPChunks.swift @@ -0,0 +1,109 @@ +import Foundation +import TableProInfluxDBCore + +// The body of a streamed read as the socket delivers it. It is the task's delegate, not the session's, so the +// session's own delegate still answers the server trust challenge. The transfer is suspended as each chunk is handed +// over and resumed once the decoder is done with it, so undecoded body never piles up. +final class InfluxDBHTTPChunks: PluginRedirectRefusingDelegate, URLSessionDataDelegate, @unchecked Sendable { + // An error body is a message for the user; InfluxDB's run to a few hundred bytes. + private static let errorBodyByteCap = 65_536 + + private let stream: AsyncThrowingStream + private let continuation: AsyncThrowingStream.Continuation + private let lock = NSLock() + private var task: URLSessionDataTask? + private var isTransferSuspended = false + private var refusal: Error? + private var failedResponse: HTTPURLResponse? + private var failureBody = Data() + + // Takes over the task's delegate; the caller resumes it. + init(task: URLSessionDataTask) { + (stream, continuation) = AsyncThrowingStream.makeStream() + self.task = task + super.init() + task.delegate = self + continuation.onTermination = { _ in task.cancel() } + } + + // The task is torn down on every way out. A reader that throws leaves it suspended mid-body with nothing left to + // resume it, which would hold the connection open until the driver disconnects. + func forEachChunk(_ body: (Data) async throws -> Void) async throws { + defer { endTransfer() } + for try await chunk in stream { + try await body(chunk) + resumeTransfer() + } + } + + func urlSession( + _ session: URLSession, + dataTask: URLSessionDataTask, + didReceive response: URLResponse, + completionHandler: @escaping (URLSession.ResponseDisposition) -> Void + ) { + guard let http = response as? HTTPURLResponse else { + lock.withLock { refusal = InfluxError.malformedResponse("Not an HTTP response") } + completionHandler(.cancel) + return + } + if !(200 ..< 300).contains(http.statusCode) { + lock.withLock { failedResponse = http } + } + completionHandler(.allow) + } + + func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) { + let isFailure = lock.withLock { () -> Bool in + guard failedResponse != nil else { return false } + if failureBody.count < Self.errorBodyByteCap { + failureBody.append(data.prefix(Self.errorBodyByteCap - failureBody.count)) + } + return true + } + guard !isFailure else { return } + + // Suspended under the lock and before the reader sees the chunk, so the flag and the task never disagree. + lock.withLock { + guard !isTransferSuspended else { return } + isTransferSuspended = true + dataTask.suspend() + } + continuation.yield(data) + } + + // The same error `send` gives for the same response, so a refused statement reads alike on both paths. + func urlSession(_ session: URLSession, task: URLSessionTask, didCompleteWithError error: Error?) { + let failure = lock.withLock { () -> Error? in + if let refusal { return refusal } + if let error { return error } + guard let response = failedResponse else { return nil } + return InfluxDBResponse(statusCode: response.statusCode, headers: response.allHeaderFields, data: failureBody) + .error + } + guard let failure else { + continuation.finish() + return + } + continuation.finish(throwing: failure) + } + + private func resumeTransfer() { + lock.withLock { + guard isTransferSuspended else { return } + isTransferSuspended = false + task?.resume() + } + } + + private func endTransfer() { + let outstanding = lock.withLock { () -> URLSessionDataTask? in + let current = task + task = nil + isTransferSuspended = false + return current + } + outstanding?.cancel() + continuation.finish() + } +} diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift new file mode 100644 index 0000000000..9bf9af67a4 --- /dev/null +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver+Stream.swift @@ -0,0 +1,106 @@ +import Foundation +import TableProInfluxDBCore +import TableProPluginKit + +// Only the grid's own pages and exports stream: their columns are the measurement's, known before the first row. Flux +// and editor statements keep the collected read, whose columns are the union of everything that comes back. +extension InfluxDBPluginDriver { + func streamRows(query: String) -> AsyncThrowingStream { + guard let remembered = streamedGrid(query) else { return collectedStream(query) } + return PluginRowStream.make { continuation, abort in + let producer = Task { + do { + try await self.streamGrid(remembered, abort: abort, continuation: continuation) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + abort.onAbort { producer.cancel() } + } + } + + // A stream carries no column metadata, so the classification `execute` gives each column is added back. + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + guard streamedGrid(query) != nil else { return nil } + let result = try await boundedQueryFromStream(query: query, rowCap: rowCap) + return PluginQueryResult( + columns: result.columns, + columnTypeNames: result.columnTypeNames, + rows: result.rows, + rowsAffected: 0, + timing: result.timing, + isTruncated: result.isTruncated, + columnMeta: Self.columnMeta(columns: result.columns, typeNames: result.columnTypeNames) + ) + } + + private func streamedGrid(_ query: String) -> RememberedGridQuery? { + guard let remembered = rememberedGrid(query), remembered.query.shape != .count else { return nil } + return remembered + } + + private func streamGrid( + _ remembered: RememberedGridQuery, + abort: PluginStreamAbort, + continuation: AsyncThrowingStream.Continuation + ) async throws { + guard !abort.isAborted else { throw CancellationError() } + let (_, server) = try requireConnection() + let (statement, schema) = try await gridStatementFromSchema( + remembered.query, + hostKinds: remembered.hostKinds, + generation: server.generation + ) + let (connection, current, request) = try await queryRequest(statement, streamed: true) + guard !abort.isAborted else { throw CancellationError() } + + var decoder = InfluxGridStream( + body: current.generation == .v3 ? .csv : .influxQLChunks, + grid: remembered.query, + schema: schema + ) + var headerSent = false + func send(_ batches: [InfluxResultTable]) { + for batch in batches { + if !headerSent { + headerSent = true + continuation.yield(.header(PluginStreamHeader(columns: batch.columns, columnTypeNames: batch.typeNames))) + } + if !batch.rows.isEmpty { + continuation.yield(.rows(Self.cells(batch.rows))) + } + } + } + try await connection.stream(request) { chunk in + guard !abort.isAborted else { throw CancellationError() } + let batches = try decoder.consume(chunk) + send(batches) + } + let rest = try decoder.finish() + send(rest) + } + + // PluginKit's default stream, which the override above replaces for every statement. + private func collectedStream(_ query: String) -> AsyncThrowingStream { + PluginRowStream.make { continuation, abort in + let producer = Task { + do { + try Task.checkCancellation() + let result = try await self.execute(query: query) + continuation.yield(.header(PluginStreamHeader( + columns: result.columns, + columnTypeNames: result.columnTypeNames + ))) + if !result.rows.isEmpty { + continuation.yield(.rows(result.rows)) + } + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + abort.onAbort { producer.cancel() } + } + } +} diff --git a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift index 99a420548f..e2bc432c05 100644 --- a/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift +++ b/apple/Plugins/InfluxDBDriverPlugin/InfluxDBPluginDriver.swift @@ -28,7 +28,7 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send var serverVersion: String? { lock.withLock { server?.version } } var supportsTransactions: Bool { false } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } var currentDatabase: String? { lock.withLock { database.isEmpty ? nil : database } } func beginTransaction() async throws {} @@ -152,7 +152,7 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send let started = Date() let (_, server) = try requireConnection() let table: InfluxResultTable - if let remembered = lock.withLock({ gridQueries[query] }) { + if let remembered = rememberedGrid(query) { table = try await runGrid(remembered.query, hostKinds: remembered.hostKinds, generation: server.generation) } else { table = try await run(InfluxStatement.route(query, generation: server.generation)) @@ -160,22 +160,30 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send return PluginQueryResult( columns: table.columns, columnTypeNames: table.typeNames, - rows: table.rows.map { row in row.map { $0.map(PluginCellValue.text) ?? .null } }, + rows: Self.cells(table.rows), rowsAffected: 0, executionTime: Date().timeIntervalSince(started), - columnMeta: zip(table.columns, table.typeNames).map { name, typeName in - Self.columnInfo(name: name, typeName: typeName, role: nil) - } + columnMeta: Self.columnMeta(columns: table.columns, typeNames: table.typeNames) ) } - // A grid statement built before the measurement's columns were known is written again from them here. - // A filter on a column the cached schema lacks means the measurement gained one since; the schema is read again. private func runGrid( _ grid: InfluxGridQuery, hostKinds: [String: InfluxColumnKind], generation: InfluxGeneration ) async throws -> InfluxResultTable { + let (statement, schema) = try await gridStatementFromSchema(grid, hostKinds: hostKinds, generation: generation) + let table = try await run(statement).typed(by: schema) + return table.columns.isEmpty ? grid.emptyResult(schema: schema) : table + } + + // A grid statement built before the measurement's columns were known is written again from them here. + // A filter on a column the cached schema lacks means the measurement gained one since; the schema is read again. + func gridStatementFromSchema( + _ grid: InfluxGridQuery, + hostKinds: [String: InfluxColumnKind], + generation: InfluxGeneration + ) async throws -> (statement: InfluxStatement, schema: [InfluxColumn]) { var schema = try await columns(of: grid.measurement) let known = Set(schema.map(\.name)) if grid.filters.contains(where: { !known.contains($0.column) }) { @@ -186,20 +194,31 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send guard let text = grid.statement(language: language, kinds: kinds, marked: false) else { throw InfluxError.filterNotExpressible(language) } - var table = try await run(InfluxStatement(language: language, text: text)).typed(by: schema) - if table.columns.isEmpty { - let names = grid.projection.isEmpty ? schema.map(\.name) : grid.projection - let types = Dictionary(schema.map { ($0.name, $0.typeName) }, uniquingKeysWith: { first, _ in first }) - table = InfluxResultTable( - columns: names, - typeNames: names.map { types[$0] ?? InfluxTypeName.string }, - rows: [] - ) - } - return table + return (InfluxStatement(language: language, text: text), schema) + } + + func rememberedGrid(_ statement: String) -> RememberedGridQuery? { + lock.withLock { gridQueries[statement] } } func run(_ statement: InfluxStatement) async throws -> InfluxResultTable { + let (connection, server, request) = try await queryRequest(statement) + let response = try await connection.send(request) + guard response.isSuccess else { throw response.error } + switch (server.generation, statement.language) { + case (.v3, _): + return try InfluxResponseDecoder.decodeV3CSV(response.data) + case (_, .flux): + return try InfluxResponseDecoder.decodeFluxCSV(response.data) + default: + return try InfluxResponseDecoder.decodeV1JSON(response.data) + } + } + + func queryRequest( + _ statement: InfluxStatement, + streamed: Bool = false + ) async throws -> (connection: InfluxDBConnection, server: InfluxServerInfo, request: InfluxRequest) { let (connection, server) = try requireConnection() let (database, organization) = lock.withLock { (self.database, self.organization) } if server.generation == .v3, statement.language == .sql, database.isEmpty { @@ -221,18 +240,10 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send retentionPolicy: target.retentionPolicy, organization: organization, generation: server.generation, - credentials: connection.credentials + credentials: connection.credentials, + streamed: streamed ) - let response = try await connection.send(request) - guard response.isSuccess else { throw response.error } - switch (server.generation, statement.language) { - case (.v3, _): - return try InfluxResponseDecoder.decodeV3CSV(response.data) - case (_, .flux): - return try InfluxResponseDecoder.decodeFluxCSV(response.data) - default: - return try InfluxResponseDecoder.decodeV1JSON(response.data) - } + return (connection, server, request) } // Safe Mode read this script with TablePro's lexer. The server parses it first, and a script it reads as writing or @@ -360,10 +371,18 @@ internal final class InfluxDBPluginDriver: PluginDatabaseDriver, @unchecked Send columnCheck: nil ) } + + static func columnMeta(columns: [String], typeNames: [String]) -> [PluginColumnInfo] { + zip(columns, typeNames).map { name, typeName in columnInfo(name: name, typeName: typeName, role: nil) } + } + + static func cells(_ rows: [[String?]]) -> [PluginRow] { + rows.map { row in row.map { $0.map(PluginCellValue.text) ?? .null } } + } } // The statements the driver built for its grid, so `execute` runs them knowing the measurement they read. -private struct RememberedGridQuery { +struct RememberedGridQuery { let query: InfluxGridQuery let hostKinds: [String: InfluxColumnKind] } diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift index 0f78551a21..60fb612887 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift @@ -325,6 +325,7 @@ final class MSSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { .uuidCellText, .userManagement, .temporaryTables, + .columnDefaultMarks, ] } diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLStatementGenerator.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLStatementGenerator.swift index a195f9ef45..d849f6674f 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLStatementGenerator.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLStatementGenerator.swift @@ -14,9 +14,6 @@ import TableProSQLQuoting /// second row that happens to look the same. That bound is only safe when every column takes part in the match: a /// match with one left out can pick a different row, so such a change is refused instead. struct MSSQLStatementGenerator { - /// What the grid stages for a column the user leaves to the server's default. It is a marker, never a value. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - let qualifiedTable: String let columns: [String] let primaryKeyColumns: [String] @@ -38,7 +35,8 @@ struct MSSQLStatementGenerator { let statement = try insertStatement( values: values, rowIndex: change.rowIndex, - expressionColumns: change.expressionColumns ?? [] + expressionColumns: change.expressionColumns ?? [], + defaultColumns: change.defaultColumns ?? [] ) writes.append(PluginRowWrite(statement: statement.statement, parameters: statement.parameters, rowIndices: [change.rowIndex])) case .update: @@ -72,10 +70,11 @@ struct MSSQLStatementGenerator { func insertStatement( values: [PluginCellValue], rowIndex: Int = 0, - expressionColumns: Set = [] + expressionColumns: Set = [], + defaultColumns: Set = [] ) throws -> (statement: String, parameters: [PluginCellValue]) { let written = zip(columns, values).enumerated() - .filter { $0.element.1 != Self.defaultMarker } + .filter { !defaultColumns.contains($0.offset) } .map { (index: $0.offset, column: $0.element.0, value: $0.element.1) } if let owned = written.first(where: { context.serverOwnedColumns.contains($0.column) }) { throw PluginRowWriteRefusal(rowIndex: rowIndex, reason: Self.serverOwnedReason(owned.column)) @@ -106,7 +105,7 @@ struct MSSQLStatementGenerator { var parameters: [PluginCellValue] = [] let assignments = change.cellChanges.map { cellChange -> String in let column = Self.quote(cellChange.columnName) - guard cellChange.newValue != Self.defaultMarker else { return "\(column) = DEFAULT" } + guard !change.isColumnDefault(cellChange.columnIndex) else { return "\(column) = DEFAULT" } if let sql = Self.expression( cellChange.newValue, isMarked: expressionColumns.contains(cellChange.columnIndex) diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift index 272d58c088..cbbbd6569a 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedConnection.swift @@ -11,11 +11,16 @@ import TableProPluginKit private final class MemcachedReplyHandler: ChannelInboundHandler, @unchecked Sendable { typealias InboundIn = ByteBuffer + private let closed: NIOLockedValueBox private var reader: MemcachedReplyReader? private var continuation: CheckedContinuation? private var failure: Error? private let lock = NIOLock() + init(closed: NIOLockedValueBox) { + self.closed = closed + } + func expect( _ shape: MemcachedReplyShape, lineLimit: Int, @@ -57,6 +62,7 @@ private final class MemcachedReplyHandler: ChannelInboundHandler, @unchecked Sen } func channelInactive(context: ChannelHandlerContext) { + closed.withLockedValue { $0 = true } finish(with: MemcachedError.connectionClosed) } @@ -124,6 +130,10 @@ actor MemcachedConnection { private var wireSlotIsTaken = false private var waitingForWireSlot: [CheckedContinuation] = [] private var requestTimeout: TimeAmount? + private nonisolated let closed = NIOLockedValueBox(false) + + // True once the channel closed under the connection, as a cancelled or timed-out request closes it. + nonisolated var hasClosed: Bool { closed.withLockedValue { $0 } } /// A reply that has not finished by then closes the connection, since the text protocol cannot skip it. func setRequestTimeout(seconds: Int) { @@ -138,7 +148,7 @@ actor MemcachedConnection { guard channel == nil else { return } let sslContext = try Self.makeSSLContext(endpoint.ssl) let serverHostname = endpoint.ssl.verifiesHostname ? endpoint.tlsServerName : nil - let replyHandler = MemcachedReplyHandler() + let replyHandler = MemcachedReplyHandler(closed: closed) let bootstrap = ClientBootstrap(group: NIOSingletons.posixEventLoopGroup) .connectTimeout(endpoint.connectTimeout) .channelOption(ChannelOptions.socketOption(.so_keepalive), value: 1) diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift index cc1021e399..93d1cda577 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift @@ -13,6 +13,7 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { /// Keys one listing reads before it stops, so a cache holding millions of items cannot exhaust memory. static let listingLimit = 1_000_000 private static let valuesPerRequest = 100 + private static let rowsPerStreamBatch = 1_000 private let endpoint: MemcachedEndpoint private let tunneledServerName: String? @@ -40,7 +41,10 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var serverVersion: String? { lock.withLock { _serverVersion } } var supportsTransactions: Bool { false } - var capabilities: PluginCapabilities { [] } + var capabilities: PluginCapabilities { [.columnDefaultMarks] } + + // A stopped request closes the channel, since the text protocol cannot skip its reply; the host then reconnects. + var hasLostConnection: Bool { lock.withLock { connection }?.hasClosed ?? false } func beginTransaction() async throws {} func commitTransaction() async throws {} @@ -208,24 +212,28 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Browsing + private struct BrowsePage { + let entries: [MemcachedKeyEntry] + let serverTime: Int64 + // The listing stopped at `listingLimit`, so the page is drawn from part of the cache. + let isCut: Bool + } + private func browse(_ spec: MemcachedBrowseSpec, started: Date) async throws -> PluginQueryResult { let connection = try liveConnection() - let serverTime = try await Self.serverTime(on: connection) - let listed = try await listKeys().filter { !$0.isExpired(at: serverTime) } - let page = try spec.page(of: listed) - - var values: [String: MemcachedValue] = [:] - var start = page.startIndex - while start < page.endIndex { - try Task.checkCancellation() - let end = min(start + Self.valuesPerRequest, page.endIndex) - let keys = page[start ..< end].map(\.key) - if case .values(let read) = try await connection.send(.get(keys: keys, withCAS: false)) { - for value in read { values[value.key] = value } - } - start = end - } + let page = try await browsePage(spec, on: connection) + let rows = try await rows(of: page.entries, serverTime: page.serverTime, on: connection) + return result( + columns: MemcachedColumn.allCases.map(\.rawValue), + types: MemcachedColumn.allCases.map(\.typeName), + rows: rows, started: started + ) + } + private func browsePage(_ spec: MemcachedBrowseSpec, on connection: MemcachedConnection) async throws -> BrowsePage { + let serverTime = try await Self.serverTime(on: connection) + let listing = try await listKeys() + let page = try spec.page(of: listing.entries.filter { !$0.isExpired(at: serverTime) }) lock.withLock { serverClockOffset = MemcachedServerClock.offset( serverTime: serverTime, localTime: Int64(Date().timeIntervalSince1970) @@ -233,8 +241,27 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { if shownExpiries.count + page.count > Self.shownExpiryLimit { shownExpiries.removeAll() } for entry in page { shownExpiries[entry.key] = MemcachedKnownExpiry(expiresAt: entry.expiresAt) } } + return BrowsePage(entries: page, serverTime: serverTime, isCut: listing.isCut) + } - let rows: [PluginRow] = page.compactMap { entry in + // A key removed since the listing has no value and no row. + private func rows( + of entries: some Collection, + serverTime: Int64, + on connection: MemcachedConnection + ) async throws -> [PluginRow] { + var values: [String: MemcachedValue] = [:] + let keys = entries.map(\.key) + var start = keys.startIndex + while start < keys.endIndex { + try Task.checkCancellation() + let end = min(start + Self.valuesPerRequest, keys.endIndex) + if case .values(let read) = try await connection.send(.get(keys: Array(keys[start ..< end]), withCAS: false)) { + for value in read { values[value.key] = value } + } + start = end + } + return entries.compactMap { entry in guard let value = values[entry.key] else { return nil } return [ .text(entry.key), @@ -243,20 +270,15 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { .text(String(value.flags)), ] } - return result( - columns: MemcachedColumn.allCases.map(\.rawValue), - types: MemcachedColumn.allCases.map(\.typeName), - rows: rows, started: started - ) } - private func listKeys() async throws -> [MemcachedKeyEntry] { + private func listKeys() async throws -> (entries: [MemcachedKeyEntry], isCut: Bool) { if lock.withLock({ keySource }) == .metadump { do { return try await withListingConnection { connection in let reply = try await connection.send(.metadump(selector: "all"), lineLimit: Self.listingLimit) - guard case .lines(let lines, _) = reply else { return [] } - return lines.compactMap(MemcachedKeyListing.metadumpEntry) + guard case .lines(let lines, let truncated) = reply else { return ([], false) } + return (lines.compactMap(MemcachedKeyListing.metadumpEntry), truncated) } } catch let error as MemcachedServerError where error.kind != .busy && !error.needsAuthentication { Self.logger.info("metadump refused, listing with cachedump") @@ -266,18 +288,69 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return try await cachedumpKeys() } - private func cachedumpKeys() async throws -> [MemcachedKeyEntry] { + private func cachedumpKeys() async throws -> (entries: [MemcachedKeyEntry], isCut: Bool) { let connection = try liveConnection() - guard case .lines(let itemStats, _) = try await connection.send(.stats(arguments: ["items"])) else { return [] } + guard case .lines(let itemStats, _) = try await connection.send(.stats(arguments: ["items"])) else { return ([], false) } var entries: [MemcachedKeyEntry] = [] for slab in MemcachedKeyListing.slabClasses(fromItemStats: itemStats) { try Task.checkCancellation() let reply = try await connection.send(.stats(arguments: ["cachedump", String(slab), "0"])) guard case .lines(let lines, _) = reply else { continue } entries += lines.compactMap(MemcachedKeyListing.cachedumpEntry) - if entries.count >= Self.listingLimit { break } + if entries.count >= Self.listingLimit { return (Array(entries.prefix(Self.listingLimit)), true) } + } + return (entries, false) + } + + // MARK: - Streaming + + // An export reads every key, so its values are read and sent a batch at a time on a connection of their own, and + // stopping closes that connection, never the session's. + func streamRows(query: String) -> AsyncThrowingStream { + let statement = query.trimmingCharacters(in: .whitespacesAndNewlines) + let spec = MemcachedBrowseSpec.refusalMessage(statement) == nil ? MemcachedBrowseSpec.parse(statement) : nil + return PluginRowStream.make { continuation, abort in + let task = Task { + do { + if let spec { + try await self.streamBrowse(spec, into: continuation) + } else { + let result = try await self.execute(query: query) + var header = PluginStreamHeader(columns: result.columns, columnTypeNames: result.columnTypeNames) + if result.isTruncated { header.rowLimit = result.rows.count } + continuation.yield(.header(header)) + if !result.rows.isEmpty { continuation.yield(.rows(result.rows)) } + } + continuation.finish() + } catch { + continuation.finish(throwing: MemcachedError.wrap(error)) + } + } + abort.onAbort { task.cancel() } + } + } + + private func streamBrowse( + _ spec: MemcachedBrowseSpec, + into continuation: AsyncThrowingStream.Continuation + ) async throws { + let page = try await browsePage(spec, on: try liveConnection()) + var header = PluginStreamHeader( + columns: MemcachedColumn.allCases.map(\.rawValue), + columnTypeNames: MemcachedColumn.allCases.map(\.typeName) + ) + if page.isCut { header.rowLimit = page.entries.count } + continuation.yield(.header(header)) + try await withListingConnection { connection in + var start = page.entries.startIndex + while start < page.entries.endIndex { + try Task.checkCancellation() + let end = min(start + Self.rowsPerStreamBatch, page.entries.endIndex) + let rows = try await self.rows(of: page.entries[start ..< end], serverTime: page.serverTime, on: connection) + if !rows.isEmpty { continuation.yield(.rows(rows)) } + start = end + } } - return entries } /// Opens a connection for one listing and closes it after, even when the listing stopped early with the rest @@ -431,6 +504,7 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let offset = lock.withLock { serverClockOffset } let expiries = lock.withLock { shownExpiries } let generator = MemcachedStatementGenerator( diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift index 05bfefbd69..88935569d4 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBPluginDriver.swift @@ -35,7 +35,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se func quoteIdentifier(_ name: String) -> String { name } var capabilities: PluginCapabilities { - [.cancelQuery, .statementPolicy] + [.cancelQuery, .statementPolicy, .columnDefaultMarks] } func defaultExportQuery(table: String) -> String? { diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift index e3ff61ce49..574e17e26f 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift @@ -40,8 +40,6 @@ nonisolated struct MongoDBStatementGenerator { /// connection. var capabilities: () -> MongoDBCapabilities = { .unknown } - private static let defaultMarker = "__DEFAULT__" - /// Where a value written whole comes from. A cell carries only text or bytes, so this is all the /// generator knows about the type the value has to keep. private enum Provenance { @@ -133,7 +131,7 @@ nonisolated struct MongoDBStatementGenerator { let absent = change.absentColumns ?? [] let written = cells.filter { cell in let serverAssignsIdentity = cell.field == MongoDBCollectionDDL.idField && cell.value.isNull - return !serverAssignsIdentity && !isLeftOut(cell.value, isAbsent: absent.contains(cell.column)) + return !serverAssignsIdentity && !absent.contains(cell.column) && !change.isColumnDefault(cell.column) } let entries = try written.map { cell in let provenance: Provenance = typed[cell.field] == cell.value ? .typedIntoNewDocument : .copiedIntoNewDocument @@ -142,10 +140,6 @@ nonisolated struct MongoDBStatementGenerator { return "\(collectionAccessor).insertOne({\(entries.joined(separator: ", "))})" } - private func isLeftOut(_ value: PluginCellValue, isAbsent: Bool) -> Bool { - isAbsent || value.asText == Self.defaultMarker - } - /// A value of a whole document the grid writes: a new row, a duplicate or a paste, or a /// deleted document put back. An `_id` the row holds is kept, typed the way the filters type it; /// the grid leaves a new row's `_id` as DEFAULT, so the server generates it. @@ -187,7 +181,7 @@ nonisolated struct MongoDBStatementGenerator { let idField = MongoDBCollectionDDL.idField var entries = ["\(quotedKey(idField)): \(try Self.identityJson(locator: locator))"] for (index, value) in row.enumerated() where index != idIndex && index < columns.count { - guard !isLeftOut(value, isAbsent: absent.contains(index)) else { continue } + guard !absent.contains(index) else { continue } let field = columns[index] entries.append("\(quotedKey(field)): \(try documentValueJson(value, field: field, provenance: .restored))") } @@ -226,7 +220,7 @@ nonisolated struct MongoDBStatementGenerator { let cellWrites = try change.cellChanges.map { try cellWrite( field: $0.columnName, from: $0.oldValue, to: $0.newValue, - removesField: removed.contains($0.columnIndex) + removesField: removed.contains($0.columnIndex), isDefault: change.isColumnDefault($0.columnIndex) ) } let update = try updateDocument(for: cellWrites) @@ -237,17 +231,18 @@ nonisolated struct MongoDBStatementGenerator { field: String, from oldValue: PluginCellValue, to newValue: PluginCellValue, - removesField: Bool + removesField: Bool, + isDefault: Bool ) throws -> CellWrite { guard field != MongoDBCollectionDDL.idField else { throw MongoDBWriteRefusal.identityChanged } guard !removesField else { return .remove(field: field) } + guard !isDefault else { throw MongoDBWriteRefusal.noDefaultValue(field: field) } switch newValue { case .null: return .whole(field: field, json: "null") case .bytes(let data): return .whole(field: field, json: try binaryJson(data, field: field, provenance: .edit(replacing: oldValue))) case .text(let text): - guard text != Self.defaultMarker else { throw MongoDBWriteRefusal.noDefaultValue(field: field) } if let nested = try nestedEdit(of: field, from: oldValue, to: text) { return nested } diff --git a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift index 9500f5c6cd..f71b06fe00 100644 --- a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift +++ b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift @@ -215,6 +215,7 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { .cellExpressions, .uuidCellText, .userManagement, + .columnDefaultMarks, ] } diff --git a/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift b/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift index 13d859d092..5d9984ebe2 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift @@ -20,16 +20,21 @@ import TableProSQLQuoting /// because a NULL bind into an object-type column fails with ORA-00932; and text for an NCHAR, NVARCHAR2 or NCLOB /// column is `UNISTR`, because a literal or bind passes through the database character set (``OracleNationalText``). internal struct OracleRowWriter { - /// What the grid stages for a column the user leaves to the server's default. It is a marker, never a value. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - /// A keyed delete carries at most this many key values. One statement with 32,767 binds took 30 seconds on 23ai /// and one with 1,000 took 0.07, and a statement that outlives the query timeout closes the session. static let maxValuesPerDelete = 1_000 static let dateMask = "YYYY-MM-DD HH24:MI:SS" static let timestampMask = "YYYY-MM-DD HH24:MI:SS.FF" - static let timestampWithTimeZoneMask = "YYYY-MM-DD HH24:MI:SS.FFTZH:TZM" + // A cell reads `….ffffff+02:00`, the offset straight after the fraction, or `….ffffff Europe/Paris`, a region after + // a blank. TZH:TZM reads the first as written and TZR the second. + static let timestampWithOffsetMask = "YYYY-MM-DD HH24:MI:SS.FFTZH:TZM" + static let timestampWithTimeZoneMask = "YYYY-MM-DD HH24:MI:SS.FF TZR" + + static func timestampWithTimeZoneMask(for text: String?) -> String { + let zone = text?.split(separator: " ").last.flatMap(\.first) + return zone?.isLetter == true ? timestampWithTimeZoneMask : timestampWithOffsetMask + } let qualifiedTable: String let columns: [String] @@ -56,7 +61,8 @@ internal struct OracleRowWriter { let write = try insert( values: values, rowIndex: change.rowIndex, - expressionColumns: change.expressionColumns ?? [] + expressionColumns: change.expressionColumns ?? [], + defaultColumns: change.defaultColumns ?? [] ) else { continue } try flushDeletes() writes.append(write) @@ -80,7 +86,8 @@ internal struct OracleRowWriter { private func insert( values: [PluginCellValue], rowIndex: Int, - expressionColumns: Set + expressionColumns: Set, + defaultColumns: Set ) throws -> PluginRowWrite? { guard let firstColumn = columns.first else { return nil } var names: [String] = [] @@ -89,7 +96,7 @@ internal struct OracleRowWriter { for (index, (column, value)) in zip(columns, values).enumerated() where !context.serverOwnedColumns.contains(column) { names.append(Self.quote(column)) - if value == Self.defaultMarker { + if defaultColumns.contains(index) { placeholders.append("DEFAULT") } else if let expression = Self.expression(value, isMarked: expressionColumns.contains(index)) { placeholders.append(expression) @@ -117,7 +124,7 @@ internal struct OracleRowWriter { var parameters: [PluginCellValue] = [] let assignments = try change.cellChanges.map { cell -> String in let column = Self.quote(cell.columnName) - guard cell.newValue != Self.defaultMarker else { return "\(column) = DEFAULT" } + guard !change.isColumnDefault(cell.columnIndex) else { return "\(column) = DEFAULT" } if let expression = Self.expression(cell.newValue, isMarked: expressionColumns.contains(cell.columnIndex)) { return "\(column) = \(expression)" } @@ -261,10 +268,10 @@ internal struct OracleRowWriter { case .text(let text): if let literal = kind.inlineSQL(for: text) { return literal } if !kind.isNumeric, let quoted = OracleBindPlaceholders.quotedLiteral(text) { - return kind.converting(quoted) + return kind.converting(quoted, text: text) } parameters.append(value) - return kind.converting("?") + return kind.converting("?", text: text) } } @@ -352,14 +359,14 @@ internal enum OracleValueKind: Equatable { /// `TO_TIMESTAMP` rejects (ORA-01830). `TO_TIMESTAMP_TZ` takes it, and text without an offset in the session's /// zone, so the value round-trips through SET and a keyless match whatever the session and database zones are /// (measured on 23ai with the session at `+07:00` and at `America/New_York` against a `+00:00` database). - func converting(_ operand: String) -> String { + func converting(_ operand: String, text: String? = nil) -> String { switch self { case .date: return "TO_DATE(\(operand), '\(OracleRowWriter.dateMask)')" case .timestamp: return "TO_TIMESTAMP(\(operand), '\(OracleRowWriter.timestampMask)')" case .timestampWithTimeZone, .timestampWithLocalTimeZone: - return "TO_TIMESTAMP_TZ(\(operand), '\(OracleRowWriter.timestampWithTimeZoneMask)')" + return "TO_TIMESTAMP_TZ(\(operand), '\(OracleRowWriter.timestampWithTimeZoneMask(for: text))')" case .number, .binaryFloat, .binaryDouble, .bfile, .nationalText, .nationalLOB, .other: return operand } diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift index ca822ad6e3..b8f4ad2a45 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver+Metadata.swift @@ -184,11 +184,16 @@ extension QdrantPluginDriver { switch change.type { case .insert: guard insertedRowIndices.contains(change.rowIndex) else { return nil } + // A cell left to the default has no text, so the point leaves it out and a defaulted id gets a new one. let cells: [QdrantCell] if let row = insertedRowData[change.rowIndex] { - cells = zip(columns, row).map { QdrantCell(column: $0, text: $1.asText) } + cells = zip(columns, row).enumerated().map { index, cell in + QdrantCell(column: cell.0, text: change.isColumnDefault(index) ? nil : cell.1.asText) + } } else { - cells = change.cellChanges.map { QdrantCell(column: $0.columnName, text: $0.newValue.asText) } + cells = change.cellChanges.map { + QdrantCell(column: $0.columnName, text: change.isColumnDefault($0.columnIndex) ? nil : $0.newValue.asText) + } } return QdrantRowChange(kind: .insert, rowIndex: change.rowIndex, id: nil, cells: cells) case .update: diff --git a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift index fe11918a98..90ea656972 100644 --- a/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift +++ b/apple/Plugins/QdrantDriverPlugin/QdrantPluginDriver.swift @@ -21,7 +21,7 @@ internal final class QdrantPluginDriver: PluginDatabaseDriver, @unchecked Sendab var supportsTransactions: Bool { false } - var capabilities: PluginCapabilities { [.cancelQuery] } + var capabilities: PluginCapabilities { [.cancelQuery, .columnDefaultMarks] } var parameterStyle: ParameterStyle { .questionMark } diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift index 4d4ffcb41b..75ff9f44ec 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift @@ -63,7 +63,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } var capabilities: PluginCapabilities { - var supported: PluginCapabilities = [.truncateTable, .cancelQuery] + var supported: PluginCapabilities = [.truncateTable, .cancelQuery, .columnDefaultMarks] if redisConnection?.supportsTransactions ?? true { supported.insert(.transactions) } return supported } @@ -657,6 +657,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let partitioned = redisConnection?.partitionsKeyspace == true let generator = RedisStatementGenerator( namespaceName: table, diff --git a/apple/Plugins/RedisDriverTests/RedisColumnDefaultTests.swift b/apple/Plugins/RedisDriverTests/RedisColumnDefaultTests.swift new file mode 100644 index 0000000000..853eff5f75 --- /dev/null +++ b/apple/Plugins/RedisDriverTests/RedisColumnDefaultTests.swift @@ -0,0 +1,50 @@ +// +// RedisColumnDefaultTests.swift +// RedisDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct RedisColumnDefaultTests { + private let driver = RedisPluginDriver(config: DriverConnectionConfig( + host: "127.0.0.1", port: 6_379, username: "", password: "", database: "0" + )) + + private func valueEdit(to value: PluginCellValue, isDefault: Bool) -> PluginRowChange { + var change = PluginRowChange( + rowIndex: 0, + type: .update, + cellChanges: [(columnIndex: 1, columnName: "Value", oldValue: "old", newValue: value)], + originalRow: ["session:abc", "old", "-1"] + ) + if isDefault { change.defaultColumns = [1] } + return change + } + + private func writes(_ change: PluginRowChange) throws -> [String] { + try driver.generateRowWrites( + table: "", schema: nil, columns: ["Key", "Value", "TTL"], primaryKeyColumns: ["Key"], + changes: [change], insertedRowData: [:], deletedRowIndices: [], insertedRowIndices: [] + )?.map(\.statement) ?? [] + } + + @Test("The driver declares that it reads the default mark") + func declaresTheMark() { + #expect(driver.capabilities.contains(.columnDefaultMarks)) + } + + @Test("The text __DEFAULT__ is written as the value") + func defaultLookingTextIsAValue() throws { + let statements = try writes(valueEdit(to: "__DEFAULT__", isDefault: false)) + #expect(statements.contains { $0.contains("__DEFAULT__") }) + } + + @Test("A cell left to a default is refused, since Redis has none") + func markedDefaultIsRefused() { + #expect(throws: PluginRowWriteRefusal.self) { + _ = try writes(valueEdit(to: "__DEFAULT__", isDefault: true)) + } + } +} diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift index d4d5ca8870..13c8010726 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift @@ -43,6 +43,7 @@ final class SnowflakePluginDriver: PluginDatabaseDriver, @unchecked Sendable { .dataCompare, .rowWrites, .cellExpressions, + .columnDefaultMarks, ] } diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeStatementGenerator.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeStatementGenerator.swift index 5eac2b6580..1d72ede5f9 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeStatementGenerator.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeStatementGenerator.swift @@ -87,9 +87,6 @@ struct SnowflakeStatementGenerator { ) } - /// What the grid stages for a column the user leaves to the server's default. It is a marker, never a value. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - static func isSemiStructured(_ typeName: String) -> Bool { let base = typeName.uppercased().components(separatedBy: "(")[0].trimmingCharacters(in: .whitespaces) return ["VARIANT", "OBJECT", "ARRAY"].contains(base) @@ -100,18 +97,22 @@ struct SnowflakeStatementGenerator { insertedRowData: [Int: [PluginCellValue]] ) -> (statement: String, parameters: [PluginCellValue])? { var values: [(column: String, value: PluginCellValue)] = [] + var hasCells = false if let rowData = insertedRowData[change.rowIndex] { for (index, column) in columns.enumerated() where index < rowData.count { + hasCells = true + guard !change.isColumnDefault(index) else { continue } values.append((column, rowData[index])) } } else { for cellChange in change.cellChanges { + hasCells = true + guard !change.isColumnDefault(cellChange.columnIndex) else { continue } values.append((cellChange.columnName, cellChange.newValue)) } } - guard !values.isEmpty else { return nil } + guard hasCells else { return nil } let expressions = Set((change.expressionColumns ?? []).compactMap { columns.indices.contains($0) ? columns[$0] : nil }) - values.removeAll { $0.value == Self.defaultMarker } guard !values.isEmpty else { guard let firstColumn = columns.first else { return nil } return ("INSERT INTO \(qualifiedTable) (\(quoteIdentifier(firstColumn))) VALUES (DEFAULT)", []) diff --git a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Editing.swift b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Editing.swift index dc205a2a6a..5352ec4480 100644 --- a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Editing.swift +++ b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver+Editing.swift @@ -188,12 +188,13 @@ extension SpannerPluginDriver { SpannerRowChange.CellChange(column: $0.columnName, value: spannerCell($0.newValue)) }, originalRow: change.originalRow?.map(spannerCell), - expressionColumns: expressionColumnNames(of: change, columns: columns) + expressionColumns: columnNames(change.expressionColumns, of: change, columns: columns), + defaultColumns: columnNames(change.defaultColumns, of: change, columns: columns) ) } - private static func expressionColumnNames(of change: PluginRowChange, columns: [String]) -> Set { - guard let marked = change.expressionColumns, !marked.isEmpty else { return [] } + private static func columnNames(_ marked: Set?, of change: PluginRowChange, columns: [String]) -> Set { + guard let marked, !marked.isEmpty else { return [] } switch change.type { case .update: return Set(change.cellChanges.filter { marked.contains($0.columnIndex) }.map(\.columnName)) diff --git a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift index cd645015f0..60f81f66fd 100644 --- a/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift +++ b/apple/Plugins/SpannerDriverPlugin/SpannerPluginDriver.swift @@ -37,7 +37,10 @@ internal final class SpannerPluginDriver: PluginDatabaseDriver, @unchecked Senda } var capabilities: PluginCapabilities { - [.multiSchema, .cancelQuery, .transactions, .truncateTable, .cellExpressions, .uuidCellText, .rowWrites] + [ + .multiSchema, .cancelQuery, .transactions, .truncateTable, .cellExpressions, .uuidCellText, .rowWrites, + .columnDefaultMarks, + ] } var serverVersion: String? { diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift index 1edab23077..fb2c57d7a3 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift @@ -38,7 +38,7 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } var capabilities: PluginCapabilities { - [.parameterizedQueries, .cancelQuery, .truncateTable, .multiSchema] + [.parameterizedQueries, .cancelQuery, .truncateTable, .multiSchema, .columnDefaultMarks] } var serverVersion: String? { diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift index c98626d2bd..4fe2be6314 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift @@ -8,13 +8,6 @@ import TableProNumberFormatting import TableProPluginKit public enum SurrealStatementGenerator { - static let autoIdMarker = "__DEFAULT__" - - static func isAutoDefault(_ value: PluginCellValue) -> Bool { - guard case let .text(text) = value else { return false } - return text.trimmingCharacters(in: .whitespaces) == autoIdMarker - } - public static func rowWrites( table: String, scope: SurrealScope, @@ -37,10 +30,15 @@ public enum SurrealStatementGenerator { writes.append(write) } + var defaults: [Int: Set] = [:] + for change in changes where change.type == .insert { + defaults[change.rowIndex, default: []].formUnion(change.defaultColumns ?? []) + } for index in insertedRowIndices.sorted() { guard let values = insertedRowData[index] else { continue } writes.append(try insert( - table: table, scope: scope, columns: columns, typing: typing, values: values, rowIndex: index + table: table, scope: scope, columns: columns, typing: typing, values: values, + defaults: defaults[index] ?? [], rowIndex: index )) } @@ -96,7 +94,7 @@ public enum SurrealStatementGenerator { } let recordKey = Self.recordText(columns: columns, originalRow: change.originalRow) guard let record = recordKey.flatMap({ SurrealQL.parseRecordId($0, fallbackTable: table) }) else { return nil } - let editable = change.cellChanges.filter { !Self.isAutoDefault($0.newValue) } + let editable = change.cellChanges.filter { !change.isColumnDefault($0.columnIndex) } guard !editable.isEmpty else { return nil } var parameters: [PluginCellValue] = [SurrealCellCoder.parameter(.recordId(record))] @@ -126,6 +124,7 @@ public enum SurrealStatementGenerator { columns: [String], typing: FieldTyping, values: [PluginCellValue], + defaults: Set, rowIndex: Int ) throws(PluginRowWriteRefusal) -> PluginRowWrite { var parameters: [PluginCellValue] = [] @@ -133,13 +132,13 @@ public enum SurrealStatementGenerator { var target = SurrealQL.quoteIdentifier(table) for (index, column) in columns.enumerated() { - guard index < values.count else { continue } + guard index < values.count, !defaults.contains(index) else { continue } let cell = values[index] if column == SurrealInfoParser.recordIdColumn { guard case let .text(text) = cell else { continue } let trimmed = text.trimmingCharacters(in: .whitespaces) - guard !trimmed.isEmpty, trimmed != Self.autoIdMarker else { continue } + guard !trimmed.isEmpty else { continue } guard let record = SurrealQL.parseRecordId(text, fallbackTable: table) else { continue } parameters.append(SurrealCellCoder.parameter(.recordId(record))) target = "$p\(parameters.count - 1)" @@ -147,7 +146,6 @@ public enum SurrealStatementGenerator { } if case .null = cell { continue } - if Self.isAutoDefault(cell) { continue } try refuseShortened(cell, in: column, rowIndex: rowIndex) let value = typing.value(of: cell, in: column) parameters.append(SurrealCellCoder.parameter(value)) diff --git a/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift b/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift index 702eaa1e56..1153452bb7 100644 --- a/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift +++ b/apple/Plugins/TableProPluginKit/PluginBoundedStream.swift @@ -31,6 +31,7 @@ public enum PluginBoundedStream { var rows: [PluginRow] = [] rows.reserveCapacity(min(cap, 10_000)) var truncated = false + var declaredLimit: Int? /// The first batch to arrive is the first row the server produced, which is the only part /// of a streamed read that separates the query's own cost from the cost of moving its rows. var firstRowTime: TimeInterval? @@ -41,6 +42,7 @@ public enum PluginBoundedStream { columns = header.columns columnTypeNames = header.columnTypeNames columnOrigins = header.columnOrigins + declaredLimit = header.rowLimit case .rows(let batch): if firstRowTime == nil, !batch.isEmpty { firstRowTime = Date().timeIntervalSince(startedAt) @@ -55,6 +57,10 @@ public enum PluginBoundedStream { } if truncated { break } } + // A stream that ended at the limit its header declared was cut there, below the cap. + if let declaredLimit, rows.count >= declaredLimit { + truncated = true + } var result = PluginQueryResult( columns: columns, diff --git a/apple/Plugins/TableProPluginKit/PluginCapabilities.swift b/apple/Plugins/TableProPluginKit/PluginCapabilities.swift index 59872dfaef..67ed3dcbf9 100644 --- a/apple/Plugins/TableProPluginKit/PluginCapabilities.swift +++ b/apple/Plugins/TableProPluginKit/PluginCapabilities.swift @@ -41,4 +41,7 @@ public struct PluginCapabilities: OptionSet, Sendable { // The driver refuses what `PluginStatementContext.allowsDestructive` and `allowsServerCode` forbid, for a language // the host cannot read in full. public static let statementPolicy = PluginCapabilities(rawValue: 1 << 22) + // The driver writes the column default only where `PluginRowChange.defaultColumns` marks one, so the text + // `__DEFAULT__` in any other cell is saved as text. + public static let columnDefaultMarks = PluginCapabilities(rawValue: 1 << 23) } diff --git a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index 53d2af6929..0810eb1d63 100644 --- a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -70,6 +70,11 @@ public struct PluginRowChange: Sendable { /// that declares `PluginCapabilities.cellExpressions` is handed any. public var expressionColumns: Set? + // The columns whose new value is the column's default rather than a value: an update's `cellChanges`, or a new + // row's `insertedRowData` cells. Such a cell still holds the text `__DEFAULT__`, which a driver built before this + // field reads as the default. Every other cell is a value, that text included. + public var defaultColumns: Set? + public init( rowIndex: Int, type: ChangeType, @@ -81,6 +86,10 @@ public struct PluginRowChange: Sendable { self.cellChanges = cellChanges self.originalRow = originalRow } + + public func isColumnDefault(_ columnIndex: Int) -> Bool { + defaultColumns?.contains(columnIndex) ?? false + } } public protocol PluginDatabaseDriver: AnyObject, Sendable { @@ -1476,7 +1485,7 @@ public extension PluginDatabaseDriver { parameters: [PluginCellValue]?, context: PluginStatementContext ) async throws -> PluginQueryResult { - try await PluginParameterBinder.$hostFeatures.withValue(context.lexicalFeatures) { + try await PluginParameterBinder.withHostContext(context) { try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) } } @@ -1486,7 +1495,7 @@ public extension PluginDatabaseDriver { parameters: [PluginCellValue], context: PluginStatementContext ) async throws -> PluginQueryResult { - try await PluginParameterBinder.$hostFeatures.withValue(context.lexicalFeatures) { + try await PluginParameterBinder.withHostContext(context) { try await executeParameterized(query: query, parameters: parameters) } } @@ -1540,9 +1549,12 @@ public extension PluginDatabaseDriver { guard !parameters.isEmpty else { return try await execute(query: query) } + let kinds = PluginParameterBinder.hostParameterKinds let sql = try PluginParameterBinder.bind( query, - literals: parameters.map(sqlLiteral(for:)), + literals: parameters.enumerated().map { index, value in + sqlLiteral(for: value, kind: kinds.flatMap { $0.indices.contains(index) ? $0[index] : nil }) + }, style: parameterStyle, features: literalFeatures ) @@ -1550,11 +1562,17 @@ public extension PluginDatabaseDriver { } func sqlLiteral(for value: PluginCellValue) -> String { + sqlLiteral(for: value, kind: nil) + } + + // A nil kind is a value nobody typed, such as a grid cell, whose text the host wrote: number-shaped text is a number. + func sqlLiteral(for value: PluginCellValue, kind: PluginColumnKind?) -> String { switch value { case .null: return "NULL" case .text(let s): - return escapedParameterValue(s) + guard let kind else { return escapedParameterValue(s) } + return typedParameterValue(s, kind: kind) case .bytes(let data): var hex = "X'" hex.reserveCapacity(2 + data.count * 2 + 1) @@ -1578,6 +1596,23 @@ public extension PluginDatabaseDriver { PluginNumericLiteral.isValid(value) } + // Text the server cannot read as the declared number or boolean is sent as a string, for the server to refuse. + func typedParameterValue(_ value: String, kind: PluginColumnKind) -> String { + switch kind { + case .integer, .decimal: + if PluginSQLLiteral.isNumericLiteral(value, kind: kind) { return value } + case .boolean: + switch PluginSQLLiteral.booleanSynonym(for: value, kind: kind) { + case .isTrue: return "TRUE" + case .isFalse: return "FALSE" + case nil: break + } + case .text, .other: + break + } + return PluginSQLQuoting.stringLiteral(value, escapes: PluginSQLQuoting.Escapes(features: literalFeatures)) + } + func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { let raw: PluginQueryResult if let parameters { diff --git a/apple/Plugins/TableProPluginKit/PluginDriverError.swift b/apple/Plugins/TableProPluginKit/PluginDriverError.swift index 9581028e68..09b1da9da8 100644 --- a/apple/Plugins/TableProPluginKit/PluginDriverError.swift +++ b/apple/Plugins/TableProPluginKit/PluginDriverError.swift @@ -31,3 +31,17 @@ public extension PluginDriverError { return desc } } + +// A statement Stop ended, after which the server rolled back the transaction it ran in. A driver throws it in place of +// the cancellation, so the host can say what the Stop undid. +public struct PluginStopRolledBackTransaction: PluginDriverError { + public let pluginErrorCode: Int? + + public init(code: Int? = nil) { + pluginErrorCode = code + } + + public var pluginErrorMessage: String { + String(localized: "The statement was stopped, and the server rolled back the transaction it ran in. Every change since that transaction began is undone.") + } +} diff --git a/apple/Plugins/TableProPluginKit/PluginParameterBinder.swift b/apple/Plugins/TableProPluginKit/PluginParameterBinder.swift index d2a06245da..1669cfdc11 100644 --- a/apple/Plugins/TableProPluginKit/PluginParameterBinder.swift +++ b/apple/Plugins/TableProPluginKit/PluginParameterBinder.swift @@ -20,6 +20,21 @@ public enum PluginParameterBinder { // The host's reading of the connection, set around a statement it hands to a driver. @TaskLocal static var hostFeatures: SQLLexicalFeatures? + @TaskLocal static var hostParameterKinds: [PluginColumnKind?]? + + // A driver whose context overload forwards to its context-free twin wraps the call in this, so the default binder + // still writes each value the way the host's context says. + public static func withHostContext( + _ context: PluginStatementContext, + operation: () async throws -> T + ) async rethrows -> T { + try await $hostFeatures.withValue(context.lexicalFeatures) { + try await $hostParameterKinds.withValue(context.parameterKinds) { + try await operation() + } + } + } + public static func bind( _ template: String, literals parameters: [String], diff --git a/apple/Plugins/TableProPluginKit/PluginRowWrite.swift b/apple/Plugins/TableProPluginKit/PluginRowWrite.swift index 317bdad935..7e85f6a811 100644 --- a/apple/Plugins/TableProPluginKit/PluginRowWrite.swift +++ b/apple/Plugins/TableProPluginKit/PluginRowWrite.swift @@ -76,4 +76,13 @@ public struct PluginRowWriteRefusal: Error, LocalizedError, Sendable, Equatable } public var errorDescription: String? { reason } + + // For a driver whose engine has no column defaults: the first change that leaves a cell to one. + public static func columnDefaultsUnsupported(in changes: [PluginRowChange]) -> PluginRowWriteRefusal? { + guard let change = changes.first(where: { !($0.defaultColumns ?? []).isEmpty }) else { return nil } + return PluginRowWriteRefusal( + rowIndex: change.rowIndex, + reason: String(localized: "This database has no column defaults, so a cell cannot be left to one. Enter a value.") + ) + } } diff --git a/apple/Plugins/TableProPluginKit/PluginStatementContext.swift b/apple/Plugins/TableProPluginKit/PluginStatementContext.swift index 2af809f03b..ef39c06a8c 100644 --- a/apple/Plugins/TableProPluginKit/PluginStatementContext.swift +++ b/apple/Plugins/TableProPluginKit/PluginStatementContext.swift @@ -31,6 +31,10 @@ public struct PluginStatementContext: Sendable, Equatable { // that does not say. public var lexicalFeatures: SQLLexicalFeatures? + // The type the user gave each parameter, by position: `.text` and `.other` are written as strings, `.integer` and + // `.decimal` as numbers, `.boolean` as TRUE or FALSE. Nil, or a nil entry, where none was given. + public var parameterKinds: [PluginColumnKind?]? + public init() {} } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift index ea42a44623..7d851354e3 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift @@ -1,4 +1,5 @@ import Foundation +import TableProPluginKit import TableProTeradataCore final class TeradataAsyncConnection: @unchecked Sendable { @@ -30,6 +31,19 @@ final class TeradataAsyncConnection: @unchecked Sendable { try await run { try $0.execute(sql) } } + func execute(_ sql: String, rowCap: Int?) async throws -> TeradataCappedResult { + try await run { try $0.execute(sql, rowCap: rowCap) } + } + + // One queue item for the whole answer: the session holds one request, so no other call may run between its replies. + func stream( + _ sql: String, + onColumns: @escaping @Sendable ([TeradataColumn]) throws -> Void, + onRows: @escaping @Sendable ([[TeradataValue]]) throws -> Bool + ) async throws { + try await run { try $0.stream(sql, onColumns: onColumns, onRows: onRows) } + } + func disconnect() { queue.sync { connection.disconnect() } } @@ -55,6 +69,9 @@ final class TeradataAsyncConnection: @unchecked Sendable { let value = try body(self.connection) try cancellation.throwIfCancelled() continuation.resume(returning: value) + } catch TeradataWireError.rolledBackByStop { + // Kept past the cancel, so the host can say what the Stop undid. + continuation.resume(throwing: PluginStopRolledBackTransaction(code: TeradataConnection.userAbortCode)) } catch { continuation.resume(throwing: cancellation.isCancelled ? CancellationError() : error) } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift index 6790d03b9d..c9dd03b96e 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift @@ -15,15 +15,19 @@ extension TeradataValue { } } +extension TeradataColumn { + var pluginTypeName: String { TeradataColumnType.wireTypeName(typeCode) } +} + extension TeradataResultSet { - func toPluginResult(executionTime: TimeInterval) -> PluginQueryResult { + func toPluginResult(executionTime: TimeInterval, isTruncated: Bool = false) -> PluginQueryResult { PluginQueryResult( columns: columns.map { $0.name }, - columnTypeNames: columns.map { TeradataColumnType.wireTypeName($0.typeCode) }, + columnTypeNames: columns.map { $0.pluginTypeName }, rows: rows.map { row in row.map { $0.asPluginCell } }, rowsAffected: activityCount, executionTime: executionTime, - isTruncated: false + isTruncated: isTruncated ) } } @@ -136,7 +140,10 @@ final class TeradataPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var supportsTransactions: Bool { true } var capabilities: PluginCapabilities { - [.cancelQuery, .transactions, .alterTableDDL, .dataCompare, .cellExpressions, .uuidCellText, .rowWrites] + [ + .cancelQuery, .transactions, .alterTableDDL, .dataCompare, .cellExpressions, .uuidCellText, .rowWrites, + .columnDefaultMarks, + ] } init(config: DriverConnectionConfig) { @@ -203,27 +210,72 @@ final class TeradataPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func execute(query: String) async throws -> PluginQueryResult { - guard let connection else { throw TeradataWireError.connectionFailed("not connected") } - let start = Date() - let result = try await connection.execute(query) - return result.toPluginResult(executionTime: Date().timeIntervalSince(start)) + try await execute(query, rowCap: nil) } func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { - guard let parameters else { return try await execute(query: query) } - return try await executeParameterized(query: query, parameters: parameters) + try await execute(bound(query, parameters ?? []), rowCap: rowCap) + } + + // Not the stream: a Stop cancels the reading task, which ends a stream before a rollback error can cross it. + func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { + try await execute(query, rowCap: rowCap) } - // Teradata reads quotes, names and comments the ANSI way, which is no lexical feature. func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { - guard !parameters.isEmpty else { return try await execute(query: query) } - let bound = try PluginParameterBinder.bind( + try await execute(bound(query, parameters), rowCap: nil) + } + + func streamRows(query: String) -> AsyncThrowingStream { + let connection = self.connection + return PluginRowStream.make { continuation, abort in + let task = Task { + guard let connection else { + continuation.finish(throwing: TeradataWireError.connectionFailed("not connected")) + return + } + do { + // Each reply's rows go out as it arrives, before the next one is asked for. + try await connection.stream( + query, + onColumns: { columns in + continuation.yield(.header(PluginStreamHeader( + columns: columns.map { $0.name }, + columnTypeNames: columns.map { $0.pluginTypeName } + ))) + }, + onRows: { rows in + guard !abort.isAborted else { return false } + continuation.yield(.rows(rows.map { row in row.map { $0.asPluginCell } })) + return true + } + ) + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + abort.onAbort { task.cancel() } + } + } + + // Teradata reads quotes, names and comments the ANSI way, which is no lexical feature. + private func bound(_ query: String, _ parameters: [PluginCellValue]) throws -> String { + guard !parameters.isEmpty else { return query } + return try PluginParameterBinder.bind( query, literals: parameters.map(TeradataRowEditSQL.literal), style: .questionMark, features: [] ) - return try await execute(query: bound) + } + + private func execute(_ query: String, rowCap: Int?) async throws -> PluginQueryResult { + guard let connection else { throw TeradataWireError.connectionFailed("not connected") } + let start = Date() + let capped = try await connection.execute(query, rowCap: rowCap) + return capped.resultSet.toPluginResult( + executionTime: Date().timeIntervalSince(start), isTruncated: capped.isTruncated) } func cancelQuery() throws { diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift index e9c842a0f7..a60e747208 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Editing.swift @@ -36,7 +36,8 @@ extension TeradataPluginDriver { target: target, columns: columns, values: values, - expressionColumns: change.expressionColumns ?? [] + expressionColumns: change.expressionColumns ?? [], + defaultColumns: change.defaultColumns ?? [] ) statements.append((insert, [])) case .update: @@ -45,7 +46,8 @@ extension TeradataPluginDriver { TeradataRowEditSQL.assignment( column: $0.columnName, value: $0.newValue, - isExpression: expressionColumns.contains($0.columnIndex) + isExpression: expressionColumns.contains($0.columnIndex), + isDefault: change.isColumnDefault($0.columnIndex) ) } guard !assignments.isEmpty, diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataRowEditSQL.swift b/apple/Plugins/TeradataDriverPlugin/TeradataRowEditSQL.swift index 7c1b7420ad..1ff5996b2d 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataRowEditSQL.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataRowEditSQL.swift @@ -11,9 +11,6 @@ import TableProTeradataCore /// Teradata literals and the INSERT a new grid row becomes. The driver sends statements as text, so every value is /// written as a literal here rather than bound. enum TeradataRowEditSQL { - /// What the grid stages for a column the user leaves to the server's default. It is a marker, never a value. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - static func literal(_ value: PluginCellValue) -> String { switch value { case .null: @@ -29,9 +26,10 @@ enum TeradataRowEditSQL { target: String, columns: [String], values: [PluginCellValue], - expressionColumns: Set = [] + expressionColumns: Set = [], + defaultColumns: Set = [] ) -> String { - let written = zip(columns, values).enumerated().filter { $0.element.1 != defaultMarker } + let written = zip(columns, values).enumerated().filter { !defaultColumns.contains($0.offset) } guard !written.isEmpty else { return "INSERT INTO \(target) DEFAULT VALUES" } let columnList = written.map { TeradataSchemaQueries.quoteIdentifier($0.element.0) }.joined(separator: ", ") let valueList = written.map { cell in @@ -40,9 +38,9 @@ enum TeradataRowEditSQL { return "INSERT INTO \(target) (\(columnList)) VALUES (\(valueList))" } - static func assignment(column: String, value: PluginCellValue, isExpression: Bool) -> String { + static func assignment(column: String, value: PluginCellValue, isExpression: Bool, isDefault: Bool = false) -> String { let name = TeradataSchemaQueries.quoteIdentifier(column) - guard value != defaultMarker else { return "\(name) = DEFAULT" } + guard !isDefault else { return "\(name) = DEFAULT" } return "\(name) = \(Self.value(value, isExpression: isExpression))" } diff --git a/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver.swift b/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver.swift index 1cdc981634..86711e30c1 100644 --- a/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver.swift +++ b/apple/Plugins/TrinoDriverPlugin/TrinoPluginDriver.swift @@ -27,7 +27,10 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } var capabilities: PluginCapabilities { - [.multiSchema, .cancelQuery, .materializedViews, .dataCompare, .rowWrites, .cellExpressions, .uuidCellText] + [ + .multiSchema, .cancelQuery, .materializedViews, .dataCompare, .rowWrites, .cellExpressions, .uuidCellText, + .columnDefaultMarks, + ] } func cacheColumnTypes(_ types: [String: String], key: String) { @@ -129,8 +132,10 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { parameters: [PluginCellValue]?, context: PluginStatementContext ) async throws -> PluginQueryResult { - try await Self.$hostSplit.withValue(context.hostSplit) { - try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) + try await PluginParameterBinder.withHostContext(context) { + try await Self.$hostSplit.withValue(context.hostSplit) { + try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters) + } } } @@ -139,8 +144,10 @@ final class TrinoPluginDriver: PluginDatabaseDriver, @unchecked Sendable { parameters: [PluginCellValue], context: PluginStatementContext ) async throws -> PluginQueryResult { - try await Self.$hostSplit.withValue(context.hostSplit) { - try await executeParameterized(query: query, parameters: parameters) + try await PluginParameterBinder.withHostContext(context) { + try await Self.$hostSplit.withValue(context.hostSplit) { + try await executeParameterized(query: query, parameters: parameters) + } } } diff --git a/apple/Plugins/TrinoDriverPlugin/TrinoRowEditValues.swift b/apple/Plugins/TrinoDriverPlugin/TrinoRowEditValues.swift index 0a4da883cf..e2cf65dcc6 100644 --- a/apple/Plugins/TrinoDriverPlugin/TrinoRowEditValues.swift +++ b/apple/Plugins/TrinoDriverPlugin/TrinoRowEditValues.swift @@ -9,10 +9,7 @@ import TableProTrinoCore /// The column values a grid change writes, in the form `TrinoRowEditSQL` renders. enum TrinoRowEditValues { - /// What the grid stages for a column the user leaves to the server's default. Trino has no `DEFAULT` keyword in - /// `VALUES`, so the column stays out of the insert and the connector fills it. - static let defaultMarker = PluginCellValue.text("__DEFAULT__") - + // Trino has no `DEFAULT` keyword in `VALUES`, so a defaulted column stays out of the insert and the connector fills it. static func insertValues( _ change: PluginRowChange, columns: [String], @@ -21,11 +18,11 @@ enum TrinoRowEditValues { ) -> [TrinoColumnValue] { if let rowData = insertedRowData[change.rowIndex] { return columns.enumerated().compactMap { index, column in - guard index < rowData.count, rowData[index] != defaultMarker else { return nil } + guard index < rowData.count, !change.isColumnDefault(index) else { return nil } return TrinoColumnValue(name: column, value: trinoValue(rowData[index]), typeName: typeName(column)) } } - return change.cellChanges.filter { $0.newValue != defaultMarker }.map { + return change.cellChanges.filter { !change.isColumnDefault($0.columnIndex) }.map { TrinoColumnValue(name: $0.columnName, value: trinoValue($0.newValue), typeName: typeName($0.columnName)) } } diff --git a/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver.swift b/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver.swift index e00f521323..4ed6c0f139 100644 --- a/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver.swift +++ b/apple/Plugins/TypesenseDriverPlugin/TypesensePluginDriver.swift @@ -28,7 +28,7 @@ internal final class TypesensePluginDriver: PluginDatabaseDriver, @unchecked Sen var supportsTransactions: Bool { false } /// `.userManagement` is what enables Database > Users & Roles, which lists Typesense API keys. - var capabilities: PluginCapabilities { [.cancelQuery, .userManagement] } + var capabilities: PluginCapabilities { [.cancelQuery, .userManagement, .columnDefaultMarks] } init(config: DriverConnectionConfig) { self.config = config @@ -237,6 +237,7 @@ internal final class TypesensePluginDriver: PluginDatabaseDriver, @unchecked Sen deletedRowIndices: Set, insertedRowIndices: Set ) throws -> [PluginRowWrite]? { + if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } let fields = lock.withLock { _schemaCache[table] }?.fieldsByName ?? [:] let generator = TypesenseStatementGenerator(collection: table, columns: columns, fields: fields) return try generator.generateRowWrites( diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift index 9c9c7f27ba..f9256485cd 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift @@ -81,7 +81,7 @@ extension QueryExecutionCoordinator { sql: String, index: Int, baseQuery: String?, - baseQueryParameterValues: [String?]? = nil, + baseQueryParameterValues: [TypedParameterValue?]? = nil, namedParameterStatement: NamedParameterStatement? = nil, tabId: UUID, anchor: StatementAnchor? = nil, diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift index c0bcf998c7..7d72575eb4 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift @@ -286,7 +286,10 @@ extension QueryExecutionCoordinator { ) await MainActor.run { [weak self] in guard let self else { return } - guard parent.tabExecution.settle(claim) else { return } + guard parent.tabExecution.settle(claim) else { + parent.reportStopRollback(error, tabId: tabId) + return + } parent.tabManager.mutate(tabId: tabId) { tab in tab.pagination.isLoadingMore = false } @@ -295,6 +298,7 @@ extension QueryExecutionCoordinator { parent.reportEndedExecutions([ EndedExecution(tabId: claim.tabId, startedAt: claim.startedAt, reason: .cancelledByUser) ]) + parent.reportStopRollback(error, tabId: tabId) return } handleQueryExecutionError( @@ -681,7 +685,7 @@ extension QueryExecutionCoordinator { sql: statement.originalSQL, index: firstNumber + index, baseQuery: statement.executableSQL, - baseQueryParameterValues: statement.parameterValues?.map { $0 as? String }, + baseQueryParameterValues: statement.parameterValues?.map { $0 as? TypedParameterValue }, namedParameterStatement: statement.parameterValues == nil ? nil : NamedParameterStatement(sql: statement.originalSQL, parameters: history.parameters), @@ -790,7 +794,7 @@ extension QueryExecutionCoordinator { prepared: prepared ) - let parameterValues = nativeParameters.map { $0 as? String } + let parameterValues = nativeParameters.map { $0 as? TypedParameterValue } let statement = originalSQL.map { NamedParameterStatement(sql: $0, parameters: originalParameters) } parent.tabManager.mutate(tabId: tabId) { $0.pagination.baseQueryParameterValues = parameterValues diff --git a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift index 19517845a3..fb8d7ce52a 100644 --- a/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift +++ b/apple/TablePro/Core/DataWrite/RowChangeStatementFactory.swift @@ -328,23 +328,30 @@ struct RowChangeStatementFactory { return marked } - // A driver reads its own spelling for a default, not the mark, so a default reaches it spelled that way and unmarked. + // A driver built before `PluginRowChange.defaultColumns` reads its own spelling for a default, so a default reaches + // every driver spelled that way, with the mark beside it for one that reads the mark. private static func spellingDefaultsForDrivers( _ changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]] - ) -> (changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]]) { + ) -> (changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]], defaults: [RowID: Set]) { var rows = insertedRowData + var defaults: [RowID: Set] = [:] let spelledChanges = changes.map { change -> RowChange in var spelled = change if change.type == .insert, var values = rows[change.rowID] { for index in change.expressionColumns where values.indices.contains(index) && values[index] == .columnDefault { values[index] = .pluginDefaultMarker spelled.expressionColumns.remove(index) + defaults[change.rowID, default: []].insert(index) } rows[change.rowID] = values } let defaulted = change.cellChanges.filter { $0.newValue.isColumnDefault(isExpression: $0.newIsExpression) } guard !defaulted.isEmpty else { return spelled } + // A new row's stored values are what a driver writes, so only they mark its defaults. + if !(change.type == .insert && insertedRowData[change.rowID] != nil) { + defaults[change.rowID, default: []].formUnion(defaulted.map(\.columnIndex)) + } spelled.cellChanges = change.cellChanges.map { cell in guard cell.newValue.isColumnDefault(isExpression: cell.newIsExpression) else { return cell } return CellChange( @@ -362,7 +369,7 @@ struct RowChangeStatementFactory { } return spelled } - return (spelledChanges, rows) + return (spelledChanges, rows, defaults) } private func rowWriteContext(for keyed: PluginKeyedChanges) -> PluginRowWriteContext { @@ -426,7 +433,9 @@ struct RowChangeStatementFactory { }, context: restoreContext(rowCount: rows.count) ) { - try refuseRestoredTextDriversReadAsDefault(rows, restorable: restorable) + if !pluginDriver.capabilities.contains(.columnDefaultMarks) { + try refuseRestoredTextDriversReadAsDefault(rows, restorable: restorable) + } try refuseUnfilteredWrites(restored.map { $0.statement }) return RestoreStatements(statements: restored.map { ParameterizedStatement(sql: $0.statement, parameters: $0.parameters.map(\.asAny)) @@ -535,7 +544,8 @@ struct RowChangeStatementFactory { changes: spelled.changes, insertedRowData: spelled.insertedRowData, deletedRowIDs: deletedRowIDs, - insertedRowIDs: insertedRowIDs + insertedRowIDs: insertedRowIDs, + defaultColumns: spelled.defaults ) let writes: [PluginRowWrite] do { @@ -566,7 +576,9 @@ struct RowChangeStatementFactory { throw DataWriteError.changeRefused(table: tableName, kind: nil, reason: error.localizedDescription) } - try refuseTextDriversReadAsDefault(in: changes, insertedRowData: insertedRowData) + if !pluginDriver.capabilities.contains(.columnDefaultMarks) { + try refuseTextDriversReadAsDefault(in: changes, insertedRowData: insertedRowData) + } try refuseUnfilteredWrites(writes.map(\.statement)) let unwritten = RowWriteCoverage.unwrittenChanges( changes, @@ -590,8 +602,8 @@ struct RowChangeStatementFactory { } } - // A driver may read the text `__DEFAULT__` as the column default, and none can say whether it does until PluginKit - // carries the mark beside the value, so the text is never handed to one as a value. + // A driver that does not read `PluginRowChange.defaultColumns` may read the text `__DEFAULT__` as the column default, + // so the text is never handed to one as a value. private func refuseTextDriversReadAsDefault( in changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]] @@ -709,7 +721,8 @@ struct PluginKeyedChanges { changes: [RowChange], insertedRowData: [RowID: [PluginCellValue]], deletedRowIDs: Set, - insertedRowIDs: Set + insertedRowIDs: Set, + defaultColumns: [RowID: Set] = [:] ) { let ordered = changes.sorted { $0.sequence < $1.sequence } var keys: [RowID: Int] = [:] @@ -720,7 +733,7 @@ struct PluginKeyedChanges { } self.rowIDs = rowIDs self.changes = ordered.compactMap { change in - keys[change.rowID].map { PluginRowChange(change, key: $0) } + keys[change.rowID].map { PluginRowChange(change, key: $0, defaultColumns: defaultColumns[change.rowID]) } } self.insertedRowData = Dictionary( uniqueKeysWithValues: insertedRowData.compactMap { rowID, values in @@ -773,7 +786,7 @@ private extension SQLUnfilteredWrite { } private extension PluginRowChange { - init(_ change: RowChange, key: Int) { + init(_ change: RowChange, key: Int, defaultColumns: Set?) { self.init( rowIndex: key, type: { @@ -790,6 +803,7 @@ private extension PluginRowChange { ) absentColumns = Self.absentColumns(after: change) expressionColumns = Self.expressionColumns(of: change) + self.defaultColumns = change.type == .delete || defaultColumns?.isEmpty != false ? nil : defaultColumns } static func expressionColumns(of change: RowChange) -> Set? { diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift index 587b6fe793..4b4ec3ab6d 100644 --- a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift @@ -106,6 +106,7 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor static func cellValue(for parameter: Any?) -> PluginCellValue { guard let parameter else { return .null } + if let typed = parameter as? TypedParameterValue { return .text(typed.text) } if let data = parameter as? Data { return .bytes(data) } if let f = parameter as? Float { guard f.isFinite else { return .null } @@ -1439,7 +1440,7 @@ extension PluginDriverAdapter { let pluginResult = try await pluginDriver.executeParameterized( query: query, parameters: parameters.map(Self.cellValue(for:)), - context: statementContext(query, approvedAs: context) + context: statementContext(query, approvedAs: context, parameters: parameters) ) return mapQueryResult(pluginResult) } @@ -1456,14 +1457,15 @@ extension PluginDriverAdapter { query: query, rowCap: rowCap, parameters: parameters?.map(Self.cellValue(for:)), - context: statementContext(query, approvedAs: context) + context: statementContext(query, approvedAs: context, parameters: parameters) ) return mapQueryResult(pluginResult) } - fileprivate func statementContext(_ query: String, approvedAs caller: PluginStatementContext) -> PluginStatementContext { + fileprivate func statementContext(_ query: String, approvedAs caller: PluginStatementContext, parameters: [Any?]? = nil) -> PluginStatementContext { var context = PluginStatementContext.statement(query, databaseType: connection.type, approvedAs: caller) context.lexicalFeatures = sessionLexicalFeatures + context.parameterKinds = TypedParameterValue.kinds(of: parameters) ?? context.parameterKinds return context } diff --git a/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift b/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift index dab2c5e5b4..05251416e1 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLParameterExtractor.swift @@ -55,7 +55,9 @@ enum SQLParameterExtractor { } if let param = paramLookup[name] { - values.append(param.isNull ? nil : param.value) + values.append( + param.isNull ? nil : TypedParameterValue(text: param.value, kind: param.type.pluginKind(for: param.value)) + ) } else { values.append(nil) } diff --git a/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift b/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift index 3c763effc6..acadf5177a 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLRowToStatementConverter.swift @@ -14,8 +14,8 @@ import TableProPluginKit internal struct SQLRowToStatementConverter { /// One row of the grid, as it is shown and as the server has it. /// - /// `defaultedColumns` names the cells the grid staged as left to the server's default. The grid spells that as the - /// text `__DEFAULT__`, which a stored value can also be, so only a cell named here is read as the marker. + /// `defaultedColumns` names the cells the grid staged as left to the server's default. The grid stages that as the + /// keyword DEFAULT marked as SQL, and a stored value can read the same, so only a cell named here is the default. /// `expressionColumns` names the cells staged as SQL, which go out unquoted; text anywhere else is a literal. internal struct SourceRow { internal let values: [PluginCellValue] diff --git a/apple/TablePro/Models/Query/QueryParameter.swift b/apple/TablePro/Models/Query/QueryParameter.swift index d7a7cc4db8..c366ce87b5 100644 --- a/apple/TablePro/Models/Query/QueryParameter.swift +++ b/apple/TablePro/Models/Query/QueryParameter.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProPluginKit enum QueryParameterType: String, Codable, CaseIterable, Sendable { case string @@ -28,3 +29,50 @@ struct QueryParameter: Identifiable, Codable, Equatable, Hashable, Sendable { self.isNull = isNull } } + +extension QueryParameterType { + // String is the panel's default, so a plain number in it still goes as a number (`LIMIT :n`, `id = :id`). Anything + // else is text, a code like `1E5` included; Decimal sends that as a number. + func pluginKind(for value: String) -> PluginColumnKind? { + switch self { + case .string: return Self.isPlainNumber(value) ? nil : .text + case .integer: return .integer + case .decimal: return .decimal + case .boolean: return .boolean + case .date: return .other + } + } + + // Digits with an optional sign and an optional fraction: `42`, `-3.5`, `.5`. + private static func isPlainNumber(_ value: String) -> Bool { + var scalars = value.unicodeScalars[...] + if scalars.first == "-" || scalars.first == "+" { scalars = scalars.dropFirst() } + var sawDot = false + var digitsAfterDot = 0 + var digits = 0 + for scalar in scalars { + if ("0"..."9").contains(scalar) { + digits += 1 + if sawDot { digitsAfterDot += 1 } + } else if scalar == ".", !sawDot { + sawDot = true + } else { + return false + } + } + return digits > 0 && (!sawDot || digitsAfterDot > 0) + } +} + +// A panel value with the type the user gave it, so a driver that writes values into the statement writes it as that. +struct TypedParameterValue: Equatable, Sendable { + let text: String + let kind: PluginColumnKind? + + // Nil when no value carries a type, so a driver reads the values as it did before types were passed. + static func kinds(of parameters: [Any?]?) -> [PluginColumnKind?]? { + guard let parameters else { return nil } + let kinds = parameters.map { ($0 as? TypedParameterValue)?.kind ?? nil } + return kinds.contains { $0 != nil } ? kinds : nil + } +} diff --git a/apple/TablePro/Models/Query/QueryTabState.swift b/apple/TablePro/Models/Query/QueryTabState.swift index 796c43ee88..48b832dd2e 100644 --- a/apple/TablePro/Models/Query/QueryTabState.swift +++ b/apple/TablePro/Models/Query/QueryTabState.swift @@ -288,10 +288,10 @@ struct PaginationState: Equatable { var hasMoreRows: Bool = false var isLoadingMore: Bool = false var baseQueryForMore: String? - var baseQueryParameterValues: [String?]? + var baseQueryParameterValues: [TypedParameterValue?]? // Set together: stale bindings sent with a placeholder-free query made Fetch All fail on the server. - mutating func setBaseQueryForMore(_ sql: String?, parameterValues: [String?]?) { + mutating func setBaseQueryForMore(_ sql: String?, parameterValues: [TypedParameterValue?]?) { baseQueryForMore = sql baseQueryParameterValues = parameterValues } diff --git a/apple/TablePro/Models/Query/ResultSet.swift b/apple/TablePro/Models/Query/ResultSet.swift index 96319f8afa..44e62e8543 100644 --- a/apple/TablePro/Models/Query/ResultSet.swift +++ b/apple/TablePro/Models/Query/ResultSet.swift @@ -39,7 +39,7 @@ final class ResultSet: ObservableObject, Identifiable { @Published var isPinned: Bool = false @Published var isTruncated: Bool = false @Published var baseQuery: String? - @Published var baseQueryParameterValues: [String?]? + @Published var baseQueryParameterValues: [TypedParameterValue?]? /// The statement a parameterized run produced these rows from, as written, with the values it bound. `baseQuery` /// and `baseQueryParameterValues` are the same run in the driver's positional form, which Fetch All sends as it diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index ebbd35aef7..570eb50fdb 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -44054,6 +44054,10 @@ } } }, + "column %@ first appears in a later chunk" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Column %@ has a type, default, generation expression or check that is not one SQL expression." : { "extractionState" : "manual", "generatesSymbol" : false @@ -194590,6 +194594,10 @@ } } }, + "The statement was stopped, and the server rolled back the transaction it ran in. Every change since that transaction began is undone." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The statement's placeholders do not match the %d values given." : { "extractionState" : "manual", "generatesSymbol" : false @@ -198401,6 +198409,10 @@ } } }, + "This database has no column defaults, so a cell cannot be left to one. Enter a value." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This database has no Create Table form" : { "extractionState" : "manual", "generatesSymbol" : false, diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift index 48861aa5be..2e1a8f6b71 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+QueryHelpers.swift @@ -20,6 +20,13 @@ extension MainContentCoordinator { ) } + // Stop ends the run before the driver answers, so a rollback the server did for it arrives on a settled claim. It is + // shown on the tab unless the tab already runs something new. + func reportStopRollback(_ error: Error, tabId: UUID) { + guard error is PluginStopRolledBackTransaction, !tabExecution.isExecuting(tabId) else { return } + tabManager.mutate(tabId: tabId) { $0.execution.errorMessage = error.localizedDescription } + } + /// A table tab's SELECT is the app's own, so it may follow a database switch it waited through /// onto a pooled connection. The choice belongs to the tab and never to the statement: an editor /// SELECT can read a temp table or sit inside the user's open transaction, and moving it to @@ -57,6 +64,7 @@ extension MainContentCoordinator { serverOutput: PluginServerOutput = .none ) { guard tabExecution.settle(claim) else { + reportStopRollback(error, tabId: tabId) traceStaleResultDropped(traceToken) return } @@ -73,6 +81,7 @@ extension MainContentCoordinator { reportEndedExecutions([ EndedExecution(tabId: claim.tabId, startedAt: claim.startedAt, reason: .cancelledByUser) ]) + reportStopRollback(error, tabId: tabId) return } if isAutoLoad, services.databaseManager.driver(for: connectionId)?.status != .connected { diff --git a/apple/TableProMobile/TableProMobile/Drivers/PluginRowWritingDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/PluginRowWritingDriver.swift index 0389ae815c..ab6c89d52a 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/PluginRowWritingDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/PluginRowWritingDriver.swift @@ -79,8 +79,8 @@ nonisolated class PluginRowWritingDriver: PluginBackedDriver, DriverRowWriting, columns: [String], values: [(column: String, value: String?)] ) throws -> [PluginRowWrite] { - // The plugin reads this text as "leave the column out", so storing it as a value cannot be expressed. - if values.contains(where: { $0.value == Self.defaultSentinel }) { + // A plugin that does not read the default mark reads this text as "leave the column out", so it cannot store it. + if !plugin.capabilities.contains(.columnDefaultMarks), values.contains(where: { $0.value == Self.defaultSentinel }) { throw DatabaseError(message: String(format: String( localized: "The text %@ cannot be inserted: this database reads it as the column's default." ), Self.defaultSentinel)) @@ -90,7 +90,9 @@ nonisolated class PluginRowWritingDriver: PluginBackedDriver, DriverRowWriting, guard let value = given[column] else { return .text(Self.defaultSentinel) } return PluginCellValue(optionalText: value) } - let rowChange = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + var rowChange = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + let defaulted = Set(columns.indices.filter { given[columns[$0]] == nil }) + rowChange.defaultColumns = defaulted.isEmpty ? nil : defaulted return try generate( table: table, schema: schema, columns: columns, changes: [rowChange], inserted: [0: row], context: PluginRowWriteContext() @@ -158,7 +160,7 @@ nonisolated class PluginRowWritingDriver: PluginBackedDriver, DriverRowWriting, DatabaseError(message: String(localized: "This database cannot write this change from iPhone or iPad.")) } - // The Mac's grid marks a cell left at the column default with this text, and the drivers leave such a column out. + // What a column left at its default holds, for a plugin built before `PluginRowChange.defaultColumns`. static let defaultSentinel = "__DEFAULT__" private func generate( diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/PluginBackedDriverTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/PluginBackedDriverTests.swift index 910302a3d6..1bbe8b04d4 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/PluginBackedDriverTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/PluginBackedDriverTests.swift @@ -231,6 +231,34 @@ struct PluginBackedDriverTests { ]) } + @Test("A plugin that reads the default mark gets every left-out column marked") + func insertMarksOmittedColumnsForAMarkingPlugin() throws { + let plugin = StubPluginDriver() + plugin.declaredCapabilities = .columnDefaultMarks + let driver = makeWriter(plugin) + + _ = try driver.insertWrites( + table: "events", schema: nil, columns: ["id", "name", "note"], values: [(column: "name", value: "x")] + ) + + #expect(plugin.generatedChanges.first?.defaultColumns == [0, 2]) + } + + @Test("A plugin that reads the default mark stores the marker's text as a value") + func markingPluginStoresTheMarkerText() throws { + let plugin = StubPluginDriver() + plugin.declaredCapabilities = .columnDefaultMarks + let driver = makeWriter(plugin) + + _ = try driver.insertWrites( + table: "events", schema: nil, columns: ["id", "note"], + values: [(column: "id", value: "1"), (column: "note", value: PluginRowWritingDriver.defaultSentinel)] + ) + + #expect(plugin.generatedInserts[0] == ["1", .text(PluginRowWritingDriver.defaultSentinel)]) + #expect(plugin.generatedChanges.first?.defaultColumns == nil) + } + @Test("A value spelled as the default marker is refused, never stored as the default") func insertRefusesTheDefaultMarker() { let plugin = StubPluginDriver() diff --git a/apple/TableProMobile/TableProMobileTests/Mocks/StubPluginDriver.swift b/apple/TableProMobile/TableProMobileTests/Mocks/StubPluginDriver.swift index 376ba32e9c..e5ede52867 100644 --- a/apple/TableProMobile/TableProMobileTests/Mocks/StubPluginDriver.swift +++ b/apple/TableProMobile/TableProMobileTests/Mocks/StubPluginDriver.swift @@ -12,8 +12,11 @@ final class StubPluginDriver: PluginDatabaseDriver, @unchecked Sendable { var checkRows: [Int] = [] var onFetchTables: (@Sendable () async -> Void)? var currentDb = "" + var declaredCapabilities: PluginCapabilities = [] private(set) var databasesAtExecute: [String] = [] + var capabilities: PluginCapabilities { declaredCapabilities } + private(set) var boundedQueries: [(query: String, rowCap: Int)] = [] private(set) var plainQueries: [String] = [] private(set) var parameterizedQueries: [(query: String, parameters: [PluginCellValue])] = [] diff --git a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift index 0109aeb1ec..376fdb75e2 100644 --- a/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift +++ b/apple/TableProTests/Core/DataWrite/RowChangeStatementFactoryExpressionTests.swift @@ -37,6 +37,12 @@ private final class ExpressionRecordingDriver: PluginDriverStub, @unchecked Send } private final class RestoringDriver: PluginDriverStub, @unchecked Sendable { + let capabilities: PluginCapabilities + + init(capabilities: PluginCapabilities = []) { + self.capabilities = capabilities + } + func generateIdentityPreservingInsert( table: String, schema: String?, @@ -286,6 +292,66 @@ struct RowChangeStatementFactoryExpressionTests { #expect(refused?.reason.contains("name") == true) } + @Test("A driver that reads the default mark gets the mark beside its old spelling") + func defaultReachesAMarkingDriverWithItsMark() throws { + let driver = ExpressionRecordingDriver(capabilities: .columnDefaultMarks) + _ = try factory(databaseType: .mssql, driver: driver).statements( + for: [edit(to: .columnDefault, isExpression: true, column: 1)] + ) + #expect(driver.changes.first?.cellChanges.first?.newValue == .pluginDefaultMarker) + #expect(driver.changes.first?.defaultColumns == [1]) + #expect(driver.changes.first?.expressionColumns == nil) + } + + @Test("A driver that reads the default mark saves the text __DEFAULT__ as text") + func textAMarkingDriverReadsIsSaved() throws { + let driver = ExpressionRecordingDriver(capabilities: .columnDefaultMarks) + _ = try factory(databaseType: .mssql, driver: driver).statements( + for: [edit(to: "__DEFAULT__", isExpression: false, column: 1)] + ) + #expect(driver.changes.first?.cellChanges.first?.newValue == "__DEFAULT__") + #expect(driver.changes.first?.defaultColumns == nil) + } + + @Test("A new row's default cells are marked for the driver, and its text cells are not") + func newRowDefaultsAreMarked() throws { + let driver = ExpressionRecordingDriver(capabilities: .columnDefaultMarks) + let rowID = RowID.inserted(UUID()) + let change = RowChange(rowID: rowID, type: .insert, expressionColumns: [0]) + _ = try factory(databaseType: .mssql, driver: driver).statements( + for: [change], + insertedRowData: [rowID: [.columnDefault, "__DEFAULT__", "2026-01-01"]], + insertedRowIDs: [rowID] + ) + #expect(driver.changes.first?.defaultColumns == [0]) + } + + @Test("A new row staged only as cell edits marks its default cells by column") + func newRowCellEditDefaultsAreMarked() throws { + let driver = ExpressionRecordingDriver(capabilities: .columnDefaultMarks) + let change = RowChange( + rowID: .inserted(UUID()), + type: .insert, + cellChanges: [ + CellChange(columnIndex: 1, columnName: "name", oldValue: .null, newValue: "__DEFAULT__"), + CellChange( + columnIndex: 2, columnName: "stamped_at", oldValue: .null, newValue: .columnDefault, + newIsExpression: true + ), + ] + ) + _ = try factory(databaseType: .mssql, driver: driver).statements(for: [change]) + #expect(driver.changes.first?.defaultColumns == [2]) + #expect(driver.changes.first?.cellChanges.first?.newValue == "__DEFAULT__") + } + + @Test("A deleted row holding that text is put back by a driver that reads the default mark") + func restoredTextAMarkingDriverReadsIsWritten() throws { + let restored = try factory(databaseType: .dameng, driver: RestoringDriver(capabilities: .columnDefaultMarks)) + .restoreStatements(rows: [["1", "__DEFAULT__", "2026-01-01"]]) + #expect(restored.statements.first?.parameters.compactMap { $0 as? String } == ["1", "__DEFAULT__", "2026-01-01"]) + } + @Test("A deleted row a driver puts back with no such text is restored") func restoredRowWithoutThatTextIsWritten() throws { let restored = try factory(databaseType: .dameng, driver: RestoringDriver()) diff --git a/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementContextTests.swift b/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementContextTests.swift index 775152ce39..8844e795ba 100644 --- a/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementContextTests.swift +++ b/apple/TableProTests/Core/Plugins/PluginDriverAdapterStatementContextTests.swift @@ -320,3 +320,42 @@ struct UnenforcedStatementPolicyTests { #expect(adapter(ContextRecordingDriver(capabilities: .readOnlyStatements)).runsReadOnlyStatements) } } + +struct PluginDriverAdapterParameterKindTests { + private func adapter(_ driver: ContextRecordingDriver) -> PluginDriverAdapter { + PluginDriverAdapter(connection: DatabaseConnection(name: "Test", type: .postgresql), pluginDriver: driver) + } + + @Test("A panel value reaches the driver with the type the user gave it") + func panelTypesReachTheDriver() async throws { + let driver = ContextRecordingDriver() + let values: [Any?] = [ + TypedParameterValue(text: "00123", kind: .text), nil, TypedParameterValue(text: "7", kind: .integer), + ] + + _ = try await adapter(driver).executeUserQuery( + query: "SELECT $1, $2, $3", rowCap: nil, parameters: values, context: PluginStatementContext() + ) + _ = try await adapter(driver).executeParameterized( + query: "SELECT $1, $2, $3", parameters: values, context: PluginStatementContext() + ) + + #expect(driver.contexts.map { $0.parameterKinds } == [[.text, nil, .integer], [.text, nil, .integer]]) + } + + @Test("Values with no type, such as a grid save's, carry none") + func untypedValuesCarryNoKinds() async throws { + let driver = ContextRecordingDriver() + + _ = try await adapter(driver).executeParameterized( + query: "UPDATE t SET a = $1", parameters: ["42"], context: PluginStatementContext() + ) + + #expect(driver.contexts.map { $0.parameterKinds } == [nil]) + } + + @Test("A typed value is written to the driver as its text") + func typedValueIsItsText() { + #expect(PluginDriverAdapter.cellValue(for: TypedParameterValue(text: "1E5", kind: .text)) == .text("1E5")) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift b/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift index 17f6c3c005..12629b8c26 100644 --- a/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift +++ b/apple/TableProTests/Core/Utilities/SQLParameterExtractorTests.swift @@ -132,8 +132,16 @@ struct SQLParameterExtractorTests { ) #expect(result.sql == "WHERE id = ? AND name = ?") #expect(result.values.count == 2) - #expect((result.values[0] as? String) == "42") - #expect((result.values[1] as? String) == "Alice") + #expect((result.values[0] as? TypedParameterValue) == TypedParameterValue(text: "42", kind: .integer)) + #expect((result.values[1] as? TypedParameterValue) == TypedParameterValue(text: "Alice", kind: .text)) + } + + @Test("A String parameter holding a plain number keeps the reading by shape; any other text is text", arguments: [ + ("42", PluginColumnKind?.none), ("-3.5", nil), ("00123", nil), ("1E5", .text), ("12e3", .text), ("1.", .text), + (".5", nil), ("abc", .text), ("", .text), ("-", .text), + ]) + func stringKindFollowsTheValue(value: String, kind: PluginColumnKind?) { + #expect(QueryParameterType.string.pluginKind(for: value) == kind) } @Test @@ -161,8 +169,8 @@ struct SQLParameterExtractorTests { ) #expect(result.sql == "WHERE ? = ?") #expect(result.values.count == 2) - #expect((result.values[0] as? String) == "42") - #expect((result.values[1] as? String) == "42") + #expect((result.values[0] as? TypedParameterValue)?.text == "42") + #expect((result.values[1] as? TypedParameterValue)?.text == "42") } @Test diff --git a/apple/TableProTests/Models/PaginationStateTests.swift b/apple/TableProTests/Models/PaginationStateTests.swift index 0235101ebe..35e00686ec 100644 --- a/apple/TableProTests/Models/PaginationStateTests.swift +++ b/apple/TableProTests/Models/PaginationStateTests.swift @@ -331,7 +331,7 @@ struct PaginationStateTests { func resetLoadMoreClearsTheQueryAndItsValues() { var state = PaginationState() state.hasMoreRows = true - state.setBaseQueryForMore("SELECT * FROM t WHERE id = ?", parameterValues: ["1"]) + state.setBaseQueryForMore("SELECT * FROM t WHERE id = ?", parameterValues: [TypedParameterValue(text: "1", kind: .text)]) state.resetLoadMore() @@ -404,7 +404,7 @@ struct PaginationStateTests { @Test("Setting the base query clears the previous run's bindings") func setBaseQueryForMoreClearsStaleBindings() { var state = PaginationState() - state.setBaseQueryForMore("SELECT * FROM orders WHERE id = $1", parameterValues: ["42"]) + state.setBaseQueryForMore("SELECT * FROM orders WHERE id = $1", parameterValues: [TypedParameterValue(text: "42", kind: .text)]) state.setBaseQueryForMore("SELECT * FROM events", parameterValues: nil) @@ -416,10 +416,11 @@ struct PaginationStateTests { func setBaseQueryForMoreCarriesBindings() { var state = PaginationState() - state.setBaseQueryForMore("SELECT * FROM orders WHERE id = $1", parameterValues: ["7"]) + let bound = TypedParameterValue(text: "7", kind: .integer) + state.setBaseQueryForMore("SELECT * FROM orders WHERE id = $1", parameterValues: [bound]) #expect(state.baseQueryForMore == "SELECT * FROM orders WHERE id = $1") - #expect(state.baseQueryParameterValues == ["7"]) + #expect(state.baseQueryParameterValues == [bound]) } @Test("Busy covers every kind of pending work") diff --git a/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift b/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift index 47cd3e727d..d968e8daf0 100644 --- a/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift +++ b/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift @@ -528,7 +528,7 @@ struct MainContentCoordinatorSortTests { let idx = try #require(tabManager.tabs.firstIndex(where: { $0.id == tabId })) let result = ResultSet(label: "audit", tableRows: rows) result.baseQuery = "SELECT msg FROM audit WHERE msg = ?" - result.baseQueryParameterValues = ["hello"] + result.baseQueryParameterValues = [TypedParameterValue(text: "hello", kind: .text)] tabManager.mutate(at: idx) { $0.display.replaceUnpinnedResults(with: [result]) } coordinator.setActiveTableRows(rows, for: tabId) diff --git a/apple/TableProTests/Views/Main/StopRollbackNoticeTests.swift b/apple/TableProTests/Views/Main/StopRollbackNoticeTests.swift new file mode 100644 index 0000000000..b75f6881d9 --- /dev/null +++ b/apple/TableProTests/Views/Main/StopRollbackNoticeTests.swift @@ -0,0 +1,40 @@ +// +// StopRollbackNoticeTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@MainActor +struct StopRollbackNoticeTests { + private func fixture() -> (CoordinatorFixture, UUID) { + let fixture = CoordinatorFixture(connection: TestFixtures.makeConnection(type: .postgresql)) + let tab = fixture.addTab(QueryTab(title: "Q1", query: "UPDATE t SET a = 1", tabType: .query)) + return (fixture, tab.id) + } + + private func errorMessage(_ fixture: CoordinatorFixture, _ tabId: UUID) -> String? { + fixture.tabManager.tabs.first { $0.id == tabId }?.execution.errorMessage + } + + @Test("A rollback the server did for a Stop is shown on the stopped tab") + func rollbackIsShown() { + let (fixture, tabId) = fixture() + + fixture.coordinator.reportStopRollback(PluginStopRolledBackTransaction(code: 3_110), tabId: tabId) + + #expect(errorMessage(fixture, tabId)?.contains("rolled back the transaction") == true) + } + + @Test("A plain cancel leaves the tab as Stop left it") + func plainCancelShowsNothing() { + let (fixture, tabId) = fixture() + + fixture.coordinator.reportStopRollback(CancellationError(), tabId: tabId) + + #expect(errorMessage(fixture, tabId) == nil) + } +} diff --git a/apple/scripts/probes/check-oracle-time-zone-regions.sh b/apple/scripts/probes/check-oracle-time-zone-regions.sh new file mode 100755 index 0000000000..bd6a1beab8 --- /dev/null +++ b/apple/scripts/probes/check-oracle-time-zone-regions.sh @@ -0,0 +1,434 @@ +#!/usr/bin/env bash +# +# Check that a TIMESTAMP WITH TIME ZONE stored with a region name reads as its wall clock and Oracle's name, and that +# the grid's write-back masks parse that text again. +# +# Such a value carries an id into the server's time zone file. OracleTimeZoneRegions reads the ids once per connection +# from SYS.V_$TIMEZONE_NAMES and assumes a layout no Oracle document states: 0x80 in byte 11 marks a region, the id is +# 13 bits over bytes 11 and 12 with the low two bits of byte 12 unused, and bytes 1 to 7 are UTC. This reads the wire +# bytes through oracle-nio and the cell text through the same OracleCoreConnection the app uses, and checks: +# 1. whether the account can read the view; ORA-00942 is a normal answer, and the cells then read as UTC +# 2. the map query's rows, for AT TIME ZONE and FROM_TZ: every row a region, no id repeated, the low bits clear +# 3. which names Foundation's TimeZone does not know +# 4. stored region values: wire bytes against SYS_EXTRACT_UTC and DUMP, cell text against TO_CHAR and EXTRACT +# 5. a session zone set to a region: CURRENT_TIMESTAMP and friends read with it, and the map outlives the ALTER +# 6. the masks in OracleRowWriter.swift parse the region, offset and zone-less spellings, into TSTZ and LTZ +# +# Usage: +# scripts/probes/check-oracle-time-zone-regions.sh [host] [port] [service] [user] [password] +# +# Defaults suit a throwaway Oracle Free container: +# docker run -d --name oracle-probe -p 1521:1521 -e ORACLE_PASSWORD=probe_pw \ +# -e APP_USER=probe -e APP_USER_PASSWORD=probe_pw gvenzl/oracle-free:23-slim-faststart +# +# Hand-run only, never in CI. The user needs CREATE TABLE. Every object it creates is prefixed TP_TZR_ and dropped +# again. Exits non-zero on a disagreement, 3 when a prerequisite is missing. + +set -euo pipefail + +HOST="${1:-127.0.0.1}" +PORT="${2:-1521}" +SERVICE="${3:-FREEPDB1}" +USER_NAME="${4:-probe}" +PASSWORD="${5:-probe_pw}" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +if [ -z "${DEVELOPER_DIR:-}" ]; then + for candidate in /Applications/Xcode-beta.app /Applications/Xcode.app; do + if [ -d "$candidate/Contents/Developer" ]; then + export DEVELOPER_DIR="$candidate/Contents/Developer" + break + fi + done +fi +command -v swift > /dev/null || { + echo "swift not found" >&2 + exit 3 +} + +MASK="$(sed -n 's/.*static let timestampWithTimeZoneMask = "\(.*\)"$/\1/p' "$ROOT/Plugins/OracleDriverPlugin/OracleRowWriter.swift")" +OFFSET_MASK="$(sed -n 's/.*static let timestampWithOffsetMask = "\(.*\)"$/\1/p' "$ROOT/Plugins/OracleDriverPlugin/OracleRowWriter.swift")" +NIO_REVISION="$(sed -n 's/.*revision: "\([0-9a-f]*\)".*/\1/p' "$ROOT/Packages/TableProOracle/Package.swift")" +if [ -z "$MASK" ] || [ -z "$OFFSET_MASK" ] || [ -z "$NIO_REVISION" ]; then + echo "cannot read the writer's mask or the oracle-nio pin" >&2 + exit 3 +fi + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT +mkdir -p "$WORK/Sources/Probe" + +cat > "$WORK/Package.swift" < "$WORK/Sources/Probe/main.swift" <<'EOF' +import Foundation +import NIOCore +import OracleNIO +import TableProOracleCore + +let arguments = CommandLine.arguments +let host = arguments[1] +let port = Int(arguments[2]) ?? 1_521 +let service = arguments[3] +let user = arguments[4] +let password = arguments[5] +let regionMask = arguments[6] +let offsetMask = arguments[7] +let view = "SYS.V_$TIMEZONE_NAMES" + +let connection = OracleCoreConnection(options: OracleConnectionOptions( + host: host, port: port, user: user, password: password, identifierMode: .service, serviceName: service +)) +do { + try await connection.connect() +} catch { + print("cannot connect: \(error)") + exit(3) +} + +func dialRaw() async -> OracleNIO.OracleConnection { + do { + return try await OracleNIO.OracleConnection.connect( + configuration: OracleNIO.OracleConnection.Configuration( + host: host, port: port, service: .serviceName(service), username: user, password: password + ), + id: 1 + ) + } catch { + print("cannot connect through oracle-nio: \(error)") + exit(3) + } +} + +let raw = await dialRaw() + +var disagreements = 0 + +@MainActor +func check(_ holds: Bool, _ claim: String) { + print("\(holds ? "ok " : "FAIL") \(claim)") + if !holds { disagreements += 1 } +} + +func info(_ line: String) { + print("info \(line)") +} + +func attempt(_ sql: String) async -> (result: OracleRawResult?, failure: String?) { + do { + return (try await connection.executeQuery(sql), nil) + } catch { + return (nil, String(describing: error)) + } +} + +@MainActor +func result(_ sql: String) async -> OracleRawResult? { + let answer = await attempt(sql) + if let failure = answer.failure { + print("FAIL \(sql.prefix(80)): \(failure)") + disagreements += 1 + } + return answer.result +} + +func scalar(_ sql: String) async -> String? { + await attempt(sql).result?.rows.first?.first?.stringValue +} + +func run(_ sql: String) async { + _ = await attempt(sql) +} + +// The cells as oracle-nio hands them over, before the core decodes anything. +func rawRows(_ sql: String) async throws -> [[OracleCell]] { + let stream = try await raw.execute( + OracleStatement(unsafeSQL: sql), options: StatementOptions(prefetchRows: 1_000, arraySize: 1_000) + ) + var rows: [[OracleCell]] = [] + for try await row in stream { + rows.append(Array(row)) + } + return rows +} + +func bytes(of cell: OracleCell) -> [UInt8] { + cell.bytes.map { Array($0.readableBytesView) } ?? [] +} + +func hex(_ bytes: [UInt8]) -> String { + bytes.map { String(format: "%02x", $0) }.joined() +} + +func listed(_ items: [String]) -> String { + items.isEmpty ? "" : " (\(items.count): \(items.prefix(5).joined(separator: "; ")))" +} + +// The writer's own choice: a region name after a blank takes the region mask, anything else the offset mask. +func mask(for text: String) -> String { + text.split(separator: " ").last?.first?.isLetter == true ? regionMask : offsetMask +} + +func parsed(_ text: String) -> String { + "TO_TIMESTAMP_TZ('\(text)', '\(mask(for: text))')" +} + +guard let release = connection.serverRelease else { + print("the login reply carried no server release") + exit(3) +} +print("Oracle \(release.major).\(release.update), writer masks '\(regionMask)' and '\(offsetMask)'") + +print("\n1. Who can read \(view)") +let roles = await result("SELECT ROLE FROM SESSION_ROLES ORDER BY ROLE")?.rows.compactMap { $0.first?.stringValue } ?? [] +info("session roles: \(roles.isEmpty ? "none" : roles.joined(separator: ", "))") +let publicGrants = await scalar(""" + SELECT COUNT(*) FROM ALL_TAB_PRIVS + WHERE TABLE_SCHEMA = 'SYS' AND TABLE_NAME = 'V_$TIMEZONE_NAMES' AND GRANTEE = 'PUBLIC' + """) +info("PUBLIC grants on \(view) in ALL_TAB_PRIVS: \(publicGrants ?? "unreadable")") +let access = await attempt("SELECT COUNT(*) FROM \(view)") +let mapReadable = access.result != nil +if let count = access.result?.rows.first?.first?.stringValue { + check(true, "\(user) reads \(view): \(count) rows") +} else { + let failure = access.failure ?? "" + check(failure.contains("ORA-00942"), "\(user) is refused \(view) with ORA-00942: \(failure.prefix(120))") +} +if let file = await scalar("SELECT FILENAME || ' version ' || VERSION FROM SYS.V_$TIMEZONE_FILE") { + info("time zone file \(file)") +} + +print("\n2. The map query, read raw") +var mapRows: [(name: String, wire: [UInt8])] = [] +var idsByForm: [String: [String: Int]] = [:] +let fromTZQuery = "SELECT TZNAME, FROM_TZ(TIMESTAMP '2000-01-01 00:00:00', TZNAME) FROM " + + "(SELECT DISTINCT TZNAME FROM \(view)) ORDER BY TZNAME" +if mapReadable { + for (form, sql) in [("AT TIME ZONE", OracleTimeZoneRegions.query), ("FROM_TZ", fromTZQuery)] { + let started = ContinuousClock.now + let rows: [[OracleCell]] + do { + rows = try await rawRows(sql) + } catch { + check(false, "\(form): the map query runs: \(error)") + continue + } + info("\(form): \(rows.count) rows in \(ContinuousClock.now - started)") + let pairs = rows.compactMap { cells -> (name: String, wire: [UInt8])? in + guard cells.count == 2, let name = try? cells[0].decode(String.self) else { return nil } + return (name, bytes(of: cells[1])) + } + let unflagged = pairs.filter { OracleTimeZoneRegions.regionID(wire: $0.wire) == nil }.map { "\($0.name) \(hex($0.wire))" } + check(unflagged.isEmpty, "\(form): every row carries a region id\(listed(unflagged))") + var namesByID: [Int: [String]] = [:] + for pair in pairs { + guard let id = OracleTimeZoneRegions.regionID(wire: pair.wire) else { continue } + namesByID[id, default: []].append(pair.name) + } + let shared = namesByID.filter { $0.value.count > 1 }.map { "\($0.key): \($0.value.joined(separator: ", "))" } + check(shared.isEmpty, "\(form): no two names share an id\(listed(shared))") + let lowBits = pairs.filter { $0.wire.count >= 13 && $0.wire[12] & 0x03 != 0 }.map { "\($0.name) \(hex($0.wire))" } + check(lowBits.isEmpty, "\(form): the low two bits of byte 12 are clear\(listed(lowBits))") + var ids: [String: Int] = [:] + for pair in pairs { + ids[pair.name] = OracleTimeZoneRegions.regionID(wire: pair.wire) + } + idsByForm[form] = ids + if form == "AT TIME ZONE" { + mapRows = pairs + let utcFields: [UInt8] = [0x78, 0x64, 0x01, 0x01, 0x01, 0x01, 0x01] + let moved = pairs.filter { Array($0.wire.prefix(7)) != utcFields }.map { "\($0.name) \(hex($0.wire))" } + check(moved.isEmpty, "AT TIME ZONE: bytes 1 to 7 of every row are 2000-01-01 00:00:00 UTC\(listed(moved))") + } + } + if idsByForm.count == 2 { + check(idsByForm["AT TIME ZONE"] == idsByForm["FROM_TZ"], "AT TIME ZONE and FROM_TZ give each name the same id") + } + info("OracleTimeZoneRegions keeps \(OracleTimeZoneRegions(rows: mapRows).count) of \(mapRows.count) rows") +} else { + info("skip: \(view) is not readable, so every region value reads as UTC") +} + +// The first name per id, as the map keeps it: the query orders by name. +var nameByID: [Int: String] = [:] +for row in mapRows { + guard let id = OracleTimeZoneRegions.regionID(wire: row.wire), nameByID[id] == nil else { continue } + nameByID[id] = row.name +} + +print("\n3. Names Foundation does not know") +let unknownNames = mapRows.map { $0.name }.filter { TimeZone(identifier: $0) == nil } +info("\(unknownNames.count) of \(mapRows.count) names read as UTC instead") +for name in unknownNames { + info("Foundation does not know \(name)") +} + +print("\n4. Stored region values") +let samples: [(id: Int, literal: String)] = [ + (1, "TIMESTAMP '2024-01-02 03:04:05.5 Europe/Paris'"), + (2, "TIMESTAMP '2024-07-01 14:00:00 Europe/Paris'"), + (3, "TIMESTAMP '2024-03-10 01:30:00 America/Los_Angeles'"), + (4, "TIMESTAMP '2024-11-03 12:00:00 US/Pacific'"), + (5, "TIMESTAMP '2024-06-15 08:00:00 UTC'"), + (6, "TIMESTAMP '2024-06-15 08:00:00 Etc/GMT+5'"), + (7, "TIMESTAMP '1880-06-01 12:00:00 Europe/Paris'"), + (8, "TIMESTAMP '2024-10-27 02:30:00 Europe/Paris CEST'"), +] +let overlapRow = 8 +await run("DROP TABLE TP_TZR_T PURGE") +if let failure = await attempt("CREATE TABLE TP_TZR_T (ID NUMBER PRIMARY KEY, TZ TIMESTAMP(6) WITH TIME ZONE)").failure { + print("cannot create TP_TZR_T: \(failure)") + exit(3) +} +var literalByID: [Int: String] = [:] +for sample in samples { + if let failure = await attempt("INSERT INTO TP_TZR_T VALUES (\(sample.id), \(sample.literal))").failure { + info("skip \(sample.literal): \(failure.prefix(100))") + } else { + literalByID[sample.id] = sample.literal + } +} +_ = await result("COMMIT") + +var wireByID: [Int: [UInt8]] = [:] +do { + for cells in try await rawRows("SELECT TO_CHAR(ID), TZ, SYS_EXTRACT_UTC(TZ), DUMP(TZ) FROM TP_TZR_T ORDER BY ID") { + guard cells.count == 4, let row = (try? cells[0].decode(String.self)).flatMap({ Int($0) }) else { continue } + let wire = bytes(of: cells[1]) + let utc = bytes(of: cells[2]) + let dump = (try? cells[3].decode(String.self)) ?? "" + let dumped = dump.split(separator: ":", maxSplits: 1).dropFirst().first.map { + $0.split(separator: ",").compactMap { UInt8($0.trimmingCharacters(in: .whitespaces)) } + } ?? [] + let id = OracleTimeZoneRegions.regionID(wire: wire) + let lowBits = wire.count >= 13 ? String(wire[12] & 0x03) : "?" + info("row \(row): \(hex(wire)), region id \(id.map { String($0) } ?? "none"), byte 12 low bits \(lowBits)") + check(id != nil, "row \(row): the stored value carries a region id") + check(Array(wire.prefix(7)) == Array(utc.prefix(7)), "row \(row): wire bytes 1 to 7 are SYS_EXTRACT_UTC's \(hex(Array(utc.prefix(7))))") + check(dump.hasPrefix("Typ=181") && dumped == wire, "row \(row): DUMP of the column matches the wire bytes: \(dump)") + wireByID[row] = wire + } +} catch { + check(false, "the stored values read raw: \(error)") +} + +let textQuery = """ + SELECT TO_CHAR(ID), TZ, TO_CHAR(TZ, 'YYYY-MM-DD HH24:MI:SS.FF6') || ' ' || EXTRACT(TIMEZONE_REGION FROM TZ), + TO_CHAR(SYS_EXTRACT_UTC(TZ), 'YYYY-MM-DD HH24:MI:SS.FF6') || '+00:00', + EXTRACT(TIMEZONE_REGION FROM TZ), TO_CHAR(TZ, 'TZR') + FROM TP_TZR_T ORDER BY ID + """ +var cellTextByID: [Int: String] = [:] +var regionByID: [Int: String] = [:] +for values in await result(textQuery)?.rows ?? [] { + guard values.count == 6, let row = values[0].stringValue.flatMap({ Int($0) }) else { continue } + let cell = values[1].stringValue ?? "NULL" + let server = values[2].stringValue ?? "" + let utcText = values[3].stringValue ?? "" + let region = values[4].stringValue ?? "" + let tzr = values[5].stringValue ?? "" + let mapName = wireByID[row].flatMap { OracleTimeZoneRegions.regionID(wire: $0) }.flatMap { nameByID[$0] } + if let mapName, mapName != region { + check(false, "row \(row): the map names its id \(mapName), the server \(region)") + } + let expected = mapName.map { TimeZone(identifier: $0) != nil } == true ? server : utcText + check(cell == expected, "row \(row) \(literalByID[row] ?? ""): reads \(cell)\(cell == expected ? "" : ", expected \(expected)")") + if tzr != region { + info("row \(row): TO_CHAR TZR spells \(tzr), EXTRACT spells \(region)") + } + cellTextByID[row] = cell + regionByID[row] = region +} + +print("\n5. A session zone set to a region") +_ = await result("ALTER SESSION SET TIME_ZONE = 'Europe/Paris'") +let sessionSuffix = nameByID.values.contains("Europe/Paris") ? " Europe/Paris" : "+00:00" +let regionForms = [ + "CURRENT_TIMESTAMP", + "FROM_TZ(TIMESTAMP '2024-01-02 03:04:05', 'Europe/Paris')", + "TIMESTAMP '2024-01-02 02:04:05 UTC' AT TIME ZONE 'Europe/Paris'", + "TIMESTAMP '2024-01-02 02:04:05 UTC' AT LOCAL", + "CAST(CAST(TIMESTAMP '2024-01-02 02:04:05 UTC' AS TIMESTAMP WITH LOCAL TIME ZONE) AS TIMESTAMP WITH TIME ZONE)", +] +if let values = await result("SELECT \(regionForms.joined(separator: ", ")) FROM DUAL")?.rows.first { + for (form, value) in zip(regionForms, values) { + let text = value.stringValue ?? "NULL" + check(text.hasSuffix(sessionSuffix), "\(form) reads \(text)") + } +} +if let first = cellTextByID.keys.min(), let before = cellTextByID[first] { + let after = await scalar("SELECT TZ FROM TP_TZR_T WHERE ID = \(first)") + check(after == before, "after the ALTER, row \(first) still reads \(after ?? "NULL")") +} + +print("\n6. Write-back through the writer's masks") +for (row, text) in cellTextByID.sorted(by: { $0.key < $1.key }) { + let matched = await scalar("SELECT COUNT(*) FROM TP_TZR_T WHERE ID = \(row) AND TZ = \(parsed(text))") + if row == overlapRow { + let resolved = await scalar("SELECT TO_CHAR(\(parsed(text)), 'YYYY-MM-DD HH24:MI:SS TZR TZD') FROM DUAL") + info("the repeated hour: \(text) matches \(matched ?? "no") stored row and resolves to \(resolved ?? "an error")") + continue + } + check(matched == "1", "\(text) parses back to the stored instant") + guard !text.hasSuffix("+00:00") else { continue } + let kept = await scalar("SELECT EXTRACT(TIMEZONE_REGION FROM \(parsed(text))) FROM DUAL") + check(kept == regionByID[row], "\(text) keeps its region: \(kept ?? "NULL")") +} + +let spellings: [(text: String, literal: String)] = [ + ("2024-01-02 03:04:05.123456-05:30", "TIMESTAMP '2024-01-02 03:04:05.123456 -05:30'"), + ("2000-01-01 00:00:00.000000+00:00", "TIMESTAMP '2000-01-01 00:00:00 +00:00'"), + ("2000-01-01 00:00:00", "FROM_TZ(TIMESTAMP '2000-01-01 00:00:00', SESSIONTIMEZONE)"), + ("2024-01-02 03:04:05.500000 Europe/Paris", "TIMESTAMP '2024-01-02 03:04:05.5 Europe/Paris'"), +] +await run("DROP TABLE TP_TZR_L PURGE") +_ = await result("CREATE TABLE TP_TZR_L (ID NUMBER PRIMARY KEY, L TIMESTAMP(6) WITH LOCAL TIME ZONE)") +for (index, spelling) in spellings.enumerated() { + let compared = await attempt("SELECT CASE WHEN \(parsed(spelling.text)) = \(spelling.literal) THEN 1 ELSE 0 END FROM DUAL") + let equal = compared.result?.rows.first?.first?.stringValue + check(equal == "1", "TIMESTAMP WITH TIME ZONE: '\(spelling.text)' parses to \(spelling.literal)\(compared.failure.map { ": \($0.prefix(100))" } ?? "")") + let inserted = await attempt("INSERT INTO TP_TZR_L VALUES (\(index), \(parsed(spelling.text)))") + var stored: String? + if inserted.failure == nil { + stored = await scalar("SELECT COUNT(*) FROM TP_TZR_L WHERE ID = \(index) AND L = \(spelling.literal)") + } + check(stored == "1", "TIMESTAMP WITH LOCAL TIME ZONE: '\(spelling.text)' writes \(spelling.literal)\(inserted.failure.map { ": \($0.prefix(100))" } ?? "")") +} + +await run("DROP TABLE TP_TZR_T PURGE") +await run("DROP TABLE TP_TZR_L PURGE") +try? await raw.close() +print(disagreements == 0 ? "\nall checks hold" : "\n\(disagreements) disagreement(s)") +exit(disagreements == 0 ? 0 : 1) +EOF + +echo "Building the probe against Packages/TableProOracle (the first build takes a minute)" +(cd "$WORK" && swift build -c debug > "$WORK/build.log" 2>&1) || { + grep -E "error:" "$WORK/build.log" | head -20 >&2 + exit 3 +} +"$WORK/.build/debug/Probe" "$HOST" "$PORT" "$SERVICE" "$USER_NAME" "$PASSWORD" "$MASK" "$OFFSET_MASK" 2> /dev/null From 21364870f89ba031e5406d0f6aee56efe6c1f70c Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 11 Nov 2026 20:32:51 +0700 Subject: [PATCH 5/7] docs(changelog): add the fragments for #4190 --- apple/changelog.d/4190.fixed.1.md | 1 + apple/changelog.d/4190.fixed.2.md | 1 + apple/changelog.d/4190.fixed.3.md | 1 + apple/changelog.d/4190.fixed.4.md | 1 + apple/changelog.d/4190.fixed.5.md | 1 + apple/changelog.d/4190.fixed.6.md | 1 + apple/changelog.d/4190.fixed.7.md | 1 + apple/changelog.d/4190.fixed.8.md | 1 + apple/changelog.d/4190.fixed.md | 1 + apple/changelog.d/4190.security.md | 1 + 10 files changed, 10 insertions(+) create mode 100644 apple/changelog.d/4190.fixed.1.md create mode 100644 apple/changelog.d/4190.fixed.2.md create mode 100644 apple/changelog.d/4190.fixed.3.md create mode 100644 apple/changelog.d/4190.fixed.4.md create mode 100644 apple/changelog.d/4190.fixed.5.md create mode 100644 apple/changelog.d/4190.fixed.6.md create mode 100644 apple/changelog.d/4190.fixed.7.md create mode 100644 apple/changelog.d/4190.fixed.8.md create mode 100644 apple/changelog.d/4190.fixed.md create mode 100644 apple/changelog.d/4190.security.md diff --git a/apple/changelog.d/4190.fixed.1.md b/apple/changelog.d/4190.fixed.1.md new file mode 100644 index 0000000000..ecc80d29b8 --- /dev/null +++ b/apple/changelog.d/4190.fixed.1.md @@ -0,0 +1 @@ +Saving the text __DEFAULT__ in a SQL Server, Oracle, MongoDB or other driver-written table was refused. diff --git a/apple/changelog.d/4190.fixed.2.md b/apple/changelog.d/4190.fixed.2.md new file mode 100644 index 0000000000..74b5117d28 --- /dev/null +++ b/apple/changelog.d/4190.fixed.2.md @@ -0,0 +1 @@ +Exporting a large Teradata, InfluxDB, SAP HANA or Memcached table loaded every row into memory first. diff --git a/apple/changelog.d/4190.fixed.3.md b/apple/changelog.d/4190.fixed.3.md new file mode 100644 index 0000000000..f49fe3cc89 --- /dev/null +++ b/apple/changelog.d/4190.fixed.3.md @@ -0,0 +1 @@ +Stopping a Teradata query in ANSI mode did not say that Teradata rolled back the open transaction. diff --git a/apple/changelog.d/4190.fixed.4.md b/apple/changelog.d/4190.fixed.4.md new file mode 100644 index 0000000000..ce44ae8112 --- /dev/null +++ b/apple/changelog.d/4190.fixed.4.md @@ -0,0 +1 @@ +Oracle TIMESTAMP WITH TIME ZONE values stored with a region name showed a placeholder instead of the time. diff --git a/apple/changelog.d/4190.fixed.5.md b/apple/changelog.d/4190.fixed.5.md new file mode 100644 index 0000000000..7a22306a09 --- /dev/null +++ b/apple/changelog.d/4190.fixed.5.md @@ -0,0 +1 @@ +A Teradata query with the row limit on read every row of the result. diff --git a/apple/changelog.d/4190.fixed.6.md b/apple/changelog.d/4190.fixed.6.md new file mode 100644 index 0000000000..76236db0a8 --- /dev/null +++ b/apple/changelog.d/4190.fixed.6.md @@ -0,0 +1 @@ +A long SAP HANA export failed when it ran past the query timeout. diff --git a/apple/changelog.d/4190.fixed.7.md b/apple/changelog.d/4190.fixed.7.md new file mode 100644 index 0000000000..b130b47acb --- /dev/null +++ b/apple/changelog.d/4190.fixed.7.md @@ -0,0 +1 @@ +Duplicating an Elasticsearch document saved the copy with the id __DEFAULT__. diff --git a/apple/changelog.d/4190.fixed.8.md b/apple/changelog.d/4190.fixed.8.md new file mode 100644 index 0000000000..29f5f55088 --- /dev/null +++ b/apple/changelog.d/4190.fixed.8.md @@ -0,0 +1 @@ +Stopping a Memcached export left the connection closed until a reconnect. diff --git a/apple/changelog.d/4190.fixed.md b/apple/changelog.d/4190.fixed.md new file mode 100644 index 0000000000..4d55e2759e --- /dev/null +++ b/apple/changelog.d/4190.fixed.md @@ -0,0 +1 @@ +Integer, Decimal and Boolean query parameters, and codes such as 1E5, were sent to Trino by how the value looked. diff --git a/apple/changelog.d/4190.security.md b/apple/changelog.d/4190.security.md new file mode 100644 index 0000000000..900541eb6a --- /dev/null +++ b/apple/changelog.d/4190.security.md @@ -0,0 +1 @@ +Browsing a Memcached cache with a key holding a line break could run a second command on the server. From 11245e61ae63a94e139251b750b917cc0631b44e Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 11 Nov 2026 20:39:13 +0700 Subject: [PATCH 6/7] fix(ios): add the new plugin strings to the iOS catalog --- apple/TableProMobile/TableProMobile/Localizable.xcstrings | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/apple/TableProMobile/TableProMobile/Localizable.xcstrings b/apple/TableProMobile/TableProMobile/Localizable.xcstrings index 6c3e6f49ca..8a20343c3c 100644 --- a/apple/TableProMobile/TableProMobile/Localizable.xcstrings +++ b/apple/TableProMobile/TableProMobile/Localizable.xcstrings @@ -26443,6 +26443,10 @@ }, "The statement before it stays applied." : { + }, + "The statement was stopped, and the server rolled back the transaction it ran in. Every change since that transaction began is undone." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The table \"%@\" and all its data will be permanently deleted." : { "localizations" : { @@ -27177,6 +27181,10 @@ } } }, + "This database has no column defaults, so a cell cannot be left to one. Enter a value." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This database has no Create Table form" : { "extractionState" : "manual", "generatesSymbol" : false, From e67167e949f476aa62ef9e409dd87607ac42188b Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Wed, 11 Nov 2026 21:56:22 +0700 Subject: [PATCH 7/7] fix(plugins): name the Memcached key check by its type, and keep untyped rerun values --- .../Sources/TableProMemcachedCore/MemcachedKeyListing.swift | 2 +- .../Coordinators/QueryExecutionCoordinator+Parameters.swift | 4 ++-- apple/TablePro/Models/Query/QueryParameter.swift | 6 ++++++ .../Views/Main/MainContentCoordinatorSortTests.swift | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift index b34e744280..b3502139a6 100644 --- a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift @@ -55,7 +55,7 @@ public enum MemcachedKeyListing { // A key stored through the binary or meta protocol can hold a space or a line break, which a text command // line cannot carry: sent in a `get`, it would end the line and start a command of its own. guard let encoded = fields["key"], let key = percentDecoded(encoded), - MemcachedCommand.isValidKey(Array(key.utf8)) else { return nil } + MemcachedCommandParser.isValidKey(Array(key.utf8)) else { return nil } let expiry = fields["exp"].flatMap { Int64($0) } return MemcachedKeyEntry( key: key, diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift index 7d72575eb4..6143d67473 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift @@ -685,7 +685,7 @@ extension QueryExecutionCoordinator { sql: statement.originalSQL, index: firstNumber + index, baseQuery: statement.executableSQL, - baseQueryParameterValues: statement.parameterValues?.map { $0 as? TypedParameterValue }, + baseQueryParameterValues: statement.parameterValues?.map(TypedParameterValue.kept), namedParameterStatement: statement.parameterValues == nil ? nil : NamedParameterStatement(sql: statement.originalSQL, parameters: history.parameters), @@ -794,7 +794,7 @@ extension QueryExecutionCoordinator { prepared: prepared ) - let parameterValues = nativeParameters.map { $0 as? TypedParameterValue } + let parameterValues = nativeParameters.map(TypedParameterValue.kept) let statement = originalSQL.map { NamedParameterStatement(sql: $0, parameters: originalParameters) } parent.tabManager.mutate(tabId: tabId) { $0.pagination.baseQueryParameterValues = parameterValues diff --git a/apple/TablePro/Models/Query/QueryParameter.swift b/apple/TablePro/Models/Query/QueryParameter.swift index c366ce87b5..56550eab6a 100644 --- a/apple/TablePro/Models/Query/QueryParameter.swift +++ b/apple/TablePro/Models/Query/QueryParameter.swift @@ -69,6 +69,12 @@ struct TypedParameterValue: Equatable, Sendable { let text: String let kind: PluginColumnKind? + // A run's value as the rerun state keeps it; plain text from a caller that gave no type stays text with no kind. + static func kept(_ value: Any?) -> TypedParameterValue? { + if let typed = value as? TypedParameterValue { return typed } + return (value as? String).map { TypedParameterValue(text: $0, kind: nil) } + } + // Nil when no value carries a type, so a driver reads the values as it did before types were passed. static func kinds(of parameters: [Any?]?) -> [PluginColumnKind?]? { guard let parameters else { return nil } diff --git a/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift b/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift index d968e8daf0..14cee2efb9 100644 --- a/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift +++ b/apple/TableProTests/Views/Main/MainContentCoordinatorSortTests.swift @@ -894,7 +894,7 @@ private final class StatementRecordingDriver: DatabaseDriver, @unchecked Sendabl } private func record(_ query: String, parameters: [Any?]?) -> QueryResult { - let bound = parameters.map { values in values.map { $0 as? String } } + let bound = parameters.map { values in values.map { ($0 as? TypedParameterValue)?.text ?? $0 as? String } } lock.withLock { statements.append(SentStatement(sql: query, parameters: bound)) } guard query.uppercased().hasPrefix("SELECT") else { return QueryResult(columns: [], columnTypes: [], rows: [], rowsAffected: 1, executionTime: 0, error: nil)