From 224d6a88abafe89b336afb264483da7fed0b2fb4 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 12 Nov 2026 06:24:19 +0700 Subject: [PATCH 1/3] fix(import-export): keep binary values, booleans, all-NULL rows and zoned timestamps through copy, import and export --- .../CSVImportPlugin/CSVImportParsing.swift | 21 +- .../CSVImportPlugin/CSVImportPlugin.swift | 17 +- .../CSVImportPluginTests.swift | 18 +- .../CSVImportRowShapeTests.swift | 28 +++ .../ExportImportRoundTripTests.swift | 7 +- .../XLSXSheetRowsTests.swift | 43 ++++ .../XLSXWriterStreamingTests.swift | 4 +- .../ParquetTypeMapper.swift | 10 + .../XLSXExportPlugin/XLSXExportPlugin.swift | 102 +++------ .../XLSXExportPlugin/XLSXSheetRows.swift | 43 ++++ .../Plugins/XLSXExportPlugin/XLSXWriter.swift | 2 +- .../SQLStatementGenerator.swift | 10 +- .../Core/Plugins/ImportDataSinkAdapter.swift | 86 +++++--- .../Core/Plugins/ImportTargetValues.swift | 100 +++++++++ .../Core/Plugins/ImportTypeMapper.swift | 4 +- .../Core/Services/Export/ImportService.swift | 2 + .../Formatting/BlobFormattingService.swift | 38 +++- .../Services/Query/RowOperationsManager.swift | 36 +++- .../Utilities/SQL/InClauseConverter.swift | 19 +- .../Utilities/SQL/RowValueCopyFormatter.swift | 4 +- .../TablePro/Extensions/String+HexDump.swift | 24 +-- .../Commands/DataGridCommands+Copy.swift | 4 +- .../Commands/DataGridCommands+Paste.swift | 6 +- .../Views/Main/Child/DataTabSQLExporter.swift | 4 +- .../Extensions/MainContentView+Helpers.swift | 3 +- .../ImportDataSinkAdapterBitColumnTests.swift | 11 +- .../ImportDataSinkAdapterMappingTests.swift | 5 +- ...mportDataSinkAdapterTargetValueTests.swift | 204 ++++++++++++++++++ .../Core/Plugins/ImportTypeMapperTests.swift | 1 + .../Services/BlobFormattingServiceTests.swift | 26 +++ .../RowOperationsManagerPasteTests.swift | 39 ++++ .../Utilities/InClauseConverterTests.swift | 28 +++ .../DataGridSetCellsToValueTests.swift | 38 ++++ .../probes/check-parquet-export-values.sh | 42 +++- 34 files changed, 861 insertions(+), 168 deletions(-) create mode 100644 apple/Plugins/FormatPluginTests/XLSXSheetRowsTests.swift create mode 100644 apple/Plugins/XLSXExportPlugin/XLSXSheetRows.swift create mode 100644 apple/TablePro/Core/Plugins/ImportTargetValues.swift create mode 100644 apple/TableProTests/Core/Plugins/ImportDataSinkAdapterTargetValueTests.swift diff --git a/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift b/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift index e81c968cb2..a319f836b9 100644 --- a/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift +++ b/apple/Plugins/CSVImportPlugin/CSVImportParsing.swift @@ -103,10 +103,27 @@ enum CSVImportParsing { } } - static func isBlank(_ fields: [String]) -> Bool { + // RFC 4180 makes an empty line a record of one empty field, which is how CSV export writes a NULL in a one-column + // file. A record of several empty fields, `,,`, is an all-NULL row. + static func isBlank(_ fields: [String], columnCount: Int) -> Bool { + columnCount > 1 && fields.count == 1 && fields[0].isEmpty + } + + // A record with nothing in it adds no sample to detection. + static func holdsNoValue(_ fields: [String]) -> Bool { fields.allSatisfy { $0.isEmpty } } + // A spreadsheet saves the empty rows under its data as lines of delimiters, the same text as all-NULL rows at + // the very end of an export. Only that trailing run is left out; empty records between rows are rows. + static func dataEnd(recordCount: Int, fields: (Int) -> [String]) -> Int { + var end = recordCount + while end > 0, holdsNoValue(fields(end - 1)) { + end -= 1 + } + return end + } + static func detectFields( in data: Data, options: CSVImportOptions, @@ -141,7 +158,7 @@ enum CSVImportParsing { for range in dataRanges { if sampled >= limit { break } let fields = parser.parseRow(buffer, range: range) - if isBlank(fields) { continue } + if holdsNoValue(fields) { continue } for column in 0..] if hasHeader { let header = parser.parseRow(buffer, range: ranges[0]) - let names = CSVImportParsing.columnNames(header: header, columnCount: header.count) - return (Array(ranges.dropFirst()), names) + names = CSVImportParsing.columnNames(header: header, columnCount: header.count) + records = Array(ranges.dropFirst()) + } else { + let firstCount = parser.parseRow(buffer, range: ranges[0]).count + names = CSVImportParsing.columnNames(header: nil, columnCount: firstCount) + records = ranges } - let firstCount = parser.parseRow(buffer, range: ranges[0]).count - let names = CSVImportParsing.columnNames(header: nil, columnCount: firstCount) - return (ranges, names) + let end = CSVImportParsing.dataEnd(recordCount: records.count) { parser.parseRow(buffer, range: records[$0]) } + return (Array(records[..= 0x20 || scalar == "\t" || scalar == "\n" || scalar == "\r" diff --git a/apple/Plugins/FormatPluginTests/XLSXSheetRowsTests.swift b/apple/Plugins/FormatPluginTests/XLSXSheetRowsTests.swift new file mode 100644 index 0000000000..cc68d53a73 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/XLSXSheetRowsTests.swift @@ -0,0 +1,43 @@ +// +// XLSXSheetRowsTests.swift +// FormatPluginTests +// + +import Foundation +import Testing + +struct XLSXSheetRowsTests { + private typealias Chunk = XLSXSheetRows.Chunk + + @Test("One batch past several sheets fills each to the limit, header included") + func oneBatchFillsSeveralSheets() { + var sheets = XLSXSheetRows(limit: 5, headerRowCount: 1) + + #expect(sheets.chunks(of: 12) == [ + Chunk(rows: 0..<4, startsNewSheet: false), + Chunk(rows: 4..<8, startsNewSheet: true), + Chunk(rows: 8..<12, startsNewSheet: true), + ]) + } + + @Test("A batch that fills a sheet exactly starts the next sheet only when another row comes") + func exactFillStartsNoEmptySheet() { + var sheets = XLSXSheetRows(limit: 5, headerRowCount: 1) + + #expect(sheets.chunks(of: 4) == [Chunk(rows: 0..<4, startsNewSheet: false)]) + #expect(sheets.chunks(of: 0).isEmpty) + #expect(sheets.chunks(of: 1) == [Chunk(rows: 0..<1, startsNewSheet: true)]) + } + + @Test("Batches add up across calls, and without a header a sheet holds the whole limit") + func batchesAddUpWithoutHeader() { + var sheets = XLSXSheetRows(limit: 5, headerRowCount: 0) + + #expect(sheets.chunks(of: 3) == [Chunk(rows: 0..<3, startsNewSheet: false)]) + #expect(sheets.chunks(of: 9) == [ + Chunk(rows: 0..<2, startsNewSheet: false), + Chunk(rows: 2..<7, startsNewSheet: true), + Chunk(rows: 7..<9, startsNewSheet: true), + ]) + } +} diff --git a/apple/Plugins/FormatPluginTests/XLSXWriterStreamingTests.swift b/apple/Plugins/FormatPluginTests/XLSXWriterStreamingTests.swift index 1853d10d42..ee35e25170 100644 --- a/apple/Plugins/FormatPluginTests/XLSXWriterStreamingTests.swift +++ b/apple/Plugins/FormatPluginTests/XLSXWriterStreamingTests.swift @@ -12,8 +12,8 @@ struct XLSXWriterStreamingTests { private static let columns = ["id", "name", "score", "blob", "note"] private static let columnTypeNames = ["int4", "text", "float8", "bytea", "varchar(8)"] - // SHA-256 of the workbook below as the writer produced it while it built every sheet in memory. - private static let workbookDigest = "7602d732018a0d8b389690d553dd671460fcc4b3dfa53d7a9fff2dcd69a63440" + // SHA-256 of the workbook below; every flush threshold has to produce it. + private static let workbookDigest = "0e3615d4591e87affd46978b977d3122f9cf93f459873835a08080ae9048f333" private static let closingTags = "".utf8.count diff --git a/apple/Plugins/ParquetExportPlugin/ParquetTypeMapper.swift b/apple/Plugins/ParquetExportPlugin/ParquetTypeMapper.swift index 6ab10742af..c716dab7fa 100644 --- a/apple/Plugins/ParquetExportPlugin/ParquetTypeMapper.swift +++ b/apple/Plugins/ParquetExportPlugin/ParquetTypeMapper.swift @@ -25,6 +25,10 @@ public enum ParquetTypeMapper { if dateTypes.contains(base) { return enginesWhereDateCarriesTime.contains(databaseTypeId) ? "TIMESTAMP" : "DATE" } + // A cast to TIMESTAMP or TIME drops the offset, so 12:00+07 would read as 12:00 UTC. + if isZoned(typeName) || base == "timestamptz" || base == "timetz" { + return timeTypes.contains(base) ? "TIMETZ" : "TIMESTAMPTZ" + } if timestampTypes.contains(base) { return "TIMESTAMP" } if timeTypes.contains(base) { return "TIME" } if binaryTypes.contains(base) { return "BLOB" } @@ -49,6 +53,12 @@ public enum ParquetTypeMapper { words(typeName).contains("unsigned") } + // Oracle's `with local time zone` counts: the driver writes it with the session's offset. + private static func isZoned(_ typeName: String) -> Bool { + let words = words(typeName).filter { $0 != "local" } + return words.indices.dropLast(2).contains { words[$0] == "with" && words[$0 + 1] == "time" && words[$0 + 2] == "zone" } + } + // `bit` and `bit(1)` hold one flag. `bit(8)` and `bit varying` hold a string of them. private static func isSingleBit(_ typeName: String) -> Bool { guard !words(typeName).contains("varying") else { return false } diff --git a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift index f773973f4a..6ed1863eac 100644 --- a/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift +++ b/apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift @@ -55,10 +55,31 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug var isFirstBatch = true var rowBatch: [[PluginCellValue]] = [] - var currentSheetRowCount = 0 var columns: [String] = [] var columnTypeNames: [String] = [] - let headerRowCount = options.includeHeaderRow ? 1 : 0 + var sheetRows = XLSXSheetRows(limit: Self.maxRowsPerSheet, headerRowCount: options.includeHeaderRow ? 1 : 0) + + func writeBatch() throws { + for chunk in sheetRows.chunks(of: rowBatch.count) { + if chunk.startsNewSheet { + try writer.continueSheet( + baseName: table.name, + columns: columns, + columnTypeNames: columnTypeNames, + includeHeader: options.includeHeaderRow, + convertNullToEmpty: options.convertNullToEmpty + ) + didSplitSheets = true + } + try autoreleasepool { + try writer.addRows(Array(rowBatch[chunk.rows]), convertNullToEmpty: options.convertNullToEmpty) + } + } + for _ in 0..= 5_000 { - let remaining = Self.maxRowsPerSheet - currentSheetRowCount - if rowBatch.count <= remaining { - try autoreleasepool { - try writer.addRows(rowBatch, convertNullToEmpty: options.convertNullToEmpty) - } - currentSheetRowCount += rowBatch.count - } else { - let fitting = Array(rowBatch.prefix(remaining)) - let overflow = Array(rowBatch.dropFirst(remaining)) - if !fitting.isEmpty { - try autoreleasepool { - try writer.addRows(fitting, convertNullToEmpty: options.convertNullToEmpty) - } - currentSheetRowCount += fitting.count - } - try writer.continueSheet( - baseName: table.name, - columns: columns, - columnTypeNames: columnTypeNames, - includeHeader: options.includeHeaderRow, - convertNullToEmpty: options.convertNullToEmpty - ) - didSplitSheets = true - currentSheetRowCount = headerRowCount - if !overflow.isEmpty { - try autoreleasepool { - try writer.addRows(overflow, convertNullToEmpty: options.convertNullToEmpty) - } - currentSheetRowCount += overflow.count - } - } - let batchCount = rowBatch.count - rowBatch.removeAll(keepingCapacity: true) - for _ in 0.. + let startsNewSheet: Bool + } + + let limit: Int + let headerRowCount: Int + private(set) var rowsInSheet: Int + + init(limit: Int, headerRowCount: Int) { + self.limit = limit + self.headerRowCount = headerRowCount + self.rowsInSheet = headerRowCount + } + + // A new sheet starts only when a row needs it, so a batch that fills a sheet exactly leaves no empty one. + mutating func chunks(of count: Int) -> [Chunk] { + var chunks: [Chunk] = [] + var start = 0 + while start < count { + var startsNewSheet = false + if rowsInSheet >= limit { + rowsInSheet = headerRowCount + startsNewSheet = true + } + let end = min(count, start + max(1, limit - rowsInSheet)) + chunks.append(Chunk(rows: start.. ParameterizedStatement? { - let firstWritable = columns.first { !generatedColumns.contains($0) }.map(quoteIdentifierFn) - return AllDefaultsInsert.sql( - into: qualifiedTableName, databaseType: databaseType, firstWritableColumn: firstWritable + allDefaultsInsertStatement(firstColumn: columns.first { !generatedColumns.contains($0) }) + } + + // For a generator built without the table's columns, the caller names the one Oracle needs. + func allDefaultsInsertStatement(firstColumn: String?) -> ParameterizedStatement? { + AllDefaultsInsert.sql( + into: qualifiedTableName, databaseType: databaseType, firstWritableColumn: firstColumn.map(quoteIdentifierFn) ).map { ParameterizedStatement(sql: $0, parameters: []) } } diff --git a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift index cb350fb44f..a4a260d168 100644 --- a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift +++ b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift @@ -31,6 +31,8 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { private let sourceFieldsPerFoldedName: [String: Int] private let rowGenerator: SQLStatementGenerator? private let typedRowGenerator = OSAllocatedUnfairLock(uncheckedState: nil) + private let targetValues = OSAllocatedUnfairLock(initialState: nil) + private let binaryText: ImportBinaryText? private let rowWriting: ImportRowWriting private let tally: OSAllocatedUnfairLock private let immutableColumns = OSAllocatedUnfairLock?>(initialState: nil) @@ -71,6 +73,7 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { columnMapping: [String: String] = [:], sourceFields: Set = [], rowWriting: ImportRowWriting = .insert, + binaryText: ImportBinaryText? = nil, joinsTransaction: Bool = false, importRun: ImportRun? = nil, fileIndex: Int? = 0, @@ -82,6 +85,7 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { } self.isCancelled = isCancelled self.rowWriting = rowWriting + self.binaryText = binaryText self.tally = OSAllocatedUnfairLock(initialState: ImportRowTally(mode: rowWriting.mode)) self.joinsTransaction = joinsTransaction self.driver = driver @@ -131,21 +135,32 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { return rowGenerator } if let typed = typedRowGenerator.withLockUnchecked({ $0 }) { return typed } - let columns = try await importedData { - try await self.driver.fetchColumns(table: targetTable, schema: self.targetSchema) - } - var typeNames: [String: String] = [:] - for column in columns where typeNames[column.name] == nil { - typeNames[column.name] = column.typeNameForClassification - } let typed = Self.makeRowGenerator( table: targetTable, schema: targetSchema, databaseType: databaseType, driver: driver, - columnTypeNames: typeNames + columnTypeNames: try await targetColumnValues().typeNames ) ?? rowGenerator typedRowGenerator.withLockUnchecked { $0 = typed } return typed } + // Read once per import, for the BIT binding above and for what a file's text means in each column. + private func targetColumnValues() async throws -> ImportTargetValues { + if let known = targetValues.withLock({ $0 }) { return known } + guard let targetTable else { + throw PluginImportError.importFailed("No target table configured for row import") + } + let columns = try await importedData { + try await self.driver.fetchColumns(table: targetTable, schema: self.targetSchema) + } + let values = ImportTargetValues( + columns: columns.map { (name: $0.name, typeName: $0.typeNameForClassification, isGenerated: $0.isGenerated) }, + databaseType: databaseType, + binaryText: binaryText + ) + targetValues.withLock { $0 = values } + return values + } + func execute(statement: String) async throws { try await execute(statement: statement, line: 1) } @@ -228,17 +243,15 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { throw PluginImportError.importFailed("Could not resolve SQL dialect for \(targetTable)") } - let row = mappedRow(values) - guard !row.columns.isEmpty else { - guard values.isEmpty else { - throw PluginImportError.importFailed(Self.unmappedRowMessage) - } - return - } + let row = try await writableRows([values])[0] guard !rowWriting.mode.matchesExistingRows else { try await writeMatching([row], generator: rowGenerator) return } + guard !row.columns.isEmpty else { + try await insertDefaultRows(1, generator: rowGenerator) + return + } guard let statement = rowGenerator.insertStatement(columns: row.columns, values: row.values) else { throw PluginImportError.importFailed( String(format: String(localized: "Could not build an INSERT for %@"), targetTable) @@ -259,11 +272,7 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { throw PluginImportError.importFailed("Could not resolve SQL dialect for row import") } - let mapped = rows.map(mappedRow) - if zip(rows, mapped).contains(where: { !$0.0.isEmpty && $0.1.columns.isEmpty }) { - throw PluginImportError.importFailed(Self.unmappedRowMessage) - } - let writable = mapped.filter { !$0.columns.isEmpty } + let writable = try await writableRows(rows) if rowWriting.mode.matchesExistingRows { try await writeMatching(writable, generator: rowGenerator) @@ -277,9 +286,17 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { let values: [PluginCellValue] } - private func mappedRow(_ values: [String: PluginCellValue]) -> MappedRow { - let (columns, mappedValues) = mappedColumnsAndValues(values) - return MappedRow(columns: columns, values: mappedValues) + // A row with no field, such as JSON's `{}` for an all-NULL row, is a row of defaults; a row whose fields all + // missed the mapping is an error. + private func writableRows(_ rows: [[String: PluginCellValue]]) async throws -> [MappedRow] { + let target = try await targetColumnValues() + return try rows.map { values in + let (columns, mappedValues) = mappedColumnsAndValues(values) + if columns.isEmpty, !values.isEmpty { + throw PluginImportError.importFailed(Self.unmappedRowMessage) + } + return MappedRow(columns: columns, values: try target.values(mappedValues, columns: columns)) + } } private func insertGrouped(_ rows: [MappedRow], generator: SQLStatementGenerator) async throws { @@ -294,7 +311,28 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { } index = next - try await insertGroup(groupValues, columns: columns, generator: generator) + if columns.isEmpty { + try await insertDefaultRows(groupValues.count, generator: generator) + } else { + try await insertGroup(groupValues, columns: columns, generator: generator) + } + } + } + + private func insertDefaultRows(_ count: Int, generator: SQLStatementGenerator) async throws { + let firstColumn = try await targetColumnValues().firstWritableColumn + guard let statement = generator.allDefaultsInsertStatement(firstColumn: firstColumn) else { + throw PluginImportError.importFailed(String( + format: String(localized: "This row has no values, and %@ has no way to insert a row of only defaults."), + databaseType.rawValue + )) + } + for _ in 0.. + private let numericBooleanColumns: Set + private let binaryText: ImportBinaryText? + + init( + columns: [(name: String, typeName: String, isGenerated: Bool)], + databaseType: DatabaseType, + binaryText: ImportBinaryText? + ) { + var typeNames: [String: String] = [:] + for column in columns where typeNames[column.name] == nil { + typeNames[column.name] = column.typeName + } + let classifier = ColumnTypeClassifier(family: SQLTypeFamily.of(databaseType)) + let types = typeNames.mapValues { classifier.classify(rawTypeName: $0) } + // MySQL strict mode refuses `true` for an integer flag, SQL Server refuses `yes` for a bit, and SQLite keeps either as text. + let writesNumericBooleans = (try? resolveSQLDialect(for: databaseType))?.booleanLiteralStyle == .numeric + self.typeNames = typeNames + self.firstWritableColumn = columns.first { !$0.isGenerated }?.name + self.binaryColumns = Set(types.filter { $0.value.isBlobType }.keys) + self.numericBooleanColumns = writesNumericBooleans ? Set(types.filter { $0.value.isBooleanType }.keys) : [] + self.binaryText = binaryText + } + + func values(_ values: [PluginCellValue], columns: [String]) throws -> [PluginCellValue] { + try zip(columns, values).map { column, value in + guard case .text(let text) = value else { return value } + if binaryColumns.contains(column) { + return try binaryValue(text, column: column) + } + if numericBooleanColumns.contains(column), let digit = Self.booleanDigit(text) { + return .text(digit) + } + return value + } + } + + // Text not in the form the export writes is bound as text, as it always was: PostgreSQL's own JSON writes bytea as + // `\x...`, which the server reads from text. + private func binaryValue(_ text: String, column: String) throws -> PluginCellValue { + switch binaryText { + case .prefixedHex: + guard BlobFormattingService.hasHexPrefix(text) else { return .text(text) } + guard let bytes = BlobFormattingService.shared.bytes(fromPrefixedHex: text) else { + throw PluginImportError.importFailed(String( + format: String(localized: "%@ is a binary column, and this value is not 0x followed by hex digits."), + column + )) + } + return .bytes(bytes) + case .base64: + return Data(base64Encoded: text).map { .bytes($0) } ?? .text(text) + case nil: + return .text(text) + } + } + + // CSV reads t, f, y and n as booleans too. + static func booleanDigit(_ text: String) -> String? { + let word = text.trimmingCharacters(in: .whitespaces) + switch PluginSQLLiteral.booleanSynonym(for: word) { + case .isTrue: return "1" + case .isFalse: return "0" + case nil: break + @unknown default: break + } + switch word.lowercased() { + case "t", "y": return "1" + case "f", "n": return "0" + default: return nil + } + } +} diff --git a/apple/TablePro/Core/Plugins/ImportTypeMapper.swift b/apple/TablePro/Core/Plugins/ImportTypeMapper.swift index 9f88679c97..c9bc57bd03 100644 --- a/apple/TablePro/Core/Plugins/ImportTypeMapper.swift +++ b/apple/TablePro/Core/Plugins/ImportTypeMapper.swift @@ -55,11 +55,13 @@ enum ImportTypeMapper { } } + // SQLite stores a boolean as the integer 1 or 0. `BOOLEAN` has NUMERIC affinity and tells the import and the grid + // that the column is one. private static func sqliteType(_ type: PluginImportFieldType) -> String { switch type { case .integer: return "INTEGER" case .real: return "REAL" - case .boolean: return "INTEGER" + case .boolean: return "BOOLEAN" case .json, .text: return "TEXT" @unknown default: return "TEXT" } diff --git a/apple/TablePro/Core/Services/Export/ImportService.swift b/apple/TablePro/Core/Services/Export/ImportService.swift index 3f0e86c491..bc6684bcff 100644 --- a/apple/TablePro/Core/Services/Export/ImportService.swift +++ b/apple/TablePro/Core/Services/Export/ImportService.swift @@ -501,6 +501,7 @@ final class ImportService: ObservableObject { targetTable: file.targetTable, columnMapping: file.columnMapping, sourceFields: file.sourceFields, + binaryText: ImportBinaryText(formatId: file.formatId), joinsTransaction: true, importRun: run, fileIndex: index, @@ -644,6 +645,7 @@ final class ImportService: ObservableObject { let sink = ImportDataSinkAdapter( driver: driver, databaseType: connection.type, targetTable: targetTable, columnMapping: columnMapping, sourceFields: sourceFields, rowWriting: rowWriting, + binaryText: ImportBinaryText(formatId: type(of: plugin).formatId), importRun: run, isCancelled: { progress.isCancelled } ) defer { diff --git a/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift b/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift index 5b2e70e303..b5e3db6c3a 100644 --- a/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift +++ b/apple/TablePro/Core/Services/Formatting/BlobFormattingService.swift @@ -22,8 +22,10 @@ final class BlobFormattingService: Sendable { func format(_ value: String, for context: BlobDisplayContext) -> String? { switch context { - case .grid, .copy: + case .grid: return value.formattedAsCompactHex() + case .copy: + return value.formattedAsCompactHex(maxBytes: .max) case .detail: return value.formattedAsHexDump() case .edit: @@ -63,6 +65,40 @@ final class BlobFormattingService: Sendable { return String(data: data, encoding: .isoLatin1) } + static func hasHexPrefix(_ text: String) -> Bool { + text.hasPrefix("0x") || text.hasPrefix("0X") + } + + // The form a copy, a CSV export and an XLSX export write bytes in, read strictly: a copy cut short with `…` + // or text that only starts like hex is not bytes. + func bytes(fromPrefixedHex text: String) -> Data? { + guard Self.hasHexPrefix(text) else { return nil } + let digits = text.utf8.dropFirst(2) + guard digits.count.isMultiple(of: 2) else { return nil } + var bytes = [UInt8]() + bytes.reserveCapacity(digits.count / 2) + var high: UInt8? + for character in digits { + guard let nibble = Self.nibble(character) else { return nil } + if let pending = high { + bytes.append(pending << 4 | nibble) + high = nil + } else { + high = nibble + } + } + return Data(bytes) + } + + private static func nibble(_ character: UInt8) -> UInt8? { + switch character { + case UInt8(ascii: "0")...UInt8(ascii: "9"): return character - UInt8(ascii: "0") + case UInt8(ascii: "a")...UInt8(ascii: "f"): return character - UInt8(ascii: "a") + 10 + case UInt8(ascii: "A")...UInt8(ascii: "F"): return character - UInt8(ascii: "A") + 10 + default: return nil + } + } + func requiresFormatting(columnType: ColumnType) -> Bool { columnType.isBlobType } diff --git a/apple/TablePro/Core/Services/Query/RowOperationsManager.swift b/apple/TablePro/Core/Services/Query/RowOperationsManager.swift index 2a33425139..558c0a2bff 100644 --- a/apple/TablePro/Core/Services/Query/RowOperationsManager.swift +++ b/apple/TablePro/Core/Services/Query/RowOperationsManager.swift @@ -360,7 +360,11 @@ final class RowOperationsManager { } if parser == nil, let cells = clipboardProvider.readGridCells(), cells.text == clipboardText { - let parsedRows = Self.rows(fromCopiedCells: cells, schema: schema, tableRows: tableRows) + guard let parsedRows = Self.readingCopiedHex( + Self.rows(fromCopiedCells: cells, schema: schema, tableRows: tableRows), tableRows: tableRows + ) else { + return PasteRowsResult(pastedRows: [], delta: .none) + } return insertParsedRows( keepsMissingFields ? parsedRows.map(Self.nullsAsMissingFields) : parsedRows, uuidKeyForms: uuidKeyForms, @@ -375,6 +379,9 @@ final class RowOperationsManager { case .success(let parsedRows): let unfilled = Self.unfilledCellValues(columns: columns, tableRows: tableRows) let filled = parsedRows.map { Self.fillingBlankCells($0, with: unfilled) } + guard let filled = Self.readingCopiedHex(filled, tableRows: tableRows) else { + return PasteRowsResult(pastedRows: [], delta: .none) + } return insertParsedRows( keepsMissingFields ? filled.map(Self.nullsAsMissingFields) : filled, uuidKeyForms: uuidKeyForms, @@ -452,6 +459,33 @@ final class RowOperationsManager { } } + // A copy writes bytes as `0x` hex, so a binary column takes back the bytes it spells. A `0x` field that is not hex, + // such as a copy cut short with `…`, refuses the paste, as it refuses a cell paste. + private static func readingCopiedHex(_ rows: [ParsedRow], tableRows: TableRows) -> [ParsedRow]? { + var read: [ParsedRow] = [] + read.reserveCapacity(rows.count) + for row in rows { + var values = row.values + for index in values.indices where !row.expressionColumns.contains(index) { + guard case .text(let text) = values[index], BlobFormattingService.hasHexPrefix(text), + tableRows.editingColumnType(at: index)?.isBlobType == true else { continue } + guard let bytes = BlobFormattingService.shared.bytes(fromPrefixedHex: text) else { + logger.warning("Paste refused: a binary column's field is not hex after 0x") + return nil + } + values[index] = .bytes(bytes) + } + read.append(ParsedRow( + values: values, + sourceLineNumber: row.sourceLineNumber, + absentColumns: row.absentColumns, + expressionColumns: row.expressionColumns, + blankColumns: row.blankColumns + )) + } + return read + } + // A blank cell of pasted text was left unfilled, so it takes what a new row's cell does: the server's value where // the column has a default, else NULL. A NOT NULL text column with no default takes the empty string it can hold. private static func unfilledCellValues( diff --git a/apple/TablePro/Core/Utilities/SQL/InClauseConverter.swift b/apple/TablePro/Core/Utilities/SQL/InClauseConverter.swift index aeb4e15d98..0c93948d3a 100644 --- a/apple/TablePro/Core/Utilities/SQL/InClauseConverter.swift +++ b/apple/TablePro/Core/Utilities/SQL/InClauseConverter.swift @@ -20,6 +20,12 @@ internal struct InClauseConverter { // The session's escapes, for when no driver escaper is passed. internal var stringEscapes: SQLStringEscapes = .standard + // SQL Server has no TRUE or FALSE: `flag IN (TRUE)` reads TRUE as a column name. + internal var booleanLiteralStyle: SQLDialectDescriptor.BooleanLiteralStyle = .truefalse + + // Picks the binary literal's spelling; without one it is the standard X'...'. + internal var databaseType: DatabaseType? + private static let maxRows = 50_000 func generateInClause(rows: [[PluginCellValue]]) -> String { @@ -39,21 +45,28 @@ internal struct InClauseConverter { private func format(cell: PluginCellValue, type: ColumnType) -> String? { switch cell { - case .null, .bytes: + case .null: return nil + case .bytes(let data): + return binaryLiteral(data) case .text(let value): return formatScalar(value, type: type) } } + private func binaryLiteral(_ data: Data) -> String { + databaseType.flatMap { CompareSQLLiteral.binaryLiteral(for: data, databaseType: $0) } ?? "X'\(data.uppercaseHex())'" + } + private func formatScalar(_ value: String, type: ColumnType) -> String { if type.isBooleanType { guard let synonym = ColumnTypeSQLQuoting.booleanSynonym(for: value) else { return quoted(value) } + let isNumeric = booleanLiteralStyle == .numeric switch synonym { case .isTrue: - return "TRUE" + return isNumeric ? "1" : "TRUE" case .isFalse: - return "FALSE" + return isNumeric ? "0" : "FALSE" @unknown default: return quoted(value) } diff --git a/apple/TablePro/Core/Utilities/SQL/RowValueCopyFormatter.swift b/apple/TablePro/Core/Utilities/SQL/RowValueCopyFormatter.swift index d337c97ebf..dc9896b0f4 100644 --- a/apple/TablePro/Core/Utilities/SQL/RowValueCopyFormatter.swift +++ b/apple/TablePro/Core/Utilities/SQL/RowValueCopyFormatter.swift @@ -13,11 +13,11 @@ internal enum RowValueCopyFormatter { return nil case .text(let value): if columnType?.isBlobType ?? false { - return value.formattedAsCompactHex() ?? value + return BlobFormattingService.shared.format(value, for: .copy) ?? value } return value case .bytes(let data): - return String(data: data, encoding: .isoLatin1)?.formattedAsCompactHex() ?? "" + return BlobFormattingService.shared.format(data, for: .copy) ?? "" } } diff --git a/apple/TablePro/Extensions/String+HexDump.swift b/apple/TablePro/Extensions/String+HexDump.swift index 60f9a6fba7..9854119ebd 100644 --- a/apple/TablePro/Extensions/String+HexDump.swift +++ b/apple/TablePro/Extensions/String+HexDump.swift @@ -118,28 +118,10 @@ extension String { return hex } - /// Returns a compact single-line hex representation for data grid cells. - /// - /// Format: `0x48656C6C6F` for short values, truncated with `…` for longer ones. - /// - Parameter maxBytes: Maximum bytes to show before truncating (default 64). func formattedAsCompactHex(maxBytes: Int = 64) -> String? { let bytes = storedBytes - - let totalCount = bytes.count - guard totalCount > 0 else { return nil } - - let displayCount = min(totalCount, maxBytes) - let bytesArray = [UInt8](bytes.prefix(displayCount)) - - var hex = "0x" - for byte in bytesArray { - hex += String(format: "%02X", byte) - } - - if totalCount > maxBytes { - hex += "…" - } - - return hex + guard !bytes.isEmpty else { return nil } + let hex = "0x" + bytes.uppercaseHex(maxBytes: maxBytes) + return bytes.count > maxBytes ? hex + "…" : hex } } diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift index fd99df077b..95da9f0384 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Copy.swift @@ -219,7 +219,9 @@ extension DataGridCommands { columnTypes: [column.type ?? .text(rawType: nil)], escapeStringLiteral: nil, stringLiteralPrefix: traits.stringLiteralPrefix, - stringEscapes: traits.databaseType.map { SQLLexicalResolver.keptTextStringEscapes(for: $0, session: session) } ?? .standard + stringEscapes: traits.databaseType.map { SQLLexicalResolver.keptTextStringEscapes(for: $0, session: session) } ?? .standard, + booleanLiteralStyle: traits.databaseType.flatMap { try? resolveSQLDialect(for: $0) }?.booleanLiteralStyle ?? .truefalse, + databaseType: traits.databaseType ).generateInClause(rows: values.map { [$0] }) guard !text.isEmpty else { return } ClipboardService.shared.writeText(text) diff --git a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Paste.swift b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Paste.swift index 33493a31ff..8b1971b7be 100644 --- a/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Paste.swift +++ b/apple/TablePro/Views/DataGrid/Commands/DataGridCommands+Paste.swift @@ -176,7 +176,7 @@ extension DataGridCommands { } // The grid's copy says which fields were SQL; any other field is text however it reads. Text from another app is - // read the way a typed cell is. + // read the way a typed cell is. A binary column takes `0x` hex as the bytes it spells, as the hex editor does. func pastedEntry( _ field: String?, of block: DataGridCellPasteBlock, @@ -184,6 +184,10 @@ extension DataGridCommands { column: DataGridColumn ) -> StagedCellEntry? { let rows = model.rows + if !isExpression, let field, BlobFormattingService.hasHexPrefix(field), + rows.editingColumnType(at: column.dataIndex)?.isBlobType == true { + return BlobFormattingService.shared.bytes(fromPrefixedHex: field).map { (value: .bytes($0), isExpression: false) } + } guard block.isGridCopy else { let text = field.flatMap { DataGridCellPasteBlock.pastedText($0, columnType: rows.editingColumnType(at: column.dataIndex)) diff --git a/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift b/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift index 7e4e542b48..2d04f2fd59 100644 --- a/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift +++ b/apple/TablePro/Views/Main/Child/DataTabSQLExporter.swift @@ -123,7 +123,9 @@ final class DataTabSQLExporter: DataGridSQLExporting { columnTypes: [column.type ?? .text(rawType: nil)], escapeStringLiteral: driver?.escapeStringLiteral, stringLiteralPrefix: SQLStringLiteralPrefix.forDatabaseType(databaseType), - stringEscapes: SQLLexicalResolver.stringEscapes(for: databaseType, session: driver?.sessionLexicalState) + stringEscapes: SQLLexicalResolver.stringEscapes(for: databaseType, session: driver?.sessionLexicalState), + booleanLiteralStyle: (try? resolveSQLDialect(for: databaseType))?.booleanLiteralStyle ?? .truefalse, + databaseType: databaseType ) return converter.generateInClause(rows: values.map { [$0] }) } diff --git a/apple/TablePro/Views/Main/Extensions/MainContentView+Helpers.swift b/apple/TablePro/Views/Main/Extensions/MainContentView+Helpers.swift index 41deb3a3e6..9d2201e6c2 100644 --- a/apple/TablePro/Views/Main/Extensions/MainContentView+Helpers.swift +++ b/apple/TablePro/Views/Main/Extensions/MainContentView+Helpers.swift @@ -133,7 +133,8 @@ extension MainContentView { case .text(let s): raw = s case .bytes(let data): - raw = BlobFormattingService.shared.format(data, for: .copy) ?? "" + // The grid's form stops at 64 bytes, under the cut below; a full copy would hex the whole value. + raw = BlobFormattingService.shared.format(data, for: .grid) ?? "" } return (raw as NSString).length > 200 ? String(raw.prefix(200)) + "…" : raw } diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift index 6af9834a0a..07307f438b 100644 --- a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift @@ -101,13 +101,16 @@ struct ImportDataSinkAdapterBitColumnTests { #expect(sql.last == "UPDATE `devices` SET `name` = ? WHERE `flags` = CAST(? AS UNSIGNED)") } + // The file's rows still read the table's types once, for its binary and boolean columns. @Test - func anEngineThatReadsBitsAsDigitsIsNotAsked() async throws { + func anEngineThatReadsBitsAsDigitsGetsNoCast() async throws { let driver = BitTableDriver() + let target = sink(on: driver, type: .postgresql) - try await sink(on: driver, type: .postgresql).insertRows([["id": .text("1"), "flags": .text("101")]]) + try await target.insertRows([["id": .text("1"), "flags": .text("101")]]) + try await target.insertRows(columns: ["id", "flags"], values: [[.text("2"), .text("110")]]) - #expect(driver.columnFetches == 0) - #expect(driver.statements.first?.sql.contains("CAST") == false) + #expect(driver.columnFetches == 1) + #expect(driver.statements.allSatisfy { !$0.sql.contains("CAST") }) } } diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterMappingTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterMappingTests.swift index c7661521e8..4875bdacd6 100644 --- a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterMappingTests.swift +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterMappingTests.swift @@ -132,9 +132,8 @@ struct ImportDataSinkAdapterMappingTests { #expect(columns.isEmpty) } - /// A row carrying nothing has nothing to lose, so it passes through. Only a row holding values - /// that reach no column is worth stopping for, and conflating the two would turn an empty - /// object in an NDJSON file into a failed import. + // An empty object is how JSON export writes an all-NULL row without its NULLs, so it is written as a row of + // defaults. Only a row holding values that reach no column is an error. @Test("A row with no values at all is not an error") func emptyRowIsNotAnError() async throws { let sink = adapter(mapping: ["name": "name"]) diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterTargetValueTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterTargetValueTests.swift new file mode 100644 index 0000000000..348e6cc792 --- /dev/null +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterTargetValueTests.swift @@ -0,0 +1,204 @@ +// +// ImportDataSinkAdapterTargetValueTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct WrittenStatement { + let sql: String + let parameters: [PluginCellValue] +} + +private final class TypedTableDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private let columns: [PluginColumnInfo] + private var written: [WrittenStatement] = [] + + init(columns: [(String, String)], generated: Set = []) { + self.columns = columns.map { PluginColumnInfo(name: $0.0, dataType: $0.1, isGenerated: generated.contains($0.0)) } + } + + var statements: [WrittenStatement] { lock.withLock { written } } + + func quoteIdentifier(_ name: String) -> String { "\"\(name)\"" } + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { columns } + + func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + lock.withLock { written.append(WrittenStatement(sql: query, parameters: parameters)) } + return .empty + } +} + +// A file holds text; what it stands for depends on the column it lands in. +struct ImportDataSinkAdapterTargetValueTests { + private func sink( + _ driver: TypedTableDriver, + type: DatabaseType, + mapping: [String], + binaryText: ImportBinaryText? = nil, + rowWriting: ImportRowWriting = .insert + ) -> ImportDataSinkAdapter { + ImportDataSinkAdapter( + driver: PluginDriverAdapter(connection: TestFixtures.makeConnection(type: type), pluginDriver: driver), + databaseType: type, + targetTable: "t", + columnMapping: Dictionary(uniqueKeysWithValues: mapping.map { ($0, $0) }), + sourceFields: Set(mapping), + rowWriting: rowWriting, + binaryText: binaryText + ) + } + + @Test("CSV and XLSX hex lands in a binary column as its bytes, and only there") + func prefixedHexBecomesBytes() async throws { + let driver = TypedTableDriver(columns: [("payload", "bytea"), ("note", "text")]) + let target = sink(driver, type: .postgresql, mapping: ["payload", "note"], binaryText: .prefixedHex) + + try await target.insertRows([ + ["payload": .text("0x89504E47"), "note": .text("0x41")], + ["payload": .text("0x"), "note": .text("x")], + ["payload": .text("plain"), "note": .text("y")], + ]) + + let parameters = try #require(driver.statements.first).parameters + #expect(parameters == [ + .text("0x41"), .bytes(Data([0x89, 0x50, 0x4E, 0x47])), + .text("x"), .bytes(Data()), + .text("y"), .text("plain"), + ]) + } + + @Test("Hex cut short fails the row and names the column") + func brokenHexFailsTheRow() async throws { + let driver = TypedTableDriver(columns: [("payload", "blob")]) + let target = sink(driver, type: .mysql, mapping: ["payload"], binaryText: .prefixedHex) + + let error = await #expect(throws: PluginImportError.self) { + try await target.insertRow(["payload": .text("0x41…")]) + } + #expect(error?.localizedDescription.contains("payload") == true) + #expect(driver.statements.isEmpty) + } + + @Test("JSON's base64 lands in a binary column as its bytes, and other text stays text") + func base64BecomesBytes() async throws { + let driver = TypedTableDriver(columns: [("payload", "bytea")]) + let target = sink(driver, type: .postgresql, mapping: ["payload"], binaryText: .base64) + + try await target.insertRow(["payload": .text(Data([0x00, 0xFF]).base64EncodedString())]) + try await target.insertRow(["payload": .text("hello world")]) + try await target.insertRow(["payload": .text("\\x4142")]) + + #expect(driver.statements.map { $0.parameters } == [ + [.bytes(Data([0x00, 0xFF]))], [.text("hello world")], [.text("\\x4142")], + ]) + } + + @Test("A format with no binary spelling, and rows in column order, keep the text") + func noBinaryTextKeepsText() async throws { + let driver = TypedTableDriver(columns: [("payload", "bytea")]) + + try await sink(driver, type: .postgresql, mapping: ["payload"]).insertRow(["payload": .text("0x41")]) + try await sink(driver, type: .postgresql, mapping: ["payload"], binaryText: .prefixedHex) + .insertRows(columns: ["payload"], values: [[.text("0x41")]]) + + #expect(driver.statements.map { $0.parameters } == [[.text("0x41")], [.text("0x41")]]) + } + + @Test( + "A boolean word becomes 1 or 0 where the engine writes booleans as numbers", + arguments: [ + (DatabaseType.mysql, "tinyint(1)"), + (DatabaseType.mssql, "bit"), + (DatabaseType.sqlite, "BOOLEAN"), + ] + ) + func booleanWordsBecomeDigits(type: DatabaseType, column: String) async throws { + let driver = TypedTableDriver(columns: [("active", column), ("label", "varchar(10)")]) + let target = sink(driver, type: type, mapping: ["active", "label"]) + + try await target.insertRows(["true", "No", "T", "y", "1", " off ", "maybe"].map { word in + ["active": .text(word), "label": .text(word)] + }) + + let parameters = try #require(driver.statements.first).parameters + let active = stride(from: 0, to: parameters.count, by: 2).map { parameters[$0] } + let labels = stride(from: 1, to: parameters.count, by: 2).map { parameters[$0] } + #expect(active == ["1", "0", "1", "1", "1", "0", "maybe"]) + #expect(labels == ["true", "No", "T", "y", "1", " off ", "maybe"]) + } + + @Test("An engine that reads TRUE and FALSE gets the file's own words") + func trueFalseEngineKeepsWords() async throws { + let driver = TypedTableDriver(columns: [("active", "boolean")]) + + try await sink(driver, type: .postgresql, mapping: ["active"]).insertRow(["active": .text("yes")]) + + #expect(driver.statements.first?.parameters == [.text("yes")]) + } + + @Test("A row with no field is written as a row of defaults and counted") + func emptyRowIsARowOfDefaults() async throws { + let driver = TypedTableDriver(columns: [("id", "serial"), ("note", "text")]) + let target = sink(driver, type: .postgresql, mapping: ["note"]) + + try await target.insertRows([["note": .text("a")], [:], [:]]) + try await target.insertRow([:]) + + #expect(driver.statements.map { $0.sql } == [ + "INSERT INTO \"t\" (\"note\") VALUES ($1)", + "INSERT INTO \"t\" DEFAULT VALUES", + "INSERT INTO \"t\" DEFAULT VALUES", + "INSERT INTO \"t\" DEFAULT VALUES", + ]) + #expect(target.rowTally.inserted == 4) + } + + @Test("Oracle's row of defaults names the first column that is not virtual") + func oracleDefaultsNameAColumn() async throws { + let driver = TypedTableDriver(columns: [("FULL_NAME", "VARCHAR2(41)"), ("ID", "NUMBER(19)")], generated: ["FULL_NAME"]) + + try await sink(driver, type: .oracle, mapping: ["ID"]).insertRow([:]) + + #expect(driver.statements.map { $0.sql } == ["INSERT INTO \"t\" (\"ID\") VALUES (DEFAULT)"]) + } + + @Test("An engine with no INSERT for a row of defaults fails the row and says why") + func noDefaultsInsertSaysWhy() async throws { + let driver = TypedTableDriver(columns: [("id", "UInt64")]) + + let error = await #expect(throws: PluginImportError.self) { + try await sink(driver, type: .clickhouse, mapping: ["id"]).insertRow([:]) + } + #expect(error?.localizedDescription.contains("row of only defaults") == true) + } + + @Test("A row with no field cannot be matched to an existing row") + func emptyRowHasNoKeyToMatch() async throws { + let driver = TypedTableDriver(columns: [("id", "int"), ("note", "text")]) + let target = sink( + driver, type: .postgresql, mapping: ["id", "note"], + rowWriting: ImportRowWriting(mode: .upsert, keyColumns: ["id"]) + ) + + await #expect(throws: PluginImportError.self) { + try await target.insertRows([[:]]) + } + #expect(driver.statements.isEmpty) + } +} + +struct ImportBinaryTextTests { + @Test("Each format reads bytes back the way its export wrote them") + func formatIds() { + #expect(ImportBinaryText(formatId: "csv") == .prefixedHex) + #expect(ImportBinaryText(formatId: "xlsx") == .prefixedHex) + #expect(ImportBinaryText(formatId: "json") == .base64) + #expect(ImportBinaryText(formatId: "sql") == nil) + } +} diff --git a/apple/TableProTests/Core/Plugins/ImportTypeMapperTests.swift b/apple/TableProTests/Core/Plugins/ImportTypeMapperTests.swift index 5e78070475..a556b4c497 100644 --- a/apple/TableProTests/Core/Plugins/ImportTypeMapperTests.swift +++ b/apple/TableProTests/Core/Plugins/ImportTypeMapperTests.swift @@ -53,6 +53,7 @@ struct ImportTypeMapperTests { #expect(ImportTypeMapper.sqlType(for: .integer, databaseType: .sqlite) == "INTEGER") #expect(ImportTypeMapper.sqlType(for: .real, databaseType: .sqlite) == "REAL") #expect(ImportTypeMapper.sqlType(for: .json, databaseType: .sqlite) == "TEXT") + #expect(ImportTypeMapper.sqlType(for: .boolean, databaseType: .sqlite) == "BOOLEAN") } @Test("Oracle gets types it has, never TEXT") diff --git a/apple/TableProTests/Core/Services/BlobFormattingServiceTests.swift b/apple/TableProTests/Core/Services/BlobFormattingServiceTests.swift index 011bea2ec7..62a28411b1 100644 --- a/apple/TableProTests/Core/Services/BlobFormattingServiceTests.swift +++ b/apple/TableProTests/Core/Services/BlobFormattingServiceTests.swift @@ -57,6 +57,32 @@ struct BlobFormattingServiceCompactHexTests { } } +struct BlobFormattingServiceCopyTests { + @Test("A copy holds every byte, where the grid shows 64") + func copyIsTheWholeValue() { + let value = String(data: Data(repeating: 0xAB, count: 100), encoding: .isoLatin1) ?? "" + let copied = BlobFormattingService.shared.format(value, for: .copy) + #expect(copied == "0x" + String(repeating: "AB", count: 100)) + #expect(BlobFormattingService.shared.format(value, for: .grid)?.hasSuffix("…") == true) + } + + @Test("The copied form reads back as the bytes it spells") + func copiedHexReadsBack() { + let service = BlobFormattingService.shared + #expect(service.bytes(fromPrefixedHex: "0x4142") == Data([0x41, 0x42])) + #expect(service.bytes(fromPrefixedHex: "0X00ff") == Data([0x00, 0xFF])) + #expect(service.bytes(fromPrefixedHex: "0x") == Data()) + let bytes = Data(0...255) + let copied = service.format(bytes, for: .copy) ?? "" + #expect(service.bytes(fromPrefixedHex: copied) == bytes) + } + + @Test("Text that only starts like hex is not bytes", arguments: ["4142", "0x414", "0x41…", "0xZZ", "0x41 42", ""]) + func notCopiedHex(_ text: String) { + #expect(BlobFormattingService.shared.bytes(fromPrefixedHex: text) == nil) + } +} + @MainActor struct BlobFormattingServiceByteCountTests { @Test("Issue #1188 exact value reports 48 bytes (not 98)") diff --git a/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift b/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift index 51e4d37d67..c0d6f36739 100644 --- a/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift +++ b/apple/TableProTests/Core/Services/RowOperationsManagerPasteTests.swift @@ -85,6 +85,45 @@ struct RowOperationsManagerPasteTests { #expect(rows.first?[2] == .null) } + @Test("A binary column takes back the bytes a copy wrote as hex, from the grid's cells and from text") + func copiedHexPastesAsBytes() { + var types: [ColumnType] = Array(repeating: .text(rawType: nil), count: Self.columns.count) + types[2] = .blob(rawType: "varbinary(16)") + let manager = makeManager() + + let cells = RecordingClipboard() + cells.writeCells(GridCellsClipboardPayload(text: "7\tAnn\t0x4142", fields: [["7", "Ann", "0x4142"]])) + var rows = TableRows.from(queryRows: [], columns: Self.columns, columnTypes: types) + let fromCells = manager.pasteRowsFromClipboard( + columns: Self.columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: cells + ) + #expect(fromCells.pastedRows.first?.values[2] == .bytes(Data([0x41, 0x42]))) + + let text = RecordingClipboard() + text.text = "7\t0x41\t0x00FF\n8\tBo\t0x" + let fromText = manager.pasteRowsFromClipboard( + columns: Self.columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: text + ) + #expect(fromText.pastedRows.map { $0.values[1] } == [.text("0x41"), .text("Bo")]) + #expect(fromText.pastedRows.map { $0.values[2] } == [.bytes(Data([0x00, 0xFF])), .bytes(Data())]) + } + + @Test("A binary field that starts with 0x but is not hex, such as a copy cut short, pastes nothing") + func brokenHexRefusesThePaste() { + var types: [ColumnType] = Array(repeating: .text(rawType: nil), count: Self.columns.count) + types[2] = .blob(rawType: "varbinary(16)") + var rows = TableRows.from(queryRows: [], columns: Self.columns, columnTypes: types) + let clipboard = RecordingClipboard() + clipboard.text = "7\tAnn\t0x4142\n8\tBo\t0x41…" + + let result = makeManager().pasteRowsFromClipboard( + columns: Self.columns, primaryKeyColumns: ["id"], tableRows: &rows, clipboard: clipboard + ) + + #expect(result.pastedRows.isEmpty) + #expect(rows.count == 0) + } + @Test("Copied cells stale against the clipboard's text are not used") func staleCopiedCellsAreIgnored() { let clipboard = RecordingClipboard() diff --git a/apple/TableProTests/Core/Utilities/InClauseConverterTests.swift b/apple/TableProTests/Core/Utilities/InClauseConverterTests.swift index 5d3279d1d6..ca411b21b4 100644 --- a/apple/TableProTests/Core/Utilities/InClauseConverterTests.swift +++ b/apple/TableProTests/Core/Utilities/InClauseConverterTests.swift @@ -135,4 +135,32 @@ struct InClauseConverterTests { let result = converter.generateInClause(rows: [["alice", "1"], ["bob", "2"]]) #expect(result == "(1, 2)") } + + @Test("An engine without TRUE and FALSE gets 1 and 0") + func numericBooleans() { + var converter = makeConverter(columnIndex: 0, columnTypes: [.boolean(rawType: "bit")]) + converter.booleanLiteralStyle = .numeric + #expect(converter.generateInClause(rows: [["1"], ["false"], ["yes"]]) == "(1, 0, 1)") + } + + @Test("Binary values are written as the engine's binary literal, not dropped") + func binaryValues() { + var converter = makeConverter(columnIndex: 0, columnTypes: [.blob(rawType: "BINARY(16)")]) + converter.databaseType = .mysql + let rows: [[PluginCellValue]] = [[.bytes(Data([0x41, 0x42]))], [.bytes(Data([0x00]))], [.null]] + #expect(converter.generateInClause(rows: rows) == "(X'4142', X'00')") + + converter.databaseType = .mssql + #expect(converter.generateInClause(rows: rows) == "(0x4142, 0x00)") + + converter.databaseType = .postgresql + #expect(converter.generateInClause(rows: [[.bytes(Data([0x41]))]]) == "(decode('41', 'hex'))") + } + + // SQLite keeps text in a BLOB column, and text never equals a blob there. + @Test("Text in a binary column stays text, and with no engine bytes take the standard literal") + func binaryColumnTextWithoutEngine() { + let converter = makeConverter(columnIndex: 0, columnTypes: [.blob(rawType: "blob")]) + #expect(converter.generateInClause(rows: [["AB"], [.bytes(Data([0xFF]))]]) == "('AB', X'FF')") + } } diff --git a/apple/TableProTests/Views/Results/Extensions/DataGridSetCellsToValueTests.swift b/apple/TableProTests/Views/Results/Extensions/DataGridSetCellsToValueTests.swift index c766504cd9..3a41435a3b 100644 --- a/apple/TableProTests/Views/Results/Extensions/DataGridSetCellsToValueTests.swift +++ b/apple/TableProTests/Views/Results/Extensions/DataGridSetCellsToValueTests.swift @@ -290,6 +290,44 @@ extension GlobalStateSuites { #expect(grid.commands.pasteAnchor(.current(grid.model.selection), block: single) == GridCoord(row: 0, displayColumn: Fixture.note)) } + @Test("A binary value copies whole and pastes back as the same bytes") + func binaryValueRoundTrips() throws { + let clipboard = try #require(RecordingClipboard.current) + let grid = Fixture.grid() + let bytes = Data((0..<100).map { UInt8($0) }) + _ = grid.owner.tableRows.edit(row: 0, column: Fixture.payload, value: .bytes(bytes)) + grid.model.reloadRows() + grid.selectCell(0, Fixture.payload) + grid.controller.copy(nil) + #expect(clipboard.text == "0x" + bytes.map { String(format: "%02X", $0) }.joined()) + + grid.selectCell(1, Fixture.payload) + grid.controller.paste(nil) + + #expect(grid.owner.staged(1, Fixture.payload)?.value == .bytes(bytes)) + } + + @Test("Hex from another app pastes into a binary column as bytes, and a cut-short copy is refused") + func foreignHexIntoBinaryColumn() throws { + let clipboard = try #require(RecordingClipboard.current) + let grid = Fixture.grid() + + clipboard.text = "0x4142" + grid.selectCell(1, Fixture.payload) + grid.controller.paste(nil) + #expect(grid.owner.staged(1, Fixture.payload)?.value == .bytes(Data([0x41, 0x42]))) + + clipboard.text = "0x41…" + grid.selectCell(2, Fixture.payload) + grid.controller.paste(nil) + #expect(grid.owner.staged(2, Fixture.payload) == nil) + + clipboard.text = "0x41" + grid.selectCell(2, Fixture.note) + grid.controller.paste(nil) + #expect(grid.owner.staged(2, Fixture.note)?.value == .text("0x41")) + } + @Test("Paste is enabled over a cell selection with no cursor when one value is on the clipboard") func pasteIsEnabledWithoutACursor() throws { try #require(RecordingClipboard.current).text = "archived" diff --git a/apple/scripts/probes/check-parquet-export-values.sh b/apple/scripts/probes/check-parquet-export-values.sh index 966b46b5bd..8036d34fe2 100755 --- a/apple/scripts/probes/check-parquet-export-values.sh +++ b/apple/scripts/probes/check-parquet-export-values.sh @@ -5,8 +5,8 @@ # The plugin links libduckdb.a, so no test bundle can hold its staging database. This compiles # DuckDBStagingDatabase.swift against the shipped library, stages a table holding the values the # export used to change (binary, a zero-length blob, bit(8), BIGINT UNSIGNED above 2^63, a TIME -# past 24 hours, an Oracle DATE with a time, text in an integer column), writes the file and reads -# it back. +# past 24 hours, an Oracle DATE with a time, text in an integer column, timestamps and a time +# with a UTC offset), writes the file and reads it back. # # Usage: # scripts/probes/check-parquet-export-values.sh @@ -74,7 +74,12 @@ enum ParquetValuesProbe { ("seen", "DATE", "Oracle"), ("count", "int", "SQLite"), ("day", "date", "PostgreSQL"), - ("note", "text", "PostgreSQL") + ("note", "text", "PostgreSQL"), + ("zoned", "timestamp with time zone", "PostgreSQL"), + ("instant", "timestamptz", "PostgreSQL"), + ("clock", "time with time zone", "PostgreSQL"), + ("oracle_tz", "TIMESTAMP(6) WITH TIME ZONE", "Oracle"), + ("oracle_ltz", "TIMESTAMP(6) WITH LOCAL TIME ZONE", "Oracle") ] let types = declared.map { ParquetTypeMapper.duckDBType(forColumnType: $0.type, databaseTypeId: $0.engine) } @@ -83,15 +88,21 @@ enum ParquetValuesProbe { try staging.insert(rows: [ [ .text("1"), .bytes(Data([0x41, 0x42, 0x43])), .text("10101010"), .text("18446744073709551615"), - .text("100:00:00"), .text("2024-01-01 12:30:00"), .text("7"), .text("2024-03-01"), .text("it's") + .text("100:00:00"), .text("2024-01-01 12:30:00"), .text("7"), .text("2024-03-01"), .text("it's"), + .text("2024-01-01 12:00:00+07"), .text("2024-10-27 02:30:00+02"), .text("12:00:00+07"), + .text("2026-10-06 15:22:20.000001-05:30"), .text("2026-10-06 15:22:20.000001+07:00") ], [ .text("2"), .bytes(Data()), .text("1"), .text("5"), - .text("10:00:00"), .text("2024-01-02 00:00:00"), .text("seven"), .text("2024-03-02"), .text("") + .text("10:00:00"), .text("2024-01-02 00:00:00"), .text("seven"), .text("2024-03-02"), .text(""), + .null, .text("2024-10-27 02:30:00+01"), .null, .null, .null ] ]) try staging.insert(rows: [ - [.text("3"), .bytes(Data([0x00, 0xFF, 0x5C, 0x78])), .null, .null, .null, .null, .null, .null, .null] + [ + .text("3"), .bytes(Data([0x00, 0xFF, 0x5C, 0x78])), .null, .null, .null, .null, .null, .null, .null, + .null, .null, .null, .null, .null + ] ]) let fallbacks = try staging.copyToParquet(fileURL: file, compression: "SNAPPY", rowGroupSize: 122_880) @@ -106,7 +117,14 @@ enum ParquetValuesProbe { ("bit", "PostgreSQL", "BOOLEAN"), ("bit(1)", "MySQL", "BOOLEAN"), ("bit", "SQL Server", "BOOLEAN"), ("bit(8)", "MySQL", "VARCHAR"), ("bit varying(8)", "PostgreSQL", "VARCHAR"), ("int unsigned", "MySQL", "BIGINT"), ("BIGINT(20) UNSIGNED", "MySQL", "UBIGINT"), - ("bigint", "MySQL", "BIGINT"), ("date", "MySQL", "DATE"), ("DATE", "Oracle", "TIMESTAMP") + ("bigint", "MySQL", "BIGINT"), ("date", "MySQL", "DATE"), ("DATE", "Oracle", "TIMESTAMP"), + ("timestamp with time zone", "PostgreSQL", "TIMESTAMPTZ"), ("timestamptz", "PostgreSQL", "TIMESTAMPTZ"), + ("timestamp(3) with time zone", "PostgreSQL", "TIMESTAMPTZ"), + ("TIMESTAMP(6) WITH TIME ZONE", "Oracle", "TIMESTAMPTZ"), ("TIMESTAMP WITH TIME ZONE", "DuckDB", "TIMESTAMPTZ"), + ("time with time zone", "PostgreSQL", "TIMETZ"), ("timetz", "PostgreSQL", "TIMETZ"), + ("timestamp", "PostgreSQL", "TIMESTAMP"), ("timestamp without time zone", "PostgreSQL", "TIMESTAMP"), + ("time", "PostgreSQL", "TIME"), ("TIMESTAMP(6) WITH LOCAL TIME ZONE", "Oracle", "TIMESTAMPTZ"), + ("timestamp", "MySQL", "TIMESTAMP"), ("datetimeoffset", "SQL Server", "VARCHAR") ] for entry in mapped { expect( @@ -126,7 +144,9 @@ enum ParquetValuesProbe { let typeIds = try reader.columnTypes("SELECT * FROM \(source)") let expectedTypes: [duckdb_type] = [ DUCKDB_TYPE_BIGINT, DUCKDB_TYPE_BLOB, DUCKDB_TYPE_VARCHAR, DUCKDB_TYPE_UBIGINT, DUCKDB_TYPE_VARCHAR, - DUCKDB_TYPE_TIMESTAMP, DUCKDB_TYPE_VARCHAR, DUCKDB_TYPE_DATE, DUCKDB_TYPE_VARCHAR + DUCKDB_TYPE_TIMESTAMP, DUCKDB_TYPE_VARCHAR, DUCKDB_TYPE_DATE, DUCKDB_TYPE_VARCHAR, + DUCKDB_TYPE_TIMESTAMP_TZ, DUCKDB_TYPE_TIMESTAMP_TZ, DUCKDB_TYPE_TIME_TZ, DUCKDB_TYPE_TIMESTAMP_TZ, + DUCKDB_TYPE_TIMESTAMP_TZ ] for (index, column) in declared.enumerated() { expect("type of \(column.name)", "\(typeIds[index].rawValue)", "\(expectedTypes[index].rawValue)") @@ -149,6 +169,12 @@ enum ParquetValuesProbe { expect("quoted text", try cell("note", id: 1), "it's") expect("empty text", try cell("note = ''", id: 2), "true") expect("null", try cell("note IS NULL AND big IS NULL AND seen IS NULL", id: 3), "true") + expect("timestamp with time zone in UTC", try cell("timezone('UTC', zoned)", id: 1), "2024-01-01 05:00:00") + expect("first 02:30 of the fall-back hour", try cell("timezone('UTC', instant)", id: 1), "2024-10-27 00:30:00") + expect("second 02:30 of the fall-back hour", try cell("timezone('UTC', instant)", id: 2), "2024-10-27 01:30:00") + expect("time with time zone in UTC", try cell("clock", id: 1), "05:00:00+00") + expect("Oracle timestamp with time zone", try cell("timezone('UTC', oracle_tz)", id: 1), "2026-10-06 20:52:20.000001") + expect("Oracle timestamp with local time zone", try cell("timezone('UTC', oracle_ltz)", id: 1), "2026-10-06 08:22:20.000001") if failures > 0 { exit(1) } print("PASS: the Parquet file holds every staged value") From 57b7e1599c7e4c1ec6735523f4a1b95c458f3d70 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 12 Nov 2026 07:27:33 +0700 Subject: [PATCH 2/3] chore(l10n): add the import error strings to the catalog --- apple/TablePro/Resources/Localizable.xcstrings | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 59ea7fab79..bd91434d45 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -3709,6 +3709,9 @@ } } } + }, + "%@ is a binary column, and this value is not 0x followed by hex digits." : { + }, "%@ is a boolean field, so it matches only true or false, not %@." : { "extractionState" : "manual", @@ -201797,6 +201800,9 @@ }, "This row has no value for this field." : { + }, + "This row has no values, and %@ has no way to insert a row of only defaults." : { + }, "This row's _id is binary data whose subtype is not known, so TablePro cannot tell which document it names." : { "extractionState" : "manual", From 465ed2b73f8792138c0141de9e1abff0968b58fe Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 12 Nov 2026 07:30:45 +0700 Subject: [PATCH 3/3] chore(changelog): add the entries for the import, export and copy fixes --- apple/changelog.d/4221.fixed.1.md | 1 + apple/changelog.d/4221.fixed.2.md | 1 + apple/changelog.d/4221.fixed.3.md | 1 + apple/changelog.d/4221.fixed.4.md | 1 + apple/changelog.d/4221.fixed.5.md | 1 + apple/changelog.d/4221.fixed.6.md | 1 + apple/changelog.d/4221.fixed.md | 1 + 7 files changed, 7 insertions(+) create mode 100644 apple/changelog.d/4221.fixed.1.md create mode 100644 apple/changelog.d/4221.fixed.2.md create mode 100644 apple/changelog.d/4221.fixed.3.md create mode 100644 apple/changelog.d/4221.fixed.4.md create mode 100644 apple/changelog.d/4221.fixed.5.md create mode 100644 apple/changelog.d/4221.fixed.6.md create mode 100644 apple/changelog.d/4221.fixed.md diff --git a/apple/changelog.d/4221.fixed.1.md b/apple/changelog.d/4221.fixed.1.md new file mode 100644 index 0000000000..9257f6ba9f --- /dev/null +++ b/apple/changelog.d/4221.fixed.1.md @@ -0,0 +1 @@ +Binary values exported to CSV, JSON or XLSX imported back as hex or Base64 text instead of the bytes. diff --git a/apple/changelog.d/4221.fixed.2.md b/apple/changelog.d/4221.fixed.2.md new file mode 100644 index 0000000000..551b2772f6 --- /dev/null +++ b/apple/changelog.d/4221.fixed.2.md @@ -0,0 +1 @@ +CSV and JSON import dropped rows of all NULL values, the way their own export writes them, and reported success. diff --git a/apple/changelog.d/4221.fixed.3.md b/apple/changelog.d/4221.fixed.3.md new file mode 100644 index 0000000000..9aa8798be9 --- /dev/null +++ b/apple/changelog.d/4221.fixed.3.md @@ -0,0 +1 @@ +Parquet export dropped the UTC offset of timestamps and times with time zone. diff --git a/apple/changelog.d/4221.fixed.4.md b/apple/changelog.d/4221.fixed.4.md new file mode 100644 index 0000000000..8217c58508 --- /dev/null +++ b/apple/changelog.d/4221.fixed.4.md @@ -0,0 +1 @@ +Importing true, false, yes or no into a boolean column failed on MySQL and SQL Server and stored text on SQLite. diff --git a/apple/changelog.d/4221.fixed.5.md b/apple/changelog.d/4221.fixed.5.md new file mode 100644 index 0000000000..ae04654c8c --- /dev/null +++ b/apple/changelog.d/4221.fixed.5.md @@ -0,0 +1 @@ +XLSX export of more than about 2.1 million rows wrote sheets past Excel's row limit. diff --git a/apple/changelog.d/4221.fixed.6.md b/apple/changelog.d/4221.fixed.6.md new file mode 100644 index 0000000000..f8af8f84ad --- /dev/null +++ b/apple/changelog.d/4221.fixed.6.md @@ -0,0 +1 @@ +Copy as IN Clause wrote TRUE and FALSE for SQL Server bit columns and left out binary values. diff --git a/apple/changelog.d/4221.fixed.md b/apple/changelog.d/4221.fixed.md new file mode 100644 index 0000000000..568e5f6590 --- /dev/null +++ b/apple/changelog.d/4221.fixed.md @@ -0,0 +1 @@ +Copying a binary cell copied only its first 64 bytes, and pasting it back wrote the hex text instead of the bytes.