Skip to content
Merged
21 changes: 19 additions & 2 deletions apple/Plugins/CSVImportPlugin/CSVImportParsing.swift
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,27 @@ enum CSVImportParsing {
}
}

static func isBlank(_ fields: [String]) -> Bool {
// RFC 4180 makes an empty line a record of one empty field, which is how CSV export writes a NULL in a one-column
// file. A record of several empty fields, `,,`, is an all-NULL row.
static func isBlank(_ fields: [String], columnCount: Int) -> Bool {
columnCount > 1 && fields.count == 1 && fields[0].isEmpty
}

// A record with nothing in it adds no sample to detection.
static func holdsNoValue(_ fields: [String]) -> Bool {
fields.allSatisfy { $0.isEmpty }
}

// A spreadsheet saves the empty rows under its data as lines of delimiters, the same text as all-NULL rows at
// the very end of an export. Only that trailing run is left out; empty records between rows are rows.
static func dataEnd(recordCount: Int, fields: (Int) -> [String]) -> Int {
var end = recordCount
while end > 0, holdsNoValue(fields(end - 1)) {
end -= 1
}
return end
}

static func detectFields(
in data: Data,
options: CSVImportOptions,
Expand Down Expand Up @@ -141,7 +158,7 @@ enum CSVImportParsing {
for range in dataRanges {
if sampled >= limit { break }
let fields = parser.parseRow(buffer, range: range)
if isBlank(fields) { continue }
if holdsNoValue(fields) { continue }
for column in 0..<columnCount {
let raw = column < fields.count ? fields[column] : ""
guard let value = sampleText(from: raw, options: options) else { continue }
Expand Down
17 changes: 11 additions & 6 deletions apple/Plugins/CSVImportPlugin/CSVImportPlugin.swift
Original file line number Diff line number Diff line change
Expand Up @@ -158,14 +158,19 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi
let buffer = UnsafeBufferPointer(start: base, count: raw.count)
let ranges = parser.indexRows(buffer)
guard !ranges.isEmpty else { return ([], []) }
let names: [String]
let records: [Range<Int>]
if hasHeader {
let header = parser.parseRow(buffer, range: ranges[0])
let names = CSVImportParsing.columnNames(header: header, columnCount: header.count)
return (Array(ranges.dropFirst()), names)
names = CSVImportParsing.columnNames(header: header, columnCount: header.count)
records = Array(ranges.dropFirst())
} else {
let firstCount = parser.parseRow(buffer, range: ranges[0]).count
names = CSVImportParsing.columnNames(header: nil, columnCount: firstCount)
records = ranges
}
let firstCount = parser.parseRow(buffer, range: ranges[0]).count
let names = CSVImportParsing.columnNames(header: nil, columnCount: firstCount)
return (ranges, names)
let end = CSVImportParsing.dataEnd(recordCount: records.count) { parser.parseRow(buffer, range: records[$0]) }
return (Array(records[..<end]), names)
}
}

Expand All @@ -186,7 +191,7 @@ final class CSVImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi
out.reserveCapacity(ranges.count)
for (offset, range) in ranges.enumerated() {
let fields = parser.parseRow(buffer, range: range)
if CSVImportParsing.isBlank(fields) { continue }
if CSVImportParsing.isBlank(fields, columnCount: columnNames.count) { continue }
let line = startIndex + offset + lineOffset
if let error = CSVImportParsing.shapeError(
fields: fields, columnCount: columnNames.count, options: options
Expand Down
18 changes: 14 additions & 4 deletions apple/Plugins/FormatPluginTests/CSVImportPluginTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -152,11 +152,21 @@ struct CSVImportPluginTests {
#expect(CSVImportParsing.importFieldType(for: .date) == .text)
}

@Test("Blank rows are detected")
@Test("Only an empty line in a file of several columns is blank; empty fields make a row")
func testIsBlank() {
#expect(CSVImportParsing.isBlank([""]))
#expect(CSVImportParsing.isBlank(["", ""]))
#expect(!CSVImportParsing.isBlank(["", "x"]))
#expect(CSVImportParsing.isBlank([""], columnCount: 3))
#expect(!CSVImportParsing.isBlank([""], columnCount: 1))
#expect(!CSVImportParsing.isBlank(["", ""], columnCount: 2))
#expect(!CSVImportParsing.isBlank(["", "x"], columnCount: 2))
}

@Test("The data ends at the last record that holds a value")
func testDataEnd() {
let records: [[String]] = [["1", "a"], ["", ""], ["2", ""], ["", ""], [""]]
let end = CSVImportParsing.dataEnd(recordCount: records.count) { records[$0] }
let allEmpty = CSVImportParsing.dataEnd(recordCount: 2) { _ in ["", ""] }
#expect(end == 3)
#expect(allEmpty == 0)
}

// MARK: - Field detection
Expand Down
28 changes: 28 additions & 0 deletions apple/Plugins/FormatPluginTests/CSVImportRowShapeTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,34 @@ struct CSVImportRowShapeTests {
#expect(sink.rows.map { $0["id"] } == [.text("1"), .text("3")])
}

@Test("A record of empty fields, how export writes an all-NULL row, imports as one; an empty line does not")
func emptyFieldsAreARow() async throws {
let (outcome, sink) = try await runImport(
"id,name,city\n1,Ada,Paris\n,,\n\n3,Lin,Hanoi\n", errorHandling: .stopAndRollback)
let result = try outcome.get()
#expect(result.executedStatements == 3)
#expect(sink.rows.map { $0["id"] } == [.text("1"), .null, .text("3")])
#expect(sink.rows[1] == ["id": .null, "name": .null, "city": .null])
}

@Test("The empty rows a spreadsheet saves under its data, and a trailing empty line, are not rows")
func trailingEmptyRecordsAreNotRows() async throws {
let (outcome, sink) = try await runImport("id,name\n1,Ada\n,\n,\n", errorHandling: .stopAndRollback)
#expect(try outcome.get().executedStatements == 1)
#expect(sink.rows.map { $0["id"] } == [.text("1")])

let (oneColumn, oneColumnSink) = try await runImport("email\na@x.io\nb@x.io\n\n", errorHandling: .stopAndRollback)
_ = try oneColumn.get()
#expect(oneColumnSink.rows.map { $0["email"] } == [.text("a@x.io"), .text("b@x.io")])
}

@Test("In a one-column file an empty line is a NULL, as export writes one")
func emptyLineInOneColumnIsARow() async throws {
let (outcome, sink) = try await runImport("name\nAda\n\nLin\n", errorHandling: .stopAndRollback)
_ = try outcome.get()
#expect(sink.rows.map { $0["name"] } == [.text("Ada"), .null, .text("Lin")])
}

@Test("A row that lacks trailing fields imports with them NULL")
func shortRowIsPadded() async throws {
let (outcome, sink) = try await runImport(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -196,11 +196,12 @@ extension FormatLimits {
// JSON has no binary type: a binary value is written as base64 text.
static let json = FormatLimits(binary: { .text($0.base64EncodedString()) })

// An empty cell is not written, so '' reads back as NULL; a binary value is written as hex text,
// and XML 1.0 cannot hold a control character other than tab and the two line breaks.
// An empty cell is not written, so '' reads back as NULL; a binary value, zero bytes too, is
// written as hex text, and XML 1.0 cannot hold a control character other than tab and the two
// line breaks.
static let xlsx = FormatLimits(
emptyTextIsNull: true,
binary: { $0.isEmpty ? .null : .text(FormatLimits.hex($0)) },
binary: { .text(FormatLimits.hex($0)) },
text: { text in
String(String.UnicodeScalarView(text.unicodeScalars.filter { scalar in
scalar.value >= 0x20 || scalar == "\t" || scalar == "\n" || scalar == "\r"
Expand Down
43 changes: 43 additions & 0 deletions apple/Plugins/FormatPluginTests/XLSXSheetRowsTests.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
//
// XLSXSheetRowsTests.swift
// FormatPluginTests
//

import Foundation
import Testing

struct XLSXSheetRowsTests {
private typealias Chunk = XLSXSheetRows.Chunk

@Test("One batch past several sheets fills each to the limit, header included")
func oneBatchFillsSeveralSheets() {
var sheets = XLSXSheetRows(limit: 5, headerRowCount: 1)

#expect(sheets.chunks(of: 12) == [
Chunk(rows: 0..<4, startsNewSheet: false),
Chunk(rows: 4..<8, startsNewSheet: true),
Chunk(rows: 8..<12, startsNewSheet: true),
])
}

@Test("A batch that fills a sheet exactly starts the next sheet only when another row comes")
func exactFillStartsNoEmptySheet() {
var sheets = XLSXSheetRows(limit: 5, headerRowCount: 1)

#expect(sheets.chunks(of: 4) == [Chunk(rows: 0..<4, startsNewSheet: false)])
#expect(sheets.chunks(of: 0).isEmpty)
#expect(sheets.chunks(of: 1) == [Chunk(rows: 0..<1, startsNewSheet: true)])
}

@Test("Batches add up across calls, and without a header a sheet holds the whole limit")
func batchesAddUpWithoutHeader() {
var sheets = XLSXSheetRows(limit: 5, headerRowCount: 0)

#expect(sheets.chunks(of: 3) == [Chunk(rows: 0..<3, startsNewSheet: false)])
#expect(sheets.chunks(of: 9) == [
Chunk(rows: 0..<2, startsNewSheet: false),
Chunk(rows: 2..<7, startsNewSheet: true),
Chunk(rows: 7..<9, startsNewSheet: true),
])
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ struct XLSXWriterStreamingTests {
private static let columns = ["id", "name", "score", "blob", "note"]
private static let columnTypeNames = ["int4", "text", "float8", "bytea", "varchar(8)"]

// SHA-256 of the workbook below as the writer produced it while it built every sheet in memory.
private static let workbookDigest = "7602d732018a0d8b389690d553dd671460fcc4b3dfa53d7a9fff2dcd69a63440"
// SHA-256 of the workbook below; every flush threshold has to produce it.
private static let workbookDigest = "0e3615d4591e87affd46978b977d3122f9cf93f459873835a08080ae9048f333"

private static let closingTags = "</sheetData></worksheet>".utf8.count

Expand Down
10 changes: 10 additions & 0 deletions apple/Plugins/ParquetExportPlugin/ParquetTypeMapper.swift
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ public enum ParquetTypeMapper {
if dateTypes.contains(base) {
return enginesWhereDateCarriesTime.contains(databaseTypeId) ? "TIMESTAMP" : "DATE"
}
// A cast to TIMESTAMP or TIME drops the offset, so 12:00+07 would read as 12:00 UTC.
if isZoned(typeName) || base == "timestamptz" || base == "timetz" {
return timeTypes.contains(base) ? "TIMETZ" : "TIMESTAMPTZ"
}
if timestampTypes.contains(base) { return "TIMESTAMP" }
if timeTypes.contains(base) { return "TIME" }
if binaryTypes.contains(base) { return "BLOB" }
Expand All @@ -49,6 +53,12 @@ public enum ParquetTypeMapper {
words(typeName).contains("unsigned")
}

// Oracle's `with local time zone` counts: the driver writes it with the session's offset.
private static func isZoned(_ typeName: String) -> Bool {
let words = words(typeName).filter { $0 != "local" }
return words.indices.dropLast(2).contains { words[$0] == "with" && words[$0 + 1] == "time" && words[$0 + 2] == "zone" }
}

// `bit` and `bit(1)` hold one flag. `bit(8)` and `bit varying` hold a string of them.
private static func isSingleBit(_ typeName: String) -> Bool {
guard !words(typeName).contains("varying") else { return false }
Expand Down
102 changes: 27 additions & 75 deletions apple/Plugins/XLSXExportPlugin/XLSXExportPlugin.swift
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,31 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug

var isFirstBatch = true
var rowBatch: [[PluginCellValue]] = []
var currentSheetRowCount = 0
var columns: [String] = []
var columnTypeNames: [String] = []
let headerRowCount = options.includeHeaderRow ? 1 : 0
var sheetRows = XLSXSheetRows(limit: Self.maxRowsPerSheet, headerRowCount: options.includeHeaderRow ? 1 : 0)

func writeBatch() throws {
for chunk in sheetRows.chunks(of: rowBatch.count) {
if chunk.startsNewSheet {
try writer.continueSheet(
baseName: table.name,
columns: columns,
columnTypeNames: columnTypeNames,
includeHeader: options.includeHeaderRow,
convertNullToEmpty: options.convertNullToEmpty
)
didSplitSheets = true
}
try autoreleasepool {
try writer.addRows(Array(rowBatch[chunk.rows]), convertNullToEmpty: options.convertNullToEmpty)
}
}
for _ in 0..<rowBatch.count {
progress.incrementRow()
}
rowBatch.removeAll(keepingCapacity: true)
}

let stream = dataSource.streamRows(for: table)
for try await element in stream {
Expand All @@ -75,87 +96,18 @@ final class XLSXExportPlugin: ObservableObject, ExportFormatPlugin, SettablePlug
includeHeader: options.includeHeaderRow,
convertNullToEmpty: options.convertNullToEmpty
)
currentSheetRowCount = headerRowCount
sheetRows = XLSXSheetRows(limit: Self.maxRowsPerSheet, headerRowCount: options.includeHeaderRow ? 1 : 0)
isFirstBatch = false
case .rows(let rows):
for row in rows {
rowBatch.append(row)
}
rowBatch.append(contentsOf: rows)
if rowBatch.count >= 5_000 {
let remaining = Self.maxRowsPerSheet - currentSheetRowCount
if rowBatch.count <= remaining {
try autoreleasepool {
try writer.addRows(rowBatch, convertNullToEmpty: options.convertNullToEmpty)
}
currentSheetRowCount += rowBatch.count
} else {
let fitting = Array(rowBatch.prefix(remaining))
let overflow = Array(rowBatch.dropFirst(remaining))
if !fitting.isEmpty {
try autoreleasepool {
try writer.addRows(fitting, convertNullToEmpty: options.convertNullToEmpty)
}
currentSheetRowCount += fitting.count
}
try writer.continueSheet(
baseName: table.name,
columns: columns,
columnTypeNames: columnTypeNames,
includeHeader: options.includeHeaderRow,
convertNullToEmpty: options.convertNullToEmpty
)
didSplitSheets = true
currentSheetRowCount = headerRowCount
if !overflow.isEmpty {
try autoreleasepool {
try writer.addRows(overflow, convertNullToEmpty: options.convertNullToEmpty)
}
currentSheetRowCount += overflow.count
}
}
let batchCount = rowBatch.count
rowBatch.removeAll(keepingCapacity: true)
for _ in 0..<batchCount {
progress.incrementRow()
}
try writeBatch()
}
}
}

if !rowBatch.isEmpty {
let remaining = Self.maxRowsPerSheet - currentSheetRowCount
if rowBatch.count <= remaining {
try autoreleasepool {
try writer.addRows(rowBatch, convertNullToEmpty: options.convertNullToEmpty)
}
currentSheetRowCount += rowBatch.count
} else {
let fitting = Array(rowBatch.prefix(remaining))
let overflow = Array(rowBatch.dropFirst(remaining))
if !fitting.isEmpty {
try autoreleasepool {
try writer.addRows(fitting, convertNullToEmpty: options.convertNullToEmpty)
}
}
try writer.continueSheet(
baseName: table.name,
columns: columns,
columnTypeNames: columnTypeNames,
includeHeader: options.includeHeaderRow,
convertNullToEmpty: options.convertNullToEmpty
)
didSplitSheets = true
currentSheetRowCount = headerRowCount
if !overflow.isEmpty {
try autoreleasepool {
try writer.addRows(overflow, convertNullToEmpty: options.convertNullToEmpty)
}
currentSheetRowCount += overflow.count
}
}
for _ in 0..<rowBatch.count {
progress.incrementRow()
}
try writeBatch()
}

if !isFirstBatch {
Expand Down
43 changes: 43 additions & 0 deletions apple/Plugins/XLSXExportPlugin/XLSXSheetRows.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
//
// XLSXSheetRows.swift
// XLSXExportPlugin
//

import Foundation

// Splits incoming rows across sheets of at most `limit` rows, the header included. A batch can be a whole in-memory
// result, so one overflow can need several new sheets.
struct XLSXSheetRows {
struct Chunk: Equatable {
let rows: Range<Int>
let startsNewSheet: Bool
}

let limit: Int
let headerRowCount: Int
private(set) var rowsInSheet: Int

init(limit: Int, headerRowCount: Int) {
self.limit = limit
self.headerRowCount = headerRowCount
self.rowsInSheet = headerRowCount
}

// A new sheet starts only when a row needs it, so a batch that fills a sheet exactly leaves no empty one.
mutating func chunks(of count: Int) -> [Chunk] {
var chunks: [Chunk] = []
var start = 0
while start < count {
var startsNewSheet = false
if rowsInSheet >= limit {
rowsInSheet = headerRowCount
startsNewSheet = true
}
let end = min(count, start + max(1, limit - rowsInSheet))
chunks.append(Chunk(rows: start..<end, startsNewSheet: startsNewSheet))
rowsInSheet += end - start
start = end
}
return chunks
}
}
Loading
Loading