diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQCopyDirection+UnsupportedMessage.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQCopyDirection+UnsupportedMessage.swift index f434425ace..aeada04b55 100644 --- a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQCopyDirection+UnsupportedMessage.swift +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQCopyDirection+UnsupportedMessage.swift @@ -4,7 +4,7 @@ nonisolated public extension LibPQCopyDirection { var unsupportedMessage: String { switch self { case .copyIn: - return String(localized: "The query editor cannot send data to COPY FROM STDIN, so no rows were sent. Use Import to load a file into a table.") + return String(localized: "COPY FROM STDIN takes its rows from the lines after the line its ; ends, so no rows were sent.") case .copyOut: return String(localized: "The query editor cannot receive the output of COPY TO STDOUT, so it was discarded. Run a SELECT, or use Export to save the rows to a file.") case .copyBoth: diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLPsqlScript.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLPsqlScript.swift index 60bbd7c47f..0860269016 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLPsqlScript.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLPsqlScript.swift @@ -31,6 +31,54 @@ public enum SQLPsqlScript { return NSRange(location: semicolon + 1, length: length - semicolon - 1) } + /// The rows of the `COPY` block `rows` spans, as ``copyRows(afterSemicolonAt:in:length:)`` found it: the lines + /// after the `;`'s line, each with its line break, up to the `\.` line or the end of the text. The `\.` line is + /// never part of them: psql 18 does not send it, and PostgreSQL 18 reads it as data in CSV. + public static func copyData(inRows rows: NSRange, of text: NSString) -> NSRange { + let end = NSMaxRange(rows) + guard let firstBreak = lineEnd(in: text, from: rows.location, length: end) else { + return NSRange(location: end, length: 0) + } + let start = firstBreak + 1 + var lastLineStart = start + if start < end { + let found = text.range( + of: "\n", + options: [.literal, .backwards], + range: NSRange(location: start, length: end - start) + ) + if found.location != NSNotFound { lastLineStart = found.location + 1 } + } + let dataEnd = endOfDataMarker(in: text, at: lastLineStart, length: text.length) == end ? lastLineStart : end + return NSRange(location: start, length: max(0, dataEnd - start)) + } + + /// Where the `\.` line that starts at `lineStart` ends, its line break excluded, or nil when the line is not one. + /// psql 18 matches `\.` and a LF or CRLF at a line's start only; a last line `\.` with no line break ends the rows + /// too. The unit after the marker is read to decide, so `text` has to hold it unless the marker ends the text. + public static func endOfDataMarker(in text: NSString, at lineStart: Int, length: Int) -> Int? { + guard lineStart + 1 < length, + text.character(at: lineStart) == backslash, + text.character(at: lineStart + 1) == period + else { + return nil + } + let after = lineStart + 2 + guard after < length else { return after } + switch text.character(at: after) { + case SqlLexer.newline: + return after + case SqlLexer.carriageReturn where after + 1 == length || text.character(at: after + 1) == SqlLexer.newline: + return after + default: + return nil + } + } + + /// How many units ``endOfDataMarker(in:at:length:)`` reads from a line's start: `\`, `.`, a carriage return and + /// the line feed after it. + public static let endOfDataMarkerLookahead = 4 + /// The `\restrict` or `\unrestrict` line whose backslash is at `offset`, its line break excluded, when only blanks /// stand before it on its line. psql reads both names case for case. public static func restrictLine(at offset: Int, in text: NSString, length: Int) -> NSRange? { @@ -71,27 +119,6 @@ public enum SQLPsqlScript { return nameEnd } - // psql 18 matches `\.` + LF or CRLF at a line's start only; a last line `\.` with no line break ends the rows too. - // The unit after the returned end, when there is one, was read to decide. - private static func endOfDataMarker(in text: NSString, at lineStart: Int, length: Int) -> Int? { - guard lineStart + 1 < length, - text.character(at: lineStart) == backslash, - text.character(at: lineStart + 1) == period - else { - return nil - } - let after = lineStart + 2 - guard after < length else { return after } - switch text.character(at: after) { - case SqlLexer.newline: - return after - case SqlLexer.carriageReturn where after + 1 == length || text.character(at: after + 1) == SqlLexer.newline: - return after - default: - return nil - } - } - private static func lineEnd(in text: NSString, from offset: Int, length: Int) -> Int? { guard offset < length else { return nil } let found = text.range( diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift index 44584b5e4f..65270ddfd5 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift @@ -77,7 +77,9 @@ public enum SQLStatementScanner { guard !isAllContent else { return range } let text = sql as NSString let content = StatementBlank.contentRange(of: text.substring(to: max(0, text.length - delimiterLength))) - return NSRange(location: offset + content.location, length: content.length) + // A COPY's rows run to their end as written: a last row's trailing tab is a column. + let end = copyRows.map { max(NSMaxRange(content), NSMaxRange($0) - offset) } ?? NSMaxRange(content) + return NSRange(location: offset + content.location, length: end - content.location) } } @@ -103,18 +105,23 @@ public enum SQLStatementScanner { public let segmentEnd: Int public let clientDelimiter: SQLClientDelimiter? + // A `COPY ... FROM STDIN`'s rows as psql sends them, empty for a block with none; nil for any other statement. + public let copyData: String? + public init( sql: String, range: NSRange, acceptsBindParameters: Bool = true, segmentEnd: Int? = nil, - clientDelimiter: SQLClientDelimiter? = nil + clientDelimiter: SQLClientDelimiter? = nil, + copyData: String? = nil ) { self.sql = sql self.range = range self.acceptsBindParameters = acceptsBindParameters self.segmentEnd = segmentEnd ?? NSMaxRange(range) self.clientDelimiter = clientDelimiter + self.copyData = copyData } public func offset(by delta: Int) -> ExecutableStatement { @@ -124,7 +131,8 @@ public enum SQLStatementScanner { range: NSRange(location: range.location + delta, length: range.length), acceptsBindParameters: acceptsBindParameters, segmentEnd: segmentEnd + delta, - clientDelimiter: clientDelimiter + clientDelimiter: clientDelimiter, + copyData: copyData ) } } @@ -271,7 +279,8 @@ public enum SQLStatementScanner { /// A `;` that belongs to the statement stays, but only when there is something before it: a unit reduced to its /// terminator is as empty as any other. /// - /// A client delimiter never reaches the server, and a `COPY ... FROM STDIN` is sent without its rows. + /// A client delimiter never reaches the server, and a `COPY ... FROM STDIN` carries its rows apart from its text, + /// in ``ExecutableStatement/copyData``. public static func executableStatement(from located: LocatedStatement) -> ExecutableStatement? { guard !located.isClientLine else { return nil } let rawSQL = located.sql @@ -291,10 +300,18 @@ public enum SQLStatementScanner { range: NSRange(location: located.offset + range.location, length: range.length), acceptsBindParameters: located.acceptsBindParameters, segmentEnd: NSMaxRange(located.range), - clientDelimiter: located.clientDelimiter + clientDelimiter: located.clientDelimiter, + copyData: copyData(of: located) ) } + private static func copyData(of located: LocatedStatement) -> String? { + guard let rows = located.copyRows else { return nil } + let text = located.sql as NSString + let local = NSRange(location: rows.location - located.offset, length: rows.length) + return text.substring(with: SQLPsqlScript.copyData(inRows: local, of: text)) + } + /// The statements after `ran`, one of those `sql` was split into, split again under `grammar`, up to `end`: once /// a statement changes how the session reads literals, the rest no longer splits as planned. The split starts at /// ``ExecutableStatement/segmentEnd``, so a `COPY`'s rows are never read as statements, and keeps the client diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLPsqlScriptTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLPsqlScriptTests.swift index b5d127c657..523cb672a4 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLPsqlScriptTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLPsqlScriptTests.swift @@ -103,6 +103,90 @@ struct SQLPsqlScriptTests { #expect(SQLStatementScanner.executableStatements(in: rows, grammar: unknown.execution).count == 2) } + private func copyData(_ sql: String) -> String { + let text = sql as NSString + let semicolon = text.range(of: ";").location + let rows = SQLPsqlScript.copyRows(afterSemicolonAt: semicolon, in: text, length: text.length) + return text.substring(with: SQLPsqlScript.copyData(inRows: rows, of: text)) + } + + @Test("A COPY's data is the lines after its ; line, up to the \\. line, which is never sent", arguments: [ + ("COPY t FROM stdin;\n1\tone\n2\ttwo\n\\.\nSELECT 2;", "1\tone\n2\ttwo\n"), + ("COPY t FROM stdin;\r\n1\tone\r\n2\ttwo\r\n\\.\r\nSELECT 2;", "1\tone\r\n2\ttwo\r\n"), + ("COPY t FROM stdin;\n1\tone\n2\ttwo\n", "1\tone\n2\ttwo\n"), + ("COPY t FROM stdin;\n1\tone\n2\ttwo", "1\tone\n2\ttwo"), + ("COPY t FROM stdin;\n1\tone\n\\.", "1\tone\n"), + ("COPY t FROM stdin;\n1\tone\n\\.\r", "1\tone\n"), + ("COPY t FROM stdin;\n\\. \n\\.\n", "\\. \n"), + ("COPY t FROM stdin;", ""), + ("COPY t FROM stdin;\n\\.\nSELECT 2;", ""), + ("COPY t FROM stdin; SELECT 8;\n1\ta\n\\.\nSELECT 7;", "1\ta\n"), + ("COPY t FROM stdin; DROP TABLE users;", ""), + ("COPY t FROM stdin; DROP TABLE users;\n", ""), + ]) + func copyDataSpan(sql: String, expected: String) { + #expect(copyData(sql) == expected) + } + + @Test("A scanned COPY carries its data, which offset(by:) keeps, and no other statement carries any") + func executableStatementCarriesCopyData() throws { + let sql = "SELECT 1;\nCOPY t (a, b) FROM stdin;\n1\tone\n2\ttwo\n\\.\nSELECT 2;" + let statements = SQLStatementScanner.executableStatements(in: sql, grammar: Self.postgreSQL.execution) + #expect(statements.map { $0.sql } == ["SELECT 1", "COPY t (a, b) FROM stdin", "SELECT 2"]) + #expect(statements.map { $0.copyData } == [nil, "1\tone\n2\ttwo\n", nil]) + + let copy = try #require(statements.dropFirst().first) + let moved = copy.offset(by: 40) + #expect(moved.copyData == "1\tone\n2\ttwo\n") + #expect(moved.range.location == copy.range.location + 40) + #expect(moved.segmentEnd == copy.segmentEnd + 40) + } + + @Test("A COPY with no rows carries empty data and still runs as a COPY", arguments: [ + "COPY t FROM stdin;", "COPY t FROM stdin;\n\\.\n", "COPY t FROM stdin; DROP TABLE users;\n", + ]) + func copyWithoutRows(sql: String) { + let statements = SQLStatementScanner.executableStatements(in: sql, grammar: Self.postgreSQL.execution) + #expect(statements.map { $0.sql } == ["COPY t FROM stdin"]) + #expect(statements.map { $0.copyData } == [""]) + } + + @Test("Run Current's text and a split resumed after a statement give the COPY the same data") + func copyDataInFragmentsAndResumedSplits() throws { + let sql = "SET search_path = public;\r\nCOPY t FROM stdin;\r\n1\tone\r\n2\t\\N\r\n\\.\r\nSELECT 2;" + let grammar = Self.postgreSQL.execution + let statements = SQLStatementScanner.executableStatements(in: sql, grammar: grammar) + let set = try #require(statements.first) + + let resumed = SQLStatementScanner.executableStatements( + in: sql, after: set, through: (sql as NSString).length, grammar: grammar + ) + #expect(resumed.map { $0.copyData } == ["1\tone\r\n2\t\\N\r\n", nil]) + + let navigable = SQLStatementScanner.navigableStatements(in: sql, grammar: grammar) + let fragment = (sql as NSString).substring(with: navigable[1].contentRange) + let rerun = SQLStatementScanner.executableStatements(in: fragment, grammar: grammar) + #expect(rerun.map { $0.copyData } == ["1\tone\r\n2\t\\N\r\n"]) + } + + @Test("Run Current's text keeps a last row's trailing tab when no \\. line ends the rows") + func copyContentKeepsTheLastRowAsWritten() throws { + let sql = "SELECT 1;\nCOPY t (a, b) FROM stdin;\n1\tone\n2\t\n" + let grammar = Self.postgreSQL.execution + let navigable = SQLStatementScanner.navigableStatements(in: sql, grammar: grammar) + let fragment = (sql as NSString).substring(with: navigable[1].contentRange) + let rerun = SQLStatementScanner.executableStatements(in: fragment, grammar: grammar) + #expect(rerun.map { $0.copyData } == ["1\tone\n2\t\n"]) + } + + @Test("An engine that does not read psql scripts carries no copy data") + func noCopyDataOutsidePsqlScripts() { + let mysql = SQLLexicalReadings.resolve(databaseTypeId: "MySQL", declared: nil, session: nil).execution + let statements = SQLStatementScanner.executableStatements(in: "COPY t FROM stdin;\n1\n", grammar: mysql) + #expect(statements.count == 2) + #expect(statements.allSatisfy { $0.copyData == nil }) + } + @Test("The text psql holds as no SQL is reported for the formatter") func scriptText() { let sql = "\\restrict k\nCOPY t FROM stdin;\n1\ta\n\\.\nSELECT 1;" diff --git a/apple/Plugins/FormatPluginTests/SQLImportCopyRowsTests.swift b/apple/Plugins/FormatPluginTests/SQLImportCopyRowsTests.swift new file mode 100644 index 0000000000..37e845c375 --- /dev/null +++ b/apple/Plugins/FormatPluginTests/SQLImportCopyRowsTests.swift @@ -0,0 +1,172 @@ +// +// SQLImportCopyRowsTests.swift +// FormatPluginTests +// + +import Foundation +import TableProPluginKit +import Testing + +private struct CopyRefusal: Error {} + +private final class CopySink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "PostgreSQL" + private let lock = NSLock() + private var log: [String] = [] + private var received = Data() + var refusesCopy = false + var onPiece: @Sendable () -> Void = {} + + var calls: [String] { lock.withLock { log } } + var copied: Data { lock.withLock { received } } + + func execute(statement: String) async throws { + lock.withLock { log.append(statement) } + } + + func copyFromStandardInput(statement: String, line: Int, rows: PluginCopyData) async throws { + lock.withLock { log.append("\(statement) @\(line)") } + if refusesCopy { throw CopyRefusal() } + while let piece = try rows.nextPiece() { + lock.withLock { received.append(piece) } + onPiece() + } + } + + func deleteAllRowsFromTargetTable() async throws {} + func beginTransaction() async throws { lock.withLock { log.append("BEGIN") } } + func commitTransaction() async throws { lock.withLock { log.append("COMMIT") } } + func rollbackTransaction() async throws { lock.withLock { log.append("ROLLBACK") } } +} + +private final class ElementSource: PluginImportSource, @unchecked Sendable { + private let elements: [PluginImportElement] + + init(_ elements: [PluginImportElement]) { + self.elements = elements + } + + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + AsyncThrowingStream { $0.finish() } + } + + func elements() async throws -> AsyncThrowingStream { + let elements = self.elements + return AsyncThrowingStream { continuation in + elements.forEach { continuation.yield($0) } + continuation.finish() + } + } + + func fileURL() -> URL { URL(fileURLWithPath: "/dev/null") } + func fileSizeBytes() -> Int64 { 0 } +} + +private final class PieceList: @unchecked Sendable { + private let lock = NSLock() + private var pieces: [Data] + + init(_ pieces: [String]) { + self.pieces = pieces.map { Data($0.utf8) } + } + + var data: PluginCopyData { + PluginCopyData { [self] in + lock.withLock { pieces.isEmpty ? nil : pieces.removeFirst() } + } + } +} + +@Suite("SQL import COPY rows", .serialized) +struct SQLImportCopyRowsTests { + private func runImport( + _ elements: [PluginImportElement], + sink: CopySink, + errorHandling: ImportErrorHandling = .stopAndRollback, + progress: PluginImportProgress = PluginImportProgress(progress: Progress()) + ) async -> Result { + let plugin = SQLImportPlugin() + let original = plugin.settings + defer { plugin.settings = original } + plugin.settings.errorHandling = errorHandling + plugin.settings.wrapInTransaction = true + plugin.settings.disableForeignKeyChecks = false + do { + return .success(try await plugin.performImport(source: ElementSource(elements), sink: sink, progress: progress)) + } catch { + return .failure(error) + } + } + + @Test("A COPY element goes to the sink with its rows, inside the import's transaction, counted as one statement") + func copyRunsInsideTheTransaction() async throws { + let sink = CopySink() + let result = try await runImport([ + .statement("CREATE TABLE t (a int)", lineNumber: 1), + .copyFromStandardInput(statement: "COPY t (a) FROM stdin", lineNumber: 2, rows: PieceList(["1\n2\n", "3\n"]).data), + .statement("SELECT 1", lineNumber: 6), + ], sink: sink).get() + + #expect(sink.calls == ["BEGIN", "CREATE TABLE t (a int)", "COPY t (a) FROM stdin @2", "SELECT 1", "COMMIT"]) + #expect(sink.copied == Data("1\n2\n3\n".utf8)) + #expect(result.executedStatements == 3) + } + + @Test("A COPY the sink refuses fails the import at its line and rolls back") + func refusedCopyStopsTheImport() async throws { + let sink = CopySink() + sink.refusesCopy = true + let outcome = await runImport([ + .copyFromStandardInput(statement: "COPY t FROM stdin", lineNumber: 7, rows: PieceList(["1\n"]).data), + ], sink: sink) + + guard case .failure(let error) = outcome, + case .statementFailed(let statement, let line, _) = error as? PluginImportError + else { + Issue.record("Expected statementFailed, got \(outcome)") + return + } + #expect(statement == "COPY t FROM stdin") + #expect(line == 7) + #expect(sink.calls.last == "ROLLBACK") + } + + @Test("A Stop while rows are sent ends the COPY as a cancelled import") + func stopDuringRows() async throws { + let sink = CopySink() + let progress = PluginImportProgress(progress: Progress()) + sink.onPiece = { progress.cancel() } + let outcome = await runImport([ + .copyFromStandardInput(statement: "COPY t FROM stdin", lineNumber: 1, rows: PieceList(["1\n", "2\n", "3\n"]).data), + ], sink: sink, progress: progress) + + guard case .failure(let error) = outcome else { + Issue.record("Expected a cancellation, got \(outcome)") + return + } + #expect(error is PluginImportCancellationError) + #expect(sink.copied == Data("1\n".utf8)) + } + + @Test("A source with statements alone still runs through the default elements") + func statementOnlySourceRuns() async throws { + final class LegacySource: PluginImportSource, @unchecked Sendable { + func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { + AsyncThrowingStream { continuation in + continuation.yield((statement: "SELECT 1", lineNumber: 3)) + continuation.finish() + } + } + + func fileURL() -> URL { URL(fileURLWithPath: "/dev/null") } + func fileSizeBytes() -> Int64 { 0 } + } + + var lines: [Int] = [] + for try await element in try await LegacySource().elements() { + lines.append(element.lineNumber) + #expect(element.statementText == "SELECT 1") + } + #expect(lines == [3]) + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/CockroachPluginDriver.swift b/apple/Plugins/PostgreSQLDriverPlugin/CockroachPluginDriver.swift index 8d47584aad..16a768935d 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/CockroachPluginDriver.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/CockroachPluginDriver.swift @@ -33,6 +33,7 @@ final class CockroachPluginDriver: LibPQBackedDriver, @unchecked Sendable { .dataCompare, .readOnlyStatements, .enforcesReadOnlyMode, + .copyFromStandardInput, ] } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQCopyIn.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQCopyIn.swift new file mode 100644 index 0000000000..2a47629565 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQCopyIn.swift @@ -0,0 +1,120 @@ +// +// LibPQCopyIn.swift +// PostgreSQLDriverPlugin +// + +import CLibPQ +import Foundation +import TableProPluginKit +import TableProPostgreSQLCore + +/// Sends a `COPY ... FROM STDIN` its rows, after the statement answered `PGRES_COPY_IN`, the way psql 18 sends a +/// script's rows: the data as `CopyData`, never the `\.` line, then `CopyDone`. Measured with libpq 17.4 against +/// PostgreSQL 17.11 and 18.6 (`apple/scripts/probes/check-postgresql-copy-in.sh`). +nonisolated enum LibPQCopyIn { + struct Sent { + // The copy's own result, or nil when libpq had none to give. The caller clears it. + let result: OpaquePointer? + // Why the rows stopped before their end, when they did. + let interruption: Interruption? + } + + enum Interruption { + case cancelled + case binaryFormat + case source(Error) + case connection + } + + // psql's own CopyFail reasons, which the server echoes as "COPY from stdin failed: ". + static let cancelledReason = "canceled by user" + static let sourceFailedReason = "aborted because of read failure" + static let binaryReason = "binary COPY data cannot come from a script" + + static var binaryRefusal: String { + String(localized: "A script cannot hold binary COPY data, so no rows were sent. Dump the rows as text or CSV.") + } + + // PQputCopyData takes an `int` length and grows its buffer to hold a whole message. + private static let sliceLength = 1 << 20 + + static func send( + _ rows: PluginCopyData, + format: LibPQCopyFormat, + conn: OpaquePointer, + isCancelled: () -> Bool + ) -> Sent { + // A script holds text, and psql would send the rest of the file as the binary stream. + let interruption: Interruption? = format == .binary + ? .binaryFormat + : sendPieces(rows, conn: conn, isCancelled: isCancelled) + let ended: Int32 + switch interruption { + case nil: + ended = PQputCopyEnd(conn, nil) + case .connection: + ended = -1 + case .cancelled: + ended = cancelledReason.withCString { PQputCopyEnd(conn, $0) } + case .binaryFormat: + ended = binaryReason.withCString { PQputCopyEnd(conn, $0) } + case .source(let error): + let reason = isStop(error) ? cancelledReason : sourceFailedReason + ended = reason.withCString { PQputCopyEnd(conn, $0) } + } + let reported = ended == 1 ? interruption : (interruption ?? .connection) + return Sent(result: finalResult(conn), interruption: reported) + } + + private static func sendPieces(_ rows: PluginCopyData, conn: OpaquePointer, isCancelled: () -> Bool) -> Interruption? { + do { + while true { + if isCancelled() { return .cancelled } + guard let piece = try rows.nextPiece() else { return nil } + guard put(piece, conn: conn) else { return .connection } + } + } catch { + return .source(error) + } + } + + // An import's Stop reaches here as its source's throw, and the server log should read it as one. + private static func isStop(_ error: Error) -> Bool { + error is PluginImportCancellationError || error is CancellationError + } + + private static func put(_ piece: Data, conn: OpaquePointer) -> Bool { + piece.withUnsafeBytes { raw in + guard let base = raw.baseAddress?.assumingMemoryBound(to: CChar.self) else { return true } + var offset = 0 + while offset < raw.count { + let length = min(sliceLength, raw.count - offset) + guard PQputCopyData(conn, base + offset, Int32(length)) == 1 else { return false } + offset += length + } + return true + } + } + + // While libpq still answers PGRES_COPY_IN the copy is not over: end it again, as psql does. The first other + // result is the copy's; the rest are drained so the connection takes the next statement. + private static func finalResult(_ conn: OpaquePointer) -> OpaquePointer? { + var copyResult: OpaquePointer? + var endsAgain = 0 + while let next = PQgetResult(conn) { + guard PQresultStatus(next) != PGRES_COPY_IN else { + PQclear(next) + endsAgain += 1 + guard endsAgain <= 3 else { break } + _ = "trying to exit copy mode".withCString { PQputCopyEnd(conn, $0) } + continue + } + if copyResult == nil { + copyResult = next + } else { + PQclear(next) + } + } + return copyResult + } +} diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift index a9ba921425..5db63931ba 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQDriverCore.swift @@ -380,6 +380,17 @@ final class LibPQDriverCore: @unchecked Sendable { return answer } + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + readOnly: LibPQReadOnlyTransaction.Reason? + ) async throws -> PluginQueryResult { + let pqConn = try connection() + let startTime = Date() + let result = try await pqConn.copyFromStandardInput(statement, rows: rows, readOnly: readOnly) + return Self.answer(result, since: startTime) + } + private static func answer(_ result: LibPQPluginQueryResult, since startTime: Date) -> PluginQueryResult { PluginQueryResult( columns: result.columns, @@ -510,6 +521,23 @@ extension LibPQBackedDriver { core.sessionLexicalState } + // Every libpq engine here reaches this through one conformance, so a sibling that does not declare the capability + // (Redshift, whose COPY reads only its cloud sources) refuses here. + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + guard capabilities.contains(.copyFromStandardInput) else { + throw PluginDriverUnsupportedOperation.copyFromStandardInput + } + return try await core.copyFromStandardInput( + statement: statement, + rows: rows, + readOnly: LibPQReadOnlyTransaction.reason(for: context, capabilities: capabilities) + ) + } + /// The new name must be bare. Every libpq engine here rejects a qualified one, because this /// statement renames in place and never moves the object; `SET SCHEMA` is the separate verb. /// diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift index 48bf94a8fd..d45c9cc798 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift @@ -951,6 +951,28 @@ final class LibPQPluginConnection: @unchecked Sendable { } } + // One dispatch holds the connection from the COPY to its last result: a PQexec sent in between would end the copy. + func copyFromStandardInput( + _ statement: String, + rows: PluginCopyData, + readOnly: LibPQReadOnlyTransaction.Reason? + ) async throws -> LibPQPluginQueryResult { + let statementToRun = String(statement) + + return try await pluginDispatchAsync(on: queue) { [self] in + guard !isShuttingDown, let conn = connectionHandle else { throw LibPQPluginError.notConnected } + guard let readOnly else { return try copyFromStandardInputSync(statementToRun, rows: rows, conn: conn) } + if let ended = sessionEndedBeforeSending(conn) { throw ended } + return try LibPQReadOnlyTransaction.run( + state: transactionStateOnQueue().sessionTransactionState, + reason: readOnly, + send: { try runReadOnlyControl($0, on: conn) } + ) { + try copyFromStandardInputSync(statementToRun, rows: rows, conn: conn) + } + } + } + // MARK: - Server Information func serverVersion() -> String? { @@ -1059,6 +1081,67 @@ final class LibPQPluginConnection: @unchecked Sendable { ) } + // A statement that is not a copy-in, the server's refusal of the COPY among them, reads as any other result. + private func copyFromStandardInputSync( + _ statement: String, + rows: PluginCopyData, + conn: OpaquePointer + ) throws -> LibPQPluginQueryResult { + let generation = cancellationGate.beginQuery() + defer { cancellationGate.endQuery(generation) } + defer { refreshStandardConformingStrings(from: conn) } + + let cancelsOutput = cancelsAbandonedOutput(conn) + if let ended = sessionEndedBeforeSending(conn) { throw ended } + guard let start = statement.withCString({ PQexec(conn, $0) }) else { + throw lostConnection(getError(from: conn), on: conn, sent: true) + } + recordTransactionState(of: conn) + guard let copy = LibPQCopyState.copy(of: start), copy.direction == .copyIn else { + return try queryResult( + from: start, + connection: conn, + generation: generation, + cancelsOutput: cancelsOutput, + allowsSideQueries: true + ) + } + PQclear(start) + + let sent = LibPQCopyIn.send(rows, format: copy.format, conn: conn) { + cancellationGate.isCancelled(generation) + } + recordTransactionState(of: conn) + guard let result = sent.result else { + throw lostConnection(getError(from: conn), on: conn, sent: true) + } + defer { PQclear(result) } + switch sent.interruption { + case .cancelled: + throw CancellationError() + case .source(let error): + throw error + case .binaryFormat: + throw LibPQPluginError(message: LibPQCopyIn.binaryRefusal, sqlState: nil, detail: nil) + case .connection, nil: + break + } + guard PQresultStatus(result) == PGRES_COMMAND_OK else { + let error = getResultError(from: result) + if cancellationGate.isCancelled(generation) { throw CancellationError() } + throw lostConnection(error, on: conn, sent: true) + } + return LibPQPluginQueryResult( + columns: [], + columnOids: [], + columnTypeNames: [], + rows: [], + affectedRows: getAffectedRows(from: result), + commandTag: getCommandTag(from: result), + isTruncated: false + ) + } + private func queryResult( from result: OpaquePointer, connection conn: OpaquePointer, diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PGlitePluginDriver.swift b/apple/Plugins/PostgreSQLDriverPlugin/PGlitePluginDriver.swift index 39684d302e..cbf6456bb1 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PGlitePluginDriver.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PGlitePluginDriver.swift @@ -22,8 +22,9 @@ final class PGlitePluginDriver: PostgreSQLPluginDriver { super.init(config: config, singleConnectionMode: true) } + // pglite-socket 0.2.19 (PGlite 0.5.16) never reads CopyData: a COPY FROM STDIN, psql's too, loses the connection. override var capabilities: PluginCapabilities { - super.capabilities.subtracting(.cancelQuery) + super.capabilities.subtracting([.cancelQuery, .copyFromStandardInput]) } // pldebugger is a server C extension and needs a second session; PGlite has neither. diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift index de3f018dc9..67de451b85 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift @@ -52,7 +52,8 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { .dataCompare, .temporaryTables, .readOnlyStatements, - .enforcesReadOnlyMode + .enforcesReadOnlyMode, + .copyFromStandardInput, ] } diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQCopyInTests.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQCopyInTests.swift new file mode 100644 index 0000000000..0375b98a04 --- /dev/null +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQCopyInTests.swift @@ -0,0 +1,224 @@ +// +// LibPQCopyInTests.swift +// PostgreSQLDriverTests +// + +import Foundation +import TableProPluginKit +import Testing + +@Suite(.timeLimit(.minutes(1))) +struct LibPQCopyInTests { + private static let statement = "COPY public.t (a, b) FROM stdin" + + private struct SourceFailure: Error, Equatable {} + + private final class Pieces: @unchecked Sendable { + private let lock = NSLock() + private var remaining: [Data] + private var read = 0 + private let failsAfter: Int? + private let onRead: @Sendable (Int) -> Void + + init(_ pieces: [String], failsAfter: Int? = nil, onRead: @escaping @Sendable (Int) -> Void = { _ in }) { + remaining = pieces.map { Data($0.utf8) } + self.failsAfter = failsAfter + self.onRead = onRead + } + + var piecesRead: Int { + lock.withLock { read } + } + + var data: PluginCopyData { + PluginCopyData { [self] in try next() } + } + + private func next() throws -> Data? { + let (count, piece) = try lock.withLock { () -> (Int, Data?) in + if let failsAfter, read == failsAfter { throw SourceFailure() } + read += 1 + return (read, remaining.isEmpty ? nil : remaining.removeFirst()) + } + onRead(count) + return piece + } + } + + private static func connected(to server: LibPQScriptedServer) async throws -> LibPQPluginConnection { + let connection = LibPQPluginConnection( + host: "127.0.0.1", port: server.port, user: "postgres", password: nil, database: "postgres" + ) + try await connection.connect() + try await connection.finishConnecting() + return connection + } + + private static func server(_ copyIn: LibPQScriptedServer.CopyIn = .init()) throws -> LibPQScriptedServer { + let server = try LibPQScriptedServer(answer: { query in + query.hasPrefix("COPY") ? LibPQScriptedServer.Answer(copyIn: copyIn) : nil + }) + server.start() + return server + } + + @Test("The rows reach the server byte for byte in the order read, and the result counts them") + func rowsArriveWhole() async throws { + let server = try Self.server() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let pieces = Pieces(["1\tone\n2\tt", "wo\n3\t\\N\n", "4\tcaf\u{e9} \u{1F600}\r\n"]) + let result = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: nil) + + #expect(server.copyData == Data("1\tone\n2\ttwo\n3\t\\N\n4\tcaf\u{e9} \u{1F600}\r\n".utf8)) + #expect(result.affectedRows == 4) + #expect(result.commandTag == "COPY 4") + #expect(server.copyFailMessages.isEmpty) + #expect(server.queries.filter { $0.hasPrefix("COPY") } == [Self.statement]) + } + + @Test("No rows still ends the copy, as a COPY of zero rows") + func noRows() async throws { + let server = try Self.server() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let result = try await connection.copyFromStandardInput(Self.statement, rows: Pieces([]).data, readOnly: nil) + + #expect(server.copyData.isEmpty) + #expect(result.affectedRows == 0) + } + + @Test("A source that fails ends the copy with CopyFail and its own error comes back, not the server's echo") + func sourceFailureEndsTheCopy() async throws { + let server = try Self.server() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let pieces = Pieces(["1\tone\n", "2\ttwo\n"], failsAfter: 1) + await #expect(throws: SourceFailure.self) { + _ = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: nil) + } + + #expect(server.copyData == Data("1\tone\n".utf8)) + #expect(server.copyFailMessages == ["aborted because of read failure"]) + _ = try await connection.executeQuery("SELECT 1") + } + + @Test("An import's Stop thrown by the source ends the copy as cancelled by the user and comes back as itself") + func importStopEndsTheCopy() async throws { + let server = try Self.server() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let rows = PluginCopyData { throw PluginImportCancellationError() } + await #expect(throws: PluginImportCancellationError.self) { + _ = try await connection.copyFromStandardInput(Self.statement, rows: rows, readOnly: nil) + } + #expect(server.copyFailMessages == ["canceled by user"]) + } + + @Test("A Stop between pieces ends the copy with CopyFail and reports cancelled") + func stopEndsTheCopy() async throws { + let server = try Self.server() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let pieces = Pieces(["1\tone\n", "2\ttwo\n", "3\tthree\n"]) { count in + if count == 1 { _ = connection.cancellationGate.cancel() } + } + await #expect(throws: CancellationError.self) { + _ = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: nil) + } + + #expect(server.copyData == Data("1\tone\n".utf8)) + #expect(server.copyFailMessages == ["canceled by user"]) + #expect(pieces.piecesRead == 1) + } + + @Test("A binary COPY is refused with CopyFail before any piece is read") + func binaryIsRefused() async throws { + let server = try Self.server(.init(binary: true)) + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let pieces = Pieces(["PGCOPY"]) + await #expect(throws: LibPQPluginError.self) { + _ = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: nil) + } + + #expect(pieces.piecesRead == 0) + #expect(server.copyData.isEmpty) + #expect(server.copyFailMessages == ["binary COPY data cannot come from a script"]) + } + + @Test("A row the server refuses fails the COPY with the server's error after every piece was sent") + func serverRefusalComesBack() async throws { + let server = try Self.server(.init(failure: .init(sqlState: "22P02", message: "invalid input syntax"))) + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + await #expect { + _ = try await connection.copyFromStandardInput(Self.statement, rows: Pieces(["x\tbad\n"]).data, readOnly: nil) + } throws: { error in + (error as? LibPQPluginError)?.sqlState == "22P02" + } + _ = try await connection.executeQuery("SELECT 1") + } + + @Test("A statement that answers no copy reads as any other, and no piece is read") + func notACopyReadsAsAStatement() async throws { + let server = try LibPQScriptedServer(answer: { _ in + LibPQScriptedServer.Answer(failure: .init(sqlState: "42P01", message: "relation does not exist")) + }) + server.start() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let pieces = Pieces(["1\tone\n"]) + await #expect { + _ = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: nil) + } throws: { error in + (error as? LibPQPluginError)?.sqlState == "42P01" + } + #expect(pieces.piecesRead == 0) + } + + @Test("Under Safe Mode's Read-Only the COPY goes inside a read-only transaction that is rolled back") + func readOnlyWrapsTheCopy() async throws { + let server = try LibPQScriptedServer( + status: { query, _ in + query.hasPrefix("START TRANSACTION READ ONLY") || query.hasPrefix("COPY") + ? LibPQScriptedServer.inTransaction + : LibPQScriptedServer.idle + }, + answer: { query in + guard query.hasPrefix("COPY") else { return nil } + return LibPQScriptedServer.Answer(failure: .init( + sqlState: "25006", message: "cannot execute COPY FROM in a read-only transaction" + )) + } + ) + server.start() + defer { server.close() } + let connection = try await Self.connected(to: server) + defer { connection.disconnect() } + + let pieces = Pieces(["1\tone\n"]) + await #expect(throws: (any Error).self) { + _ = try await connection.copyFromStandardInput(Self.statement, rows: pieces.data, readOnly: .safeMode) + } + + #expect(Array(server.queries.suffix(3)) == ["START TRANSACTION READ ONLY; SELECT 1", Self.statement, "ROLLBACK"]) + #expect(pieces.piecesRead == 0) + } +} diff --git a/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift b/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift index 2db121008f..b7fd8c19d0 100644 --- a/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift +++ b/apple/Plugins/PostgreSQLDriverTests/LibPQScriptedServer.swift @@ -12,7 +12,8 @@ import Foundation // or a parsed statement gets what `answer` returns for it; without one, each statement of a simple query gets an empty // completion, except that a statement holding `failingStatement` gets an error and the ones after it nothing. Cancel // requests arrive on connections of their own: each is counted, and a statement waiting for one then fails as a -// cancelled statement does. +// cancelled statement does. A query answered with `copyIn` starts a copy-in: the server keeps every CopyData byte, and +// answers CopyDone with `COPY ` (or the scripted failure) and CopyFail with 57014, as PostgreSQL does. final class LibPQScriptedServer: @unchecked Sendable { struct SetupFailure: Error {} @@ -34,6 +35,13 @@ final class LibPQScriptedServer: @unchecked Sendable { // After the rows, wait for a cancel request and fail with 57014. A statement nobody cancels completes. var waitsForCancel = false var failure: Failure? + var copyIn: CopyIn? + } + + struct CopyIn: Sendable { + var binary = false + // Sent after CopyDone in place of the completion, as a row the server refuses is reported. + var failure: Failure? } static let idle = UInt8(ascii: "I") @@ -68,6 +76,8 @@ final class LibPQScriptedServer: @unchecked Sendable { private var bindings: [[String?]] = [] private var cancels = 0 private var isClosed = false + private var copied = Data() + private var copyFailures: [String] = [] var queries: [String] { lock.withLock { received } @@ -82,6 +92,15 @@ final class LibPQScriptedServer: @unchecked Sendable { lock.withLock { cancels } } + var copyData: Data { + lock.withLock { copied } + } + + // Each CopyFail's message. + var copyFailMessages: [String] { + lock.withLock { copyFailures } + } + init( status: @escaping @Sendable (_ query: String, _ earlier: [String]) -> UInt8 = { _, _ in LibPQScriptedServer.idle }, onQuery: @escaping @Sendable (String) -> Void = { _ in }, @@ -148,7 +167,10 @@ final class LibPQScriptedServer: @unchecked Sendable { let earlier = record(query) onQuery(query) var simpleFailed = false - if let scripted = answer(query) { + let scripted = answer(query) + if let copyIn = scripted?.copyIn { + guard copy(copyIn, on: client, failed: &simpleFailed) else { return } + } else if let scripted { if scripted.failure == nil, !scripted.columns.isEmpty { guard send(client, Self.rowDescription(scripted.columns)) else { return } } @@ -215,6 +237,38 @@ final class LibPQScriptedServer: @unchecked Sendable { return send(client, Self.message("C", Array("SELECT \(answer.rows.count)".utf8) + [0])) } + // CopyInResponse with every column in the copy's format, then CopyData until CopyDone or CopyFail. + private func copy(_ copyIn: CopyIn, on client: Int32, failed: inout Bool) -> Bool { + let format: UInt8 = copyIn.binary ? 1 : 0 + guard send(client, Self.message("G", [format] + Self.bytes(UInt16(1)) + Self.bytes(UInt16(format)))) else { + return false + } + var lines = 0 + while let type = receive(client, 1)?.first, let body = receiveBody(client) { + switch type { + case UInt8(ascii: "d"): + lines += body.filter { $0 == UInt8(ascii: "\n") }.count + lock.withLock { copied.append(contentsOf: body) } + case UInt8(ascii: "c"): + if let failure = copyIn.failure { + failed = true + return send(client, Self.errorResponse(failure)) + } + return send(client, Self.message("C", Array("COPY \(lines)".utf8) + [0])) + case UInt8(ascii: "f"): + let reason = Self.cString(body, from: 0).text + lock.withLock { copyFailures.append(reason) } + failed = true + return send(client, Self.errorResponse(Failure( + sqlState: "57014", message: "COPY from stdin failed: \(reason)" + ))) + default: + return false + } + } + return false + } + private func acceptCancelRequests() { while !lock.withLock({ isClosed }) { var pollFD = pollfd(fd: listener, events: Int16(POLLIN), revents: 0) diff --git a/apple/Plugins/SQLImportPlugin/SQLImportCopyRows.swift b/apple/Plugins/SQLImportPlugin/SQLImportCopyRows.swift new file mode 100644 index 0000000000..1b6034c571 --- /dev/null +++ b/apple/Plugins/SQLImportPlugin/SQLImportCopyRows.swift @@ -0,0 +1,53 @@ +// +// SQLImportCopyRows.swift +// SQLImportPlugin +// + +import Foundation +import TableProPluginKit + +// The driver pulls a COPY's rows on its own thread, so the check for a Stop and the progress note ride on each pull. +internal final class SQLImportCopyRows: @unchecked Sendable { + private static let lineFeed = UInt8(ascii: "\n") + private static let linesPerNote = 10_000 + + private let rows: PluginCopyData + private let line: Int + private let progress: PluginImportProgress + private let lock = NSLock() + private var linesSent = 0 + private var linesNoted: Int? + + private init(_ rows: PluginCopyData, line: Int, progress: PluginImportProgress) { + self.rows = rows + self.line = line + self.progress = progress + } + + internal static func watching(_ rows: PluginCopyData, line: Int, progress: PluginImportProgress) -> PluginCopyData { + let watched = SQLImportCopyRows(rows, line: line, progress: progress) + return PluginCopyData { try watched.nextPiece() } + } + + private func nextPiece() throws -> Data? { + try progress.checkCancellation() + guard let piece = try rows.nextPiece() else { return nil } + let lines = piece.reduce(into: 0) { count, byte in + if byte == Self.lineFeed { count += 1 } + } + let note = lock.withLock { () -> Int? in + linesSent += lines + guard linesNoted.map({ linesSent - $0 >= Self.linesPerNote }) ?? true else { return nil } + linesNoted = linesSent + return linesSent + } + if let note { + progress.setStatus(String( + format: String(localized: "COPY at line %@: %@ lines sent"), + line.formatted(), + note.formatted() + )) + } + return piece + } +} diff --git a/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift b/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift index 137d0a7ac3..bdff60a382 100644 --- a/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift +++ b/apple/Plugins/SQLImportPlugin/SQLImportPlugin.swift @@ -69,13 +69,15 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi try await sink.beginTransaction() } - let stream = try await source.statements() + let stream = try await source.elements() - for try await (statement, lineNumber) in stream { + for try await element in stream { try progress.checkCancellation() + let statement = element.statementText + let lineNumber = element.lineNumber do { - try await sink.execute(statement: statement, line: lineNumber) + try await Self.run(element, on: sink, progress: progress) executedCount += 1 progress.incrementStatement() } catch is PluginImportCancellationError { @@ -178,4 +180,23 @@ final class SQLImportPlugin: ObservableObject, ImportFormatPlugin, SettablePlugi errors: errors ) } + + private static func run( + _ element: PluginImportElement, + on sink: any PluginImportDataSink, + progress: PluginImportProgress + ) async throws { + switch element { + case .statement(let statement, let lineNumber): + try await sink.execute(statement: statement, line: lineNumber) + case .copyFromStandardInput(let statement, let lineNumber, let rows): + let watched = SQLImportCopyRows.watching(rows, line: lineNumber, progress: progress) + try await sink.copyFromStandardInput(statement: statement, line: lineNumber, rows: watched) + @unknown default: + throw PluginImportError.importFailed(String( + format: String(localized: "This version of SQL Import cannot run line %@."), + element.lineNumber.formatted() + )) + } + } } diff --git a/apple/Plugins/TableProPluginKit/PluginCapabilities.swift b/apple/Plugins/TableProPluginKit/PluginCapabilities.swift index 1efed41fc8..0eb929ecbc 100644 --- a/apple/Plugins/TableProPluginKit/PluginCapabilities.swift +++ b/apple/Plugins/TableProPluginKit/PluginCapabilities.swift @@ -47,4 +47,7 @@ public struct PluginCapabilities: OptionSet, Sendable { // A `PluginStatementContext.enforcesReadOnly` statement runs in the engine's read-only mode or throws, from every // entry point that takes a context, `executeBoundedQuery(query:rowCap:context:)` included. public static let enforcesReadOnlyMode = PluginCapabilities(rawValue: 1 << 24) + // The driver runs a `COPY ... FROM STDIN` and sends its rows through + // `PluginDatabaseDriver.copyFromStandardInput(statement:rows:context:)`. + public static let copyFromStandardInput = PluginCapabilities(rawValue: 1 << 25) } diff --git a/apple/Plugins/TableProPluginKit/PluginCopyData.swift b/apple/Plugins/TableProPluginKit/PluginCopyData.swift new file mode 100644 index 0000000000..7a49c89e39 --- /dev/null +++ b/apple/Plugins/TableProPluginKit/PluginCopyData.swift @@ -0,0 +1,24 @@ +// +// PluginCopyData.swift +// TableProPluginKit +// + +import Foundation + +/// The rows a `COPY ... FROM STDIN` loads, handed over a piece at a time so a large block is never held whole. +/// +/// The driver pulls each piece on the thread that holds its connection for the whole copy, so a read may block on a +/// file. A piece can end anywhere, inside a row too, and holds row data only: never psql's `\.` line. +public struct PluginCopyData: Sendable { + private let read: @Sendable () throws -> Data? + + public init(read: @escaping @Sendable () throws -> Data?) { + self.read = read + } + + /// The next piece, encoded for the session (UTF-8 here), or nil once every row was read. A throw ends the copy + /// with a failure, so the server keeps none of its rows, and the driver rethrows it. + public func nextPiece() throws -> Data? { + try read() + } +} diff --git a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index 642043655e..991d7721a6 100644 --- a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -145,6 +145,15 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { /// statements after it may still have run and returned results. func executeBatch(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginBatchResult? + /// Runs `statement`, a `COPY ... FROM STDIN`, and sends it `rows` a piece at a time through the engine's copy + /// protocol, holding the session until the copy ends. A driver declares `PluginCapabilities.copyFromStandardInput` + /// for it; the default throws `PluginDriverUnsupportedOperation.copyFromStandardInput`. + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext + ) async throws -> PluginQueryResult + func fetchTables(schema: String?) async throws -> [PluginTableInfo] /// What `fetchTables(schema:)` lists for every schema `fetchSchemas()` lists, in one call, each @@ -1510,6 +1519,14 @@ public extension PluginDatabaseDriver { nil } + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + throw PluginDriverUnsupportedOperation.copyFromStandardInput + } + /// The bounded read for a driver whose `streamRows` yields rows as they arrive and whose /// producer stops when its consumer does. Opt in by returning this from `executeBoundedQuery`. /// diff --git a/apple/Plugins/TableProPluginKit/PluginDriverUnsupportedOperation.swift b/apple/Plugins/TableProPluginKit/PluginDriverUnsupportedOperation.swift index 75d947a168..695612e291 100644 --- a/apple/Plugins/TableProPluginKit/PluginDriverUnsupportedOperation.swift +++ b/apple/Plugins/TableProPluginKit/PluginDriverUnsupportedOperation.swift @@ -22,6 +22,7 @@ public enum PluginDriverUnsupportedOperation: Error, LocalizedError, Sendable { case writeDocument case explainPlan case importExportedDatabase + case copyFromStandardInput public var errorDescription: String? { switch self { @@ -37,6 +38,8 @@ public enum PluginDriverUnsupportedOperation: Error, LocalizedError, Sendable { return String(localized: "This driver cannot explain a statement") case .importExportedDatabase: return String(localized: "This driver cannot restore an exported folder. Update it in Settings > Extensions.") + case .copyFromStandardInput: + return String(localized: "This driver cannot send rows to COPY FROM STDIN.") } } } diff --git a/apple/Plugins/TableProPluginKit/PluginImportDataSink.swift b/apple/Plugins/TableProPluginKit/PluginImportDataSink.swift index 21d25f4d95..440057949f 100644 --- a/apple/Plugins/TableProPluginKit/PluginImportDataSink.swift +++ b/apple/Plugins/TableProPluginKit/PluginImportDataSink.swift @@ -15,6 +15,9 @@ public protocol PluginImportDataSink: AnyObject, Sendable { /// its lines from the batch's first. func execute(statement: String, line: Int) async throws + /// Runs a `COPY ... FROM STDIN` read from the file's `line` and sends it `rows`. + func copyFromStandardInput(statement: String, line: Int, rows: PluginCopyData) async throws + func insertRow(_ values: [String: PluginCellValue]) async throws func insertRows(_ rows: [[String: PluginCellValue]]) async throws func deleteAllRowsFromTargetTable() async throws @@ -30,6 +33,11 @@ public extension PluginImportDataSink { func execute(statement: String, line: Int) async throws { try await execute(statement: statement) } + func copyFromStandardInput(statement: String, line: Int, rows: PluginCopyData) async throws { + throw PluginImportError.importFailed( + String(localized: "This connection cannot load the rows of COPY FROM STDIN.") + ) + } func insertRow(_ values: [String: PluginCellValue]) async throws { throw PluginImportError.importFailed("Row-based import is not supported by this connection") } diff --git a/apple/Plugins/TableProPluginKit/PluginImportElement.swift b/apple/Plugins/TableProPluginKit/PluginImportElement.swift new file mode 100644 index 0000000000..0f26191005 --- /dev/null +++ b/apple/Plugins/TableProPluginKit/PluginImportElement.swift @@ -0,0 +1,31 @@ +// +// PluginImportElement.swift +// TableProPluginKit +// + +import Foundation + +/// One thing a SQL file asks the import to run, in file order. +public enum PluginImportElement: Sendable { + case statement(String, lineNumber: Int) + + /// A `COPY ... FROM STDIN` and the rows the file holds after it. The rows are read from the file while they are + /// sent; once the next element is asked for, any left unread are passed over. + case copyFromStandardInput(statement: String, lineNumber: Int, rows: PluginCopyData) +} + +public extension PluginImportElement { + var statementText: String { + switch self { + case .statement(let statement, _), .copyFromStandardInput(let statement, _, _): + return statement + } + } + + var lineNumber: Int { + switch self { + case .statement(_, let lineNumber), .copyFromStandardInput(_, let lineNumber, _): + return lineNumber + } + } +} diff --git a/apple/Plugins/TableProPluginKit/PluginImportSource.swift b/apple/Plugins/TableProPluginKit/PluginImportSource.swift index ca602a0ee5..9d857a1781 100644 --- a/apple/Plugins/TableProPluginKit/PluginImportSource.swift +++ b/apple/Plugins/TableProPluginKit/PluginImportSource.swift @@ -7,6 +7,11 @@ import Foundation public protocol PluginImportSource: AnyObject, Sendable { func statements() async throws -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> + + /// The file's statements and `COPY ... FROM STDIN` blocks, in order. A source that reads no `COPY` rows answers + /// with its statements. + func elements() async throws -> AsyncThrowingStream + func fileURL() -> URL func fileSizeBytes() -> Int64 func cleanup() @@ -14,4 +19,25 @@ public protocol PluginImportSource: AnyObject, Sendable { public extension PluginImportSource { func cleanup() {} + + func elements() async throws -> AsyncThrowingStream { + let statements = StatementElements(try await statements()) + return AsyncThrowingStream(unfolding: { + try await statements.next() + }) + } +} + +// The stream asks for one element at a time, so its iterator is never used from two places at once. +private final class StatementElements: @unchecked Sendable { + private var statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>.AsyncIterator + + init(_ statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>) { + self.statements = statements.makeAsyncIterator() + } + + func next() async throws -> PluginImportElement? { + guard let (statement, lineNumber) = try await statements.next() else { return nil } + return .statement(statement, lineNumber: lineNumber) + } } diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift index 86c162abfd..e9d278d3dc 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift @@ -55,14 +55,21 @@ extension QueryExecutionCoordinator { } } - // A context is Read-Only's, and every call below carries it. + // A context is Read-Only's, and every call below carries it. A COPY with rows never goes as a header alone: a driver + // without copy-in refuses before anything is sent, where some servers drop the connection on the header. func executeStatement( rowCap: Int?, originalSQL: String, driver: DatabaseDriver, parameters: [Any?]? = nil, + copyData: String? = nil, context: PluginStatementContext? = nil ) async throws -> QueryResult { + if let copyData { + return try await driver.copyFromStandardInput( + statement: originalSQL, rows: CopyDataPieces.reading(copyData), context: context + ) + } if let context { return try await executeStatement( rowCap: rowCap, originalSQL: originalSQL, driver: driver, parameters: parameters, carrying: context diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift index f413c7d2e9..5c92b35101 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift @@ -414,7 +414,8 @@ extension QueryExecutionCoordinator { prepared: prepared, grammar: grammar, script: Self.scriptHolds(statements, in: script) ? script : nil, - end: statements.last.map { NSMaxRange($0.range) } ?? 0 + // The segment's end, so a last COPY cut again keeps its rows. + end: statements.last?.segmentEnd ?? 0 ) { statement, grammar in self.prepareStatement( statement: statement, @@ -617,6 +618,7 @@ extension QueryExecutionCoordinator { originalSQL: statement.sentSQL, driver: driver, parameters: statement.parameterValues, + copyData: statement.statement?.copyData, context: context ) } diff --git a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift index cb24fe0eac..800db08fcd 100644 --- a/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift +++ b/apple/TablePro/Core/Database/Access/DatabaseAccessBridge.swift @@ -290,6 +290,10 @@ internal actor DatabaseAccessBridge { ) case .batches: return PlannedText(statement: OperationStatement(sql: query), unit: .script) + case .statements(let statements) where statements.contains(where: { $0.copyData != nil }): + throw DatabaseAccessError.invalidArgument( + String(localized: "COPY FROM STDIN takes its rows from the lines after it, which only the query editor and SQL import send. Run it there.") + ) case .statements: throw DatabaseAccessError.invalidArgument( String(localized: "Update the database driver in Settings > Extensions to run several statements in one call.") diff --git a/apple/TablePro/Core/Database/DatabaseDriver.swift b/apple/TablePro/Core/Database/DatabaseDriver.swift index 830a54fc0e..8a7ee251c0 100644 --- a/apple/TablePro/Core/Database/DatabaseDriver.swift +++ b/apple/TablePro/Core/Database/DatabaseDriver.swift @@ -144,6 +144,17 @@ protocol DatabaseDriver: AnyObject, Sendable { /// text statement by statement instead. A server error inside the batch is part of the answer, not a throw. func executeBatch(query: String, rowCap: Int?, parameters: [Any?]?) async throws -> QueryBatchResult? + /// Whether ``copyFromStandardInput(statement:rows:context:)`` sends a `COPY ... FROM STDIN` its rows. + var supportsCopyFromStandardInput: Bool { get } + + /// Runs `statement`, a `COPY ... FROM STDIN`, and sends it `rows` a piece at a time, holding the session until the + /// copy ends. Throws when the driver cannot. + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext? + ) async throws -> QueryResult + func explainPlan(statement: String, variantId: String) async throws -> QueryResult // MARK: - Schema Operations @@ -624,6 +635,16 @@ extension DatabaseDriver { func executeBatch(query: String, rowCap: Int?, parameters: [Any?]?) async throws -> QueryBatchResult? { nil } + var supportsCopyFromStandardInput: Bool { false } + + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext? + ) async throws -> QueryResult { + throw PluginDriverUnsupportedOperation.copyFromStandardInput + } + func explainPlan(statement: String, variantId: String) async throws -> QueryResult { throw PluginDriverUnsupportedOperation.explainPlan } diff --git a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift index 8898c1c535..4b71c40824 100644 --- a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift +++ b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift @@ -234,6 +234,13 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { } } + func copyFromStandardInput(statement: String, line: Int, rows: PluginCopyData) async throws { + guard driver.supportsCopyFromStandardInput else { throw SQLFileParseError.copyFromStandardInput(line: line) } + try await importedData(writes: true, statement: statement) { + _ = try await self.driver.copyFromStandardInput(statement: statement, rows: rows, context: nil) + } + } + // T-SQL scopes a variable and a TRY...CATCH to its batch, so a batch goes whole where the driver can send it so. private func send(_ statement: String, line: Int, grammar: SQLLexicalGrammar) async throws { guard grammar.contains(.batchSeparatorLines) else { diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter+CopyIn.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter+CopyIn.swift new file mode 100644 index 0000000000..df153629f4 --- /dev/null +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter+CopyIn.swift @@ -0,0 +1,27 @@ +// +// PluginDriverAdapter+CopyIn.swift +// TablePro +// + +import Foundation +import TableProPluginKit + +extension PluginDriverAdapter { + var supportsCopyFromStandardInput: Bool { + schemaPluginDriver.capabilities.contains(.copyFromStandardInput) + } + + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext? + ) async throws -> QueryResult { + try StatementTextValidator.validate(statement) + // The rows come from the editor's text or an imported file, never from an external caller. + guard supportsCopyFromStandardInput, context?.origin != .external else { + throw PluginDriverUnsupportedOperation.copyFromStandardInput + } + let sent = context.map { statementContext(statement, approvedAs: $0) } ?? statementContext(statement) + return mapQueryResult(try await schemaPluginDriver.copyFromStandardInput(statement: statement, rows: rows, context: sent)) + } +} diff --git a/apple/TablePro/Core/Plugins/SqlFileImportSource.swift b/apple/TablePro/Core/Plugins/SqlFileImportSource.swift index 3262cad7d5..aa5fe6971f 100644 --- a/apple/TablePro/Core/Plugins/SqlFileImportSource.swift +++ b/apple/TablePro/Core/Plugins/SqlFileImportSource.swift @@ -63,23 +63,29 @@ final class SqlFileImportSource: PluginImportSource, @unchecked Sendable { let grammar = self.grammar.withLock { $0 } let parsed = parser.parseFile(url: fileURL, encoding: encoding, grammar: grammar) guard !sendsFileBytesUnchanged else { return parsed } - return Self.sentAsUTF8(parsed, decodedFrom: encoding, family: family, grammar: grammar) + let rewriting = TranscodedDumpStatements(parsed, rewriter: rewriter(grammar: grammar)) + return AsyncThrowingStream(unfolding: { + try await rewriting.next() + }) + } + + func elements() async throws -> AsyncThrowingStream { + let fileURL = try await resolveURL() + let grammar = self.grammar.withLock { $0 } + let parsed = parser.parseElements(url: fileURL, encoding: encoding, grammar: grammar) + guard !sendsFileBytesUnchanged else { return parsed } + let rewriting = TranscodedDumpElements(parsed, rewriter: rewriter(grammar: grammar)) + return AsyncThrowingStream(unfolding: { + try await rewriting.next() + }) } private var sendsFileBytesUnchanged: Bool { encoding == .utf8 || encoding == .ascii } - private static func sentAsUTF8( - _ statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>, - decodedFrom encoding: String.Encoding, - family: TransactionEngineFamily, - grammar: @escaping @Sendable () -> SQLLexicalGrammar - ) -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { - let rewriting = TranscodedDumpStatements(statements, decodedFrom: encoding, family: family, grammar: grammar) - return AsyncThrowingStream(unfolding: { - try await rewriting.next() - }) + private func rewriter(grammar: @escaping @Sendable () -> SQLLexicalGrammar) -> TranscodedStatementRewriter { + TranscodedStatementRewriter(encoding: encoding, family: family, grammar: grammar) } /// `ownsDecompressedFile` answers only whether the caller handed over a file to delete. A file @@ -158,34 +164,52 @@ internal enum SqlFileImportError: LocalizedError, Equatable { } private final class TranscodedDumpStatements: @unchecked Sendable { - private static let logger = Logger(subsystem: "com.TablePro", category: "SqlFileImportSource") - private var statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>.AsyncIterator - private let encoding: String.Encoding - private let family: TransactionEngineFamily - private let grammar: @Sendable () -> SQLLexicalGrammar + private let rewriter: TranscodedStatementRewriter - init( - _ statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>, - decodedFrom encoding: String.Encoding, - family: TransactionEngineFamily, - grammar: @escaping @Sendable () -> SQLLexicalGrammar - ) { + init(_ statements: AsyncThrowingStream<(statement: String, lineNumber: Int), Error>, rewriter: TranscodedStatementRewriter) { self.statements = statements.makeAsyncIterator() - self.encoding = encoding - self.family = family - self.grammar = grammar + self.rewriter = rewriter } func next() async throws -> (statement: String, lineNumber: Int)? { guard let (statement, lineNumber) = try await statements.next() else { return nil } + return (try rewriter.rewritten(statement, at: lineNumber), lineNumber) + } +} + +// COPY rows pass unchanged: the parser hands them out as UTF-8 already. +private final class TranscodedDumpElements: @unchecked Sendable { + private var elements: AsyncThrowingStream.AsyncIterator + private let rewriter: TranscodedStatementRewriter + + init(_ elements: AsyncThrowingStream, rewriter: TranscodedStatementRewriter) { + self.elements = elements.makeAsyncIterator() + self.rewriter = rewriter + } + + func next() async throws -> PluginImportElement? { + guard let element = try await elements.next() else { return nil } + guard case let .statement(statement, lineNumber) = element else { return element } + return .statement(try rewriter.rewritten(statement, at: lineNumber), lineNumber: lineNumber) + } +} + +private struct TranscodedStatementRewriter: Sendable { + private static let logger = Logger(subsystem: "com.TablePro", category: "SqlFileImportSource") + + let encoding: String.Encoding + let family: TransactionEngineFamily + let grammar: @Sendable () -> SQLLexicalGrammar + + func rewritten(_ statement: String, at lineNumber: Int) throws -> String { let grammar = self.grammar() let literals = try rewrittenLiterals(of: statement, at: lineNumber, grammar: grammar) ?? statement guard let rewritten = SQLClientEncodingRewriter.rewritten(literals, family: family, grammar: grammar) else { - return (literals, lineNumber) + return literals } Self.logger.info("Kept the session's client encoding at line \(lineNumber, privacy: .public), since the import sends UTF-8") - return (rewritten, lineNumber) + return rewritten } private func rewrittenLiterals(of statement: String, at lineNumber: Int, grammar: SQLLexicalGrammar) throws -> String? { diff --git a/apple/TablePro/Core/Services/Execution/CopyDataPieces.swift b/apple/TablePro/Core/Services/Execution/CopyDataPieces.swift new file mode 100644 index 0000000000..86300eea35 --- /dev/null +++ b/apple/TablePro/Core/Services/Execution/CopyDataPieces.swift @@ -0,0 +1,26 @@ +// +// CopyDataPieces.swift +// TablePro +// + +import Foundation +import os +import TableProPluginKit + +enum CopyDataPieces { + static let pieceSize = 64 * 1_024 + + // A piece may end inside a row, even inside a character: the server reads a copy's data as one byte stream. + static func reading(_ text: String) -> PluginCopyData { + let data = Data(text.utf8) + let offset = OSAllocatedUnfairLock(initialState: data.startIndex) + return PluginCopyData { + offset.withLock { start in + guard start < data.endIndex else { return nil } + let end = min(start + Self.pieceSize, data.endIndex) + defer { start = end } + return data.subdata(in: start.. QueryExecutionRoute? { let statements = batches.flatMap(\.statements) guard let first = statements.first else { return nil } - guard sendsBatchesWhole else { + let carriesCopyData = statements.contains { $0.copyData != nil } + guard sendsBatchesWhole, !carriesCopyData else { guard batches.allSatisfy({ $0.repeatCount == 1 }) else { return .needsBatchDriver } - return statements.count == 1 ? .single(first) : .statements(statements) + return statements.count == 1 && !carriesCopyData ? .single(first) : .statements(statements) } if statements.count == 1, batches.first?.repeatCount == 1, isPlainQuery(first.sql) { return .single(first) diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift new file mode 100644 index 0000000000..87ef982eaa --- /dev/null +++ b/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift @@ -0,0 +1,153 @@ +// +// SQLFileParser+CopyRows.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar + +extension SQLFileParser { + static let defaultChunkSize = 65_536 + + // A UTF-16 unit takes at most three UTF-8 bytes, so a piece of COPY rows stays within 1 MiB. + static let defaultCopyPieceLength = 1_048_576 / 3 + + struct ReadLimits { + let batchCutLength: Int + let chunkSize: Int + let copyPieceLength: Int + } + + // Reads the rows as psql 17 and 18 do (SQLPsqlScript.copyRows), from a buffer that holds part of the file. + struct CopyRowsScanner { + enum Step { + // Read more, then scan again from the same start. + case needsMoreText + // `length` units of data, then `skipped` units that are never sent: the rest of the `;`'s line, or the + // `\.` line with its line break. + case advance(length: Int, skipped: Int, lineFeeds: Int, endsRows: Bool) + } + + private let pieceLength: Int + private var passesCommandLine = true + // Where the scan stands relative to `start`, which stays put until a step hands units out. + private var lineStart = 0 + private var searchedTo = 0 + private var lineFeeds = 0 + private var checksLine = true + + init(pieceLength: Int) { + // A cut inside a line steps back off a lead surrogate and must still take a unit. + self.pieceLength = max(2, pieceLength) + } + + mutating func scan(_ text: NSString, from start: Int, atEndOfInput: Bool) -> Step { + let available = text.length - start + if passesCommandLine { + return passCommandLine(text, from: start, available: available, atEndOfInput: atEndOfInput) + } + while true { + if checksLine { + if lineStart >= pieceLength { + return cut(at: lineStart, insideLine: false) + } + if available - lineStart < SQLPsqlScript.endOfDataMarkerLookahead, !atEndOfInput { + return .needsMoreText + } + if let markerEnd = SQLPsqlScript.endOfDataMarker(in: text, at: start + lineStart, length: text.length) { + return endAtMarker(text, from: start, markerEnd: markerEnd) + } + if lineStart == available { + return .advance(length: lineStart, skipped: 0, lineFeeds: lineFeeds, endsRows: true) + } + checksLine = false + searchedTo = lineStart + } + let found = text.range( + of: "\n", + options: .literal, + range: NSRange(location: start + searchedTo, length: available - searchedTo) + ) + guard found.location != NSNotFound else { + if available > pieceLength { + return cutBeforeOverflow(text, from: start) + } + if atEndOfInput { + return .advance(length: available, skipped: 0, lineFeeds: lineFeeds, endsRows: true) + } + searchedTo = available + return .needsMoreText + } + let lineEnd = found.location - start + 1 + if lineEnd > pieceLength { + return cutBeforeOverflow(text, from: start) + } + lineFeeds += 1 + lineStart = lineEnd + checksLine = true + } + } + + private mutating func passCommandLine( + _ text: NSString, + from start: Int, + available: Int, + atEndOfInput: Bool + ) -> Step { + let found = text.range(of: "\n", options: .literal, range: NSRange(location: start, length: available)) + if found.location != NSNotFound { + passesCommandLine = false + return .advance(length: 0, skipped: found.location - start + 1, lineFeeds: 1, endsRows: false) + } + guard available > 0 || atEndOfInput else { return .needsMoreText } + return .advance(length: 0, skipped: available, lineFeeds: 0, endsRows: atEndOfInput) + } + + // The line being read runs past the piece: end the piece at the line before it, or inside it when it is the + // piece's first. + private mutating func cutBeforeOverflow(_ text: NSString, from start: Int) -> Step { + if lineStart > 0 { + return cut(at: lineStart, insideLine: false) + } + let splitsPair = UTF16.isLeadSurrogate(text.character(at: start + pieceLength - 1)) + return cut(at: splitsPair ? pieceLength - 1 : pieceLength, insideLine: true) + } + + private mutating func cut(at length: Int, insideLine: Bool) -> Step { + let step = Step.advance(length: length, skipped: 0, lineFeeds: lineFeeds, endsRows: false) + lineStart = 0 + searchedTo = 0 + lineFeeds = 0 + checksLine = !insideLine + return step + } + + private func endAtMarker(_ text: NSString, from start: Int, markerEnd: Int) -> Step { + var end = markerEnd + var feeds = lineFeeds + if end < text.length, text.character(at: end) == SqlLexer.carriageReturn { + end += 1 + } + if end < text.length, text.character(at: end) == SqlLexer.newline { + end += 1 + feeds += 1 + } + return .advance(length: lineStart, skipped: end - start - lineStart, lineFeeds: feeds, endsRows: true) + } + } +} + +extension SQLFileParser.ParserContext { + // SQL resumes on the line after a COPY's rows with nothing of a statement read. + mutating func resumeAfterCopyRows() { + SQLFileParser.resetStatement(&self) + state = .normal + lineHasCode = false + lineHoldsOnlyBlanks = true + previousUnit = SQLFileParser.kNewline + previousUnitInWord = false + pendingSlashLine = false + pendingSlashTrailing.removeAll() + separatorLineSearched = 0 + } +} diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift index 0edc2c3d64..475e977f90 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift @@ -18,7 +18,7 @@ enum SQLFileParseError: LocalizedError, Equatable { case .copyFromStandardInput(let line): return String( format: String( - localized: "Line %lld runs COPY FROM STDIN, whose rows TablePro cannot send. Run the file with psql, or dump it again with --inserts." + localized: "Line %lld runs COPY FROM STDIN, and this connection cannot load its rows. Run the file with psql, or dump it again with --inserts." ), line ) @@ -36,10 +36,14 @@ final class SQLFileParser: Sendable { /// and held one piece at a time rather than whole. static let defaultBatchCutLength = 67_108_864 - private let batchCutLength: Int + private let limits: ReadLimits - init(batchCutLength: Int = SQLFileParser.defaultBatchCutLength) { - self.batchCutLength = batchCutLength + init( + batchCutLength: Int = SQLFileParser.defaultBatchCutLength, + chunkSize: Int = SQLFileParser.defaultChunkSize, + copyPieceLength: Int = SQLFileParser.defaultCopyPieceLength + ) { + limits = ReadLimits(batchCutLength: batchCutLength, chunkSize: chunkSize, copyPieceLength: copyPieceLength) } /// One statement the file holds, or on a grammar that cuts scripts into batches, one batch, with how many times @@ -48,6 +52,7 @@ final class SQLFileParser: Sendable { let statement: String let lineNumber: Int let repeatCount: Int + var readsCopyRows = false } enum ParserState { @@ -250,8 +255,6 @@ final class SQLFileParser: Sendable { var lineHasCode = false /// Whether the statement's first unit was the first code on its line, where a `DELIMITER` can stand. var statementOpensLine = false - /// The line of a `COPY ... FROM STDIN` the file holds, which no driver here can send rows to. - var copyFromStandardInputLine: Int? var pendingSlashLine = false var pendingSlashTrailing: [unichar] = [] @@ -330,7 +333,7 @@ final class SQLFileParser: Sendable { .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" } - private static func resetStatement(_ ctx: inout ParserContext) { + static func resetStatement(_ ctx: inout ParserContext) { ctx.currentStatement?.setString("") ctx.hasStatementContent = false ctx.insideConditionalComment = false @@ -781,7 +784,10 @@ final class SQLFileParser: Sendable { return } if ctx.boundaries?.copiesFromStandardInput == true { - ctx.copyFromStandardInputLine = ctx.statementStartLine + ctx.collected.append(ParsedStatement( + statement: trimmedStatement(ctx), lineNumber: ctx.statementStartLine, repeatCount: 1, readsCopyRows: true + )) + resetStatement(&ctx) return } if ctx.boundaries?.awaitsContinuation == true, ctx.hasStatementContent { @@ -963,23 +969,39 @@ final class SQLFileParser: Sendable { encoding: String.Encoding, grammar: @escaping @Sendable () -> SQLLexicalGrammar ) -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { - let session = ParseSession( - url: url, encoding: encoding, grammar: grammar, countOnly: false, batchCutLength: batchCutLength - ) + let session = ParseSession(url: url, encoding: encoding, grammar: grammar, countOnly: false, limits: limits) return AsyncThrowingStream(unfolding: { try await session.nextRun() }) } - // Runs are added up rather than walked: a `GO n` count can reach `Int32.max`. + func parseElements( + url: URL, + encoding: String.Encoding, + grammar: SQLLexicalGrammar + ) -> AsyncThrowingStream { + parseElements(url: url, encoding: encoding, grammar: { grammar }) + } + + // The statements as parseFile yields them, and each `COPY ... FROM STDIN` with a reader over its rows. + func parseElements( + url: URL, + encoding: String.Encoding, + grammar: @escaping @Sendable () -> SQLLexicalGrammar + ) -> AsyncThrowingStream { + let session = ParseSession(url: url, encoding: encoding, grammar: grammar, countOnly: false, limits: limits) + return AsyncThrowingStream(unfolding: { + try await session.nextElement() + }) + } + + // Runs are added up rather than walked: a `GO n` count can reach `Int32.max`. A COPY counts as one. func countStatements( url: URL, encoding: String.Encoding, grammar: SQLLexicalGrammar ) async throws -> Int { - let session = ParseSession( - url: url, encoding: encoding, grammar: { grammar }, countOnly: true, batchCutLength: batchCutLength - ) + let session = ParseSession(url: url, encoding: encoding, grammar: { grammar }, countOnly: true, limits: limits) var count = 0 while let statement = try await session.nextStatement() { @@ -1005,8 +1027,7 @@ extension SQLFileParser { private let url: URL private let encoding: String.Encoding private let grammar: @Sendable () -> SQLLexicalGrammar - private let batchCutLength: Int - private let chunkSize = 65_536 + private let limits: ReadLimits private var fileHandle: FileHandle? private var ctx: ParserContext @@ -1020,18 +1041,24 @@ extension SQLFileParser { private var repeating: ParsedStatement? private var remainingRuns = 0 + // The rows of the COPY last yielded while any are unread, and which COPY that is, so a reader handed out + // for an earlier one answers nil. + private var copyRows: CopyRowsScanner? + private var copyGeneration = 0 + private var rowsFailure: Error? + init( url: URL, encoding: String.Encoding, grammar: @escaping @Sendable () -> SQLLexicalGrammar, countOnly: Bool, - batchCutLength: Int, + limits: ReadLimits, reading: Reading = .fromFile ) { self.url = url self.encoding = encoding self.grammar = grammar - self.batchCutLength = batchCutLength + self.limits = limits self.decoder = SQLChunkDecoder(encoding: encoding) self.readingSettled = reading == .batches let initialGrammar = grammar() @@ -1040,7 +1067,7 @@ extension SQLFileParser { self.ctx = ParserContext( grammar: initialGrammar, currentStatement: keepsText ? NSMutableString() : nil, - batchCutLength: batchCutLength, + batchCutLength: limits.batchCutLength, readsBatches: reading == .batches ) } @@ -1050,21 +1077,48 @@ extension SQLFileParser { } func nextRun() async throws -> (statement: String, lineNumber: Int)? { + let element = try await nextElement() + // The rows after the `COPY` would otherwise reach the server as statements. + if case let .copyFromStandardInput(_, line, _)? = element { + finished = true + closeFile() + throw SQLFileParseError.copyFromStandardInput(line: line) + } + guard case let .statement(statement, lineNumber)? = element else { return nil } + return (statement, lineNumber) + } + + func nextElement() async throws -> PluginImportElement? { if let repeating, remainingRuns > 0 { remainingRuns -= 1 - return (repeating.statement, repeating.lineNumber) + return .statement(repeating.statement, lineNumber: repeating.lineNumber) } guard let next = try await nextStatement() else { return nil } - repeating = next - remainingRuns = next.repeatCount - 1 - return (next.statement, next.lineNumber) + guard next.readsCopyRows else { + repeating = next + remainingRuns = next.repeatCount - 1 + return .statement(next.statement, lineNumber: next.lineNumber) + } + let generation = copyGeneration + // The driver reads pieces on its own queue while the import awaits the copy, and the import asks for the + // next element only once the copy returns, so the session is never used from two places at once. + let rows = PluginCopyData { [self] in try nextRowsPiece(generation: generation) } + return .copyFromStandardInput(statement: next.statement, lineNumber: next.lineNumber, rows: rows) } func nextStatement() async throws -> ParsedStatement? { + if let failure = rowsFailure { + rowsFailure = nil + throw failure + } while true { if emitIndex < ctx.collected.count { let item = ctx.collected[emitIndex] emitIndex += 1 + if item.readsCopyRows { + copyGeneration += 1 + copyRows = CopyRowsScanner(pieceLength: limits.copyPieceLength) + } return item } ctx.collected.removeAll(keepingCapacity: true) @@ -1075,6 +1129,7 @@ extension SQLFileParser { } do { + try passUnreadRows() try settleReading() guard !Task.isCancelled else { finished = true @@ -1103,14 +1158,14 @@ extension SQLFileParser { encoding: encoding, grammar: { grammar }, countOnly: true, - batchCutLength: batchCutLength, + limits: limits, reading: .batches ) guard try scan.findsBatchSeparator() else { return } ctx = ParserContext( grammar: grammar, currentStatement: ctx.currentStatement, - batchCutLength: batchCutLength, + batchCutLength: limits.batchCutLength, readsBatches: true ) } @@ -1125,18 +1180,20 @@ extension SQLFileParser { } private func readNextStatement() throws { - try refuseCopyFromStandardInput() processBuffer() guard ctx.collected.isEmpty else { return } - try refuseCopyFromStandardInput() guard !ctx.atEndOfInput else { emitTrailingStatement() finished = true closeFile() return } + try readChunk() + } + + private func readChunk() throws { let handle = try openFileIfNeeded() - let rawData = handle.readData(ofLength: chunkSize) + let rawData = handle.readData(ofLength: limits.chunkSize) if rawData.isEmpty && !decoder.hasPendingBytes { ctx.atEndOfInput = true @@ -1261,16 +1318,50 @@ extension SQLFileParser { nsBuffer, from: stepStart, to: min(i, bufLen), before: ctx.lineHoldsOnlyBlanks ) } - if !ctx.collected.isEmpty || ctx.copyFromStandardInputLine != nil { break } + if !ctx.collected.isEmpty { break } } readOffset = min(i, bufLen) } - // The rows after the `COPY` would otherwise reach the server as statements. - private func refuseCopyFromStandardInput() throws { - guard let line = ctx.copyFromStandardInputLine else { return } - throw SQLFileParseError.copyFromStandardInput(line: line) + private func nextRowsPiece(generation: Int) throws -> Data? { + do { + guard let piece = try nextRowsRange(generation: generation) else { return nil } + return Data(nsBuffer.substring(with: piece).utf8) + } catch { + rowsFailure = error + copyRows = nil + finished = true + closeFile() + SQLFileParser.logger.error("SQL file parsing failed: \(error.localizedDescription)") + throw error + } + } + + private func passUnreadRows() throws { + while copyRows != nil, !finished, !Task.isCancelled { + _ = try nextRowsRange(generation: copyGeneration) + } + } + + // The next piece's range in the buffer, valid until the next chunk is read. + private func nextRowsRange(generation: Int) throws -> NSRange? { + while !finished, generation == copyGeneration, + let step = copyRows?.scan(nsBuffer, from: readOffset, atEndOfInput: ctx.atEndOfInput) { + guard case let .advance(length, skipped, lineFeeds, endsRows) = step else { + try readChunk() + continue + } + let piece = NSRange(location: readOffset, length: length) + readOffset += length + skipped + ctx.currentLine += lineFeeds + if endsRows { + copyRows = nil + ctx.resumeAfterCopyRows() + } + if length > 0 { return piece } + } + return nil } private func emitTrailingStatement() { diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 93949bd22a..0e3b7a5837 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -17048,6 +17048,10 @@ } } }, + "A script cannot hold binary COPY data, so no rows were sent. Dump the rows as text or CSV." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "A script wants to run a query on \"%@\"" : { "localizations" : { "fr" : { @@ -51323,6 +51327,10 @@ }, "Copy as UPDATE Statements" : { + }, + "COPY at line %@: %@ lines sent" : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Copy Average" : { @@ -51797,6 +51805,13 @@ }, "Copy Fix SQL" : { + }, + "COPY FROM STDIN takes its rows from the lines after it, which only the query editor and SQL import send. Run it there." : { + + }, + "COPY FROM STDIN takes its rows from the lines after the line its ; ends, so no rows were sent." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Copy ID" : { "localizations" : { @@ -103782,6 +103797,9 @@ } } } + }, + "Line %lld runs COPY FROM STDIN, and this connection cannot load its rows. Run the file with psql, or dump it again with --inserts." : { + }, "Line %lld runs COPY FROM STDIN, whose rows TablePro cannot send. Run the file with psql, or dump it again with --inserts." : { @@ -198106,6 +198124,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "This connection cannot load the rows of COPY FROM STDIN." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This connection comes from a file in a linked folder, so its author chose the server and how it signs in." : { }, @@ -199734,6 +199756,10 @@ } } }, + "This driver cannot send rows to COPY FROM STDIN." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This driver has no TLS fallback. Preferred forces TLS, same as Required." : { "localizations" : { "fr" : { @@ -203545,6 +203571,10 @@ } } }, + "This version of SQL Import cannot run line %@." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This version of TablePro can no longer sync with iCloud. Update TablePro to resume." : { }, diff --git a/apple/TableProMobile/TableProMobile/Localizable.xcstrings b/apple/TableProMobile/TableProMobile/Localizable.xcstrings index 852d55e896..9c8eb437a2 100644 --- a/apple/TableProMobile/TableProMobile/Localizable.xcstrings +++ b/apple/TableProMobile/TableProMobile/Localizable.xcstrings @@ -6630,6 +6630,10 @@ } } }, + "COPY FROM STDIN takes its rows from the lines after the line its ; ends, so no rows were sent." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Copy Key" : { }, @@ -26840,6 +26844,10 @@ } } }, + "This connection cannot load the rows of COPY FROM STDIN." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This connection goes through an SSH jump host, which TablePro on iPhone and iPad cannot dial. Open it in TablePro on the Mac." : { "localizations" : { "ko" : { @@ -27328,6 +27336,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "This driver cannot send rows to COPY FROM STDIN." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This file also holds saved queries, which import only on a Mac." : { "localizations" : { "ko" : { diff --git a/apple/TableProTests/Core/Coordinators/CopyFromStandardInputRunTests.swift b/apple/TableProTests/Core/Coordinators/CopyFromStandardInputRunTests.swift new file mode 100644 index 0000000000..de62791f4c --- /dev/null +++ b/apple/TableProTests/Core/Coordinators/CopyFromStandardInputRunTests.swift @@ -0,0 +1,95 @@ +// +// CopyFromStandardInputRunTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProSQLGrammar +import Testing + +@MainActor +struct CopyFromStandardInputRunTests { + private static let script = "CREATE TABLE t (a int, b text);\n" + + "COPY t (a, b) FROM stdin;\n1\tone\n2\té\n\\.\n" + + "ANALYZE t;" + + private let connection = TestFixtures.makeConnection(type: .postgresql) + + private func run(_ script: String, on driver: CopyInRecordingDriver) async throws -> QueryTab { + let fixture = CoordinatorFixture(connection: connection) + var session = ConnectionSession(connection: connection, driver: driver) + session.status = .connected + session.browseDatabase = connection.database + DatabaseManager.shared.injectSession(session, for: connection.id) + defer { DatabaseManager.shared.removeSession(for: connection.id) } + + let tab = fixture.addTab(QueryTab(title: "Query", query: script, tabType: .query)) + let statements = SQLStatementScanner.executableStatements( + in: script, grammar: fixture.coordinator.lexicalGrammar + ) + fixture.coordinator.queryExecutionCoordinator.executeMultipleStatements(statements) + let handle = try #require(fixture.coordinator.queryTasks.task(for: tab.id)) + await handle.value + return try #require(fixture.tabManager.tabs.first { $0.id == tab.id }) + } + + @Test("A run sends a COPY's rows through copy-in with its header, byte for byte, and never the \\. line") + func copyGoesThroughCopyIn() async throws { + let driver = CopyInRecordingDriver(connection: connection, copiesIn: true) + + let tab = try await run(Self.script, on: driver) + + #expect(driver.sentCopies == [ + CopyInRecordingDriver.Copy(statement: "COPY t (a, b) FROM stdin", data: Data("1\tone\n2\té\n".utf8)), + ]) + #expect(driver.sentStatements == ["CREATE TABLE t (a int, b text)", "ANALYZE t"]) + #expect(!tab.display.resultSets.isEmpty) + #expect(tab.display.resultSets.allSatisfy { $0.errorMessage == nil }) + } + + @Test("A driver without copy-in refuses the COPY before anything is sent for it, and the run fails there") + func driverWithoutCopyInRefuses() async throws { + let driver = CopyInRecordingDriver(connection: connection, copiesIn: false) + + let tab = try await run(Self.script, on: driver) + + #expect(driver.sentCopies.isEmpty) + #expect(driver.sentStatements == ["CREATE TABLE t (a int, b text)"]) + let failed = try #require(tab.display.resultSets.last) + #expect(failed.failedQuery == "COPY t (a, b) FROM stdin") + let refusal = try #require(PluginDriverUnsupportedOperation.copyFromStandardInput.errorDescription) + #expect(failed.errorMessage?.contains(refusal) == true) + } + + @Test("A COPY with no rows still goes through copy-in, as a copy of none") + func copyWithoutRows() async throws { + let driver = CopyInRecordingDriver(connection: connection, copiesIn: true) + + _ = try await run("COPY t FROM stdin;", on: driver) + + #expect(driver.sentCopies == [CopyInRecordingDriver.Copy(statement: "COPY t FROM stdin", data: Data())]) + #expect(driver.sentStatements.isEmpty) + } + + @Test("A COPY that ends a run cut again after a SET keeps its rows") + func replannedCopyKeepsItsRows() async throws { + let standard = PluginSessionLexicalState(determined: [.backslashEscapesInSingleQuotes], enabled: []) + let escapes = PluginSessionLexicalState( + determined: [.backslashEscapesInSingleQuotes], enabled: [.backslashEscapesInSingleQuotes] + ) + #expect(SQLLexicalResolver.readings(for: .postgresql, session: standard).execution + != SQLLexicalResolver.readings(for: .postgresql, session: escapes).execution) + let driver = CopyInRecordingDriver( + connection: connection, copiesIn: true, lexicalState: standard, lexicalStateAfterFirstStatement: escapes + ) + + _ = try await run("SET standard_conforming_strings = off;\nCOPY t FROM stdin;\n1\tC:\\\\dir\n\\.", on: driver) + + #expect(driver.sentStatements == ["SET standard_conforming_strings = off"]) + #expect(driver.sentCopies == [ + CopyInRecordingDriver.Copy(statement: "COPY t FROM stdin", data: Data("1\tC:\\\\dir\n".utf8)), + ]) + } +} diff --git a/apple/TableProTests/Core/Database/DatabaseAccessBridgeCopyTests.swift b/apple/TableProTests/Core/Database/DatabaseAccessBridgeCopyTests.swift new file mode 100644 index 0000000000..842b5f1064 --- /dev/null +++ b/apple/TableProTests/Core/Database/DatabaseAccessBridgeCopyTests.swift @@ -0,0 +1,37 @@ +// +// DatabaseAccessBridgeCopyTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@Suite(.serialized) +@MainActor +struct DatabaseAccessBridgeCopyTests { + @Test("An external caller's COPY FROM STDIN is refused before anything is sent, rows or not", arguments: [ + "COPY t FROM stdin;\n1\tone\n\\.\n", + "COPY t FROM stdin;", + ]) + func copyIsRefused(sql: String) async throws { + let driver = ScriptAnsweringDriver( + connection: TestFixtures.makeConnection(database: "app", type: .postgresql), + sendsBatchesWhole: false + ) + var session = ConnectionSession(connection: driver.connection) + session.driver = driver + DatabaseManager.shared.injectSession(session, for: driver.connection.id) + defer { DatabaseManager.shared.removeSession(for: driver.connection.id) } + let scope = DatabaseScope(connectionId: driver.connection.id, database: "app", schema: nil) + + await #expect { + _ = try await DatabaseAccessBridge().plannedScript(scope: scope, query: sql, maxRows: 10) + } throws: { error in + guard case .invalidArgument(let message) = error as? DatabaseAccessError else { return false } + return message.contains("COPY FROM STDIN") + } + #expect(driver.sentStatements.isEmpty) + } +} diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterCopyTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterCopyTests.swift new file mode 100644 index 0000000000..4417e5f628 --- /dev/null +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterCopyTests.swift @@ -0,0 +1,82 @@ +// +// ImportDataSinkAdapterCopyTests.swift +// TableProTests +// + +import Foundation +import os +@testable import TablePro +import TableProPluginKit +import Testing + +private final class CopyingDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private let declaresCopy: Bool + private var copies: [(statement: String, data: Data)] = [] + + init(declaresCopy: Bool) { + self.declaresCopy = declaresCopy + } + + var capabilities: PluginCapabilities { + declaresCopy ? [.transactions, .copyFromStandardInput] : [.transactions] + } + + var received: [(statement: String, data: Data)] { + lock.withLock { copies } + } + + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + var data = Data() + while let piece = try rows.nextPiece() { + data.append(piece) + } + lock.withLock { copies.append((statement, data)) } + return .empty + } +} + +struct ImportDataSinkAdapterCopyTests { + private static func rows(_ text: String) -> PluginCopyData { + let remaining = OSAllocatedUnfairLock(initialState: Data(text.utf8)) + return PluginCopyData { + remaining.withLock { data in + defer { data = nil } + return data + } + } + } + + private static func sink(on driver: CopyingDriver) -> ImportDataSinkAdapter { + let connection = TestFixtures.makeConnection(type: .postgresql) + let adapter = PluginDriverAdapter(connection: connection, pluginDriver: driver) + return ImportDataSinkAdapter(driver: adapter, databaseType: .postgresql) + } + + @Test("A COPY reaches a driver that declares copy-in with its header and every row byte") + func copyReachesTheDriver() async throws { + let driver = CopyingDriver(declaresCopy: true) + let sink = Self.sink(on: driver) + + try await sink.copyFromStandardInput(statement: "COPY t (a) FROM stdin", line: 4, rows: Self.rows("1\n2\n")) + + let received = driver.received + #expect(received.map { $0.statement } == ["COPY t (a) FROM stdin"]) + #expect(received.map { $0.data } == [Data("1\n2\n".utf8)]) + } + + @Test("A driver without copy-in refuses the COPY at its line before anything is sent") + func driverWithoutCopyRefuses() async throws { + let driver = CopyingDriver(declaresCopy: false) + let sink = Self.sink(on: driver) + + await #expect(throws: SQLFileParseError.copyFromStandardInput(line: 4)) { + try await sink.copyFromStandardInput(statement: "COPY t (a) FROM stdin", line: 4, rows: Self.rows("1\n")) + } + #expect(driver.received.isEmpty) + } +} diff --git a/apple/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift b/apple/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift index 1e8662832e..dd89bb8dab 100644 --- a/apple/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift +++ b/apple/TableProTests/Core/Plugins/SqlFileImportSourceEncodingTests.swift @@ -44,6 +44,36 @@ struct SqlFileImportSourceEncodingTests { #expect(result.first == "SET client_encoding = 'UTF8'") } + @Test("A Latin-1 pg_dump's COPY rows arrive as UTF-8, and its statements still run with a UTF-8 client encoding") + func latin1PgDumpCopyRows() async throws { + let dump = "SET client_encoding = 'LATIN1';\nCOPY t (a) FROM stdin;\nCafé\nZürich\n\\.\nSELECT 'é';\n" + let bytes = try #require(dump.data(using: .isoLatin1, allowLossyConversion: false)) + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("SqlFileImportSourceEncodingTests-\(UUID().uuidString).sql") + try bytes.write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + let source = SqlFileImportSource(url: url, encoding: .isoLatin1, grammar: { TestGrammar.postgres }, family: .postgres) + + var statements: [String] = [] + var rows = Data() + for try await element in try await source.elements() { + switch element { + case let .statement(statement, _): + statements.append(statement) + case let .copyFromStandardInput(statement, _, copy): + statements.append(statement) + while let piece = try copy.nextPiece() { + rows.append(piece) + } + @unknown default: + Issue.record("An element this test does not read: \(element)") + } + } + + #expect(statements == ["SET client_encoding = 'UTF8'", "COPY t (a) FROM stdin", "SELECT 'é'"]) + #expect(rows == Data("Café\nZürich\n".utf8)) + } + @Test("A UTF-8 file keeps its own character set statement, since its bytes go to the server unchanged") func utf8FileKeepsItsDeclaration() async throws { let dump = "/*!40101 SET NAMES latin1 */;\nINSERT INTO t VALUES ('é');\n" diff --git a/apple/TableProTests/Core/Services/Execution/CopyDataRouteTests.swift b/apple/TableProTests/Core/Services/Execution/CopyDataRouteTests.swift new file mode 100644 index 0000000000..61dbc7f5b4 --- /dev/null +++ b/apple/TableProTests/Core/Services/Execution/CopyDataRouteTests.swift @@ -0,0 +1,77 @@ +// +// CopyDataRouteTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import TableProSQLGrammar +import Testing + +@MainActor +struct CopyDataRouteTests { + private static func route(_ text: String, sendsBatchesWhole: Bool) -> QueryExecutionRoute? { + QueryExecutionRoute.resolve( + QueryBatchPlanner.batches(in: text, model: .sql, grammar: TestGrammar.postgres), + sendsBatchesWhole: sendsBatchesWhole, + isPlainQuery: { _ in true } + ) + } + + @Test("A COPY carrying rows runs statement by statement even alone, where a lone statement takes the single path", arguments: [ + false, true, + ]) + func copyTakesTheStatementPath(sendsBatchesWhole: Bool) { + let route = Self.route("COPY t (a, b) FROM stdin;\n1\tone\n\\.\n", sendsBatchesWhole: sendsBatchesWhole) + guard case .statements(let statements) = route else { + Issue.record("expected statements, got \(String(describing: route))") + return + } + #expect(statements.map { $0.sql } == ["COPY t (a, b) FROM stdin"]) + #expect(statements.map { $0.copyData } == ["1\tone\n"]) + + guard case .single = Self.route("SELECT 1;", sendsBatchesWhole: sendsBatchesWhole) else { + Issue.record("expected a lone query on the single path") + return + } + } + + @Test("A COPY with no rows still takes the statement path, the one that runs a copy") + func copyWithoutRowsTakesTheStatementPath() { + guard case .statements(let statements) = Self.route("COPY t FROM stdin;", sendsBatchesWhole: false) else { + Issue.record("expected statements") + return + } + #expect(statements.map { $0.copyData } == [""]) + } +} + +struct CopyDataPiecesTests { + private static func pieces(of text: String) throws -> [Data] { + let rows = CopyDataPieces.reading(text) + var pieces: [Data] = [] + while let piece = try rows.nextPiece() { + pieces.append(piece) + } + #expect(try rows.nextPiece() == nil) + return pieces + } + + @Test("No rows make no piece") + func noRows() throws { + #expect(try Self.pieces(of: "").isEmpty) + } + + @Test("Rows go as UTF-8 in pieces of at most 64 KiB, cut wherever the limit falls, even inside a character") + func piecesCutAnywhere() throws { + let text = "12\t" + String(repeating: "é", count: 70_000) + "\n" + + let pieces = try Self.pieces(of: text) + + #expect(pieces.map { $0.count } == [65_536, 65_536, 8_932]) + #expect(Data(pieces.joined()) == Data(text.utf8)) + let first = try #require(pieces.first) + #expect(String(data: first, encoding: .utf8) == nil) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserCopyRowsTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserCopyRowsTests.swift new file mode 100644 index 0000000000..e227718272 --- /dev/null +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserCopyRowsTests.swift @@ -0,0 +1,256 @@ +// +// SQLFileParserCopyRowsTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import TableProSQLGrammar +import Testing + +@testable import TablePro + +struct SQLFileParserCopyRowsTests { + private enum Element: Equatable { + case statement(String, line: Int) + case copy(String, line: Int, rows: Data) + + static func copying(_ statement: String, line: Int, rows: String) -> Element { + .copy(statement, line: line, rows: Data(rows.utf8)) + } + } + + private static func withFile(_ sql: String, _ body: (URL) async throws -> T) async throws -> T { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("SQLFileParserCopyRowsTests-\(UUID().uuidString).sql") + try Data(sql.utf8).write(to: url) + defer { try? FileManager.default.removeItem(at: url) } + return try await body(url) + } + + private static func pieces(of rows: PluginCopyData) throws -> [Data] { + var pieces: [Data] = [] + while let piece = try rows.nextPiece() { + pieces.append(piece) + } + return pieces + } + + private static func read(_ element: PluginImportElement) throws -> Element { + switch element { + case let .statement(statement, lineNumber): + return .statement(statement, line: lineNumber) + case let .copyFromStandardInput(statement, lineNumber, rows): + return .copy(statement, line: lineNumber, rows: Data(try pieces(of: rows).joined())) + @unknown default: + Issue.record("An element this test does not read: \(element)") + return .statement("", line: 0) + } + } + + private static func elements( + _ sql: String, + parser: SQLFileParser = SQLFileParser(), + grammar: SQLLexicalGrammar = TestGrammar.postgres + ) async throws -> [Element] { + try await withFile(sql) { url in + var result: [Element] = [] + for try await element in parser.parseElements(url: url, encoding: .utf8, grammar: grammar) { + result.append(try read(element)) + } + return result + } + } + + // The pieces of every COPY the file holds, in order. + private static func copyPieces(_ sql: String, parser: SQLFileParser = SQLFileParser()) async throws -> [Data] { + try await withFile(sql) { url in + var result: [Data] = [] + for try await element in parser.parseElements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) { + if case let .copyFromStandardInput(_, _, rows) = element { + result += try pieces(of: rows) + } + } + return result + } + } + + @Test("COPY FROM STDIN yields its rows up to the \\. line, and the statements after it keep their file lines") + func linefeedRows() async throws { + let sql = "CREATE TABLE t (a int, b text);\nCOPY public.t (a, b) FROM stdin;\n1\tone\n2\tDROP TABLE users;\n\\.\n" + + "SELECT count(*) FROM t;\nSELECT 2;\n" + #expect(try await Self.elements(sql) == [ + .statement("CREATE TABLE t (a int, b text)", line: 1), + .copying("COPY public.t (a, b) FROM stdin", line: 2, rows: "1\tone\n2\tDROP TABLE users;\n"), + .statement("SELECT count(*) FROM t", line: 6), + .statement("SELECT 2", line: 7), + ]) + } + + @Test("CRLF rows keep their CRLF, and a \\. line ending in CRLF ends them") + func carriageReturnRows() async throws { + let sql = "COPY t (a) FROM stdin;\r\n1\r\n2\r\n\\.\r\nSELECT 1;\r\n" + #expect(try await Self.elements(sql) == [ + .copying("COPY t (a) FROM stdin", line: 1, rows: "1\r\n2\r\n"), + .statement("SELECT 1", line: 5), + ]) + } + + @Test("Rows end at the end of the file when no \\. line ends them first", arguments: [ + ("COPY t (a) FROM stdin;\n1\n2\n", "1\n2\n"), + ("COPY t (a) FROM stdin;\n1\n2", "1\n2"), + ("COPY t (a) FROM stdin;\n1\n\\.", "1\n"), + ("COPY t (a) FROM stdin;\n1\n\\.\r", "1\n"), + ("COPY t (a) FROM stdin;\n\\.\n", ""), + ("COPY t (a) FROM stdin;\n", ""), + ("COPY t (a) FROM stdin;", ""), + ]) + func rowsToTheEndOfTheFile(sql: String, rows: String) async throws { + #expect(try await Self.elements(sql) == [.copying("COPY t (a) FROM stdin", line: 1, rows: rows)]) + } + + @Test("Text after the ; on the COPY line is neither data nor run") + func restOfTheCopyLineRunsNothing() async throws { + let sql = "COPY t (a) FROM stdin; SELECT 'not run';\n1\n\\.\nSELECT 2;\n" + #expect(try await Self.elements(sql) == [ + .copying("COPY t (a) FROM stdin", line: 1, rows: "1\n"), + .statement("SELECT 2", line: 4), + ]) + } + + @Test("Two COPY blocks each get their own rows") + func twoCopyBlocks() async throws { + let sql = "COPY a (x) FROM stdin;\n1\n\\.\nCOPY b (y) FROM stdin;\n2\n3\n\\.\nSELECT 3;\n" + #expect(try await Self.elements(sql) == [ + .copying("COPY a (x) FROM stdin", line: 1, rows: "1\n"), + .copying("COPY b (y) FROM stdin", line: 4, rows: "2\n3\n"), + .statement("SELECT 3", line: 8), + ]) + } + + @Test("Rows left unread are passed over, and their reader answers nil once the next element is asked for") + func unreadRowsArePassedOver() async throws { + let sql = "COPY t (a) FROM stdin;\n1\n2\n\\.\nSELECT 5;\n" + try await Self.withFile(sql) { url in + var iterator = SQLFileParser().parseElements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) + .makeAsyncIterator() + let copy = try #require(try await iterator.next()) + guard case let .copyFromStandardInput(_, _, rows) = copy else { + Issue.record("Expected the COPY, got \(copy)") + return + } + let next = try #require(try await iterator.next()) + #expect(try Self.read(next) == .statement("SELECT 5", line: 5)) + #expect(try rows.nextPiece() == nil) + let end = try await iterator.next().map { try Self.read($0) } + #expect(end == nil) + } + } + + @Test("Rows read half way are passed over from where the reading stopped") + func halfReadRowsArePassedOver() async throws { + let sql = "COPY t (a) FROM stdin;\n1\n2\n3\n4\n5\n\\.\nSELECT 5;\n" + try await Self.withFile(sql) { url in + let parser = SQLFileParser(copyPieceLength: 4) + var iterator = parser.parseElements(url: url, encoding: .utf8, grammar: TestGrammar.postgres).makeAsyncIterator() + let copy = try #require(try await iterator.next()) + guard case let .copyFromStandardInput(_, _, rows) = copy else { + Issue.record("Expected the COPY, got \(copy)") + return + } + #expect(try rows.nextPiece() == Data("1\n2\n".utf8)) + let next = try #require(try await iterator.next()) + #expect(try Self.read(next) == .statement("SELECT 5", line: 8)) + #expect(try rows.nextPiece() == nil) + } + } + + @Test("A CSV header line is data, sent as the server reads it") + func csvHeaderIsData() async throws { + let sql = "COPY t (a, b) FROM stdin WITH (FORMAT csv, HEADER true);\na,b\n1,\"x\"\n\\.\n" + #expect(try await Self.elements(sql) == [ + .copying("COPY t (a, b) FROM stdin WITH (FORMAT csv, HEADER true)", line: 1, rows: "a,b\n1,\"x\"\n"), + ]) + } + + @Test("Rows keep non-ASCII text and emoji byte for byte, whatever the chunk and piece size", + arguments: [2, 3, 5, SQLFileParser.defaultCopyPieceLength]) + func nonASCIIRowsAreByteExact(pieceLength: Int) async throws { + let rows = "1\tcafé\n2\t日本語\n3\t😀😀😀\n" + let sql = "COPY t (a, b) FROM stdin;\n\(rows)\\.\nSELECT 1;\n" + for chunkSize in [1, 2, 3, 7, SQLFileParser.defaultChunkSize] { + let parser = SQLFileParser(chunkSize: chunkSize, copyPieceLength: pieceLength) + let pieces = try await Self.copyPieces(sql, parser: parser) + #expect(Data(pieces.joined()) == Data(rows.utf8), "chunk \(chunkSize)") + } + } + + @Test("A chunk end anywhere, inside a \\. line too, changes nothing") + func chunkEndAnywhere() async throws { + let sql = "SET a = 1;\nCOPY t (a) FROM stdin;\r\n1\r\n\\.x\n\\.\r2\n\\\n22\n\\.\r\nSELECT 2;\nCOPY u FROM stdin;\n3\n\\." + let expected: [Element] = [ + .statement("SET a = 1", line: 1), + .copying("COPY t (a) FROM stdin", line: 2, rows: "1\r\n\\.x\n\\.\r2\n\\\n22\n"), + .statement("SELECT 2", line: 9), + .copying("COPY u FROM stdin", line: 10, rows: "3\n"), + ] + for chunkSize in 1...12 { + for pieceLength in [2, 3, 5, 64] { + let parser = SQLFileParser(chunkSize: chunkSize, copyPieceLength: pieceLength) + #expect(try await Self.elements(sql, parser: parser) == expected, "chunk \(chunkSize), piece \(pieceLength)") + } + } + } + + @Test("A row longer than a piece arrives in pieces no longer than the limit") + func rowLongerThanAPiece() async throws { + let row = String(repeating: "x", count: 50) + "\n" + let pieces = try await Self.copyPieces("COPY t (a) FROM stdin;\n\(row)\\.\n", parser: SQLFileParser(copyPieceLength: 8)) + #expect(pieces.count == 7) + #expect(pieces.allSatisfy { piece in piece.count <= 8 }) + #expect(Data(pieces.joined()) == Data(row.utf8)) + } + + @Test("Pieces end at a line end when the line fits") + func piecesEndAtLineEnds() async throws { + let rows = (1...500).map { "\($0)\tvalue \($0)\n" }.joined() + let pieces = try await Self.copyPieces("COPY t (a, b) FROM stdin;\n\(rows)\\.\n", parser: SQLFileParser(copyPieceLength: 100)) + #expect(pieces.count > 1) + #expect(pieces.allSatisfy { piece in piece.count <= 100 && piece.last == 0x0A }) + #expect(Data(pieces.joined()) == Data(rows.utf8)) + } + + @Test("A piece holds at most 1 MiB of UTF-8, even of three-byte characters") + func pieceStaysWithinOneMebibyte() async throws { + let row = String(repeating: "日", count: 400_000) + "\n" + let pieces = try await Self.copyPieces("COPY t (a) FROM stdin;\n\(row)\\.\n") + #expect(pieces.count > 1) + #expect(pieces.allSatisfy { piece in piece.count <= 1_048_576 }) + #expect(Data(pieces.joined()) == Data(row.utf8)) + } + + @Test("The count reads each COPY as one statement and none of its rows") + func countReadsCopyAsOne() async throws { + let sql = "CREATE TABLE t (a text);\nCOPY t (a) FROM stdin;\nSELECT 1;\nDROP TABLE t;\n\\.\n" + + "COPY t (a) FROM stdin;\nx\n\\.\nSELECT 2;\n" + let count = try await Self.withFile(sql) { url in + try await SQLFileParser().countStatements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) + } + #expect(count == 4) + #expect(try await Self.elements(sql).count == count) + } + + @Test("The element stream yields each run the statement stream yields, GO n repeats included") + func elementsRepeatLikeStatements() async throws { + let sql = "SELECT 1\nGO 3\nSELECT 2\nGO\n" + let statements = try await Self.withFile(sql) { url in + var result: [Element] = [] + for try await (statement, line) in SQLFileParser().parseFile(url: url, encoding: .utf8, grammar: TestGrammar.sqlServer) { + result.append(.statement(statement, line: line)) + } + return result + } + #expect(statements.count == 4) + #expect(try await Self.elements(sql, grammar: TestGrammar.sqlServer) == statements) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift index 4bc0a135cf..796c73cbb8 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift @@ -615,8 +615,8 @@ struct SQLFileParserTests { ]) } - @Test("Postgres: COPY FROM STDIN stops the file at its line before the COPY or its rows are sent") - func postgresCopyFromStandardInputIsRefused() async throws { + @Test("Postgres: the statement stream stops at COPY FROM STDIN before the COPY or its rows are sent, and the count reads the COPY as one statement") + func postgresStatementStreamRefusesCopyFromStandardInputAndCountReadsIt() async throws { let sql = """ CREATE TABLE t (a int, b text); COPY public.t (a, b) FROM stdin; @@ -636,9 +636,7 @@ struct SQLFileParserTests { } } #expect(statements == ["CREATE TABLE t (a int, b text)"]) - await #expect(throws: SQLFileParseError.copyFromStandardInput(line: 2)) { - _ = try await SQLFileParser().countStatements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) - } + #expect(try await SQLFileParser().countStatements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) == 3) } @Test("Postgres: a COPY that reads a server file or writes rows out still runs", arguments: [ diff --git a/apple/TableProTests/Helpers/CopyInRecordingDriver.swift b/apple/TableProTests/Helpers/CopyInRecordingDriver.swift new file mode 100644 index 0000000000..5fd0bd7b0e --- /dev/null +++ b/apple/TableProTests/Helpers/CopyInRecordingDriver.swift @@ -0,0 +1,135 @@ +// +// CopyInRecordingDriver.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit + +// Records each text sent as a statement and each COPY sent through copy-in, with every piece of its data read. +final class CopyInRecordingDriver: DatabaseDriver, @unchecked Sendable { + struct Copy: Equatable { + let statement: String + let data: Data + } + + let connection: DatabaseConnection + var status: ConnectionStatus = .connected + var serverVersion: String? { nil } + + private let copiesIn: Bool + private let lexicalState: PluginSessionLexicalState? + private let lexicalStateAfterFirstStatement: PluginSessionLexicalState? + private let lock = NSLock() + private var statements: [String] = [] + private var copies: [Copy] = [] + + // `lexicalStateAfterFirstStatement` is what the session reads once a statement ran, as after a `SET`. + init( + connection: DatabaseConnection, + copiesIn: Bool, + lexicalState: PluginSessionLexicalState? = nil, + lexicalStateAfterFirstStatement: PluginSessionLexicalState? = nil + ) { + self.connection = connection + self.copiesIn = copiesIn + self.lexicalState = lexicalState + self.lexicalStateAfterFirstStatement = lexicalStateAfterFirstStatement + } + + var sentStatements: [String] { + lock.withLock { statements } + } + + var sentCopies: [Copy] { + lock.withLock { copies } + } + + var sessionLexicalState: PluginSessionLexicalState? { + lock.withLock { + guard let lexicalStateAfterFirstStatement, !(statements.isEmpty && copies.isEmpty) else { return lexicalState } + return lexicalStateAfterFirstStatement + } + } + + var supportsCopyFromStandardInput: Bool { copiesIn } + + func copyFromStandardInput( + statement: String, + rows: PluginCopyData, + context: PluginStatementContext? + ) async throws -> QueryResult { + guard copiesIn else { throw PluginDriverUnsupportedOperation.copyFromStandardInput } + var data = Data() + while let piece = try rows.nextPiece() { + data.append(piece) + } + lock.withLock { copies.append(Copy(statement: statement, data: data)) } + return Self.emptyResult + } + + private static var emptyResult: QueryResult { + QueryResult(columns: [], columnTypes: [], rows: [], rowsAffected: 0, executionTime: 0, error: nil) + } + + private func record(_ query: String) -> QueryResult { + lock.withLock { statements.append(query) } + return Self.emptyResult + } + + func execute(query: String) async throws -> QueryResult { record(query) } + func executeParameterized(query: String, parameters: [Any?]) async throws -> QueryResult { record(query) } + func executeUserQuery(query: String, rowCap: Int?, parameters: [Any?]?) async throws -> QueryResult { + record(query) + } + + func connect() async throws {} + func disconnect() {} + func testConnection() async throws -> Bool { true } + func ping() async throws {} + func cancelQuery() throws {} + func applyQueryTimeout(_ seconds: Int) async throws {} + + func fetchTables() async throws -> [TableInfo] { [] } + func fetchTables(schema: String?) async throws -> [TableInfo] { [] } + func fetchColumns(table: String) async throws -> [ColumnInfo] { [] } + func fetchAllColumns() async throws -> [String: [ColumnInfo]] { [:] } + func fetchIndexes(table: String) async throws -> [IndexInfo] { [] } + func fetchForeignKeys(table: String) async throws -> [ForeignKeyInfo] { [] } + func fetchApproximateRowCount(table: String) async throws -> Int? { nil } + func fetchDatabases() async throws -> [String] { [] } + func fetchDatabaseMetadata(_ database: String) async throws -> DatabaseMetadata { + DatabaseMetadata( + id: database, + name: database, + tableCount: nil, + sizeBytes: nil, + lastAccessed: nil, + isSystemDatabase: false, + icon: "cylinder" + ) + } + + func fetchTableDDL(table: String) async throws -> String { "" } + func fetchTableMetadata(tableName: String) async throws -> TableMetadata { + TableMetadata( + tableName: tableName, + dataSize: nil, + indexSize: nil, + totalSize: nil, + avgRowLength: nil, + rowCount: nil, + comment: nil, + engine: nil, + collation: nil, + createTime: nil, + updateTime: nil + ) + } + + func fetchViewDefinition(view: String) async throws -> String { "" } + func beginTransaction() async throws {} + func commitTransaction() async throws {} + func rollbackTransaction() async throws {} +} diff --git a/apple/changelog.d/4255.added.md b/apple/changelog.d/4255.added.md new file mode 100644 index 0000000000..9cbf52dd22 --- /dev/null +++ b/apple/changelog.d/4255.added.md @@ -0,0 +1 @@ +SQL import and the query editor load the rows of a PostgreSQL COPY ... FROM STDIN, as psql sends them. diff --git a/apple/scripts/probes/check-postgresql-copy-in.sh b/apple/scripts/probes/check-postgresql-copy-in.sh new file mode 100755 index 0000000000..ea1722620f --- /dev/null +++ b/apple/scripts/probes/check-postgresql-copy-in.sh @@ -0,0 +1,239 @@ +#!/usr/bin/env bash +# +# Checks what the PostgreSQL driver's copy-in (LibPQCopyIn) relies on, with the libpq the app links +# (apple/Libs/libpq.a) against a live server: rows go as CopyData without psql's `\.` line and end with +# CopyDone; pieces may split a row or a UTF-8 sequence; CRLF rows and a last row with no line break load; +# a bad row and a CopyFail both fail the COPY and abort an open block; a binary COPY shows on its +# PGRES_COPY_IN result; a refused COPY never enters copy-in; a read-only transaction refuses it with +# 25006; a cancel request mid-stream fails it with 57014. It also prints what a sent `\.` line does, +# which PostgreSQL 18 reads as data in CSV. +# +# Usage: +# apple/scripts/probes/check-postgresql-copy-in.sh "host=127.0.0.1 port=5432 user=postgres dbname=postgres" +# +# Needs apple/scripts/download-libs.sh to have run. Creates and drops the table copy_probe. Measured on +# PostgreSQL 17.11 and 18.6 with libpq 17.4. Exits non-zero on a failure. + +set -euo pipefail + +CONNINFO="${1:?usage: check-postgresql-copy-in.sh }" +APPLE="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +LIBS="$APPLE/Libs" +[ -f "$LIBS/libpq.a" ] || { echo "apple/Libs/libpq.a not found; run apple/scripts/download-libs.sh" >&2; exit 3; } + +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +cat > "$WORK_DIR/probe.c" << 'C_SOURCE' +#include +#include +#include +#include +#include +#include + +static int failures = 0; + +static void check(const char *label, int ok, const char *detail) { + printf("%s %s%s%s\n", ok ? "ok " : "FAIL", label, detail ? ": " : "", detail ? detail : ""); + if (!ok) failures++; +} + +static void run(PGconn *c, const char *sql) { + PGresult *r = PQexec(c, sql); + ExecStatusType s = PQresultStatus(r); + if (s != PGRES_COMMAND_OK && s != PGRES_TUPLES_OK) { + fprintf(stderr, "setup failed: %s: %s", sql, PQerrorMessage(c)); + exit(3); + } + PQclear(r); +} + +static char *scalar(PGconn *c, const char *sql) { + static char buf[4096]; + PGresult *r = PQexec(c, sql); + snprintf(buf, sizeof buf, "%s", PQresultStatus(r) == PGRES_TUPLES_OK && PQntuples(r) > 0 && !PQgetisnull(r, 0, 0) + ? PQgetvalue(r, 0, 0) : ""); + PQclear(r); + return buf; +} + +typedef struct { + ExecStatusType status; + char tuples[32]; + char tag[64]; + char state[8]; + char message[512]; + int extra; +} Outcome; + +// Sends `data` in pieces of `piece` bytes after a header that answered COPY_IN, ends it, reads every result. +static Outcome copy_in(PGconn *c, const char *header, const char *data, size_t length, size_t piece, + const char *fail_reason, int *binary) { + Outcome o; + memset(&o, 0, sizeof o); + PGresult *start = PQexec(c, header); + o.status = PQresultStatus(start); + if (binary) *binary = PQbinaryTuples(start); + if (o.status != PGRES_COPY_IN) { + const char *st = PQresultErrorField(start, PG_DIAG_SQLSTATE); + snprintf(o.state, sizeof o.state, "%s", st ? st : ""); + snprintf(o.message, sizeof o.message, "%s", PQresultErrorField(start, PG_DIAG_MESSAGE_PRIMARY) ?: ""); + PQclear(start); + while ((start = PQgetResult(c))) { o.extra++; PQclear(start); } + return o; + } + PQclear(start); + for (size_t at = 0; at < length && !fail_reason; at += piece) { + size_t n = length - at < piece ? length - at : piece; + if (PQputCopyData(c, data + at, (int)n) != 1) { snprintf(o.message, sizeof o.message, "put failed"); break; } + } + if (PQputCopyEnd(c, fail_reason) != 1) snprintf(o.message, sizeof o.message, "end failed"); + PGresult *r = PQgetResult(c); + o.status = PQresultStatus(r); + snprintf(o.tuples, sizeof o.tuples, "%s", PQcmdTuples(r)); + snprintf(o.tag, sizeof o.tag, "%s", PQcmdStatus(r)); + const char *st = PQresultErrorField(r, PG_DIAG_SQLSTATE); + snprintf(o.state, sizeof o.state, "%s", st ? st : ""); + const char *msg = PQresultErrorField(r, PG_DIAG_MESSAGE_PRIMARY); + if (msg) snprintf(o.message, sizeof o.message, "%s", msg); + PQclear(r); + while ((r = PQgetResult(c))) { o.extra++; PQclear(r); } + return o; +} + +static int is_ok(Outcome o, const char *tuples) { + return o.status == PGRES_COMMAND_OK && strcmp(o.tuples, tuples) == 0 && o.extra == 0; +} + +static void describe(char *out, size_t size, Outcome o) { + snprintf(out, size, "status=%s tuples=%s tag='%s' state=%s msg='%s' extra=%d", PQresStatus(o.status), o.tuples, + o.tag, o.state, o.message, o.extra); +} + +typedef struct { PGconn *conn; } CancelArgs; + +static void *cancel_after(void *arg) { + usleep(300000); + PGcancel *cancel = PQgetCancel(((CancelArgs *)arg)->conn); + char err[256]; + PQcancel(cancel, err, sizeof err); + PQfreeCancel(cancel); + return NULL; +} + +int main(int argc, char **argv) { + PGconn *c = PQconnectdb(argv[1]); + if (PQstatus(c) != CONNECTION_OK) { fprintf(stderr, "no server: %s", PQerrorMessage(c)); return 3; } + printf("libpq %d, server %d\n", PQlibVersion(), PQserverVersion(c)); + char d[1024]; + Outcome o; + + run(c, "DROP TABLE IF EXISTS copy_probe; CREATE TABLE copy_probe (a int, b text)"); + + const char *rows = "1\tone\n2\ttwo\n3\t\\N\n"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", rows, strlen(rows), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("text rows without \\. end with CopyDone as COPY 3", is_ok(o, "3") && strcmp(o.tag, "COPY 3") == 0, d); + check("a NULL marker loads as NULL", strcmp(scalar(c, "SELECT count(*) FROM copy_probe WHERE b IS NULL"), "1") == 0, NULL); + check("an autocommit COPY leaves the session idle", PQtransactionStatus(c) == PQTRANS_IDLE, NULL); + + const char *with_marker = "4\tfour\n\\.\n"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", with_marker, strlen(with_marker), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + printf("info text rows with the \\. line sent: %s\n", d); + + run(c, "TRUNCATE copy_probe"); + const char *csv = "a,b\n5,\"five, quoted\"\n6,\"multi\nline\"\n"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN WITH (FORMAT csv, HEADER true)", csv, strlen(csv), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("CSV with a header line and a quoted line break is COPY 2", is_ok(o, "2"), d); + check("the quoted line break is kept", strcmp(scalar(c, "SELECT b FROM copy_probe WHERE a = 6"), "multi\nline") == 0, NULL); + + const char *csv_marker = "7,seven\n\\.\n"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN WITH (FORMAT csv)", csv_marker, strlen(csv_marker), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + printf("info CSV rows with the \\. line sent: %s\n", d); + + run(c, "TRUNCATE copy_probe"); + const char *utf8 = "8\tcaf\xc3\xa9 \xf0\x9f\x98\x80\n9\tlast row without a line break"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", utf8, strlen(utf8), 3, NULL, NULL); + describe(d, sizeof d, o); + check("3-byte pieces that split a UTF-8 sequence and a last row with no line break load", is_ok(o, "2"), d); + check("the split sequence arrives whole", strcmp(scalar(c, "SELECT b FROM copy_probe WHERE a = 8"), "caf\xc3\xa9 \xf0\x9f\x98\x80") == 0, NULL); + + run(c, "TRUNCATE copy_probe"); + const char *crlf = "10\tten\r\n11\televen\r\n"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", crlf, strlen(crlf), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("CRLF text rows load", is_ok(o, "2"), d); + check("a CRLF row keeps no carriage return", strcmp(scalar(c, "SELECT b FROM copy_probe WHERE a = 10"), "ten") == 0, NULL); + + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", "", 0, 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("no rows is COPY 0", is_ok(o, "0"), d); + + run(c, "BEGIN"); + const char *bad = "12\ttwelve\nx\tbad\n13\tthirteen\n"; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", bad, strlen(bad), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("a bad row fails the COPY after CopyDone with 22P02", o.status == PGRES_FATAL_ERROR && strcmp(o.state, "22P02") == 0 && o.extra == 0, d); + check("inside a block the failed COPY leaves it aborted", PQtransactionStatus(c) == PQTRANS_INERROR, NULL); + run(c, "ROLLBACK"); + + run(c, "BEGIN"); + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", rows, strlen(rows), 1 << 16, "canceled by user", NULL); + describe(d, sizeof d, o); + check("CopyFail answers 57014 'COPY from stdin failed: canceled by user'", + o.status == PGRES_FATAL_ERROR && strcmp(o.state, "57014") == 0 && strstr(o.message, "canceled by user") != NULL, d); + check("CopyFail aborts the block", PQtransactionStatus(c) == PQTRANS_INERROR, NULL); + run(c, "ROLLBACK"); + + run(c, "BEGIN"); + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", rows, strlen(rows), 1 << 16, NULL, NULL); + check("a COPY inside a block leaves it open", is_ok(o, "3") && PQtransactionStatus(c) == PQTRANS_INTRANS, NULL); + run(c, "ROLLBACK"); + + int binary = 0; + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN WITH (FORMAT binary)", "", 0, 1, "binary COPY data cannot come from a script", &binary); + describe(d, sizeof d, o); + check("a binary COPY reports binary tuples on its COPY_IN result", binary == 1, NULL); + check("ending a binary COPY with CopyFail fails it with 57014", o.status == PGRES_FATAL_ERROR && strcmp(o.state, "57014") == 0, d); + + o = copy_in(c, "COPY no_such_table FROM STDIN", rows, strlen(rows), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("a COPY the server refuses never enters COPY_IN (42P01)", o.status == PGRES_FATAL_ERROR && strcmp(o.state, "42P01") == 0, d); + + run(c, "START TRANSACTION READ ONLY; SELECT 1"); + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", rows, strlen(rows), 1 << 16, NULL, NULL); + describe(d, sizeof d, o); + check("a read-only transaction refuses the COPY before COPY_IN (25006)", o.status == PGRES_FATAL_ERROR && strcmp(o.state, "25006") == 0, d); + run(c, "ROLLBACK"); + + // Cancel while rows stream: the server keeps reading pieces and answers 57014 after CopyDone. + size_t big = 64u * 1024u * 1024u; + char *lots = malloc(big); + size_t usable = big - (big % 7); + for (size_t i = 0; i + 7 <= usable; i += 7) memcpy(lots + i, "1\tabcd\n", 7); + pthread_t thread; + CancelArgs args = { c }; + pthread_create(&thread, NULL, cancel_after, &args); + o = copy_in(c, "COPY copy_probe (a, b) FROM STDIN", lots, usable, 1 << 16, NULL, NULL); + pthread_join(thread, NULL); + describe(d, sizeof d, o); + check("a cancel request mid-stream fails the COPY with 57014", o.status == PGRES_FATAL_ERROR && strcmp(o.state, "57014") == 0, d); + free(lots); + + run(c, "DROP TABLE copy_probe"); + PQfinish(c); + printf("%d failure(s)\n", failures); + return failures ? 1 : 0; +} +C_SOURCE + +clang -O1 -o "$WORK_DIR/probe" "$WORK_DIR/probe.c" \ + -I"$APPLE/Plugins/PostgreSQLDriverPlugin/CLibPQ/include" \ + "$LIBS/libpq.a" "$LIBS/libpgcommon.a" "$LIBS/libpgport.a" \ + -L"$LIBS/dylibs" -lssl.3 -lcrypto.3 -lz -framework GSS -Wl,-rpath,"$LIBS/dylibs" + +"$WORK_DIR/probe" "$CONNINFO"