diff --git a/Cargo.lock b/Cargo.lock index 1c42a9049f..d6ef499938 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5404,8 +5404,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" +source = "git+https://github.com/TableProApp/rust-postgres?rev=f8bdf78d12df410a1c1b485445a9d0683361a173#f8bdf78d12df410a1c1b485445a9d0683361a173" dependencies = [ "base64 0.22.1", "byteorder", @@ -5422,8 +5421,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" +source = "git+https://github.com/TableProApp/rust-postgres?rev=f8bdf78d12df410a1c1b485445a9d0683361a173#f8bdf78d12df410a1c1b485445a9d0683361a173" dependencies = [ "bytes", "chrono", @@ -8497,8 +8495,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" +source = "git+https://github.com/TableProApp/rust-postgres?rev=f8bdf78d12df410a1c1b485445a9d0683361a173#f8bdf78d12df410a1c1b485445a9d0683361a173" dependencies = [ "async-trait", "byteorder", diff --git a/Cargo.toml b/Cargo.toml index 93589a4a14..98d254b261 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -197,6 +197,16 @@ testcontainers = { version = "0.27", default-features = false, features = ["aws- testcontainers-modules = { version = "0.15", default-features = false, features = ["aws-lc-rs"] } oo7 = { version = "0.6", default-features = false, features = ["tokio", "native_crypto", "unstable"] } +# TablePro's fork of rust-postgres, pinned by revision: the tokio-postgres-v0.7.18 tag plus the +# copy_in fix in rust-postgres/rust-postgres#1402, without which a COPY the server refuses before +# copy-in mode leaves the connection out of step. All three crates come from the fork, so the +# driver's postgres-types and postgres-protocol stay the ones tokio-postgres uses. Remove this +# once a tokio-postgres release has the fix. +[patch.crates-io] +tokio-postgres = { git = "https://github.com/TableProApp/rust-postgres", rev = "f8bdf78d12df410a1c1b485445a9d0683361a173" } +postgres-types = { git = "https://github.com/TableProApp/rust-postgres", rev = "f8bdf78d12df410a1c1b485445a9d0683361a173" } +postgres-protocol = { git = "https://github.com/TableProApp/rust-postgres", rev = "f8bdf78d12df410a1c1b485445a9d0683361a173" } + [workspace.lints.rust] unsafe_op_in_unsafe_fn = "deny" diff --git a/core/crates/core/src/copy_in.rs b/core/crates/core/src/copy_in.rs new file mode 100644 index 0000000000..82af8aaceb --- /dev/null +++ b/core/crates/core/src/copy_in.rs @@ -0,0 +1,27 @@ +use tokio::sync::mpsc; +use tokio_util::bytes::Bytes; + +use crate::result_event::RunEnd; +use crate::transaction_state::TransactionState; + +/// One message of the rows a `COPY ... FROM STDIN` reads. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CopyInPiece { + /// Bytes of the rows as the file holds them, cut anywhere. + Data(Bytes), + /// The rows ended. A source that closes without it gave up, and the + /// copy is abandoned, never finished with the rows it has. + End, +} + +/// Where a session takes the rows of a copy from, a piece at a time. +pub type CopyInRows = mpsc::Receiver; + +#[derive(Debug, Clone, PartialEq)] +pub struct CopyInSummary { + pub end: RunEnd, + /// The rows the server took, from its `COPY n`. 0 unless the copy + /// completed. + pub rows: u64, + pub transaction_state: TransactionState, +} diff --git a/core/crates/core/src/editor_session.rs b/core/crates/core/src/editor_session.rs index 670d05dabb..aeb3d68ae3 100644 --- a/core/crates/core/src/editor_session.rs +++ b/core/crates/core/src/editor_session.rs @@ -1,6 +1,7 @@ use async_trait::async_trait; use crate::call_options::CallOptions; +use crate::copy_in::{CopyInRows, CopyInSummary}; use crate::error::DriverError; use crate::result_event::RunSummary; use crate::result_sink::ResultSink; @@ -75,6 +76,24 @@ pub trait EditorSession: Send + Sync { async fn transaction_state(&self, options: CallOptions) -> Result; + /// Run `statement`, a `COPY ... FROM STDIN`, with the pieces `rows` + /// gives as its data, sent on as they come. A cancel, or a source that + /// closes before `CopyInPiece::End`, abandons the copy and ends it + /// cancelled, whatever the server answers to that. A copy the server + /// refused ends `Failed`. `Unsupported`, which a session without + /// copy-in returns, fails only this statement; any other error is the + /// session failing, as with `run`. + async fn copy_in( + &self, + _statement: UserSql, + _rows: CopyInRows, + _options: RunOptions, + ) -> Result { + Err(DriverError::Unsupported { + feature: crate::UnsupportedFeature::CopyFromStdin, + }) + } + /// The database an unqualified name resolves to now, as the server /// last said. `None` where the driver does not follow it: the session /// is still in the database it opened in. diff --git a/core/crates/core/src/import/sql_file.rs b/core/crates/core/src/import/sql_file.rs index eadcb8bacf..7bc082394b 100644 --- a/core/crates/core/src/import/sql_file.rs +++ b/core/crates/core/src/import/sql_file.rs @@ -49,13 +49,16 @@ pub struct SqlFileReader { line: u64, first: bool, failed: bool, + /// The error that ended the reading, given once the units read before + /// it are taken. + error: Option, /// Set by whoever stopped taking units, so a long run of lines that /// make none, such as a refused COPY block, is not read to its end. stop: Arc, } impl SqlFileReader { - pub fn new(file: Box, grammar: SqlGrammar, copy_overriding: Option<&str>) -> Self { + pub fn new(file: Box, grammar: SqlGrammar) -> Self { let consumed = Arc::new(AtomicU64::new(0)); let counted = Counted { inner: file, @@ -72,19 +75,18 @@ impl SqlFileReader { }; Self { source, - reader: Some(ScriptReader::new(grammar).with_copy_overriding(copy_overriding)), + reader: Some(ScriptReader::new(grammar)), units: VecDeque::new(), consumed, partial: Vec::new(), line: 1, first: true, failed: false, + error: None, stop: Arc::new(AtomicBool::new(false)), } } - /// The bytes of the file read so far, shared so another thread can - /// show progress. /// How the session reads a string as the file starts. pub fn with_settings(mut self, settings: LexicalSettings) -> Self { self.reader = self.reader.map(|reader| reader.with_settings(settings)); @@ -96,6 +98,8 @@ impl SqlFileReader { Arc::clone(&self.stop) } + /// The bytes of the file read so far, shared so another thread can + /// show progress. pub fn consumed(&self) -> Arc { Arc::clone(&self.consumed) } @@ -106,11 +110,15 @@ impl SqlFileReader { match self.source.read(&mut bytes) { Ok(read) => break read, Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, - Err(error) => return Err(SqlFileError::Read(error.to_string())), + Err(error) => { + self.plan_held()?; + return Err(SqlFileError::Read(error.to_string())); + } } }; if read == 0 { if !self.partial.is_empty() { + self.plan_held()?; return Err(SqlFileError::NotUtf8 { line: self.line }); } let Some(reader) = self.reader.take() else { @@ -137,6 +145,10 @@ impl SqlFileReader { Err(error) => { let before = chunk.get(..error.valid_up_to()).unwrap_or_default(); let line = self.line + before.iter().filter(|byte| **byte == b'\n').count() as u64; + // The statements that end before the byte still run. + if let (Ok(text), Some(reader)) = (std::str::from_utf8(before), self.reader.as_mut()) { + self.units.extend(reader.push_last(text)?); + } return Err(SqlFileError::NotUtf8 { line }); } }; @@ -151,6 +163,17 @@ impl SqlFileReader { } } +impl SqlFileReader { + /// Plan what the reader holds now, so the statements read whole before + /// the file stops being readable still run. + fn plan_held(&mut self) -> Result<(), SqlFileError> { + if let Some(reader) = self.reader.as_mut() { + self.units.extend(reader.push_last("")?); + } + Ok(()) + } +} + impl Iterator for SqlFileReader { type Item = Result; @@ -159,6 +182,9 @@ impl Iterator for SqlFileReader { if let Some(unit) = self.units.pop_front() { return Some(Ok(unit)); } + if let Some(error) = self.error.take() { + return Some(Err(error)); + } if self.failed || self.stop.load(Ordering::Relaxed) { return None; } @@ -167,7 +193,7 @@ impl Iterator for SqlFileReader { Ok(false) => return None, Err(error) => { self.failed = true; - return Some(Err(error)); + self.error = Some(error); } } } @@ -197,7 +223,7 @@ mod tests { use super::*; fn units(bytes: Vec) -> Vec> { - SqlFileReader::new(Box::new(std::io::Cursor::new(bytes)), SqlGrammar::Sqlite, None).collect() + SqlFileReader::new(Box::new(std::io::Cursor::new(bytes)), SqlGrammar::Sqlite).collect() } fn texts(bytes: Vec) -> Vec { @@ -231,9 +257,94 @@ mod tests { fn bytes_that_are_not_utf8_name_their_line() { let read = units(b"SELECT 1;\nSELECT 2;\nSELECT X'00', '\xff';\n".to_vec()); + let ran: Vec<&str> = read + .iter() + .filter_map(|unit| unit.as_ref().ok()) + .map(|unit| unit.text.as_str()) + .collect(); + assert_eq!(ran, ["SELECT 1", "SELECT 2"], "the statements before the byte"); + assert_eq!(read.last(), Some(&Err(SqlFileError::NotUtf8 { line: 3 }))); + } + + #[test] + fn a_copy_row_that_is_not_utf8_names_its_line() { + let file = b"COPY t (a) FROM stdin;\nok\n\xff\n\\.\n".to_vec(); + + let read: Vec> = + SqlFileReader::new(Box::new(std::io::Cursor::new(file)), SqlGrammar::PostgreSql).collect(); + assert_eq!(read.last(), Some(&Err(SqlFileError::NotUtf8 { line: 3 }))); } + /// Gives at most `chunk` bytes a read, as a pipe can, and then fails + /// when `fails`. + struct Trickle { + bytes: Vec, + at: usize, + chunk: usize, + fails: bool, + } + + impl Trickle { + fn new(bytes: &[u8], fails: bool) -> Self { + Self { + bytes: bytes.to_vec(), + at: 0, + chunk: 20, + fails, + } + } + } + + impl Read for Trickle { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let rest = self.bytes.get(self.at..).unwrap_or_default(); + if rest.is_empty() && self.fails { + return Err(std::io::Error::other("the copy was cut off")); + } + let read = rest.len().min(self.chunk).min(buffer.len()); + buffer[..read].copy_from_slice(&rest[..read]); + self.at += read; + Ok(read) + } + } + + fn read_trickle(file: Trickle) -> (Vec, Option) { + let read: Vec> = + SqlFileReader::new(Box::new(file), SqlGrammar::Sqlite).collect(); + let ran = read + .iter() + .filter_map(|unit| unit.as_ref().ok()) + .map(|unit| unit.text.clone()) + .collect(); + (ran, read.last().and_then(|unit| unit.clone().err())) + } + + #[test] + fn the_statements_read_whole_before_the_file_stops_being_readable_run() { + let cases = [ + ( + Trickle::new(b"SELECT 1;\nSELECT 2;\nSELECT 3;\n'\xff';\n", false), + SqlFileError::NotUtf8 { line: 4 }, + ), + ( + Trickle::new(b"SELECT 1;\nSELECT 2;\nSELECT 3;\n\xc3", false), + SqlFileError::NotUtf8 { line: 4 }, + ), + ( + Trickle::new(b"SELECT 1;\nSELECT 2;\nSELECT 3;\n", true), + SqlFileError::Read("the copy was cut off".to_owned()), + ), + ]; + + for (file, error) in cases { + let (ran, last) = read_trickle(file); + + assert_eq!(ran, ["SELECT 1", "SELECT 2", "SELECT 3"], "{error:?}"); + assert_eq!(last, Some(error)); + } + } + #[test] fn a_character_cut_between_reads_is_whole() { // The read ends between the two bytes of the é. diff --git a/core/crates/core/src/import/sql_import.rs b/core/crates/core/src/import/sql_import.rs index c4c0ddbbf7..5742856a87 100644 --- a/core/crates/core/src/import/sql_import.rs +++ b/core/crates/core/src/import/sql_import.rs @@ -5,9 +5,11 @@ use std::time::{Duration, Instant}; use async_trait::async_trait; use tokio::sync::mpsc; +use tokio_util::bytes::Bytes; use tokio_util::sync::CancellationToken; use crate::call_options::CallOptions; +use crate::copy_in::{CopyInPiece, CopyInSummary}; use crate::dialect::SqlDialect; use crate::dump::DumpDialect; use crate::editor_session::EditorSession; @@ -38,6 +40,9 @@ const KEPT_STATEMENT_CHARS: usize = 2_000; /// Units read ahead of the one running. const UNITS_AHEAD: usize = 8; +/// Pieces of COPY rows handed to the session ahead of the one it sends. +const PIECES_AHEAD: usize = 2; + /// How long a rollback or commit after the file may take. const FINISH_DEADLINE: Duration = Duration::from_secs(120); @@ -142,7 +147,7 @@ pub async fn run_sql_import( let started = Instant::now(); let grammar = dialect.grammar(); let settings = session_settings(editor, grammar, cancel).await; - let reader = SqlFileReader::new(file, grammar, dialect.insert_overriding_clause()).with_settings(settings); + let reader = SqlFileReader::new(file, grammar).with_settings(settings); let mut file = FileUnits::start(reader); let transaction = dialect.dump().filter(|dump| { options.one_transaction && dump.transactional_ddl() && options.on_error == SqlImportErrorPolicy::Stop @@ -352,6 +357,9 @@ impl<'a> Run<'a> { break; }; match next { + Ok(unit) if unit.kind == ScriptUnitKind::CopyFromStdin => { + first = self.copy(unit, file, progress, cancel).await.map(Ok); + } Ok(unit) => self.unit(unit, cancel).await, Err(error) => self.report.end = SqlImportEnd::File(error), } @@ -359,6 +367,87 @@ impl<'a> Run<'a> { } } + /// Run a `COPY ... FROM STDIN` with the rows after it, sent on as the + /// file is read. It counts as one statement, and one that fails loses + /// its whole block, as in psql. Returns a unit it read past its rows, + /// which runs next. + async fn copy( + &mut self, + unit: ScriptUnit, + file: &mut FileUnits, + progress: &mut (dyn FnMut(SqlImportProgress) + Send), + cancel: &CancellationToken, + ) -> Option { + if cancel.is_cancelled() { + self.report.end = SqlImportEnd::Cancelled; + return None; + } + let (sender, rows) = mpsc::channel(PIECES_AHEAD); + let editor = self.editor; + let copying = editor.copy_in( + UserSql::new(unit.text.as_str()), + rows, + RunOptions::new(cancel.child_token()), + ); + let feeding = feed(file, sender, &self.report, progress, cancel); + let (copied, fed) = tokio::join!(copying, feeding); + let (rows_left, past_rows) = match fed { + Fed::Cancelled => { + self.report.end = SqlImportEnd::Cancelled; + return None; + } + Fed::File(error) => { + self.report.end = SqlImportEnd::File(error); + return None; + } + Fed::Ended => (false, None), + Fed::Refused { rows_left } => (rows_left, None), + Fed::Unfinished(past_rows) => (false, past_rows), + }; + match copy_answer(copied) { + CopyAnswer::Done => self.report.statements += 1, + CopyAnswer::Failed(error) => { + self.failure(&unit, SqlImportFault::Statement(error)); + if rows_left && matches!(self.report.end, SqlImportEnd::Completed) { + return self.skip_rows(file, progress, cancel).await; + } + } + CopyAnswer::Ends(end) => self.report.end = end, + } + past_rows + } + + /// Pass over the rest of the rows of a COPY that failed before it + /// took them all, up to their end. Returns a unit that is not one of + /// them. + async fn skip_rows( + &mut self, + file: &mut FileUnits, + progress: &mut (dyn FnMut(SqlImportProgress) + Send), + cancel: &CancellationToken, + ) -> Option { + loop { + let next = tokio::select! { + () = cancel.cancelled() => { + self.report.end = SqlImportEnd::Cancelled; + return None; + } + next = file.units.recv() => next, + }; + progress(file.progress(&self.report)); + match next { + Some(Ok(unit)) if unit.kind == ScriptUnitKind::CopyData => {} + Some(Ok(unit)) if unit.kind == ScriptUnitKind::CopyEnd => return None, + Some(Ok(unit)) => return Some(unit), + None => return None, + Some(Err(error)) => { + self.report.end = SqlImportEnd::File(error); + return None; + } + } + } + } + /// Commit or roll back the import's transaction, as the run ended, or /// roll back one the file left open. async fn close( @@ -416,6 +505,9 @@ impl<'a> Run<'a> { } ScriptUnitKind::Refused(refused) => Some(refused.clone()), ScriptUnitKind::Run => None, + // Rows come only after their COPY, which `copy` reads to their + // end. + ScriptUnitKind::CopyFromStdin | ScriptUnitKind::CopyData | ScriptUnitKind::CopyEnd => return, }; if let Some(refused) = refused { self.failure(&unit, SqlImportFault::Refused(refused)); @@ -574,6 +666,85 @@ impl<'a> Run<'a> { } } +/// How handing a COPY its rows ended. +enum Fed { + /// Every row went, and then the end. + Ended, + /// The session stopped taking rows. `rows_left` once the file still + /// holds some of them, false when the refused piece was their end. + Refused { + rows_left: bool, + }, + Cancelled, + File(SqlFileError), + /// The file's units stopped short of the end of the rows, with the + /// unit read in their place. + Unfinished(Option), +} + +/// Hand the session the rows that follow a COPY, a piece at a time, up to +/// their end. Dropping `sender` before the end abandons the copy. +async fn feed( + file: &mut FileUnits, + sender: mpsc::Sender, + report: &SqlImportReport, + progress: &mut (dyn FnMut(SqlImportProgress) + Send), + cancel: &CancellationToken, +) -> Fed { + loop { + let next = tokio::select! { + biased; + () = cancel.cancelled() => return Fed::Cancelled, + next = file.units.recv() => next, + }; + let piece = match next { + Some(Ok(unit)) if unit.kind == ScriptUnitKind::CopyData => CopyInPiece::Data(Bytes::from(unit.text)), + Some(Ok(unit)) if unit.kind == ScriptUnitKind::CopyEnd => CopyInPiece::End, + Some(Ok(unit)) => return Fed::Unfinished(Some(unit)), + None => return Fed::Unfinished(None), + Some(Err(error)) => return Fed::File(error), + }; + let last = piece == CopyInPiece::End; + let sent = tokio::select! { + biased; + () = cancel.cancelled() => return Fed::Cancelled, + sent = sender.send(piece) => sent, + }; + progress(file.progress(report)); + match (sent, last) { + (Err(_), last) => return Fed::Refused { rows_left: !last }, + (Ok(()), true) => return Fed::Ended, + (Ok(()), false) => {} + } + } +} + +/// What a copy's answer means to the import. +enum CopyAnswer { + Done, + /// A failure the error policy decides on. + Failed(DriverError), + Ends(SqlImportEnd), +} + +fn copy_answer(copied: Result) -> CopyAnswer { + match copied { + Ok(summary) => match summary.end { + RunEnd::Completed => CopyAnswer::Done, + RunEnd::Failed(error) => CopyAnswer::Failed(error), + RunEnd::TimedOut { server_cancelled } => CopyAnswer::Failed(DriverError::Timeout { + phase: TimeoutPhase::Statement, + server_cancelled, + }), + RunEnd::Cancelled => CopyAnswer::Ends(SqlImportEnd::Cancelled), + }, + // The session cannot send COPY data, which fails this statement, + // not the session. + Err(error @ DriverError::Unsupported { .. }) => CopyAnswer::Failed(error), + Err(error) => CopyAnswer::Ends(SqlImportEnd::Session(error)), + } +} + /// How the session reads a backslash in a string as the import starts. /// The server's configuration decides it, and a file need not set it: /// MySQL's `NO_BACKSLASH_ESCAPES` and PostgreSQL's diff --git a/core/crates/core/src/lib.rs b/core/crates/core/src/lib.rs index 3307e8280a..cb750b59df 100644 --- a/core/crates/core/src/lib.rs +++ b/core/crates/core/src/lib.rs @@ -17,6 +17,7 @@ mod connect_options; mod connect_target; mod connection; pub mod connection_url; +mod copy_in; pub mod credentials; pub mod ddl; pub mod dialect; @@ -120,6 +121,7 @@ pub use connect_environment::{APPLICATION_NAME, ConnectEnvironment}; pub use connect_options::ConnectOptions; pub use connect_target::ConnectTarget; pub use connection::Connection; +pub use copy_in::{CopyInPiece, CopyInRows, CopyInSummary}; pub use credentials::Credentials; pub use driver::DatabaseDriver; pub use editor_session::{EditorSession, EditorSessionId, EditorSessionOptions, EditorSessionState}; diff --git a/core/crates/core/src/sql_syntax/script/copy_rows.rs b/core/crates/core/src/sql_syntax/script/copy_rows.rs index 50b1be9210..bb4364a016 100644 --- a/core/crates/core/src/sql_syntax/script/copy_rows.rs +++ b/core/crates/core/src/sql_syntax/script/copy_rows.rs @@ -4,40 +4,38 @@ use crate::sql_syntax::script::{ScriptToken, ScriptTokenKind}; use super::script_reader::{RefusedUnit, ScriptUnit, ScriptUnitKind}; -/// Rows one INSERT made from COPY data carries, as a dump's INSERTs do. -const ROWS_PER_INSERT: usize = 1_000; -const BYTES_PER_INSERT: usize = 1_046_528; +/// The most one piece of COPY rows holds: what the file is read by. +pub(super) const COPY_PIECE_BYTES: usize = 256 * 1024; -/// The rows of a `COPY table (columns) FROM stdin` block as INSERTs. -/// -/// A driver sends no COPY data, so the rows go in as statements instead: -/// each field of the text format as a string the server casts to its -/// column, as it casts COPY's own text. pg_dump writes this form. +/// psql's end of the rows, a line of its own. psql 18 never sends it, and +/// PostgreSQL 18 reads it as data in CSV. +const END_MARKER: &str = "\\."; + +/// The rows of a `COPY ... FROM stdin` block, passed on as the file holds +/// them. A piece is cut where the rows reach `COPY_PIECE_BYTES`, not where +/// a read of the file ended, so a file splits the same however it is read. #[derive(Debug)] pub(super) struct CopyRows { - start: String, - refused: Option, - tuples: String, - rows: usize, - first_line: u64, + binary: bool, + data: String, + /// The line `data` starts on. + data_line: u64, + /// The line the next byte taken is on. line: u64, + /// Part of the line being read was taken already, so it is a row and + /// not the end marker. + in_row: bool, } impl CopyRows { /// The block a statement opens, when it is a COPY from stdin. - pub(super) fn starting(text: &str, tokens: &[ScriptToken], overriding: Option<&str>) -> Option { + pub(super) fn starting(text: &str, tokens: &[ScriptToken]) -> Option { let code: Vec<&ScriptToken> = tokens .iter() .filter(|token| !token.kind.is_trivia() && token.kind != ScriptTokenKind::Semicolon) .collect(); - let word = |token: &ScriptToken, wanted: &str| { - token.kind == ScriptTokenKind::Word - && text - .get(token.range.clone()) - .is_some_and(|word| word.eq_ignore_ascii_case(wanted)) - }; let copy = code.first()?; - if !word(copy, "COPY") { + if !is_word(text, copy, "COPY") { return None; } let mut depth = 0i32; @@ -46,211 +44,221 @@ impl CopyRows { match token.kind { ScriptTokenKind::OpenParen => depth += 1, ScriptTokenKind::CloseParen => depth -= 1, - _ if depth == 0 && word(token, "FROM") => { + _ if depth == 0 && is_word(text, token, "FROM") => { from = Some(index); break; } _ => {} } } - let from = from?; - let source = code.get(from + 1)?; - if !word(source, "STDIN") { + let source = code.get(from? + 1)?; + if !is_word(text, source, "STDIN") { return None; } - let target = text.get(copy.range.end..code[from].range.start)?.trim(); - let mut start = format!("INSERT INTO {target} "); - if let Some(clause) = overriding { - start.push_str(clause); - start.push(' '); - } - start.push_str("VALUES\n"); - let refused = (code.len() > from + 2).then_some(RefusedUnit::CopyFormat); Some(Self { - start, - refused, - tuples: String::new(), - rows: 0, - first_line: 0, + binary: reads_binary(text, &code), + data: String::new(), + data_line: 0, line: 0, + in_row: false, }) } pub(super) fn refusal(&self) -> Option { - self.refused.clone() + self.binary.then_some(RefusedUnit::CopyBinary) } /// The data's first line. pub(super) fn at_line(mut self, line: u64) -> Self { self.line = line; - self.first_line = line; + self.data_line = line; self } - pub(super) fn row(&mut self, line: &str, units: &mut VecDeque) { - let number = self.line; - self.line += 1; - if self.refused.is_some() { - return; + /// Take the rows `pending` holds. Gives how many bytes were taken and + /// whether the rows ended. A part of a line that could still be the + /// end marker is left until the rest of the line comes. + pub(super) fn take(&mut self, pending: &str, end: bool, units: &mut VecDeque) -> (usize, bool) { + let mut consumed = 0; + loop { + let rest = pending.get(consumed..).unwrap_or_default(); + match rest.find('\n') { + Some(at) => { + let line = rest.get(..at).unwrap_or_default(); + if !self.in_row && is_end_marker(line) { + self.finish(units); + return (consumed + at + 1, true); + } + self.push(rest.get(..=at).unwrap_or_default(), units); + self.in_row = false; + consumed += at + 1; + } + None if end => { + if self.in_row || !is_end_marker(rest) { + self.push(rest, units); + } + self.finish(units); + return (pending.len(), true); + } + None => { + if !rest.is_empty() && (self.in_row || !may_be_end_marker(rest)) { + self.push(rest, units); + self.in_row = true; + consumed = pending.len(); + } + return (consumed, false); + } + } } - let Some(tuple) = tuple(line) else { - self.flush(units); - units.push_back(ScriptUnit { - text: line.to_owned(), - line: number, - repeat: 1, - kind: ScriptUnitKind::Refused(RefusedUnit::CopyNotText), - }); + } + + fn push(&mut self, rows: &str, units: &mut VecDeque) { + if self.binary || rows.is_empty() { return; - }; - if self.rows > 0 && self.tuples.len() + tuple.len() + self.start.len() > BYTES_PER_INSERT { - self.flush(units); } - if self.rows == 0 { - self.first_line = number; - } else { - self.tuples.push_str(",\n"); + if self.data.is_empty() { + self.data_line = self.line; } - self.tuples.push_str(&tuple); - self.rows += 1; - if self.rows >= ROWS_PER_INSERT { - self.flush(units); + self.line += rows.matches('\n').count() as u64; + self.data.push_str(rows); + while self.data.len() >= COPY_PIECE_BYTES { + let mut cut = COPY_PIECE_BYTES; + while !self.data.is_char_boundary(cut) { + cut -= 1; + } + let rest = self.data.split_off(cut); + let piece = std::mem::replace(&mut self.data, rest); + self.emit(piece, units); } } - pub(super) fn finish(&mut self, units: &mut VecDeque) { - self.flush(units); - } - - fn flush(&mut self, units: &mut VecDeque) { - if self.rows == 0 { + fn finish(&mut self, units: &mut VecDeque) { + if self.binary { return; } - let mut text = String::with_capacity(self.start.len() + self.tuples.len()); - text.push_str(&self.start); - text.push_str(&self.tuples); + let piece = std::mem::take(&mut self.data); + if !piece.is_empty() { + self.emit(piece, units); + } units.push_back(ScriptUnit { - text, - line: self.first_line, + text: String::new(), + line: self.line, repeat: 1, - kind: ScriptUnitKind::Run, + kind: ScriptUnitKind::CopyEnd, }); - self.tuples.clear(); - self.rows = 0; } -} -/// One line of COPY's text format as a parenthesised list of literals: -/// fields split at tabs, `\N` as NULL, backslash escapes read. `None` when -/// the escapes make bytes that are not UTF-8. -fn tuple(line: &str) -> Option { - let mut out = String::from("("); - for (index, field) in line.split('\t').enumerate() { - if index > 0 { - out.push_str(", "); - } - if field == "\\N" { - out.push_str("NULL"); - continue; - } - let value = String::from_utf8(unescape(field)).ok()?; - // An E'' string reads the same whatever standard_conforming_strings is. - out.push_str("E'"); - for character in value.chars() { - match character { - '\'' => out.push_str("''"), - '\\' => out.push_str("\\\\"), - other => out.push(other), - } - } - out.push('\''); + fn emit(&mut self, piece: String, units: &mut VecDeque) { + let line = self.data_line; + self.data_line += piece.matches('\n').count() as u64; + units.push_back(ScriptUnit { + text: piece, + line, + repeat: 1, + kind: ScriptUnitKind::CopyData, + }); } - out.push(')'); - Some(out) } -/// COPY's backslash sequences: `\b \f \n \r \t \v`, up to three octal -/// digits, `\x` and up to two hex digits, and a backslash before any other -/// character standing for that character. -fn unescape(field: &str) -> Vec { - let bytes = field.as_bytes(); - let mut out = Vec::with_capacity(bytes.len()); - let mut index = 0; - while let Some(&byte) = bytes.get(index) { - if byte != b'\\' { - out.push(byte); - index += 1; - continue; - } - let Some(&next) = bytes.get(index + 1) else { - out.push(b'\\'); - break; - }; - index += 2; - match next { - b'b' => out.push(0x08), - b'f' => out.push(0x0c), - b'n' => out.push(b'\n'), - b'r' => out.push(b'\r'), - b't' => out.push(b'\t'), - b'v' => out.push(0x0b), - b'0'..=b'7' => { - let mut value = u32::from(next - b'0'); - for _ in 0..2 { - match bytes.get(index) { - Some(digit @ b'0'..=b'7') => { - value = value * 8 + u32::from(digit - b'0'); - index += 1; - } - _ => break, - } - } - out.push((value & 0xff) as u8); - } - b'x' => { - let mut value = 0u32; - let mut digits = 0; - while digits < 2 { - match bytes.get(index).and_then(|digit| char::from(*digit).to_digit(16)) { - Some(digit) => { - value = value * 16 + digit; - index += 1; - digits += 1; - } - None => break, - } - } - match digits { - 0 => out.push(b'x'), - _ => out.push((value & 0xff) as u8), - } - } - other => out.push(other), +fn is_end_marker(line: &str) -> bool { + line.strip_suffix('\r').unwrap_or(line) == END_MARKER +} + +/// Whether the start of a line can still turn out to be the end marker. +fn may_be_end_marker(part: &str) -> bool { + END_MARKER.starts_with(part) || is_end_marker(part) +} + +fn is_word(text: &str, token: &ScriptToken, wanted: &str) -> bool { + token.kind == ScriptTokenKind::Word + && text + .get(token.range.clone()) + .is_some_and(|word| word.eq_ignore_ascii_case(wanted)) +} + +/// Whether the copy reads the binary format, which a script cannot hold: +/// `COPY BINARY t`, the old `WITH BINARY`, or `FORMAT binary` among the +/// options. A WHERE clause ends the options. +fn reads_binary(text: &str, code: &[&ScriptToken]) -> bool { + let mut depth = 0i32; + let mut after_format = false; + for token in code { + match token.kind { + ScriptTokenKind::OpenParen => depth += 1, + ScriptTokenKind::CloseParen => depth -= 1, + _ if depth == 0 && is_word(text, token, "WHERE") => return false, + _ if depth == 0 && is_word(text, token, "BINARY") => return true, + _ if after_format && names_binary(text, token) => return true, + _ => {} } + after_format = depth == 1 && is_word(text, token, "FORMAT"); } - out + false +} + +/// `binary`, `'binary'` or `"binary"`. +fn names_binary(text: &str, token: &ScriptToken) -> bool { + let quoted = matches!( + token.kind, + ScriptTokenKind::Word | ScriptTokenKind::StringLiteral | ScriptTokenKind::QuotedIdentifier + ); + quoted + && text + .get(token.range.clone()) + .map(|name| name.trim_matches(|character| character == '\'' || character == '"')) + .is_some_and(|name| name.eq_ignore_ascii_case("binary")) } #[cfg(test)] mod tests { + use crate::sql_syntax::SqlGrammar; + use crate::sql_syntax::script::{LexicalSettings, ScriptPlan}; + use super::*; - #[test] - fn a_text_format_line_becomes_literals() { - assert_eq!( - tuple("1\tit's\t\\N\ta\\tb\\nc\\\\d\t\\\\x00ff").as_deref(), - Some("(E'1', E'it''s', NULL, E'a\tb\nc\\\\d', E'\\\\x00ff')") + fn binary(statement: &str) -> Option { + let plan = ScriptPlan::build( + statement, + SqlGrammar::PostgreSql, + LexicalSettings { + backslash_escapes: false, + }, ); + let tokens: Vec = plan.tokens(0..statement.len()).collect(); + CopyRows::starting(statement, &tokens).map(|copy| copy.binary) } #[test] - fn octal_and_hex_escapes_read_as_bytes() { - assert_eq!(unescape("\\101\\x42\\x4"), b"AB\x04"); - assert_eq!(tuple("\\377"), None, "a lone 0xFF byte is not UTF-8"); + fn every_spelling_of_the_binary_format_is_found() { + for statement in [ + "COPY t FROM stdin (FORMAT binary)", + "copy t from stdin with (format 'binary')", + "COPY t FROM STDIN WITH (FREEZE, FORMAT \"binary\")", + "COPY BINARY t FROM stdin", + "COPY t FROM stdin WITH BINARY", + "COPY t (a, b) FROM stdin BINARY", + ] { + assert_eq!(binary(statement), Some(true), "{statement}"); + } + } + + #[test] + fn text_and_csv_with_any_options_are_not_binary() { + for statement in [ + "COPY t FROM stdin", + "COPY t (a, b) FROM stdin WITH (FORMAT csv, HEADER true, NULL 'x', DELIMITER ';')", + "COPY t FROM stdin WITH CSV HEADER QUOTE AS '\"'", + "COPY t (\"binary\", format) FROM stdin", + "COPY t FROM stdin (FORMAT text) WHERE (format = 'binary')", + ] { + assert_eq!(binary(statement), Some(false), "{statement}"); + } } #[test] - fn an_empty_field_is_an_empty_string() { - assert_eq!(tuple("").as_deref(), Some("(E'')")); + fn only_a_copy_from_stdin_opens_a_block() { + assert_eq!(binary("COPY t FROM '/tmp/rows'"), None); + assert_eq!(binary("COPY (SELECT 1) TO STDOUT"), None); + assert_eq!(binary("SELECT 'COPY t FROM stdin'"), None); } } diff --git a/core/crates/core/src/sql_syntax/script/script_reader/mod.rs b/core/crates/core/src/sql_syntax/script/script_reader/mod.rs index c15f8d84cf..894a840eb3 100644 --- a/core/crates/core/src/sql_syntax/script/script_reader/mod.rs +++ b/core/crates/core/src/sql_syntax/script/script_reader/mod.rs @@ -31,6 +31,15 @@ pub struct ScriptUnit { pub enum ScriptUnitKind { /// Sent to the server. Run, + /// A `COPY ... FROM stdin`, sent as written with the `CopyData` units + /// that follow it as its rows, up to the `CopyEnd`. + CopyFromStdin, + /// A piece of the rows, as the file holds them: the lines after the + /// COPY's line, cut anywhere. + CopyData, + /// The rows ended, at psql's `\.` line or the end of the file. The + /// marker itself is never sent. + CopyEnd, /// A line only psql reads, such as pg_dump's `\restrict`, which keeps /// psql from running meta-commands a server could plant in the data. /// Nothing here runs meta-commands, so it is passed over. @@ -43,11 +52,9 @@ pub enum ScriptUnitKind { pub enum RefusedUnit { /// A psql meta-command other than `\restrict` and `\unrestrict`. ClientCommand { command: String }, - /// `COPY ... FROM stdin` in a form other than the text format pg_dump - /// writes, or with options. - CopyFormat, - /// `COPY` data that is not UTF-8 once its escapes are read. - CopyNotText, + /// `COPY ... FROM stdin` in the binary format, whose rows a script + /// cannot hold. Its rows are passed over. + CopyBinary, } #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] @@ -63,10 +70,11 @@ pub enum ScriptReadError { /// `DELIMITER` line sets, at SQL Server `GO` lines. Only text up to the /// last line break is planned until the end, since a directive is a whole /// line. On PostgreSQL it also reads what psql reads in a pg_dump file: -/// `COPY ... FROM stdin` rows up to `\.`, which become INSERTs, and -/// backslash lines. A `SET` of `standard_conforming_strings` or of a -/// `sql_mode` with `NO_BACKSLASH_ESCAPES` changes how the rest reads, as -/// it does on the server. +/// `COPY ... FROM stdin` rows up to `\.`, given as the file holds them in +/// bounded pieces, and backslash lines. A `SET` of +/// `standard_conforming_strings` or of a `sql_mode` with +/// `NO_BACKSLASH_ESCAPES` changes how the rest reads, as it does on the +/// server. #[derive(Debug)] pub struct ScriptReader { grammar: SqlGrammar, @@ -77,9 +85,12 @@ pub struct ScriptReader { delimiter: String, plan_at: usize, copy: Option, - /// What lets an INSERT made from COPY rows set an identity column, as - /// COPY itself may. - copy_overriding: Option, + /// What follows a COPY on its own line, which psql runs once the rows + /// end, with the line it is on. + after_rows: Option<(String, u64)>, + /// The line of the first line of `pending` when it is not `line`: the + /// rest of a COPY's line, put back in front of what follows its rows. + first_line: Option, tracker: LexicalTracker, units: VecDeque, } @@ -94,7 +105,8 @@ impl ScriptReader { delimiter: ";".to_owned(), plan_at: 0, copy: None, - copy_overriding: None, + after_rows: None, + first_line: None, tracker: LexicalTracker::default(), units: VecDeque::new(), } @@ -107,13 +119,6 @@ impl ScriptReader { self } - /// The clause `SqlDialect::insert_overriding_clause` gives, for the - /// INSERTs COPY rows become. - pub fn with_copy_overriding(mut self, clause: Option<&str>) -> Self { - self.copy_overriding = clause.map(str::to_owned); - self - } - /// Take the next piece of the script, and give back the units it /// completed. pub fn push(&mut self, text: &str) -> Result, ScriptReadError> { @@ -122,6 +127,13 @@ impl ScriptReader { Ok(self.units.drain(..).collect()) } + /// Take the last piece before the text stops being readable, and give + /// back every unit it completed, without waiting for more text first. + pub(crate) fn push_last(&mut self, text: &str) -> Result, ScriptReadError> { + self.plan_at = 0; + self.push(text) + } + /// The end of the script: what is left, whole or not, as the client /// sends a last statement with no terminator. pub fn finish(mut self) -> Result, ScriptReadError> { @@ -159,7 +171,7 @@ impl ScriptReader { } Planned::Nothing => { if self.pending.len() > MAX_UNIT_BYTES { - return Err(ScriptReadError::TooLong { line: self.line }); + return Err(ScriptReadError::TooLong { line: self.line_of(0) }); } self.plan_at = (self.pending.len() * 2).max(self.pending.len() + PLAN_STEP); break; @@ -182,6 +194,10 @@ impl ScriptReader { false => settled.offset, }; let mut found = Vec::new(); + let mut after_rows = None; + // A COPY whose line has not all been read yet, on a line too long + // to wait for whole. + let mut waits_at = None; let mut interrupted_at = None; let mut delimiter_at_cut = None; for batch in plan.batches() { @@ -195,21 +211,34 @@ impl ScriptReader { } if self.grammar == SqlGrammar::PostgreSql { let tokens: Vec = plan.tokens(batch.range.clone()).collect(); - if let Some(copy) = CopyRows::starting(text, &tokens, self.copy_overriding.as_deref()) { - let after_line = text - .get(batch.range.end..) - .and_then(|rest| rest.find('\n')) - .map_or(limit, |at| batch.range.end + at + 1); - // A COPY the reader turns into INSERTs is never sent; - // one it cannot read is reported and its rows passed over. - if let Some(refused) = copy.refusal() { - found.push((batch.range.clone(), 1, Some(refused))); + if let Some(copy) = CopyRows::starting(text, &tokens) { + // psql reads the rows from the line after the COPY's + // `;`, and runs the rest of that line once they end. + let ends = plan + .statements() + .get(batch.statements.start) + .map_or(batch.range.end, |statement| statement.extent.end.max(batch.range.end)); + let line_end = text.get(ends..).and_then(|rest| rest.find('\n')).map(|at| ends + at); + if line_end.is_none() && !end { + waits_at = Some(batch.range.start); + break; } + let after_line = line_end.map_or(limit, |at| at + 1); + let rest = text.get(ends..line_end.unwrap_or(limit)).unwrap_or_default(); + if !rest.trim().is_empty() { + let newlines = text.get(..ends).unwrap_or_default().matches('\n').count() as u64; + after_rows = Some((rest.to_owned(), self.line_of(newlines))); + } + let kind = match copy.refusal() { + Some(refused) => ScriptUnitKind::Refused(refused), + None => ScriptUnitKind::CopyFromStdin, + }; + found.push((batch.range.clone(), 1, kind)); interrupted_at = Some((after_line, Some(copy))); break; } } - found.push((batch.range.clone(), batch.repeat.get(), None)); + found.push((batch.range.clone(), batch.repeat.get(), ScriptUnitKind::Run)); // A statement that changes how a backslash reads changes how // the rest of the text splits, so it is planned again from here. if let Some(settings) = self.tracker.after(unit_text, self.grammar, self.settings) { @@ -224,24 +253,26 @@ impl ScriptReader { } } let lines_before = |offset: usize| text.get(..offset).unwrap_or_default().matches('\n').count() as u64; - for (range, repeat, refused) in &found { + for (range, repeat, kind) in found { let unit_text = text.get(range.clone()).unwrap_or_default().trim().to_owned(); if unit_text.is_empty() { continue; } self.units.push_back(ScriptUnit { text: unit_text, - line: self.line + lines_before(range.start), - repeat: *repeat, - kind: match refused { - Some(refused) => ScriptUnitKind::Refused(refused.clone()), - None => ScriptUnitKind::Run, - }, + line: self.line_of(lines_before(range.start)), + repeat, + kind, }); } + if let Some(start) = waits_at { + self.cut(start); + return Planned::Nothing; + } match interrupted_at { Some((cut, copy)) => { self.cut(cut); + self.after_rows = after_rows; self.copy = copy.map(|copy| copy.at_line(self.line)); self.delimiter = delimiter_at_cut.unwrap_or_else(|| self.delimiter.clone()); Planned::Interrupted @@ -304,7 +335,7 @@ impl ScriptReader { .unwrap_or_default() .trim_end() .to_owned(); - let starts_on = self.line + self.pending.get(..offset).unwrap_or_default().matches('\n').count() as u64; + let starts_on = self.line_of(self.pending.get(..offset).unwrap_or_default().matches('\n').count() as u64); let command = line.split_whitespace().next().unwrap_or_default().to_owned(); let kind = match command.as_str() { "\\restrict" | "\\unrestrict" => ScriptUnitKind::ClientOnly, @@ -321,37 +352,21 @@ impl ScriptReader { } } - /// Read COPY rows a line at a time. True when the data ended and the - /// script goes on. + /// Read COPY rows. True when the data ended and the script goes on. fn copy_lines(&mut self, end: bool) -> bool { let Some(mut copy) = self.copy.take() else { return false; }; - let mut consumed = 0; - let mut finished = false; - while let Some(at) = self.pending.get(consumed..).and_then(|rest| rest.find('\n')) { - let line = self.pending.get(consumed..consumed + at).unwrap_or_default(); - let line = line.strip_suffix('\r').unwrap_or(line); - consumed += at + 1; - if line == "\\." { - finished = true; - break; - } - copy.row(line, &mut self.units); - } - if !finished && end { - let line = self.pending.get(consumed..).unwrap_or_default(); - let line = line.strip_suffix('\r').unwrap_or(line); - if !line.is_empty() && line != "\\." { - copy.row(line, &mut self.units); - } - consumed = self.pending.len(); - finished = true; - } + let (consumed, finished) = copy.take(&self.pending, end, &mut self.units); self.cut(consumed); if finished { - copy.finish(&mut self.units); self.plan_at = 0; + if let Some((rest, line)) = self.after_rows.take() { + self.pending.insert(0, '\n'); + self.pending.insert_str(0, &rest); + self.line = self.line.saturating_sub(1); + self.first_line = Some(line); + } return true; } self.copy = Some(copy); @@ -360,16 +375,29 @@ impl ScriptReader { fn cut(&mut self, offset: usize) { let offset = offset.min(self.pending.len()); - self.line += self.pending.get(..offset).unwrap_or_default().matches('\n').count() as u64; + let newlines = self.pending.get(..offset).unwrap_or_default().matches('\n').count() as u64; + if newlines > 0 { + self.first_line = None; + } + self.line += newlines; self.pending.drain(..offset); } + + /// The line of text that starts after `newlines` line breaks of + /// `pending`. + fn line_of(&self, newlines: u64) -> u64 { + match (newlines, self.first_line) { + (0, Some(line)) => line, + _ => self.line + newlines, + } + } } enum Planned { /// A COPY or a backslash line stopped the plan; read on from there. Interrupted, Settled, - /// No unit is whole yet. + /// No unit is whole yet, or a COPY waits for the rest of its line. Nothing, } diff --git a/core/crates/core/src/sql_syntax/script/script_reader/tests.rs b/core/crates/core/src/sql_syntax/script/script_reader/tests.rs index 036189ad74..4e3042142e 100644 --- a/core/crates/core/src/sql_syntax/script/script_reader/tests.rs +++ b/core/crates/core/src/sql_syntax/script/script_reader/tests.rs @@ -1,7 +1,9 @@ +use crate::sql_syntax::script::copy_rows::COPY_PIECE_BYTES; + use super::*; fn read_in_pieces(grammar: SqlGrammar, text: &str, piece: usize) -> Vec { - let mut reader = ScriptReader::new(grammar).with_copy_overriding(Some("OVERRIDING SYSTEM VALUE")); + let mut reader = ScriptReader::new(grammar); let mut units = Vec::new(); let mut start = 0; while start < text.len() { @@ -29,6 +31,19 @@ fn texts(units: &[ScriptUnit]) -> Vec<&str> { units.iter().map(|unit| unit.text.as_str()).collect() } +fn kinds(units: &[ScriptUnit]) -> Vec<&ScriptUnitKind> { + units.iter().map(|unit| &unit.kind).collect() +} + +/// Every piece of COPY rows joined, as the session receives them. +fn copy_data(units: &[ScriptUnit]) -> String { + units + .iter() + .filter(|unit| unit.kind == ScriptUnitKind::CopyData) + .map(|unit| unit.text.as_str()) + .collect() +} + #[test] fn statements_split_at_semicolons_with_their_lines() { let units = read_every_way( @@ -96,41 +111,241 @@ fn a_sql_server_batch_ends_at_go_and_keeps_its_count() { } #[test] -fn pg_dump_copy_rows_become_inserts_and_restrict_lines_are_passed_over() { +fn pg_dump_copy_rows_follow_their_copy_and_restrict_lines_are_passed_over() { let dump = "--\n-- PostgreSQL database dump\n--\n\n\\restrict abc123\n\nSET client_encoding = 'UTF8';\n\nCOPY app.child (id, note) FROM stdin;\n1\tit's\n2\t\\N\n\\.\n\nSELECT pg_catalog.setval('app.child_id_seq', 5, true);\n\n\\unrestrict abc123\n"; let units = read_every_way(SqlGrammar::PostgreSql, dump); - let kinds: Vec<&ScriptUnitKind> = units.iter().map(|unit| &unit.kind).collect(); assert_eq!( - kinds, + kinds(&units), [ &ScriptUnitKind::ClientOnly, &ScriptUnitKind::Run, - &ScriptUnitKind::Run, + &ScriptUnitKind::CopyFromStdin, + &ScriptUnitKind::CopyData, + &ScriptUnitKind::CopyEnd, &ScriptUnitKind::Run, &ScriptUnitKind::ClientOnly ], "{units:?}" ); + assert_eq!(units[2].text, "COPY app.child (id, note) FROM stdin"); + assert_eq!(units[3].text, "1\tit's\n2\t\\N\n"); assert_eq!( - units[2].text, - "INSERT INTO app.child (id, note) OVERRIDING SYSTEM VALUE VALUES\n(E'1', E'it''s'),\n(E'2', NULL)" + [units[2].line, units[3].line, units[4].line, units[5].line], + [9, 10, 12, 14] ); - assert_eq!(units[2].line, 10); - assert_eq!(units[3].line, 14); } #[test] -fn a_copy_with_options_is_refused_and_its_rows_skipped() { +fn crlf_rows_pass_byte_exact_and_a_crlf_end_marker_is_not_sent() { let units = read_every_way( SqlGrammar::PostgreSql, - "COPY t FROM stdin WITH (FORMAT csv);\n1,2\n\\.\nSELECT 1;\n", + "COPY t FROM stdin;\r\n1\tx\r\n\\.x\r\n2\t\\\\.\r\n\\.\r\nSELECT 1;\r\n", ); - assert_eq!(units.len(), 2, "{units:?}"); - assert_eq!(units[0].kind, ScriptUnitKind::Refused(RefusedUnit::CopyFormat)); - assert_eq!(units[1].text, "SELECT 1"); + assert_eq!( + kinds(&units), + [ + &ScriptUnitKind::CopyFromStdin, + &ScriptUnitKind::CopyData, + &ScriptUnitKind::CopyEnd, + &ScriptUnitKind::Run + ], + "{units:?}" + ); + assert_eq!(copy_data(&units), "1\tx\r\n\\.x\r\n2\t\\\\.\r\n"); + assert_eq!(units[3].text, "SELECT 1"); +} + +#[test] +fn rows_without_an_end_marker_run_to_the_end_of_the_file() { + let units = read_every_way(SqlGrammar::PostgreSql, "COPY t FROM stdin;\n1\n2"); + + assert_eq!( + kinds(&units), + [ + &ScriptUnitKind::CopyFromStdin, + &ScriptUnitKind::CopyData, + &ScriptUnitKind::CopyEnd + ], + "{units:?}" + ); + assert_eq!(copy_data(&units), "1\n2"); +} + +#[test] +fn an_end_marker_on_the_last_line_without_a_line_break_is_not_sent() { + for file in ["COPY t FROM stdin;\n1\n\\.", "COPY t FROM stdin;\n1\n\\.\r"] { + let units = read_every_way(SqlGrammar::PostgreSql, file); + + assert_eq!(copy_data(&units), "1\n", "{file:?}"); + assert_eq!(units.last().map(|unit| &unit.kind), Some(&ScriptUnitKind::CopyEnd)); + } +} + +#[test] +fn what_follows_a_copy_on_its_line_runs_after_its_rows() { + let units = read_every_way( + SqlGrammar::PostgreSql, + "COPY t FROM stdin; SELECT 1; -- after\n1\n\\.\nSELECT 2;\n", + ); + + assert_eq!( + kinds(&units), + [ + &ScriptUnitKind::CopyFromStdin, + &ScriptUnitKind::CopyData, + &ScriptUnitKind::CopyEnd, + &ScriptUnitKind::Run, + &ScriptUnitKind::Run + ], + "{units:?}" + ); + assert_eq!(copy_data(&units), "1\n"); + assert_eq!(texts(&units)[3..], ["SELECT 1", "SELECT 2"]); + assert_eq!([units[3].line, units[4].line], [1, 4]); +} + +#[test] +fn a_second_copy_on_the_line_reads_the_rows_after_the_first() { + let units = read_every_way( + SqlGrammar::PostgreSql, + "COPY a FROM stdin; COPY b FROM stdin;\n1\n\\.\n2\n\\.\nSELECT 3;\n", + ); + + assert_eq!( + kinds(&units), + [ + &ScriptUnitKind::CopyFromStdin, + &ScriptUnitKind::CopyData, + &ScriptUnitKind::CopyEnd, + &ScriptUnitKind::CopyFromStdin, + &ScriptUnitKind::CopyData, + &ScriptUnitKind::CopyEnd, + &ScriptUnitKind::Run + ], + "{units:?}" + ); + assert_eq!(texts(&units)[3], "COPY b FROM stdin"); + assert_eq!(units[4].text, "2\n"); + let lines: Vec = units.iter().map(|unit| unit.line).collect(); + assert_eq!(lines, [1, 2, 3, 1, 4, 5, 6]); +} + +#[test] +fn a_copy_on_a_long_line_waits_for_the_rest_of_its_line() { + // The line reaches the length the reader plans a long line at just + // after the COPY's `;`, partway through the statement after it. + let inserts = "INSERT INTO t VALUES (1);".repeat(2_620); + let file = format!("{inserts} COPY t FROM stdin; DELETE FROM t WHERE a = 'tmp';\nkeep me\n\\.\n"); + assert!((inserts.len() + 19..inserts.len() + 50).contains(&PLAN_STEP)); + + let units = read_every_way(SqlGrammar::PostgreSql, &file); + + assert_eq!(units.len(), 2_624, "{:?}", &units[2_615..]); + assert_eq!( + kinds(&units)[2_620..], + [ + &ScriptUnitKind::CopyFromStdin, + &ScriptUnitKind::CopyData, + &ScriptUnitKind::CopyEnd, + &ScriptUnitKind::Run + ] + ); + assert_eq!(copy_data(&units), "keep me\n"); + assert_eq!(units[2_623].text, "DELETE FROM t WHERE a = 'tmp'"); + assert_eq!(units[2_623].line, 1); +} + +#[test] +fn the_rows_start_on_the_line_after_the_copys_terminator() { + let units = read_every_way(SqlGrammar::PostgreSql, "COPY t FROM stdin\n;\n1\n\\.\nSELECT 1;\n"); + + assert_eq!(texts(&units)[0], "COPY t FROM stdin"); + assert_eq!(copy_data(&units), "1\n"); + assert_eq!([units[1].line, units[3].line], [3, 5]); +} + +#[test] +fn a_copy_with_no_rows_still_ends() { + let units = read_every_way(SqlGrammar::PostgreSql, "COPY t FROM stdin;\n\\.\nSELECT 1;\n"); + + assert_eq!( + kinds(&units), + [ + &ScriptUnitKind::CopyFromStdin, + &ScriptUnitKind::CopyEnd, + &ScriptUnitKind::Run + ], + "{units:?}" + ); +} + +#[test] +fn a_csv_copy_with_options_passes_its_rows_as_written() { + let rows = "id;note\n1;\"two\nlines\"\n2;x\n"; + let units = read_every_way( + SqlGrammar::PostgreSql, + &format!("COPY t FROM stdin WITH (FORMAT csv, HEADER true, DELIMITER ';', NULL 'x');\n{rows}\\.\nSELECT 1;\n"), + ); + + assert_eq!( + units[0].text, + "COPY t FROM stdin WITH (FORMAT csv, HEADER true, DELIMITER ';', NULL 'x')" + ); + assert_eq!(units[0].kind, ScriptUnitKind::CopyFromStdin); + assert_eq!(copy_data(&units), rows); + assert_eq!(units.last().map(|unit| unit.text.as_str()), Some("SELECT 1")); +} + +#[test] +fn a_binary_copy_is_refused_and_its_rows_passed_over() { + for copy in [ + "COPY t FROM stdin (FORMAT binary);", + "COPY t FROM STDIN WITH (format 'binary');", + "COPY BINARY t FROM stdin;", + "COPY t FROM stdin WITH BINARY;", + ] { + let units = read_every_way( + SqlGrammar::PostgreSql, + &format!("{copy}\nPGCOPY\n\u{1}\u{2}\n\\.\nSELECT 1;\n"), + ); + + assert_eq!( + kinds(&units), + [&ScriptUnitKind::Refused(RefusedUnit::CopyBinary), &ScriptUnitKind::Run], + "{copy}: {units:?}" + ); + assert_eq!(units[1].text, "SELECT 1"); + } +} + +#[test] +fn long_rows_come_in_pieces_cut_by_size_whatever_the_reads() { + let rows: String = (0..60_000).map(|n| format!("{n}\tr\u{e9}sum\u{e9} {n}\n")).collect(); + let file = format!("COPY t (a, b) FROM stdin;\n{rows}\\.\nSELECT 1;\n"); + + let whole = read_in_pieces(SqlGrammar::PostgreSql, &file, file.len()); + for piece in [4_096, 100_003] { + assert_eq!( + read_in_pieces(SqlGrammar::PostgreSql, &file, piece), + whole, + "reads of {piece}" + ); + } + + let pieces: Vec<&ScriptUnit> = whole + .iter() + .filter(|unit| unit.kind == ScriptUnitKind::CopyData) + .collect(); + assert!(pieces.len() >= 3, "{} pieces", pieces.len()); + assert!(pieces.iter().all(|piece| piece.text.len() <= COPY_PIECE_BYTES)); + assert_eq!(copy_data(&whole), rows); + assert_eq!(pieces[0].line, 2); + let second_starts = 2 + pieces[0].text.matches('\n').count() as u64; + assert_eq!(pieces[1].line, second_starts); + assert_eq!(whole.last().map(|unit| unit.line), Some(60_003)); } #[test] diff --git a/core/crates/core/src/unsupported_feature.rs b/core/crates/core/src/unsupported_feature.rs index 6c9e50e76c..c6a9d43c71 100644 --- a/core/crates/core/src/unsupported_feature.rs +++ b/core/crates/core/src/unsupported_feature.rs @@ -19,6 +19,9 @@ pub enum UnsupportedFeature { }, CopyFromStdinInEditor, CopyToStdoutInEditor, + /// The rows of a `COPY ... FROM STDIN`, on a session that cannot send + /// them. + CopyFromStdin, /// A TIMESTAMP in the hour the session time zone repeats, which the /// server cannot store as the one instant it was given. TimestampInRepeatedHour, @@ -78,6 +81,7 @@ impl fmt::Display for UnsupportedFeature { Self::MinimumVersion { required } => write!(f, "{required} or later"), Self::CopyFromStdinInEditor => f.write_str("COPY FROM STDIN in the editor"), Self::CopyToStdoutInEditor => f.write_str("COPY TO STDOUT in the editor"), + Self::CopyFromStdin => f.write_str("COPY FROM STDIN rows"), Self::TimestampInRepeatedHour => { f.write_str("single instant for a TIMESTAMP in the hour its time zone repeats") } diff --git a/core/crates/drivers/postgres/Cargo.toml b/core/crates/drivers/postgres/Cargo.toml index 0215c3619d..fd0c9fa3a3 100644 --- a/core/crates/drivers/postgres/Cargo.toml +++ b/core/crates/drivers/postgres/Cargo.toml @@ -14,7 +14,7 @@ chrono.workspace = true deadpool.workspace = true deadpool-postgres.workspace = true fallible-iterator.workspace = true -futures-util.workspace = true +futures-util = { workspace = true, features = ["sink"] } postgres-protocol.workspace = true postgres-types.workspace = true secrecy.workspace = true diff --git a/core/crates/drivers/postgres/src/editor/actor.rs b/core/crates/drivers/postgres/src/editor/actor.rs index fda5f7a693..3bc64daa50 100644 --- a/core/crates/drivers/postgres/src/editor/actor.rs +++ b/core/crates/drivers/postgres/src/editor/actor.rs @@ -1,7 +1,10 @@ use std::sync::Arc; use std::time::Duration; -use tablepro_core::{DriverError, LossPhase, ResultSink, RunOptions, RunSummary, ServerNotice, UserSql}; +use tablepro_core::{ + CopyInRows, CopyInSummary, DriverError, LossPhase, ResultSink, RunOptions, RunSummary, ServerNotice, + TransactionState, UserSql, +}; use tokio::sync::{mpsc, oneshot}; use tokio_util::sync::CancellationToken; @@ -17,6 +20,12 @@ pub(crate) enum Command { sink: Box, reply: oneshot::Sender>, }, + CopyIn { + statement: UserSql, + rows: CopyInRows, + options: RunOptions, + reply: oneshot::Sender>, + }, Close, } @@ -41,37 +50,74 @@ pub(crate) async fn session_actor( () = &mut driving => break, command = commands.recv() => command, }; - let Some(Command::Run { - sql, - options, - mut sink, - reply, - }) = command - else { + let going_on = match command { + Some(Command::Run { + sql, + options, + mut sink, + reply, + }) => { + let outcome = tokio::select! { + () = &mut driving => None, + outcome = core.run(&sql, options, sink.as_mut(), &shutdown) => outcome.ok(), + }; + let outcome = outcome.map(|summary| { + let state = summary.transaction_state; + (summary, state) + }); + answer(outcome, reply, &status) + } + Some(Command::CopyIn { + statement, + rows, + options, + reply, + }) => { + let outcome = tokio::select! { + () = &mut driving => None, + outcome = core.copy_in(&statement, rows, options, &shutdown) => outcome.ok(), + }; + let outcome = outcome.map(|summary| { + let state = summary.transaction_state; + (summary, state) + }); + answer(outcome, reply, &status) + } // A close, or every handle is gone. Dropping the client first // lets the connection send Terminate before it ends. - drop(core); - let _ = tokio::time::timeout(close_timeout, &mut driving).await; - status.lose(); - return; - }; - let outcome = tokio::select! { - () = &mut driving => None, - outcome = core.run(&sql, options, sink.as_mut(), &shutdown) => outcome.ok(), - }; - match outcome { - Some(summary) => { - status.finish(summary.transaction_state); - let _ = reply.send(Ok(summary)); - } - None => { + Some(Command::Close) | None => { + drop(core); + let _ = tokio::time::timeout(close_timeout, &mut driving).await; status.lose(); - let _ = reply.send(Err(DriverError::ConnectionLost { - during: LossPhase::Statement, - })); - break; + return; } + }; + if !going_on { + break; } } status.lose(); } + +/// Send a command's answer, or the connection's loss when there is none. +/// False once the session is lost. +fn answer( + outcome: Option<(T, TransactionState)>, + reply: oneshot::Sender>, + status: &SessionStatus, +) -> bool { + match outcome { + Some((outcome, state)) => { + status.finish(state); + let _ = reply.send(Ok(outcome)); + true + } + None => { + status.lose(); + let _ = reply.send(Err(DriverError::ConnectionLost { + during: LossPhase::Statement, + })); + false + } + } +} diff --git a/core/crates/drivers/postgres/src/editor/copy_flight.rs b/core/crates/drivers/postgres/src/editor/copy_flight.rs new file mode 100644 index 0000000000..e0868d1a69 --- /dev/null +++ b/core/crates/drivers/postgres/src/editor/copy_flight.rs @@ -0,0 +1,210 @@ +use std::future::Future; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use futures_util::SinkExt; +use tablepro_core::{CopyInPiece, CopyInRows, DriverError, ResultEvent, ResultSink, RunEnd, SinkFlow}; +use tokio_postgres::CancelToken; + +use super::editor_dispatch::{CopyFlight, copy_plan}; +use super::run::{Run, end_after_stop}; +use super::session_core::Lost; +use super::stop_watch::StopWatch; +use super::transaction_tracker::KnownState; +use crate::errors::map_error; +use crate::physical::PgEndpoint; + +/// How a copy ended. +pub(crate) struct Copied { + pub(crate) end: RunEnd, + pub(crate) rows: u64, +} + +impl Copied { + fn ended(end: RunEnd) -> Self { + Self { end, rows: 0 } + } +} + +/// What the copy came to, before the driver's own transaction ends. +enum Flown { + Copied(u64), + /// The server refused the statement or a row, or the end of the copy. + Failed(DriverError), + /// Stopped, or the rows' source gave up, so the copy was failed on + /// purpose and the server's answer to that is not the end. + Abandoned, +} + +impl Run<'_> { + /// Run a `COPY ... FROM STDIN` with `rows` as its data, through + /// tokio-postgres's copy-in on the session's own client. + pub(crate) async fn copy_in(&mut self, statement: &str, rows: &mut CopyInRows) -> Result { + if let Some(interrupt) = self.watch.due() { + self.watch.fire(interrupt, None); + } + if let Some(interrupt) = self.watch.fired() { + return Ok(Copied::ended(end_after_stop(interrupt, &DriverError::Cancelled))); + } + self.watch.begin_statement(self.core.statement_timeout()); + let begin = match copy_plan(self.core.tracker.state(), self.core.read_only) { + CopyFlight::Wrapped { read_only: true } => Some("START TRANSACTION READ ONLY"), + CopyFlight::Wrapped { read_only: false } => Some("BEGIN"), + CopyFlight::Plain => None, + }; + let wrapped = begin.is_some(); + if let Some(begin) = begin + && let Some(error) = self.control(begin).await? + { + return Ok(Copied::ended(self.failed_end(error))); + } + let outcome = match (self.copy_rows(statement, rows).await?, self.watch.fired()) { + (Flown::Copied(rows), None) => Ok(rows), + (Flown::Failed(error), _) => Err(self.failed_end(error)), + (Flown::Abandoned, None) => Err(RunEnd::Cancelled), + (Flown::Copied(_) | Flown::Abandoned, Some(interrupt)) => { + Err(end_after_stop(interrupt, &DriverError::Cancelled)) + } + }; + let outcome = match wrapped { + true => self.end_wrap(outcome).await?, + false => { + if outcome.is_err() && self.core.tracker.state() == KnownState::InTransaction { + self.core.tracker.set(KnownState::Failed); + } + outcome + } + }; + Ok(match outcome { + Ok(rows) => Copied { + end: RunEnd::Completed, + rows, + }, + Err(end) => Copied::ended(end), + }) + } + + /// Prepare the COPY, start it and send the rows as they come, then + /// finish it for the server's count. A stop or a source that gives up + /// drops the sink, which tokio-postgres turns into CopyFail. + async fn copy_rows(&mut self, statement: &str, rows: &mut CopyInRows) -> Result { + self.watch.settle().await; + let running: Running = (Arc::clone(&self.core.endpoint), self.core.cancel_token.clone()); + let client = &self.core.client; + let prepared = match interruptible(&mut self.watch, &running, client.prepare(statement)).await? { + Ok(prepared) => prepared, + Err(error) => return server_error(&error).map(Flown::Failed), + }; + if self.watch.fired().is_some() { + return Ok(Flown::Abandoned); + } + let started = interruptible(&mut self.watch, &running, client.copy_in::<_, Bytes>(&prepared)).await?; + let sink = match started { + Ok(sink) => sink, + Err(error) => return server_error(&error).map(Flown::Failed), + }; + let mut sink = std::pin::pin!(sink); + loop { + if self.watch.fired().is_some() { + return Ok(Flown::Abandoned); + } + let piece = tokio::select! { + biased; + interrupt = self.watch.triggered() => { + self.watch.fire(interrupt, Some(running.clone())); + return Ok(Flown::Abandoned); + } + piece = rows.recv() => piece, + }; + let bytes = match piece { + Some(CopyInPiece::Data(bytes)) => bytes, + Some(CopyInPiece::End) => break, + None => return Ok(Flown::Abandoned), + }; + let sent = tokio::select! { + biased; + interrupt = self.watch.triggered() => { + self.watch.fire(interrupt, Some(running.clone())); + return Ok(Flown::Abandoned); + } + sent = sink.send(bytes) => sent, + }; + if let Err(error) = sent { + return server_error(&error).map(Flown::Failed); + } + } + match interruptible(&mut self.watch, &running, sink.as_mut().finish()).await? { + Ok(rows) => Ok(Flown::Copied(rows)), + Err(error) => server_error(&error).map(Flown::Failed), + } + } + + /// End the driver's own transaction: COMMIT what copied, ROLLBACK the + /// rest. A failed COMMIT has rolled back, unless a cancel reached it + /// before it ran and left the block open, which the ROLLBACK ends. + async fn end_wrap(&mut self, outcome: Result) -> Result, Lost> { + let ending = match outcome { + Ok(_) => "COMMIT", + Err(_) => "ROLLBACK", + }; + let ended = self.control(ending).await?; + if ended.is_some() { + self.control("ROLLBACK").await?; + } + self.core.tracker.set(KnownState::Idle); + Ok(match (outcome, ended) { + (Ok(_), Some(error)) => Err(self.failed_end(error)), + (outcome, _) => outcome, + }) + } + + fn failed_end(&self, error: DriverError) -> RunEnd { + match self.watch.fired() { + Some(interrupt) => end_after_stop(interrupt, &error), + None => RunEnd::Failed(error), + } + } +} + +/// Where a cancel request for the running statement goes. +type Running = (Arc, CancelToken); + +/// Wait for `future`, sending the cancel request for it if the run is +/// stopped meanwhile. Lost once the grace after the stop runs out. +async fn interruptible( + watch: &mut StopWatch, + running: &Running, + future: impl Future, +) -> Result { + let mut future = std::pin::pin!(future); + loop { + tokio::select! { + biased; + done = &mut future => return Ok(done), + interrupt = watch.triggered(), if watch.fired().is_none() => { + watch.fire(interrupt, Some(running.clone())); + } + () = watch.gave_up(), if watch.fired().is_some() => return Err(Lost), + } + } +} + +/// A server error is the copy's own answer; any other error means the +/// connection failed. +fn server_error(error: &tokio_postgres::Error) -> Result { + match error.as_db_error() { + Some(_) => Ok(map_error(error)), + None => Err(Lost), + } +} + +/// A copy's notices and the driver's own statements' answers go nowhere. +pub(crate) struct NoRows; + +#[async_trait] +impl ResultSink for NoRows { + async fn accept(&mut self, _event: ResultEvent) -> SinkFlow { + SinkFlow::Continue + } +} diff --git a/core/crates/drivers/postgres/src/editor/editor_dispatch.rs b/core/crates/drivers/postgres/src/editor/editor_dispatch.rs index 47f87298af..5dfdb47ba3 100644 --- a/core/crates/drivers/postgres/src/editor/editor_dispatch.rs +++ b/core/crates/drivers/postgres/src/editor/editor_dispatch.rs @@ -71,6 +71,28 @@ pub(crate) fn plan(class: StatementClass, state: KnownState, read_only: bool) -> } } +/// How a `COPY ... FROM STDIN` with its rows goes to the server. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CopyFlight { + /// In a transaction of the driver's own, so the statement tokio-postgres + /// prepares under a name and the Bind that runs it reach the same + /// server connection behind a transaction pooler. A COPY on its own + /// is atomic already, so the wrap changes nothing else. A read-only + /// session starts it READ ONLY, and the server refuses the COPY as it + /// refuses any other write. + Wrapped { read_only: bool }, + /// Inside the session's transaction, which a read-only session started + /// READ ONLY, or its failed one, which refuses the COPY itself. + Plain, +} + +pub(crate) fn copy_plan(state: KnownState, read_only: bool) -> CopyFlight { + match state { + KnownState::Idle => CopyFlight::Wrapped { read_only }, + KnownState::InTransaction | KnownState::Failed => CopyFlight::Plain, + } +} + /// The plan for a row source whose DECLARE was refused, which then runs /// as a plain statement under the same read-only rule. pub(crate) fn fallback(owner: CursorOwner) -> Wrap { @@ -147,6 +169,22 @@ mod tests { } } + #[test] + fn a_copy_is_wrapped_when_idle_and_read_only_when_the_session_is() { + use KnownState::{Failed, Idle, InTransaction}; + + let cases = [ + (Idle, false, CopyFlight::Wrapped { read_only: false }), + (Idle, true, CopyFlight::Wrapped { read_only: true }), + (InTransaction, false, CopyFlight::Plain), + (InTransaction, true, CopyFlight::Plain), + (Failed, false, CopyFlight::Plain), + ]; + for (state, read_only, expected) in cases { + assert_eq!(copy_plan(state, read_only), expected, "{state:?} {read_only}"); + } + } + #[test] fn copy_from_stdin_is_refused_before_anything_is_sent() { assert!(matches!( diff --git a/core/crates/drivers/postgres/src/editor/editor_session.rs b/core/crates/drivers/postgres/src/editor/editor_session.rs index 2e1a692aa2..2779f84602 100644 --- a/core/crates/drivers/postgres/src/editor/editor_session.rs +++ b/core/crates/drivers/postgres/src/editor/editor_session.rs @@ -4,8 +4,8 @@ use std::time::Duration; use async_trait::async_trait; use tablepro_core::{ - CallOptions, DriverError, EditorSession, EditorSessionId, EditorSessionOptions, EditorSessionState, LossPhase, - ResultSink, RunOptions, RunSummary, TransactionState, UserSql, + CallOptions, CopyInRows, CopyInSummary, DriverError, EditorSession, EditorSessionId, EditorSessionOptions, + EditorSessionState, LossPhase, ResultSink, RunOptions, RunSummary, TransactionState, UserSql, }; use tokio::sync::{mpsc, oneshot}; use tokio::task::JoinHandle; @@ -80,6 +80,26 @@ impl PgEditorSession { close_timeout, })) } + + /// Hand the actor a command the session was claimed for, and wait for + /// its answer. + async fn ask( + &self, + command: Command, + answer: oneshot::Receiver>, + cancel: CancellationToken, + ) -> Result { + // Dropping the caller is itself a cancel: the statement is + // already running on the server and has to be stopped. + let guard = cancel.drop_guard(); + if self.commands.send(command).await.is_err() { + self.status.lose(); + return Err(lost()); + } + let outcome = answer.await.unwrap_or_else(|_| Err(lost())); + drop(guard.disarm()); + outcome + } } #[async_trait] @@ -103,9 +123,7 @@ impl EditorSession for PgEditorSession { sink: Box, ) -> Result { self.status.begin()?; - // Dropping the caller is itself a cancel: the statement is - // already running on the server and has to be stopped. - let guard = options.cancel.clone().drop_guard(); + let cancel = options.cancel.clone(); let (reply, answer) = oneshot::channel(); let command = Command::Run { sql, @@ -113,13 +131,25 @@ impl EditorSession for PgEditorSession { sink, reply, }; - if self.commands.send(command).await.is_err() { - self.status.lose(); - return Err(lost()); - } - let outcome = answer.await.unwrap_or_else(|_| Err(lost())); - drop(guard.disarm()); - outcome + self.ask(command, answer, cancel).await + } + + async fn copy_in( + &self, + statement: UserSql, + rows: CopyInRows, + options: RunOptions, + ) -> Result { + self.status.begin()?; + let cancel = options.cancel.clone(); + let (reply, answer) = oneshot::channel(); + let command = Command::CopyIn { + statement, + rows, + options, + reply, + }; + self.ask(command, answer, cancel).await } async fn transaction_state(&self, _options: CallOptions) -> Result { diff --git a/core/crates/drivers/postgres/src/editor/mod.rs b/core/crates/drivers/postgres/src/editor/mod.rs index aea718bba9..725e746d4c 100644 --- a/core/crates/drivers/postgres/src/editor/mod.rs +++ b/core/crates/drivers/postgres/src/editor/mod.rs @@ -2,6 +2,7 @@ // with the transaction status followed statement by statement. mod actor; +mod copy_flight; mod cursor_flight; mod editor_dispatch; mod editor_session; diff --git a/core/crates/drivers/postgres/src/editor/run.rs b/core/crates/drivers/postgres/src/editor/run.rs index 6d49dbdb8f..cdb71ad36e 100644 --- a/core/crates/drivers/postgres/src/editor/run.rs +++ b/core/crates/drivers/postgres/src/editor/run.rs @@ -38,9 +38,9 @@ enum Role { } pub(crate) struct Run<'a> { - core: &'a mut SessionCore, + pub(super) core: &'a mut SessionCore, sink: &'a mut dyn ResultSink, - watch: StopWatch, + pub(super) watch: StopWatch, limit: RowLimit, sink_gone: bool, notices_sent: u64, @@ -368,6 +368,11 @@ impl<'a> Run<'a> { Err(Lost) } + /// One of the driver's own statements, its notices dropped. + pub(super) async fn control(&mut self, sql: &str) -> Result, Lost> { + self.exchange(sql, Role::Control).await + } + /// Send one message and read everything it returns, keeping the first /// error. Its notices reach the sink unless it is the driver's own. async fn exchange(&mut self, sql: &str, role: Role) -> Result, Lost> { @@ -574,7 +579,7 @@ fn truncation(truncated: bool, cursor: bool) -> Truncation { /// How a run that was stopped ends. The server's 57014 is the /// acknowledgement; any other error was the statement's own. -fn end_after_stop(interrupt: Interrupt, error: &DriverError) -> RunEnd { +pub(super) fn end_after_stop(interrupt: Interrupt, error: &DriverError) -> RunEnd { match interrupt { Interrupt::Cancelled | Interrupt::ConsumerGone => RunEnd::Cancelled, Interrupt::DeadlineElapsed => RunEnd::TimedOut { diff --git a/core/crates/drivers/postgres/src/editor/session_core.rs b/core/crates/drivers/postgres/src/editor/session_core.rs index bfdb6e5d42..b78459f6eb 100644 --- a/core/crates/drivers/postgres/src/editor/session_core.rs +++ b/core/crates/drivers/postgres/src/editor/session_core.rs @@ -3,12 +3,13 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::task::Poll; use std::time::Duration; -use tablepro_core::{ResultSink, RunOptions, RunSummary, ServerCode, ServerNotice, UserSql}; +use tablepro_core::{CopyInRows, CopyInSummary, ResultSink, RunOptions, RunSummary, ServerCode, ServerNotice, UserSql}; use tokio::time::Instant; use tokio_postgres::error::DbError; use tokio_postgres::{AsyncMessage, CancelToken, Client}; use tokio_util::sync::CancellationToken; +use super::copy_flight::NoRows; use super::notice_queue::{NoticeQueue, NoticeSender}; use super::run::Run; use super::stop_watch::StopWatch; @@ -97,6 +98,31 @@ impl SessionCore { transaction_state: self.tracker.state().into(), }) } + + /// Run a `COPY ... FROM STDIN` with `rows` as its data. + pub(crate) async fn copy_in( + &mut self, + statement: &UserSql, + mut rows: CopyInRows, + options: RunOptions, + shutdown: &CancellationToken, + ) -> Result { + self.notices.start_run(); + let deadline = options.deadline.map(|deadline| Instant::now() + deadline); + let watch = StopWatch::new(options.cancel, shutdown.clone(), deadline, self.grace); + let mut sink = NoRows; + let mut run = Run::new(self, &mut sink, watch, options.row_limit); + let copied = run.copy_in(statement.as_str(), &mut rows).await?; + if run.stopped() { + run.probe().await?; + } + self.notices.discard(); + Ok(CopyInSummary { + end: copied.end, + rows: copied.rows, + transaction_state: self.tracker.state().into(), + }) + } } /// Drive the connection's I/O, which the actor polls itself so notices diff --git a/core/crates/drivers/postgres/tests/integration/copy_in.rs b/core/crates/drivers/postgres/tests/integration/copy_in.rs new file mode 100644 index 0000000000..47a200f199 --- /dev/null +++ b/core/crates/drivers/postgres/tests/integration/copy_in.rs @@ -0,0 +1,331 @@ +//! Copy-in on an editor session: the rows of a `COPY ... FROM STDIN` +//! sent as they come, in the session's transaction or in one of the +//! driver's own. + +use std::time::Duration; + +use bytes::Bytes; +use tablepro_core::{ + Connection, CopyInPiece, CopyInSummary, DriverError, EditorSession, EditorSessionState, ReadOnlyRefusal, RunEnd, + RunOptions, TransactionState, UserSql, +}; +use tokio::sync::mpsc; +use tokio_postgres::SimpleQueryMessage; +use tokio_postgres::config::SslMode; +use tokio_util::sync::CancellationToken; + +use crate::support::{CALL, connect, run_ok, scalar, session, setup, start, text, within}; + +const TABLE: &str = "CREATE TABLE t (a int, b text)"; + +/// The pieces already queued, then their end. +pub async fn queued(pieces: &[&[u8]]) -> mpsc::Receiver { + let (sender, rows) = mpsc::channel(pieces.len() + 1); + for piece in pieces { + sender + .send(CopyInPiece::Data(Bytes::copy_from_slice(piece))) + .await + .expect("queued"); + } + sender.send(CopyInPiece::End).await.expect("queued"); + rows +} + +async fn copy(session: &dyn EditorSession, statement: &str, pieces: &[&[u8]]) -> CopyInSummary { + session + .copy_in( + UserSql::new(statement), + queued(pieces).await, + RunOptions::new(CancellationToken::new()), + ) + .await + .expect("the copy") +} + +async fn value(connection: &dyn Connection, sql: &str) -> String { + text(&scalar(connection, sql).await) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn text_rows_cut_anywhere_load_and_commit() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &[TABLE]).await; + let session = session(connection.as_ref()).await; + + let summary = copy( + session.as_ref(), + "COPY t (a, b) FROM stdin", + &[b"1\tone\r\n2\tt".as_slice(), b"wo\r\n3\t\\N\r\n".as_slice()], + ) + .await; + + assert_eq!(summary.end, RunEnd::Completed); + assert_eq!(summary.rows, 3); + assert_eq!(summary.transaction_state, TransactionState::Idle); + assert_eq!(value(connection.as_ref(), "SELECT b FROM t WHERE a = 2").await, "two"); + assert_eq!( + value(connection.as_ref(), "SELECT count(*) FROM t WHERE b IS NULL").await, + "1" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn csv_with_a_header_and_a_quoted_line_break_loads() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &[TABLE]).await; + let session = session(connection.as_ref()).await; + + let summary = copy( + session.as_ref(), + "COPY t (a, b) FROM stdin WITH (FORMAT csv, HEADER true)", + &[b"a,b\n1,\"x\ny\"\n2,z\n"], + ) + .await; + + assert_eq!(summary.end, RunEnd::Completed); + assert_eq!(summary.rows, 2); + assert_eq!(value(connection.as_ref(), "SELECT b FROM t WHERE a = 1").await, "x\ny"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_bad_row_fails_the_copy_and_the_session_goes_on() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &[TABLE]).await; + let session = session(connection.as_ref()).await; + + let summary = copy( + session.as_ref(), + "COPY t (a, b) FROM stdin", + &[b"1\tok\nnot a number\tx\n"], + ) + .await; + + let RunEnd::Failed(DriverError::Server(diagnostics)) = &summary.end else { + panic!("a bad row ended {:?}", summary.end); + }; + assert_eq!(diagnostics.sqlstate(), Some("22P02")); + assert_eq!(summary.transaction_state, TransactionState::Idle); + assert_eq!(value(connection.as_ref(), "SELECT count(*) FROM t").await, "0"); + run_ok(session.as_ref(), "SELECT 1").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_copy_in_the_users_transaction_is_kept_by_its_commit() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &[TABLE]).await; + let session = session(connection.as_ref()).await; + + run_ok(session.as_ref(), "BEGIN").await; + let summary = copy(session.as_ref(), "COPY t (a, b) FROM stdin", &[b"1\ta\n2\tb\n"]).await; + let before_commit = value(connection.as_ref(), "SELECT count(*) FROM t").await; + run_ok(session.as_ref(), "COMMIT").await; + + assert_eq!(summary.end, RunEnd::Completed); + assert_eq!(summary.transaction_state, TransactionState::InTransaction); + assert_eq!(before_commit, "0", "the copy committed on its own"); + assert_eq!(value(connection.as_ref(), "SELECT count(*) FROM t").await, "2"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_copy_cancelled_while_it_waits_for_rows_keeps_nothing() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &[TABLE]).await; + let session = session(connection.as_ref()).await; + let (sender, rows) = mpsc::channel(4); + sender + .send(CopyInPiece::Data(Bytes::from_static(b"1\tone\n"))) + .await + .expect("queued"); + let cancel = CancellationToken::new(); + let stop = cancel.clone(); + + let copying = session.copy_in(UserSql::new("COPY t (a, b) FROM stdin"), rows, RunOptions::new(cancel)); + let stopping = async move { + tokio::time::sleep(Duration::from_millis(300)).await; + stop.cancel(); + }; + let (summary, ()) = tokio::join!(copying, stopping); + drop(sender); + + assert_eq!(summary.expect("the copy").end, RunEnd::Cancelled); + assert_eq!(value(connection.as_ref(), "SELECT count(*) FROM t").await, "0"); + assert_eq!( + session.transaction_state(within(CALL)).await, + Ok(TransactionState::Idle) + ); + run_ok(session.as_ref(), "SELECT 1").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_source_that_gives_up_abandons_the_copy() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &[TABLE]).await; + let session = session(connection.as_ref()).await; + let (sender, rows) = mpsc::channel(4); + sender + .send(CopyInPiece::Data(Bytes::from_static(b"1\tone\n"))) + .await + .expect("queued"); + drop(sender); + + let summary = session + .copy_in( + UserSql::new("COPY t (a, b) FROM stdin"), + rows, + RunOptions::new(CancellationToken::new()), + ) + .await + .expect("the copy"); + + assert_eq!(summary.end, RunEnd::Cancelled); + assert_eq!(value(connection.as_ref(), "SELECT count(*) FROM t").await, "0"); + run_ok(session.as_ref(), "SELECT 1").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_read_only_session_has_the_server_refuse_a_copy() { + let fixture = start().await; + let writer = connect(&fixture, false).await; + setup(writer.as_ref(), &[TABLE]).await; + let reader = connect(&fixture, true).await; + let session = session(reader.as_ref()).await; + + let summary = copy(session.as_ref(), "COPY t (a, b) FROM stdin", &[b"1\tone\n"]).await; + + let RunEnd::Failed(DriverError::ReadOnly(ReadOnlyRefusal::ServerRejected(diagnostics))) = &summary.end else { + panic!("a read-only copy ended {:?}", summary.end); + }; + assert_eq!(diagnostics.sqlstate(), Some("25006")); + assert_eq!(summary.transaction_state, TransactionState::Idle); + assert_eq!(value(writer.as_ref(), "SELECT count(*) FROM t").await, "0"); + run_ok(session.as_ref(), "SELECT 1").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_copy_refused_at_its_start_says_why_and_the_session_goes_on() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + let summary = copy(session.as_ref(), "COPY missing FROM stdin", &[b"1\n"]).await; + + let RunEnd::Failed(DriverError::Server(diagnostics)) = &summary.end else { + panic!("a missing table ended {:?}", summary.end); + }; + assert_eq!(diagnostics.sqlstate(), Some("42P01")); + assert_eq!(summary.transaction_state, TransactionState::Idle); + assert_eq!(session.state(), EditorSessionState::Ready); + let events = run_ok(session.as_ref(), "SELECT 41 + 1").await; + assert_eq!(text(&events.rows()[0][0]), "42"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_copy_refused_in_the_users_transaction_fails_it_until_rollback() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + let session = session(connection.as_ref()).await; + + run_ok(session.as_ref(), "BEGIN").await; + let summary = copy(session.as_ref(), "COPY missing FROM stdin", &[b"1\n"]).await; + run_ok(session.as_ref(), "ROLLBACK").await; + + assert!( + matches!(summary.end, RunEnd::Failed(DriverError::Server(_))), + "{:?}", + summary.end + ); + assert_eq!(summary.transaction_state, TransactionState::Failed); + assert_eq!( + session.transaction_state(within(CALL)).await, + Ok(TransactionState::Idle) + ); + run_ok(session.as_ref(), "SELECT 1").await; +} + +/// The COPY waits on the lock at its Execute, so the cancel request +/// refuses it before copy-in mode. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_copy_stopped_while_it_waits_on_a_lock_is_cancelled_and_the_session_goes_on() { + let fixture = start().await; + let connection = connect(&fixture, false).await; + setup(connection.as_ref(), &[TABLE]).await; + let locker = session(connection.as_ref()).await; + run_ok(locker.as_ref(), "BEGIN").await; + run_ok(locker.as_ref(), "LOCK TABLE t").await; + let session = session(connection.as_ref()).await; + let cancel = CancellationToken::new(); + let stop = cancel.clone(); + + let copying = session.copy_in( + UserSql::new("COPY t (a, b) FROM stdin"), + queued(&[b"1\tone\n"]).await, + RunOptions::new(cancel), + ); + let stopping = async move { + tokio::time::sleep(Duration::from_millis(300)).await; + stop.cancel(); + }; + let (summary, ()) = tokio::join!(copying, stopping); + run_ok(locker.as_ref(), "COMMIT").await; + + assert_eq!(summary.expect("the copy").end, RunEnd::Cancelled); + assert_eq!(session.state(), EditorSessionState::Ready); + assert_eq!( + session.transaction_state(within(CALL)).await, + Ok(TransactionState::Idle) + ); + assert_eq!(value(connection.as_ref(), "SELECT count(*) FROM t").await, "0"); + run_ok(session.as_ref(), "SELECT 1").await; +} + +/// tokio-postgres 0.7.18 sent a Sync with the COPY's Execute and another +/// with its CopyFail, so a COPY refused before copy-in mode got two +/// ReadyForQuery and the next request read the second. The driver relies +/// on the fork's fix, rust-postgres/rust-postgres#1402. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn tokio_postgres_answers_the_request_after_a_copy_refused_at_its_start() { + let fixture = start().await; + let credentials = fixture.password_credentials(); + let mut config = tokio_postgres::Config::new(); + config + .host(fixture.host()) + .port(fixture.port()) + .user(&credentials.username) + .password(&credentials.password) + .dbname(fixture.database()) + .ssl_mode(SslMode::Disable); + let (client, connection) = config.connect(tokio_postgres::NoTls).await.expect("a client"); + let driving = tokio::spawn(connection); + + let refused = client.copy_in::<_, Bytes>("COPY missing FROM STDIN").await; + let next = tokio::time::timeout(CALL, client.simple_query("SELECT 1")) + .await + .expect("an answer in time") + .expect("the request's own answer"); + drop(client); + let _ = driving.await; + + assert!(refused.is_err(), "a COPY into a missing table started"); + let row = next.iter().find_map(|message| match message { + SimpleQueryMessage::Row(row) => Some(row.get(0)), + _ => None, + }); + assert_eq!(row, Some(Some("1"))); +} diff --git a/core/crates/drivers/postgres/tests/integration/main.rs b/core/crates/drivers/postgres/tests/integration/main.rs index 57f303971a..6926bba934 100644 --- a/core/crates/drivers/postgres/tests/integration/main.rs +++ b/core/crates/drivers/postgres/tests/integration/main.rs @@ -9,6 +9,7 @@ mod cancellation; mod catalog_ddl; mod conformance_engine; +mod copy_in; mod csv_import; mod ddl; mod decode; diff --git a/core/crates/drivers/postgres/tests/integration/pgbouncer.rs b/core/crates/drivers/postgres/tests/integration/pgbouncer.rs index a174fb61c0..54c359cf22 100644 --- a/core/crates/drivers/postgres/tests/integration/pgbouncer.rs +++ b/core/crates/drivers/postgres/tests/integration/pgbouncer.rs @@ -8,13 +8,14 @@ use std::time::Duration; use tablepro_core::meta::ObjectScope; use tablepro_core::{ - Connection, DriverError, ReadOnlyRefusal, RowLimit, RunEnd, RunOptions, TlsConfig, Value, WorkspaceSettings, - WriteKind, + Connection, DriverError, ReadOnlyRefusal, RowLimit, RunEnd, RunOptions, TlsConfig, UserSql, Value, + WorkspaceSettings, WriteKind, }; use tablepro_test_fixtures::{PgBouncerFixture, PoolMode}; use tablepro_test_support::conformance::test_liveness; use tokio_util::sync::CancellationToken; +use crate::copy_in::queued; use crate::support::{ CALL, apply, connect_with, direct, options_over, password, rows, run, run_ok, run_with, scalar, session, setup, statement, text, within, @@ -165,6 +166,56 @@ async fn enum_citext_10000_row_flight_never_prepares() { } } +/// The copy prepares its statement under a name, which only holds +/// because the driver's own transaction keeps one server connection from +/// the Parse to the end of the rows. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn transaction_pool_copy_in_loads_its_rows() { + let bouncer = PgBouncerFixture::start(PoolMode::Transaction).await.expect("PgBouncer"); + let connection = through(&bouncer, false).await; + setup(connection.as_ref(), &["CREATE TABLE t (id int, label text)"]).await; + let session = session(connection.as_ref()).await; + + let summary = session + .copy_in( + UserSql::new("COPY t (id, label) FROM stdin"), + queued(&[b"1\ta\n2\tb\n"]).await, + RunOptions::new(CancellationToken::new()), + ) + .await + .expect("the copy"); + + assert_eq!(summary.end, RunEnd::Completed); + assert_eq!(summary.rows, 2); + assert_eq!(text(&scalar(connection.as_ref(), "SELECT count(*) FROM t").await), "2"); + run_ok(session.as_ref(), "SELECT 1").await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn transaction_pool_copy_refused_at_its_start_keeps_the_session() { + let bouncer = PgBouncerFixture::start(PoolMode::Transaction).await.expect("PgBouncer"); + let connection = through(&bouncer, false).await; + let session = session(connection.as_ref()).await; + + let summary = session + .copy_in( + UserSql::new("COPY missing FROM stdin"), + queued(&[b"1\n"]).await, + RunOptions::new(CancellationToken::new()), + ) + .await + .expect("the copy"); + + let RunEnd::Failed(DriverError::Server(diagnostics)) = &summary.end else { + panic!("a missing table ended {:?}", summary.end); + }; + assert_eq!(diagnostics.sqlstate(), Some("42P01")); + let events = run_ok(session.as_ref(), "SELECT 41 + 1").await; + assert_eq!(text(&events.rows()[0][0]), "42"); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "requires docker"] async fn session_pool_editor_state_persists() { diff --git a/core/crates/session/src/session/byte_limited_editor.rs b/core/crates/session/src/session/byte_limited_editor.rs index a12f8d765e..2412cae06c 100644 --- a/core/crates/session/src/session/byte_limited_editor.rs +++ b/core/crates/session/src/session/byte_limited_editor.rs @@ -2,8 +2,8 @@ use std::sync::Arc; use async_trait::async_trait; use tablepro_core::{ - ByteLimitSink, CallOptions, DriverError, EditorSession, EditorSessionId, EditorSessionState, ResultSink, - RunOptions, RunSummary, TransactionState, UserSql, + ByteLimitSink, CallOptions, CopyInRows, CopyInSummary, DriverError, EditorSession, EditorSessionId, + EditorSessionState, ResultSink, RunOptions, RunSummary, TransactionState, UserSql, }; // Every app opens its editor sessions here, so none can run one without the run's byte limit. @@ -41,6 +41,15 @@ impl EditorSession for ByteLimitedEditor { self.inner.transaction_state(options).await } + async fn copy_in( + &self, + statement: UserSql, + rows: CopyInRows, + options: RunOptions, + ) -> Result { + self.inner.copy_in(statement, rows, options).await + } + fn database(&self) -> Option { self.inner.database() } diff --git a/core/crates/test-support/src/fake_copy_in.rs b/core/crates/test-support/src/fake_copy_in.rs new file mode 100644 index 0000000000..23ee222410 --- /dev/null +++ b/core/crates/test-support/src/fake_copy_in.rs @@ -0,0 +1,13 @@ +use tablepro_core::UserSql; + +/// One `COPY ... FROM STDIN` a `FakeEditorSession` was given. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FakeCopyIn { + pub statement: UserSql, + /// Every piece of the rows joined, as far as they came. + pub data: Vec, + pub pieces: usize, + /// The source said the rows ended, rather than closing or the copy + /// being cancelled. + pub ended: bool, +} diff --git a/core/crates/test-support/src/fake_editor_session.rs b/core/crates/test-support/src/fake_editor_session.rs index 9f0235f719..c35a3fa846 100644 --- a/core/crates/test-support/src/fake_editor_session.rs +++ b/core/crates/test-support/src/fake_editor_session.rs @@ -3,14 +3,15 @@ use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use async_trait::async_trait; use tablepro_core::{ - CallOptions, DriverError, EditorSession, EditorSessionId, EditorSessionState, ErrorCategory, LossPhase, ResultSink, - RunEnd, RunOptions, RunSummary, SinkFlow, TimeoutPhase, TransactionState, UserSql, + CallOptions, CopyInPiece, CopyInRows, CopyInSummary, DriverError, EditorSession, EditorSessionId, + EditorSessionState, ErrorCategory, LossPhase, ResultEvent, ResultSink, RunEnd, RunOptions, RunSummary, SinkFlow, + TimeoutPhase, TransactionState, UserSql, }; use tokio::time::Instant; use tokio_util::sync::CancellationToken; use crate::fake_reply::Stop; -use crate::{CloseBehaviour, FakeRun, Replies, RunWait, lock}; +use crate::{CloseBehaviour, FakeCopyIn, FakeRun, Replies, RunWait, lock}; /// An `EditorSession` that answers each run from a script and records /// what it was asked to run. @@ -18,11 +19,18 @@ use crate::{CloseBehaviour, FakeRun, Replies, RunWait, lock}; /// It keeps the session rules the editor relies on: one run at a time, /// a lost session stays lost, and the transaction state is whatever the /// last run left. +/// +/// A copy answers from the same script, by its statement. One scripted +/// to end other than `Completed` refuses the COPY before it takes any +/// rows, as a server refuses a missing table, and lets go of them once +/// its scripted wait is over; the rest take every row up to the end and +/// then answer. #[derive(Debug)] pub struct FakeEditorSession { id: EditorSessionId, runs: Mutex>, texts: Mutex>, + copies: Mutex>, state: Mutex, transaction_state: Mutex, database: Mutex>, @@ -45,6 +53,7 @@ impl FakeEditorSession { id: EditorSessionId::new(), runs: Mutex::new(runs), texts: Mutex::new(Vec::new()), + copies: Mutex::new(Vec::new()), state: Mutex::new(EditorSessionState::Ready), transaction_state: Mutex::new(TransactionState::Idle), database: Mutex::new(None), @@ -58,11 +67,17 @@ impl FakeEditorSession { } } - /// The text of every run, in the order the runs started. + /// The text of every run and every copy's statement, in the order + /// they started. pub fn texts(&self) -> Vec { lock(&self.texts).clone() } + /// Every copy, with the rows it took so far. + pub fn copies(&self) -> Vec { + lock(&self.copies).clone() + } + pub fn close_count(&self) -> usize { self.closes.load(Ordering::SeqCst) } @@ -142,6 +157,66 @@ impl FakeEditorSession { *lock(&self.transaction_state) = run.transaction_state; run.end } + + /// What the end of a run or a copy leaves the session in. + fn settle(&self, loses_session: bool, end: &Result) { + if loses_session { + *lock(&self.state) = EditorSessionState::Lost; + } + match end { + Ok(RunEnd::Cancelled | RunEnd::TimedOut { .. }) => { + self.stopped_runs.fetch_add(1, Ordering::SeqCst); + } + Err(error) if error.category() == ErrorCategory::ConnectionLost => { + *lock(&self.state) = EditorSessionState::Lost; + } + _ => {} + } + } + + /// Take the rows of copy `index` up to their end. False when the copy + /// was cancelled or its source gave up first. + async fn take_rows(&self, index: usize, rows: &mut CopyInRows, options: &RunOptions) -> bool { + loop { + let piece = tokio::select! { + biased; + () = options.cancel.cancelled() => return false, + () = self.closing.cancelled() => return false, + piece = rows.recv() => piece, + }; + let mut copies = lock(&self.copies); + let Some(copy) = copies.get_mut(index) else { + return false; + }; + match piece { + Some(CopyInPiece::Data(bytes)) => { + copy.data.extend_from_slice(&bytes); + copy.pieces += 1; + } + Some(CopyInPiece::End) => { + copy.ended = true; + return true; + } + None => return false, + } + } + } +} + +/// Lines of COPY data, a last one without a line break included. +fn rows_in(data: &[u8]) -> u64 { + let breaks = data.iter().filter(|byte| **byte == b'\n').count() as u64; + breaks + u64::from(data.last().is_some_and(|byte| *byte != b'\n')) +} + +/// Where the events of a copy's scripted run go. +struct Discard; + +#[async_trait] +impl ResultSink for Discard { + async fn accept(&mut self, _event: ResultEvent) -> SinkFlow { + SinkFlow::Continue + } } /// Puts the session back to Ready however the run ends, including when @@ -186,18 +261,7 @@ impl EditorSession for FakeEditorSession { }; let loses_session = run.loses_session; let end = self.play(run, &options, sink.as_mut()).await; - if loses_session { - *lock(&self.state) = EditorSessionState::Lost; - } - match &end { - Ok(RunEnd::Cancelled | RunEnd::TimedOut { .. }) => { - self.stopped_runs.fetch_add(1, Ordering::SeqCst); - } - Err(error) if error.category() == ErrorCategory::ConnectionLost => { - *lock(&self.state) = EditorSessionState::Lost; - } - _ => {} - } + self.settle(loses_session, &end); Ok(RunSummary { elapsed: started.elapsed(), end: end?, @@ -205,6 +269,55 @@ impl EditorSession for FakeEditorSession { }) } + async fn copy_in( + &self, + statement: UserSql, + mut rows: CopyInRows, + options: RunOptions, + ) -> Result { + let _running = self.begin()?; + lock(&self.texts).push(statement.clone()); + let index = { + let mut copies = lock(&self.copies); + copies.push(FakeCopyIn { + statement: statement.clone(), + data: Vec::new(), + pieces: 0, + ended: false, + }); + copies.len() - 1 + }; + let run = lock(&self.runs).next(Some(statement.as_str())); + let Some(run) = run else { + return Err(DriverError::Protocol( + "FakeEditorSession has no scripted run".to_owned(), + )); + }; + let refused = !matches!(run.end, Ok(RunEnd::Completed)); + if !refused && !self.take_rows(index, &mut rows, &options).await { + self.stopped_runs.fetch_add(1, Ordering::SeqCst); + return Ok(CopyInSummary { + end: RunEnd::Cancelled, + rows: 0, + transaction_state: *lock(&self.transaction_state), + }); + } + let loses_session = run.loses_session; + let end = self.play(run, &options, &mut Discard).await; + drop(rows); + self.settle(loses_session, &end); + let end = end?; + let copied = match end { + RunEnd::Completed => lock(&self.copies).get(index).map_or(0, |copy| rows_in(©.data)), + _ => 0, + }; + Ok(CopyInSummary { + end, + rows: copied, + transaction_state: *lock(&self.transaction_state), + }) + } + async fn transaction_state(&self, _options: CallOptions) -> Result { Ok(*lock(&self.transaction_state)) } @@ -238,24 +351,15 @@ mod tests { use std::sync::Arc; use std::time::Duration; - use async_trait::async_trait; use futures::future::BoxFuture; + use tablepro_core::RowLimit; use tablepro_core::value::Value; - use tablepro_core::{ResultEvent, RowLimit}; + use tokio::sync::mpsc; + use tokio_util::bytes::Bytes; use super::*; use crate::close_race::{ROUNDS, pending_closes}; - #[derive(Default)] - struct Discard; - - #[async_trait] - impl ResultSink for Discard { - async fn accept(&mut self, _event: ResultEvent) -> SinkFlow { - SinkFlow::Continue - } - } - fn run_options() -> RunOptions { RunOptions::new(CancellationToken::new()) } @@ -289,6 +393,65 @@ mod tests { assert_eq!(session.state(), EditorSessionState::Ready); } + #[tokio::test] + async fn a_copy_takes_its_rows_and_a_refused_one_takes_none() { + let mut runs = Replies::default(); + runs.when_sql_contains( + "missing", + FakeRun::failed(DriverError::Protocol("no such table".to_owned())), + ) + .always(FakeRun::completed(Vec::new())); + let session = FakeEditorSession::new(runs); + let (sender, rows) = mpsc::channel(4); + sender + .send(CopyInPiece::Data(Bytes::from_static(b"1\tone\n2"))) + .await + .unwrap(); + sender.send(CopyInPiece::End).await.unwrap(); + let (_unread, refused_rows) = mpsc::channel(1); + + let copied = session + .copy_in(UserSql::new("COPY t FROM stdin"), rows, run_options()) + .await + .unwrap(); + let refused = session + .copy_in(UserSql::new("COPY missing FROM stdin"), refused_rows, run_options()) + .await + .unwrap(); + + assert_eq!(copied.end, RunEnd::Completed); + assert_eq!(copied.rows, 2); + assert!(matches!(refused.end, RunEnd::Failed(_)), "{refused:?}"); + let copies = session.copies(); + assert_eq!(copies[0].data, b"1\tone\n2"); + assert!(copies[0].ended); + assert!(!copies[1].ended); + assert_eq!(session.state(), EditorSessionState::Ready); + } + + #[tokio::test] + async fn a_copy_whose_source_closes_before_the_end_is_cancelled() { + let mut runs = Replies::default(); + runs.always(FakeRun::completed(Vec::new())); + let session = FakeEditorSession::new(runs); + let (sender, rows) = mpsc::channel(4); + sender + .send(CopyInPiece::Data(Bytes::from_static(b"1\n"))) + .await + .unwrap(); + drop(sender); + + let copied = session + .copy_in(UserSql::new("COPY t FROM stdin"), rows, run_options()) + .await + .unwrap(); + + assert_eq!(copied.end, RunEnd::Cancelled); + assert_eq!(session.copies()[0].data, b"1\n"); + assert!(!session.copies()[0].ended); + assert_eq!(session.stopped_runs(), 1); + } + #[tokio::test(start_paused = true)] async fn a_second_run_while_one_is_running_is_busy_and_cancel_is_observed() { let mut runs = Replies::default(); diff --git a/core/crates/test-support/src/lib.rs b/core/crates/test-support/src/lib.rs index bf87b7d1e6..066d78d4a7 100644 --- a/core/crates/test-support/src/lib.rs +++ b/core/crates/test-support/src/lib.rs @@ -16,6 +16,7 @@ mod connection_script; mod csv_import; mod executable; mod fake_connection; +mod fake_copy_in; mod fake_dialect; mod fake_driver; mod fake_editor_session; @@ -28,6 +29,8 @@ pub mod history_cases; mod replies; mod round_trip_counter; mod snippet_text; +#[cfg(test)] +mod sql_import_copy_in; mod stream_end; pub use call_counters::CallCounters; @@ -37,6 +40,7 @@ pub use connection_script::ConnectionScript; pub use csv_import::{CsvImportFailure, import_csv}; pub use executable::write_executable; pub use fake_connection::FakeConnection; +pub use fake_copy_in::FakeCopyIn; pub use fake_dialect::FakeDialect; pub use fake_driver::FakeDriver; pub use fake_editor_session::FakeEditorSession; diff --git a/core/crates/test-support/src/sql_import_copy_in.rs b/core/crates/test-support/src/sql_import_copy_in.rs new file mode 100644 index 0000000000..1f6783eec6 --- /dev/null +++ b/core/crates/test-support/src/sql_import_copy_in.rs @@ -0,0 +1,255 @@ +use std::io::{Cursor, Read}; +use std::sync::Arc; +use std::time::Duration; + +use tablepro_core::import::{ + SqlImportEnd, SqlImportErrorPolicy, SqlImportFault, SqlImportOptions, SqlImportReport, run_sql_import, +}; +use tablepro_core::{DriverError, UserSql}; +use tokio_util::sync::CancellationToken; + +use crate::{FakeDialect, FakeEditorSession, FakeRun, Replies}; + +fn options(on_error: SqlImportErrorPolicy) -> SqlImportOptions { + SqlImportOptions { + on_error, + one_transaction: false, + } +} + +async fn import(session: &FakeEditorSession, file: &str, on_error: SqlImportErrorPolicy) -> SqlImportReport { + run_sql_import( + session, + &FakeDialect, + Box::new(Cursor::new(file.as_bytes().to_vec())), + options(on_error), + &mut |_| {}, + &CancellationToken::new(), + ) + .await +} + +fn texts(session: &FakeEditorSession) -> Vec { + session.texts().iter().map(|text| text.as_str().to_owned()).collect() +} + +fn refusing_copies() -> FakeEditorSession { + let mut runs = Replies::default(); + runs.when_sql_contains( + "COPY", + FakeRun::failed(DriverError::Protocol("relation \"t\" does not exist".to_owned())), + ) + .always(FakeRun::completed(Vec::new())); + FakeEditorSession::new(runs) +} + +#[tokio::test] +async fn copy_rows_reach_the_session_byte_exact_as_one_statement() { + let mut runs = Replies::default(); + runs.always(FakeRun::completed(Vec::new())); + let session = FakeEditorSession::new(runs); + + let report = import( + &session, + "CREATE TABLE t (a int, b text);\nCOPY t (a, b) FROM stdin;\n1\tone\r\n2\t\\N\n\\.\r\nSELECT 1;\n", + SqlImportErrorPolicy::Stop, + ) + .await; + + assert_eq!(report.end, SqlImportEnd::Completed, "{report:?}"); + assert_eq!((report.statements, report.failed), (3, 0)); + let copies = session.copies(); + assert_eq!(copies.len(), 1); + assert_eq!(copies[0].statement, UserSql::new("COPY t (a, b) FROM stdin")); + assert_eq!(copies[0].data, b"1\tone\r\n2\t\\N\n"); + assert!(copies[0].ended); + assert_eq!( + texts(&session)[1..], + ["CREATE TABLE t (a int, b text)", "COPY t (a, b) FROM stdin", "SELECT 1"] + ); +} + +#[tokio::test] +async fn a_refused_copy_under_continue_passes_over_its_rows_and_goes_on() { + let session = refusing_copies(); + + let report = import( + &session, + "COPY t FROM stdin;\n1\n2\n\\.\nSELECT 2;\n", + SqlImportErrorPolicy::Continue, + ) + .await; + + assert_eq!(report.end, SqlImportEnd::Completed, "{report:?}"); + assert_eq!((report.statements, report.failed), (1, 1)); + assert_eq!(report.failures[0].line, 1); + assert_eq!(report.failures[0].statement, "COPY t FROM stdin"); + assert!(matches!(report.failures[0].fault, SqlImportFault::Statement(_))); + assert_eq!(texts(&session).last().map(String::as_str), Some("SELECT 2")); +} + +/// The session holds on to the rows until the server refuses, so two +/// pieces fill the queue and the refusal lands on their end. +#[tokio::test] +async fn a_copy_refused_after_its_end_was_queued_keeps_the_statement_after_it() { + let mut runs = Replies::default(); + runs.when_sql_contains( + "COPY", + FakeRun::failed(DriverError::Protocol("relation \"t\" does not exist".to_owned())) + .taking(Duration::from_millis(200)), + ) + .always(FakeRun::completed(Vec::new())); + let session = FakeEditorSession::new(runs); + let rows: String = (0..60_000).map(|n| format!("{n}\n")).collect(); + assert!( + (256 * 1024..512 * 1024).contains(&rows.len()), + "{} bytes is not two pieces", + rows.len() + ); + + let report = import( + &session, + &format!("COPY t FROM stdin;\n{rows}\\.\nSELECT 2;\n"), + SqlImportErrorPolicy::Continue, + ) + .await; + + assert_eq!(report.end, SqlImportEnd::Completed, "{report:?}"); + assert_eq!((report.statements, report.failed), (1, 1)); + assert_eq!(texts(&session).last().map(String::as_str), Some("SELECT 2")); +} + +/// More pieces than the queue holds, so the refusal leaves rows in the +/// file for the import to pass over. +#[tokio::test] +async fn a_copy_refused_with_rows_left_passes_over_them_to_the_statement_after() { + let mut runs = Replies::default(); + runs.when_sql_contains( + "COPY", + FakeRun::failed(DriverError::Protocol("relation \"t\" does not exist".to_owned())) + .taking(Duration::from_millis(200)), + ) + .always(FakeRun::completed(Vec::new())); + let session = FakeEditorSession::new(runs); + let rows: String = (0..130_000).map(|n| format!("{n}\n")).collect(); + assert!( + rows.len() > 3 * 256 * 1024, + "{} bytes is not over three pieces", + rows.len() + ); + + let report = import( + &session, + &format!("COPY t FROM stdin;\n{rows}\\.\nSELECT 2;\n"), + SqlImportErrorPolicy::Continue, + ) + .await; + + assert_eq!(report.end, SqlImportEnd::Completed, "{report:?}"); + assert_eq!((report.statements, report.failed), (1, 1)); + let texts = texts(&session); + assert_eq!(texts.last().map(String::as_str), Some("SELECT 2")); + assert_eq!(texts.iter().filter(|text| text.as_str() == "SELECT 2").count(), 1); + assert!(!texts.iter().any(|text| text.contains("129999")), "a row ran as SQL"); +} + +#[tokio::test] +async fn a_refused_copy_under_stop_ends_the_import_at_it() { + let session = refusing_copies(); + + let report = import( + &session, + "COPY t FROM stdin;\n1\n2\n\\.\nSELECT 2;\n", + SqlImportErrorPolicy::Stop, + ) + .await; + + assert_eq!(report.end, SqlImportEnd::Stopped); + assert_eq!((report.statements, report.failed), (0, 1)); + assert!(!texts(&session).contains(&"SELECT 2".to_owned())); +} + +#[tokio::test] +async fn a_binary_copy_is_refused_and_the_statements_after_its_rows_run() { + let mut runs = Replies::default(); + runs.always(FakeRun::completed(Vec::new())); + let session = FakeEditorSession::new(runs); + + let report = import( + &session, + "COPY t FROM stdin (FORMAT binary);\nPGCOPY\n\\.\nSELECT 3;\n", + SqlImportErrorPolicy::Continue, + ) + .await; + + assert_eq!((report.statements, report.failed), (1, 1)); + assert!(session.copies().is_empty()); + assert_eq!(texts(&session).last().map(String::as_str), Some("SELECT 3")); +} + +/// A file whose reads stop after `head` until `gate` opens. +struct GatedFile { + head: Cursor>, + gate: std::sync::mpsc::Receiver<()>, + tail: Cursor>, +} + +impl Read for GatedFile { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let read = self.head.read(buffer)?; + if read > 0 { + return Ok(read); + } + // Ends when the sender sends or goes away. + let _ = self.gate.recv(); + self.tail.read(buffer) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancelling_in_the_middle_of_a_copy_abandons_it() { + let mut runs = Replies::default(); + runs.always(FakeRun::completed(Vec::new())); + let session = Arc::new(FakeEditorSession::new(runs)); + let rows: String = (0..30_000).map(|n| format!("{n}\trow {n}\n")).collect(); + let (open, gate) = std::sync::mpsc::channel(); + let file = GatedFile { + head: Cursor::new(format!("COPY t (a, b) FROM stdin;\n{rows}").into_bytes()), + gate, + tail: Cursor::new(b"\\.\nSELECT 1;\n".to_vec()), + }; + let cancel = CancellationToken::new(); + let importing = { + let session = Arc::clone(&session); + let cancel = cancel.clone(); + tokio::spawn(async move { + let dialect = FakeDialect; + run_sql_import( + session.as_ref(), + &dialect, + Box::new(file), + options(SqlImportErrorPolicy::Stop), + &mut |_| {}, + &cancel, + ) + .await + }) + }; + let mut waited = 0; + while session.copies().first().is_none_or(|copy| copy.pieces == 0) { + assert!(waited < 1_000, "no rows reached the session"); + tokio::time::sleep(Duration::from_millis(10)).await; + waited += 1; + } + + cancel.cancel(); + drop(open); + let report = importing.await.expect("the import"); + + assert_eq!(report.end, SqlImportEnd::Cancelled, "{report:?}"); + assert_eq!(report.statements, 0); + let copies = session.copies(); + assert!(!copies[0].ended, "a cancelled copy was finished"); + assert!(copies[0].data.len() < rows.len()); + assert!(!texts(&session).contains(&"SELECT 1".to_owned())); +} diff --git a/core/docs/decisions/0011-sql-dump-and-import-in-process.md b/core/docs/decisions/0011-sql-dump-and-import-in-process.md index 02380bb415..7da87480a1 100644 --- a/core/docs/decisions/0011-sql-dump-and-import-in-process.md +++ b/core/docs/decisions/0011-sql-dump-and-import-in-process.md @@ -1,6 +1,6 @@ # 0011: SQL dumps and SQL file imports run in process -- **Status**: Accepted +- **Status**: Accepted; the import of COPY rows as INSERTs superseded by [0024](0024-copy-rows-through-copy-in.md) - **Date**: 2026-11-05 ## Context diff --git a/core/docs/decisions/0024-copy-rows-through-copy-in.md b/core/docs/decisions/0024-copy-rows-through-copy-in.md new file mode 100644 index 0000000000..ce09b06cbe --- /dev/null +++ b/core/docs/decisions/0024-copy-rows-through-copy-in.md @@ -0,0 +1,37 @@ +# 0024: An imported COPY block loads through PostgreSQL's copy-in + +- **Status**: Accepted +- **Date**: 2026-11-13 + +## Context + +[0011](0011-sql-dump-and-import-in-process.md) has Import SQL File turn the rows of pg_dump's `COPY ... FROM stdin` blocks into INSERTs, and refuse a COPY with any option. A file in CSV, or with `NULL`, `DELIMITER` or `HEADER`, does not load, and a large block loads as many INSERTs instead of one COPY. The macOS app now loads such blocks through libpq's copy-in. + +## Decision + +On PostgreSQL the `ScriptReader` gives a COPY from stdin as a unit of its own, then its rows as the file holds them in pieces of at most 256 KiB, then their end; `run_sql_import` sends them with `EditorSession::copy_in`, which the PostgreSQL driver runs with tokio-postgres's copy-in on the session's own connection. + +## Rationale + +- It is what psql does. The rows reach the server unchanged, so text and CSV with any option load as the server reads them, and the server names the line of a bad row, which the import shows with the failure. +- The file is still never held whole: the pieces go through the import's bounded queue, and the import's progress and Stop work during the rows. +- One COPY loads a block much faster than INSERTs of 1,000 rows each. + +## Consequences + +- The rows end at a line that is exactly `\.`, with LF or CRLF, or at the end of the file. The `\.` is never sent, as psql 18 does, since PostgreSQL 18 reads it as data in CSV. What follows the COPY on its own line runs once the rows end, as in psql. A `\.` line inside a CSV quoted field ends the rows too, as in psql. +- The binary format is refused and its rows passed over, since a script cannot hold it. Binary rows are almost never UTF-8, so such a block usually stops the import at its first row, as the next point says. +- The reader takes the whole file as UTF-8 before it splits it, so a row that is not UTF-8 stops the import at its line, and the rows reach the session's UTF8 `client_encoding` as written. +- A COPY counts as one statement. One the server refuses loses its whole block, as in psql, and the import's error policy decides whether the rest runs. Stop, or a read of the file that fails, fails the copy with CopyFail, and the import reports the stop or the read, not the server's answer to the CopyFail. +- A COPY runs in the import's transaction when there is one, and otherwise in one of the driver's own, so the statement tokio-postgres prepares under a name and the Bind after it reach one server connection behind a transaction pooler. A COPY on its own is atomic, so the wrap changes nothing else. +- A read-only session starts that transaction READ ONLY, and the server refuses the COPY as it refuses any other write. +- tokio-postgres 0.7.18 sends a Sync with the COPY's Execute and another with its CopyDone or CopyFail. A COPY the server refuses before copy-in mode (a missing table or column, a permission error, a Stop while it waits on a lock) then gets two ReadyForQuery, and the next request reads the second. The workspace takes tokio-postgres, postgres-types and postgres-protocol from `TableProApp/rust-postgres` (root `Cargo.toml`, `[patch.crates-io]`): the 0.7.18 tag plus [rust-postgres#1402](https://github.com/rust-postgres/rust-postgres/pull/1402), which sends Flush in place of the first Sync. Such a COPY fails like any statement, and the session goes on. The pin goes once a release has the fix; the integration test `tokio_postgres_answers_the_request_after_a_copy_refused_at_its_start` fails if it goes earlier. +- The query editor still refuses `COPY ... FROM STDIN`, and Export Database still writes INSERTs, which every client loads. +- A session without copy-in fails the COPY with `UnsupportedFeature::CopyFromStdin`. + +## Alternatives considered + +- **Keep the INSERTs and read CSV and the options too.** A second reader of every format and option the server already reads, and still slower. +- **Read a block into memory and send it in one call.** A block can be larger than memory. +- **Give the session up after a COPY refused at its start.** A missing table, a permission error or a Stop while the COPY waits on a lock would each lose the session, and under Continue end the import. +- **Skip the CopyFail when no CopyInResponse came, in tokio-postgres.** A Stop drops the copy while it waits on a lock; once the lock frees, the server enters copy-in mode and waits for rows that never come. diff --git a/core/docs/decisions/README.md b/core/docs/decisions/README.md index 153e454636..331672cfe2 100644 --- a/core/docs/decisions/README.md +++ b/core/docs/decisions/README.md @@ -18,7 +18,7 @@ The format, the statuses and how a later ADR changes an earlier one follow [the | [0008](0008-importers-from-other-apps.md) | Importers from other apps live in the core | Accepted | One crate reads each app's files on every platform into one record shape, takes keychains and directories from the app, says what it cannot carry, and reviews the result against the library with the macOS duplicate handling. | | [0009](0009-in-process-ssh-on-windows.md) | SSH on Windows runs in process on russh | Accepted | russh behind `tablepro-ssh`'s `in-process` feature, on every target; Windows uses it, Linux keeps OpenSSH. Forwards are channels with no address, so MySQL and MongoDB relay them through a private listener that serves only this process. | | [0010](0010-macos-on-the-core.md) | The macOS app moves to the core in slices, starting with the connection library | Accepted | UniFFI Swift bindings in an XCFramework of the static library, without the engines, at the app's deployment target; UI, Keychain and the CloudKit transport stay Swift; the core maps the library's CloudKit records both ways with the existing fields and merges a pull field by field. | -| [0011](0011-sql-dump-and-import-in-process.md) | SQL dumps and SQL file imports run in process | Accepted | Export Database writes one `.sql` file from each driver's catalog read in `pg_dump` and `mysqldump` order, and Import SQL File splits a file of any size as the engine's client does and runs it on an editor session; no app calls `pg_dump` or `mysqldump`. | +| [0011](0011-sql-dump-and-import-in-process.md) | SQL dumps and SQL file imports run in process | Accepted; the import of COPY rows as INSERTs superseded by 0024 | Export Database writes one `.sql` file from each driver's catalog read in `pg_dump` and `mysqldump` order, and Import SQL File splits a file of any size as the engine's client does and runs it on an editor session; no app calls `pg_dump` or `mysqldump`. | | [0012](0012-copies-before-a-format-upgrade.md) | A file in an older format is copied beside itself before its first write in the newer one | Accepted | `write_document_blocking` and the history's open copy a file an older build wrote to `.v-` before replacing it, the history through `VACUUM INTO`; three kept per file; no copy, no write. | | [0013](0013-catalog-operations.md) | Truncating a table and creating or dropping a schema or database | Accepted | `DdlDialect` plans TRUNCATE and CREATE and DROP for schemas and databases per engine, MongoDB through its shell; a database change runs outside any transaction as `WriteKind::DatabaseChanges`; a login to a missing database fails as `DatabaseNotFound` and parks. | | [0014](0014-tls-mode-of-a-new-connection.md) | A new connection starts on the TLS mode the engine's own client starts on | Accepted | `TlsMode::Prefer` for PostgreSQL and MySQL, Verify full for SQL Server, Disable for the engines that decide TLS by the port, with a DNS seed list and ClickHouse's HTTPS ports on Verify full; the core holds the rule, and every TLS failure a mode change fixes offers Change Encryption. | @@ -31,3 +31,4 @@ The format, the statuses and how a later ADR changes an earlier one follow [the | [0021](0021-an-int32-keeps-its-width.md) | An int32 keeps its width, and an edited number keeps its type | Accepted | `Value::Int32` holds MongoDB's int32, equal to the `Int` of the same number; a save carries each edited cell's read value, and a number binds in the read's BSON type when it is exactly one, else the column's. | | [0022](0022-browsing-through-the-ffi.md) | A table page is read on the session with its filter bound, and structure, definitions, export and history cross the FFI | Accepted | `read_table_page` builds the page with the filter, binds its values and reads it with `read_page` on a pooled connection as run events; `table_structure`, `object_definition`, `export_rows`, `export_table` and `Core::history` give the Linux app's reads to an app on the FFI. | | [0023](0023-mcp-client-setup-from-one-catalog.md) | Both apps set up MCP clients from one catalog | Accepted | `core/crates/mcp/clients.json`, bundled, lists each client per OS; install by deeplink, then the client's CLI by argv, then a comment-keeping splice into its config file with a `.tablepro-backup`, else copy; macOS writes the stdio bridge and no token, Linux the URL and token, and the Linux server serves only the user's own account; brand marks from Simple Icons or the brand, still their owners' trademarks. | +| [0024](0024-copy-rows-through-copy-in.md) | An imported COPY block loads through PostgreSQL's copy-in | Accepted | The `ScriptReader` gives a `COPY ... FROM stdin` then its rows as written in pieces of at most 256 KiB, up to `\.`, which is never sent; `run_sql_import` streams them through `EditorSession::copy_in`, in the import's transaction or the driver's own; binary is refused; tokio-postgres comes from a fork with the copy-in fix (rust-postgres#1402) until a release has it. | diff --git a/deny.toml b/deny.toml index 1efa63b5e1..874b395136 100644 --- a/deny.toml +++ b/deny.toml @@ -67,4 +67,5 @@ deny = [ [sources] unknown-registry = "deny" unknown-git = "deny" -allow-git = [] +# tokio-postgres with the copy_in fix, until a release has it (Cargo.toml, patch.crates-io). +allow-git = ["https://github.com/TableProApp/rust-postgres"] diff --git a/linux/README.md b/linux/README.md index 4e17741bbf..8da355c712 100644 --- a/linux/README.md +++ b/linux/README.md @@ -474,8 +474,9 @@ connection, on a session of its own so what the file sets stays there. The file is read as it runs, so its size does not matter. It is split as the engine's client splits it: at `;`, at a MySQL `DELIMITER`, and at SQL Server `GO` lines, a `GO 3` batch running three times. Files from -`pg_dump` load too: their `COPY ... FROM stdin` rows run as INSERTs, and -the `\restrict` and `\unrestrict` lines that only psql reads are passed +`pg_dump` load too: their `COPY ... FROM stdin` rows go through +PostgreSQL's copy protocol as psql sends them, text or CSV, and the +`\restrict` and `\unrestrict` lines that only psql reads are passed over. Any other psql command, such as `\connect`, is refused with its line. diff --git a/linux/ROADMAP.md b/linux/ROADMAP.md index 1e3f1694fb..a8380d2c7e 100644 --- a/linux/ROADMAP.md +++ b/linux/ROADMAP.md @@ -128,7 +128,7 @@ The app runs on GNOME 50 ([ADR 0015](docs/decisions/0015-gnome-50-platform-floor | Capability | Detail | |---|---| -| SQL files | Import SQL File in the main menu runs a `.sql` or `.sql.gz` file of any size on an editor session of its own, split as the engine's client splits it (`DELIMITER`, `GO n`, routine bodies, dollar quotes), with progress, Stop and a summary of what failed and where. pg_dump's `COPY ... FROM stdin` rows run as INSERTs and its `\restrict` lines are passed over | +| SQL files | Import SQL File in the main menu runs a `.sql` or `.sql.gz` file of any size on an editor session of its own, split as the engine's client splits it (`DELIMITER`, `GO n`, routine bodies, dollar quotes), with progress, Stop and a summary of what failed and where. pg_dump's `COPY ... FROM stdin` rows load through the copy protocol, text or CSV, and its `\restrict` lines are passed over | | Errors and transactions | Stop at First Error, or run the rest and list each failure with its line. On PostgreSQL, SQL Server, SQLite and DuckDB the file can run as one transaction, so a failure or Stop undoes it all; MySQL, MariaDB and ClickHouse commit each statement as it runs. Safe Mode Read-Only offers neither import, and the import dialog is the one question Confirm Writes asks for the whole file | ### Windows and sessions diff --git a/linux/build-aux/rpm/tablepro.spec b/linux/build-aux/rpm/tablepro.spec index a09fa168b2..65803944fe 100644 --- a/linux/build-aux/rpm/tablepro.spec +++ b/linux/build-aux/rpm/tablepro.spec @@ -164,6 +164,9 @@ an SSH bastion. # Writes .cargo/config.toml with the vendored crates as the only source. # Cargo reads it from the parent of the Meson build directory it runs in. %cargo_prep -v vendor +# %%cargo_prep replaces only crates.io. The vendored crates that come from +# a git repository need the source replacement cargo vendor wrote for them. +sed -n '/^\[source\."git+/,/^$/p' vendor/cargo-config.toml >> .cargo/config.toml # Some crates ship executable .rs files, whose leading #![...] attribute # brp-mangle-shebangs then rejects as a shebang in the debugsource package. find vendor -type f -name '*.rs' -perm /111 -exec chmod a-x {} + diff --git a/linux/changelog.d/4258.added.md b/linux/changelog.d/4258.added.md new file mode 100644 index 0000000000..b08a1d4a6a --- /dev/null +++ b/linux/changelog.d/4258.added.md @@ -0,0 +1 @@ +Import SQL File loads PostgreSQL COPY blocks in CSV or with options, and large blocks load faster. diff --git a/linux/crates/app/src/ui/error_text/tests.rs b/linux/crates/app/src/ui/error_text/tests.rs index b10a381970..de00476bdf 100644 --- a/linux/crates/app/src/ui/error_text/tests.rs +++ b/linux/crates/app/src/ui/error_text/tests.rs @@ -851,3 +851,25 @@ fn a_catalog_option_the_statement_cannot_carry_says_which() { "This database takes no template for a new database." ); } + +#[test] +fn an_import_failure_names_where_the_server_failed() { + let diagnostics = ServerDiagnostics { + where_context: Some("COPY events, line 3, column ts: \"2024-13-01\"".to_owned()), + ..ServerDiagnostics::new( + Some(ServerCode::SqlState("22008".to_owned())), + "date/time field value out of range", + ) + }; + + let message = sql_import_fault_message(&tablepro_core::import::SqlImportFault::Statement( + DriverError::reported(diagnostics), + )); + let without = sql_import_fault_message(&tablepro_core::import::SqlImportFault::Statement(DriverError::server( + "syntax error", + ))); + + assert!(message.contains("out of range"), "{message}"); + assert!(message.contains("COPY events, line 3, column ts"), "{message}"); + assert!(!without.contains('\n'), "{without}"); +} diff --git a/linux/crates/app/src/ui/error_text/transfer.rs b/linux/crates/app/src/ui/error_text/transfer.rs index daaad391fe..d1c3c8f4f8 100644 --- a/linux/crates/app/src/ui/error_text/transfer.rs +++ b/linux/crates/app/src/ui/error_text/transfer.rs @@ -1,3 +1,4 @@ +use tablepro_core::DriverError; use tablepro_core::dump::DumpError; use tablepro_core::import::{SqlFileError, SqlImportFault}; use tablepro_core::sql_syntax::script::{RefusedUnit, ScriptReadError}; @@ -36,17 +37,32 @@ pub fn sql_file_message(error: &SqlFileError) -> String { /// Why one statement of a SQL file did not run. pub fn sql_import_fault_message(fault: &SqlImportFault) -> String { match fault { - SqlImportFault::Statement(error) => driver_message(error), + SqlImportFault::Statement(error) => statement_failure(error), SqlImportFault::Refused(RefusedUnit::ClientCommand { command }) => gettext_f( "{command} is a psql command, which only psql runs.", &[("command", command)], ), - SqlImportFault::Refused(RefusedUnit::CopyFormat) => { - gettext("Only COPY data in the text format pg_dump writes can be imported, with no options.") + SqlImportFault::Refused(RefusedUnit::CopyBinary) => { + gettext("A SQL file cannot hold COPY rows in the binary format, so these rows were skipped.") } - SqlImportFault::Refused(RefusedUnit::CopyNotText) => gettext("This COPY row is not UTF-8 text."), SqlImportFault::FileEndsTransaction => gettext( "The file ends its own transaction, which would end the one the import runs it in. Turn off Run as One Transaction to run it as written.", ), } } + +/// The server's context names where in a long statement it failed, such +/// as the row of a COPY block, which the statement's line alone does not. +fn statement_failure(error: &DriverError) -> String { + let mut message = driver_message(error); + if let DriverError::Server(diagnostics) = error + && let Some(context) = diagnostics + .where_context + .as_deref() + .filter(|context| !context.is_empty()) + { + message.push('\n'); + message.push_str(&gettext_f("Context: {context}", &[("context", context)])); + } + message +} diff --git a/linux/crates/app/src/ui/error_text/unsupported.rs b/linux/crates/app/src/ui/error_text/unsupported.rs index 36ed7d6feb..e8e61ff6a7 100644 --- a/linux/crates/app/src/ui/error_text/unsupported.rs +++ b/linux/crates/app/src/ui/error_text/unsupported.rs @@ -21,6 +21,7 @@ pub(super) fn unsupported_message(feature: UnsupportedFeature) -> String { } UnsupportedFeature::CopyFromStdinInEditor => gettext("COPY FROM STDIN cannot run in the editor."), UnsupportedFeature::CopyToStdoutInEditor => gettext("COPY TO STDOUT cannot run in the editor."), + UnsupportedFeature::CopyFromStdin => gettext("This database cannot take the rows of COPY FROM STDIN."), UnsupportedFeature::TimestampInRepeatedHour => gettext( "This time falls in the hour the session's time zone repeats, so the database cannot store it as one instant.", ), @@ -103,6 +104,7 @@ mod tests { UnsupportedFeature::EditorCommand { command: "MONITOR" }, UnsupportedFeature::AuthPlugin, UnsupportedFeature::SqlDump, + UnsupportedFeature::CopyFromStdin, ]; let messages: Vec = features.iter().map(|feature| unsupported_message(*feature)).collect(); diff --git a/linux/docs/adding-drivers.md b/linux/docs/adding-drivers.md index 6f0d31a00b..e71709fdfd 100644 --- a/linux/docs/adding-drivers.md +++ b/linux/docs/adding-drivers.md @@ -300,6 +300,7 @@ An editor tab holds one session for as long as it is open, because the state a u - A statement the server refused ends the run with `RunEnd::Failed` in the `RunSummary`; an `Err` from `run` means the session itself failed. - `RunSummary` carries `elapsed`, the `RunEnd` (`Completed`, `Failed`, `Cancelled`, `TimedOut { server_cancelled }`) and the `TransactionState` after the run. `Unknown` is an answer of its own: the tab's close guard warns instead of guessing. - `close` stops a running statement, then closes the connection within `close_timeout`. +- `copy_in(statement, rows, options)` runs a `COPY ... FROM STDIN` from an imported file and sends it the `CopyInPiece`s it receives, ending at `CopyInPiece::End`; a closed channel abandons the copy. The default refuses with `Unsupported { CopyFromStdin }`; only PostgreSQL implements it. ### `ResultSink` and backpressure diff --git a/linux/docs/drivers/postgresql.md b/linux/docs/drivers/postgresql.md index a6480d028b..6d58991c0e 100644 --- a/linux/docs/drivers/postgresql.md +++ b/linux/docs/drivers/postgresql.md @@ -112,6 +112,7 @@ An editor tab runs SQL the way psql does: - No session-level `SET` is ever sent. The run's deadline and the session's statement timeout are enforced with cancel requests, and nothing new goes to the server until an outstanding cancel request has finished. A statement that has not ended by the end of the cancel grace ends the session. - Notices (`RAISE NOTICE`, warnings) arrive as `ResultEvent::Notice` with the statement that raised them, those of the cursor's `DECLARE` and its closing `COMMIT` or `RELEASE` included. Those of the driver's own statements, such as its `BEGIN`, `ROLLBACK` and the transaction probe, are dropped. A run keeps 10,000; past that the connection drops the rest and the run sends `NoticeLimitReached` once. - `COPY FROM STDIN` and `COPY TO STDOUT` are refused with `Unsupported`. +- `copy_in` runs an imported `COPY ... FROM STDIN` with its rows through tokio-postgres's copy-in, in the session's transaction, or in one of its own when none is open so a transaction pooler keeps the prepared statement and its Bind on one server connection. A read-only session starts its own transaction READ ONLY, so the server refuses the COPY. A COPY the server refuses before the copy starts fails like any statement; the driver relies on the patched tokio-postgres for that (`core/docs/decisions/0024-copy-rows-through-copy-in.md`). - Result columns are text: the simple protocol gives a column no type, so every value arrives as `Value::Text`. Typed columns wait on rust-postgres #1358. ### The row limit