Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 3 additions & 6 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 10 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,16 @@ testcontainers = { version = "0.27", default-features = false, features = ["aws-
testcontainers-modules = { version = "0.15", default-features = false, features = ["aws-lc-rs"] }
oo7 = { version = "0.6", default-features = false, features = ["tokio", "native_crypto", "unstable"] }

# TablePro's fork of rust-postgres, pinned by revision: the tokio-postgres-v0.7.18 tag plus the
# copy_in fix in rust-postgres/rust-postgres#1402, without which a COPY the server refuses before
# copy-in mode leaves the connection out of step. All three crates come from the fork, so the
# driver's postgres-types and postgres-protocol stay the ones tokio-postgres uses. Remove this
# once a tokio-postgres release has the fix.
[patch.crates-io]
tokio-postgres = { git = "https://github.com/TableProApp/rust-postgres", rev = "f8bdf78d12df410a1c1b485445a9d0683361a173" }
postgres-types = { git = "https://github.com/TableProApp/rust-postgres", rev = "f8bdf78d12df410a1c1b485445a9d0683361a173" }
postgres-protocol = { git = "https://github.com/TableProApp/rust-postgres", rev = "f8bdf78d12df410a1c1b485445a9d0683361a173" }

[workspace.lints.rust]
unsafe_op_in_unsafe_fn = "deny"

Expand Down
27 changes: 27 additions & 0 deletions core/crates/core/src/copy_in.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
use tokio::sync::mpsc;
use tokio_util::bytes::Bytes;

use crate::result_event::RunEnd;
use crate::transaction_state::TransactionState;

/// One message of the rows a `COPY ... FROM STDIN` reads.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CopyInPiece {
/// Bytes of the rows as the file holds them, cut anywhere.
Data(Bytes),
/// The rows ended. A source that closes without it gave up, and the
/// copy is abandoned, never finished with the rows it has.
End,
}

/// Where a session takes the rows of a copy from, a piece at a time.
pub type CopyInRows = mpsc::Receiver<CopyInPiece>;

#[derive(Debug, Clone, PartialEq)]
pub struct CopyInSummary {
pub end: RunEnd,
/// The rows the server took, from its `COPY n`. 0 unless the copy
/// completed.
pub rows: u64,
pub transaction_state: TransactionState,
}
19 changes: 19 additions & 0 deletions core/crates/core/src/editor_session.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use async_trait::async_trait;

use crate::call_options::CallOptions;
use crate::copy_in::{CopyInRows, CopyInSummary};
use crate::error::DriverError;
use crate::result_event::RunSummary;
use crate::result_sink::ResultSink;
Expand Down Expand Up @@ -75,6 +76,24 @@ pub trait EditorSession: Send + Sync {

async fn transaction_state(&self, options: CallOptions) -> Result<TransactionState, DriverError>;

/// Run `statement`, a `COPY ... FROM STDIN`, with the pieces `rows`
/// gives as its data, sent on as they come. A cancel, or a source that
/// closes before `CopyInPiece::End`, abandons the copy and ends it
/// cancelled, whatever the server answers to that. A copy the server
/// refused ends `Failed`. `Unsupported`, which a session without
/// copy-in returns, fails only this statement; any other error is the
/// session failing, as with `run`.
async fn copy_in(
&self,
_statement: UserSql,
_rows: CopyInRows,
_options: RunOptions,
) -> Result<CopyInSummary, DriverError> {
Err(DriverError::Unsupported {
feature: crate::UnsupportedFeature::CopyFromStdin,
})
}

/// The database an unqualified name resolves to now, as the server
/// last said. `None` where the driver does not follow it: the session
/// is still in the database it opened in.
Expand Down
125 changes: 118 additions & 7 deletions core/crates/core/src/import/sql_file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,16 @@ pub struct SqlFileReader {
line: u64,
first: bool,
failed: bool,
/// The error that ended the reading, given once the units read before
/// it are taken.
error: Option<SqlFileError>,
/// Set by whoever stopped taking units, so a long run of lines that
/// make none, such as a refused COPY block, is not read to its end.
stop: Arc<AtomicBool>,
}

impl SqlFileReader {
pub fn new(file: Box<dyn Read + Send>, grammar: SqlGrammar, copy_overriding: Option<&str>) -> Self {
pub fn new(file: Box<dyn Read + Send>, grammar: SqlGrammar) -> Self {
let consumed = Arc::new(AtomicU64::new(0));
let counted = Counted {
inner: file,
Expand All @@ -72,19 +75,18 @@ impl SqlFileReader {
};
Self {
source,
reader: Some(ScriptReader::new(grammar).with_copy_overriding(copy_overriding)),
reader: Some(ScriptReader::new(grammar)),
units: VecDeque::new(),
consumed,
partial: Vec::new(),
line: 1,
first: true,
failed: false,
error: None,
stop: Arc::new(AtomicBool::new(false)),
}
}

/// The bytes of the file read so far, shared so another thread can
/// show progress.
/// How the session reads a string as the file starts.
pub fn with_settings(mut self, settings: LexicalSettings) -> Self {
self.reader = self.reader.map(|reader| reader.with_settings(settings));
Expand All @@ -96,6 +98,8 @@ impl SqlFileReader {
Arc::clone(&self.stop)
}

/// The bytes of the file read so far, shared so another thread can
/// show progress.
pub fn consumed(&self) -> Arc<AtomicU64> {
Arc::clone(&self.consumed)
}
Expand All @@ -106,11 +110,15 @@ impl SqlFileReader {
match self.source.read(&mut bytes) {
Ok(read) => break read,
Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue,
Err(error) => return Err(SqlFileError::Read(error.to_string())),
Err(error) => {
self.plan_held()?;
return Err(SqlFileError::Read(error.to_string()));
}
}
};
if read == 0 {
if !self.partial.is_empty() {
self.plan_held()?;
return Err(SqlFileError::NotUtf8 { line: self.line });
}
let Some(reader) = self.reader.take() else {
Expand All @@ -137,6 +145,10 @@ impl SqlFileReader {
Err(error) => {
let before = chunk.get(..error.valid_up_to()).unwrap_or_default();
let line = self.line + before.iter().filter(|byte| **byte == b'\n').count() as u64;
// The statements that end before the byte still run.
if let (Ok(text), Some(reader)) = (std::str::from_utf8(before), self.reader.as_mut()) {
self.units.extend(reader.push_last(text)?);
}
return Err(SqlFileError::NotUtf8 { line });
}
};
Expand All @@ -151,6 +163,17 @@ impl SqlFileReader {
}
}

impl SqlFileReader {
/// Plan what the reader holds now, so the statements read whole before
/// the file stops being readable still run.
fn plan_held(&mut self) -> Result<(), SqlFileError> {
if let Some(reader) = self.reader.as_mut() {
self.units.extend(reader.push_last("")?);
}
Ok(())
}
}

impl Iterator for SqlFileReader {
type Item = Result<ScriptUnit, SqlFileError>;

Expand All @@ -159,6 +182,9 @@ impl Iterator for SqlFileReader {
if let Some(unit) = self.units.pop_front() {
return Some(Ok(unit));
}
if let Some(error) = self.error.take() {
return Some(Err(error));
}
if self.failed || self.stop.load(Ordering::Relaxed) {
return None;
}
Expand All @@ -167,7 +193,7 @@ impl Iterator for SqlFileReader {
Ok(false) => return None,
Err(error) => {
self.failed = true;
return Some(Err(error));
self.error = Some(error);
}
}
}
Expand Down Expand Up @@ -197,7 +223,7 @@ mod tests {
use super::*;

fn units(bytes: Vec<u8>) -> Vec<Result<ScriptUnit, SqlFileError>> {
SqlFileReader::new(Box::new(std::io::Cursor::new(bytes)), SqlGrammar::Sqlite, None).collect()
SqlFileReader::new(Box::new(std::io::Cursor::new(bytes)), SqlGrammar::Sqlite).collect()
}

fn texts(bytes: Vec<u8>) -> Vec<String> {
Expand Down Expand Up @@ -231,9 +257,94 @@ mod tests {
fn bytes_that_are_not_utf8_name_their_line() {
let read = units(b"SELECT 1;\nSELECT 2;\nSELECT X'00', '\xff';\n".to_vec());

let ran: Vec<&str> = read
.iter()
.filter_map(|unit| unit.as_ref().ok())
.map(|unit| unit.text.as_str())
.collect();
assert_eq!(ran, ["SELECT 1", "SELECT 2"], "the statements before the byte");
assert_eq!(read.last(), Some(&Err(SqlFileError::NotUtf8 { line: 3 })));
}

#[test]
fn a_copy_row_that_is_not_utf8_names_its_line() {
let file = b"COPY t (a) FROM stdin;\nok\n\xff\n\\.\n".to_vec();

let read: Vec<Result<ScriptUnit, SqlFileError>> =
SqlFileReader::new(Box::new(std::io::Cursor::new(file)), SqlGrammar::PostgreSql).collect();

assert_eq!(read.last(), Some(&Err(SqlFileError::NotUtf8 { line: 3 })));
}

/// Gives at most `chunk` bytes a read, as a pipe can, and then fails
/// when `fails`.
struct Trickle {
bytes: Vec<u8>,
at: usize,
chunk: usize,
fails: bool,
}

impl Trickle {
fn new(bytes: &[u8], fails: bool) -> Self {
Self {
bytes: bytes.to_vec(),
at: 0,
chunk: 20,
fails,
}
}
}

impl Read for Trickle {
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
let rest = self.bytes.get(self.at..).unwrap_or_default();
if rest.is_empty() && self.fails {
return Err(std::io::Error::other("the copy was cut off"));
}
let read = rest.len().min(self.chunk).min(buffer.len());
buffer[..read].copy_from_slice(&rest[..read]);
self.at += read;
Ok(read)
}
}

fn read_trickle(file: Trickle) -> (Vec<String>, Option<SqlFileError>) {
let read: Vec<Result<ScriptUnit, SqlFileError>> =
SqlFileReader::new(Box::new(file), SqlGrammar::Sqlite).collect();
let ran = read
.iter()
.filter_map(|unit| unit.as_ref().ok())
.map(|unit| unit.text.clone())
.collect();
(ran, read.last().and_then(|unit| unit.clone().err()))
}

#[test]
fn the_statements_read_whole_before_the_file_stops_being_readable_run() {
let cases = [
(
Trickle::new(b"SELECT 1;\nSELECT 2;\nSELECT 3;\n'\xff';\n", false),
SqlFileError::NotUtf8 { line: 4 },
),
(
Trickle::new(b"SELECT 1;\nSELECT 2;\nSELECT 3;\n\xc3", false),
SqlFileError::NotUtf8 { line: 4 },
),
(
Trickle::new(b"SELECT 1;\nSELECT 2;\nSELECT 3;\n", true),
SqlFileError::Read("the copy was cut off".to_owned()),
),
];

for (file, error) in cases {
let (ran, last) = read_trickle(file);

assert_eq!(ran, ["SELECT 1", "SELECT 2", "SELECT 3"], "{error:?}");
assert_eq!(last, Some(error));
}
}

#[test]
fn a_character_cut_between_reads_is_whole() {
// The read ends between the two bytes of the é.
Expand Down
Loading
Loading