diff --git a/core/crates/core/src/sql_syntax/script/lexer.rs b/core/crates/core/src/sql_syntax/script/lexer.rs index 51f55a5a86..1d3c63deab 100644 --- a/core/crates/core/src/sql_syntax/script/lexer.rs +++ b/core/crates/core/src/sql_syntax/script/lexer.rs @@ -1,3 +1,4 @@ +use std::cell::RefCell; use std::collections::HashSet; use std::ops::Range; @@ -5,6 +6,7 @@ use crate::sql_syntax::script::script_rules::{ BackslashPolicy, BracketIdentifier, DollarQuote, HashComment, QuoteKind, QuoteRule, ScriptSyntax, }; use crate::sql_syntax::script::server_reading::{CommentRead, NameQuotes, ServerReading, dollar_tag_end, read_at}; +use crate::sql_syntax::script::unicode_spaces::{self, UnicodeSpaces}; use crate::sql_syntax::script::{LexicalSettings, OpenConstruct, ScriptTokenKind}; /// MariaDB's MSSQL mode ends a `[name]` at a `]` that is not doubled. @@ -30,6 +32,11 @@ pub(crate) struct Lexer<'a> { missing_heredoc_closers: HashSet<&'a str>, /// Where the executable comment the lexer is inside opened. executable: Option, + /// DuckDB's pass over the statement being read, when the text holds a + /// space it may blank. + unicode_spaces: Option>, + /// The statement being read has had its first code token. + in_statement: bool, } impl<'a> Lexer<'a> { @@ -48,6 +55,11 @@ impl<'a> Lexer<'a> { pos: 0, missing_heredoc_closers: HashSet::new(), executable: None, + unicode_spaces: (syntax.blanks_unicode_spaces + && reading.blanks_unicode_spaces() + && unicode_spaces::any(text.as_bytes())) + .then(|| RefCell::new(UnicodeSpaces::new(0))), + in_statement: false, } } @@ -70,6 +82,13 @@ impl<'a> Lexer<'a> { pub fn next_token(&mut self, delimiter: Option<&str>) -> Option { let start = self.pos; let first = *self.bytes.get(start)?; + // Each statement goes to DuckDB from its first code token on, and + // DuckDB starts its pass there. + if !self.in_statement + && let Some(spaces) = &self.unicode_spaces + { + *spaces.borrow_mut() = UnicodeSpaces::new(start); + } let token = match delimiter { // The server ends an executable comment at the first `*/` // that stands where a token could start. @@ -83,6 +102,11 @@ impl<'a> Lexer<'a> { _ => self.lex_at(start, first, delimiter), }; self.pos = token.range.end; + if token.kind == ScriptTokenKind::Semicolon { + self.in_statement = false; + } else if !token.kind.is_trivia() { + self.in_statement = true; + } Some(token) } @@ -95,6 +119,7 @@ impl<'a> Lexer<'a> { } match first { b if is_space(b) => self.plain(ScriptTokenKind::Whitespace, self.whitespace_end(start)), + _ if self.blank_at(start).is_some() => self.plain(ScriptTokenKind::Whitespace, self.whitespace_end(start)), 0xEF if self.syntax.byte_order_mark_is_space && self.bytes[start..].starts_with(BYTE_ORDER_MARK) => { self.plain(ScriptTokenKind::Whitespace, start + BYTE_ORDER_MARK.len()) } @@ -141,6 +166,10 @@ impl<'a> Lexer<'a> { fn whitespace_end(&self, start: usize) -> usize { let mut end = start; while let Some(&b) = self.bytes.get(end) { + if let Some(len) = self.blank_at(end) { + end += len; + continue; + } if !is_space(b) { break; } @@ -359,8 +388,8 @@ impl<'a> Lexer<'a> { DollarQuote::Tagged => { if let Some(opener_end) = self.tagged_opener_end(start) { let opener = &self.text[start..opener_end]; - return match self.text[opener_end..].find(opener) { - Some(offset) => self.plain(ScriptTokenKind::DollarQuoted, opener_end + offset + opener.len()), + return match self.closer(opener_end, opener) { + Some(closer) => self.plain(ScriptTokenKind::DollarQuoted, closer + opener.len()), None => self.unterminated(start, OpenConstruct::DollarQuoted), }; } @@ -452,13 +481,13 @@ impl<'a> Lexer<'a> { let mut index = start + 1; let first = *self.bytes.get(index)?; if first != b'$' { - if !(first.is_ascii_alphabetic() || first == b'_' || first >= 0x80) { + if !(first.is_ascii_alphabetic() || first == b'_' || self.non_ascii_name_byte(index)) { return None; } while self .bytes .get(index) - .is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'_' || *b >= 0x80) + .is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'_' || self.non_ascii_name_byte(index)) { index += 1; } @@ -469,6 +498,21 @@ impl<'a> Lexer<'a> { Some(index + 1) } + /// Where the first `opener` after `from` starts. One that holds a space + /// DuckDB's pass blanks is no longer the opener's text. + fn closer(&self, from: usize, opener: &str) -> Option { + let holds_space = (0..opener.len()).any(|at| unicode_spaces::space_len(opener.as_bytes(), at).is_some()); + let mut search = from; + loop { + let at = search + self.text.get(search..)?.find(opener)?; + if !holds_space || (at..at + opener.len()).all(|byte| self.blank_at(byte).is_none()) { + return Some(at); + } + // The opener starts with `$`, so the next byte starts a character. + search = at + 1; + } + } + fn heredoc_end(&mut self, start: usize) -> Option { let close = find_byte(self.bytes, start + 1, b'$')?; let opener = &self.text[start..=close]; @@ -500,7 +544,10 @@ impl<'a> Lexer<'a> { let delimiter = delimiter.filter(|d| d.bytes().next().is_some_and(|b| self.is_word_part(b))); let mut end = from; while let Some(&b) = self.bytes.get(end) { - if !self.is_word_part(b) || delimiter.is_some_and(|d| self.bytes[end..].starts_with(d.as_bytes())) { + if !self.is_word_part(b) + || (b >= 0x80 && self.blank_at(end).is_some()) + || delimiter.is_some_and(|d| self.bytes[end..].starts_with(d.as_bytes())) + { break; } end += 1; @@ -555,6 +602,18 @@ impl<'a> Lexer<'a> { fn is_word_part(&self, b: u8) -> bool { b.is_ascii_alphanumeric() || b == b'_' || b >= 0x80 || self.syntax.word_part_extra.contains(&b) } + + fn non_ascii_name_byte(&self, at: usize) -> bool { + self.bytes.get(at).is_some_and(|&b| b >= 0x80) && self.blank_at(at).is_none() + } + + /// How long the non-ASCII space at `at` is, when DuckDB's pass turns it + /// into a space. + fn blank_at(&self, at: usize) -> Option { + let spaces = self.unicode_spaces.as_ref()?; + let len = unicode_spaces::space_len(self.bytes, at)?; + spaces.borrow_mut().blanks(self.bytes, at).then_some(len) + } } fn is_space(b: u8) -> bool { diff --git a/core/crates/core/src/sql_syntax/script/mod.rs b/core/crates/core/src/sql_syntax/script/mod.rs index 7423c01ab6..753aa1a703 100644 --- a/core/crates/core/src/sql_syntax/script/mod.rs +++ b/core/crates/core/src/sql_syntax/script/mod.rs @@ -16,6 +16,7 @@ mod script_token; mod script_token_kind; pub(crate) mod server_reading; pub mod text_offsets; +pub mod unicode_spaces; pub use batch_error_policy::BatchErrorPolicy; pub use lexical_settings::LexicalSettings; diff --git a/core/crates/core/src/sql_syntax/script/script_plan.rs b/core/crates/core/src/sql_syntax/script/script_plan.rs index a0fbb0659b..dac24b57f6 100644 --- a/core/crates/core/src/sql_syntax/script/script_plan.rs +++ b/core/crates/core/src/sql_syntax/script/script_plan.rs @@ -74,7 +74,8 @@ impl ScriptPlan { /// Every plan the client or a server may make of `text`: as the client /// splits it and, on MySQL, as each server reads what it is sent, at /// each version its executable comments name. On SQL Server, with a - /// `--` after a number's `e` a comment and not. Each with backslash + /// `--` after a number's `e` a comment and not. On DuckDB, with its + /// non-ASCII spaces blanked as its pass does and kept. Each with backslash /// escapes on and off, since a session setting decides. Built one at a /// time. `None` when the text names too many versions to try. pub fn server_readings(text: &str, grammar: SqlGrammar) -> Option + '_> { diff --git a/core/crates/core/src/sql_syntax/script/script_rules.rs b/core/crates/core/src/sql_syntax/script/script_rules.rs index 5e39ea78e1..e8bfb35d7e 100644 --- a/core/crates/core/src/sql_syntax/script/script_rules.rs +++ b/core/crates/core/src/sql_syntax/script/script_rules.rs @@ -93,6 +93,12 @@ pub(crate) struct ScriptSyntax { /// SQLite's `sqlite3GetToken` reads it. Inside a name it stays a name /// character, there as elsewhere. pub byte_order_mark_is_space: bool, + /// DuckDB turns its non-ASCII spaces into spaces before it lexes, where + /// its own reading of quotes is outside one (`unicode_spaces`). The + /// driver sends each statement alone, from its first code token, and + /// DuckDB reads each text it gets afresh, so the reading starts again + /// there. + pub blanks_unicode_spaces: bool, pub word_start_extra: &'static [u8], pub word_part_extra: &'static [u8], } @@ -132,6 +138,7 @@ const POSTGRESQL: ScriptRules = ScriptRules { digit_separators: true, tcl_parameters: false, byte_order_mark_is_space: false, + blanks_unicode_spaces: false, word_start_extra: b"", word_part_extra: b"$", }, @@ -159,6 +166,7 @@ const MYSQL: ScriptRules = ScriptRules { digit_separators: false, tcl_parameters: false, byte_order_mark_is_space: false, + blanks_unicode_spaces: false, word_start_extra: b"$", word_part_extra: b"$", }, @@ -186,6 +194,7 @@ const SQLITE: ScriptRules = ScriptRules { digit_separators: true, tcl_parameters: true, byte_order_mark_is_space: true, + blanks_unicode_spaces: false, word_start_extra: b"$", word_part_extra: b"$", }, @@ -213,6 +222,7 @@ const MSSQL: ScriptRules = ScriptRules { digit_separators: false, tcl_parameters: false, byte_order_mark_is_space: false, + blanks_unicode_spaces: false, word_start_extra: b"@#", word_part_extra: b"@#$", }, @@ -240,6 +250,7 @@ const CLICKHOUSE: ScriptRules = ScriptRules { digit_separators: true, tcl_parameters: false, byte_order_mark_is_space: false, + blanks_unicode_spaces: false, word_start_extra: b"", word_part_extra: b"", }, @@ -267,6 +278,7 @@ const CQL: ScriptRules = ScriptRules { digit_separators: false, tcl_parameters: false, byte_order_mark_is_space: false, + blanks_unicode_spaces: false, word_start_extra: b"", word_part_extra: b"", }, @@ -275,6 +287,7 @@ const CQL: ScriptRules = ScriptRules { /// DuckDB 2.0's PEG tokenizer: a string escapes only with an `E` prefix /// and ends at its quote, with no continuation on the next line; block /// comments nest, `$tag$` quotes, and a backtick quotes a name as `"` does. +/// A byte from 0x80 up is a name byte, except a space its pass blanks. const DUCKDB: ScriptRules = ScriptRules { boundary: Boundary::Semicolon, compound: CompoundRule::None, @@ -297,6 +310,7 @@ const DUCKDB: ScriptRules = ScriptRules { digit_separators: true, tcl_parameters: false, byte_order_mark_is_space: false, + blanks_unicode_spaces: true, word_start_extra: b"", word_part_extra: b"$", }, diff --git a/core/crates/core/src/sql_syntax/script/server_reading.rs b/core/crates/core/src/sql_syntax/script/server_reading.rs index 34dee66507..e41b1408f6 100644 --- a/core/crates/core/src/sql_syntax/script/server_reading.rs +++ b/core/crates/core/src/sql_syntax/script/server_reading.rs @@ -2,11 +2,12 @@ use std::collections::{BTreeSet, HashSet}; use std::ops::RangeInclusive; use crate::sql_syntax::SqlGrammar; +use crate::sql_syntax::script::unicode_spaces; /// How the lexer reads the text whose meaning hangs on who reads it. On /// MySQL, an executable comment (`/*! … */`, `/*!50100 … */`, MariaDB's /// `/*M!100100 … */`), a `$tag$` quote and a `DELIMITER` line. On SQL -/// Server, a `--` after a number's `e`. +/// Server, a `--` after a number's `e`. On DuckDB, a non-ASCII space. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum ServerReading { /// As the mysql client reads a script to split it: every executable @@ -26,6 +27,11 @@ pub(crate) enum ServerReading { /// and the `--` of `1E--1` would be that sign and a minus where the /// client reads a comment. Nobody measured that, so both are read. SignedEmptyExponents, + /// As the client reads it, but with no DuckDB pass over its non-ASCII + /// spaces: each stays a name character, as DuckDB reads one the pass + /// keeps. The pass is mirrored from one DuckDB version, so a gate also + /// reads the text without it. + UnicodeSpacesInNames, } impl ServerReading { @@ -33,6 +39,10 @@ impl ServerReading { !matches!(self, Self::Server { .. }) } + pub(crate) fn blanks_unicode_spaces(self) -> bool { + self != Self::UnicodeSpacesInNames + } + pub(crate) fn signs_empty_exponents(self) -> bool { self == Self::SignedEmptyExponents } @@ -49,7 +59,7 @@ impl ServerReading { pub(crate) fn name_quotes(self) -> NameQuotes { match self { - Self::Client | Self::SignedEmptyExponents => NameQuotes::Default, + Self::Client | Self::SignedEmptyExponents | Self::UnicodeSpacesInNames => NameQuotes::Default, Self::Server { quotes, .. } => quotes, } } @@ -201,13 +211,17 @@ pub(crate) fn dollar_tag_end(bytes: &[u8], at: usize) -> Option { /// The client's reading of `text`, then each distinct one the servers it /// may meet give it: each kind of server at version 0 and at each version /// the text's comments name, since only those change which comments run. -/// On SQL Server, one more where a `--` follows an `e`. `None` when there -/// are more than are worth trying. +/// On SQL Server, one more where a `--` follows an `e`. On DuckDB, one more +/// without its pass when the text holds a space the pass may blank. `None` +/// when there are more than are worth trying. pub(crate) fn server_readings(text: &str, grammar: SqlGrammar) -> Option> { let mut readings = vec![ServerReading::Client]; if grammar == SqlGrammar::MsSql && (text.contains("e--") || text.contains("E--")) { readings.push(ServerReading::SignedEmptyExponents); } + if grammar == SqlGrammar::DuckDb && unicode_spaces::any(text.as_bytes()) { + readings.push(ServerReading::UnicodeSpacesInNames); + } if grammar != SqlGrammar::MySql { return Some(readings); } @@ -446,6 +460,20 @@ mod tests { server_readings("SELECT /*! 1 */", SqlGrammar::PostgreSql), Some(vec![ServerReading::Client]) ); + assert_eq!( + server_readings("SELECT\u{a0}1", SqlGrammar::DuckDb), + Some(vec![ServerReading::Client, ServerReading::UnicodeSpacesInNames]) + ); + for (text, grammar) in [ + ("SELECT 'caf\u{e9}'", SqlGrammar::DuckDb), + ("SELECT\u{a0}1", SqlGrammar::PostgreSql), + ] { + assert_eq!( + server_readings(text, grammar), + Some(vec![ServerReading::Client]), + "{text:?}" + ); + } } #[test] diff --git a/core/crates/core/src/sql_syntax/script/unicode_spaces.rs b/core/crates/core/src/sql_syntax/script/unicode_spaces.rs new file mode 100644 index 0000000000..6207f736a4 --- /dev/null +++ b/core/crates/core/src/sql_syntax/script/unicode_spaces.rs @@ -0,0 +1,224 @@ +//! DuckDB's pass that turns non-ASCII spaces into spaces before its lexer +//! runs. + +use std::ops::Range; + +/// DuckDB 2.0.0's `Parser::StripUnicodeSpaces` (`src/parser/parser.cpp`), +/// run only as far as the lexer asks. It turns U+00A0, U+2000 to U+200B, +/// U+202F, U+205F, U+2060, U+3000 and U+FEFF into spaces wherever it is +/// outside what it reads as a `'` or `"` quote, a `$tag$` quote or a `--` +/// comment. It reads no block comment, backslash or backtick, so it can be +/// inside a quote where the lexer is not, and there the space stays a name +/// character. The loops, their bounds and their order are DuckDB's. +#[derive(Debug)] +pub(crate) struct UnicodeSpaces { + state: State, + pos: usize, + /// Where each space it turns into a space starts, in order. + blanked: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum State { + Regular, + Quote(u8), + /// After a `$` that may open a quote, reading its tag. + Tag { + start: usize, + }, + Dollar { + start: usize, + len: usize, + }, + Comment, + Done, +} + +impl UnicodeSpaces { + pub(crate) fn new(from: usize) -> Self { + Self { + state: State::Regular, + pos: from, + blanked: Vec::new(), + } + } + + /// Whether the space that starts at `at` becomes a space. `at` is not + /// before where the pass started. + pub(crate) fn blanks(&mut self, bytes: &[u8], at: usize) -> bool { + while self.pos <= at && self.state != State::Done { + self.step(bytes); + } + self.blanked.binary_search(&at).is_ok() + } + + /// One turn of the loop DuckDB is in. + fn step(&mut self, q: &[u8]) { + let size = q.len(); + let pos = self.pos; + match self.state { + State::Done => {} + State::Regular if pos + 2 >= size => self.state = State::Done, + State::Regular => { + if space_len(q, pos).is_some() { + self.blanked.push(pos); + } + match q[pos] { + b'"' | b'\'' => { + self.state = State::Quote(q[pos]); + self.pos += 1; + return; + } + b'$' if q[pos + 1] == b'$' || is_tag_start(q[pos + 1]) => { + self.pos += 1; + self.state = State::Tag { start: self.pos }; + return; + } + b'-' if q[pos + 1] == b'-' => { + self.state = State::Comment; + return; + } + _ => {} + } + self.pos += 1; + } + State::Tag { .. } if pos + 2 >= size => self.state = State::Done, + State::Tag { start } => match q[pos] { + b'$' => { + self.state = State::Dollar { + start, + len: pos - start, + } + } + b if !is_tag_part(b) => self.state = State::Regular, + _ => self.pos += 1, + }, + State::Dollar { .. } if pos + 2 >= size => self.state = State::Done, + State::Dollar { start, len } => { + let closes = q[pos] == b'$' + && size - (pos + 1) > len + && q[pos + len + 1] == b'$' + && q[pos + 1..pos + 1 + len] == q[start..start + len]; + if closes { + self.pos += len + 1; + self.state = State::Regular; + } else { + self.pos += 1; + } + } + State::Quote(_) if pos + 1 >= size => self.state = State::Done, + State::Quote(quote) => { + if q[pos] != quote { + self.pos += 1; + } else if q[pos + 1] == quote { + self.pos += 2; + } else { + self.pos += 1; + self.state = State::Regular; + } + } + State::Comment if pos >= size => self.state = State::Done, + State::Comment => match q[pos] { + b'\n' | b'\r' => self.state = State::Regular, + _ => self.pos += 1, + }, + } + } +} + +/// How many bytes the space that starts at `at` takes, when one of the +/// spaces DuckDB turns into a space starts there. +pub(crate) fn space_len(bytes: &[u8], at: usize) -> Option { + match bytes.get(at..)? { + [0xC2, 0xA0, ..] => Some(2), + [0xE2, 0x80, 0x80..=0x8B | 0xAF, ..] | [0xE2, 0x81, 0x9F | 0xA0, ..] | [0xE3, 0x80, 0x80, ..] => Some(3), + [0xEF, 0xBB, 0xBF, ..] => Some(3), + _ => None, + } +} + +/// The spaces DuckDB turns into spaces in `text`, sent to it whole. +pub fn blanked_spaces(text: &str) -> Vec> { + let bytes = text.as_bytes(); + let mut pass = UnicodeSpaces::new(0); + (0..bytes.len()) + .filter_map(|at| space_len(bytes, at).map(|len| at..at + len)) + .filter(|space| pass.blanks(bytes, space.start)) + .collect() +} + +/// Whether `bytes` hold any of the spaces, so the pass can change them. +pub(crate) fn any(bytes: &[u8]) -> bool { + (0..bytes.len()).any(|at| space_len(bytes, at).is_some()) +} + +fn is_tag_start(b: u8) -> bool { + b.is_ascii_alphabetic() || b == b'_' || b >= 0x80 +} + +fn is_tag_part(b: u8) -> bool { + is_tag_start(b) || b.is_ascii_digit() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn blanked(text: &str) -> Vec { + blanked_spaces(text).into_iter().map(|space| space.start).collect() + } + + #[test] + fn every_space_it_lists_is_turned_into_a_space_and_no_other() { + for space in [ + '\u{a0}', '\u{2000}', '\u{2005}', '\u{200a}', '\u{200b}', '\u{202f}', '\u{205f}', '\u{2060}', '\u{3000}', + '\u{feff}', + ] { + assert_eq!(blanked(&format!("a{space}b c")), [1], "{space:?}"); + } + for other in [ + '\u{85}', '\u{1680}', '\u{200c}', '\u{2028}', '\u{2061}', '\u{2019}', '\u{fffe}', + ] { + assert!(blanked(&format!("a{other}b c")).is_empty(), "{other:?}"); + } + } + + #[test] + fn a_space_inside_what_it_reads_as_a_quote_or_a_comment_stays() { + assert!(blanked("'a\u{200b}b' x").is_empty()); + assert!(blanked("\"a\u{200b}b\" x").is_empty()); + assert!(blanked("-- a\u{200b}b\nx").is_empty()); + assert!(blanked("$$a\u{200b}b$$ x").is_empty()); + assert!(blanked("$t$a\u{200b}$t$ x").is_empty()); + assert_eq!(blanked("'it''s' \u{200b} x"), [8]); + assert_eq!(blanked("-- a\n\u{200b} x"), [5]); + assert_eq!(blanked("$t$a$t$ \u{200b} x"), [8]); + } + + #[test] + fn it_reads_quotes_its_own_way() { + // No block comment and no backslash: the quote opens in the comment and closes at E'. + assert!(blanked("/* ' */ a\u{200b}E'x").is_empty()); + assert!(blanked("/* ' */ 'a' \u{200b}x").is_empty()); + // `a$$` opens a `$$` quote to it, as a tag of the next `$`. + assert_eq!(blanked("a$$ b\u{200b}$$ c\u{200b}d x"), [12]); + // A tag's space is read as part of the tag, and a bad tag byte goes back to the regular loop. + assert_eq!(blanked("$a\u{200b}$x$a\u{200b}$ \u{200b} x"), [14]); + assert_eq!(blanked("$a- \u{200b} x"), [4]); + } + + #[test] + fn the_last_two_bytes_are_never_read() { + assert!(blanked("a\u{a0}").is_empty()); + assert_eq!(blanked("a\u{a0}b"), [1]); + assert_eq!(blanked("a\u{200b}"), [1]); + } + + #[test] + fn it_reads_on_from_where_it_starts() { + let text = "' a; \u{200b} x"; + let mut pass = UnicodeSpaces::new(4); + assert!(pass.blanks(text.as_bytes(), 5)); + assert!(!blanked(text).contains(&5)); + } +} diff --git a/core/crates/core/src/sql_syntax/statement_access.rs b/core/crates/core/src/sql_syntax/statement_access.rs index 254236ac6c..fa2e65a1ab 100644 --- a/core/crates/core/src/sql_syntax/statement_access.rs +++ b/core/crates/core/src/sql_syntax/statement_access.rs @@ -1423,6 +1423,31 @@ mod tests { } } + #[test] + fn duckdb_is_read_with_and_without_the_spaces_it_blanks_before_lexing() { + // Measured on DuckDB 2.0.0: each ran the DROP. In the first it kept one U+200B, inside what its pass took + // for a `$$` quote, and blanked the other; the second ran it once its last statement went alone. + for sql in [ + "SELECT 1 AS a$$, 2 AS b\u{200b}$$, varchar\u{200b}E'\\''; DROP TABLE t; --'x$$", + "/* ' */ SELECT 1; SELECT varchar\u{200b}E'\\''; DROP TABLE t; --'", + "/* ' */ SELECT varchar\u{200b}E'\\''; DROP TABLE t; --'", + "SELECT varchar\u{200b}E'\\''; DROP TABLE t; --'", + "SELECT 1 AS a,\u{200b}$$'$$; DROP TABLE t; SELECT 'x'", + "SELECT 1 AS a$$, 2 AS b\u{200b}$$; DROP TABLE t; --$$", + ] { + assert_eq!(access(sql, SqlGrammar::DuckDb), StatementAccess::Destructive, "{sql:?}"); + } + // DuckDB blanks this space and reads one statement, but without its pass the text is two. + assert_eq!( + access("SELECT 1 AS x\u{200b}$$; DROP TABLE t; --$$", SqlGrammar::DuckDb), + StatementAccess::Destructive + ); + assert_eq!( + access("SELECT 'a\u{200b}b' FROM t", SqlGrammar::DuckDb), + StatementAccess::Read + ); + } + #[test] fn a_unicode_blank_before_a_keyword_does_not_hide_it() { for (sql, expected) in [ diff --git a/core/crates/core/tests/script_corpus.rs b/core/crates/core/tests/script_corpus.rs index 9ed007bfba..73f9e1171d 100644 --- a/core/crates/core/tests/script_corpus.rs +++ b/core/crates/core/tests/script_corpus.rs @@ -133,6 +133,116 @@ fn duckdb_ends_an_escape_string_at_its_quote_whatever_follows() { assert_eq!(statements(SqlGrammar::PostgreSql, text), [text]); } +// Measured on DuckDB 2.0.0: each of these spaces ended `varchar` and the DROP ran; each other unit kept the name. +#[test] +fn duckdb_ends_a_name_at_the_spaces_it_blanks_before_lexing() { + for space in [ + "\u{a0}", "\u{2000}", "\u{2005}", "\u{2009}", "\u{200a}", "\u{200b}", "\u{202f}", "\u{205f}", "\u{2060}", + "\u{3000}", "\u{feff}", + ] { + let text = format!("SELECT varchar{space}E'\\''; DROP TABLE t; --'"); + let first = format!("SELECT varchar{space}E'\\''"); + assert_eq!( + statements(SqlGrammar::DuckDb, &text), + [first.as_str(), "DROP TABLE t"], + "{space:?}" + ); + assert_eq!(statements(SqlGrammar::PostgreSql, &text), [text.as_str()], "{space:?}"); + } + for kept in [ + "\u{85}", "\u{1680}", "\u{200c}", "\u{200d}", "\u{2019}", "\u{2028}", "\u{2029}", + ] { + let text = format!("SELECT varchar{kept}E'\\''; DROP TABLE t; --'"); + assert_eq!(statements(SqlGrammar::DuckDb, &text), [text.as_str()], "{kept:?}"); + } + assert_eq!( + statements( + SqlGrammar::DuckDb, + "SELECT 1 AS a,\u{200b}$$'$$; DROP TABLE t; SELECT 'x'" + ), + ["SELECT 1 AS a,\u{200b}$$'$$", "DROP TABLE t", "SELECT 'x'"] + ); + let words: Vec<&str> = { + let text = "SELECT\u{a0}*\u{a0}FROM\u{3000}t"; + plan(SqlGrammar::DuckDb, text) + .tokens(0..text.len()) + .filter(|token| token.kind == ScriptTokenKind::Word) + .map(|token| &text[token.range.clone()]) + .collect() + }; + assert_eq!(words, ["SELECT", "FROM", "t"]); +} + +// Measured on DuckDB 2.0.0. Its pass reads `'`, `"`, `$tag$` and `--` its own way, not block comments or +// backslashes, and keeps a space inside what it takes for a quote: there the space is a name character. +#[test] +fn duckdb_keeps_a_space_where_its_pass_sees_a_quote() { + for (text, expected) in [ + ( + "SELECT 1 AS a$$, 2 AS b\u{200b}$$, varchar\u{200b}E'\\''; DROP TABLE t; --'x$$", + vec![ + "SELECT 1 AS a$$, 2 AS b\u{200b}$$, varchar\u{200b}E'\\''", + "DROP TABLE t", + ], + ), + ( + "SELECT 1 AS a$$, 2 AS b\u{200b}$$; DROP TABLE t; --$$", + vec!["SELECT 1 AS a$$, 2 AS b\u{200b}$$", "DROP TABLE t"], + ), + ( + "SELECT $a\u{200b}$x$a\u{200b}$, varchar\u{200b}E'\\''; DROP TABLE t; --'", + vec!["SELECT $a\u{200b}$x$a\u{200b}$, varchar\u{200b}E'\\''", "DROP TABLE t"], + ), + ( + "-- it's\nSELECT varchar\u{200b}E'\\''; DROP TABLE t; --'", + vec!["SELECT varchar\u{200b}E'\\''", "DROP TABLE t"], + ), + ] { + assert_eq!(statements(SqlGrammar::DuckDb, text), expected, "{text:?}"); + } +} + +// Measured on DuckDB 2.0.0: whole, the pass is inside the comment's quote at the space and keeps it; the second +// statement sent alone, as the driver sends it, has the space blanked and runs the DROP. +#[test] +fn duckdb_reads_each_statement_as_it_is_sent_alone() { + assert_eq!( + statements( + SqlGrammar::DuckDb, + "/* ' */ SELECT 1; SELECT varchar\u{200b}E'\\''; DROP TABLE t; --'" + ), + ["SELECT 1", "SELECT varchar\u{200b}E'\\''", "DROP TABLE t"] + ); + // It gets each one without the comments before it. + assert_eq!( + statements( + SqlGrammar::DuckDb, + "/* ' */ SELECT varchar\u{200b}E'\\''; /* $$ */ SELECT 1 AS a,\u{200b}$$'$$; DROP TABLE t; --'" + ), + [ + "SELECT varchar\u{200b}E'\\''", + "SELECT 1 AS a,\u{200b}$$'$$", + "DROP TABLE t" + ] + ); +} + +// Measured on PostgreSQL 17.11: with a domain of each name, the server ran the DROP. +#[test] +fn postgresql_continues_a_name_with_a_dollar_and_every_non_ascii_unit() { + for glue in [ + "$", "\u{e9}", "\u{a0}", "\u{200b}", "\u{2060}", "\u{3000}", "\u{feff}", "\u{2019}", "\u{ff0c}", + ] { + let text = format!("SELECT x{glue}e'\\'; DROP TABLE t; --'"); + let first = format!("SELECT x{glue}e'\\'"); + assert_eq!( + statements(SqlGrammar::PostgreSql, &text), + [first.as_str(), "DROP TABLE t"], + "{glue:?}" + ); + } +} + #[test] fn a_sqlite_parameter_runs_to_its_closing_parenthesis() { for parameter in [ @@ -702,7 +812,7 @@ fn char_byte_roundtrip_multibyte() { proptest! { #[test] fn ranges_on_char_boundaries_and_batches_disjoint( - text in "[ -~\n\té😀]{0,160}", + text in "[ -~\n\té😀\u{a0}\u{200b}\u{3000}]{0,160}", backslash_escapes in any::(), ) { for grammar in SqlGrammar::SQL.into_iter().chain([SqlGrammar::Redis, SqlGrammar::Mongo]) { diff --git a/core/crates/drivers/duckdb/src/dialect.rs b/core/crates/drivers/duckdb/src/dialect.rs index e9dee7b9f5..cdbfc308bd 100644 --- a/core/crates/drivers/duckdb/src/dialect.rs +++ b/core/crates/drivers/duckdb/src/dialect.rs @@ -574,6 +574,8 @@ mod tests { "PRAGMA http_proxy_password = 'v'", "PRAGMA s3_secret_access_key='v'", "SET http_proxy = 'http://u:p@proxy:3128'", + // DuckDB turns the U+200B into a space, so the E'' string ends early and the secret is a statement. + "SELECT varchar\u{200b}E'\\''; CREATE SECRET s (TYPE s3, KEY_ID 'k', SECRET 'v'); --'", ] { assert_eq!(DuckDbDialect::new().history_recording(sql), withheld, "{sql}"); } diff --git a/core/crates/drivers/duckdb/src/result_stream.rs b/core/crates/drivers/duckdb/src/result_stream.rs index 5fc49ed813..bfaaefad38 100644 --- a/core/crates/drivers/duckdb/src/result_stream.rs +++ b/core/crates/drivers/duckdb/src/result_stream.rs @@ -6,12 +6,14 @@ //! instead of DuckDB producing the whole result first. use std::ffi::{OsStr, OsString}; +use std::ops::Range; use std::path::PathBuf; use std::sync::Arc; use duckdb::arrow::array::{Array, StructArray}; use tablepro_core::column::{ColumnOrigin, ResultColumn}; use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::unicode_spaces::blanked_spaces; use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptDiagnostic, ScriptPlan, ScriptToken, ScriptTokenKind}; use tablepro_core::statement::{BoundParam, Statement}; use tablepro_core::value::Value; @@ -195,8 +197,8 @@ pub(crate) fn run_script( if traced.is_some() && controls_transaction(statement) { traced = None; } - match attach_literal(statement) { - Some(literal) => { + match attach_path(statement) { + Some(path) => { // DuckDB leaves a HIDDEN database out of what it lists, so // no check could tell another window it is open. if attach_hides(statement) { @@ -204,7 +206,7 @@ pub(crate) fn run_script( feature: UnsupportedFeature::HiddenAttach, }); } - let targets = attach_targets(link, literal)?; + let targets = attach_targets(link, statement, path)?; let ran = attach_unless_open(link, &targets, || { // A stop that came while the ATTACH waited its turn: // DuckDB clears an interrupt as a statement starts. @@ -468,11 +470,11 @@ pub(crate) fn controls_transaction(sql: &str) -> bool { /// The words an ATTACH may carry before its path. const ATTACH_WORDS: [&str; 6] = ["DATABASE", "IF", "NOT", "EXISTS", "OR", "REPLACE"]; -/// The source text of the path an ATTACH statement names, which only an -/// alias or the options may follow. DuckDB 2.0 takes an expression there: -/// a parenthesis right after a word is a call's or a type's, and the -/// options are a last group of their own. -fn attach_literal(statement: &str) -> Option<&str> { +/// Where the path an ATTACH statement names stands, which only an alias or +/// the options may follow. DuckDB 2.0 takes an expression there: a +/// parenthesis right after a word is a call's or a type's, and the options +/// are a last group of their own. +fn attach_path(statement: &str) -> Option> { let grammar = SqlGrammar::DuckDb; let plan = ScriptPlan::build(statement, grammar, LexicalSettings::default_for(grammar)); let tokens: Vec = plan @@ -513,7 +515,8 @@ fn attach_literal(statement: &str) -> Option<&str> { _ => {} } } - statement.get(start..end).map(str::trim_end) + let path = statement.get(start..end)?.trim_end(); + Some(start..start + path.len()) } /// Whether the group the parenthesis at `open` starts is the last thing @@ -543,14 +546,38 @@ fn closes_last(tokens: &[ScriptToken], open: usize) -> bool { /// /// A path that does not evaluate refuses the ATTACH: a misread boundary /// would otherwise let DuckDB open a file nothing checked. -fn attach_targets(link: &Link, literal: &str) -> Result, DriverError> { +fn attach_targets(link: &Link, statement: &str, path: Range) -> Result, DriverError> { + // DuckDB turns some non-ASCII spaces into spaces before it reads the + // ATTACH, as far as its own reading of quotes goes, and starts again + // in the query below. So the path goes in as DuckDB reads it there. + let mut literal = String::new(); + let mut at = path.start; + for space in blanked_spaces(statement) + .into_iter() + .filter(|space| path.start <= space.start && space.end <= path.end) + { + literal.push_str(statement.get(at..space.start).unwrap_or_default()); + literal.push(' '); + at = space.end; + } + literal.push_str(statement.get(at..path.end).unwrap_or_default()); // The path stands on lines of its own, so a comment after it hides // nothing. - let read = link.connection.query_row( - &format!("SELECT current_setting('home_directory'), CAST(path AS VARCHAR) FROM (SELECT\n{literal}\n) AS attach_path(path)"), - [], - |row| Ok((row.get::<_, Option>(0)?, row.get::<_, Option>(1)?)), - ); + let head = "SELECT current_setting('home_directory'), CAST(path AS VARCHAR) FROM (SELECT\n"; + let query = format!("{head}{literal}\n) AS attach_path(path)"); + // `query_row` runs every statement of its text, and the pass must + // leave the path as it is. + let in_path = head.len()..head.len() + literal.len(); + let reads_alike = one_statement(&query) + && blanked_spaces(&query) + .iter() + .all(|space| space.end <= in_path.start || space.start >= in_path.end); + if !reads_alike { + return Err(DriverError::server("the ATTACH path does not read as one value")); + } + let read = link.connection.query_row(&query, [], |row| { + Ok((row.get::<_, Option>(0)?, row.get::<_, Option>(1)?)) + }); let (home, path) = match read.map_err(map_error)? { (home, Some(path)) => (home, path), // DuckDB refuses a NULL path itself. @@ -673,6 +700,17 @@ fn expand_path(path: &str, home: Option<&OsStr>) -> PathBuf { PathBuf::from(path.get(url_offset..).unwrap_or(path)) } +/// Whether DuckDB reads `sql`, sent whole, as one statement with nothing +/// left open. With no `;` the lexer reads it as DuckDB does in one call. +fn one_statement(sql: &str) -> bool { + let grammar = SqlGrammar::DuckDb; + let plan = ScriptPlan::build(sql, grammar, LexicalSettings::default_for(grammar)); + plan.diagnostics().is_empty() + && !plan + .tokens(0..sql.len()) + .any(|token| token.kind == ScriptTokenKind::Semicolon) +} + /// The words of `sql`, upper case, past comments and outside strings /// and quoted names. fn words(sql: &str) -> Vec { @@ -749,6 +787,42 @@ mod tests { assert!(split_statements(" -- only a comment\n").is_empty()); } + // DuckDB turns U+200B and U+00A0 into spaces before it lexes, so the E'' string ends at its second quote. + #[test] + fn a_space_duckdb_blanks_ends_the_name_before_a_string() { + assert_eq!( + split_statements("SELECT varchar\u{200b}E'\\''; DROP TABLE t; --'"), + ["SELECT varchar\u{200b}E'\\''", "DROP TABLE t"] + ); + assert_eq!( + split_statements("/* ' */ SELECT 1; SELECT varchar\u{200b}E'\\''; DROP TABLE t; --'"), + ["SELECT 1", "SELECT varchar\u{200b}E'\\''", "DROP TABLE t"] + ); + assert!(controls_transaction( + "SELECT varchar\u{a0}E'\\''; COMMIT; DROP TABLE t; --'" + )); + assert_eq!(main_verb("SELECT\u{a0}1").as_deref(), Some("SELECT")); + // DuckDB gets each statement without the comment before it, so the quote in it does not count. + assert_eq!( + split_statements("/* ' */ SELECT varchar\u{200b}E'\\''; DROP TABLE t; --'"), + ["SELECT varchar\u{200b}E'\\''", "DROP TABLE t"] + ); + assert!(controls_transaction( + "SELECT 1; /* ' */ SELECT varchar\u{200b}E'\\''; COMMIT; --'" + )); + } + + #[test] + fn a_text_is_one_statement_only_without_a_semicolon_or_an_open_construct() { + assert!(one_statement("SELECT\n'p' || varchar\u{200b}E'x'\n) AS a")); + assert!(one_statement("SELECT 'a;b' -- c;\n, 1")); + assert!(!one_statement( + "SELECT\n'p' || varchar\u{200b}E'\\'') x(path); DROP TABLE t; --'\n) AS a" + )); + assert!(!one_statement("SELECT 1;")); + assert!(!one_statement("SELECT 'open")); + } + #[test] fn a_cte_named_like_a_verb_is_not_the_statement() { assert_eq!( @@ -786,6 +860,10 @@ mod tests { assert_eq!(main_verb("(SELECT 1)").as_deref(), Some("SELECT")); } + fn attach_literal(statement: &str) -> Option<&str> { + statement.get(attach_path(statement)?) + } + #[test] fn an_attach_path_runs_up_to_its_alias_or_options() { assert_eq!( diff --git a/core/crates/drivers/duckdb/tests/integration/open_semantics.rs b/core/crates/drivers/duckdb/tests/integration/open_semantics.rs index 596c5a2b91..8fe9948cb4 100644 --- a/core/crates/drivers/duckdb/tests/integration/open_semantics.rs +++ b/core/crates/drivers/duckdb/tests/integration/open_semantics.rs @@ -207,9 +207,15 @@ async fn an_exported_query_cannot_write() { let read = export("SELECT 'COMMIT' AS word").await; // DuckDB quotes an identifier in backticks, and a string that // continues an E-string on the next line keeps no backslash escape. + // It turns U+200B into a space before it lexes, except where its + // pass takes `$$` for a quote. let hidden = [ export("SELECT 1 AS `'`; COMMIT; INSERT INTO t VALUES (3); --'").await, export("SELECT E'x'\n'\\'; COMMIT; INSERT INTO t VALUES (3); --'").await, + export("SELECT varchar\u{200b}E'\\''; COMMIT; INSERT INTO t VALUES (3); --'").await, + export("SELECT 1 AS a$$, 2 AS b\u{200b}$$, varchar\u{200b}E'\\''; COMMIT; INSERT INTO t VALUES (3); --'x$$") + .await, + export("/* ' */ SELECT varchar\u{200b}E'\\''; COMMIT; INSERT INTO t VALUES (3); --'").await, ]; assert_eq!( @@ -232,6 +238,53 @@ async fn an_exported_query_cannot_write() { connection.close().await.unwrap(); } +/// The path of an ATTACH is read again in a query of its own, where +/// DuckDB's pass over non-ASCII spaces starts afresh: a U+200B the ATTACH +/// kept, behind the quote in its comment, ends a name there, and the rest +/// of the path would run as statements of their own. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_attach_path_read_again_runs_nothing_else() { + let file = TestFile::new(); + let connection = file.open().await; + apply(connection.as_ref(), &["CREATE TABLE t (a INTEGER)"]) + .await + .unwrap(); + let editor = session(connection.as_ref()).await; + + let sql = + "ATTACH /* ' */ 'p' || varchar\u{200b}E'\\'') x(path); DROP TABLE t; SELECT * FROM (SELECT 1 AS path --' AS a"; + let (summary, _) = run(editor.as_ref(), sql).await; + + assert!(matches!(summary.end, RunEnd::Failed(_)), "{:?}", summary.end); + let rows = query(connection.as_ref(), "SELECT count(*) FROM t").await; + assert_eq!(rows.rows, vec![vec![Value::Int(0)]]); + editor.close().await; + connection.close().await.unwrap(); +} + +/// The quote in the comment puts DuckDB's pass outside the string, so it +/// turns the U+00A0 into a space and opens the file another window has. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_attach_path_names_the_file_duckdb_opens_after_its_spaces() { + let file = TestFile::new(); + let other = TestFile::new(); + let spaced = other.path.with_file_name("a b.duckdb"); + std::fs::rename(&other.path, &spaced).unwrap(); + let connection = file.open().await; + let opened = connect(FileEndpoint::open_existing(spaced.clone()).unwrap(), writable()) + .await + .unwrap(); + let editor = session(connection.as_ref()).await; + + let path = spaced.display().to_string().replace(' ', "\u{a0}"); + let (summary, _) = run(editor.as_ref(), &format!("ATTACH /* ' */ '{path}' AS o")).await; + + assert_eq!(summary.end, RunEnd::Failed(DriverError::Busy(BusyReason::FileLocked))); + editor.close().await; + opened.close().await.unwrap(); + connection.close().await.unwrap(); +} + /// A hard link is the same file, so it joins the database already open. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn a_second_name_for_the_file_shares_its_database() { diff --git a/core/crates/mcp/src/query_tools.rs b/core/crates/mcp/src/query_tools.rs index c1364a03c7..bd01a4e1ef 100644 --- a/core/crates/mcp/src/query_tools.rs +++ b/core/crates/mcp/src/query_tools.rs @@ -465,6 +465,16 @@ mod tests { ) .is_err() ); + // DuckDB turns the U+200B into a space before it lexes and runs two statements. + assert!(single_batch("SELECT varchar\u{200b}E'\\''; DROP TABLE t; --'", SqlGrammar::DuckDb).is_err()); + assert!( + single_batch( + "/* ' */ SELECT varchar\u{200b}E'\\''; DROP TABLE t; --'", + SqlGrammar::DuckDb + ) + .is_err() + ); + assert!(single_batch("SELECT\u{a0}1", SqlGrammar::DuckDb).is_ok()); } #[test] diff --git a/linux/changelog.d/4239.security.md b/linux/changelog.d/4239.security.md new file mode 100644 index 0000000000..d298797ef7 --- /dev/null +++ b/linux/changelog.d/4239.security.md @@ -0,0 +1 @@ +Safe Mode missed a DuckDB statement hidden after a non-breaking or zero-width space.