Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 64 additions & 5 deletions core/crates/core/src/sql_syntax/script/lexer.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
use std::cell::RefCell;
use std::collections::HashSet;
use std::ops::Range;

use crate::sql_syntax::script::script_rules::{
BackslashPolicy, BracketIdentifier, DollarQuote, HashComment, QuoteKind, QuoteRule, ScriptSyntax,
};
use crate::sql_syntax::script::server_reading::{CommentRead, NameQuotes, ServerReading, dollar_tag_end, read_at};
use crate::sql_syntax::script::unicode_spaces::{self, UnicodeSpaces};
use crate::sql_syntax::script::{LexicalSettings, OpenConstruct, ScriptTokenKind};

/// MariaDB's MSSQL mode ends a `[name]` at a `]` that is not doubled.
Expand All @@ -30,6 +32,11 @@ pub(crate) struct Lexer<'a> {
missing_heredoc_closers: HashSet<&'a str>,
/// Where the executable comment the lexer is inside opened.
executable: Option<usize>,
/// DuckDB's pass over the statement being read, when the text holds a
/// space it may blank.
unicode_spaces: Option<RefCell<UnicodeSpaces>>,
/// The statement being read has had its first code token.
in_statement: bool,
}

impl<'a> Lexer<'a> {
Expand All @@ -48,6 +55,11 @@ impl<'a> Lexer<'a> {
pos: 0,
missing_heredoc_closers: HashSet::new(),
executable: None,
unicode_spaces: (syntax.blanks_unicode_spaces
&& reading.blanks_unicode_spaces()
&& unicode_spaces::any(text.as_bytes()))
.then(|| RefCell::new(UnicodeSpaces::new(0))),
in_statement: false,
}
}

Expand All @@ -70,6 +82,13 @@ impl<'a> Lexer<'a> {
pub fn next_token(&mut self, delimiter: Option<&str>) -> Option<LexedToken> {
let start = self.pos;
let first = *self.bytes.get(start)?;
// Each statement goes to DuckDB from its first code token on, and
// DuckDB starts its pass there.
if !self.in_statement
&& let Some(spaces) = &self.unicode_spaces
{
*spaces.borrow_mut() = UnicodeSpaces::new(start);
}
let token = match delimiter {
// The server ends an executable comment at the first `*/`
// that stands where a token could start.
Expand All @@ -83,6 +102,11 @@ impl<'a> Lexer<'a> {
_ => self.lex_at(start, first, delimiter),
};
self.pos = token.range.end;
if token.kind == ScriptTokenKind::Semicolon {
self.in_statement = false;
} else if !token.kind.is_trivia() {
self.in_statement = true;
}
Some(token)
}

Expand All @@ -95,6 +119,7 @@ impl<'a> Lexer<'a> {
}
match first {
b if is_space(b) => self.plain(ScriptTokenKind::Whitespace, self.whitespace_end(start)),
_ if self.blank_at(start).is_some() => self.plain(ScriptTokenKind::Whitespace, self.whitespace_end(start)),
0xEF if self.syntax.byte_order_mark_is_space && self.bytes[start..].starts_with(BYTE_ORDER_MARK) => {
self.plain(ScriptTokenKind::Whitespace, start + BYTE_ORDER_MARK.len())
}
Expand Down Expand Up @@ -141,6 +166,10 @@ impl<'a> Lexer<'a> {
fn whitespace_end(&self, start: usize) -> usize {
let mut end = start;
while let Some(&b) = self.bytes.get(end) {
if let Some(len) = self.blank_at(end) {
end += len;
continue;
}
if !is_space(b) {
break;
}
Expand Down Expand Up @@ -359,8 +388,8 @@ impl<'a> Lexer<'a> {
DollarQuote::Tagged => {
if let Some(opener_end) = self.tagged_opener_end(start) {
let opener = &self.text[start..opener_end];
return match self.text[opener_end..].find(opener) {
Some(offset) => self.plain(ScriptTokenKind::DollarQuoted, opener_end + offset + opener.len()),
return match self.closer(opener_end, opener) {
Some(closer) => self.plain(ScriptTokenKind::DollarQuoted, closer + opener.len()),
None => self.unterminated(start, OpenConstruct::DollarQuoted),
};
}
Expand Down Expand Up @@ -452,13 +481,13 @@ impl<'a> Lexer<'a> {
let mut index = start + 1;
let first = *self.bytes.get(index)?;
if first != b'$' {
if !(first.is_ascii_alphabetic() || first == b'_' || first >= 0x80) {
if !(first.is_ascii_alphabetic() || first == b'_' || self.non_ascii_name_byte(index)) {
return None;
}
while self
.bytes
.get(index)
.is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'_' || *b >= 0x80)
.is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'_' || self.non_ascii_name_byte(index))
{
index += 1;
}
Expand All @@ -469,6 +498,21 @@ impl<'a> Lexer<'a> {
Some(index + 1)
}

/// Where the first `opener` after `from` starts. One that holds a space
/// DuckDB's pass blanks is no longer the opener's text.
fn closer(&self, from: usize, opener: &str) -> Option<usize> {
let holds_space = (0..opener.len()).any(|at| unicode_spaces::space_len(opener.as_bytes(), at).is_some());
let mut search = from;
loop {
let at = search + self.text.get(search..)?.find(opener)?;
if !holds_space || (at..at + opener.len()).all(|byte| self.blank_at(byte).is_none()) {
return Some(at);
}
// The opener starts with `$`, so the next byte starts a character.
search = at + 1;
}
}

fn heredoc_end(&mut self, start: usize) -> Option<usize> {
let close = find_byte(self.bytes, start + 1, b'$')?;
let opener = &self.text[start..=close];
Expand Down Expand Up @@ -500,7 +544,10 @@ impl<'a> Lexer<'a> {
let delimiter = delimiter.filter(|d| d.bytes().next().is_some_and(|b| self.is_word_part(b)));
let mut end = from;
while let Some(&b) = self.bytes.get(end) {
if !self.is_word_part(b) || delimiter.is_some_and(|d| self.bytes[end..].starts_with(d.as_bytes())) {
if !self.is_word_part(b)
|| (b >= 0x80 && self.blank_at(end).is_some())
|| delimiter.is_some_and(|d| self.bytes[end..].starts_with(d.as_bytes()))
{
break;
}
end += 1;
Expand Down Expand Up @@ -555,6 +602,18 @@ impl<'a> Lexer<'a> {
fn is_word_part(&self, b: u8) -> bool {
b.is_ascii_alphanumeric() || b == b'_' || b >= 0x80 || self.syntax.word_part_extra.contains(&b)
}

fn non_ascii_name_byte(&self, at: usize) -> bool {
self.bytes.get(at).is_some_and(|&b| b >= 0x80) && self.blank_at(at).is_none()
}

/// How long the non-ASCII space at `at` is, when DuckDB's pass turns it
/// into a space.
fn blank_at(&self, at: usize) -> Option<usize> {
let spaces = self.unicode_spaces.as_ref()?;
let len = unicode_spaces::space_len(self.bytes, at)?;
spaces.borrow_mut().blanks(self.bytes, at).then_some(len)
}
}

fn is_space(b: u8) -> bool {
Expand Down
1 change: 1 addition & 0 deletions core/crates/core/src/sql_syntax/script/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ mod script_token;
mod script_token_kind;
pub(crate) mod server_reading;
pub mod text_offsets;
pub mod unicode_spaces;

pub use batch_error_policy::BatchErrorPolicy;
pub use lexical_settings::LexicalSettings;
Expand Down
3 changes: 2 additions & 1 deletion core/crates/core/src/sql_syntax/script/script_plan.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,8 @@ impl ScriptPlan {
/// Every plan the client or a server may make of `text`: as the client
/// splits it and, on MySQL, as each server reads what it is sent, at
/// each version its executable comments name. On SQL Server, with a
/// `--` after a number's `e` a comment and not. Each with backslash
/// `--` after a number's `e` a comment and not. On DuckDB, with its
/// non-ASCII spaces blanked as its pass does and kept. Each with backslash
/// escapes on and off, since a session setting decides. Built one at a
/// time. `None` when the text names too many versions to try.
pub fn server_readings(text: &str, grammar: SqlGrammar) -> Option<impl Iterator<Item = ScriptPlan> + '_> {
Expand Down
14 changes: 14 additions & 0 deletions core/crates/core/src/sql_syntax/script/script_rules.rs
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,12 @@ pub(crate) struct ScriptSyntax {
/// SQLite's `sqlite3GetToken` reads it. Inside a name it stays a name
/// character, there as elsewhere.
pub byte_order_mark_is_space: bool,
/// DuckDB turns its non-ASCII spaces into spaces before it lexes, where
/// its own reading of quotes is outside one (`unicode_spaces`). The
/// driver sends each statement alone, from its first code token, and
/// DuckDB reads each text it gets afresh, so the reading starts again
/// there.
pub blanks_unicode_spaces: bool,
pub word_start_extra: &'static [u8],
pub word_part_extra: &'static [u8],
}
Expand Down Expand Up @@ -132,6 +138,7 @@ const POSTGRESQL: ScriptRules = ScriptRules {
digit_separators: true,
tcl_parameters: false,
byte_order_mark_is_space: false,
blanks_unicode_spaces: false,
word_start_extra: b"",
word_part_extra: b"$",
},
Expand Down Expand Up @@ -159,6 +166,7 @@ const MYSQL: ScriptRules = ScriptRules {
digit_separators: false,
tcl_parameters: false,
byte_order_mark_is_space: false,
blanks_unicode_spaces: false,
word_start_extra: b"$",
word_part_extra: b"$",
},
Expand Down Expand Up @@ -186,6 +194,7 @@ const SQLITE: ScriptRules = ScriptRules {
digit_separators: true,
tcl_parameters: true,
byte_order_mark_is_space: true,
blanks_unicode_spaces: false,
word_start_extra: b"$",
word_part_extra: b"$",
},
Expand Down Expand Up @@ -213,6 +222,7 @@ const MSSQL: ScriptRules = ScriptRules {
digit_separators: false,
tcl_parameters: false,
byte_order_mark_is_space: false,
blanks_unicode_spaces: false,
word_start_extra: b"@#",
word_part_extra: b"@#$",
},
Expand Down Expand Up @@ -240,6 +250,7 @@ const CLICKHOUSE: ScriptRules = ScriptRules {
digit_separators: true,
tcl_parameters: false,
byte_order_mark_is_space: false,
blanks_unicode_spaces: false,
word_start_extra: b"",
word_part_extra: b"",
},
Expand Down Expand Up @@ -267,6 +278,7 @@ const CQL: ScriptRules = ScriptRules {
digit_separators: false,
tcl_parameters: false,
byte_order_mark_is_space: false,
blanks_unicode_spaces: false,
word_start_extra: b"",
word_part_extra: b"",
},
Expand All @@ -275,6 +287,7 @@ const CQL: ScriptRules = ScriptRules {
/// DuckDB 2.0's PEG tokenizer: a string escapes only with an `E` prefix
/// and ends at its quote, with no continuation on the next line; block
/// comments nest, `$tag$` quotes, and a backtick quotes a name as `"` does.
/// A byte from 0x80 up is a name byte, except a space its pass blanks.
const DUCKDB: ScriptRules = ScriptRules {
boundary: Boundary::Semicolon,
compound: CompoundRule::None,
Expand All @@ -297,6 +310,7 @@ const DUCKDB: ScriptRules = ScriptRules {
digit_separators: true,
tcl_parameters: false,
byte_order_mark_is_space: false,
blanks_unicode_spaces: true,
word_start_extra: b"",
word_part_extra: b"$",
},
Expand Down
36 changes: 32 additions & 4 deletions core/crates/core/src/sql_syntax/script/server_reading.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@ use std::collections::{BTreeSet, HashSet};
use std::ops::RangeInclusive;

use crate::sql_syntax::SqlGrammar;
use crate::sql_syntax::script::unicode_spaces;

/// How the lexer reads the text whose meaning hangs on who reads it. On
/// MySQL, an executable comment (`/*! … */`, `/*!50100 … */`, MariaDB's
/// `/*M!100100 … */`), a `$tag$` quote and a `DELIMITER` line. On SQL
/// Server, a `--` after a number's `e`.
/// Server, a `--` after a number's `e`. On DuckDB, a non-ASCII space.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum ServerReading {
/// As the mysql client reads a script to split it: every executable
Expand All @@ -26,13 +27,22 @@ pub(crate) enum ServerReading {
/// and the `--` of `1E--1` would be that sign and a minus where the
/// client reads a comment. Nobody measured that, so both are read.
SignedEmptyExponents,
/// As the client reads it, but with no DuckDB pass over its non-ASCII
/// spaces: each stays a name character, as DuckDB reads one the pass
/// keeps. The pass is mirrored from one DuckDB version, so a gate also
/// reads the text without it.
UnicodeSpacesInNames,
}

impl ServerReading {
pub(crate) fn reads_directives(self) -> bool {
!matches!(self, Self::Server { .. })
}

pub(crate) fn blanks_unicode_spaces(self) -> bool {
self != Self::UnicodeSpacesInNames
}

pub(crate) fn signs_empty_exponents(self) -> bool {
self == Self::SignedEmptyExponents
}
Expand All @@ -49,7 +59,7 @@ impl ServerReading {

pub(crate) fn name_quotes(self) -> NameQuotes {
match self {
Self::Client | Self::SignedEmptyExponents => NameQuotes::Default,
Self::Client | Self::SignedEmptyExponents | Self::UnicodeSpacesInNames => NameQuotes::Default,
Self::Server { quotes, .. } => quotes,
}
}
Expand Down Expand Up @@ -201,13 +211,17 @@ pub(crate) fn dollar_tag_end(bytes: &[u8], at: usize) -> Option<usize> {
/// The client's reading of `text`, then each distinct one the servers it
/// may meet give it: each kind of server at version 0 and at each version
/// the text's comments name, since only those change which comments run.
/// On SQL Server, one more where a `--` follows an `e`. `None` when there
/// are more than are worth trying.
/// On SQL Server, one more where a `--` follows an `e`. On DuckDB, one more
/// without its pass when the text holds a space the pass may blank. `None`
/// when there are more than are worth trying.
pub(crate) fn server_readings(text: &str, grammar: SqlGrammar) -> Option<Vec<ServerReading>> {
let mut readings = vec![ServerReading::Client];
if grammar == SqlGrammar::MsSql && (text.contains("e--") || text.contains("E--")) {
readings.push(ServerReading::SignedEmptyExponents);
}
if grammar == SqlGrammar::DuckDb && unicode_spaces::any(text.as_bytes()) {
readings.push(ServerReading::UnicodeSpacesInNames);
}
if grammar != SqlGrammar::MySql {
return Some(readings);
}
Expand Down Expand Up @@ -446,6 +460,20 @@ mod tests {
server_readings("SELECT /*! 1 */", SqlGrammar::PostgreSql),
Some(vec![ServerReading::Client])
);
assert_eq!(
server_readings("SELECT\u{a0}1", SqlGrammar::DuckDb),
Some(vec![ServerReading::Client, ServerReading::UnicodeSpacesInNames])
);
for (text, grammar) in [
("SELECT 'caf\u{e9}'", SqlGrammar::DuckDb),
("SELECT\u{a0}1", SqlGrammar::PostgreSql),
] {
assert_eq!(
server_readings(text, grammar),
Some(vec![ServerReading::Client]),
"{text:?}"
);
}
}

#[test]
Expand Down
Loading
Loading