Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,12 @@ public struct SQLLexicalGrammar: OptionSet, Hashable, Sendable {
/// See ``SQLPsqlScript``. Curated only.
public static let psqlScripts = SQLLexicalGrammar(rawValue: 1 << 41)

/// The non-ASCII spaces ``SQLNonCodeSpan/spacesBeforeLexing`` lists end a name, as DuckDB turns them into spaces
/// before it lexes where its pre-lexing pass sees no quote, so `x` + U+200B + `E'...'` is a name and an escape
/// string. PostgreSQL reads every byte from 0x80 up as part of a name. Measured on DuckDB 1.5.4 and 2.0.0 and
/// PostgreSQL 17.11. That pass reads quotes its own way, so DuckDB's profile also doubts this fact. Curated only.
public static let nonASCIISpacesEndNames = SQLLexicalGrammar(rawValue: 1 << 42)

/// Standard SQL: quotes close only on a doubled quote, `--` and flat `/* */` are the only comments.
public static let ansi: SQLLexicalGrammar = []

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,10 +127,11 @@ public struct SQLLexicalProfile: Sendable, Hashable {
/// quotes a name and a `'...'` after a line break is a string of its own, not part of an `E'...'`. 1.5.4 continues
/// the `E'...'` as PostgreSQL does, a registry plugin built before 2.0 links 1.5, and `enable_legacy_parser` brings
/// 1.5's parser back on 2.0, so a gate reads both. It has no `BEGIN ... END` body, so it takes PostgreSQL's
/// narrower fact.
/// narrower fact. Both versions turn the non-ASCII spaces into blanks before they lex, outside the quotes their
/// pre-lexing pass sees.
static let duckDB: SQLLexicalGrammar = [
.taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix, .backtickQuotes,
.carriageReturnEndsLineComments, .atomicRoutineBodies,
.carriageReturnEndsLineComments, .atomicRoutineBodies, .nonASCIISpacesEndNames,
]

/// Oracle 23.26, measured: a backslash is literal, block comments do not nest, `q'[...]'` is a literal, `$` and
Expand Down Expand Up @@ -295,7 +296,13 @@ public struct SQLLexicalProfile: Sendable, Hashable {
"libSQL": sqliteFamily,
"Turso": sqliteFamily,
"Cloudflare D1": sqliteFamily,
"DuckDB": SQLLexicalProfile(grammar: duckDB, undetermined: [.escapeStringContinuations]),
// DuckDB blanks a non-ASCII space only where its own pre-lexing pass is outside a quote, and that pass reads
// quotes its own way: after `a$$`, `E'\''` or `/* ' */` it keeps the space, a name character again
// (measured by hand on 1.5.4 and 2.0.0; the probe checks the execution reading only). So a gate reads both.
"DuckDB": SQLLexicalProfile(
grammar: duckDB,
undetermined: [.escapeStringContinuations, .nonASCIISpacesEndNames]
),
"Oracle": SQLLexicalProfile(grammar: oracle),
"Dameng": SQLLexicalProfile(
grammar: dameng,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,8 @@ enum SQLLexicalRelevance {
if next == SqlLexer.singleQuote { facts.insert(.alternativeQuoting) }
case capitalG, smallG:
facts.insert(.batchSeparatorLines)
case let space where space >= 0xA0 && SQLNonCodeSpan.spacesBeforeLexing.contains(space):
facts.insert(.nonASCIISpacesEndNames)
default:
break
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -119,15 +119,36 @@ public enum SQLNonCodeSpan {
return span.end
}

/// Whether `unit` continues a word, so a prefix like `E` or `q` glued to it is part of an identifier rather than
/// the start of a literal.
public static func isWordUnit(_ unit: UInt16) -> Bool {
if unit < 0x80 {
return SqlDollarQuote.isIdentifierPart(unit)
}
return !SQLSeparatingCharacter.isSeparating(unit)
}

/// Whether a literal prefix right after `unit` is part of the name `unit` belongs to, so `x$e'\'` is the name
/// `x$e` and a plain string. PostgreSQL 17.11 continues a name with `$` and every unit from 0x80 up, and DuckDB
/// 1.5.4 and 2.0.0 do too, less the spaces they blank (measured). CockroachDB's scanner reads bytes the same way, and
/// KingbaseES, a PostgreSQL fork, is taken to. Oracle and Dameng document `$` and `#` as name characters, and run
/// one statement per call.
public static func continuesName(beforePrefix unit: UInt16, grammar: SQLLexicalGrammar) -> Bool {
if unit >= 0x80 {
return !endsNameAsBlank(unit, grammar: grammar)
}
return SqlDollarQuote.isIdentifierPart(unit) || unit == SqlDollarQuote.dollar
|| (unit == SqlLexer.hash && grammar.contains(.dollarAndHashInIdentifiers))
}

/// Whether a `$` right after `unit` is part of the name `unit` belongs to, as in `a$$b`, rather than a dollar
/// quote's opener. On DuckDB `SELECT` + U+200B + `$$...$$` opens one, measured on 2.0.0.
public static func continuesName(beforeDollar unit: UInt16, grammar: SQLLexicalGrammar) -> Bool {
SqlDollarQuote.isIdentifierContinuation(unit) && !endsNameAsBlank(unit, grammar: grammar)
}

private static func endsNameAsBlank(_ unit: UInt16, grammar: SQLLexicalGrammar) -> Bool {
grammar.contains(.nonASCIISpacesEndNames) && spacesBeforeLexing.contains(unit)
}

// MARK: - Comments

private static func commentSpan(
Expand Down Expand Up @@ -272,16 +293,18 @@ public enum SQLNonCodeSpan {
return closedSpan(.quoted, at: index, span, closerLength: 1)
}

/// `E'...'`, `b'...'` and `q'[...]'`, which only start where a word could: `xE'` is an identifier followed by a
/// string.
/// `E'...'`, `b'...'` and `q'[...]'`, which only start where a name does not go on: `xE'` and `x$E'` are a name
/// followed by a string.
private static func prefixedLiteralSpan(
at index: Int,
character: UInt16,
in text: NSString,
length: Int,
grammar: SQLLexicalGrammar
) -> Span? {
guard index == 0 || !isWordUnit(text.character(at: index - 1)) else { return nil }
guard index == 0 || !continuesName(beforePrefix: text.character(at: index - 1), grammar: grammar) else {
return nil
}
let startsEscapeString = grammar.contains(.escapeStringPrefix) && (character == capitalE || character == smallE)
let startsEscapeBytes = grammar.contains(.escapeBytesPrefix) && character == smallB
if startsEscapeString || startsEscapeBytes,
Expand Down Expand Up @@ -327,9 +350,10 @@ public enum SQLNonCodeSpan {

private static let verticalTab: UInt16 = 0x0B

// DuckDB 1.5 turns these into spaces before it lexes (measured on 1.5.4). PostgreSQL and CockroachDB do not, and
// the text a join there leaves them does not parse, so they run none of it.
private static let spacesBeforeLexing: Set<UInt16> = Set([0x00A0, 0x202F, 0x205F, 0x2060, 0x3000, 0xFEFF])
// DuckDB turns these into spaces before it lexes, outside what its pre-lexing pass reads as a quote, and no other
// non-ASCII unit (measured on 1.5.4 and 2.0.0).
// PostgreSQL and CockroachDB do not, and the text a join there leaves them does not parse, so they run none of it.
static let spacesBeforeLexing: Set<UInt16> = Set([0x00A0, 0x202F, 0x205F, 0x2060, 0x3000, 0xFEFF])
.union(0x2000...0x200B)

/// The quote that carries on the string closed just before `index`, as PostgreSQL's `quotecontinue` finds it: only
Expand Down Expand Up @@ -376,7 +400,8 @@ public enum SQLNonCodeSpan {
at: index,
in: text,
bufLen: length,
style: style
style: style,
continuesName: { SQLNonCodeSpan.continuesName(beforeDollar: $0, grammar: grammar) }
)
else {
return nil
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,16 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking {
case list(depth: Int)
}

// How a routine's body began outside a block. A statement never holds a block. After a `RETURN` a block opens
// only behind `AS` or `IS`, as the header `RETURN INT AS BEGIN` writes it. A control statement such as `IF` may
// hold blocks, and statement words inside it begin nothing.
private enum RoutineBody {
case unknown
case statement
case returnValue
case controlStatement
}

private enum Header {
case opener
/// After `CREATE`, `ALTER` or `REPLACE`. `accountWords` is what is left of an unquoted `DEFINER = user@host`.
Expand All @@ -34,10 +44,23 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking {

private static let dot = UInt16(UnicodeScalar(".").value)
private static let colon = UInt16(UnicodeScalar(":").value)
private static let comma = UInt16(UnicodeScalar(",").value)

/// The words a statement inside a body can follow without a `;`, as in `IF x THEN BEGIN ... END; END IF`.
private static let statementIntroducers: Set<String> = ["THEN", "ELSE", "DO", "LOOP", "REPEAT"]

/// The words that open a routine body written as one statement, as in MySQL's
/// `CREATE PROCEDURE p() SELECT ... ORDER BY begin DESC`, where `begin` is a column.
private static let singleStatementBodyOpeners: Set<String> = [
"SELECT", "INSERT", "UPDATE", "DELETE", "REPLACE", "MERGE",
]

/// A trigger's event follows these, as in `BEFORE INSERT`, `INSTEAD OF DELETE` and `INSERT OR UPDATE`, so the
/// word after one is part of the header rather than the body.
private static let triggerEventLeaders: Set<String> = ["BEFORE", "AFTER", "OF", "OR", "FOR"]

private static let controlStatementOpeners: Set<String> = ["IF", "WHILE", "LOOP", "REPEAT"]

private let grammar: SQLLexicalGrammar
private let readsBlocksEverywhere: Bool
// T-SQL outside a routine: where a block's `BEGIN` can stand, as the statement's first word, after `IF`, `WHILE`
Expand All @@ -50,6 +73,14 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking {
private var header = Header.opener
private var definesRoutine = false
private var openedBlockBody = false
private var routineBody = RoutineBody.unknown
private var previousWord: String?
// A trigger's events stand before its table, as in SQLite's `CREATE TRIGGER tr DELETE ON t`.
private var readsTriggerEvents = false
// A trigger's event list goes on after a word in `triggerEventLeaders`, or after a `,` that follows an event, as
// in `FOR INSERT, UPDATE`.
private var expectsTriggerEvent = false
private var followsTriggerEvent = false
private var depth = 0

/// The open `CASE`s that have held no `;`, which only a `CASE` statement can. The next `END` closes one of them,
Expand Down Expand Up @@ -102,6 +133,11 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking {

public mutating func observeWord(_ word: String) {
merge?.observeWord(word)
let previous = previousWord
previousWord = word
let isTriggerEvent = expectsTriggerEvent && Self.singleStatementBodyOpeners.contains(word)
expectsTriggerEvent = Self.triggerEventLeaders.contains(word)
followsTriggerEvent = isTriggerEvent
let isMember = followsDot
followsName = true
followsDot = false
Expand All @@ -124,13 +160,22 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking {
observeRuleWord(word)
}
guard parenDepth == 0, readsBlocksEverywhere || (pastHeader && definesRoutine) else { return }
if !readsBlocksEverywhere, depth == 0 {
observeBodyStart(word, after: previous, isTriggerEvent: isTriggerEvent)
}
switch word {
case "BEGIN" where readsBlocksEverywhere:
pendingBegin = true
pendingBeginMayOpen = opensNestedBlock(atStatementPosition: statementPosition, isFirstWord: isFirstWord)
case "BEGIN" where opensNestedBlock(atStatementPosition: statementPosition, isFirstWord: isFirstWord):
pendingBegin = true
pendingBeginMayOpen = true
pendingBeginMayOpen = opensNestedBlock(
atStatementPosition: statementPosition,
isFirstWord: isFirstWord,
after: previous
)
case "BEGIN":
if opensNestedBlock(atStatementPosition: statementPosition, isFirstWord: isFirstWord, after: previous) {
pendingBegin = true
pendingBeginMayOpen = true
}
case "IF" where readsBlocksEverywhere:
followsControlFlow = true
if depth == 0 {
Expand All @@ -155,6 +200,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking {

public mutating func observeSymbol(_ symbol: UInt16) {
merge?.observeSymbol(symbol)
previousWord = nil
expectsTriggerEvent = symbol == Self.comma && followsTriggerEvent
followsTriggerEvent = false
if ruleAction == .afterDo, symbol != SqlLexer.openParen {
ruleAction = .none
}
Expand Down Expand Up @@ -184,6 +232,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking {

public mutating func observeOpaqueToken() {
merge?.observeOpaqueToken()
previousWord = nil
expectsTriggerEvent = false
followsTriggerEvent = false
if ruleAction == .afterDo {
ruleAction = .none
}
Expand All @@ -201,6 +252,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking {

public mutating func observeSemicolon() -> Bool {
pendingBegin = false
previousWord = nil
expectsTriggerEvent = false
followsTriggerEvent = false
if pendingEnd {
pendingEnd = false
closeBlock()
Expand Down Expand Up @@ -273,6 +327,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking {
}
if SqlBlockStructure.isRoutineKind(word, grammar: grammar) {
definesRoutine = true
readsTriggerEvents = word == "TRIGGER"
header = .name
} else if word == "DEFINER" {
header = .kind(accountWords: 2)
Expand Down Expand Up @@ -303,12 +358,36 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking {
}
}

private func opensNestedBlock(atStatementPosition: Bool, isFirstWord: Bool) -> Bool {
private func opensNestedBlock(atStatementPosition: Bool, isFirstWord: Bool, after previous: String?) -> Bool {
if readsBlocksEverywhere {
return depth > 0 || definesRoutine || isFirstWord || followsControlFlow || openedBlockBody
}
guard depth > 0, !grammar.hasSingleBodyForm else { return true }
return atStatementPosition || inHandlerDeclaration
if grammar.hasSingleBodyForm { return true }
switch routineBody {
case .statement:
return false
case .returnValue:
return depth == 0 && (previous == "AS" || previous == "IS")
case .unknown, .controlStatement:
guard depth > 0 else { return true }
return atStatementPosition || inHandlerDeclaration
}
}

// Where a routine's body is one statement, the word that starts it.
private mutating func observeBodyStart(_ word: String, after previous: String?, isTriggerEvent: Bool) {
if word == "ON" {
readsTriggerEvents = false
}
guard routineBody == .unknown else { return }
if Self.controlStatementOpeners.contains(word) {
routineBody = .controlStatement
} else if word == "RETURN", previous != "ON" {
// DB2's `COMMIT ON RETURN NO` is a header clause.
routineBody = .returnValue
} else if Self.singleStatementBodyOpeners.contains(word), !isTriggerEvent, !readsTriggerEvents {
routineBody = .statement
}
}

private mutating func observeRuleWord(_ word: String) {
Expand Down Expand Up @@ -388,6 +467,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking {
// A trigger body holds no other; `BEGIN ATOMIC` is never a name, so a routine defined inside one nests.
private mutating func openBlockBody() {
guard !grammar.contains(.triggerBodies) || depth == 0 else { return }
if routineBody == .returnValue {
routineBody = .unknown
}
depth += 1
openedBlockBody = true
startsBodyStatement = true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,19 @@ public enum SqlDollarQuote {
}

/// Resolves a `$` at `pos` to a dollar-quote opener, a positional parameter like `$1`, or a non-tag dollar. A `$`
/// glued to a preceding identifier is not an opener. Returns `needsMoreData` when the buffer ends mid-tag; a
/// whole-string caller treats that as `notOpener`.
/// glued to a preceding identifier, as `continuesName` reads the unit before it, is not an opener. Returns
/// `needsMoreData` when the buffer ends mid-tag; a whole-string caller treats that as `notOpener`.
///
/// A tag starts with a letter or an underscore and continues with letters, digits and underscores, where a letter
/// is any non-ASCII character as well, which is how PostgreSQL accepts `$ü$`.
public static func scanOpener(at pos: Int, in buffer: NSString, bufLen: Int, style: Style) -> Opener {
if pos > 0, isIdentifierContinuation(buffer.character(at: pos - 1)) {
public static func scanOpener(
at pos: Int,
in buffer: NSString,
bufLen: Int,
style: Style,
continuesName: (unichar) -> Bool = SqlDollarQuote.isIdentifierContinuation
) -> Opener {
if pos > 0, continuesName(buffer.character(at: pos - 1)) {
return .notOpener
}
guard pos + 1 < bufLen else { return .needsMoreData }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,32 @@ struct SQLStatementSplittingTests {
).count == 1)
}

// Measured on DuckDB 1.5.4 and 2.0.0, the engine whose plugin reads E'...' through the kit.
@Test("A prefix glued to a name is part of it, and the non-ASCII spaces DuckDB blanks end the name")
func prefixAfterAName() {
let duckDB: SQLLexicalFeatures = [.taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix, .backtickQuotes]
for glue in ["$", "\u{E9}", "\u{2019}", "\u{FF0C}"] {
#expect(SQLStatementSplitting.statements(
in: "SELECT x\(glue)e'\\'; DROP TABLE t; SELECT 'x'",
lexicalFeatures: duckDB
).count == 3, "\(glue)")
}
for space in ["\u{00A0}", "\u{200B}", "\u{2060}", "\u{3000}", "\u{FEFF}"] {
#expect(SQLStatementSplitting.statements(
in: "SELECT varchar\(space)E'\\''; DROP TABLE t; SELECT 'x'",
lexicalFeatures: duckDB
).count == 3, "\(space.unicodeScalars.first.map { String($0.value, radix: 16) } ?? "")")
}
#expect(SQLStatementSplitting.statements(
in: "SELECT 1 AS a,\u{200B}$$'$$; DROP TABLE t; SELECT 'x'",
lexicalFeatures: duckDB
).count == 3)
let oracle: SQLLexicalFeatures = [.alternativeQuoting, .dollarAndHashInIdentifiers]
#expect(SQLStatementSplitting.statements(in: "SELECT x#q'['; SELECT 2 --]'", lexicalFeatures: oracle).count == 2)
#expect(SQLStatementSplitting.statements(in: "SELECT x#q'['; SELECT 2 --]'", lexicalFeatures: .alternativeQuoting)
.count == 1)
}

@Test("SQLite's $name(...) parameter keeps a semicolon, and a $ inside a name starts none")
func featuresReadParenthesizedParameters() {
#expect(SQLStatementSplitting.statements(in: "SELECT $a(;)", lexicalFeatures: .parenthesizedParameterNames)
Expand Down
Loading
Loading