Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,12 @@ public struct SQLLexicalGrammar: OptionSet, Hashable, Sendable {
// `BT` and `ET` begin and end a transaction, as Teradata reads them; anywhere else they are names. Curated only.
public static let transactionShortForms = SQLLexicalGrammar(rawValue: 1 << 43)

/// A routine's body is one statement, as MySQL's `routine_body` reads it, and a compound one holds the `;`s inside
/// it: `IF ... END IF`, `WHILE ... END WHILE`, `LOOP ... END LOOP` and `REPEAT ... END REPEAT` as `BEGIN ... END`
/// does. A `SET` outside the return type starts the body, so a `begin` after it is a name. Measured on MariaDB
/// 13.0.2. Curated only.
public static let compoundStatementBodies = SQLLexicalGrammar(rawValue: 1 << 44)

/// Standard SQL: quotes close only on a doubled quote, `--` and flat `/* */` are the only comments.
public static let ansi: SQLLexicalGrammar = []

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,10 +68,11 @@ public struct SQLLexicalProfile: Sendable, Hashable {
/// own; an engine that runs statements without one says so through its plugin. A `MERGE` that keeps its `;` is left
/// out for the same reason: an engine that refuses one without it says so through its plugin. So is a CQL batch,
/// which would read one statement where an unknown engine may run several. So are the two single body forms, which
/// would cut the `BEGIN ... END` body of an engine that has the usual one. So is a statement inside another, which
/// would make every `update` or `access` column an unknown engine names a write, and Snowflake's pipe operator,
/// which would cut a statement at the `->>` that is JSON's operator on an unknown engine. So are psql's `COPY` rows,
/// which would hide lines an unknown engine runs as statements.
/// would cut the `BEGIN ... END` body of an engine that has the usual one. So is MySQL's compound statement body,
/// which would hold a `LOOP` or a `WHILE ... DO` open past the `;` an unknown engine ends it at. So is a statement
/// inside another, which would make every `update` or `access` column an unknown engine names a write, and Snowflake's
/// pipe operator, which would cut a statement at the `->>` that is JSON's operator on an unknown engine. So are
/// psql's `COPY` rows, which would hide lines an unknown engine runs as statements.
public static let everyKnownReading: [SQLLexicalGrammar] = {
var seen: Set<SQLLexicalGrammar> = []
var result: [SQLLexicalGrammar] = []
Expand All @@ -80,7 +81,7 @@ public struct SQLLexicalProfile: Sendable, Hashable {
for reading in profile.readings {
let unitless = reading.subtracting([
.plsqlBlocks, .delimiterDirective, .unterminatedStatements, .terminatedMergeStatements,
.sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies,
.sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies, .compoundStatementBodies,
.statementsAsValues, .bracketedStatementSources, .pipeOperatorChains, .psqlScripts,
// An unknown engine reads `BT` and `ET` as names, not as transaction control.
.transactionShortForms,
Expand Down Expand Up @@ -112,10 +113,11 @@ public struct SQLLexicalProfile: Sendable, Hashable {

/// MySQL 8.4.11 and MariaDB 11.8.9, measured: a backslash escapes in `'` and `"` strings and is literal in a
/// backtick identifier, block comments do not nest, `#` is a comment, `/*! */` runs, `--` needs whitespace after
/// it, and only a line feed ends a line comment.
/// it, and only a line feed ends a line comment. From the MySQL 8.4 manual, and by hand on MariaDB 13.0.2 (the probe
/// does not check it): a routine's body is one statement, which may be a compound one.
static let mySQL: SQLLexicalGrammar = [
.backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .hashLineComments,
.executableComments, .dashCommentsNeedWhitespace, .delimiterDirective,
.executableComments, .dashCommentsNeedWhitespace, .delimiterDirective, .compoundStatementBodies,
]

/// SQLite 3.54, measured: brackets and backticks quote identifiers, `]]` does not escape, a backslash is literal,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,8 +137,8 @@ enum SQLLexicalRelevance {
let length = text.length
var facts: SQLLexicalGrammar = [
.plsqlBlocks, .delimiterDirective, .unterminatedStatements, .terminatedMergeStatements,
.sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies, .statementsAsValues,
.pipeOperatorChains, .databendTokens, .psqlScripts, .transactionShortForms,
.sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies, .compoundStatementBodies,
.statementsAsValues, .pipeOperatorChains, .databendTokens, .psqlScripts, .transactionShortForms,
]
var blockCommentOpeners = 0
var index = 0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -319,7 +319,8 @@ public enum SQLNonCodeSpan {
)
if grammar.contains(.escapeStringContinuations) {
var newlines = span.newlines
while span.isClosed, let continued = continuingQuote(after: span.next, in: text, length: length) {
while span.isClosed,
let continued = continuingQuote(after: span.next, in: text, length: length, grammar: grammar) {
span = SqlLexer.skipQuotedString(
text,
from: continued.quote,
Expand Down Expand Up @@ -351,15 +352,20 @@ public enum SQLNonCodeSpan {
private static let verticalTab: UInt16 = 0x0B

// DuckDB turns these into spaces before it lexes, outside what its pre-lexing pass reads as a quote, and no other
// non-ASCII unit (measured on 1.5.4 and 2.0.0).
// PostgreSQL and CockroachDB do not, and the text a join there leaves them does not parse, so they run none of it.
// non-ASCII unit (measured on 1.5.4 and 2.0.0). PostgreSQL and CockroachDB read them as part of a name.
static let spacesBeforeLexing: Set<UInt16> = Set([0x00A0, 0x202F, 0x205F, 0x2060, 0x3000, 0xFEFF])
.union(0x2000...0x200B)

/// The quote that carries on the string closed just before `index`, as PostgreSQL's `quotecontinue` finds it: only
/// blanks and `--` comments between, a line break among them. PostgreSQL 16 and later take a vertical tab for a
/// blank; an older server refuses the text.
private static func continuingQuote(after index: Int, in text: NSString, length: Int) -> (quote: Int, newlines: Int)? {
/// blank; an older server refuses the text. A non-ASCII space is a blank only where DuckDB 1.5 has blanked it
/// first: PostgreSQL 17.11 reads `E'a'` + U+00A0 + line break + `'b'` as a name between two strings.
private static func continuingQuote(
after index: Int,
in text: NSString,
length: Int,
grammar: SQLLexicalGrammar
) -> (quote: Int, newlines: Int)? {
var cursor = index
var lineBroke = false
var newlines = 0
Expand All @@ -374,7 +380,7 @@ public enum SQLNonCodeSpan {
cursor += 1
case SqlLexer.space, SqlLexer.tab, verticalTab, SqlLexer.formFeed:
cursor += 1
case let unit where spacesBeforeLexing.contains(unit):
case let unit where endsNameAsBlank(unit, grammar: grammar):
cursor += 1
case SqlLexer.dash where SqlLexer.startsLineComment(text, at: cursor, length: length):
cursor = SqlLexer.endOfLineComment(
Expand Down
Loading
Loading