From b5d98b9b532c13f7ce5b3822ee595df74b2a5e3a Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Fri, 13 Nov 2026 05:23:56 +0700 Subject: [PATCH 1/2] fix(editor): read a MySQL SET or IF routine body as one statement and open no body at a begin name --- .../SQLLexicalGrammar.swift | 6 + .../SQLLexicalProfile.swift | 16 +- .../SQLLexicalReadings.swift | 4 +- .../TableProSQLGrammar/SQLNonCodeSpan.swift | 18 +- .../SQLRoutineBodyTracker.swift | 213 +++++++++++++++++- .../SQLNonCodeSpanTests.swift | 27 ++- .../SQLRoutineBodyNameTests.swift | 134 ++++++++++- .../SQL/InvisibleCharacterRemoverTests.swift | 7 + .../SQL/QueryClassifierLexicalTests.swift | 27 +++ .../SQLLexicalFeatureMappingTests.swift | 14 +- .../src/sql_syntax/script/compound_tracker.rs | 48 +++- core/crates/core/tests/script_corpus.rs | 31 ++- 12 files changed, 505 insertions(+), 40 deletions(-) diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift index 187eb5dd2b..8690940f80 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift @@ -164,6 +164,12 @@ public struct SQLLexicalGrammar: OptionSet, Hashable, Sendable { /// PostgreSQL 17.11. That pass reads quotes its own way, so DuckDB's profile also doubts this fact. Curated only. public static let nonASCIISpacesEndNames = SQLLexicalGrammar(rawValue: 1 << 42) + /// A routine's body is one statement, as MySQL's `routine_body` reads it, and a compound one holds the `;`s inside + /// it: `IF ... END IF`, `WHILE ... END WHILE`, `LOOP ... END LOOP` and `REPEAT ... END REPEAT` as `BEGIN ... END` + /// does. A `SET` outside the return type starts the body, so a `begin` after it is a name. Measured on MariaDB + /// 13.0.2. Curated only. + public static let compoundStatementBodies = SQLLexicalGrammar(rawValue: 1 << 43) + /// Standard SQL: quotes close only on a doubled quote, `--` and flat `/* */` are the only comments. public static let ansi: SQLLexicalGrammar = [] diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift index eb5701d9aa..7e2d700305 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift @@ -68,10 +68,11 @@ public struct SQLLexicalProfile: Sendable, Hashable { /// own; an engine that runs statements without one says so through its plugin. A `MERGE` that keeps its `;` is left /// out for the same reason: an engine that refuses one without it says so through its plugin. So is a CQL batch, /// which would read one statement where an unknown engine may run several. So are the two single body forms, which - /// would cut the `BEGIN ... END` body of an engine that has the usual one. So is a statement inside another, which - /// would make every `update` or `access` column an unknown engine names a write, and Snowflake's pipe operator, - /// which would cut a statement at the `->>` that is JSON's operator on an unknown engine. So are psql's `COPY` rows, - /// which would hide lines an unknown engine runs as statements. + /// would cut the `BEGIN ... END` body of an engine that has the usual one. So is MySQL's compound statement body, + /// which would hold a `LOOP` or a `WHILE ... DO` open past the `;` an unknown engine ends it at. So is a statement + /// inside another, which would make every `update` or `access` column an unknown engine names a write, and Snowflake's + /// pipe operator, which would cut a statement at the `->>` that is JSON's operator on an unknown engine. So are + /// psql's `COPY` rows, which would hide lines an unknown engine runs as statements. public static let everyKnownReading: [SQLLexicalGrammar] = { var seen: Set = [] var result: [SQLLexicalGrammar] = [] @@ -80,7 +81,7 @@ public struct SQLLexicalProfile: Sendable, Hashable { for reading in profile.readings { let unitless = reading.subtracting([ .plsqlBlocks, .delimiterDirective, .unterminatedStatements, .terminatedMergeStatements, - .sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies, + .sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies, .compoundStatementBodies, .statementsAsValues, .bracketedStatementSources, .pipeOperatorChains, .psqlScripts, ]) if seen.insert(unitless).inserted { @@ -110,10 +111,11 @@ public struct SQLLexicalProfile: Sendable, Hashable { /// MySQL 8.4.11 and MariaDB 11.8.9, measured: a backslash escapes in `'` and `"` strings and is literal in a /// backtick identifier, block comments do not nest, `#` is a comment, `/*! */` runs, `--` needs whitespace after - /// it, and only a line feed ends a line comment. + /// it, and only a line feed ends a line comment. From the MySQL 8.4 manual, and by hand on MariaDB 13.0.2 (the probe + /// does not check it): a routine's body is one statement, which may be a compound one. static let mySQL: SQLLexicalGrammar = [ .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .hashLineComments, - .executableComments, .dashCommentsNeedWhitespace, .delimiterDirective, + .executableComments, .dashCommentsNeedWhitespace, .delimiterDirective, .compoundStatementBodies, ] /// SQLite 3.54, measured: brackets and backticks quote identifiers, `]]` does not escape, a backslash is literal, diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift index be5805d4a9..b5da9fde92 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift @@ -137,8 +137,8 @@ enum SQLLexicalRelevance { let length = text.length var facts: SQLLexicalGrammar = [ .plsqlBlocks, .delimiterDirective, .unterminatedStatements, .terminatedMergeStatements, - .sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies, .statementsAsValues, - .pipeOperatorChains, .databendTokens, .psqlScripts, + .sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies, .compoundStatementBodies, + .statementsAsValues, .pipeOperatorChains, .databendTokens, .psqlScripts, ] var blockCommentOpeners = 0 var index = 0 diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift index 81ad5eaf37..52e67f17ff 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift @@ -319,7 +319,8 @@ public enum SQLNonCodeSpan { ) if grammar.contains(.escapeStringContinuations) { var newlines = span.newlines - while span.isClosed, let continued = continuingQuote(after: span.next, in: text, length: length) { + while span.isClosed, + let continued = continuingQuote(after: span.next, in: text, length: length, grammar: grammar) { span = SqlLexer.skipQuotedString( text, from: continued.quote, @@ -351,15 +352,20 @@ public enum SQLNonCodeSpan { private static let verticalTab: UInt16 = 0x0B // DuckDB turns these into spaces before it lexes, outside what its pre-lexing pass reads as a quote, and no other - // non-ASCII unit (measured on 1.5.4 and 2.0.0). - // PostgreSQL and CockroachDB do not, and the text a join there leaves them does not parse, so they run none of it. + // non-ASCII unit (measured on 1.5.4 and 2.0.0). PostgreSQL and CockroachDB read them as part of a name. static let spacesBeforeLexing: Set = Set([0x00A0, 0x202F, 0x205F, 0x2060, 0x3000, 0xFEFF]) .union(0x2000...0x200B) /// The quote that carries on the string closed just before `index`, as PostgreSQL's `quotecontinue` finds it: only /// blanks and `--` comments between, a line break among them. PostgreSQL 16 and later take a vertical tab for a - /// blank; an older server refuses the text. - private static func continuingQuote(after index: Int, in text: NSString, length: Int) -> (quote: Int, newlines: Int)? { + /// blank; an older server refuses the text. A non-ASCII space is a blank only where DuckDB 1.5 has blanked it + /// first: PostgreSQL 17.11 reads `E'a'` + U+00A0 + line break + `'b'` as a name between two strings. + private static func continuingQuote( + after index: Int, + in text: NSString, + length: Int, + grammar: SQLLexicalGrammar + ) -> (quote: Int, newlines: Int)? { var cursor = index var lineBroke = false var newlines = 0 @@ -374,7 +380,7 @@ public enum SQLNonCodeSpan { cursor += 1 case SqlLexer.space, SqlLexer.tab, verticalTab, SqlLexer.formFeed: cursor += 1 - case let unit where spacesBeforeLexing.contains(unit): + case let unit where endsNameAsBlank(unit, grammar: grammar): cursor += 1 case SqlLexer.dash where SqlLexer.startsLineComment(text, at: cursor, length: length): cursor = SqlLexer.endOfLineComment( diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift index efc70b6c5c..6585b020fe 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift @@ -14,6 +14,9 @@ import Foundation /// /// A `;` inside parentheses ends the statement, so a stray `(` cannot swallow what follows it, except in a PostgreSQL /// rule's action list, `DO ALSO (INSERT ...; INSERT ...)`, which holds its commands together. +/// +/// On MySQL a routine's body is one statement (``SQLLexicalGrammar/compoundStatementBodies``), so an `IF`, `WHILE`, +/// `LOOP` or `REPEAT` where a statement starts holds its `;`s up to its own `END IF` or kin, as `BEGIN ... END` does. public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { // A PostgreSQL `CREATE RULE`: its header, the `DO [ALSO | INSTEAD]` before the action list, and the list itself. private enum RuleAction: Equatable { @@ -33,6 +36,30 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { case controlStatement } + // A MySQL compound statement whose block is open, with the depth inside it, so `END IF` closes only an `IF`. + private struct ControlStatement { + let keyword: String + let depth: Int + } + + // A handler declaration, `DECLARE ... HANDLER FOR condition [, condition] statement`, read up to its statement's + // first word, the one place in it where a statement starts. + private enum HandlerPart { + case none + case header + case condition + case sqlState + case notFound + case afterCondition + case statement + } + + private enum NestedDefinition { + case none + case opener + case routine + } + private enum Header { case opener /// After `CREATE`, `ALTER` or `REPLACE`. `accountWords` is what is left of an unquoted `DEFINER = user@host`. @@ -45,6 +72,8 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { private static let dot = UInt16(UnicodeScalar(".").value) private static let colon = UInt16(UnicodeScalar(":").value) private static let comma = UInt16(UnicodeScalar(",").value) + private static let zero = UInt16(UnicodeScalar("0").value) + private static let nine = UInt16(UnicodeScalar("9").value) /// The words a statement inside a body can follow without a `;`, as in `IF x THEN BEGIN ... END; END IF`. private static let statementIntroducers: Set = ["THEN", "ELSE", "DO", "LOOP", "REPEAT"] @@ -61,8 +90,27 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { private static let controlStatementOpeners: Set = ["IF", "WHILE", "LOOP", "REPEAT"] + private static let handlerKinds: Set = ["CONTINUE", "EXIT", "UNDO"] + + /// The words before a `SET` that is part of MySQL's return type: the `SET(...)` type, `CHARACTER SET`, `CHAR SET`. + private static let wordsBeforeTypeSet: Set = ["RETURNS", "CHARACTER", "CHAR"] + private let grammar: SQLLexicalGrammar private let readsBlocksEverywhere: Bool + private let readsCompoundBodies: Bool + private var openControlStatements: [ControlStatement] = [] + // An `IF` or `WHILE` where a statement can start, until the word that ends its condition at the same depth: + // `IF(a, 1, 2)` is a function wherever a value can stand. + private var conditionAwaitingBody: ControlStatement? + // The `CASE`s open here that are expressions, by the depth inside each: their `THEN` and `ELSE` start no statement. + private var caseExpressionDepths: [Int] = [] + // Inside PostgreSQL's `BEGIN ATOMIC` body, how far the statement being read is into `CREATE FUNCTION`: only such a + // statement has a body of its own, and elsewhere `begin atomic` is a column and its alias. + private var nestedDefinition = NestedDefinition.none + // A `REPEAT` block just opened, which a `(` right after it turns into the `REPEAT()` function. + private var repeatMayBeFunction = false + // A MySQL event's body follows its `DO`, and its schedule before that starts nothing. + private var awaitsEventBody = false // T-SQL outside a routine: where a block's `BEGIN` can stand, as the statement's first word, after `IF`, `WHILE` // or `ELSE`, or after a block the statement already opened. Anywhere else `begin` is in a name's place. private var followsControlFlow = false @@ -100,12 +148,13 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { private var atStatementPosition = false private var labelMayFollow = false private var inDeclaration = false - private var inHandlerDeclaration = false + private var handler = HandlerPart.none private var merge: SQLMergeStatementTracker? public init(grammar: SQLLexicalGrammar) { self.grammar = grammar self.readsBlocksEverywhere = grammar.contains(.unterminatedStatements) + self.readsCompoundBodies = grammar.contains(.compoundStatementBodies) self.merge = grammar.contains(.terminatedMergeStatements) ? SQLMergeStatementTracker() : nil } @@ -133,6 +182,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeWord(_ word: String) { merge?.observeWord(word) + repeatMayBeFunction = false let previous = previousWord previousWord = word let isTriggerEvent = expectsTriggerEvent && Self.singleStatementBodyOpeners.contains(word) @@ -148,8 +198,8 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { } let startsStatement = startsBodyStatement startsBodyStatement = false - let statementPosition = atStatementPosition - observeStatementPosition(after: word, wasAtStatementPosition: statementPosition) + let statementPosition = atStatementPosition || readHandlerWord(word) + observeStatementPosition(after: word, previous: previous, wasAtStatementPosition: statementPosition) guard !isMember else { return } var isFirstWord = false if case .opener = header { @@ -160,9 +210,17 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { observeRuleWord(word) } guard parenDepth == 0, readsBlocksEverywhere || (pastHeader && definesRoutine) else { return } + let startsBody = depth == 0 && routineBody == .unknown && !awaitsEventBody if !readsBlocksEverywhere, depth == 0 { observeBodyStart(word, after: previous, isTriggerEvent: isTriggerEvent) } + let atStatementStart = depth == 0 ? startsBody : statementPosition + if readsCompoundBodies { + observeCompoundStatement(word, atStatementStart: atStatementStart) + } + if depth > 0, grammar.contains(.atomicRoutineBodies) { + observeNestedDefinition(word, startsStatement: startsStatement) + } switch word { case "BEGIN" where readsBlocksEverywhere: pendingBegin = true @@ -191,6 +249,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { case "CASE" where !grammar.hasSingleBodyForm: depth += 1 openCaseExpressions += 1 + if !atStatementStart { + caseExpressionDepths.append(depth) + } case "END" where !grammar.hasSingleBodyForm || startsStatement: pendingEnd = true default: @@ -200,7 +261,15 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeSymbol(_ symbol: UInt16) { merge?.observeSymbol(symbol) + if repeatMayBeFunction { + repeatMayBeFunction = false + if symbol == SqlLexer.openParen { + openControlStatements.removeLast() + depth -= 1 + } + } previousWord = nil + readHandlerSymbol(symbol) expectsTriggerEvent = symbol == Self.comma && followsTriggerEvent followsTriggerEvent = false if ruleAction == .afterDo, symbol != SqlLexer.openParen { @@ -232,6 +301,15 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeOpaqueToken() { merge?.observeOpaqueToken() + repeatMayBeFunction = false + switch handler { + case .condition, .sqlState: + handler = .afterCondition + case .afterCondition: + handler = .statement + default: + break + } previousWord = nil expectsTriggerEvent = false followsTriggerEvent = false @@ -252,6 +330,11 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeSemicolon() -> Bool { pendingBegin = false + repeatMayBeFunction = false + conditionAwaitingBody = nil + // A `CASE` that holds a `;` is a statement. + caseExpressionDepths.removeAll() + nestedDefinition = .none previousWord = nil expectsTriggerEvent = false followsTriggerEvent = false @@ -282,7 +365,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { atStatementPosition = depth > 0 labelMayFollow = false inDeclaration = false - inHandlerDeclaration = false + handler = .none merge?.observeSemicolon(endsStatement: endsStatement) endedBeforeElse = endsStatement && readsBlocksEverywhere && conditionalsAwaitingElse > 0 return endsStatement @@ -328,6 +411,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { if SqlBlockStructure.isRoutineKind(word, grammar: grammar) { definesRoutine = true readsTriggerEvents = word == "TRIGGER" + awaitsEventBody = readsCompoundBodies && word == "EVENT" header = .name } else if word == "DEFINER" { header = .kind(accountWords: 2) @@ -346,15 +430,51 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { } } - // A label, `lbl:`, is a word at a statement's start followed by `:`. A handler's body follows its condition. - private mutating func observeStatementPosition(after word: String, wasAtStatementPosition: Bool) { - atStatementPosition = depth > 0 && Self.statementIntroducers.contains(word) + // A label, `lbl:`, is a word at a statement's start followed by `:`. On MySQL a `DO` where a statement starts is + // the `DO` statement, an expression after it; any other ends a `WHILE` or `FOR` header. + private mutating func observeStatementPosition(after word: String, previous: String?, wasAtStatementPosition: Bool) { + atStatementPosition = depth > 0 && caseExpressionDepths.last != depth && Self.statementIntroducers.contains(word) + && !(readsCompoundBodies && word == "DO" && wasAtStatementPosition) labelMayFollow = wasAtStatementPosition if wasAtStatementPosition, word == "DECLARE" { inDeclaration = true } - if inDeclaration, word == "HANDLER" { - inHandlerDeclaration = true + // `handler` is not reserved, so `DECLARE handler CURSOR FOR ...` declares a cursor. + if inDeclaration, word == "HANDLER", let previous, Self.handlerKinds.contains(previous) { + handler = .header + } + } + + /// Returns whether `word` starts a handler's statement. + private mutating func readHandlerWord(_ word: String) -> Bool { + switch handler { + case .header where word == "FOR": + handler = .condition + case .condition where word == "SQLSTATE": + handler = .sqlState + case .condition where word == "NOT": + handler = .notFound + case .condition, .notFound: + handler = .afterCondition + case .afterCondition: + handler = .statement + return true + default: + break + } + return false + } + + // A MySQL error code arrives one digit at a time. + private mutating func readHandlerSymbol(_ symbol: UInt16) { + let isDigit = symbol >= Self.zero && symbol <= Self.nine + switch handler { + case .condition: + handler = .afterCondition + case .afterCondition where !isDigit: + handler = symbol == Self.comma ? .condition : .statement + default: + break } } @@ -362,6 +482,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { if readsBlocksEverywhere { return depth > 0 || definesRoutine || isFirstWord || followsControlFlow || openedBlockBody } + if grammar.contains(.atomicRoutineBodies) { return depth == 0 || nestedDefinition == .routine } if grammar.hasSingleBodyForm { return true } switch routineBody { case .statement: @@ -369,8 +490,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { case .returnValue: return depth == 0 && (previous == "AS" || previous == "IS") case .unknown, .controlStatement: - guard depth > 0 else { return true } - return atStatementPosition || inHandlerDeclaration + // A MySQL body is the one control statement, so a `begin` beside it is a name. + guard depth > 0 else { return routineBody == .unknown || !readsCompoundBodies } + return atStatementPosition } } @@ -379,6 +501,10 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { if word == "ON" { readsTriggerEvents = false } + if awaitsEventBody { + awaitsEventBody = word != "DO" + return + } guard routineBody == .unknown else { return } if Self.controlStatementOpeners.contains(word) { routineBody = .controlStatement @@ -387,7 +513,68 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { routineBody = .returnValue } else if Self.singleStatementBodyOpeners.contains(word), !isTriggerEvent, !readsTriggerEvents { routineBody = .statement + } else if readsCompoundBodies, word == "SET", !Self.wordsBeforeTypeSet.contains(previous ?? "") { + routineBody = .statement + } else if readsCompoundBodies, word == "DO" { + routineBody = .statement + } + } + + // A compound statement opens its block where a statement starts. An `IF` or a `WHILE` opens it only at the word + // that ends its condition, since a condition holds no statement and `IF(a, 1, 2)` is a function. + private mutating func observeCompoundStatement(_ word: String, atStatementStart: Bool) { + guard routineBody == .unknown || routineBody == .controlStatement else { return } + let awaited = conditionAwaitingBody.flatMap { $0.depth == depth ? $0.keyword : nil } + switch word { + case "IF", "WHILE": + if atStatementStart, conditionAwaitingBody == nil { + conditionAwaitingBody = ControlStatement(keyword: word, depth: depth) + } + case "THEN" where awaited == "IF": + conditionAwaitingBody = nil + openControlStatement("IF", startsStatement: true) + case "DO" where awaited == "WHILE": + conditionAwaitingBody = nil + openControlStatement("WHILE", startsStatement: true) + // MariaDB's Oracle mode writes `WHILE x LOOP ... END LOOP`. + case "LOOP" where awaited == "WHILE": + conditionAwaitingBody = nil + openControlStatement(word, startsStatement: true) + case "LOOP" where atStatementStart: + openControlStatement(word, startsStatement: true) + case "REPEAT" where atStatementStart: + openControlStatement(word, startsStatement: true) + repeatMayBeFunction = true + case "WHEN", "ELSE", "END": + if awaited != nil { + conditionAwaitingBody = nil + } + default: + break + } + } + + private mutating func observeNestedDefinition(_ word: String, startsStatement: Bool) { + if startsStatement { + nestedDefinition = SqlBlockStructure.opensRoutineDefinition(word) ? .opener : .none + } else if nestedDefinition == .opener, !SqlBlockStructure.precedesRoutineKind(word) { + nestedDefinition = SqlBlockStructure.isRoutineKind(word, grammar: grammar) ? .routine : .none + } + } + + private mutating func openControlStatement(_ keyword: String, startsStatement: Bool) { + depth += 1 + openControlStatements.append(ControlStatement(keyword: keyword, depth: depth)) + atStatementPosition = startsStatement + } + + // `END IF` and its kin close the block their own keyword opened, and nothing where it opened none. + private mutating func closeControlStatement(_ keyword: String) { + guard let innermost = openControlStatements.last, innermost.keyword == keyword, innermost.depth == depth else { + return } + openControlStatements.removeLast() + depth -= 1 } private mutating func observeRuleWord(_ word: String) { @@ -436,6 +623,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { } switch SqlBlockStructure.endingFollowedBy(word, closesCaseExpression: openCaseExpressions > 0, grammar: grammar) { case .closesControlFlow: + closeControlStatement(word) return true case .closesCaseStatement: closeBlock() @@ -474,10 +662,13 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { openedBlockBody = true startsBodyStatement = true atStatementPosition = true + nestedDefinition = .none } private mutating func closeBlock() { depth = max(0, depth - 1) openCaseExpressions = max(0, openCaseExpressions - 1) + openControlStatements.removeAll { $0.depth > depth } + caseExpressionDepths.removeAll { $0 > depth } } } diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift index 5d492a2bae..04cfded890 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift @@ -198,11 +198,36 @@ struct SQLNonCodeSpanTests { #expect(end("E'x' -- c\n'y' z", postgreSQL) == 13) #expect(end("E'x'\n-- c\n'y' z", postgreSQL) == 13) #expect(end("E'x'\n'y'\n\n'z' w", postgreSQL) == 13) - #expect(end("E'x'\u{00A0}\n\u{3000}'y' z", postgreSQL) == 10) #expect(end("E'x'\u{200C}\n'y' z", postgreSQL) == 4) #expect(SQLNonCodeSpan.span(at: 0, in: "E'x'\r\n'y'\n'z'" as NSString, grammar: postgreSQL)?.newlines == 2) } + // Measured: PostgreSQL 17.11 answers a syntax error at the second string, and DuckDB 1.5.4 returns `xy`. + @Test("A non-ASCII space between the parts of a continued string is a name on PostgreSQL and a blank on DuckDB 1.5") + func nonASCIISpaceBeforeAContinuedString() { + let text = "E'x'\u{00A0}\n\u{3000}'y' z" + for engine in ["PostgreSQL", "CockroachDB", "Redshift", "KingbaseES"] { + let readings = SQLLexicalReadings.resolve(databaseTypeId: engine, declared: nil, session: nil) + for grammar in readings.all where grammar.contains(.escapeStringPrefix) { + #expect(end(text, grammar) == 4, "\(engine)") + #expect(end("E'x'\n\u{00A0}'y' z", grammar) == 4, "\(engine)") + } + } + let duckDB15 = Self.execution("DuckDB").union(.escapeStringContinuations) + #expect(end(text, duckDB15) == 10) + #expect(end(text, duckDB15.subtracting(.nonASCIISpacesEndNames)) == 4) + } + + // Measured on PostgreSQL 17.11: psql ran the statement after `'\'`. + @Test("Every gate reading sees the statement after a string a non-ASCII space keeps from continuing") + func nonASCIISpaceBeforeAContinuedStringHidesNothing() { + let sql = "SELECT E'a'\u{00A0}\n'\\'; DROP TABLE t; --'" + for engine in ["PostgreSQL", "CockroachDB", "Redshift", "KingbaseES", "DuckDB", "Nonesuch"] { + #expect(Self.tier(sql, engine) == .destructive, "\(engine)") + } + #expect(Self.statements(sql, "PostgreSQL") == 2) + } + @Test("An E'...' ends at its quote without a line break, across a block comment, or on DuckDB") func escapeStringEndsAtItsQuote() { let postgreSQL = Self.execution("PostgreSQL") diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLRoutineBodyNameTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLRoutineBodyNameTests.swift index 2ade004a42..e748f5bf38 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLRoutineBodyNameTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLRoutineBodyNameTests.swift @@ -80,6 +80,8 @@ struct SQLRoutineBodyNameTests { "CREATE FUNCTION f() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT 1 AS x$case, 2 AS \u{E9}case; END", "CREATE PROCEDURE p() LANGUAGE sql BEGIN ATOMIC CREATE FUNCTION g() RETURNS int BEGIN ATOMIC SELECT 1; END; END", "CREATE FUNCTION f() RETURNS void BEGIN ATOMIC /* nothing */ ; -- still nothing\n END", + "CREATE FUNCTION f() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT begin atomic FROM (SELECT 1 AS begin) s; END", + "CREATE PROCEDURE p() LANGUAGE sql BEGIN ATOMIC CREATE OR REPLACE FUNCTION g() RETURNS int BEGIN ATOMIC SELECT begin atomic FROM t; END; SELECT begin atomic FROM t; END", ]) func atomicBodies(function: String) { #expect(sent(function + "; DROP TABLE users", "PostgreSQL") == [function, "DROP TABLE users"]) @@ -95,6 +97,7 @@ struct SQLRoutineBodyNameTests { "CREATE PROCEDURE p(begin int, \"case\" int) BEGIN SELECT 1; END", "CREATE PROCEDURE p() BEGIN SELECT t.begin, t.case FROM t; END", "CREATE PROCEDURE p() BEGIN SELECT (SELECT begin FROM t); END", + "CREATE PROCEDURE p() BEGIN SELECT CASE WHEN closed THEN begin ELSE NULL END FROM shifts; SELECT 2; END", "CREATE EVENT TRIGGER begin ON ddl_command_start EXECUTE FUNCTION f()", ]) func namesOnOtherEngines(first: String) { @@ -131,6 +134,8 @@ struct SQLRoutineBodyNameTests { "CREATE PROCEDURE p() BEGIN inner_block: BEGIN SELECT 1; END inner_block; SELECT 2; END", "CREATE PROCEDURE db.p() outer_block: BEGIN SELECT (CASE WHEN 1 THEN 2 END); SELECT 2; END outer_block", "CREATE FUNCTION f() RETURNS VARCHAR(10) CHARACTER SET utf8mb4 DETERMINISTIC BEGIN SET @a = 1; RETURN 'x'; END", + "CREATE FUNCTION f() RETURNS VARCHAR(10) CHAR SET utf8mb4 DETERMINISTIC BEGIN SET @a = 1; RETURN 'x'; END", + "CREATE FUNCTION f() RETURNS SET('a', 'b') DETERMINISTIC BEGIN SET @a = 1; RETURN 'a'; END", "CREATE TRIGGER tr AFTER DELETE ON t FOR EACH ROW BEGIN DELETE FROM log; INSERT INTO log VALUES (1); END", "CREATE TRIGGER tr BEFORE UPDATE ON t FOR EACH ROW FOLLOWS other BEGIN UPDATE log SET n = 1; SELECT 1; END", ]) @@ -171,6 +176,124 @@ struct SQLRoutineBodyNameTests { } } + // Measured on MariaDB 13.0.2: each routine was stored with its whole body. + @Test("On MySQL a SET body is the routine's one statement, so a begin column in it opens no block", arguments: [ + "CREATE PROCEDURE p() SET @x = begin IS NULL", + "CREATE PROCEDURE p(begin INT) SET @half = begin DIV 2", + "CREATE DEFINER=root@localhost PROCEDURE p() SQL SECURITY INVOKER SET @x = begin IS NULL", + "CREATE EVENT cleanup ON SCHEDULE EVERY 1 DAY DO SET @open = begin IS NULL", + "ALTER EVENT cleanup DO SET @open = begin IS NULL", + ]) + func setBodiesOnMySQL(routine: String) { + for typeId in ["MySQL", "MariaDB", "OceanBase"] { + #expect( + sent(routine + ";\nCALL p();\nDROP TABLE users", typeId) == [routine, "CALL p()", "DROP TABLE users"], + "\(typeId)" + ) + } + } + + // Measured on MariaDB 13.0.2: each routine was stored with its whole body. + @Test("On MySQL an IF, WHILE, LOOP or REPEAT body holds its statements, and a begin column in it opens no block", arguments: [ + "CREATE PROCEDURE p() IF 1 THEN SELECT begin FROM shifts; SELECT 2; END IF", + "CREATE PROCEDURE p(begin INT) IF begin IS NULL THEN SELECT 1; END IF", + "CREATE PROCEDURE p() IF (SELECT COUNT(*) FROM shifts WHERE begin IS NULL) > 0 THEN SELECT 1; END IF", + "CREATE PROCEDURE p() IF @a THEN SELECT 1; ELSEIF @b THEN SELECT begin FROM t; ELSE IF @c THEN SELECT 3; END IF; END IF", + "CREATE PROCEDURE p() IF 1 THEN BEGIN SELECT begin FROM t; SELECT 2; END; END IF", + "CREATE PROCEDURE p() IF 1 THEN SELECT CASE WHEN 1 THEN IF(1, 2, 3) ELSE REPEAT('a', 2) END; SELECT begin FROM t; END IF", + "CREATE PROCEDURE p() CASE 1 WHEN 1 THEN IF 2 THEN SELECT begin FROM t; END IF; ELSE SELECT 2; END CASE", + "CREATE PROCEDURE p() WHILE @i < 3 DO SET @i = @i + 1; SELECT begin FROM shifts; END WHILE", + "CREATE PROCEDURE p() lbl: LOOP SELECT begin FROM shifts; LEAVE lbl; END LOOP lbl", + "CREATE PROCEDURE p() REPEAT SET @i = @i + 1; SELECT begin FROM shifts; UNTIL @i > 3 END REPEAT", + "CREATE PROCEDURE p() WHILE 1 DO REPEAT SET @s = REPEAT('x', 2); UNTIL 1 END REPEAT; SELECT begin FROM t; END WHILE", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR NOT FOUND IF 1 THEN SET @d = 1; END IF; SELECT begin FROM t; END", + "CREATE TRIGGER tr BEFORE UPDATE ON shifts FOR EACH ROW IF NEW.closed THEN SET NEW.id = 0; SELECT begin FROM shifts INTO @b; END IF", + "CREATE EVENT cleanup ON SCHEDULE EVERY 1 DAY DO IF 1 THEN DELETE FROM shifts WHERE begin IS NULL; SELECT 1; END IF", + "CREATE EVENT cleanup ON SCHEDULE AT IF(1, NOW(), NOW()) + INTERVAL 1 DAY DO BEGIN SELECT begin FROM t; SELECT 2; END", + "CREATE PROCEDURE p() DO REPEAT('a', 2)", + "CREATE PROCEDURE p() IF 1 THEN SELECT CASE WHEN closed THEN begin ELSE NULL END FROM shifts; END IF", + "CREATE PROCEDURE p() IF CASE WHEN @a THEN IF(@b, 1, 0) END THEN SELECT 1; SELECT 2; END IF", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR NOT FOUND SET @d = CASE WHEN IF(@a, 1, 0) THEN 1 END; SELECT 1; END", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR NOT FOUND CASE @x WHEN 1 THEN BEGIN SELECT 1; END; ELSE SELECT 2; END CASE; SELECT 3; END", + "CREATE PROCEDURE p() BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION SELECT begin INTO @b FROM t; SELECT 1; END", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR 1062, SQLSTATE VALUE '23000', NOT FOUND SET @d = CASE WHEN @a THEN begin ELSE 0 END; SELECT 1; END", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR SQLWARNING lbl: BEGIN SELECT 1; END lbl; SELECT 2; END", + "CREATE PROCEDURE p(begin INT) DO CASE WHEN @a THEN begin ELSE 0 END", + "CREATE PROCEDURE p(begin INT) BEGIN DO CASE WHEN @a THEN begin ELSE 0 END; SELECT 2; END", + "CREATE PROCEDURE p() BEGIN FOR i IN 1..3 DO BEGIN SELECT i; SELECT 2; END; END FOR; SELECT 3; END", + "CREATE PROCEDURE p() BEGIN DECLARE handler CURSOR FOR SELECT begin FROM t; SELECT 1; END", + ]) + func compoundBodiesOnMySQL(routine: String) { + for typeId in ["MySQL", "MariaDB", "OceanBase"] { + #expect( + sent(routine + ";\nCALL p();\nDROP TABLE users", typeId) == [routine, "CALL p()", "DROP TABLE users"], + "\(typeId)" + ) + } + } + + @Test("A handler's own BEGIN ... END body still opens on every engine with the usual body", arguments: [ + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR SQLSTATE '23000' BEGIN SELECT 1; END; SELECT 2; END", + "CREATE PROCEDURE p() BEGIN DECLARE EXIT HANDLER FOR SQLEXCEPTION, my_condition BEGIN SELECT 1; END; SELECT 2; END", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR 1062 BEGIN SELECT 1; END; SELECT 2; END", + "CREATE PROCEDURE p() BEGIN DECLARE EXIT HANDLER FOR 1062, 1213 BEGIN SELECT 1; END; SELECT 2; END", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR NOT FOUND BEGIN SELECT 1; END; SELECT 2; END", + "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR SQLSTATE VALUE '23000' BEGIN SELECT 1; END; SELECT 2; END", + ]) + func handlerBodies(routine: String) { + for typeId in ["MySQL", "Snowflake", "BigQuery", "Nonesuch"] { + #expect(sent(routine + ";\nSELECT 3", typeId) == [routine, "SELECT 3"], "\(typeId)") + } + } + + // Measured on MariaDB 13.0.2 with `sql_mode = ORACLE`. + @Test("MariaDB's Oracle mode WHILE ... LOOP closes at its END LOOP") + func oracleModeWhileLoop() { + let routine = "CREATE PROCEDURE p AS BEGIN WHILE 1 < 3 LOOP SELECT begin FROM t; EXIT; END LOOP; END" + #expect(sent(routine + ";\nDROP TABLE users", "MariaDB") == [routine, "DROP TABLE users"]) + } + + // Measured on PostgreSQL 17.11: the function was stored and the DROP after it ran in the same simple query. + @Test("A begin column with the alias atomic inside a BEGIN ATOMIC body hides nothing from the gate") + func beginAtomicAliasHidesNothing() { + let sql = "CREATE FUNCTION f() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT begin atomic FROM (SELECT 1 AS begin) s; " + + "END; DROP TABLE users; END" + for typeId in ["PostgreSQL", "Redshift", "CockroachDB", "Nonesuch"] { + let readings = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil) + #expect(SQLStatementClassifier.classify(sql, databaseTypeId: typeId, readings: readings).tier == .destructive) + } + } + + @Test("Every gate reading sees the statement after a MySQL SET or IF body") + func mySQLGatesSeeTheStatementAfterTheBody() { + for body in ["SET @x = begin IS NULL", "IF 1 THEN SELECT begin FROM t; END IF", "LOOP SELECT begin FROM t; END LOOP"] { + let sql = "CREATE PROCEDURE p() \(body);\nDROP TABLE audit_tmp;" + for typeId in ["MySQL", "MariaDB"] { + let readings = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil) + #expect(SQLStatementClassifier.classify(sql, databaseTypeId: typeId, readings: readings).tier == .destructive) + for grammar in readings.all { + #expect(SQLStatementScanner.allStatements(in: sql, grammar: grammar).last == "DROP TABLE audit_tmp") + } + } + } + } + + @Test("An engine TablePro does not know reads IF and WHILE as T-SQL does, so the statement after one stays its own", arguments: [ + "IF @x = 1 SELECT 1; DROP TABLE users", + "IF @x = 1 SELECT 1; ELSE SELECT 2; DROP TABLE users", + "WHILE @i < 3 SET @i = @i + 1; DROP TABLE users", + "CREATE PROCEDURE p AS IF @x = 1 SELECT 1; DROP TABLE users", + "CREATE PROCEDURE p() IF a THEN SELECT 1; DROP TABLE users", + "CREATE PROCEDURE p AS WHILE @i < 3 SET @i = @i + 1; DROP TABLE users", + "CREATE PROCEDURE p() WHILE 1 DO SELECT 1; DROP TABLE users", + "CREATE PROCEDURE p() LOOP SELECT 1; DROP TABLE users", + ]) + func unknownEnginesKeepTSQLControlFlow(sql: String) { + let readings = SQLLexicalReadings.resolve(databaseTypeId: "Nonesuch", declared: nil, session: nil) + #expect(SQLStatementClassifier.classify(sql, databaseTypeId: "Nonesuch", readings: readings).tier == .destructive) + #expect(readings.all.allSatisfy { !SQLStatementScanner.endsInsideOpenBlock(sql, grammar: $0) }) + } + @Test("A body that is a control statement keeps the blocks in it, and header clauses stay header", arguments: [ "CREATE PROCEDURE p AS IF @x = 1 DELETE FROM t ELSE BEGIN SELECT 1; SELECT 2; END", "CREATE PROCEDURE p() LANGUAGE SQL SET OPTION DBGVIEW = *SOURCE BEGIN SELECT 1; SELECT 2; END", @@ -254,6 +377,10 @@ struct SQLRoutineBodyNameTests { ("MySQL", "CREATE PROCEDURE p() BEGIN SELECT 1; END", false), ("MySQL", "CREATE PROCEDURE p() BEGIN SELECT 1; END;", false), ("MySQL", "CREATE PROCEDURE p() SELECT 1 AS case; DROP TABLE users", true), + ("MySQL", "CREATE PROCEDURE p() IF 1 THEN SELECT 1; DROP TABLE users", true), + ("MySQL", "CREATE PROCEDURE p() WHILE 1 DO SELECT 1; DROP TABLE users", true), + ("MySQL", "CREATE PROCEDURE p() IF 1 THEN SELECT 1; END IF", false), + ("MySQL", "CREATE PROCEDURE p() DO IF(1, 2, 3); DROP TABLE users", false), ("Snowflake", "DECLARE c CURSOR FOR SELECT 1 AS case; DROP TABLE users", true), ("PostgreSQL", "CREATE FUNCTION f() RETURNS int BEGIN ATOMIC SELECT 1; DROP TABLE users", true), ("PostgreSQL", "CREATE TABLE begin(a int); DROP TABLE users", false), @@ -270,13 +397,18 @@ struct SQLRoutineBodyNameTests { @Test("An engine TablePro does not know keeps the usual BEGIN ... END body") func unknownEnginesKeepTheUsualBody() { #expect(SQLLexicalProfile.everyKnownReading.allSatisfy { !$0.hasSingleBodyForm }) + #expect(SQLLexicalProfile.everyKnownReading.allSatisfy { !$0.contains(.compoundStatementBodies) }) let procedure = "CREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END" #expect(sent(procedure + "; SELECT 3", "Nonesuch") == [procedure, "SELECT 3"]) } @Test("A reading keeps the form of body its engine has, whatever the text holds") func relevanceKeepsTheBodyForm() { - for (typeId, fact) in [("Redshift", SQLLexicalGrammar.atomicRoutineBodies), ("PostgreSQL", .atomicRoutineBodies)] { + let bodyForms: [(String, SQLLexicalGrammar)] = [ + ("Redshift", .atomicRoutineBodies), ("PostgreSQL", .atomicRoutineBodies), + ("MySQL", .compoundStatementBodies), ("MariaDB", .compoundStatementBodies), + ] + for (typeId, fact) in bodyForms { let readings = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil) #expect(readings.distinct(for: "CREATE TABLE begin(a int)").allSatisfy { $0.contains(fact) }, "\(typeId)") } diff --git a/apple/TableProTests/Core/Utilities/SQL/InvisibleCharacterRemoverTests.swift b/apple/TableProTests/Core/Utilities/SQL/InvisibleCharacterRemoverTests.swift index d3f128521f..8c1667f800 100644 --- a/apple/TableProTests/Core/Utilities/SQL/InvisibleCharacterRemoverTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/InvisibleCharacterRemoverTests.swift @@ -108,6 +108,13 @@ struct InvisibleCharacterRemoverTests { #expect(clean("/* a /* b */\u{A0} */ SELECT\u{A0}1", grammar: grammar) == "/* a /* b */\u{A0} */ SELECT 1") } + // Measured on PostgreSQL 17.11: the NBSP left a syntax error, and the cleaned text returns `ab`. + @Test("A non-ASCII space between the parts of a PostgreSQL string is cleaned, so the parts join as they run") + func nonASCIISpaceBeforeAContinuedString() { + #expect(clean("SELECT E'a'\u{A0}\n'b'", grammar: TestGrammar.postgres) == "SELECT E'a' \n'b'") + #expect(clean("SELECT E'a'\n\u{3000}'b\u{A0}c'", grammar: TestGrammar.postgres) == "SELECT E'a'\n 'b\u{A0}c'") + } + @Test("A bracketed identifier keeps its characters where brackets quote names") func bracketedIdentifier() { let grammar = TestGrammar.sqlite diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierLexicalTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierLexicalTests.swift index 6fc2683ea8..e65f1ad426 100644 --- a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierLexicalTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierLexicalTests.swift @@ -146,6 +146,33 @@ struct QueryClassifierLexicalTests { #expect(QueryClassifier.isDangerousQuery(bypass.sql, databaseType: bypass.engine)) } + // Measured on 2026-11-13: MariaDB 13.0.2 stores each routine body whole and alone, psql 17.11 ran the statement + // after `'\'`, the NBSP being part of a name, and PostgreSQL 17.11 ran the DROP after the function in one query. + @Test("The statement after a MySQL SET or IF body, or after a string a non-ASCII space ends, is its own", arguments: [ + (DatabaseType.mysql, "CREATE PROCEDURE p() SET @x = begin IS NULL; DROP TABLE users"), + (DatabaseType.mariadb, "CREATE PROCEDURE p() IF 1 THEN SELECT begin FROM t; END IF; DROP TABLE users"), + (DatabaseType.postgresql, "SELECT E'a'\u{A0}\n'\\'; DROP TABLE users; --'"), + ( + DatabaseType.postgresql, + "CREATE FUNCTION f() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT begin atomic FROM (SELECT 1 AS begin) s; END; " + + "DROP TABLE users; END" + ), + (DatabaseType(rawValue: "Nonesuch"), "IF @x = 1 SELECT 1; DROP TABLE users"), + ]) + func statementAfterABodyIsItsOwn(engine: DatabaseType, sql: String) { + #expect(QueryClassifier.classifyTier(sql, databaseType: engine) == .destructive) + #expect(QueryClassifier.isMultiStatement(sql, databaseType: engine)) + #expect(QueryClassifier.isDangerousQuery(sql, databaseType: engine)) + } + + @Test("A MySQL IF body is one statement for an external client, and one left open is several") + func mySQLIfBodyForExternalClients() { + let closed = "CREATE PROCEDURE p() IF 1 THEN SELECT begin FROM t; SELECT 2; END IF" + #expect(!QueryClassifier.isMultiStatement(closed, databaseType: .mysql)) + #expect(QueryClassifier.classifyTier(closed, databaseType: .mysql) == .write) + #expect(QueryClassifier.isMultiStatement("CREATE PROCEDURE p() IF 1 THEN SELECT 1; DROP TABLE users", databaseType: .mysql)) + } + @Test("A COPY TO PROGRAM hidden after a literal backslash reaches the filesystem, measured on PostgreSQL 17") func hiddenCopyToProgramRunsCode() { let sql = "SELECT 'a\\'; COPY users TO PROGRAM 'touch /tmp/pwned'; --'" diff --git a/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift b/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift index e469284b29..af99cd0e6a 100644 --- a/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift +++ b/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift @@ -30,9 +30,10 @@ struct SQLLexicalFeatureMappingTests { .backslashEscapesInAngleQuotes, .escapeBytesPrefix, .exponentSignsNeedNoDigits, .formFeedEndsLineComments, .atomicRoutineBodies, .triggerBodies, .statementsAsValues, .bracketedStatementSources, .pipeOperatorChains, .withoutDashComments, .escapeStringContinuations, .databendTokens, .psqlScripts, .nonASCIISpacesEndNames, + .compoundStatementBodies, ] #expect(grammarBits.isDisjoint(with: curatedOnly)) - #expect(grammarBits.union(curatedOnly).rawValue == (1 << 43) - 1) + #expect(grammarBits.union(curatedOnly).rawValue == (1 << 44) - 1) } @Test("SAP HANA's SQLScript blocks stay in the app's curated grammar and never reach the kit") @@ -97,11 +98,13 @@ struct SQLLexicalFeatureMappingTests { let curatedOnly: SQLLexicalGrammar = [.atomicRoutineBodies, .escapeStringContinuations, .psqlScripts] #expect(grammar.isSuperset(of: curatedOnly)) #expect(SQLLexicalGrammar(pluginFeatures: grammar.pluginFeatures) == grammar.subtracting(curatedOnly)) - #expect(SQLLexicalGrammar(pluginFeatures: DatabaseType.mysql.lexicalGrammar.pluginFeatures) == DatabaseType.mysql.lexicalGrammar) + let mySQL = DatabaseType.mysql.lexicalGrammar + #expect(mySQL.contains(.compoundStatementBodies)) + #expect(SQLLexicalGrammar(pluginFeatures: mySQL.pluginFeatures) == mySQL.subtracting(.compoundStatementBodies)) } /// MySQLDriverTests' MySQLLexicalFeatureListTests pins MySQLLexicalFeatures to these same sets. - @Test("The grammars the MySQL plugin keeps are the curated ones") + @Test("The grammars the MySQL plugin keeps are the curated ones, less the routine body only the app reads") func pluginGrammarsMatchTheCuratedTable() { let mySQL: SQLLexicalFeatures = [ .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .hashLineComments, @@ -110,7 +113,10 @@ struct SQLLexicalFeatureMappingTests { let databend: SQLLexicalFeatures = [ .backslashEscapesInSingleQuotes, .backslashEscapesInDoubleQuotes, .backtickQuotes, .untaggedDollarQuotes, ] - #expect(SQLLexicalGrammar(pluginFeatures: mySQL) == DatabaseType.mysql.lexicalGrammar) + #expect( + SQLLexicalGrammar(pluginFeatures: mySQL) + == DatabaseType.mysql.lexicalGrammar.subtracting(.compoundStatementBodies) + ) let curatedDatabend = DatabaseType.databend.lexicalGrammar.subtracting([.formFeedEndsLineComments, .databendTokens]) #expect(SQLLexicalGrammar(pluginFeatures: databend) == curatedDatabend) } diff --git a/core/crates/core/src/sql_syntax/script/compound_tracker.rs b/core/crates/core/src/sql_syntax/script/compound_tracker.rs index 9540bc00c7..56b2b5b80e 100644 --- a/core/crates/core/src/sql_syntax/script/compound_tracker.rs +++ b/core/crates/core/src/sql_syntax/script/compound_tracker.rs @@ -32,6 +32,12 @@ pub(crate) struct CompoundTracker { identifier_count: usize, identifiers: [u8; 4], begin_depth: u32, + /// PostgreSQL's only routine body outside a literal is `BEGIN ATOMIC + /// ... END` (gram.y, `opt_routine_body`), so a `begin` opens one only + /// in a statement that defines a routine and only when `atomic` is the + /// next token: `CREATE FUNCTION begin()`, `RETURN begin` and, inside a + /// body, `SELECT begin atomic FROM t` name things. + follows_begin: bool, trigger: TriggerState, batch: BatchState, } @@ -86,6 +92,7 @@ impl CompoundTracker { identifier_count: 0, identifiers: [0; 4], begin_depth: 0, + follows_begin: false, trigger: TriggerState::Start, batch: BatchState::default(), } @@ -99,11 +106,10 @@ impl CompoundTracker { } match self.rule { CompoundRule::None => {} - CompoundRule::RoutineBody => { - if kind == ScriptTokenKind::Word { - self.observe_routine_word(text); - } - } + CompoundRule::RoutineBody => match kind { + ScriptTokenKind::Word => self.observe_routine_word(text), + _ => self.follows_begin = false, + }, CompoundRule::TriggerBody => self.trigger = advance_trigger(self.trigger, classify_trigger(kind, text)), CompoundRule::Batch => self.batch.observe(kind, text), } @@ -122,7 +128,10 @@ impl CompoundTracker { self.reset(); } else if self.rule == CompoundRule::TriggerBody { self.trigger = TriggerState::Semi; + } else if self.begin_depth > 0 { + self.start_body_statement(); } + self.follows_begin = false; ends } @@ -142,11 +151,17 @@ impl CompoundTracker { && (second == b'f' || second == b'p' || (second == b'o' && third == b'r' && (fourth == b'f' || fourth == b'p'))); - if !creates_routine || self.paren_depth != 0 { + let follows_begin = std::mem::take(&mut self.follows_begin); + if self.paren_depth != 0 || (self.begin_depth == 0 && !creates_routine) { return; } if word.eq_ignore_ascii_case("begin") { - self.begin_depth += 1; + self.follows_begin = creates_routine; + } else if word.eq_ignore_ascii_case("atomic") { + if follows_begin { + self.begin_depth += 1; + self.start_body_statement(); + } } else if word.eq_ignore_ascii_case("case") { if self.begin_depth >= 1 { self.begin_depth += 1; @@ -155,6 +170,13 @@ impl CompoundTracker { self.begin_depth = self.begin_depth.saturating_sub(1); } } + + /// A statement inside the body starts, and its own words decide + /// whether it defines a routine. + fn start_body_statement(&mut self) { + self.identifier_count = 0; + self.identifiers = [0; 4]; + } } fn classify_trigger(kind: ScriptTokenKind, text: &str) -> TriggerToken { @@ -218,6 +240,18 @@ mod tests { assert!(tracker.semicolon_ends_statement()); } + #[test] + fn a_begin_before_anything_but_atomic_is_a_name() { + let mut tracker = CompoundTracker::new(CompoundRule::RoutineBody, true); + feed( + &mut tracker, + &[ + "CREATE", "FUNCTION", "begin", "(", ")", "RETURNS", "int", "RETURN", "begin", + ], + ); + assert!(tracker.semicolon_ends_statement()); + } + #[test] fn transaction_begin_is_not_counted() { let mut tracker = CompoundTracker::new(CompoundRule::RoutineBody, true); diff --git a/core/crates/core/tests/script_corpus.rs b/core/crates/core/tests/script_corpus.rs index 73f9e1171d..c2d575258d 100644 --- a/core/crates/core/tests/script_corpus.rs +++ b/core/crates/core/tests/script_corpus.rs @@ -94,11 +94,13 @@ fn escape_string_continues_past_a_line_break() { } // Without a line break, past a block comment, or past anything but a // blank or a `--` comment, the server ends the string and so must the - // lexer. + // lexer. PostgreSQL 17.11 reads a non-ASCII space as part of a name. for apart in [ "SELECT E'a' '\\'", "SELECT E'a' /* note */\n'\\'", "SELECT E'a'\n, '\\'", + "SELECT E'a'\u{a0}\n'\\'", + "SELECT E'a'\n\u{3000}'\\'", ] { let text = format!("{apart}; SELECT 2;"); assert_eq!( @@ -372,6 +374,33 @@ fn routine_case_end_inside_body() { ); } +// Measured on PostgreSQL 17.11: each definition ran in one simple query +// with the statement after it. +#[test] +fn a_begin_name_in_a_routine_opens_no_body() { + for routine in [ + "CREATE FUNCTION begin() RETURNS int AS 'select 1' LANGUAGE sql", + "CREATE OR REPLACE PROCEDURE s.begin() LANGUAGE sql AS 'select 1'", + "CREATE FUNCTION f(begin int) RETURNS int RETURN begin", + "CREATE FUNCTION f() RETURNS int LANGUAGE sql SET search_path = begin RETURN 1", + "CREATE FUNCTION f() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT begin FROM t; SELECT 2; END", + "CREATE FUNCTION f() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT begin atomic FROM (SELECT 1 AS begin) s; END", + "CREATE PROCEDURE p() LANGUAGE sql BEGIN ATOMIC CREATE OR REPLACE FUNCTION g() RETURNS int BEGIN ATOMIC SELECT begin atomic FROM t; END; SELECT begin atomic FROM t; END", + ] { + let text = format!("{routine};\nDROP TABLE users;"); + assert_eq!( + statements(SqlGrammar::PostgreSql, &text), + [routine, "DROP TABLE users"], + "{routine}" + ); + assert_eq!( + statement_access(&text, SqlGrammar::PostgreSql), + StatementAccess::Destructive, + "{routine}" + ); + } +} + #[test] fn begin_column_outside_routine_is_identifier() { let text = "SELECT begin, end FROM spans; SELECT 2;"; From 8d03dbd5e77af9d7a4548e3267d54296ca8b9dd0 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Fri, 13 Nov 2026 05:24:27 +0700 Subject: [PATCH 2/2] docs(changelog): add fragments for #4254 --- apple/changelog.d/4254.fixed.1.md | 1 + apple/changelog.d/4254.fixed.md | 1 + apple/changelog.d/4254.security.md | 1 + linux/changelog.d/4254.security.md | 1 + 4 files changed, 4 insertions(+) create mode 100644 apple/changelog.d/4254.fixed.1.md create mode 100644 apple/changelog.d/4254.fixed.md create mode 100644 apple/changelog.d/4254.security.md create mode 100644 linux/changelog.d/4254.security.md diff --git a/apple/changelog.d/4254.fixed.1.md b/apple/changelog.d/4254.fixed.1.md new file mode 100644 index 0000000000..480bf46a22 --- /dev/null +++ b/apple/changelog.d/4254.fixed.1.md @@ -0,0 +1 @@ +Remove Invisible Characters left a non-ASCII space between the two parts of a PostgreSQL string, so the query failed. diff --git a/apple/changelog.d/4254.fixed.md b/apple/changelog.d/4254.fixed.md new file mode 100644 index 0000000000..e9eb48b304 --- /dev/null +++ b/apple/changelog.d/4254.fixed.md @@ -0,0 +1 @@ +A MySQL routine with a `SET` or `IF` body naming a `begin` column swallowed the rest of the script on Run All. diff --git a/apple/changelog.d/4254.security.md b/apple/changelog.d/4254.security.md new file mode 100644 index 0000000000..56d30d1b2d --- /dev/null +++ b/apple/changelog.d/4254.security.md @@ -0,0 +1 @@ +Safe Mode and MCP missed a PostgreSQL statement hidden after a `BEGIN ATOMIC` body that selects `begin atomic`. diff --git a/linux/changelog.d/4254.security.md b/linux/changelog.d/4254.security.md new file mode 100644 index 0000000000..2359aff95b --- /dev/null +++ b/linux/changelog.d/4254.security.md @@ -0,0 +1 @@ +Safe Mode missed a PostgreSQL statement hidden after a routine named `begin` or one that names a `begin` column.