From b670f121ffd34f4382d43e243cdfcb9203bebea2 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 15 Nov 2026 04:19:03 +0700 Subject: [PATCH 01/10] refactor(sql): read every SQL span through SQLNonCodeSpan and run one shared script corpus against the core --- .../Sources/TableProQuery/QueryScript.swift | 5 + .../TableProSQLGrammar/SQLNonCodeSpan.swift | 74 ++- .../Sources/TableProSQLGrammar/SqlLexer.swift | 33 +- .../TableProQueryTests/QueryScriptTests.swift | 7 + .../SQLNonCodeSpanTests.swift | 38 ++ .../SQLScriptSharedCorpusTests.swift | 73 +++ .../ConsoleRequestDiagnosticsProducer.swift | 2 +- .../MongoDiagnosticsProducer.swift | 2 +- .../Diagnostics/QueryBracketScanner.swift | 113 ++-- .../Core/Diagnostics/QueryDiagnostic.swift | 7 +- .../Diagnostics/SQLDiagnosticsProducer.swift | 5 +- .../Services/Formatting/SQLTokenizer.swift | 169 ------ .../QueryHistoryStorage+PlanSnapshots.swift | 1 + .../Core/Storage/QueryHistoryStorage.swift | 42 +- .../SQL/SQLFileParser+CopyRows.swift | 1 - .../SQL/SQLFileParser+Literals.swift | 235 -------- .../Core/Utilities/SQL/SQLFileParser.swift | 545 +++--------------- .../Utilities/SQL/SQLQueryFingerprint.swift | 79 +-- .../TransactionRollBackClassifier.swift | 46 +- .../ViewModels/QueryEditorViewModel.swift | 4 +- .../TransactionRollBackClassifierTests.swift | 17 +- .../Diagnostics/QueryDiagnosticsTests.swift | 36 +- .../Services/SQLFormatterServiceTests.swift | 2 +- .../Core/Services/SQLTokenizerTests.swift | 154 ++--- ...ueryHistoryFingerprintMigrationTests.swift | 26 + .../Storage/QueryHistoryMigrationTests.swift | 4 +- .../SQL/SQLFileParserSharedCorpusTests.swift | 68 +++ .../Utilities/SQL/SQLFileParserTests.swift | 70 +++ .../Helpers/SharedSQLScriptCorpus.swift | 45 ++ .../SQLLexicalFeatureMappingTests.swift | 35 ++ .../tests/fixtures/sql_script_corpus.json | 545 ++++++++++++++++++ .../crates/core/tests/script_shared_corpus.rs | 137 +++++ scripts/ci/detect-changed-paths.sh | 8 +- scripts/ci/test_detect_changed_paths.py | 11 + 34 files changed, 1540 insertions(+), 1099 deletions(-) create mode 100644 apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLScriptSharedCorpusTests.swift delete mode 100644 apple/TablePro/Core/Utilities/SQL/SQLFileParser+Literals.swift create mode 100644 apple/TableProTests/Core/Utilities/SQL/SQLFileParserSharedCorpusTests.swift create mode 100644 apple/TableProTests/Helpers/SharedSQLScriptCorpus.swift create mode 100644 core/crates/core/tests/fixtures/sql_script_corpus.json create mode 100644 core/crates/core/tests/script_shared_corpus.rs diff --git a/apple/Packages/TableProCore/Sources/TableProQuery/QueryScript.swift b/apple/Packages/TableProCore/Sources/TableProQuery/QueryScript.swift index 94c3c23d55..5c57b7f65c 100644 --- a/apple/Packages/TableProCore/Sources/TableProQuery/QueryScript.swift +++ b/apple/Packages/TableProCore/Sources/TableProQuery/QueryScript.swift @@ -120,6 +120,11 @@ public struct QueryScript: Sendable, Equatable { return SQLStatementScanner.executableStatements(in: unit.sql, grammar: grammar).map(\.sql) } + // The grammar the units were split by; standard SQL for a script built from units alone. + public var lexicalGrammar: SQLLexicalGrammar { + databaseType.map { Self.executionGrammar(for: $0, session: session) } ?? .ansi + } + // A bare Redis argument may end in a byte Foundation trims, such as a vertical tab or a no-break space. public static func trimmed(_ text: String, databaseType: DatabaseType) -> String { guard databaseType == .redis else { return text.trimmingCharacters(in: .whitespacesAndNewlines) } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift index 52e67f17ff..e6d2754b4a 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift @@ -58,6 +58,8 @@ public enum SQLNonCodeSpan { private static let capitalE = UInt16(UnicodeScalar("E").value) private static let smallE = UInt16(UnicodeScalar("e").value) private static let smallB = UInt16(UnicodeScalar("b").value) + private static let at = UInt16(UnicodeScalar("@").value) + private static let colon = UInt16(UnicodeScalar(":").value) /// The comment, literal or quoted identifier starting at `index`, or nil when `index` starts code. public static func span(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Span? { @@ -119,6 +121,57 @@ public enum SQLNonCodeSpan { return span.end } + // A reader that walks every unit asks this first, and `span(at:)` only where the answer can be a span. + public static func mayStart(_ unit: UInt16) -> Bool { + switch unit { + case SqlLexer.dash, SqlLexer.hash, SqlLexer.slash, SqlLexer.singleQuote, SqlLexer.doubleQuote, SqlLexer.backtick, + openBracket, SqlLexer.openAngleQuote, SqlDollarQuote.dollar, at, colon, capitalE, smallE, smallB, + SqlLexer.smallQ, SqlLexer.capitalQ, SqlLexer.smallN, SqlLexer.capitalN: + return true + default: + return false + } + } + + // Units past a span's start that decide its kind (`nq'` and a delimiter outside the BMP), and past its end that + // decide where it ends (a doubled quote or `]]` after the closer). + private static let openerLookahead = 8 + private static let closerLookahead = 1 + + // Whether text appended after `text` could change what `span(at: index)` answered, which a chunked reader waits on. + public static func needsMoreText(_ span: Span?, at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Bool { + let length = text.length + guard length - index > openerLookahead else { return true } + guard let span else { return opensPastTheEnd(at: index, in: text, grammar: grammar) } + if !span.isTerminated || span.end + closerLookahead > length { + return true + } + guard grammar.contains(.escapeStringContinuations), span.kind == .quoted, + [capitalE, smallE, smallB].contains(text.character(at: span.start)), + case .runsOut = continuation(after: span.end, in: text, length: length, grammar: grammar) + else { + return false + } + return true + } + + // A dollar tag or a Tcl-style parameter name can run past any fixed lookahead before it shows what it opens. + private static func opensPastTheEnd(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Bool { + let length = text.length + if let style = grammar.dollarQuoteStyle, text.character(at: index) == SqlDollarQuote.dollar, + case .needsMoreData = SqlDollarQuote.scanOpener( + at: index, + in: text, + bufLen: length, + style: style, + continuesName: { SQLNonCodeSpan.continuesName(beforeDollar: $0, grammar: grammar) } + ) { + return true + } + guard grammar.contains(.parenthesizedParameterNames) else { return false } + return SqlLexer.parameterNameEnd(text, at: index, length: length) == length + } + public static func isWordUnit(_ unit: UInt16) -> Bool { if unit < 0x80 { return SqlDollarQuote.isIdentifierPart(unit) @@ -320,7 +373,7 @@ public enum SQLNonCodeSpan { if grammar.contains(.escapeStringContinuations) { var newlines = span.newlines while span.isClosed, - let continued = continuingQuote(after: span.next, in: text, length: length, grammar: grammar) { + case .quote(let continued) = continuation(after: span.next, in: text, length: length, grammar: grammar) { span = SqlLexer.skipQuotedString( text, from: continued.quote, @@ -360,12 +413,19 @@ public enum SQLNonCodeSpan { /// blanks and `--` comments between, a line break among them. PostgreSQL 16 and later take a vertical tab for a /// blank; an older server refuses the text. A non-ASCII space is a blank only where DuckDB 1.5 has blanked it /// first: PostgreSQL 17.11 reads `E'a'` + U+00A0 + line break + `'b'` as a name between two strings. - private static func continuingQuote( + private enum Continuation { + case quote((quote: Int, newlines: Int)) + case none + // Only blanks and comments up to the end of the text, so more text could still carry the string on. + case runsOut + } + + private static func continuation( after index: Int, in text: NSString, length: Int, grammar: SQLLexicalGrammar - ) -> (quote: Int, newlines: Int)? { + ) -> Continuation { var cursor = index var lineBroke = false var newlines = 0 @@ -390,13 +450,15 @@ public enum SQLNonCodeSpan { carriageReturnEnds: true, formFeedEnds: false ) + case SqlLexer.dash where cursor + 1 == length: + return .runsOut case SqlLexer.singleQuote where lineBroke: - return (cursor, newlines) + return .quote((cursor, newlines)) default: - return nil + return .none } } - return nil + return .runsOut } private static func dollarQuotedSpan(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Span? { diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift index 4f5e31e623..77ff38ba36 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlLexer.swift @@ -393,20 +393,11 @@ public enum SqlLexer { /// `::` continues a name, as in `$a::b(...)`. A `$` inside a word starts no parameter: `a$b(` is the name `a$b` /// and a `(`. public static func skipParenthesizedParameterName(_ text: NSString, at offset: Int, length: Int) -> Span? { - guard parameterPrefixes.contains(text.character(at: offset)), !continuesSQLiteWord(text, at: offset) else { + guard var cursor = parameterNameEnd(text, at: offset, length: length), + cursor > offset + 1, cursor < length, text.character(at: cursor) == openParen + else { return nil } - var cursor = offset + 1 - while cursor < length { - let character = text.character(at: cursor) - if isSQLiteIdentifierCharacter(character) { - cursor += 1 - continue - } - guard character == colonUnit, cursor + 1 < length, text.character(at: cursor + 1) == colonUnit else { break } - cursor += 2 - } - guard cursor > offset + 1, cursor < length, text.character(at: cursor) == openParen else { return nil } cursor += 1 while cursor < length { let character = text.character(at: cursor) @@ -421,6 +412,24 @@ public enum SqlLexer { return Span(next: length, newlines: 0, isClosed: false) } + // Where a Tcl-style parameter's name ends, before its `(`; nil when no parameter starts at `offset`. + public static func parameterNameEnd(_ text: NSString, at offset: Int, length: Int) -> Int? { + guard parameterPrefixes.contains(text.character(at: offset)), !continuesSQLiteWord(text, at: offset) else { + return nil + } + var cursor = offset + 1 + while cursor < length { + let character = text.character(at: cursor) + if isSQLiteIdentifierCharacter(character) { + cursor += 1 + continue + } + guard character == colonUnit, cursor + 1 < length, text.character(at: cursor + 1) == colonUnit else { break } + cursor += 2 + } + return cursor + } + /// Whether the prefix at `offset` is a `$` inside a word SQLite is already reading. `$` is one of its identifier /// characters; `@`, `:` and `#` are not, and always start a token. /// diff --git a/apple/Packages/TableProCore/Tests/TableProQueryTests/QueryScriptTests.swift b/apple/Packages/TableProCore/Tests/TableProQueryTests/QueryScriptTests.swift index fa0eeb768c..02410160ee 100644 --- a/apple/Packages/TableProCore/Tests/TableProQueryTests/QueryScriptTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQueryTests/QueryScriptTests.swift @@ -204,4 +204,11 @@ struct QueryScriptTests { #expect(QueryScript.trimmed(" \n ; ", databaseType: .redis).isEmpty) #expect(QueryScript.trimmed(" \nSELECT 1; \n", databaseType: .sqlite) == "SELECT 1;") } + + @Test("A script hands out the grammar it was split by, and standard SQL when it knows no engine") + func lexicalGrammarIsTheSplittingGrammar() { + #expect(plan("SELECT 1", .mysql).lexicalGrammar.contains(.hashLineComments)) + #expect(plan("SELECT 1", .postgresql).lexicalGrammar.contains(.nestedBlockComments)) + #expect(QueryScript(units: [.init(sql: "SELECT 1")], unitKind: .statement).lexicalGrammar == .ansi) + } } diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift index 04cfded890..2cd75ab150 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift @@ -402,4 +402,42 @@ struct SQLNonCodeSpanTests { ) #expect(sqlite == "SELECT eval ( )") } + + private func needsMoreText(_ text: String, at index: Int = 0, _ grammar: SQLLexicalGrammar) -> Bool { + let buffer = text as NSString + let span = SQLNonCodeSpan.span(at: index, in: buffer, grammar: grammar) + return SQLNonCodeSpan.needsMoreText(span, at: index, in: buffer, grammar: grammar) + } + + private static let continued: SQLLexicalGrammar = [.escapeStringPrefix, .escapeStringContinuations] + private static let longTag: String = "$" + String(repeating: "t", count: 40) + private static let longName: String = "$" + String(repeating: "p", count: 40) + private static let blanks = String(repeating: " ", count: 20) + + private static let unsettled: [(String, SQLLexicalGrammar)] = [ + (longTag, .taggedDollarQuotes), + ("'a doubled quote may follow'", .ansi), + ("-- a comment with no line break", .ansi), + ("/* open comment ", .ansi), + (longName, .parenthesizedParameterNames), + ("E'x'" + blanks + "-", continued), + ("E'x'\n" + blanks + "-- c", continued), + ] + + private static let settled: [(String, SQLLexicalGrammar)] = [ + ("'abc' + 1 ", .ansi), + ("E'x'" + blanks + "+ 1", continued), + ("-- note\nSELECT 1 FROM t", .ansi), + ("SELECT 1 FROM t WHERE a = 1", .ansi), + ] + + @Test("A reader of a buffer waits when more text could change the span", arguments: unsettled) + func waitsForMoreText(text: String, grammar: SQLLexicalGrammar) { + #expect(needsMoreText(text, grammar), "\(text)") + } + + @Test("A span whose end no later text can move is settled", arguments: settled) + func settledSpan(text: String, grammar: SQLLexicalGrammar) { + #expect(!needsMoreText(text, grammar), "\(text)") + } } diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLScriptSharedCorpusTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLScriptSharedCorpusTests.swift new file mode 100644 index 0000000000..80193f0434 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLScriptSharedCorpusTests.swift @@ -0,0 +1,73 @@ +import Foundation +@testable import TableProSQLGrammar +import Testing + +// The core's script reader runs the same file in its own tests, so the two read each case where the engine does. +struct SQLScriptSharedCorpusTests { + struct Batch: Decodable, Equatable, CustomStringConvertible { + let text: String + let repeatCount: Int + + enum CodingKeys: String, CodingKey { + case text + case repeatCount = "repeat" + } + + var description: String { "\(repeatCount)x \(text)" } + } + + struct Case: Decodable { + let name: String + let dialect: String + let sql: String + let statements: [String] + let batches: [Batch]? + + var grammar: SQLLexicalGrammar? { + Self.databaseTypeIds[dialect].flatMap { SQLLexicalProfile.curated(forDatabaseTypeId: $0)?.grammar } + } + + private static let databaseTypeIds = [ + "postgresql": "PostgreSQL", "mysql": "MySQL", "sqlite": "SQLite", "mssql": "SQL Server", + "clickhouse": "ClickHouse", "duckdb": "DuckDB", "cql": "Cassandra", + ] + } + + private struct Corpus: Decodable { + let cases: [Case] + } + + static let corpusURL = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("core/crates/core/tests/fixtures/sql_script_corpus.json") + + static func cases() throws -> [Case] { + try JSONDecoder().decode(Corpus.self, from: Data(contentsOf: corpusURL)).cases + } + + @Test("Every dialect the corpus names has a curated grammar") + func everyDialectIsKnown() throws { + let cases = try Self.cases() + #expect(!cases.isEmpty) + #expect(cases.filter { $0.grammar == nil }.map(\.dialect).isEmpty) + } + + @Test("The scanner splits each case where the engine does") + func scannerSplitsEachCase() throws { + for testCase in try Self.cases() { + guard let grammar = testCase.grammar else { continue } + let statements = SQLStatementScanner.executableStatements(in: testCase.sql, grammar: grammar) + #expect(statements.map(\.sql) == testCase.statements, "\(testCase.dialect): \(testCase.name)") + guard let expected = testCase.batches else { continue } + let separators = SQLStatementScanner.batchSeparators(in: testCase.sql, grammar: grammar) + let batches = SQLStatementScanner.batches(in: testCase.sql, statements: statements, separators: separators) + let read = batches.map { Batch(text: $0.sql, repeatCount: $0.repeatCount) } + #expect(read == expected, "\(testCase.dialect): \(testCase.name)") + } + } +} diff --git a/apple/TablePro/Core/Diagnostics/ConsoleRequestDiagnosticsProducer.swift b/apple/TablePro/Core/Diagnostics/ConsoleRequestDiagnosticsProducer.swift index 2bdb04b545..45333fafcb 100644 --- a/apple/TablePro/Core/Diagnostics/ConsoleRequestDiagnosticsProducer.swift +++ b/apple/TablePro/Core/Diagnostics/ConsoleRequestDiagnosticsProducer.swift @@ -17,7 +17,7 @@ struct ConsoleRequestDiagnosticsProducer: QueryDiagnosticsProducing { // The body highlights as JavaScript, so `//` and `/* */` read as comments on screen and the // scan has to read them the same way. - let structure = QueryBracketScanner.scan(blankingRequestLine(source), comments: .javaScript) + let structure = QueryBracketScanner.scan(blankingRequestLine(source), language: .javaScript) var results: [QueryDiagnostic] = [] if let range = structure.unmatchedClose { diff --git a/apple/TablePro/Core/Diagnostics/MongoDiagnosticsProducer.swift b/apple/TablePro/Core/Diagnostics/MongoDiagnosticsProducer.swift index 67e22d521e..2ff569f3e8 100644 --- a/apple/TablePro/Core/Diagnostics/MongoDiagnosticsProducer.swift +++ b/apple/TablePro/Core/Diagnostics/MongoDiagnosticsProducer.swift @@ -16,7 +16,7 @@ struct MongoDiagnosticsProducer: QueryDiagnosticsProducing { // The bracket scanner has no regex state, so `db.c.find({x:/[)]/})` reads as an unmatched // closer. The syntax checker below knows JavaScript properly, so the structural pass is // only there to stay quiet while a query is still being typed. - let structure = QueryBracketScanner.scan(source, comments: .javaScript) + let structure = QueryBracketScanner.scan(source, language: .javaScript) let hasRegexLiteral = source.range(of: "/", options: .literal).location != NSNotFound if let range = structure.unmatchedClose, !hasRegexLiteral { diff --git a/apple/TablePro/Core/Diagnostics/QueryBracketScanner.swift b/apple/TablePro/Core/Diagnostics/QueryBracketScanner.swift index afca4cf051..59b14f6996 100644 --- a/apple/TablePro/Core/Diagnostics/QueryBracketScanner.swift +++ b/apple/TablePro/Core/Diagnostics/QueryBracketScanner.swift @@ -1,4 +1,5 @@ import Foundation +import TableProSQLGrammar /// Structural scan shared by the diagnostic producers. Reports only what more typing cannot fix: /// a closer with no opener, and an unterminated block comment. A missing closer is treated as @@ -20,27 +21,90 @@ enum QueryBracketScanner { private static let backslash = UInt16(UnicodeScalar("\\").value) private static let slash = UInt16(UnicodeScalar("/").value) private static let star = UInt16(UnicodeScalar("*").value) - private static let dash = UInt16(UnicodeScalar("-").value) private static let newline = UInt16(UnicodeScalar("\n").value) private static let singleQuote = UInt16(UnicodeScalar("'").value) private static let doubleQuote = UInt16(UnicodeScalar("\"").value) private static let backtick = UInt16(UnicodeScalar("`").value) - /// Which line-comment spelling the language uses. - /// - /// The two are mutually exclusive and taking both is wrong in both directions: `--` is the - /// decrement operator in JavaScript, so reading it as a comment swallows the rest of the line - /// and hides everything the scan was called to check. - enum CommentStyle { - /// `--` opens a line comment, and `//` does not. - case sql - /// `//` opens a line comment, and `--` is an operator. + // `--` is JavaScript's decrement operator: read as a comment it would hide the rest of the line. + enum Language { + case sql(SQLLexicalGrammar) case javaScript } - static func scan(_ text: NSString, comments: CommentStyle) -> Result { + static func scan(_ text: NSString, language: Language) -> Result { + switch language { + case .sql(let grammar): + return scanSQL(text, grammar: grammar) + case .javaScript: + return scanJavaScript(text) + } + } + + private struct Brackets { var stack: [UInt16] = [] var unmatchedClose: NSRange? + + mutating func read(_ character: UInt16, at index: Int) { + switch character { + case openParen, openBrace, openBracket: + stack.append(character) + case closeParen, closeBrace, closeBracket: + if stack.last == QueryBracketScanner.opener(for: character) { + stack.removeLast() + } else if unmatchedClose == nil { + unmatchedClose = NSRange(location: index, length: 1) + } + default: + break + } + } + } + + private static func scanSQL(_ text: NSString, grammar: SQLLexicalGrammar) -> Result { + var brackets = Brackets() + var unterminatedComment: NSRange? + var hasUnterminatedString = false + var index = 0 + + while index < text.length { + let character = text.character(at: index) + guard SQLNonCodeSpan.mayStart(character), + let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) + else { + brackets.read(character, at: index) + index += 1 + continue + } + switch span.kind { + case .lineComment, .parameter: + break + case .blockComment, .executableComment: + if !span.isTerminated { + unterminatedComment = NSRange(location: span.start, length: 2) + } + case .quoted: + hasUnterminatedString = hasUnterminatedString || !span.isTerminated + } + // The server runs the body of `/*! ... */`, so its brackets are code. + if span.kind == .executableComment, + let opener = SqlLexer.executableCommentOpenerLength(text, at: index, length: text.length) { + index += opener + continue + } + index = max(span.end, index + 1) + } + + return Result( + unmatchedClose: brackets.unmatchedClose, + unterminatedComment: unterminatedComment, + hasUnclosedOpener: !brackets.stack.isEmpty, + hasUnterminatedString: hasUnterminatedString + ) + } + + private static func scanJavaScript(_ text: NSString) -> Result { + var brackets = Brackets() var commentStart: Int? var stringDelimiter: UInt16 = 0 var isInBlockComment = false @@ -85,46 +149,27 @@ enum QueryBracketScanner { index += 2 continue } - if next == slash, comments == .javaScript { + if next == slash { isInLineComment = true index += 2 continue } } - if comments == .sql, character == dash, index + 1 < text.length, - text.character(at: index + 1) == dash { - isInLineComment = true - index += 2 - continue - } - if character == singleQuote || character == doubleQuote || character == backtick { stringDelimiter = character index += 1 continue } - switch character { - case openParen, openBrace, openBracket: - stack.append(character) - case closeParen, closeBrace, closeBracket: - if stack.last == opener(for: character) { - stack.removeLast() - } else if unmatchedClose == nil { - unmatchedClose = NSRange(location: index, length: 1) - } - default: - break - } - + brackets.read(character, at: index) index += 1 } return Result( - unmatchedClose: unmatchedClose, + unmatchedClose: brackets.unmatchedClose, unterminatedComment: isInBlockComment ? commentStart.map { NSRange(location: $0, length: 2) } : nil, - hasUnclosedOpener: !stack.isEmpty, + hasUnclosedOpener: !brackets.stack.isEmpty, hasUnterminatedString: stringDelimiter != 0 ) } diff --git a/apple/TablePro/Core/Diagnostics/QueryDiagnostic.swift b/apple/TablePro/Core/Diagnostics/QueryDiagnostic.swift index 96d91478f3..ad366cc4ca 100644 --- a/apple/TablePro/Core/Diagnostics/QueryDiagnostic.swift +++ b/apple/TablePro/Core/Diagnostics/QueryDiagnostic.swift @@ -56,16 +56,17 @@ enum QueryDiagnosticsFactory { case .javascript: return ConsoleRequestDiagnosticsProducer() case .sql: + let grammar = grammar ?? resolvedType.lexicalGrammar return CombinedQueryDiagnosticsProducer(producers: [ - SQLDiagnosticsProducer(), - SQLConfusableCharacterDiagnosticsProducer(grammar: grammar ?? resolvedType.lexicalGrammar) + SQLDiagnosticsProducer(grammar: grammar), + SQLConfusableCharacterDiagnosticsProducer(grammar: grammar) ]) case .bash: // A command line's arguments are plain text, so `SET smile :)` holds a bracket that closes // nothing and is still a valid command. There is no structure to check. return CombinedQueryDiagnosticsProducer(producers: []) case .custom: - return SQLDiagnosticsProducer() + return SQLDiagnosticsProducer(grammar: grammar ?? resolvedType.lexicalGrammar) } } } diff --git a/apple/TablePro/Core/Diagnostics/SQLDiagnosticsProducer.swift b/apple/TablePro/Core/Diagnostics/SQLDiagnosticsProducer.swift index c9ed916965..f6b09c419c 100644 --- a/apple/TablePro/Core/Diagnostics/SQLDiagnosticsProducer.swift +++ b/apple/TablePro/Core/Diagnostics/SQLDiagnosticsProducer.swift @@ -1,13 +1,16 @@ import Foundation +import TableProSQLGrammar /// Reports only structural problems more typing cannot fix. TablePro has no SQL grammar, and a /// half-written statement must never be flagged, so an unclosed bracket is deliberately silent. struct SQLDiagnosticsProducer: QueryDiagnosticsProducing { + let grammar: SQLLexicalGrammar + func diagnostics(for text: String) -> [QueryDiagnostic] { let source = text as NSString guard source.length > 0, source.length <= QueryDiagnosticsLimits.maximumDocumentLength else { return [] } - let structure = QueryBracketScanner.scan(source, comments: .sql) + let structure = QueryBracketScanner.scan(source, language: .sql(grammar)) var results: [QueryDiagnostic] = [] if let range = structure.unmatchedClose { diff --git a/apple/TablePro/Core/Services/Formatting/SQLTokenizer.swift b/apple/TablePro/Core/Services/Formatting/SQLTokenizer.swift index 4c95295691..4ca0ccc1c4 100644 --- a/apple/TablePro/Core/Services/Formatting/SQLTokenizer.swift +++ b/apple/TablePro/Core/Services/Formatting/SQLTokenizer.swift @@ -35,9 +35,6 @@ struct SQLToken: Equatable { // Comments, literals, quoted identifiers and batch lines end where the engine's grammar says (`SQLNonCodeSpan`), so a // formatted query keeps every one of them byte for byte. Only code between them is split into words and operators. -// -// Without a grammar the text is read exactly as before the formatter read grammars, because the query fingerprint is -// the one caller and its hashes are stored: a different reading would split or merge history groups after an upgrade. struct SQLTokenizer { private static let standardKeywords: Set = [ // DML @@ -103,10 +100,6 @@ struct SQLTokenizer { self.operatorsByFirstUnit = byFirst.mapValues { $0.sorted { $0.utf16.count > $1.utf16.count } } } - func tokenize(_ sql: String) -> [SQLToken] { - readsWithoutGrammar(sql) - } - func tokenize(_ sql: String, grammar: SQLLexicalGrammar) -> [SQLToken] { var lexer = Lexer(text: sql as NSString, grammar: grammar, tokenizer: self) return lexer.run() @@ -130,168 +123,6 @@ struct SQLTokenizer { fileprivate func operatorCandidates(startingWith unit: UInt16) -> [String] { operatorsByFirstUnit[unit] ?? [] } - - private func readsWithoutGrammar(_ sql: String) -> [SQLToken] { - var tokens: [SQLToken] = [] - let chars = Array(sql) - let count = chars.count - var i = 0 - - while i < count { - let ch = chars[i] - - // Line comment: -- ... - if ch == "-" && i + 1 < count && chars[i + 1] == "-" { - let start = i - i += 2 - while i < count && chars[i] != "\n" { - i += 1 - } - tokens.append(SQLToken(type: .comment, value: String(chars[start..=, <=, <>, !=, ||, ::, ->>, -> - if i + 1 < count { - let twoChar = String([chars[i], chars[i + 1]]) - if [">=", "<=", "<>", "!=", "||", "::", "->"].contains(twoChar) { - if twoChar == "->" && i + 2 < count && chars[i + 2] == ">" { - tokens.append(SQLToken(type: .operator, value: "->>")) - i += 3 - } else { - tokens.append(SQLToken(type: .operator, value: twoChar)) - i += 2 - } - continue - } - } - - // Single-character operators - if "=<>+-*/%&|^~!".contains(ch) { - tokens.append(SQLToken(type: .operator, value: String(ch))) - i += 1 - continue - } - - // Punctuation: ( ) , ; . - if "(),;.".contains(ch) { - tokens.append(SQLToken(type: .punctuation, value: String(ch))) - i += 1 - continue - } - - // Words: keywords or identifiers - if ch.isLetter || ch == "_" { - let start = i - i += 1 - while i < count && (chars[i].isLetter || chars[i].isNumber || chars[i] == "_") { - i += 1 - } - let word = String(chars[start.. [QueryPlanSnapshotSummary] { let boundedLimit = min(max(limit, 0), Self.maximumPlanSnapshotListLength) guard db != nil, boundedLimit > 0 else { return [] } + ensureFingerprintsBackfilled() var clause = QueryHistorySqlClause() clause.append(""" diff --git a/apple/TablePro/Core/Storage/QueryHistoryStorage.swift b/apple/TablePro/Core/Storage/QueryHistoryStorage.swift index 7394509bfc..750bce3b45 100644 --- a/apple/TablePro/Core/Storage/QueryHistoryStorage.swift +++ b/apple/TablePro/Core/Storage/QueryHistoryStorage.swift @@ -106,6 +106,7 @@ actor QueryHistoryStorage { createFingerprintIndex() createPlanSnapshotStorage() createRewindSnapshotStorage() + resetFingerprintsReadWithoutGrammar() protectDatabaseFiles(at: dbPath) } @@ -222,31 +223,46 @@ actor QueryHistoryStorage { else { return false } - setUserVersion(5) + if userVersion() < 5 { + setUserVersion(5) + } return true } + // Before version 6 a fingerprint was read without the engine's grammar. The backfill hashes a zero again, so a + // group keeps the runs from before the upgrade. + private func resetFingerprintsReadWithoutGrammar() { + guard userVersion() < 6, beginTransaction() else { return } + guard execute("UPDATE history SET fingerprint_hash = 0;"), + execute("UPDATE plan_snapshots SET fingerprint_hash = 0;"), + setUserVersion(6) + else { + rollbackTransaction() + return + } + commitTransaction() + } + // first_row_time and server_time stay NULL for older rows: COALESCE falls back to execution_time. private func addColumnIfMissing(_ column: String, definition: String) -> Bool { guard let present = hasColumn(column, inTable: "history") else { return false } return present || execute("ALTER TABLE history ADD COLUMN \(column) \(definition);") } - /// Filling the column is deliberately not part of `init`. - /// - /// An actor's initializer runs synchronously on whoever calls it, and the first caller is a - /// `Task` inside `applicationWillFinishLaunching`, which inherits main-actor isolation. Doing - /// the backfill there tokenizes every stored row on the main thread before any window appears, - /// for as long as the user's history is. Running it from an actor method instead puts it on the - /// actor's own executor, and the only caller that needs it is the one that reads groups. - private func ensureFingerprintsBackfilled() { + // Not in `init`, which runs on the main actor at launch: the readers of groups and plans call it on the actor. + func ensureFingerprintsBackfilled() { guard !didBackfillFingerprints else { return } didBackfillFingerprints = true backfillFingerprints() } private func backfillFingerprints() { - let pending = readRowsMissingFingerprint() + backfillFingerprints(table: "history", sqlColumn: "query") + backfillFingerprints(table: "plan_snapshots", sqlColumn: "subject_sql") + } + + private func backfillFingerprints(table: String, sqlColumn: String) { + let pending = readRowsMissingFingerprint(table: table, sqlColumn: sqlColumn) guard !pending.isEmpty else { return } beginTransaction() @@ -256,7 +272,7 @@ actor QueryHistoryStorage { defer { sqlite3_finalize(statement) } guard sqlite3_prepare_v2( db, - "UPDATE history SET fingerprint_hash = ? WHERE id = ?;", + "UPDATE \(table) SET fingerprint_hash = ? WHERE id = ?;", -1, &statement, nil @@ -276,14 +292,14 @@ actor QueryHistoryStorage { } } - private func readRowsMissingFingerprint() -> [(id: String, fingerprint: Int64)] { + private func readRowsMissingFingerprint(table: String, sqlColumn: String) -> [(id: String, fingerprint: Int64)] { var pending: [(id: String, fingerprint: Int64)] = [] var statement: OpaquePointer? defer { sqlite3_finalize(statement) } guard sqlite3_prepare_v2( db, - "SELECT id, query, database_type FROM history WHERE fingerprint_hash = 0;", + "SELECT id, \(sqlColumn), database_type FROM \(table) WHERE fingerprint_hash = 0;", -1, &statement, nil diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift index 87ef982eaa..8d0538aeaf 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift @@ -144,7 +144,6 @@ extension SQLFileParser.ParserContext { state = .normal lineHasCode = false lineHoldsOnlyBlanks = true - previousUnit = SQLFileParser.kNewline previousUnitInWord = false pendingSlashLine = false pendingSlashTrailing.removeAll() diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser+Literals.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser+Literals.swift deleted file mode 100644 index bdf0e0b269..0000000000 --- a/apple/TablePro/Core/Utilities/SQL/SQLFileParser+Literals.swift +++ /dev/null @@ -1,235 +0,0 @@ -import Foundation -import TableProSQLGrammar - -// A literal or quoted name runs to its closer here, and stops at the end of a chunk when the next one decides it. -extension SQLFileParser { - static func processAlternativeQuote( - _ ctx: inout ParserContext, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - if ch == ctx.alternativeQuoteCloser { - let quoteAt = pos + (ctx.alternativeQuoteCloserTrail == nil ? 1 : 2) - if quoteAt >= bufLen { - if ctx.atEndOfInput { - pos += 1 - continue - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - let trailMatches = ctx.alternativeQuoteCloserTrail.map { nsBuffer.character(at: pos + 1) == $0 } ?? true - if trailMatches, nsBuffer.character(at: quoteAt) == kSingleQuote { - pos = quoteAt + 1 - ctx.state = .normal - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - } - pos += 1 - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - - /// Runs a triple-quoted literal to the three quotes that close it, holding back a quote or a backslash the buffer - /// ends on until the next chunk shows what follows. - static func processTripleQuotedString( - _ ctx: inout ParserContext, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - if ctx.backslashEscapesActive && ch == kBackslash { - guard pos + 1 < bufLen || ctx.atEndOfInput else { - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - pos += 2 - continue - } - if ch == ctx.quoteChar { - guard pos + 2 < bufLen || ctx.atEndOfInput else { - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - if SqlLexer.startsTripleQuote(nsBuffer, at: pos, length: bufLen) { - pos += 3 - ctx.state = .normal - ctx.backslashEscapesActive = false - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - } - pos += 1 - } - appendRange(&ctx, from: start, to: min(pos, bufLen), in: nsBuffer) - i = min(pos, bufLen) - return StepResult(advanced: true, deferred: false) - } - - /// Runs a `[...]` identifier to its `]`, which `]]` does not close where the grammar escapes it. - static func processBracketedIdentifier( - _ ctx: inout ParserContext, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - let doubledEscapes = ctx.grammar.contains(.doubledClosingBracketEscapes) - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - guard ch == kCloseBracket else { - pos += 1 - continue - } - if doubledEscapes { - guard pos + 1 < bufLen || ctx.atEndOfInput else { - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - if pos + 1 < bufLen, nsBuffer.character(at: pos + 1) == kCloseBracket { - pos += 2 - continue - } - } - pos += 1 - ctx.state = .normal - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - - static func processQuotedString( - _ ctx: inout ParserContext, - quoteChar: unichar, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - let escapesActive = ctx.backslashEscapesActive - - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - - if escapesActive && ch == kBackslash { - if pos + 1 >= bufLen { - if ctx.atEndOfInput { - pos += 1 - continue - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - let next = nsBuffer.character(at: pos + 1) - if next == kNewline { ctx.currentLine += 1 } - pos += 2 - continue - } - - if ch == quoteChar { - if pos + 1 >= bufLen && !ctx.atEndOfInput { - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - let next: unichar? = pos + 1 < bufLen ? nsBuffer.character(at: pos + 1) : nil - if next == quoteChar { - pos += 2 - continue - } - pos += 1 - ctx.state = .normal - ctx.backslashEscapesActive = false - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - - pos += 1 - } - - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - - static func processDollarQuote( - _ ctx: inout ParserContext, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - let closeLen = (ctx.dollarTag as NSString).length + 2 - - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - - if ch == kDollar { - if pos + closeLen > bufLen { - if ctx.atEndOfInput { - pos += 1 - continue - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - if SqlDollarQuote.matchesClose(at: pos, tag: ctx.dollarTag, in: nsBuffer, bufLen: bufLen) { - pos += closeLen - ctx.state = .normal - ctx.dollarTag = "" - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - } - pos += 1 - } - - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } -} diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift index 047b9e3885..404bd95fba 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift @@ -57,81 +57,32 @@ final class SQLFileParser: Sendable { enum ParserState { case normal - case inSingleLineComment /// A psql `\restrict` line, which runs to its line feed whatever ends a comment. case inClientLine - case inMultiLineComment - case inSingleQuotedString - case inDoubleQuotedString - case inBacktickQuotedString - case inTripleQuotedString - case inBracketedIdentifier - case inDollarQuote - case inAlternativeQuote } + // Units already read that stay in the buffer: whether `E'` or `$` opens a literal depends on the unit before it. + static let lookBehind = 256 + private static let kSemicolon: unichar = 0x3B - static let kSingleQuote: unichar = 0x27 - private static let kDoubleQuote: unichar = 0x22 - private static let kBacktick: unichar = 0x60 - static let kBackslash: unichar = 0x5C - private static let kDash: unichar = 0x2D - private static let kSlash: unichar = 0x2F - private static let kStar: unichar = 0x2A - private static let kHash: unichar = 0x23 static let kNewline: unichar = 0x0A + private static let kSlash: unichar = 0x2F + private static let kBackslash: unichar = 0x5C private static let kSpace: unichar = 0x20 private static let kTab: unichar = 0x09 private static let kCarriageReturn: unichar = 0x0D - static let kDollar: unichar = 0x24 - private static let kCapitalE: unichar = 0x45 - private static let kSmallE: unichar = 0x65 - private static let kSmallB: unichar = 0x62 - private static let kOpenBracket: unichar = 0x5B - static let kCloseBracket: unichar = 0x5D private static let kCapitalG: unichar = 0x47 private static let kSmallG: unichar = 0x67 + // A client delimiter is matched whole, so its first unit waits until the buffer holds as many units as it has. nonisolated private static func needsLookahead( _ char: unichar, + available: Int, state: ParserState, - grammar: SQLLexicalGrammar, delimiter: NSString, isSingleCharDelimiter: Bool ) -> Bool { - switch state { - case .normal: - var result = char == kDash || char == kSlash || char == kBackslash || char == kStar - || char == kSingleQuote || char == kDoubleQuote || char == kBacktick - if char == kDollar - && (grammar.dollarQuoteStyle != nil || grammar.contains(.dollarAndHashInIdentifiers)) { - result = true - } - if grammar.contains(.escapeStringPrefix) && (char == kCapitalE || char == kSmallE) { - result = true - } - if grammar.contains(.escapeBytesPrefix) && char == kSmallB { - result = true - } - if !isSingleCharDelimiter && char == delimiter.character(at: 0) { - result = true - } - return result - case .inSingleQuotedString, .inDoubleQuotedString, .inTripleQuotedString: - return char == kSingleQuote || char == kDoubleQuote || char == kBackslash - case .inBacktickQuotedString: - return char == kBacktick || char == kBackslash - case .inBracketedIdentifier: - return char == kCloseBracket - case .inMultiLineComment: - return char == kStar || char == kSlash - case .inSingleLineComment, .inClientLine: - return false - case .inDollarQuote: - return char == kDollar - case .inAlternativeQuote: - return false - } + state == .normal && !isSingleCharDelimiter && available < delimiter.length && char == delimiter.character(at: 0) } nonisolated private static func isWhitespace(_ char: unichar) -> Bool { @@ -198,17 +149,13 @@ final class SQLFileParser: Sendable { var hasStatementContent = false var currentLine = 1 var statementStartLine = 1 - // A run `/*! ... */` body is code to the server, comments and quotes included, so it is read as code and sent - // whole, and a `;` inside it ends nothing. - var insideConditionalComment = false - var commentDepth = 0 var currentDelimiter: NSString = ";" as NSString var isSingleCharDelimiter = true - var dollarTag: String = "" - var quoteChar: unichar = 0 - var backslashEscapesActive = false var collected: [ParsedStatement] = [] + // The next read takes as much again, so a literal of many megabytes is lexed a few times, not once per chunk. + var heldSpanLength = 0 + /// SQL Server's tools read a script the way sqlcmd does: a line holding only `GO` ends a batch, the batch goes /// to the server whole with its `;` and its comments, and the `GO` line goes nowhere. The comments stay so the /// server's line numbers count the file's lines and a routine keeps the comments written inside it. @@ -249,20 +196,13 @@ final class SQLFileParser: Sendable { /// Where the word being read starts in the statement text, and on which line. var wordStart = 0 var wordStartLine = 1 - var alternativeQuoteCloser: unichar = 0 - // A delimiter outside the Basic Multilingual Plane is two units, and the literal closes at both. - var alternativeQuoteCloserTrail: unichar? var lineHasCode = false /// Whether the statement's first unit was the first code on its line, where a `DELIMITER` can stand. var statementOpensLine = false var pendingSlashLine = false var pendingSlashTrailing: [unichar] = [] - /// The unit before the one being read, carried across chunks, so a prefix like `E'` or `q'` is only a - /// literal where a word could start. - var previousUnit: unichar = 0x20 - - /// Whether that unit belongs to a word or a name, which a letter glued to it continues. A number is neither, + /// Whether the unit before the one being read belongs to a word or a name, which a letter glued to it continues. A number is neither, /// so `1MERGE` is a number and a keyword, as the statement scanner and SQL Server both read it. var previousUnitInWord = false @@ -297,7 +237,7 @@ final class SQLFileParser: Sendable { // A SQL Server statement is a batch of its own and keeps the comments inside it, not the ones before it. var keepsComments: Bool { - readsBatches || (grammar.contains(.batchSeparatorLines) && hasStatementContent) || insideConditionalComment + readsBatches || (grammar.contains(.batchSeparatorLines) && hasStatementContent) } } @@ -336,7 +276,6 @@ final class SQLFileParser: Sendable { static func resetStatement(_ ctx: inout ParserContext) { ctx.currentStatement?.setString("") ctx.hasStatementContent = false - ctx.insideConditionalComment = false ctx.boundaries?.reset() ctx.heldEnd = nil ctx.statementOpensLine = false @@ -357,96 +296,34 @@ final class SQLFileParser: Sendable { ctx.boundaries?.observeWord(word) } - /// Tracks the word `char` belongs to and returns whether `char` starts an Oracle `q'...'` literal, which it does - /// only at the start of a word. private static func observeWordCharacter( _ ctx: inout ParserContext, char: unichar, i: Int, nsBuffer: NSString, bufLen: Int - ) -> WordStep { - let readsAlternativeQuotes = ctx.grammar.contains(.alternativeQuoting) - guard ctx.boundaries != nil || readsAlternativeQuotes else { return .none } + ) -> Bool { + guard ctx.boundaries != nil else { return false } let followsWord = ctx.previousUnitInWord ctx.previousUnitInWord = (followsWord && SqlBlockStructure.continuesWord(char, grammar: ctx.grammar)) || (char >= 0x80 && SQLNonCodeSpan.isWordUnit(char)) if !ctx.word.isEmpty { if SqlBlockStructure.continuesWord(char, grammar: ctx.grammar) { ctx.word.append(char) - return .continues + return true } flushWord(&ctx) } - guard SqlBlockStructure.startsWord(nsBuffer, at: i, length: bufLen, grammar: ctx.grammar) else { return .none } - // `E'...'` and CockroachDB's `b'...'` are literals, and the branch that reads them never runs once the prefix - // starts a word. - let startsEscapeLiteral = (ctx.grammar.contains(.escapeStringPrefix) && (char == kCapitalE || char == kSmallE)) - || (ctx.grammar.contains(.escapeBytesPrefix) && char == kSmallB) - if startsEscapeLiteral, !SQLNonCodeSpan.continuesName(beforePrefix: ctx.previousUnit, grammar: ctx.grammar) { - if i + 1 < bufLen { - if nsBuffer.character(at: i + 1) == kSingleQuote { return .none } - } else if !ctx.atEndOfInput { - ctx.previousUnitInWord = followsWord - return .needsMoreData - } - } - if readsAlternativeQuotes, !SQLNonCodeSpan.continuesName(beforePrefix: ctx.previousUnit, grammar: ctx.grammar), - let quoteLength = alternativeQuotePrefixLength(nsBuffer, at: i, bufLen: bufLen) { - if quoteLength > 0 { - return .opensAlternativeQuote(prefixLength: quoteLength) - } - guard ctx.atEndOfInput else { - ctx.previousUnitInWord = followsWord - return .needsMoreData - } + guard SqlBlockStructure.startsWord(nsBuffer, at: i, length: bufLen, grammar: ctx.grammar), !followsWord else { + return false } - guard !followsWord else { return .none } ctx.previousUnitInWord = true if ctx.heldEnd != nil || ctx.boundaries?.needsWords == true { ctx.wordStart = ctx.currentStatement?.length ?? 0 ctx.wordStartLine = ctx.currentLine ctx.word.append(char) } - return .continues - } - - private enum WordStep { - case none - case continues - case needsMoreData - case opensAlternativeQuote(prefixLength: Int) - } - - /// The length of `q'` or `nq'` at `i`, zero when the buffer ends before the delimiter, or nil - /// when `i` does not start one. - private static func alternativeQuotePrefixLength(_ buffer: NSString, at i: Int, bufLen: Int) -> Int? { - var cursor = i - let first = buffer.character(at: cursor) - if first == SqlLexer.smallN || first == SqlLexer.capitalN { - cursor += 1 - } - guard cursor < bufLen else { return 0 } - let prefix = buffer.character(at: cursor) - guard prefix == SqlLexer.smallQ || prefix == SqlLexer.capitalQ else { return nil } - guard cursor + 1 < bufLen else { return 0 } - guard buffer.character(at: cursor + 1) == kSingleQuote else { return nil } - guard cursor + 2 < bufLen else { return 0 } - let delimiter = buffer.character(at: cursor + 2) - guard !isWhitespace(delimiter) else { return nil } - guard UTF16.isLeadSurrogate(delimiter) else { return cursor + 3 - i } - guard cursor + 3 < bufLen else { return 0 } - return UTF16.isTrailSurrogate(buffer.character(at: cursor + 3)) ? cursor + 4 - i : cursor + 3 - i - } - - private static func alternativeQuoteCloser(for opener: unichar) -> unichar { - switch opener { - case 0x5B: return 0x5D - case 0x28: return 0x29 - case 0x7B: return 0x7D - case 0x3C: return 0x3E - default: return opener - } + return true } /// Settles a `/` held back at the start of a line once the line's end shows whether it stood alone. @@ -512,7 +389,6 @@ final class SQLFileParser: Sendable { private static func processNormalChar( _ ctx: inout ParserContext, char: unichar, - nextChar: unichar?, i: inout Int, nsBuffer: NSString, bufLen: Int @@ -530,147 +406,29 @@ final class SQLFileParser: Sendable { return step } - switch observeWordCharacter(&ctx, char: char, i: i, nsBuffer: nsBuffer, bufLen: bufLen) { - case .continues: - if !ctx.hasStatementContent { - ctx.statementOpensLine = !ctx.lineHasCode - } - (ctx.hasStatementContent, ctx.statementStartLine) = markContent( - ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) - appendChar(char, to: ctx.currentStatement) - return StepResult(advanced: false, deferred: false) - case .needsMoreData: - return StepResult(advanced: false, deferred: true) - case let .opensAlternativeQuote(prefixLength): - settleHeldEnd(&ctx, word: nil) - (ctx.hasStatementContent, ctx.statementStartLine) = markContent( - ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) - appendRange(&ctx, from: i, to: i + prefixLength, in: nsBuffer) - let delimiterEnd = nsBuffer.character(at: i + prefixLength - 1) - if UTF16.isTrailSurrogate(delimiterEnd) { - ctx.alternativeQuoteCloser = nsBuffer.character(at: i + prefixLength - 2) - ctx.alternativeQuoteCloserTrail = delimiterEnd - } else { - ctx.alternativeQuoteCloser = alternativeQuoteCloser(for: delimiterEnd) - ctx.alternativeQuoteCloserTrail = nil - } - ctx.state = .inAlternativeQuote - ctx.boundaries?.observeQuotedToken(opener: char) - i += prefixLength + // The mysql client looks for its delimiter before it looks for a comment, as the statement scanner does. + if !ctx.isSingleCharDelimiter, !readsDelimiterDirective(ctx), + matchesDelimiter(at: i, delimiter: ctx.currentDelimiter, in: nsBuffer, bufLen: bufLen) { + yieldAndReset(&ctx) + i += ctx.currentDelimiter.length return StepResult(advanced: true, deferred: false) - case .none: - break } - processDelimiterChange(&ctx, char: char) - - if char == kDash && nextChar == kDash, !ctx.grammar.contains(.withoutDashComments) { - if ctx.grammar.contains(.dashCommentsNeedWhitespace), i + 2 >= bufLen, !ctx.atEndOfInput { - return StepResult(advanced: false, deferred: true) - } - if SqlLexer.startsDashComment( - nsBuffer, - at: i, - length: bufLen, - needsWhitespace: ctx.grammar.contains(.dashCommentsNeedWhitespace) - ) { - ctx.state = .inSingleLineComment - ctx.boundaries?.observeGap() - if ctx.keepsComments { - appendRange(&ctx, from: i, to: i + 2, in: nsBuffer) - } - i += 2 - return StepResult(advanced: true, deferred: false) - } - } - - if (char == kHash && ctx.grammar.contains(.hashLineComments)) - || (char == kSlash && nextChar == kSlash && ctx.grammar.contains(.doubleSlashLineComments)) { - ctx.state = .inSingleLineComment - ctx.boundaries?.observeGap() - if ctx.keepsComments { - appendChar(char, to: ctx.currentStatement) - } - return StepResult(advanced: false, deferred: false) - } - - if let step = processBlockCommentMarker(&ctx, char: char, nextChar: nextChar, i: &i, nsBuffer: nsBuffer, bufLen: bufLen) { + if SQLNonCodeSpan.mayStart(char), let step = readSpan(&ctx, at: &i, nsBuffer: nsBuffer, bufLen: bufLen) { return step } - let startsEscapeString = ctx.grammar.contains(.escapeStringPrefix) && (char == kCapitalE || char == kSmallE) - let startsEscapeBytes = ctx.grammar.contains(.escapeBytesPrefix) && char == kSmallB - if (startsEscapeString || startsEscapeBytes) - && nextChar == kSingleQuote - && !SQLNonCodeSpan.continuesName(beforePrefix: ctx.previousUnit, grammar: ctx.grammar) { - settleHeldEnd(&ctx, word: nil) - (ctx.hasStatementContent, ctx.statementStartLine) = markContent( - ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) - appendChar(char, to: ctx.currentStatement) - appendChar(kSingleQuote, to: ctx.currentStatement) - ctx.state = .inSingleQuotedString - ctx.quoteChar = kSingleQuote - ctx.backslashEscapesActive = true - ctx.boundaries?.observeQuotedToken(opener: char) - i += 2 - return StepResult(advanced: true, deferred: false) - } - - if char == kOpenBracket, ctx.grammar.contains(.bracketQuotedIdentifiers) { - settleHeldEnd(&ctx, word: nil) + if observeWordCharacter(&ctx, char: char, i: i, nsBuffer: nsBuffer, bufLen: bufLen) { + if !ctx.hasStatementContent { + ctx.statementOpensLine = !ctx.lineHasCode + } (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) appendChar(char, to: ctx.currentStatement) - ctx.state = .inBracketedIdentifier - ctx.boundaries?.observeQuotedToken(opener: char) return StepResult(advanced: false, deferred: false) } - if let style = ctx.grammar.dollarQuoteStyle, char == kDollar { - let grammar = ctx.grammar - switch SqlDollarQuote.scanOpener( - at: i, - in: nsBuffer, - bufLen: bufLen, - style: style, - continuesName: { SQLNonCodeSpan.continuesName(beforeDollar: $0, grammar: grammar) } - ) { - case .opener(let length, let tag): - settleHeldEnd(&ctx, word: nil) - (ctx.hasStatementContent, ctx.statementStartLine) = markContent( - ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) - if let target = ctx.currentStatement { - let openerRange = NSRange(location: i, length: length) - target.append(nsBuffer.substring(with: openerRange)) - } - ctx.boundaries?.observeQuotedToken(opener: char) - ctx.state = .inDollarQuote - ctx.dollarTag = tag - i += length - return StepResult(advanced: true, deferred: false) - case .needsMoreData where !ctx.atEndOfInput: - return StepResult(advanced: false, deferred: true) - case .needsMoreData, .notOpener: - break - } - } - - if isQuote(char, grammar: ctx.grammar) { - if ctx.grammar.contains(.tripleQuotedStrings), char != kBacktick, i + 2 >= bufLen, !ctx.atEndOfInput { - return StepResult(advanced: false, deferred: true) - } - if ctx.grammar.contains(.tripleQuotedStrings), SqlLexer.startsTripleQuote(nsBuffer, at: i, length: bufLen) { - openTripleQuote(&ctx, quote: char, at: i, in: nsBuffer) - i += 3 - return StepResult(advanced: true, deferred: false) - } - } - - if let advanced = processQuoteOpen(&ctx, char: char, nextChar: nextChar) { - ctx.boundaries?.observeQuotedToken(opener: char) - if advanced { i += 2 } - return StepResult(advanced: advanced, deferred: false) - } + processDelimiterChange(&ctx, char: char) if ctx.isSingleCharDelimiter && char == kSemicolon { processSemicolon(&ctx, at: i) @@ -682,17 +440,59 @@ final class SQLFileParser: Sendable { return StepResult(advanced: false, deferred: false) } - if !ctx.isSingleCharDelimiter, !ctx.insideConditionalComment, !readsDelimiterDirective(ctx), - matchesDelimiter(at: i, delimiter: ctx.currentDelimiter, in: nsBuffer, bufLen: bufLen) { - yieldAndReset(&ctx) - i += ctx.currentDelimiter.length - return StepResult(advanced: true, deferred: false) - } - appendPlainUnit(&ctx, char: char) return StepResult(advanced: false, deferred: false) } + private static func readSpan( + _ ctx: inout ParserContext, + at i: inout Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult? { + let span = SQLNonCodeSpan.span(at: i, in: nsBuffer, grammar: ctx.grammar) + if !ctx.atEndOfInput, SQLNonCodeSpan.needsMoreText(span, at: i, in: nsBuffer, grammar: ctx.grammar) { + ctx.heldSpanLength = bufLen - i + return StepResult(advanced: false, deferred: true) + } + guard let span else { return nil } + flushWord(&ctx) + ctx.previousUnitInWord = false + switch span.kind { + case .lineComment, .blockComment: + ctx.boundaries?.observeGap() + if ctx.keepsComments { + appendRange(&ctx, from: span.start, to: span.end, in: nsBuffer) + } else if span.kind == .blockComment { + // The server reads a comment as a blank, so `a/* x */FROM` must not become `aFROM`. + appendChar(kSpace, to: ctx.currentStatement) + } + case .executableComment, .quoted, .parameter: + settleHeldEnd(&ctx, word: nil) + (ctx.hasStatementContent, ctx.statementStartLine) = markContent( + ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) + appendRange(&ctx, from: span.start, to: span.end, in: nsBuffer) + if span.kind == .quoted { + ctx.boundaries?.observeQuotedToken(opener: nsBuffer.character(at: span.start)) + } else if span.kind == .parameter { + ctx.boundaries?.observeOpaqueToken() + } + } + ctx.currentLine += lineFeeds(in: nsBuffer, from: span.start + 1, to: span.end) + i = span.end + return StepResult(advanced: true, deferred: false) + } + + private static func lineFeeds(in buffer: NSString, from start: Int, to end: Int) -> Int { + var count = 0 + var index = start + while index < end { + if buffer.character(at: index) == kNewline { count += 1 } + index += 1 + } + return count + } + // Whitespace, or a symbol no rule above claimed. private static func appendPlainUnit(_ ctx: inout ParserContext, char: unichar) { if isWhitespace(char) { @@ -709,53 +509,11 @@ final class SQLFileParser: Sendable { appendChar(char, to: ctx.currentStatement) } - /// A `/*` opens a comment, or the code of a conditional comment the server runs, which a `*/` read as code closes. - private static func processBlockCommentMarker( - _ ctx: inout ParserContext, - char: unichar, - nextChar: unichar?, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult? { - if ctx.insideConditionalComment, char == kStar, nextChar == kSlash { - appendRange(&ctx, from: i, to: i + 2, in: nsBuffer) - ctx.insideConditionalComment = false - i += 2 - return StepResult(advanced: true, deferred: false) - } - guard char == kSlash, let next = nextChar, next == kStar else { return nil } - if ctx.grammar.contains(.executableComments) { - // `/*M!` takes four units to tell apart from a plain comment. - if i + 3 >= bufLen, !ctx.atEndOfInput { - return StepResult(advanced: false, deferred: true) - } - if SqlLexer.executableCommentOpenerLength(nsBuffer, at: i, length: bufLen) != nil { - settleHeldEnd(&ctx, word: nil) - (ctx.hasStatementContent, ctx.statementStartLine) = markContent( - ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) - ctx.insideConditionalComment = true - appendRange(&ctx, from: i, to: i + 2, in: nsBuffer) - i += 2 - return StepResult(advanced: true, deferred: false) - } - } - ctx.commentDepth = 1 - ctx.state = .inMultiLineComment - ctx.boundaries?.observeGap() - if ctx.keepsComments { - appendChar(char, to: ctx.currentStatement) - appendChar(next, to: ctx.currentStatement) - } - i += 2 - return StepResult(advanced: true, deferred: false) - } - /// A `;` ends the statement unless the grammar holds it inside a PL/SQL unit or a routine body, and a statement /// that owns the `;` that ends it keeps it. In a batch it ends nothing, unless the batch has grown past what the /// server takes in one request. private static func processSemicolon(_ ctx: inout ParserContext, at i: Int) { - if ctx.insideConditionalComment || readsDelimiterDirective(ctx) { + if readsDelimiterDirective(ctx) { appendChar(kSemicolon, to: ctx.currentStatement) return } @@ -804,48 +562,6 @@ final class SQLFileParser: Sendable { yieldAndReset(&ctx) } - private static func isQuote(_ char: unichar, grammar: SQLLexicalGrammar) -> Bool { - grammar.isQuote(char) - } - - private static func processQuoteOpen( - _ ctx: inout ParserContext, - char: unichar, - nextChar: unichar? - ) -> Bool? { - guard isQuote(char, grammar: ctx.grammar) else { return nil } - settleHeldEnd(&ctx, word: nil) - if let next = nextChar, next == char { - (ctx.hasStatementContent, ctx.statementStartLine) = markContent( - ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) - appendChar(char, to: ctx.currentStatement) - appendChar(next, to: ctx.currentStatement) - return true - } - switch char { - case kSingleQuote: ctx.state = .inSingleQuotedString - case kDoubleQuote: ctx.state = .inDoubleQuotedString - default: ctx.state = .inBacktickQuotedString - } - ctx.quoteChar = char - ctx.backslashEscapesActive = ctx.grammar.backslashEscapes(inQuote: char) - (ctx.hasStatementContent, ctx.statementStartLine) = markContent( - ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) - appendChar(char, to: ctx.currentStatement) - return false - } - - private static func openTripleQuote(_ ctx: inout ParserContext, quote: unichar, at i: Int, in buffer: NSString) { - settleHeldEnd(&ctx, word: nil) - (ctx.hasStatementContent, ctx.statementStartLine) = markContent( - ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) - appendRange(&ctx, from: i, to: i + 3, in: buffer) - ctx.state = .inTripleQuotedString - ctx.quoteChar = quote - ctx.backslashEscapesActive = ctx.grammar.backslashEscapes(inQuote: quote) - ctx.boundaries?.observeQuotedToken(opener: quote) - } - private static func yieldAndReset(_ ctx: inout ParserContext) { if ctx.hasStatementContent { let text = trimmedStatement(ctx) @@ -918,37 +634,6 @@ final class SQLFileParser: Sendable { return .separator(separator) } - private static func processMultiLineComment( - _ ctx: inout ParserContext, - char: unichar, - nextChar: unichar?, - i: inout Int - ) -> Bool { - if ctx.keepsComments { - appendChar(char, to: ctx.currentStatement) - } - if char == kSlash, nextChar == kStar, ctx.grammar.contains(.nestedBlockComments) { - if ctx.keepsComments { - appendChar(kStar, to: ctx.currentStatement) - } - ctx.commentDepth += 1 - i += 2 - return true - } - if char == kStar, let next = nextChar, next == kSlash { - if ctx.keepsComments { - appendChar(next, to: ctx.currentStatement) - } - ctx.commentDepth -= 1 - i += 2 - guard ctx.commentDepth <= 0 else { return true } - ctx.state = .normal - ctx.commentDepth = 0 - return true - } - return false - } - static func appendRange( _ ctx: inout ParserContext, from start: Int, @@ -1198,6 +883,11 @@ extension SQLFileParser { return } try readChunk() + let held = ctx.heldSpanLength + ctx.heldSpanLength = 0 + while held > 0, !ctx.atEndOfInput, nsBuffer.length - readOffset < 2 * held { + try readChunk() + } } } @@ -1228,10 +918,11 @@ extension SQLFileParser { ) } - if readOffset > 0 { - nsBuffer.deleteCharacters(in: NSRange(location: 0, length: readOffset)) - ctx.unitsBeforeBuffer += readOffset - readOffset = 0 + let consumed = readOffset - min(readOffset, SQLFileParser.lookBehind) + if consumed > 0 { + nsBuffer.deleteCharacters(in: NSRange(location: 0, length: consumed)) + ctx.unitsBeforeBuffer += consumed + readOffset -= consumed } nsBuffer.append(chunk) } @@ -1244,12 +935,10 @@ extension SQLFileParser { while i < bufLen { let stepStart = i let char = nsBuffer.character(at: i) - let nextChar: unichar? = (i + 1 < bufLen) ? nsBuffer.character(at: i + 1) : nil - - if nextChar == nil && !ctx.atEndOfInput && SQLFileParser.needsLookahead( + if !ctx.atEndOfInput && SQLFileParser.needsLookahead( char, + available: bufLen - i, state: ctx.state, - grammar: ctx.grammar, delimiter: ctx.currentDelimiter, isSingleCharDelimiter: ctx.isSingleCharDelimiter ) { @@ -1261,58 +950,13 @@ extension SQLFileParser { switch ctx.state { case .normal: - step = SQLFileParser.processNormalChar( - &ctx, char: char, nextChar: nextChar, i: &i, nsBuffer: nsBuffer, bufLen: bufLen) - - case .inSingleLineComment: - if ctx.keepsComments { - SQLFileParser.appendChar(char, to: ctx.currentStatement) - } - if char == SQLFileParser.kNewline - || (char == SQLFileParser.kCarriageReturn - && ctx.grammar.contains(.carriageReturnEndsLineComments)) - || (char == SqlLexer.formFeed && ctx.grammar.contains(.formFeedEndsLineComments)) { - ctx.state = .normal - } - // `DELIMITER // -- note` ends at this line feed, which the comment took. - if char == SQLFileParser.kNewline { - SQLFileParser.processDelimiterChange(&ctx, char: char) - } - step = StepResult(advanced: false, deferred: false) + step = SQLFileParser.processNormalChar(&ctx, char: char, i: &i, nsBuffer: nsBuffer, bufLen: bufLen) case .inClientLine: if char == SQLFileParser.kNewline { ctx.state = .normal } step = StepResult(advanced: false, deferred: false) - - case .inMultiLineComment: - let advanced = SQLFileParser.processMultiLineComment(&ctx, char: char, nextChar: nextChar, i: &i) - step = StepResult(advanced: advanced, deferred: false) - - case .inSingleQuotedString: - step = SQLFileParser.processQuotedString( - &ctx, quoteChar: SQLFileParser.kSingleQuote, i: &i, nsBuffer: nsBuffer, bufLen: bufLen) - - case .inDoubleQuotedString: - step = SQLFileParser.processQuotedString( - &ctx, quoteChar: SQLFileParser.kDoubleQuote, i: &i, nsBuffer: nsBuffer, bufLen: bufLen) - - case .inBacktickQuotedString: - step = SQLFileParser.processQuotedString( - &ctx, quoteChar: SQLFileParser.kBacktick, i: &i, nsBuffer: nsBuffer, bufLen: bufLen) - - case .inTripleQuotedString: - step = SQLFileParser.processTripleQuotedString(&ctx, i: &i, nsBuffer: nsBuffer, bufLen: bufLen) - - case .inBracketedIdentifier: - step = SQLFileParser.processBracketedIdentifier(&ctx, i: &i, nsBuffer: nsBuffer, bufLen: bufLen) - - case .inDollarQuote: - step = SQLFileParser.processDollarQuote(&ctx, i: &i, nsBuffer: nsBuffer, bufLen: bufLen) - - case .inAlternativeQuote: - step = SQLFileParser.processAlternativeQuote(&ctx, i: &i, nsBuffer: nsBuffer, bufLen: bufLen) } if step.deferred { break } @@ -1322,9 +966,6 @@ extension SQLFileParser { ctx.lineHasCode = true } if !step.advanced { i += 1 } - if i > 0, i <= bufLen { - ctx.previousUnit = nsBuffer.character(at: i - 1) - } if ctx.state != .normal { ctx.previousUnitInWord = false } diff --git a/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift b/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift index 61e756f2ae..65c326ad7d 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift @@ -49,7 +49,7 @@ enum SQLQueryFingerprint { /// The readable shape, used to label a group in the UI. static func normalize(_ sql: String, databaseType: DatabaseType) -> String { - let source = strippingDollarQuoted(truncated(sql)) + let source = truncated(sql) let treatsDoubleQuoteAsString = Self.treatsDoubleQuoteAsString(databaseType) var parts: [Part] = [] @@ -57,7 +57,7 @@ enum SQLQueryFingerprint { var previousType: SQLTokenType? var previousValue = "" - for token in SQLTokenizer().tokenize(source) { + for token in SQLTokenizer().tokenize(source, grammar: databaseType.lexicalGrammar) { defer { if token.type != .comment, token.type != .whitespace { previousType = token.type @@ -69,6 +69,15 @@ enum SQLQueryFingerprint { case .comment, .whitespace, .directive: continue + // MySQL's `/*! ... */` hint arrives as one token, and it was a comment to the shape before grammars. + case .string where token.value.hasPrefix("/*"): + continue + + // `@@version` names a server variable: two of them are two shapes, and neither is a literal. + case .placeholder where token.value.hasPrefix("@@"): + parts.append(Part(token.value)) + previousWasLiteral = false + case .number, .placeholder: appendLiteral(to: &parts, previousWasLiteral: &previousWasLiteral) @@ -110,35 +119,7 @@ enum SQLQueryFingerprint { } } - return join(collapsingLists(mergingBracketIdentifiers(parts))) - } - - /// The tokenizer has no rule for T-SQL's `[name]`, so it arrives as three tokens and the - /// identifier never matches its unbracketed spelling. Only a bracket wrapping exactly one - /// identifier is merged, which leaves a PostgreSQL array subscript (`a[?]`) alone. - private static func mergingBracketIdentifiers(_ parts: [Part]) -> [Part] { - guard parts.contains(where: { $0.text == "[" }) else { return parts } - - var out: [Part] = [] - out.reserveCapacity(parts.count) - var index = 0 - - while index < parts.count { - if parts[index].text == "[", index + 2 < parts.count, parts[index + 2].text == "]", - isPlainIdentifier(parts[index + 1].text) { - out.append(parts[index + 1]) - index += 3 - continue - } - out.append(parts[index]) - index += 1 - } - return out - } - - private static func isPlainIdentifier(_ value: String) -> Bool { - guard let first = value.first, first.isLetter || first == "_" else { return false } - return value.allSatisfy { $0.isLetter || $0.isNumber || $0 == "_" } + return join(collapsingLists(parts)) } /// A process-stable digest of `normalize`. `Hasher` is seeded per process, so it can never back @@ -222,42 +203,6 @@ enum SQLQueryFingerprint { return source.substring(to: maxSourceLength) } - /// The tokenizer has no dollar-quote rule, so `$$body$$` reaches it as loose identifiers and - /// the literal survives into the shape. Replacing each one with an empty string literal lets - /// the tokenizer see the placeholder it would have seen for any other literal. - private static func strippingDollarQuoted(_ sql: String) -> String { - guard sql.contains("$") else { return sql } - - let buffer = sql as NSString - let length = buffer.length - var out = "" - out.reserveCapacity(length) - var index = 0 - - while index < length { - let character = buffer.character(at: index) - guard character == SqlDollarQuote.dollar, - case .opener(let openerLength, let tag) = SqlDollarQuote.scanOpener( - at: index, - in: buffer, - bufLen: length - ) - else { - out += buffer.substring(with: NSRange(location: index, length: 1)) - index += 1 - continue - } - - var scan = index + openerLength - while scan < length, !SqlDollarQuote.matchesClose(at: scan, tag: tag, in: buffer, bufLen: length) { - scan += 1 - } - out += "''" - index = scan < length ? scan + (tag as NSString).length + 2 : length - } - return out - } - /// `IN (?, ?)` and `IN (?, ?, ?)` are one lookup asked with different cardinality, and a /// multi-row `INSERT … VALUES (?, ?), (?, ?)` is one insert with a different batch size. /// Both collapse to MySQL's `(...)` spelling, which a reader of `DIGEST_TEXT` will recognise. diff --git a/apple/TableProMobile/TableProMobile/Helpers/TransactionRollBackClassifier.swift b/apple/TableProMobile/TableProMobile/Helpers/TransactionRollBackClassifier.swift index 10c37fbb27..ef2f9b4f60 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/TransactionRollBackClassifier.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/TransactionRollBackClassifier.swift @@ -1,4 +1,5 @@ import Foundation +import TableProSQLGrammar nonisolated enum TransactionRollBackClassifier { private static let optionalNouns: Set = ["WORK", "TRANSACTION", "TRAN"] @@ -7,12 +8,12 @@ nonisolated enum TransactionRollBackClassifier { // A name after ROLLBACK TRAN may be a savepoint, and TO, PREPARED or FORCE target something else. private static let otherTarget = "?" - static func rollsBackTransaction(_ sql: String, statusMessage: String? = nil) -> Bool { + static func rollsBackTransaction(_ sql: String, grammar: SQLLexicalGrammar, statusMessage: String? = nil) -> Bool { // PostgreSQL answers a COMMIT in a failed transaction by rolling it back. if let statusMessage, statusMessage == PostgreSQLSessionTransaction.failedTransactionRolledBackMessage { return true } - let words = leadingWords(of: sql, limit: 3) + let words = leadingWords(of: sql, limit: 3, grammar: grammar) switch words.first ?? "" { case "ABORT": return true @@ -28,26 +29,35 @@ nonisolated enum TransactionRollBackClassifier { } } - private static func leadingWords(of sql: String, limit: Int) -> [String] { + // Comments end where the engine ends them: a MySQL `#` line, a nested PostgreSQL `/* */`. + private static func leadingWords(of sql: String, limit: Int, grammar: SQLLexicalGrammar) -> [String] { + let text = sql as NSString var words: [String] = [] - var rest = Substring(sql) - while words.count < limit { - rest = rest.drop(while: \.isWhitespace) - if rest.hasPrefix("--") { - rest = rest.drop { !$0.isNewline } - } else if rest.hasPrefix("/*") { - // Past the opener: its `*` closes nothing, so `/*/` is still open. - guard let end = rest.dropFirst(2).range(of: "*/") else { break } - rest = rest[end.upperBound...] - } else { - let word = rest.prefix { $0.isLetter || $0.isNumber || $0 == "_" } - guard !word.isEmpty else { - if let next = rest.first, next != ";" { words.append(otherTarget) } + var index = 0 + while words.count < limit, index < text.length { + let unit = text.character(at: index) + if UnicodeScalar(unit).map(CharacterSet.whitespacesAndNewlines.contains) == true { + index += 1 + continue + } + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + guard span.kind.isComment else { + words.append(otherTarget) break } - words.append(word.uppercased()) - rest = rest.dropFirst(word.count) + index = max(span.end, index + 1) + continue + } + var end = index + while end < text.length, SQLNonCodeSpan.isWordUnit(text.character(at: end)) { + end += 1 + } + guard end > index else { + if unit != SqlLexer.semicolon { words.append(otherTarget) } + break } + words.append(text.substring(with: NSRange(location: index, length: end - index)).uppercased()) + index = end } return words } diff --git a/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift b/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift index c72841ce51..73e0a7427f 100644 --- a/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift +++ b/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift @@ -372,7 +372,9 @@ final class QueryEditorViewModel { } rowsAffectedTotal += buffer.rowsAffected ?? 0 if run == runID, script.statements(in: unit).contains(where: { - TransactionRollBackClassifier.rollsBackTransaction($0, statusMessage: buffer.statusMessage) + TransactionRollBackClassifier.rollsBackTransaction( + $0, grammar: script.lexicalGrammar, statusMessage: buffer.statusMessage + ) }) { ranTransactionRollBack = true } diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/TransactionRollBackClassifierTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/TransactionRollBackClassifierTests.swift index a1824f91ae..e3e65df7fe 100644 --- a/apple/TableProMobile/TableProMobileTests/Helpers/TransactionRollBackClassifierTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Helpers/TransactionRollBackClassifierTests.swift @@ -1,4 +1,5 @@ import Foundation +import TableProSQLGrammar @testable import TableProMobile import Testing @@ -20,7 +21,7 @@ struct TransactionRollBackClassifierTests { ] ) func wholeTransactionRollBacks(_ sql: String) { - #expect(TransactionRollBackClassifier.rollsBackTransaction(sql)) + #expect(TransactionRollBackClassifier.rollsBackTransaction(sql, grammar: .ansi)) } @Test( @@ -39,15 +40,25 @@ struct TransactionRollBackClassifierTests { ] ) func otherStatements(_ sql: String) { - #expect(!TransactionRollBackClassifier.rollsBackTransaction(sql)) + #expect(!TransactionRollBackClassifier.rollsBackTransaction(sql, grammar: .ansi)) } @Test("A PostgreSQL COMMIT the server answered by rolling back counts as a Roll Back") func commitRolledBackByPostgreSQL() { #expect(TransactionRollBackClassifier.rollsBackTransaction( "COMMIT", + grammar: .ansi, statusMessage: PostgreSQLSessionTransaction.failedTransactionRolledBackMessage )) - #expect(!TransactionRollBackClassifier.rollsBackTransaction("COMMIT", statusMessage: "COMMIT")) + #expect(!TransactionRollBackClassifier.rollsBackTransaction("COMMIT", grammar: .ansi, statusMessage: "COMMIT")) + } + + @Test("A comment ends where the engine ends it before the ROLLBACK") + func commentsFollowTheGrammar() throws { + let mysql = try #require(SQLLexicalProfile.curated(forDatabaseTypeId: "MySQL")).grammar + let postgreSQL = try #require(SQLLexicalProfile.curated(forDatabaseTypeId: "PostgreSQL")).grammar + #expect(TransactionRollBackClassifier.rollsBackTransaction("# undo\nROLLBACK", grammar: mysql)) + #expect(TransactionRollBackClassifier.rollsBackTransaction("/* a /* b */ c */ ROLLBACK", grammar: postgreSQL)) + #expect(!TransactionRollBackClassifier.rollsBackTransaction("--x\nROLLBACK", grammar: mysql)) } } diff --git a/apple/TableProTests/Core/Diagnostics/QueryDiagnosticsTests.swift b/apple/TableProTests/Core/Diagnostics/QueryDiagnosticsTests.swift index 92178fed7a..6dbbe044e8 100644 --- a/apple/TableProTests/Core/Diagnostics/QueryDiagnosticsTests.swift +++ b/apple/TableProTests/Core/Diagnostics/QueryDiagnosticsTests.swift @@ -7,12 +7,13 @@ // import Foundation +import TableProSQLGrammar import Testing @testable import TablePro struct QueryDiagnosticsTests { - private let sql = SQLDiagnosticsProducer() + private let sql = SQLDiagnosticsProducer(grammar: TestGrammar.standard) private let mql = MongoDiagnosticsProducer() // MARK: - Restraint @@ -98,6 +99,39 @@ struct QueryDiagnosticsTests { #expect(sql.diagnostics(for: "SELECT 1 // )").count == 1) } + @Test("A bracket inside a span the engine's grammar reads is not code", arguments: [ + (TestGrammar.postgres, "SELECT $$ ) $$"), + (TestGrammar.postgres, "SELECT $f$ ) $f$"), + (TestGrammar.postgres, "SELECT 'C:\\', ')'"), + (TestGrammar.postgres, "SELECT E'\\')'"), + (TestGrammar.postgres, "SELECT 1 /* a /* b */ ) */"), + (TestGrammar.mysql, "SELECT 1 # )\n"), + (TestGrammar.mysql, "SELECT 'it\\'s )'"), + (TestGrammar.sqlServer, "SELECT [a)] FROM t"), + (TestGrammar.sqlServer, "SELECT [a]])] FROM t"), + (TestGrammar.sqlite, "SELECT [a)] FROM t"), + (DatabaseType.clickhouse.lexicalGrammar, "SELECT 1 // )\n"), + ]) + func bracketInsideAGrammarSpan(grammar: SQLLexicalGrammar, text: String) { + #expect(SQLDiagnosticsProducer(grammar: grammar).diagnostics(for: text).isEmpty, "\(text)") + } + + @Test("A closer the engine reads as code is reported", arguments: [ + (TestGrammar.postgres, "SELECT 'C:\\' )"), + (TestGrammar.postgres, "SELECT 1 # )"), + (TestGrammar.mysql, "SELECT 1 /*! ) */"), + (TestGrammar.sqlite, "SELECT `a` )"), + ]) + func closerReadAsCode(grammar: SQLLexicalGrammar, text: String) { + #expect(SQLDiagnosticsProducer(grammar: grammar).diagnostics(for: text).count == 1, "\(text)") + } + + @Test("A block comment PostgreSQL still holds open is reported where it starts") + func nestedCommentLeftOpen() { + let results = SQLDiagnosticsProducer(grammar: TestGrammar.postgres).diagnostics(for: "SELECT 1 /* a /* b */") + #expect(results.map(\.range) == [NSRange(location: 9, length: 2)]) + } + @Test("a double slash starts a comment in a MongoDB query") func testMqlHasLineComments() { #expect(mql.diagnostics(for: "db.users.find({}) // )").isEmpty) diff --git a/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift b/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift index 305a631b6c..5892faa275 100644 --- a/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift +++ b/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift @@ -829,7 +829,7 @@ struct SQLFormatterServiceTests { dialectFunctions: ["toString"], rewritesFunctionCase: false ) - let tokens = SQLTokenizer(dialectKeywords: ["toString"]).tokenize("select toString(x), count(*) from t") + let tokens = SQLTokenizer(dialectKeywords: ["toString"]).tokenize("select toString(x), count(*) from t", grammar: .ansi) #expect(tokenFormatter.format(tokens) == """ SELECT toString(x), count(*) diff --git a/apple/TableProTests/Core/Services/SQLTokenizerTests.swift b/apple/TableProTests/Core/Services/SQLTokenizerTests.swift index 13e35076d3..30691f9ecb 100644 --- a/apple/TableProTests/Core/Services/SQLTokenizerTests.swift +++ b/apple/TableProTests/Core/Services/SQLTokenizerTests.swift @@ -5,17 +5,20 @@ import Foundation import TableProPluginKit +import TableProSQLGrammar @testable import TablePro import Testing struct SQLTokenizerTests { - let tokenizer = SQLTokenizer() + private func tokenize(_ sql: String, _ grammar: SQLLexicalGrammar = .ansi) -> [SQLToken] { + SQLTokenizer().tokenize(sql, grammar: grammar) + } // MARK: - Keywords @Test("Recognizes standard SQL keywords") func standardKeywords() { - let tokens = tokenizer.tokenize("SELECT FROM WHERE") + let tokens = tokenize("SELECT FROM WHERE") let nonWS = tokens.filter { $0.type != .whitespace } #expect(nonWS.count == 3) #expect(nonWS.allSatisfy { $0.type == .keyword }) @@ -23,7 +26,7 @@ struct SQLTokenizerTests { @Test("Keywords are case-insensitive") func keywordCaseInsensitive() { - let tokens = tokenizer.tokenize("select FROM Where") + let tokens = tokenize("select FROM Where") let nonWS = tokens.filter { $0.type != .whitespace } #expect(nonWS.allSatisfy { $0.type == .keyword }) } @@ -32,7 +35,7 @@ struct SQLTokenizerTests { @Test("Recognizes identifiers") func identifiers() { - let tokens = tokenizer.tokenize("users my_table col1") + let tokens = tokenize("users my_table col1") let nonWS = tokens.filter { $0.type != .whitespace } #expect(nonWS.count == 3) #expect(nonWS.allSatisfy { $0.type == .identifier }) @@ -40,7 +43,7 @@ struct SQLTokenizerTests { @Test("Backtick-quoted identifiers") func backtickIdentifiers() { - let tokens = tokenizer.tokenize("`my table`") + let tokens = tokenize("`my table`", TestGrammar.mysql) #expect(tokens.count == 1) #expect(tokens[0].type == .identifier) #expect(tokens[0].value == "`my table`") @@ -50,7 +53,7 @@ struct SQLTokenizerTests { @Test("Single-quoted string") func singleQuotedString() { - let tokens = tokenizer.tokenize("'hello world'") + let tokens = tokenize("'hello world'") #expect(tokens.count == 1) #expect(tokens[0].type == .string) #expect(tokens[0].value == "'hello world'") @@ -58,7 +61,7 @@ struct SQLTokenizerTests { @Test("String with escaped quote") func stringWithEscapedQuote() { - let tokens = tokenizer.tokenize("'it''s'") + let tokens = tokenize("'it''s'") #expect(tokens.count == 1) #expect(tokens[0].type == .string) #expect(tokens[0].value == "'it''s'") @@ -66,7 +69,7 @@ struct SQLTokenizerTests { @Test("String with backslash escape") func stringWithBackslashEscape() { - let tokens = tokenizer.tokenize("'it\\'s'") + let tokens = tokenize("'it\\'s'", TestGrammar.mysql) #expect(tokens.count == 1) #expect(tokens[0].type == .string) } @@ -75,7 +78,7 @@ struct SQLTokenizerTests { @Test("Unterminated string ending in a backslash") func unterminatedStringEndingInBackslash() { - let tokens = tokenizer.tokenize("'C:\\") + let tokens = tokenize("'C:\\") #expect(tokens.count == 1) #expect(tokens[0].type == .string) #expect(tokens[0].value == "'C:\\") @@ -83,7 +86,7 @@ struct SQLTokenizerTests { @Test("Unterminated double-quoted string ending in a backslash") func unterminatedDoubleQuotedStringEndingInBackslash() { - let tokens = tokenizer.tokenize("\"C:\\") + let tokens = tokenize("\"C:\\") #expect(tokens.count == 1) #expect(tokens[0].type == .string) #expect(tokens[0].value == "\"C:\\") @@ -91,7 +94,7 @@ struct SQLTokenizerTests { @Test("Unterminated backtick identifier ending in a backslash") func unterminatedBacktickEndingInBackslash() { - let tokens = tokenizer.tokenize("`C:\\") + let tokens = tokenize("`C:\\", TestGrammar.mysql) #expect(tokens.count == 1) #expect(tokens[0].type == .identifier) #expect(tokens[0].value == "`C:\\") @@ -99,7 +102,7 @@ struct SQLTokenizerTests { @Test("Unterminated string ending in a backslash inside a statement") func unterminatedStringEndingInBackslashInStatement() { - let tokens = tokenizer.tokenize("select * from t where c like 'C:\\") + let tokens = tokenize("select * from t where c like 'C:\\") let nonWS = tokens.filter { $0.type != .whitespace } #expect(nonWS.last?.type == .string) #expect(nonWS.last?.value == "'C:\\") @@ -107,7 +110,7 @@ struct SQLTokenizerTests { @Test("Unterminated string keeps a doubled trailing backslash") func unterminatedStringWithDoubledBackslash() { - let tokens = tokenizer.tokenize("'C:\\\\") + let tokens = tokenize("'C:\\\\") #expect(tokens.count == 1) #expect(tokens[0].type == .string) #expect(tokens[0].value == "'C:\\\\") @@ -115,7 +118,7 @@ struct SQLTokenizerTests { @Test("Unterminated string without an escape") func unterminatedStringWithoutEscape() { - let tokens = tokenizer.tokenize("'abc") + let tokens = tokenize("'abc") #expect(tokens.count == 1) #expect(tokens[0].type == .string) #expect(tokens[0].value == "'abc") @@ -123,7 +126,7 @@ struct SQLTokenizerTests { @Test("Lone quote") func loneQuote() { - let tokens = tokenizer.tokenize("'") + let tokens = tokenize("'") #expect(tokens.count == 1) #expect(tokens[0].type == .string) #expect(tokens[0].value == "'") @@ -133,7 +136,7 @@ struct SQLTokenizerTests { @Test("Unterminated block comment keeps its last character") func unterminatedBlockComment() { - let tokens = tokenizer.tokenize("/* abc") + let tokens = tokenize("/* abc") #expect(tokens.count == 1) #expect(tokens[0].type == .comment) #expect(tokens[0].value == "/* abc") @@ -141,7 +144,7 @@ struct SQLTokenizerTests { @Test("Block comment opener with nothing after it") func emptyUnterminatedBlockComment() { - let tokens = tokenizer.tokenize("/*") + let tokens = tokenize("/*") #expect(tokens.count == 1) #expect(tokens[0].type == .comment) #expect(tokens[0].value == "/*") @@ -149,7 +152,7 @@ struct SQLTokenizerTests { @Test("Terminated block comment is unchanged") func terminatedBlockComment() { - let tokens = tokenizer.tokenize("/* abc */") + let tokens = tokenize("/* abc */") #expect(tokens.count == 1) #expect(tokens[0].type == .comment) #expect(tokens[0].value == "/* abc */") @@ -159,7 +162,7 @@ struct SQLTokenizerTests { @Test("Integer") func integer() { - let tokens = tokenizer.tokenize("42") + let tokens = tokenize("42") #expect(tokens.count == 1) #expect(tokens[0].type == .number) #expect(tokens[0].value == "42") @@ -167,7 +170,7 @@ struct SQLTokenizerTests { @Test("Decimal number") func decimal() { - let tokens = tokenizer.tokenize("3.14") + let tokens = tokenize("3.14") #expect(tokens.count == 1) #expect(tokens[0].type == .number) } @@ -176,7 +179,7 @@ struct SQLTokenizerTests { @Test("Line comment") func lineComment() { - let tokens = tokenizer.tokenize("-- this is a comment\nSELECT 1") + let tokens = tokenize("-- this is a comment\nSELECT 1") let comments = tokens.filter { $0.type == .comment } #expect(comments.count == 1) #expect(comments[0].value == "-- this is a comment") @@ -184,7 +187,7 @@ struct SQLTokenizerTests { @Test("Block comment") func blockComment() { - let tokens = tokenizer.tokenize("/* block */ SELECT 1") + let tokens = tokenize("/* block */ SELECT 1") let comments = tokens.filter { $0.type == .comment } #expect(comments.count == 1) #expect(comments[0].value == "/* block */") @@ -194,7 +197,7 @@ struct SQLTokenizerTests { @Test("Multi-character operators") func multiCharOperators() { - let tokens = tokenizer.tokenize(">= <= <> !=") + let tokens = tokenize(">= <= <> !=") let ops = tokens.filter { $0.type == .operator } #expect(ops.count == 4) #expect(ops.map(\.value) == [">=", "<=", "<>", "!="]) @@ -204,14 +207,14 @@ struct SQLTokenizerTests { @Test("Punctuation tokens") func punctuation() { - let tokens = tokenizer.tokenize("(a, b)") + let tokens = tokenize("(a, b)") let puncts = tokens.filter { $0.type == .punctuation } #expect(puncts.map(\.value) == ["(", ",", ")"]) } @Test("Semicolons") func semicolons() { - let tokens = tokenizer.tokenize("SELECT 1; SELECT 2;") + let tokens = tokenize("SELECT 1; SELECT 2;") let semis = tokens.filter { $0.type == .punctuation && $0.value == ";" } #expect(semis.count == 2) } @@ -220,7 +223,7 @@ struct SQLTokenizerTests { @Test("Question mark placeholder") func questionMarkPlaceholder() { - let tokens = tokenizer.tokenize("WHERE id = ?") + let tokens = tokenize("WHERE id = ?") let placeholders = tokens.filter { $0.type == .placeholder } #expect(placeholders.count == 1) #expect(placeholders[0].value == "?") @@ -228,7 +231,7 @@ struct SQLTokenizerTests { @Test("Named placeholders") func namedPlaceholders() { - let tokens = tokenizer.tokenize("$1 :name @var") + let tokens = tokenize("$1 :name @var") let placeholders = tokens.filter { $0.type == .placeholder } #expect(placeholders.count == 3) #expect(placeholders.map(\.value) == ["$1", ":name", "@var"]) @@ -238,7 +241,7 @@ struct SQLTokenizerTests { @Test("Full SELECT statement tokens") func fullSelectStatement() { - let tokens = tokenizer.tokenize("SELECT id, name FROM users WHERE active = true") + let tokens = tokenize("SELECT id, name FROM users WHERE active = true") let nonWS = tokens.filter { $0.type != .whitespace } // SELECT id , name FROM users WHERE active = true #expect(nonWS.count == 10) @@ -256,7 +259,7 @@ struct SQLTokenizerTests { @Test("Preserves original token values") func preservesOriginalValues() { - let tokens = tokenizer.tokenize("select 'Hello World' from users") + let tokens = tokenize("select 'Hello World' from users") let nonWS = tokens.filter { $0.type != .whitespace } #expect(nonWS[0].value == "select") // preserves original case #expect(nonWS[1].value == "'Hello World'") @@ -274,9 +277,10 @@ struct SQLTokenizerTests { #expect(code("[select]", .mssql) == [SQLToken(type: .identifier, value: "[select]")]) } - @Test("Without a grammar a bracket stays apart, as the query fingerprint reads it") - func bracketWithoutGrammar() { - #expect(tokenizer.tokenize("[users]").map(\.value) == ["[", "users", "]"]) + @Test("Where brackets quote no name, a bracket stays apart") + func bracketWithoutBracketNames() { + #expect(tokenize("[users]").map(\.value) == ["[", "users", "]"]) + #expect(code("tags[i]", .postgresql).map(\.value) == ["tags", "[", "i", "]"]) } @Test("A dollar-quoted body is one string") @@ -345,10 +349,10 @@ struct SQLTokenizerTests { #expect(tokens.map(\.type) == [.number, .string, .string, .string, .identifier]) } - @Test("Without a grammar a word before a quote stays a word, so the fingerprint keeps the column") - func wordBeforeQuoteWithoutGrammar() { - for word in ["a", "b", "N"] { - let tokens = tokenizer.tokenize("SELECT \(word)'alias' FROM t").filter { $0.type != .whitespace } + @Test("A word before a quote stays a word, so the fingerprint keeps the column") + func wordBeforeQuote() { + for word in ["a", "r", "foo"] { + let tokens = tokenize("SELECT \(word)'alias' FROM t").filter { $0.type != .whitespace } #expect(tokens.map(\.value) == ["SELECT", word, "'alias'", "FROM", "t"]) #expect(tokens[1].type == .identifier) } @@ -391,9 +395,10 @@ struct SQLTokenizerTests { #expect(code("a += b == c ** d // e ^= f", .sqlite).map(\.value) == ["a", "+=", "b", "==", "c", "**", "d", "//", "e", "^=", "f"]) } - @Test("A system variable fingerprints as it did before, apart from a literal") + @Test("A system variable keeps its name in the fingerprint, apart from a literal and from another variable") func systemVariableFingerprint() { - #expect(SQLQueryFingerprint.normalize("SELECT @@version", databaseType: .mysql) == "SELECT @ ?") + #expect(SQLQueryFingerprint.normalize("SELECT @@version", databaseType: .mysql) == "SELECT @@version") + #expect(SQLQueryFingerprint.hash("SELECT @@version", databaseType: .mysql) != SQLQueryFingerprint.hash("SELECT @@sql_mode", databaseType: .mysql)) #expect(SQLQueryFingerprint.hash("SELECT @@version", databaseType: .mysql) != SQLQueryFingerprint.hash("SELECT 1", databaseType: .mysql)) } @@ -466,52 +471,51 @@ struct SQLTokenizerTests { } } -// What the query fingerprint read on main, before the formatter read grammars: its hashes are stored, so the reading -// without a grammar must not move. Each expected shape was produced by main's own SQLTokenizer and SQLQueryFingerprint. +// Stored hashes move only with the history store's version, which hashes every row again, so each shape is pinned. struct SQLFingerprintGoldenTests { static let corpus: [(DatabaseType, String, String)] = [ (.postgresql, "SELECT * FROM t WHERE id = 1", "SELECT * FROM t WHERE id = ?"), (.postgresql, "select id, name from users where active = true and role = 'admin'", "SELECT id, name FROM users WHERE active = ? AND role = ?"), - (.mysql, "SELECT @@version", "SELECT @ ?"), - (.mysql, "SELECT @@sql_mode", "SELECT @ ?"), + (.mysql, "SELECT @@version", "SELECT @@version"), + (.mysql, "SELECT @@sql_mode", "SELECT @@sql_mode"), (.mysql, "SELECT 1", "SELECT ?"), - (.mssql, "SELECT @@ROWCOUNT, @@ERROR", "SELECT @ ?, @ ?"), + (.mssql, "SELECT @@ROWCOUNT, @@ERROR", "SELECT @@ROWCOUNT, @@ERROR"), (.sqlite, "SELECT @1, @1ab FROM t WHERE a = @2", "SELECT ?, ? FROM t WHERE a = ?"), (.mssql, "SELECT @x, @y FROM t WHERE a = @p", "SELECT ?, ? FROM t WHERE a = ?"), (.mysql, "SELECT apply - 1 FROM t", "SELECT apply - ? FROM t"), (.mysql, "SELECT apply + 1 FROM t", "SELECT apply + ? FROM t"), (.mysql, "SELECT * FROM apply a", "SELECT * FROM apply a"), - (.mysql, "SELECT 0bar FROM t", "SELECT ? bar FROM t"), - (.mysql, "SELECT 0baz FROM t", "SELECT ? baz FROM t"), - (.mysql, "SELECT 0xFF, 0x1A, 0b101 FROM t", "SELECT ? xFF, ? x1A, ? b101 FROM t"), - (.mysql, "SELECT 2fa_codes.id FROM 2fa_codes", "SELECT ? fa_codes.id FROM ? fa_codes"), - (.mssql, "SELECT N'x', N'longer text' FROM t", "SELECT N ?, N ? FROM t"), - (.mysql, "SELECT X'4D', B'101', _utf8mb4'a' FROM t", "SELECT X ?, B ?, _utf8mb4 ? FROM t"), - (.mysql, "SELECT a'alias', b'alias' FROM t", "SELECT a ?, b ? FROM t"), - (.postgresql, "SELECT U&'d\\0061t', E'it\\'s' FROM t", "SELECT U & ?, E ? FROM t"), - (.oracle, "SELECT q'[it's]' FROM dual", "SELECT q ? s] ?"), - (.mssql, "SELECT [order id], [group] FROM [my table] WHERE [select] = 1", "SELECT[ORDER id], GROUP FROM[my TABLE] WHERE SELECT = ?"), + (.mysql, "SELECT 0bar FROM t", "SELECT 0bar FROM t"), + (.mysql, "SELECT 0baz FROM t", "SELECT 0baz FROM t"), + (.mysql, "SELECT 0xFF, 0x1A, 0b101 FROM t", "SELECT ?, ?, ? FROM t"), + (.mysql, "SELECT 2fa_codes.id FROM 2fa_codes", "SELECT 2fa_codes.id FROM 2fa_codes"), + (.mssql, "SELECT N'x', N'longer text' FROM t", "SELECT ?, ? FROM t"), + (.mysql, "SELECT X'4D', B'101', _utf8mb4'a' FROM t", "SELECT ?, ?, ? FROM t"), + (.mysql, "SELECT a'alias', b'alias' FROM t", "SELECT a ?, ? FROM t"), + (.postgresql, "SELECT U&'d\\0061t', E'it\\'s' FROM t", "SELECT ?, ? FROM t"), + (.oracle, "SELECT q'[it's]' FROM dual", "SELECT ? FROM dual"), + (.mssql, "SELECT [order id], [group] FROM [my table] WHERE [select] = 1", "SELECT order id, group FROM my table WHERE select = ?"), (.mssql, "SELECT * FROM [users]", "SELECT * FROM users"), (.mssql, "SELECT * FROM users", "SELECT * FROM users"), - (.mssql, "SELECT * INTO #tmp FROM t; SELECT * FROM ##global", "SELECT * INTO # tmp FROM t; SELECT * FROM # # global"), - (.mssql, "SELECT Part# FROM Parts", "SELECT Part # FROM Parts"), - (.mssql, "SET @x += 1", "SET ? + = ?"), - (.sqlite, "SELECT * FROM t WHERE a == b", "SELECT * FROM t WHERE a = = b"), - (.oracle, "SELECT y ** 2 FROM dual WHERE a ^= b", "SELECT y * * ? FROM dual WHERE a ^ = b"), - (.duckdb, "SELECT 7 // 2", "SELECT ? / / ?"), - (.mysql, "SELECT a <=> b, @a := 1 FROM t", "SELECT a <= > b, ? : = ? FROM t"), + (.mssql, "SELECT * INTO #tmp FROM t; SELECT * FROM ##global", "SELECT * INTO #tmp FROM t; SELECT * FROM ##global"), + (.mssql, "SELECT Part# FROM Parts", "SELECT Part# FROM Parts"), + (.mssql, "SET @x += 1", "SET ? += ?"), + (.sqlite, "SELECT * FROM t WHERE a == b", "SELECT * FROM t WHERE a == b"), + (.oracle, "SELECT y ** 2 FROM dual WHERE a ^= b", "SELECT y ** ? FROM dual WHERE a ^= b"), + (.duckdb, "SELECT 7 // 2", "SELECT ? // ?"), + (.mysql, "SELECT a <=> b, @a := 1 FROM t", "SELECT a <=> b, ? := ? FROM t"), (.postgresql, "SELECT $$secret payload$$ FROM t", "SELECT ? FROM t"), (.postgresql, "SELECT $tag$body here$tag$ FROM t", "SELECT ? FROM t"), (.postgresql, "CREATE FUNCTION f() RETURNS int AS $$ BEGIN RETURN 1; END; $$ LANGUAGE plpgsql", "CREATE FUNCTION f() RETURNS INT AS ? LANGUAGE plpgsql"), (.postgresql, "SELECT a::int, b::text FROM t", "SELECT a :: INT, b :: TEXT FROM t"), - (.postgresql, "SELECT data->>'name', data->'a', data #>> '{a,b}' FROM t", "SELECT data ->> ?, data -> ?, data # > > ? FROM t"), - (.postgresql, "SELECT * FROM t WHERE tags @> ARRAY['x'] AND d ?| array['a']", "SELECT * FROM t WHERE tags @ > ARRAY[?] AND d ? | array[?]"), - (.postgresql, "SELECT * FROM t WHERE name ~* 'abc' AND name !~ 'x'", "SELECT * FROM t WHERE name ~ * ? AND name ! ~ ?"), - (.postgresql, "SELECT embedding <-> '[1,2,3]' FROM items", "SELECT embedding < -> ? FROM items"), + (.postgresql, "SELECT data->>'name', data->'a', data #>> '{a,b}' FROM t", "SELECT data ->> ?, data -> ?, data #>> ? FROM t"), + (.postgresql, "SELECT * FROM t WHERE tags @> ARRAY['x'] AND d ?| array['a']", "SELECT * FROM t WHERE tags @> ARRAY[?] AND d ?| array[?]"), + (.postgresql, "SELECT * FROM t WHERE name ~* 'abc' AND name !~ 'x'", "SELECT * FROM t WHERE name ~* ? AND name !~ ?"), + (.postgresql, "SELECT embedding <-> '[1,2,3]' FROM items", "SELECT embedding <-> ? FROM items"), (.postgresql, "-- pick\nSELECT * FROM t /* hint */ WHERE id = 1", "SELECT * FROM t WHERE id = ?"), - (.mysql, "SELECT 1 # comment\nFROM t", "SELECT ? # comment FROM t"), - (.mysql, "SELECT 5--3", "SELECT ?"), - (.postgresql, "SELECT 1 /* a /* b */ c */ FROM t", "SELECT ? c * / FROM t"), + (.mysql, "SELECT 1 # comment\nFROM t", "SELECT ? FROM t"), + (.mysql, "SELECT 5--3", "SELECT ? - ?"), + (.postgresql, "SELECT 1 /* a /* b */ c */ FROM t", "SELECT ? FROM t"), (.postgresql, "select a, b -- note\nfrom t where x = 1", "SELECT a, b FROM t WHERE x = ?"), (.mysql, "SELECT /*!40001 SQL_NO_CACHE */ a FROM t", "SELECT a FROM t"), (.postgresql, "SELECT * FROM t WHERE id IN (1, 2, 3)", "SELECT * FROM t WHERE id IN (...)"), @@ -524,15 +528,15 @@ struct SQLFingerprintGoldenTests { (.mysql, "SELECT * FROM t WHERE n = \"bob\"", "SELECT * FROM t WHERE n = ?"), (.postgresql, "SELECT \"email\" FROM \"Users\" WHERE id = 1", "SELECT email FROM Users WHERE id = ?"), (.mysql, "SELECT * FROM `users`", "SELECT * FROM users"), - (.oracle, "SELECT * FROM v$session WHERE sid = 1", "SELECT * FROM v ? WHERE sid = ?"), - (.oracle, "BEGIN x := 1; END;", "BEGIN x : = ?; END;"), - (.mssql, "SELECT 1\nGO\nSELECT 2", "SELECT ? GO SELECT ?"), - (.mysql, "DELIMITER //\nCREATE PROCEDURE p() BEGIN SELECT 1; END //\nDELIMITER ;", "DELIMITER / / CREATE PROCEDURE p() BEGIN SELECT ?; END / / DELIMITER;"), - (.mysql, "DELIMITER $$\nEND$$", "DELIMITER ?"), + (.oracle, "SELECT * FROM v$session WHERE sid = 1", "SELECT * FROM v$session WHERE sid = ?"), + (.oracle, "BEGIN x := 1; END;", "BEGIN x := ?; END;"), + (.mssql, "SELECT 1\nGO\nSELECT 2", "SELECT ? SELECT ?"), + (.mysql, "DELIMITER //\nCREATE PROCEDURE p() BEGIN SELECT 1; END //\nDELIMITER ;", "CREATE PROCEDURE p() BEGIN SELECT ?; END //"), + (.mysql, "DELIMITER $$\nEND$$", "END $$"), (.clickhouse, "SELECT database, table FROM system.columns WHERE database = 'default'", "SELECT DATABASE, TABLE FROM system.columns WHERE DATABASE = ?"), (.postgresql, "SELECT count(*), sum(total) FROM t GROUP BY 1", "SELECT COUNT(*), SUM(total) FROM t GROUP BY ?"), (.postgresql, "SELECT * FROM t WHERE created > now() - interval '1 day'", "SELECT * FROM t WHERE created > now() - INTERVAL ?"), - (.postgresql, "SELECT 'C:\\', 'hello world' FROM t", "SELECT ? hello world ?"), + (.postgresql, "SELECT 'C:\\', 'hello world' FROM t", "SELECT ?, ? FROM t"), (.mysql, "SELECT 'it\\'s' FROM t", "SELECT ? FROM t"), (.postgresql, "SELECT café, naïve FROM t", "SELECT café, naïve FROM t"), (.postgresql, "SELECT * FROM t WHERE id = $1 AND name = :name", "SELECT * FROM t WHERE id = ? AND name = ?"), @@ -546,8 +550,8 @@ struct SQLFingerprintGoldenTests { (.postgresql, "SELECT a FROM t WHERE b = 'x''y'", "SELECT a FROM t WHERE b = ?"), ] - @Test("Every query fingerprints exactly as it did before the formatter read grammars") - func fingerprintsMatchMain() { + @Test("Every query fingerprints as its engine's grammar reads it") + func fingerprintsFollowTheGrammar() { for (type, sql, shape) in Self.corpus { #expect(SQLQueryFingerprint.normalize(sql, databaseType: type) == shape, "\(type.rawValue): \(sql)") } diff --git a/apple/TableProTests/Core/Storage/QueryHistoryFingerprintMigrationTests.swift b/apple/TableProTests/Core/Storage/QueryHistoryFingerprintMigrationTests.swift index bc4d70ee73..d420e01eaf 100644 --- a/apple/TableProTests/Core/Storage/QueryHistoryFingerprintMigrationTests.swift +++ b/apple/TableProTests/Core/Storage/QueryHistoryFingerprintMigrationTests.swift @@ -162,4 +162,30 @@ struct QueryHistoryFingerprintMigrationTests { #expect(snapshot.totals.distinctShapeCount == 1) #expect(snapshot.mostRun.first?.callCount == 2) } + + @Test("Opening a v5 database hashes every row again with the engine's grammar") + func versionSixRehashesWithTheGrammar() async { + let connectionId = UUID() + let url = makeVersion3Database( + queries: ["SELECT 'C:\\' AS p, a FROM t", "SELECT 'D:\\' AS p, a FROM t"], + connectionId: connectionId + ) + let migrated = QueryHistoryStorage(databaseURL: url, removeDatabaseOnDeinit: false) + #expect(await migrated.count(scope: .all) == 2) + + // A v5 store hashed each row without the grammar, and a backslash kept PostgreSQL's literal open. + var db: OpaquePointer? + if sqlite3_open(url.path(percentEncoded: false), &db) == SQLITE_OK { + sqlite3_exec(db, "UPDATE history SET fingerprint_hash = rowid; PRAGMA user_version = 5;", nil, nil, nil) + } + sqlite3_close_v2(db) + + let reopened = QueryHistoryStorage(databaseURL: url, removeDatabaseOnDeinit: true) + let snapshot = await reopened.insights( + QueryInsightsRequest(scope: .connection(connectionId)), + slowestRanking: .totalTime + ) + #expect(snapshot.totals.distinctShapeCount == 1) + #expect(snapshot.mostRun.first?.normalizedQuery == "SELECT ? AS p, a FROM t") + } } diff --git a/apple/TableProTests/Core/Storage/QueryHistoryMigrationTests.swift b/apple/TableProTests/Core/Storage/QueryHistoryMigrationTests.swift index d15c55474a..b0dfcd9976 100644 --- a/apple/TableProTests/Core/Storage/QueryHistoryMigrationTests.swift +++ b/apple/TableProTests/Core/Storage/QueryHistoryMigrationTests.swift @@ -291,7 +291,7 @@ struct QueryHistoryMigrationTests { "database_name", "database_type", "schema_name", "variant_key", "format", "raw_plan", "byte_count", "execution_time", "captured_at", "is_pinned" ]) - #expect(scalarInt(in: url, sql: "PRAGMA user_version;") == 5) + #expect(scalarInt(in: url, sql: "PRAGMA user_version;") == 6) let second = QueryHistoryStorage(databaseURL: url, removeDatabaseOnDeinit: true) #expect(await second.count(scope: .all) == 1) @@ -313,7 +313,7 @@ struct QueryHistoryMigrationTests { let second = QueryHistoryStorage(databaseURL: url, removeDatabaseOnDeinit: true) #expect(await second.count(scope: .all) == 0) - #expect(scalarInt(in: url, sql: "PRAGMA user_version;") == 5) + #expect(scalarInt(in: url, sql: "PRAGMA user_version;") == 6) #expect( scalarInt( in: url, diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserSharedCorpusTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserSharedCorpusTests.swift new file mode 100644 index 0000000000..bf4ac203b8 --- /dev/null +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserSharedCorpusTests.swift @@ -0,0 +1,68 @@ +// +// SQLFileParserSharedCorpusTests.swift +// TableProTests +// +// The corpus the statement scanner and the core's script reader run, read by the import a chunk at a time. The import +// drops the comments between statements' code, so each statement is compared as its code without comments or blanks. +// + +import Foundation +import TableProSQLGrammar +import Testing + +@testable import TablePro + +struct SQLFileParserSharedCorpusTests { + // The code a statement holds: comments dropped, blanks outside literals dropped. + private static func code(_ statement: String, grammar: SQLLexicalGrammar) -> String { + let text = statement as NSString + var units: [UInt16] = [] + var index = 0 + while index < text.length { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + if !span.kind.isComment { + for offset in span.start.. [String] { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try sql.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + var statements: [String] = [] + let parser = SQLFileParser(chunkSize: chunkSize) + for try await (statement, _) in parser.parseFile(url: url, encoding: .utf8, grammar: grammar) { + statements.append(statement) + } + return statements + } + + @Test("The import cuts each corpus case where the engine does, however the file is read") + func importCutsEachCase() async throws { + for testCase in try SharedSQLScriptCorpus.load() { + let type = try #require(testCase.databaseType, "\(testCase.dialect)") + let grammar = type.lexicalGrammar + let runs = testCase.batches.map { batches in + batches.flatMap { batch in Array(repeating: batch.text, count: batch.repeat) } + } ?? testCase.statements + let expected = runs.map { Self.code($0, grammar: grammar) } + for chunkSize in [1, 7, SQLFileParser.defaultChunkSize] { + let read = try await Self.imported(testCase.sql, grammar: grammar, chunkSize: chunkSize) + .map { Self.code($0, grammar: grammar) } + #expect(read == expected, "\(testCase.dialect): \(testCase.name), chunks of \(chunkSize)") + } + } + } +} diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift index f2be88f9fb..6e556dec07 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift @@ -26,6 +26,17 @@ struct SQLFileParserTests { return statements } + private static func parse(_ sql: String, grammar: SQLLexicalGrammar, chunkSize: Int) async throws -> [String] { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try sql.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + var statements: [String] = [] + for try await (statement, _) in SQLFileParser(chunkSize: chunkSize).parseFile(url: url, encoding: .utf8, grammar: grammar) { + statements.append(statement) + } + return statements + } + @Test("Postgres: trailing backslash in value does not desync parser") func postgres_trailing_backslash_value() async throws { let sql = """ @@ -351,6 +362,65 @@ struct SQLFileParserTests { } } + @Test("A dropped comment leaves a blank, so the words around it stay apart") + func droppedCommentKeepsWordsApart() async throws { + let sql = "SELECT a/* note */FROM t;\nSELECT b-- note\nFROM t;" + #expect(try await Self.parse(sql, grammar: TestGrammar.postgres) == ["SELECT a FROM t", "SELECT b\nFROM t"]) + } + + @Test("A string after a line break carries on an E string, as PostgreSQL reads it") + func escapeStringContinuation() async throws { + let sql = "SELECT E'x'\n'\\'; SELECT 2; --';\nSELECT 3;" + #expect(try await Self.parse(sql, grammar: TestGrammar.postgres) == ["SELECT E'x'\n'\\'; SELECT 2; --'", "SELECT 3"]) + } + + @Test("A SQLite Tcl parameter runs to its closing parenthesis, quotes and semicolons included") + func sqliteTclParameter() async throws { + let sql = "SELECT $a('); DROP TABLE t; --')\nSELECT 3;" + #expect(try await Self.parse(sql, grammar: TestGrammar.sqlite) == ["SELECT $a(')", "DROP TABLE t", "SELECT 3"]) + } + + @Test("A client delimiter a chunk cuts is still read whole") + func longDelimiterAcrossChunks() async throws { + let sql = "DELIMITER $$$\nSELECT 1$$$\nDELIMITER ;\nSELECT 2;" + for chunkSize in 1...7 { + let statements = try await Self.parse(sql, grammar: TestGrammar.mysql, chunkSize: chunkSize) + #expect(statements == ["SELECT 1", "SELECT 2"], "chunks of \(chunkSize)") + } + } + + @Test("A dollar tag longer than any fixed lookahead still opens its body when a chunk cuts the tag") + func longDollarTagAcrossChunks() async throws { + let tag = "$" + String(repeating: "t", count: 300) + "$" + let sql = "SELECT \(tag) a; b \(tag);\nSELECT 2;" + for chunkSize in [64, 100, 257, 305] { + let statements = try await Self.parse(sql, grammar: TestGrammar.postgres, chunkSize: chunkSize) + #expect(statements == ["SELECT \(tag) a; b \(tag)", "SELECT 2"], "chunks of \(chunkSize)") + } + } + + @Test("An E string waits for the line after a comment a chunk cuts after its first dash") + func escapeStringContinuationAcrossASplitComment() async throws { + let head = "SELECT E'x'" + String(repeating: " ", count: 300) + let sql = head + "-- c\n'\\'; SELECT 2; --';\nSELECT 3;" + let first = (head as NSString).length + 1 + for chunkSize in [first, first - 1, first + 1, 64] { + let statements = try await Self.parse(sql, grammar: TestGrammar.postgres, chunkSize: chunkSize) + #expect(statements.count == 2, "chunks of \(chunkSize)") + #expect(statements.last == "SELECT 3", "chunks of \(chunkSize)") + } + } + + @Test("A literal many chunks long is read whole, wherever the chunks cut it") + func literalAcrossManyChunks() async throws { + let body = String(repeating: "x;'' ", count: 40_000) + let sql = "SELECT '\(body)';\nSELECT 2;" + for chunkSize in [64, 4_099] { + let statements = try await Self.parse(sql, grammar: TestGrammar.postgres, chunkSize: chunkSize) + #expect(statements == ["SELECT '\(body)'", "SELECT 2"], "chunks of \(chunkSize)") + } + } + @Test("Multi-byte UTF-8 char straddling 64KB chunk boundary parses correctly") func multibyte_utf8_at_chunk_boundary() async throws { let chunkSize = 65_536 diff --git a/apple/TableProTests/Helpers/SharedSQLScriptCorpus.swift b/apple/TableProTests/Helpers/SharedSQLScriptCorpus.swift new file mode 100644 index 0000000000..c3d3ff70b9 --- /dev/null +++ b/apple/TableProTests/Helpers/SharedSQLScriptCorpus.swift @@ -0,0 +1,45 @@ +// +// SharedSQLScriptCorpus.swift +// TableProTests +// +// The script corpus the core's tests and the statement scanner's tests run, with each case's expected statements. +// + +import Foundation +@testable import TablePro + +struct SharedSQLScriptCorpus: Decodable { + struct Batch: Decodable { + let text: String + let `repeat`: Int + } + + struct Case: Decodable { + let name: String + let dialect: String + let sql: String + let statements: [String] + let batches: [Batch]? + + var databaseType: DatabaseType? { + SharedSQLScriptCorpus.dialects[dialect] + } + } + + let cases: [Case] + + private static let dialects: [String: DatabaseType] = [ + "postgresql": .postgresql, "mysql": .mysql, "sqlite": .sqlite, "mssql": .mssql, "clickhouse": .clickhouse, + "duckdb": .duckdb, "cql": .cassandra, + ] + + static func load() throws -> [Case] { + let url = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .appendingPathComponent("core/crates/core/tests/fixtures/sql_script_corpus.json") + return try JSONDecoder().decode(SharedSQLScriptCorpus.self, from: Data(contentsOf: url)).cases + } +} diff --git a/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift b/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift index 871a96b9c0..0124f71b32 100644 --- a/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift +++ b/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift @@ -164,4 +164,39 @@ struct SQLLexicalFeatureMappingTests { #expect(scanned == split, "\(engine.rawValue): \(sql)") } } + + // Each `;` outside what the app's lexer reads as a span ends a piece; a piece of comments and blanks is none. + private static func lexicalPieces(_ sql: String, grammar: SQLLexicalGrammar) -> Int { + let text = sql as NSString + var pieces = 0 + var holdsCode = false + var index = 0 + while index < text.length { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + holdsCode = holdsCode || !span.kind.isComment + index = max(span.end, index + 1) + continue + } + let unit = text.character(at: index) + if unit == SqlLexer.semicolon { + pieces += holdsCode ? 1 : 0 + holdsCode = false + } else if !SqlLexer.isWhitespace(unit) { + holdsCode = true + } + index += 1 + } + return pieces + (holdsCode ? 1 : 0) + } + + @Test("The kit's lexer and the app's lexer end every span of the shared script corpus alike") + func kitLexerAgreesOnTheSharedCorpus() throws { + for testCase in try SharedSQLScriptCorpus.load() { + let type = try #require(testCase.databaseType, "\(testCase.dialect)") + let features = type.lexicalGrammar.pluginFeatures + let split = SQLStatementSplitting.statements(in: testCase.sql, lexicalFeatures: features).count + let lexed = Self.lexicalPieces(testCase.sql, grammar: SQLLexicalGrammar(pluginFeatures: features)) + #expect(split == lexed, "\(testCase.dialect): \(testCase.name)") + } + } } diff --git a/core/crates/core/tests/fixtures/sql_script_corpus.json b/core/crates/core/tests/fixtures/sql_script_corpus.json new file mode 100644 index 0000000000..66954aa0e3 --- /dev/null +++ b/core/crates/core/tests/fixtures/sql_script_corpus.json @@ -0,0 +1,545 @@ +{ + "cases": [ + { + "name": "a semicolon inside a string ends nothing", + "dialect": "postgresql", + "sql": "SELECT 'a;b'; SELECT 2;", + "statements": [ + "SELECT 'a;b'", + "SELECT 2" + ] + }, + { + "name": "a backslash is literal in a plain string", + "dialect": "postgresql", + "sql": "SELECT 'C:\\'; SELECT 2;", + "statements": [ + "SELECT 'C:\\'", + "SELECT 2" + ] + }, + { + "name": "a backslash escapes the quote of an E string", + "dialect": "postgresql", + "sql": "SELECT E'it\\'s; fine'; SELECT 2;", + "statements": [ + "SELECT E'it\\'s; fine'", + "SELECT 2" + ] + }, + { + "name": "a string after a line break carries on an E string with its escapes", + "dialect": "postgresql", + "sql": "SELECT E'x'\n'\\'; SELECT 2; --';\nSELECT 3;", + "statements": [ + "SELECT E'x'\n'\\'; SELECT 2; --'", + "SELECT 3" + ] + }, + { + "name": "a name glued to E is a name before a plain string", + "dialect": "postgresql", + "sql": "SELECT xE'\\'; SELECT 2;", + "statements": [ + "SELECT xE'\\'", + "SELECT 2" + ] + }, + { + "name": "a dollar body hides its semicolons", + "dialect": "postgresql", + "sql": "CREATE FUNCTION f() RETURNS int AS $$ SELECT 1; SELECT 2; $$ LANGUAGE sql;\nSELECT 3;", + "statements": [ + "CREATE FUNCTION f() RETURNS int AS $$ SELECT 1; SELECT 2; $$ LANGUAGE sql", + "SELECT 3" + ] + }, + { + "name": "a tagged dollar body holds an untagged one", + "dialect": "postgresql", + "sql": "DO $body$ BEGIN RAISE NOTICE '$$;'; END $body$;\nSELECT 1;", + "statements": [ + "DO $body$ BEGIN RAISE NOTICE '$$;'; END $body$", + "SELECT 1" + ] + }, + { + "name": "a dollar inside a name opens no body", + "dialect": "postgresql", + "sql": "SELECT a$b$c FROM t; SELECT $1;", + "statements": [ + "SELECT a$b$c FROM t", + "SELECT $1" + ] + }, + { + "name": "block comments nest", + "dialect": "postgresql", + "sql": "SELECT 1 /* a /* b; */ c; */ + 1; SELECT 2;", + "statements": [ + "SELECT 1 /* a /* b; */ c; */ + 1", + "SELECT 2" + ] + }, + { + "name": "a carriage return ends a dash comment", + "dialect": "postgresql", + "sql": "SELECT 1 -- c\r+ 1; SELECT 2;", + "statements": [ + "SELECT 1 -- c\r+ 1", + "SELECT 2" + ] + }, + { + "name": "a hash is an operator", + "dialect": "postgresql", + "sql": "SELECT 1 # 2; SELECT 3;", + "statements": [ + "SELECT 1 # 2", + "SELECT 3" + ] + }, + { + "name": "a quoted name holds a semicolon", + "dialect": "postgresql", + "sql": "SELECT \"a;b\" FROM t; SELECT 2;", + "statements": [ + "SELECT \"a;b\" FROM t", + "SELECT 2" + ] + }, + { + "name": "an unterminated string runs to the end", + "dialect": "postgresql", + "sql": "SELECT 1; SELECT 'open; SELECT 2", + "statements": [ + "SELECT 1", + "SELECT 'open; SELECT 2" + ] + }, + { + "name": "an unterminated dollar body runs to the end", + "dialect": "postgresql", + "sql": "SELECT 1; SELECT $$ open; SELECT 2", + "statements": [ + "SELECT 1", + "SELECT $$ open; SELECT 2" + ] + }, + { + "name": "a backslash escapes in a single-quoted string", + "dialect": "mysql", + "sql": "SELECT 'it\\'s; x'; SELECT 2;", + "statements": [ + "SELECT 'it\\'s; x'", + "SELECT 2" + ] + }, + { + "name": "a backslash escapes in a double-quoted string", + "dialect": "mysql", + "sql": "SELECT \"a\\\"; b\"; SELECT 2;", + "statements": [ + "SELECT \"a\\\"; b\"", + "SELECT 2" + ] + }, + { + "name": "a backslash is literal in a backtick name", + "dialect": "mysql", + "sql": "SELECT `a\\`; SELECT 2;", + "statements": [ + "SELECT `a\\`", + "SELECT 2" + ] + }, + { + "name": "a backtick name holds a semicolon", + "dialect": "mysql", + "sql": "SELECT `a;b` FROM t; SELECT 2;", + "statements": [ + "SELECT `a;b` FROM t", + "SELECT 2" + ] + }, + { + "name": "a hash starts a comment", + "dialect": "mysql", + "sql": "SELECT 1 # c; d\n+ 1; SELECT 2;", + "statements": [ + "SELECT 1 # c; d\n+ 1", + "SELECT 2" + ] + }, + { + "name": "a dash comment needs a space after it", + "dialect": "mysql", + "sql": "SELECT 1--1; SELECT 2 -- c; d\n+ 1; SELECT 3;", + "statements": [ + "SELECT 1--1", + "SELECT 2 -- c; d\n+ 1", + "SELECT 3" + ] + }, + { + "name": "a carriage return does not end a dash comment", + "dialect": "mysql", + "sql": "SELECT 1 -- c\r; SELECT 2\n+ 1; SELECT 3;", + "statements": [ + "SELECT 1 -- c\r; SELECT 2\n+ 1", + "SELECT 3" + ] + }, + { + "name": "block comments do not nest", + "dialect": "mysql", + "sql": "SELECT 1 /* a /* b */ + 1; SELECT 2;", + "statements": [ + "SELECT 1 /* a /* b */ + 1", + "SELECT 2" + ] + }, + { + "name": "an executable comment is code", + "dialect": "mysql", + "sql": "/*!40101 SET NAMES utf8 */; SELECT 2;", + "statements": [ + "/*!40101 SET NAMES utf8 */", + "SELECT 2" + ] + }, + { + "name": "an executable comment ends at the first close read as code", + "dialect": "mysql", + "sql": "SELECT 1 /*! AS '*/' */; SELECT 2 /*!50100 -- */\n*/; SELECT 3;", + "statements": [ + "SELECT 1 /*! AS '*/' */", + "SELECT 2 /*!50100 -- */\n*/", + "SELECT 3" + ] + }, + { + "name": "a DELIMITER line sets what ends a routine", + "dialect": "mysql", + "sql": "DELIMITER //\nCREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END//\nDELIMITER ;\nSELECT 3;", + "statements": [ + "CREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END", + "SELECT 3" + ] + }, + { + "name": "a delimiter inside a string ends nothing", + "dialect": "mysql", + "sql": "DELIMITER $$\nSELECT '$$'$$\nDELIMITER ;\nSELECT 2;", + "statements": [ + "SELECT '$$'", + "SELECT 2" + ] + }, + { + "name": "a bracket name holds a semicolon", + "dialect": "sqlite", + "sql": "SELECT [a;b] FROM t; SELECT 2;", + "statements": [ + "SELECT [a;b] FROM t", + "SELECT 2" + ] + }, + { + "name": "a bracket name ends at its first close", + "dialect": "sqlite", + "sql": "SELECT [a]]; SELECT 2;", + "statements": [ + "SELECT [a]]", + "SELECT 2" + ] + }, + { + "name": "a backslash is literal", + "dialect": "sqlite", + "sql": "SELECT 'C:\\'; SELECT 2;", + "statements": [ + "SELECT 'C:\\'", + "SELECT 2" + ] + }, + { + "name": "a backtick quotes a name", + "dialect": "sqlite", + "sql": "SELECT `a;b` FROM t; SELECT 2;", + "statements": [ + "SELECT `a;b` FROM t", + "SELECT 2" + ] + }, + { + "name": "a Tcl parameter runs to its closing parenthesis", + "dialect": "sqlite", + "sql": "SELECT $a('); SELECT 2; --')", + "statements": [ + "SELECT $a(')", + "SELECT 2" + ] + }, + { + "name": "block comments do not nest", + "dialect": "sqlite", + "sql": "SELECT 1 /* a /* b */ + 1; SELECT 2;", + "statements": [ + "SELECT 1 /* a /* b */ + 1", + "SELECT 2" + ] + }, + { + "name": "a hash is no comment", + "dialect": "sqlite", + "sql": "SELECT 1; SELECT #a;", + "statements": [ + "SELECT 1", + "SELECT #a" + ] + }, + { + "name": "a doubled close bracket escapes", + "dialect": "mssql", + "sql": "SELECT [a]];b] FROM t\nGO\nSELECT 2", + "statements": [ + "SELECT [a]];b] FROM t", + "SELECT 2" + ], + "batches": [ + { + "text": "SELECT [a]];b] FROM t", + "repeat": 1 + }, + { + "text": "SELECT 2", + "repeat": 1 + } + ] + }, + { + "name": "block comments nest", + "dialect": "mssql", + "sql": "SELECT 1 /* a /* b; */ c;\nGO\n*/ + 1\nGO\nSELECT 2", + "statements": [ + "SELECT 1 /* a /* b; */ c;\nGO\n*/ + 1", + "SELECT 2" + ], + "batches": [ + { + "text": "SELECT 1 /* a /* b; */ c;\nGO\n*/ + 1", + "repeat": 1 + }, + { + "text": "SELECT 2", + "repeat": 1 + } + ] + }, + { + "name": "a backslash is literal", + "dialect": "mssql", + "sql": "SELECT 'C:\\'\nGO\nSELECT 2", + "statements": [ + "SELECT 'C:\\'", + "SELECT 2" + ], + "batches": [ + { + "text": "SELECT 'C:\\'", + "repeat": 1 + }, + { + "text": "SELECT 2", + "repeat": 1 + } + ] + }, + { + "name": "GO lines end batches and repeat them", + "dialect": "mssql", + "sql": "INSERT INTO t VALUES (1)\nGO 3\nSELECT COUNT(*) FROM t\ngo\n", + "statements": [ + "INSERT INTO t VALUES (1)", + "SELECT COUNT(*) FROM t" + ], + "batches": [ + { + "text": "INSERT INTO t VALUES (1)", + "repeat": 3 + }, + { + "text": "SELECT COUNT(*) FROM t", + "repeat": 1 + } + ] + }, + { + "name": "GO inside a string separates nothing", + "dialect": "mssql", + "sql": "SELECT 'a\nGO\nb'\nGO\nSELECT 2", + "statements": [ + "SELECT 'a\nGO\nb'", + "SELECT 2" + ], + "batches": [ + { + "text": "SELECT 'a\nGO\nb'", + "repeat": 1 + }, + { + "text": "SELECT 2", + "repeat": 1 + } + ] + }, + { + "name": "GO inside a bracket name separates nothing", + "dialect": "mssql", + "sql": "SELECT [a]]\nGO\nb] FROM t\nGO\nSELECT 2", + "statements": [ + "SELECT [a]]\nGO\nb] FROM t", + "SELECT 2" + ], + "batches": [ + { + "text": "SELECT [a]]\nGO\nb] FROM t", + "repeat": 1 + }, + { + "text": "SELECT 2", + "repeat": 1 + } + ] + }, + { + "name": "GO inside a comment separates nothing", + "dialect": "mssql", + "sql": "SELECT 1 /*\nGO\n*/ + 1\nGO\nSELECT 2", + "statements": [ + "SELECT 1 /*\nGO\n*/ + 1", + "SELECT 2" + ], + "batches": [ + { + "text": "SELECT 1 /*\nGO\n*/ + 1", + "repeat": 1 + }, + { + "text": "SELECT 2", + "repeat": 1 + } + ] + }, + { + "name": "a backslash escapes in a string", + "dialect": "clickhouse", + "sql": "SELECT 'a\\'; b'; SELECT 2;", + "statements": [ + "SELECT 'a\\'; b'", + "SELECT 2" + ] + }, + { + "name": "a hash and a space start a comment", + "dialect": "clickhouse", + "sql": "SELECT 1 # c; d\n+ 1; SELECT 2;", + "statements": [ + "SELECT 1 # c; d\n+ 1", + "SELECT 2" + ] + }, + { + "name": "a double slash starts a comment", + "dialect": "clickhouse", + "sql": "SELECT 1 // c; d\n+ 1; SELECT 2;", + "statements": [ + "SELECT 1 // c; d\n+ 1", + "SELECT 2" + ] + }, + { + "name": "a heredoc hides its semicolons", + "dialect": "clickhouse", + "sql": "SELECT $doc$a;b$doc$; SELECT 2;", + "statements": [ + "SELECT $doc$a;b$doc$", + "SELECT 2" + ] + }, + { + "name": "a dollar body hides its semicolons", + "dialect": "duckdb", + "sql": "SELECT $$a;b$$; SELECT 2;", + "statements": [ + "SELECT $$a;b$$", + "SELECT 2" + ] + }, + { + "name": "a backslash escapes only in an E string", + "dialect": "duckdb", + "sql": "SELECT E'it\\'s; x'; SELECT 'a\\'; SELECT 2;", + "statements": [ + "SELECT E'it\\'s; x'", + "SELECT 'a\\'", + "SELECT 2" + ] + }, + { + "name": "block comments nest", + "dialect": "duckdb", + "sql": "SELECT 1 /* a /* b; */ c; */ + 1; SELECT 2;", + "statements": [ + "SELECT 1 /* a /* b; */ c; */ + 1", + "SELECT 2" + ] + }, + { + "name": "a backtick quotes a name", + "dialect": "duckdb", + "sql": "SELECT `a;b` FROM t; SELECT 2;", + "statements": [ + "SELECT `a;b` FROM t", + "SELECT 2" + ] + }, + { + "name": "a dollar body hides its semicolons", + "dialect": "cql", + "sql": "SELECT $$a;b$$ FROM t; SELECT 2 FROM t;", + "statements": [ + "SELECT $$a;b$$ FROM t", + "SELECT 2 FROM t" + ] + }, + { + "name": "a double slash starts a comment", + "dialect": "cql", + "sql": "SELECT a FROM t // c; d\nWHERE a = 1; SELECT 2 FROM t;", + "statements": [ + "SELECT a FROM t // c; d\nWHERE a = 1", + "SELECT 2 FROM t" + ] + }, + { + "name": "a doubled quote escapes and a backslash does not", + "dialect": "cql", + "sql": "SELECT 'x''y;z', 'd\\' FROM t; SELECT 2 FROM t;", + "statements": [ + "SELECT 'x''y;z', 'd\\' FROM t", + "SELECT 2 FROM t" + ] + }, + { + "name": "a batch is one statement", + "dialect": "cql", + "sql": "BEGIN BATCH INSERT INTO t (a) VALUES (1); INSERT INTO t (a) VALUES (2); APPLY BATCH; SELECT a FROM t;", + "statements": [ + "BEGIN BATCH INSERT INTO t (a) VALUES (1); INSERT INTO t (a) VALUES (2); APPLY BATCH", + "SELECT a FROM t" + ] + } + ] +} diff --git a/core/crates/core/tests/script_shared_corpus.rs b/core/crates/core/tests/script_shared_corpus.rs new file mode 100644 index 0000000000..c437f4f332 --- /dev/null +++ b/core/crates/core/tests/script_shared_corpus.rs @@ -0,0 +1,137 @@ +//! The macOS app reads scripts with its own Swift lexer, and its tests +//! run this same corpus, so both read each case where the engine does. + +use serde::Deserialize; +use tablepro_core::sql_syntax::SqlGrammar; +use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptReadError, ScriptReader, ScriptUnitKind}; + +#[derive(Deserialize)] +struct Corpus { + cases: Vec, +} + +#[derive(Deserialize)] +struct Case { + name: String, + dialect: String, + sql: String, + statements: Vec, + batches: Option>, +} + +#[derive(Deserialize, Clone, PartialEq, Debug)] +struct Batch { + text: String, + repeat: u32, +} + +fn grammar(dialect: &str) -> Option { + match dialect { + "postgresql" => Some(SqlGrammar::PostgreSql), + "mysql" => Some(SqlGrammar::MySql), + "sqlite" => Some(SqlGrammar::Sqlite), + "mssql" => Some(SqlGrammar::MsSql), + "clickhouse" => Some(SqlGrammar::ClickHouse), + "duckdb" => Some(SqlGrammar::DuckDb), + "cql" => Some(SqlGrammar::Cql), + _ => None, + } +} + +fn cases() -> serde_json::Result> { + let corpus: Corpus = serde_json::from_str(include_str!("fixtures/sql_script_corpus.json"))?; + Ok(corpus.cases) +} + +fn read_in_pieces(grammar: SqlGrammar, text: &str, piece: usize) -> Result, ScriptReadError> { + let mut reader = ScriptReader::new(grammar); + let mut units = Vec::new(); + let mut start = 0; + while start < text.len() { + let mut end = (start + piece).min(text.len()); + while !text.is_char_boundary(end) { + end += 1; + } + units.extend(reader.push(&text[start..end])?); + start = end; + } + units.extend(reader.finish()?); + Ok(units + .into_iter() + .filter(|unit| unit.kind == ScriptUnitKind::Run) + .map(|unit| Batch { + text: unit.text, + repeat: unit.repeat, + }) + .collect()) +} + +#[test] +fn every_dialect_the_corpus_names_has_a_grammar() { + let unknown: Vec = cases() + .expect("the corpus") + .into_iter() + .filter(|case| grammar(&case.dialect).is_none()) + .map(|case| case.dialect) + .collect(); + assert!(unknown.is_empty(), "{unknown:?}"); +} + +#[test] +fn the_plan_splits_each_case_where_the_engine_does() { + let mut misread = Vec::new(); + for case in cases().expect("the corpus") { + let Some(grammar) = grammar(&case.dialect) else { + continue; + }; + let plan = ScriptPlan::build(&case.sql, grammar, LexicalSettings::default_for(grammar)); + let statements: Vec<&str> = plan + .statements() + .iter() + .map(|statement| statement.text(&case.sql)) + .collect(); + if statements != case.statements { + misread.push(format!("{}: {statements:?}", case.name)); + } + if let Some(batches) = &case.batches { + let read: Vec = plan + .batches() + .iter() + .map(|batch| Batch { + text: case.sql[batch.range.clone()].to_owned(), + repeat: batch.repeat.get(), + }) + .collect(); + if &read != batches { + misread.push(format!("{} batches: {read:?}", case.name)); + } + } + } + assert!(misread.is_empty(), "{misread:#?}"); +} + +#[test] +fn the_reader_sends_each_case_as_the_client_does_however_it_is_cut() { + let mut misread = Vec::new(); + for case in cases().expect("the corpus") { + let Some(grammar) = grammar(&case.dialect) else { + continue; + }; + let expected = case.batches.clone().unwrap_or_else(|| { + case.statements + .iter() + .map(|text| Batch { + text: text.clone(), + repeat: 1, + }) + .collect() + }); + for piece in [1, 2, 7, case.sql.len().max(1)] { + let read = read_in_pieces(grammar, &case.sql, piece).expect("a script the reader holds"); + if read != expected { + misread.push(format!("{} in pieces of {piece}: {read:?}", case.name)); + } + } + } + assert!(misread.is_empty(), "{misread:#?}"); +} diff --git a/scripts/ci/detect-changed-paths.sh b/scripts/ci/detect-changed-paths.sh index da6e08fb66..583347cce0 100755 --- a/scripts/ci/detect-changed-paths.sh +++ b/scripts/ci/detect-changed-paths.sh @@ -35,7 +35,9 @@ case "$SUITE" in 'apple/scripts/check-pluginkit-abi\.sh$' 'apple/scripts/release/notes\.py$' '\.github/actions/(setup-xcode(gen)?|prepare-build)/' '\.github/workflows/(ci|macos-tests)\.yml$' # The app bundles the MCP client catalog the Linux app compiles in. - 'core/crates/mcp/clients\.json$') ;; + 'core/crates/mcp/clients\.json$' + # The app's tests run the script corpus the core's tests run. + 'core/crates/core/tests/fixtures/sql_script_corpus\.json$') ;; plugins) # Registry plugins link the TableProCore and TableProOracle products. PATTERNS=('apple/Plugins/' 'apple/Packages/TableProCore/' 'apple/Packages/TableProOracle/' 'apple/Native/' @@ -51,8 +53,8 @@ case "$SUITE" in 'apple/scripts/ci/summarize-xcresult\.sh$' 'apple/scripts/localization\.py$' 'scripts/ci/detect-changed-paths\.sh$' '\.github/actions/setup-xcode(gen)?/' '\.github/workflows/(ci|ios-tests)\.yml$' - # The TableProCore package tests read the function names the Linux app shares. - 'core/crates/core/tests/fixtures/state_changing_functions\.json$') + # The TableProCore package tests read the function names and the script corpus the Linux app shares. + 'core/crates/core/tests/fixtures/(state_changing_functions|sql_script_corpus)\.json$') # The iOS project compiles files from Plugins/, Packages/ and TablePro/, so every path its # spec reaches outside TableProMobile/ is read from the spec instead of copied here. SHARED="$(grep -oE '\.\./(Configs|Packages|Plugins|TablePro)/[^"[:space:]]*' apple/TableProMobile/project.yml \ diff --git a/scripts/ci/test_detect_changed_paths.py b/scripts/ci/test_detect_changed_paths.py index e337359cf2..79580a0514 100644 --- a/scripts/ci/test_detect_changed_paths.py +++ b/scripts/ci/test_detect_changed_paths.py @@ -418,6 +418,17 @@ def test_ios_and_linux_run_on_the_function_names_both_apps_share(self): self.assertEqual(self.verdict_after("ios", "core/crates/core/tests/fixtures/other.json"), "false") self.assertEqual(self.verdict_after("macos", shared), "false") + def test_both_apple_suites_and_linux_run_on_the_script_corpus_the_core_shares(self): + spec = self.repo / "apple/TableProMobile/project.yml" + spec.parent.mkdir(parents=True) + spec.write_text("sources:\n - path: ../Packages/TableProCore\n", encoding="utf-8") + self.seed = self.commit([]) + corpus = "core/crates/core/tests/fixtures/sql_script_corpus.json" + for suite in ["macos", "ios", "linux", "core"]: + with self.subTest(suite=suite): + self.assertEqual(self.verdict_after(suite, corpus), "true") + self.assertEqual(self.verdict_after("plugins", corpus), "false") + def test_the_mac_app_runs_on_the_client_catalog_it_bundles(self): catalog = "core/crates/mcp/clients.json" for suite in ["macos", "release", "shipped", "linux", "core"]: From 90761136cb3342769982e60acdd3f69559f91bb7 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 15 Nov 2026 04:20:00 +0700 Subject: [PATCH 02/10] docs(changelog): note the import, diagnostics, grouping and iOS roll back fixes --- apple/changelog.d/4289.fixed.1.md | 1 + apple/changelog.d/4289.fixed.2.md | 1 + apple/changelog.d/4289.fixed.3.md | 1 + apple/changelog.d/4289.fixed.4.md | 1 + apple/changelog.d/4289.fixed.md | 1 + 5 files changed, 5 insertions(+) create mode 100644 apple/changelog.d/4289.fixed.1.md create mode 100644 apple/changelog.d/4289.fixed.2.md create mode 100644 apple/changelog.d/4289.fixed.3.md create mode 100644 apple/changelog.d/4289.fixed.4.md create mode 100644 apple/changelog.d/4289.fixed.md diff --git a/apple/changelog.d/4289.fixed.1.md b/apple/changelog.d/4289.fixed.1.md new file mode 100644 index 0000000000..6d17eb336d --- /dev/null +++ b/apple/changelog.d/4289.fixed.1.md @@ -0,0 +1 @@ +Importing a SQL file split a PostgreSQL `E'...'` string continued on the next line, or a SQLite `$name(...)` parameter. diff --git a/apple/changelog.d/4289.fixed.2.md b/apple/changelog.d/4289.fixed.2.md new file mode 100644 index 0000000000..bb861867b9 --- /dev/null +++ b/apple/changelog.d/4289.fixed.2.md @@ -0,0 +1 @@ +The editor flagged a bracket inside a `$$` body, a `[name]`, a MySQL `#` comment or PostgreSQL's `'C:\'` as unmatched. diff --git a/apple/changelog.d/4289.fixed.3.md b/apple/changelog.d/4289.fixed.3.md new file mode 100644 index 0000000000..09a554fc1e --- /dev/null +++ b/apple/changelog.d/4289.fixed.3.md @@ -0,0 +1 @@ +Query Insights grouped statements without the engine's quoting and comment rules, such as PostgreSQL's `'C:\'`. diff --git a/apple/changelog.d/4289.fixed.4.md b/apple/changelog.d/4289.fixed.4.md new file mode 100644 index 0000000000..18f742414d --- /dev/null +++ b/apple/changelog.d/4289.fixed.4.md @@ -0,0 +1 @@ +On iOS, a ROLLBACK after a MySQL `#` comment or a nested PostgreSQL comment left the transaction shown as open. diff --git a/apple/changelog.d/4289.fixed.md b/apple/changelog.d/4289.fixed.md new file mode 100644 index 0000000000..1714191654 --- /dev/null +++ b/apple/changelog.d/4289.fixed.md @@ -0,0 +1 @@ +Importing a SQL file joined the words around a dropped comment, so `a/* x */FROM t` ran as `aFROM t`. From 77081e60b2afafe3ef80b736350f1350665e1b99 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 15 Nov 2026 04:30:53 +0700 Subject: [PATCH 03/10] fix(import): refuse a string or comment longer than the core's statement limit instead of holding it --- .../SQL/SQLFileParser+CopyRows.swift | 1 + .../Core/Utilities/SQL/SQLFileParser.swift | 26 ++++++++++++++++--- .../TablePro/Resources/Localizable.xcstrings | 3 +++ .../Utilities/SQL/SQLFileParserTests.swift | 24 +++++++++++++++++ 4 files changed, 50 insertions(+), 4 deletions(-) diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift index 8d0538aeaf..ace89683df 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLFileParser+CopyRows.swift @@ -16,6 +16,7 @@ extension SQLFileParser { let batchCutLength: Int let chunkSize: Int let copyPieceLength: Int + let spanLimit: Int } // Reads the rows as psql 17 and 18 do (SQLPsqlScript.copyRows), from a buffer that holds part of the file. diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift index 404bd95fba..eb76fd8404 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift @@ -12,6 +12,7 @@ import TableProUtilities /// A file the import refuses before it sends the statement that needs what it cannot give. enum SQLFileParseError: LocalizedError, Equatable { case copyFromStandardInput(line: Int) + case spanTooLong(line: Int) var errorDescription: String? { switch self { @@ -22,6 +23,11 @@ enum SQLFileParseError: LocalizedError, Equatable { ), line ) + case .spanTooLong(let line): + return String( + format: String(localized: "The string or comment that starts on line %lld is longer than the import reads."), + line + ) } } } @@ -38,12 +44,21 @@ final class SQLFileParser: Sendable { private let limits: ReadLimits + // As much as the core's script reader holds of one statement: a string or comment longer is not one a server takes. + static let defaultSpanLimit = 134_217_728 + init( batchCutLength: Int = SQLFileParser.defaultBatchCutLength, chunkSize: Int = SQLFileParser.defaultChunkSize, - copyPieceLength: Int = SQLFileParser.defaultCopyPieceLength + copyPieceLength: Int = SQLFileParser.defaultCopyPieceLength, + spanLimit: Int = SQLFileParser.defaultSpanLimit ) { - limits = ReadLimits(batchCutLength: batchCutLength, chunkSize: chunkSize, copyPieceLength: copyPieceLength) + limits = ReadLimits( + batchCutLength: batchCutLength, + chunkSize: chunkSize, + copyPieceLength: copyPieceLength, + spanLimit: spanLimit + ) } /// One statement the file holds, or on a grammar that cuts scripts into batches, one batch, with how many times @@ -882,10 +897,13 @@ extension SQLFileParser { closeFile() return } - try readChunk() let held = ctx.heldSpanLength ctx.heldSpanLength = 0 - while held > 0, !ctx.atEndOfInput, nsBuffer.length - readOffset < 2 * held { + // A span is held whole until it ends, so one that never ends is refused rather than read into memory. + guard held <= limits.spanLimit else { throw SQLFileParseError.spanTooLong(line: ctx.currentLine) } + try readChunk() + let wanted = min(2 * held, limits.spanLimit + 1) + while held > 0, !ctx.atEndOfInput, !Task.isCancelled, nsBuffer.length - readOffset < wanted { try readChunk() } } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 51b62f8333..d5d1920a0e 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -199043,6 +199043,9 @@ } } } + }, + "The string or comment that starts on line %lld is longer than the import reads." : { + }, "The structure of a table in an attached database cannot be changed here." : { "extractionState" : "manual", diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift index 6e556dec07..4523f695f1 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift @@ -411,6 +411,30 @@ struct SQLFileParserTests { } } + @Test("A comment or string longer than the import holds is refused at its line, by the count and the import") + func spanLongerThanTheLimitIsRefused() async throws { + let sql = "SELECT 1;\nSELECT 2 /*" + String(repeating: "x", count: 5_000) + "*/;\nSELECT 3;" + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try sql.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + let parser = SQLFileParser(chunkSize: 64, spanLimit: 1_000) + + await #expect(throws: SQLFileParseError.spanTooLong(line: 2)) { + _ = try await parser.countStatements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) + } + var read: [String] = [] + do { + for try await (statement, _) in parser.parseFile(url: url, encoding: .utf8, grammar: TestGrammar.postgres) { + read.append(statement) + } + Issue.record("the import read the whole file") + } catch let error as SQLFileParseError { + #expect(error == .spanTooLong(line: 2)) + } + #expect(read == ["SELECT 1"]) + #expect(try await Self.parse(sql, grammar: TestGrammar.postgres, chunkSize: 64).count == 3) + } + @Test("A literal many chunks long is read whole, wherever the chunks cut it") func literalAcrossManyChunks() async throws { let body = String(repeating: "x;'' ", count: 40_000) From 1ec456886304e1d71877ffad834391804e98c3f7 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 15 Nov 2026 04:41:21 +0700 Subject: [PATCH 04/10] fix(history): keep a psql COPY command and a SurrealQL angle-quoted name in the fingerprint --- .../Core/Services/Formatting/SQLTokenizer.swift | 3 ++- .../Core/Utilities/SQL/SQLQueryFingerprint.swift | 16 +++++++++++++++- .../Utilities/SQL/SQLQueryFingerprintTests.swift | 15 +++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/apple/TablePro/Core/Services/Formatting/SQLTokenizer.swift b/apple/TablePro/Core/Services/Formatting/SQLTokenizer.swift index 4ca0ccc1c4..d96072eae4 100644 --- a/apple/TablePro/Core/Services/Formatting/SQLTokenizer.swift +++ b/apple/TablePro/Core/Services/Formatting/SQLTokenizer.swift @@ -268,7 +268,8 @@ private struct Lexer { emit(.identifier, from: start) return } - emit(opener == Self.backtick || opener == Self.openBracket ? .identifier : .string, from: start) + let quotesName = [Self.backtick, Self.openBracket, SqlLexer.openAngleQuote].contains(opener) + emit(quotesName ? .identifier : .string, from: start) } } diff --git a/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift b/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift index 65c326ad7d..de1692a53e 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift @@ -57,7 +57,7 @@ enum SQLQueryFingerprint { var previousType: SQLTokenType? var previousValue = "" - for token in SQLTokenizer().tokenize(source, grammar: databaseType.lexicalGrammar) { + for token in tokens(of: source, grammar: databaseType.lexicalGrammar) { defer { if token.type != .comment, token.type != .whitespace { previousType = token.type @@ -188,6 +188,7 @@ enum SQLQueryFingerprint { case "`": closing = "`" case "\"": closing = "\"" case "[": closing = "]" + case "\u{27E8}": closing = "\u{27E9}" default: return value } var inner = value.dropFirst() @@ -197,6 +198,19 @@ enum SQLQueryFingerprint { return String(inner) } + // A psql `COPY ... FROM STDIN` is one directive token with its rows; the command is the shape, the rows are data. + private static func tokens(of source: String, grammar: SQLLexicalGrammar) -> [SQLToken] { + let tokenizer = SQLTokenizer() + return tokenizer.tokenize(source, grammar: grammar).flatMap { token -> [SQLToken] in + guard token.type == .directive, token.upperValue.hasPrefix("COPY"), + let command = SQLStatementScanner.executableStatements(in: token.value, grammar: grammar).first?.sql + else { + return [token] + } + return tokenizer.tokenize(command, grammar: grammar.subtracting(.psqlScripts)) + } + } + private static func truncated(_ sql: String) -> String { let source = sql as NSString guard source.length > maxSourceLength else { return sql } diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift index 6978da6293..6f85465d9c 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift @@ -181,4 +181,19 @@ struct SQLQueryFingerprintTests { #expect(normalize("") == "") #expect(normalize(" \n\t ") == "") } + + @Test("A psql COPY keeps its command in the shape, and its rows stay out") + func copyFromStandardInputKeepsItsCommand() { + let users = "COPY users FROM stdin;\n1\tada\n\\.\n" + let orders = "COPY orders FROM stdin;\n1\tada\n\\.\n" + #expect(normalize(users) == "COPY users FROM stdin") + #expect(hash(users) != hash(orders)) + #expect(hash(users) == hash("COPY users FROM stdin;\n2\tbob\n\\.\n")) + } + + @Test("A SurrealQL angle-quoted name is the name it quotes") + func angleQuotedNamesStayNames() { + #expect(hash("SELECT * FROM \u{27E8}users\u{27E9}", .surrealdb) == hash("SELECT * FROM users", .surrealdb)) + #expect(hash("SELECT * FROM \u{27E8}users\u{27E9}", .surrealdb) != hash("SELECT * FROM \u{27E8}orders\u{27E9}", .surrealdb)) + } } From 9c6d8a9168e52bd8910194315abe6532a25253dc Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 15 Nov 2026 04:52:43 +0700 Subject: [PATCH 05/10] fix(history): keep a quoted name's quotes when it needs them and read any directive as what it sends --- .../Utilities/SQL/SQLQueryFingerprint.swift | 19 ++++++++++++------- .../Core/Services/SQLTokenizerTests.swift | 2 +- .../SQL/SQLQueryFingerprintTests.swift | 9 +++++++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift b/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift index de1692a53e..08ef358611 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift @@ -195,19 +195,24 @@ enum SQLQueryFingerprint { if inner.last == closing { inner = inner.dropLast() } - return String(inner) + // `[a, b]` is one column and `a, b` two, so only a name that needs no quotes loses them. + return isPlainIdentifier(inner) ? String(inner) : value } - // A psql `COPY ... FROM STDIN` is one directive token with its rows; the command is the shape, the rows are data. + private static func isPlainIdentifier(_ name: Substring) -> Bool { + guard let first = name.first, first.isLetter || first == "_" else { return false } + return name.allSatisfy { $0.isLetter || $0.isNumber || $0 == "_" || $0 == "$" } + } + + // A psql `COPY ... FROM STDIN` is one directive token with its rows: the command it sends is the shape, the rows are + // data. A directive that sends nothing, such as `GO`, leaves nothing. private static func tokens(of source: String, grammar: SQLLexicalGrammar) -> [SQLToken] { let tokenizer = SQLTokenizer() return tokenizer.tokenize(source, grammar: grammar).flatMap { token -> [SQLToken] in - guard token.type == .directive, token.upperValue.hasPrefix("COPY"), - let command = SQLStatementScanner.executableStatements(in: token.value, grammar: grammar).first?.sql - else { - return [token] + guard token.type == .directive else { return [token] } + return SQLStatementScanner.executableStatements(in: token.value, grammar: grammar).flatMap { statement in + tokenizer.tokenize(statement.sql, grammar: grammar.subtracting(.psqlScripts)) } - return tokenizer.tokenize(command, grammar: grammar.subtracting(.psqlScripts)) } } diff --git a/apple/TableProTests/Core/Services/SQLTokenizerTests.swift b/apple/TableProTests/Core/Services/SQLTokenizerTests.swift index 30691f9ecb..d05561c036 100644 --- a/apple/TableProTests/Core/Services/SQLTokenizerTests.swift +++ b/apple/TableProTests/Core/Services/SQLTokenizerTests.swift @@ -494,7 +494,7 @@ struct SQLFingerprintGoldenTests { (.mysql, "SELECT a'alias', b'alias' FROM t", "SELECT a ?, ? FROM t"), (.postgresql, "SELECT U&'d\\0061t', E'it\\'s' FROM t", "SELECT ?, ? FROM t"), (.oracle, "SELECT q'[it's]' FROM dual", "SELECT ? FROM dual"), - (.mssql, "SELECT [order id], [group] FROM [my table] WHERE [select] = 1", "SELECT order id, group FROM my table WHERE select = ?"), + (.mssql, "SELECT [order id], [group] FROM [my table] WHERE [select] = 1", "SELECT [order id], group FROM [my table] WHERE select = ?"), (.mssql, "SELECT * FROM [users]", "SELECT * FROM users"), (.mssql, "SELECT * FROM users", "SELECT * FROM users"), (.mssql, "SELECT * INTO #tmp FROM t; SELECT * FROM ##global", "SELECT * INTO #tmp FROM t; SELECT * FROM ##global"), diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift index 6f85465d9c..d2382adc43 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift @@ -189,6 +189,15 @@ struct SQLQueryFingerprintTests { #expect(normalize(users) == "COPY users FROM stdin") #expect(hash(users) != hash(orders)) #expect(hash(users) == hash("COPY users FROM stdin;\n2\tbob\n\\.\n")) + #expect(hash("/* load */ " + users) == hash(users)) + #expect(hash("/* load */ " + users) != hash("/* load */ " + orders)) + } + + @Test("A quoted name that needs its quotes keeps them, so it never reads as the list it spells") + func quotedNamesKeepNeededQuotes() { + #expect(hash("SELECT [a, b] FROM t", .mssql) != hash("SELECT a, b FROM t", .mssql)) + #expect(hash(#"SELECT "a, b" FROM t"#) != hash("SELECT a, b FROM t")) + #expect(hash("SELECT [a] FROM t", .mssql) == hash("SELECT a FROM t", .mssql)) } @Test("A SurrealQL angle-quoted name is the name it quotes") From 9f4b68f624cfe0d3274692e0a174be792f87282c Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 15 Nov 2026 05:07:21 +0700 Subject: [PATCH 06/10] fix(history): keep the store closed when the fingerprint reset fails and read a MySQL hint as a comment --- .../Core/Storage/QueryHistoryStorage.swift | 17 ++++++++++++----- .../Utilities/SQL/SQLQueryFingerprint.swift | 8 ++++---- .../SQL/SQLQueryFingerprintTests.swift | 6 ++++++ 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/apple/TablePro/Core/Storage/QueryHistoryStorage.swift b/apple/TablePro/Core/Storage/QueryHistoryStorage.swift index 750bce3b45..0cbaca00b8 100644 --- a/apple/TablePro/Core/Storage/QueryHistoryStorage.swift +++ b/apple/TablePro/Core/Storage/QueryHistoryStorage.swift @@ -106,7 +106,13 @@ actor QueryHistoryStorage { createFingerprintIndex() createPlanSnapshotStorage() createRewindSnapshotStorage() - resetFingerprintsReadWithoutGrammar() + guard resetFingerprintsReadWithoutGrammar() else { + // Hashes of two readings would split every group, so the store stays closed as for a failed migration. + Self.logger.error("Query history fingerprint reset failed; the store stays closed") + sqlite3_close_v2(pointer) + dbHandle.pointer = nil + return + } protectDatabaseFiles(at: dbPath) } @@ -231,16 +237,17 @@ actor QueryHistoryStorage { // Before version 6 a fingerprint was read without the engine's grammar. The backfill hashes a zero again, so a // group keeps the runs from before the upgrade. - private func resetFingerprintsReadWithoutGrammar() { - guard userVersion() < 6, beginTransaction() else { return } + private func resetFingerprintsReadWithoutGrammar() -> Bool { + guard userVersion() < 6 else { return true } + guard beginTransaction() else { return false } guard execute("UPDATE history SET fingerprint_hash = 0;"), execute("UPDATE plan_snapshots SET fingerprint_hash = 0;"), setUserVersion(6) else { rollbackTransaction() - return + return false } - commitTransaction() + return commitTransaction() } // first_row_time and server_time stay NULL for older rows: COALESCE falls back to execution_time. diff --git a/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift b/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift index 08ef358611..478be2da12 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift @@ -69,10 +69,6 @@ enum SQLQueryFingerprint { case .comment, .whitespace, .directive: continue - // MySQL's `/*! ... */` hint arrives as one token, and it was a comment to the shape before grammars. - case .string where token.value.hasPrefix("/*"): - continue - // `@@version` names a server variable: two of them are two shapes, and neither is a literal. case .placeholder where token.value.hasPrefix("@@"): parts.append(Part(token.value)) @@ -209,6 +205,10 @@ enum SQLQueryFingerprint { private static func tokens(of source: String, grammar: SQLLexicalGrammar) -> [SQLToken] { let tokenizer = SQLTokenizer() return tokenizer.tokenize(source, grammar: grammar).flatMap { token -> [SQLToken] in + // MySQL's `/*! ... */` hint arrives as one string token; to the shape it is a comment. + if token.type == .string, token.value.hasPrefix("/*") { + return [SQLToken(type: .comment, value: token.value)] + } guard token.type == .directive else { return [token] } return SQLStatementScanner.executableStatements(in: token.value, grammar: grammar).flatMap { statement in tokenizer.tokenize(statement.sql, grammar: grammar.subtracting(.psqlScripts)) diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift index d2382adc43..7d2f1282ce 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift @@ -205,4 +205,10 @@ struct SQLQueryFingerprintTests { #expect(hash("SELECT * FROM \u{27E8}users\u{27E9}", .surrealdb) == hash("SELECT * FROM users", .surrealdb)) #expect(hash("SELECT * FROM \u{27E8}users\u{27E9}", .surrealdb) != hash("SELECT * FROM \u{27E8}orders\u{27E9}", .surrealdb)) } + + @Test("A MySQL hint leaves the sign after it on its literal") + func hintIsNotAnOperand() { + #expect(hash("SELECT /*!40001 SQL_NO_CACHE */ -1", .mysql) == hash("SELECT /*!40001 SQL_NO_CACHE */ 1", .mysql)) + #expect(normalize("SELECT /*!40001 SQL_NO_CACHE */ -1", .mysql) == "SELECT ?") + } } From 46556397e7338b124db38cd1c33c56307159d7b0 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 15 Nov 2026 05:17:34 +0700 Subject: [PATCH 07/10] fix(ios): read the words a MySQL executable comment runs when classifying a roll back --- .../Helpers/TransactionRollBackClassifier.swift | 13 +++++++++++++ .../TransactionRollBackClassifierTests.swift | 3 +++ 2 files changed, 16 insertions(+) diff --git a/apple/TableProMobile/TableProMobile/Helpers/TransactionRollBackClassifier.swift b/apple/TableProMobile/TableProMobile/Helpers/TransactionRollBackClassifier.swift index ef2f9b4f60..bcb626efd4 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/TransactionRollBackClassifier.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/TransactionRollBackClassifier.swift @@ -34,13 +34,26 @@ nonisolated enum TransactionRollBackClassifier { let text = sql as NSString var words: [String] = [] var index = 0 + // The server runs the body of `/*! ... */`, so its words are read and its closer is a blank. + var executableBody: SQLNonCodeSpan.Span? while words.count < limit, index < text.length { + if let body = executableBody, index >= body.contentEnd { + index = body.end + executableBody = nil + continue + } let unit = text.character(at: index) if UnicodeScalar(unit).map(CharacterSet.whitespacesAndNewlines.contains) == true { index += 1 continue } if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + if span.kind == .executableComment, + let opener = SqlLexer.executableCommentOpenerLength(text, at: index, length: text.length) { + executableBody = span + index += opener + continue + } guard span.kind.isComment else { words.append(otherTarget) break diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/TransactionRollBackClassifierTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/TransactionRollBackClassifierTests.swift index e3e65df7fe..c03118bdd3 100644 --- a/apple/TableProMobile/TableProMobileTests/Helpers/TransactionRollBackClassifierTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Helpers/TransactionRollBackClassifierTests.swift @@ -60,5 +60,8 @@ struct TransactionRollBackClassifierTests { #expect(TransactionRollBackClassifier.rollsBackTransaction("# undo\nROLLBACK", grammar: mysql)) #expect(TransactionRollBackClassifier.rollsBackTransaction("/* a /* b */ c */ ROLLBACK", grammar: postgreSQL)) #expect(!TransactionRollBackClassifier.rollsBackTransaction("--x\nROLLBACK", grammar: mysql)) + #expect(TransactionRollBackClassifier.rollsBackTransaction("ROLLBACK /*! AND CHAIN */", grammar: mysql)) + #expect(TransactionRollBackClassifier.rollsBackTransaction("/*!40001 ROLLBACK */", grammar: mysql)) + #expect(!TransactionRollBackClassifier.rollsBackTransaction("ROLLBACK /*! TO a */", grammar: mysql)) } } From 784e7a1077ca5ccf2c5876fdd9cb4bc0f29c85a3 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 15 Nov 2026 05:30:47 +0700 Subject: [PATCH 08/10] fix(history): keep the word glued to a literal in the fingerprint shape --- .../Core/Utilities/SQL/SQLQueryFingerprint.swift | 15 +++++++++++++++ .../Core/Services/SQLTokenizerTests.swift | 10 +++++----- .../Utilities/SQL/SQLQueryFingerprintTests.swift | 7 +++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift b/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift index 478be2da12..af86ec1817 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLQueryFingerprint.swift @@ -84,6 +84,11 @@ enum SQLQueryFingerprint { if isDoubleQuoted(token.value), !treatsDoubleQuoteAsString { parts.append(Part(unquotedIdentifier(token.value))) previousWasLiteral = false + } else if let prefix = literalPrefix(token.value) { + // `N'x'` and SQLite's `b'alias'`, a column and its alias, stay apart from a bare `'x'`. + parts.append(Part(prefix)) + previousWasLiteral = false + appendLiteral(to: &parts, previousWasLiteral: &previousWasLiteral) } else { appendLiteral(to: &parts, previousWasLiteral: &previousWasLiteral) } @@ -171,6 +176,16 @@ enum SQLQueryFingerprint { } } + private static func literalPrefix(_ value: String) -> String? { + let prefix = value.prefix { $0 != "'" && $0 != "\"" } + guard let first = prefix.first, first.isLetter || first == "_", prefix.count < value.count, + prefix.allSatisfy({ $0.isLetter || $0.isNumber || $0 == "_" || $0 == "&" }) + else { + return nil + } + return String(prefix) + } + private static func isDoubleQuoted(_ value: String) -> Bool { value.hasPrefix("\"") } diff --git a/apple/TableProTests/Core/Services/SQLTokenizerTests.swift b/apple/TableProTests/Core/Services/SQLTokenizerTests.swift index d05561c036..20d2d966cc 100644 --- a/apple/TableProTests/Core/Services/SQLTokenizerTests.swift +++ b/apple/TableProTests/Core/Services/SQLTokenizerTests.swift @@ -489,11 +489,11 @@ struct SQLFingerprintGoldenTests { (.mysql, "SELECT 0baz FROM t", "SELECT 0baz FROM t"), (.mysql, "SELECT 0xFF, 0x1A, 0b101 FROM t", "SELECT ?, ?, ? FROM t"), (.mysql, "SELECT 2fa_codes.id FROM 2fa_codes", "SELECT 2fa_codes.id FROM 2fa_codes"), - (.mssql, "SELECT N'x', N'longer text' FROM t", "SELECT ?, ? FROM t"), - (.mysql, "SELECT X'4D', B'101', _utf8mb4'a' FROM t", "SELECT ?, ?, ? FROM t"), - (.mysql, "SELECT a'alias', b'alias' FROM t", "SELECT a ?, ? FROM t"), - (.postgresql, "SELECT U&'d\\0061t', E'it\\'s' FROM t", "SELECT ?, ? FROM t"), - (.oracle, "SELECT q'[it's]' FROM dual", "SELECT ? FROM dual"), + (.mssql, "SELECT N'x', N'longer text' FROM t", "SELECT N ?, N ? FROM t"), + (.mysql, "SELECT X'4D', B'101', _utf8mb4'a' FROM t", "SELECT X ?, B ?, _utf8mb4 ? FROM t"), + (.mysql, "SELECT a'alias', b'alias' FROM t", "SELECT a ?, b ? FROM t"), + (.postgresql, "SELECT U&'d\\0061t', E'it\\'s' FROM t", "SELECT U& ?, E ? FROM t"), + (.oracle, "SELECT q'[it's]' FROM dual", "SELECT q ? FROM dual"), (.mssql, "SELECT [order id], [group] FROM [my table] WHERE [select] = 1", "SELECT [order id], group FROM [my table] WHERE select = ?"), (.mssql, "SELECT * FROM [users]", "SELECT * FROM users"), (.mssql, "SELECT * FROM users", "SELECT * FROM users"), diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift index 7d2f1282ce..77a2c63aeb 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLQueryFingerprintTests.swift @@ -211,4 +211,11 @@ struct SQLQueryFingerprintTests { #expect(hash("SELECT /*!40001 SQL_NO_CACHE */ -1", .mysql) == hash("SELECT /*!40001 SQL_NO_CACHE */ 1", .mysql)) #expect(normalize("SELECT /*!40001 SQL_NO_CACHE */ -1", .mysql) == "SELECT ?") } + + @Test("A word glued to a literal stays in the shape, so SQLite's column and alias never read as one literal") + func literalPrefixStaysInTheShape() { + #expect(hash("SELECT b'alias' FROM t", .sqlite) != hash("SELECT n'alias' FROM t", .sqlite)) + #expect(normalize("SELECT b'alias' FROM t", .sqlite) == "SELECT b ? FROM t") + #expect(hash("SELECT N'x' FROM t", .mssql) == hash("SELECT N'other' FROM t", .mssql)) + } } From 1245d74bf7080b7610aad44bcce9857ad2b16c33 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 15 Nov 2026 05:45:41 +0700 Subject: [PATCH 09/10] fix(history): keep the previous stable's fingerprint column and group by a new shape column --- .../Diagnostics/QueryBracketScanner.swift | 9 ++- .../QueryHistoryStorage+PlanSnapshots.swift | 18 +++-- .../Core/Storage/QueryHistoryStorage.swift | 72 ++++++++----------- .../Diagnostics/QueryDiagnosticsTests.swift | 2 + ...ueryHistoryFingerprintMigrationTests.swift | 40 +++++++---- .../Storage/QueryHistoryMigrationTests.swift | 6 +- 6 files changed, 84 insertions(+), 63 deletions(-) diff --git a/apple/TablePro/Core/Diagnostics/QueryBracketScanner.swift b/apple/TablePro/Core/Diagnostics/QueryBracketScanner.swift index 59b14f6996..f84e88887e 100644 --- a/apple/TablePro/Core/Diagnostics/QueryBracketScanner.swift +++ b/apple/TablePro/Core/Diagnostics/QueryBracketScanner.swift @@ -65,9 +65,15 @@ enum QueryBracketScanner { var brackets = Brackets() var unterminatedComment: NSRange? var hasUnterminatedString = false + var executableBody: SQLNonCodeSpan.Span? var index = 0 while index < text.length { + if let body = executableBody, index >= body.contentEnd { + index = max(body.end, index) + executableBody = nil + continue + } let character = text.character(at: index) guard SQLNonCodeSpan.mayStart(character), let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) @@ -86,9 +92,10 @@ enum QueryBracketScanner { case .quoted: hasUnterminatedString = hasUnterminatedString || !span.isTerminated } - // The server runs the body of `/*! ... */`, so its brackets are code. + // The server runs the body of `/*! ... */`, so its brackets are code, and its closer is not. if span.kind == .executableComment, let opener = SqlLexer.executableCommentOpenerLength(text, at: index, length: text.length) { + executableBody = executableBody ?? span index += opener continue } diff --git a/apple/TablePro/Core/Storage/QueryHistoryStorage+PlanSnapshots.swift b/apple/TablePro/Core/Storage/QueryHistoryStorage+PlanSnapshots.swift index 84b1ca61d1..6ef385fff8 100644 --- a/apple/TablePro/Core/Storage/QueryHistoryStorage+PlanSnapshots.swift +++ b/apple/TablePro/Core/Storage/QueryHistoryStorage+PlanSnapshots.swift @@ -42,9 +42,16 @@ extension QueryHistoryStorage { byte_count INTEGER NOT NULL, execution_time REAL NOT NULL, captured_at REAL NOT NULL, - is_pinned INTEGER NOT NULL DEFAULT 0 + is_pinned INTEGER NOT NULL DEFAULT 0, + shape_hash INTEGER NOT NULL DEFAULT 0 ); """) + // A plan from the previous stable has no shape yet; the backfill reads it from `subject_sql`. + _ = addColumnIfMissing("shape_hash", definition: "INTEGER NOT NULL DEFAULT 0", inTable: "plan_snapshots") + execute(""" + CREATE INDEX IF NOT EXISTS idx_plan_snapshots_shape + ON plan_snapshots(shape_hash, connection_id, database_name, variant_key, captured_at DESC); + """) execute(""" CREATE INDEX IF NOT EXISTS idx_plan_snapshots_identity ON plan_snapshots(fingerprint_hash, connection_id, database_name, variant_key, captured_at DESC); @@ -61,12 +68,14 @@ extension QueryHistoryStorage { func recordPlanSnapshot(_ capture: QueryPlanCapture) -> Bool { guard let db, capture.isWithinPlanSizeLimit else { return false } + // The previous stable matches plans on `fingerprint_hash`, and its reading agrees with the grammar's for nearly + // every statement, so the shape goes in both. let sql = """ INSERT INTO plan_snapshots ( id, history_id, fingerprint_hash, subject_sql, connection_id, database_name, database_type, schema_name, variant_key, format, raw_plan, byte_count, - execution_time, captured_at, is_pinned - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0); + execution_time, captured_at, is_pinned, shape_hash + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?); """ var statement: OpaquePointer? defer { sqlite3_finalize(statement) } @@ -91,6 +100,7 @@ extension QueryHistoryStorage { .int64(Int64(capture.byteCount)), .double(capture.executionTime), .double(capture.capturedAt.timeIntervalSince1970), + .int64(identity.fingerprintHash), ] bind(bindings, to: statement) @@ -167,7 +177,7 @@ extension QueryHistoryStorage { clause.append(""" SELECT id, subject_sql, execution_time, captured_at, is_pinned, byte_count FROM plan_snapshots - WHERE fingerprint_hash = ? + WHERE shape_hash = ? AND connection_id = ? AND database_name = ? AND database_type = ? diff --git a/apple/TablePro/Core/Storage/QueryHistoryStorage.swift b/apple/TablePro/Core/Storage/QueryHistoryStorage.swift index 0cbaca00b8..8dab2a4651 100644 --- a/apple/TablePro/Core/Storage/QueryHistoryStorage.swift +++ b/apple/TablePro/Core/Storage/QueryHistoryStorage.swift @@ -106,13 +106,6 @@ actor QueryHistoryStorage { createFingerprintIndex() createPlanSnapshotStorage() createRewindSnapshotStorage() - guard resetFingerprintsReadWithoutGrammar() else { - // Hashes of two readings would split every group, so the store stays closed as for a failed migration. - Self.logger.error("Query history fingerprint reset failed; the store stays closed") - sqlite3_close_v2(pointer) - dbHandle.pointer = nil - return - } protectDatabaseFiles(at: dbPath) } @@ -148,7 +141,8 @@ actor QueryHistoryStorage { error_message TEXT, fingerprint_hash INTEGER NOT NULL DEFAULT 0, first_row_time REAL, - server_time REAL + server_time REAL, + shape_hash INTEGER NOT NULL DEFAULT 0 ); """ @@ -213,6 +207,14 @@ actor QueryHistoryStorage { ON history(fingerprint_hash, connection_id, executed_at DESC); """) execute("DROP INDEX IF EXISTS idx_history_fingerprint;") + execute(""" + CREATE INDEX IF NOT EXISTS idx_history_shape_executed + ON history(shape_hash, executed_at DESC); + """) + execute(""" + CREATE INDEX IF NOT EXISTS idx_history_shape_connection_executed + ON history(shape_hash, connection_id, executed_at DESC); + """) } // MARK: - Migration @@ -223,37 +225,23 @@ actor QueryHistoryStorage { if !hasSource { guard migrateToVersion3() else { return false } } + // `shape_hash` is the shape read with the engine's grammar. `fingerprint_hash` stays the previous stable's: this + // build writes it as 0, which that release's backfill fills with its own reading. guard addColumnIfMissing("fingerprint_hash", definition: "INTEGER NOT NULL DEFAULT 0"), addColumnIfMissing("first_row_time", definition: "REAL"), - addColumnIfMissing("server_time", definition: "REAL") + addColumnIfMissing("server_time", definition: "REAL"), + addColumnIfMissing("shape_hash", definition: "INTEGER NOT NULL DEFAULT 0") else { return false } - if userVersion() < 5 { - setUserVersion(5) - } + setUserVersion(5) return true } - // Before version 6 a fingerprint was read without the engine's grammar. The backfill hashes a zero again, so a - // group keeps the runs from before the upgrade. - private func resetFingerprintsReadWithoutGrammar() -> Bool { - guard userVersion() < 6 else { return true } - guard beginTransaction() else { return false } - guard execute("UPDATE history SET fingerprint_hash = 0;"), - execute("UPDATE plan_snapshots SET fingerprint_hash = 0;"), - setUserVersion(6) - else { - rollbackTransaction() - return false - } - return commitTransaction() - } - // first_row_time and server_time stay NULL for older rows: COALESCE falls back to execution_time. - private func addColumnIfMissing(_ column: String, definition: String) -> Bool { - guard let present = hasColumn(column, inTable: "history") else { return false } - return present || execute("ALTER TABLE history ADD COLUMN \(column) \(definition);") + func addColumnIfMissing(_ column: String, definition: String, inTable table: String = "history") -> Bool { + guard let present = hasColumn(column, inTable: table) else { return false } + return present || execute("ALTER TABLE \(table) ADD COLUMN \(column) \(definition);") } // Not in `init`, which runs on the main actor at launch: the readers of groups and plans call it on the actor. @@ -279,7 +267,7 @@ actor QueryHistoryStorage { defer { sqlite3_finalize(statement) } guard sqlite3_prepare_v2( db, - "UPDATE \(table) SET fingerprint_hash = ? WHERE id = ?;", + "UPDATE \(table) SET shape_hash = ? WHERE id = ?;", -1, &statement, nil @@ -306,7 +294,7 @@ actor QueryHistoryStorage { defer { sqlite3_finalize(statement) } guard sqlite3_prepare_v2( db, - "SELECT id, \(sqlColumn), database_type FROM \(table) WHERE fingerprint_hash = 0;", + "SELECT id, \(sqlColumn), database_type FROM \(table) WHERE shape_hash = 0;", -1, &statement, nil @@ -517,7 +505,7 @@ actor QueryHistoryStorage { INSERT INTO history ( id, query, connection_id, database_name, database_type, schema_name, source, statement_type, executed_at, execution_time, row_count, - was_successful, error_message, fingerprint_hash, first_row_time, server_time + was_successful, error_message, shape_hash, first_row_time, server_time ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); """ @@ -843,7 +831,7 @@ actor QueryHistoryStorage { clause.append(""" LEFT JOIN history \(alias) ON \(alias).rowid = ( SELECT r.rowid FROM history r - WHERE r.fingerprint_hash = \(groupAlias).fingerprint_hash\(predicate) + WHERE r.shape_hash = \(groupAlias).shape_hash\(predicate) """) appendScope(&clause) clause.append(" ORDER BY r.executed_at DESC, r.rowid DESC LIMIT 1)") @@ -854,7 +842,7 @@ actor QueryHistoryStorage { clause.append(""" SELECT COUNT(*), SUM(CASE WHEN was_successful = 0 THEN 1 ELSE 0 END), - COUNT(DISTINCT fingerprint_hash), + COUNT(DISTINCT shape_hash), SUM(\(Self.databaseTimeExpression)), MAX(\(Self.databaseTimeExpression)) FROM history @@ -889,11 +877,11 @@ actor QueryHistoryStorage { let failedRuns = " AND r.was_successful = 0" var clause = QueryHistorySqlClause() clause.append(""" - SELECT g.fingerprint_hash, g.call_count, g.failure_count, g.total_duration, g.max_duration, + SELECT g.shape_hash, g.call_count, g.failure_count, g.total_duration, g.max_duration, g.total_rows, rep.query, rep.statement_type, rep.database_type, \(ranking.needsErrorMessage ? "err.error_message" : "NULL") FROM ( - SELECT fingerprint_hash, + SELECT shape_hash, COUNT(*) AS call_count, SUM(CASE WHEN was_successful = 0 THEN 1 ELSE 0 END) AS failure_count, SUM(\(Self.databaseTimeExpression)) AS total_duration, @@ -903,7 +891,7 @@ actor QueryHistoryStorage { WHERE 1 = 1 """) appendInsightsFilters(request, to: &clause, columnPrefix: "") - clause.append(" GROUP BY fingerprint_hash\(ranking.having)") + clause.append(" GROUP BY shape_hash\(ranking.having)") // Ordered and limited inside the derived table, so the joined lookups below run once per // returned row rather than once per distinct shape in the whole history. clause.append( @@ -964,7 +952,7 @@ actor QueryHistoryStorage { var clause = QueryHistorySqlClause() clause.append(""" WITH windowed AS ( - SELECT fingerprint_hash, + SELECT shape_hash, CASE WHEN executed_at >= ? THEN 1 ELSE 0 END AS is_recent, \(Self.databaseTimeExpression) AS execution_time FROM history @@ -974,15 +962,15 @@ actor QueryHistoryStorage { clause.append(""" ), paired AS ( - SELECT fingerprint_hash, + SELECT shape_hash, SUM(is_recent) AS recent_count, SUM(1 - is_recent) AS prior_count, AVG(CASE WHEN is_recent = 1 THEN execution_time END) AS recent_mean, AVG(CASE WHEN is_recent = 0 THEN execution_time END) AS prior_mean FROM windowed - GROUP BY fingerprint_hash + GROUP BY shape_hash ) - SELECT p.fingerprint_hash, p.recent_count, p.prior_count, p.recent_mean, p.prior_mean, + SELECT p.shape_hash, p.recent_count, p.prior_count, p.recent_mean, p.prior_mean, rep.query, rep.database_type FROM paired p """) diff --git a/apple/TableProTests/Core/Diagnostics/QueryDiagnosticsTests.swift b/apple/TableProTests/Core/Diagnostics/QueryDiagnosticsTests.swift index 6dbbe044e8..37bea5b17b 100644 --- a/apple/TableProTests/Core/Diagnostics/QueryDiagnosticsTests.swift +++ b/apple/TableProTests/Core/Diagnostics/QueryDiagnosticsTests.swift @@ -111,6 +111,8 @@ struct QueryDiagnosticsTests { (TestGrammar.sqlServer, "SELECT [a]])] FROM t"), (TestGrammar.sqlite, "SELECT [a)] FROM t"), (DatabaseType.clickhouse.lexicalGrammar, "SELECT 1 // )\n"), + (TestGrammar.mysql, "SELECT /*! 1 */*2"), + (TestGrammar.mysql, "SELECT /*! (1) */ + 2"), ]) func bracketInsideAGrammarSpan(grammar: SQLLexicalGrammar, text: String) { #expect(SQLDiagnosticsProducer(grammar: grammar).diagnostics(for: text).isEmpty, "\(text)") diff --git a/apple/TableProTests/Core/Storage/QueryHistoryFingerprintMigrationTests.swift b/apple/TableProTests/Core/Storage/QueryHistoryFingerprintMigrationTests.swift index d420e01eaf..4f810dc609 100644 --- a/apple/TableProTests/Core/Storage/QueryHistoryFingerprintMigrationTests.swift +++ b/apple/TableProTests/Core/Storage/QueryHistoryFingerprintMigrationTests.swift @@ -163,29 +163,43 @@ struct QueryHistoryFingerprintMigrationTests { #expect(snapshot.mostRun.first?.callCount == 2) } - @Test("Opening a v5 database hashes every row again with the engine's grammar") - func versionSixRehashesWithTheGrammar() async { + @Test("Rows the previous stable wrote get their shape, and this build leaves that release its own fingerprint") + func shapeColumnLeavesThePreviousStableItsFingerprint() async { let connectionId = UUID() let url = makeVersion3Database( queries: ["SELECT 'C:\\' AS p, a FROM t", "SELECT 'D:\\' AS p, a FROM t"], connectionId: connectionId ) - let migrated = QueryHistoryStorage(databaseURL: url, removeDatabaseOnDeinit: false) - #expect(await migrated.count(scope: .all) == 2) - - // A v5 store hashed each row without the grammar, and a backslash kept PostgreSQL's literal open. - var db: OpaquePointer? - if sqlite3_open(url.path(percentEncoded: false), &db) == SQLITE_OK { - sqlite3_exec(db, "UPDATE history SET fingerprint_hash = rowid; PRAGMA user_version = 5;", nil, nil, nil) - } - sqlite3_close_v2(db) + let storage = QueryHistoryStorage(databaseURL: url, removeDatabaseOnDeinit: true) + _ = await storage.record(QueryHistoryEntry( + query: "SELECT 'E:\\' AS p, a FROM t", + connectionId: connectionId, + databaseName: "legacydb", + databaseType: .postgresql, + source: .editor, + executionTime: 0.25, + rowCount: 7, + wasSuccessful: true + )) - let reopened = QueryHistoryStorage(databaseURL: url, removeDatabaseOnDeinit: true) - let snapshot = await reopened.insights( + let snapshot = await storage.insights( QueryInsightsRequest(scope: .connection(connectionId)), slowestRanking: .totalTime ) #expect(snapshot.totals.distinctShapeCount == 1) + #expect(snapshot.mostRun.first?.callCount == 3) #expect(snapshot.mostRun.first?.normalizedQuery == "SELECT ? AS p, a FROM t") + + // The previous stable fills a zero `fingerprint_hash` with its own reading when it opens the store. + var db: OpaquePointer? + guard sqlite3_open(url.path(percentEncoded: false), &db) == SQLITE_OK else { return } + defer { sqlite3_close_v2(db) } + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + let sql = "SELECT fingerprint_hash, shape_hash FROM history WHERE query LIKE '%E:%';" + #expect(sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK) + #expect(sqlite3_step(statement) == SQLITE_ROW) + #expect(sqlite3_column_int64(statement, 0) == 0) + #expect(sqlite3_column_int64(statement, 1) != 0) } } diff --git a/apple/TableProTests/Core/Storage/QueryHistoryMigrationTests.swift b/apple/TableProTests/Core/Storage/QueryHistoryMigrationTests.swift index b0dfcd9976..0b5ff19f7a 100644 --- a/apple/TableProTests/Core/Storage/QueryHistoryMigrationTests.swift +++ b/apple/TableProTests/Core/Storage/QueryHistoryMigrationTests.swift @@ -289,9 +289,9 @@ struct QueryHistoryMigrationTests { #expect(columnNames(in: url, table: "plan_snapshots") == [ "id", "history_id", "fingerprint_hash", "subject_sql", "connection_id", "database_name", "database_type", "schema_name", "variant_key", "format", - "raw_plan", "byte_count", "execution_time", "captured_at", "is_pinned" + "raw_plan", "byte_count", "execution_time", "captured_at", "is_pinned", "shape_hash" ]) - #expect(scalarInt(in: url, sql: "PRAGMA user_version;") == 6) + #expect(scalarInt(in: url, sql: "PRAGMA user_version;") == 5) let second = QueryHistoryStorage(databaseURL: url, removeDatabaseOnDeinit: true) #expect(await second.count(scope: .all) == 1) @@ -313,7 +313,7 @@ struct QueryHistoryMigrationTests { let second = QueryHistoryStorage(databaseURL: url, removeDatabaseOnDeinit: true) #expect(await second.count(scope: .all) == 0) - #expect(scalarInt(in: url, sql: "PRAGMA user_version;") == 6) + #expect(scalarInt(in: url, sql: "PRAGMA user_version;") == 5) #expect( scalarInt( in: url, From e53afa17cf384971ee502bac81fe280a75f1362c Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sun, 15 Nov 2026 06:12:36 +0700 Subject: [PATCH 10/10] fix(history): keep writing the previous stable's fingerprint, and read a versioned MySQL comment both ways on iOS --- .../QueryHistoryStorage+PlanSnapshots.swift | 5 +- .../Core/Storage/QueryHistoryStorage.swift | 8 +- .../Utilities/SQL/SQLLegacyFingerprint.swift | 495 ++++++++++++++++++ .../TransactionRollBackClassifier.swift | 23 +- .../TransactionRollBackClassifierTests.swift | 2 + ...ueryHistoryFingerprintMigrationTests.swift | 9 +- .../QueryPlanSnapshotStorageTests.swift | 22 +- .../SQL/SQLLegacyFingerprintTests.swift | 103 ++++ 8 files changed, 652 insertions(+), 15 deletions(-) create mode 100644 apple/TablePro/Core/Utilities/SQL/SQLLegacyFingerprint.swift create mode 100644 apple/TableProTests/Core/Utilities/SQL/SQLLegacyFingerprintTests.swift diff --git a/apple/TablePro/Core/Storage/QueryHistoryStorage+PlanSnapshots.swift b/apple/TablePro/Core/Storage/QueryHistoryStorage+PlanSnapshots.swift index 6ef385fff8..3b31644573 100644 --- a/apple/TablePro/Core/Storage/QueryHistoryStorage+PlanSnapshots.swift +++ b/apple/TablePro/Core/Storage/QueryHistoryStorage+PlanSnapshots.swift @@ -68,8 +68,7 @@ extension QueryHistoryStorage { func recordPlanSnapshot(_ capture: QueryPlanCapture) -> Bool { guard let db, capture.isWithinPlanSizeLimit else { return false } - // The previous stable matches plans on `fingerprint_hash`, and its reading agrees with the grammar's for nearly - // every statement, so the shape goes in both. + // The previous stable matches plans on `fingerprint_hash`, read its own way; this build matches on `shape_hash`. let sql = """ INSERT INTO plan_snapshots ( id, history_id, fingerprint_hash, subject_sql, connection_id, database_name, @@ -88,7 +87,7 @@ extension QueryHistoryStorage { let bindings: [QueryHistorySqlBinding?] = [ .text(capture.id.uuidString), capture.historyId.map { .text($0.uuidString) }, - .int64(identity.fingerprintHash), + .int64(SQLLegacyFingerprint.hash(capture.subjectSQL, databaseType: identity.scope.databaseType)), .text(capture.subjectSQL), .text(identity.scope.connectionId.uuidString), .text(identity.scope.databaseName), diff --git a/apple/TablePro/Core/Storage/QueryHistoryStorage.swift b/apple/TablePro/Core/Storage/QueryHistoryStorage.swift index 8dab2a4651..f29dd8b2a0 100644 --- a/apple/TablePro/Core/Storage/QueryHistoryStorage.swift +++ b/apple/TablePro/Core/Storage/QueryHistoryStorage.swift @@ -225,8 +225,7 @@ actor QueryHistoryStorage { if !hasSource { guard migrateToVersion3() else { return false } } - // `shape_hash` is the shape read with the engine's grammar. `fingerprint_hash` stays the previous stable's: this - // build writes it as 0, which that release's backfill fills with its own reading. + // `shape_hash` is the shape read with the engine's grammar; `fingerprint_hash` keeps the previous stable's reading. guard addColumnIfMissing("fingerprint_hash", definition: "INTEGER NOT NULL DEFAULT 0"), addColumnIfMissing("first_row_time", definition: "REAL"), addColumnIfMissing("server_time", definition: "REAL"), @@ -505,8 +504,8 @@ actor QueryHistoryStorage { INSERT INTO history ( id, query, connection_id, database_name, database_type, schema_name, source, statement_type, executed_at, execution_time, row_count, - was_successful, error_message, shape_hash, first_row_time, server_time - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); + was_successful, error_message, shape_hash, first_row_time, server_time, fingerprint_hash + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?); """ var statement: OpaquePointer? @@ -549,6 +548,7 @@ actor QueryHistoryStorage { } else { sqlite3_bind_null(statement, 16) } + sqlite3_bind_int64(statement, 17, SQLLegacyFingerprint.hash(entry.query, databaseType: entry.databaseType)) guard sqlite3_step(statement) == SQLITE_DONE else { logSqliteError(context: "insert") diff --git a/apple/TablePro/Core/Utilities/SQL/SQLLegacyFingerprint.swift b/apple/TablePro/Core/Utilities/SQL/SQLLegacyFingerprint.swift new file mode 100644 index 0000000000..620f53a061 --- /dev/null +++ b/apple/TablePro/Core/Utilities/SQL/SQLLegacyFingerprint.swift @@ -0,0 +1,495 @@ +// +// SQLLegacyFingerprint.swift +// TablePro +// + +import CryptoKit +import Foundation +import TableProPluginKit +import TableProSQLGrammar + +// The query shape as the previous stable reads it, frozen: it fills `fingerprint_hash`, the column that release groups +// history and matches saved plans by. This build groups by `shape_hash`. Delete once no supported release reads it. +enum SQLLegacyFingerprint { + static let maxSourceLength = 20_000 + + private struct Part { + let text: String + let isKeyword: Bool + + init(_ text: String, isKeyword: Bool = false) { + self.text = text + self.isKeyword = isKeyword + } + } + + private static let placeholder = "?" + private static let collapsedList = "(...)" + private static let noSpaceBefore: Set = [",", ")", ";", ".", "]", "["] + private static let noSpaceAfter: Set = ["(", ".", "["] + + private static let signPrecedingPunctuation: Set = ["(", ",", ";"] + + static func normalize(_ sql: String, databaseType: DatabaseType) -> String { + let source = strippingDollarQuoted(truncated(sql)) + let treatsDoubleQuoteAsString = Self.treatsDoubleQuoteAsString(databaseType) + + var parts: [Part] = [] + var previousWasLiteral = false + var previousType: SQLTokenType? + var previousValue = "" + + for token in tokens(source) { + defer { + if token.type != .comment, token.type != .whitespace { + previousType = token.type + previousValue = token.value + } + } + + switch token.type { + case .comment, .whitespace, .directive: + continue + + case .number, .placeholder: + appendLiteral(to: &parts, previousWasLiteral: &previousWasLiteral) + + case .string: + // The tokenizer classifies every `"…"` as a string, but outside MySQL that is a + // quoted identifier. Replacing it with `?` erased table and column names and + // merged unrelated queries into one group. + if isDoubleQuoted(token.value), !treatsDoubleQuoteAsString { + parts.append(Part(unquotedIdentifier(token.value))) + previousWasLiteral = false + } else { + appendLiteral(to: &parts, previousWasLiteral: &previousWasLiteral) + } + + case .keyword: + // NULL, TRUE and FALSE are literals wearing a keyword's clothes. + if ["NULL", "TRUE", "FALSE"].contains(token.upperValue) { + appendLiteral(to: &parts, previousWasLiteral: &previousWasLiteral) + } else { + parts.append(Part(token.upperValue, isKeyword: true)) + previousWasLiteral = false + } + + case .identifier: + parts.append(Part(unquotedIdentifier(token.value))) + previousWasLiteral = false + + case .operator: + let isSign = (token.value == "-" || token.value == "+") + && signIntroducesLiteral(previousType: previousType, previousValue: previousValue) + if !isSign { + parts.append(Part(token.value)) + } + previousWasLiteral = false + + case .punctuation: + parts.append(Part(token.value)) + previousWasLiteral = false + } + } + + return join(collapsingLists(mergingBracketIdentifiers(parts))) + } + + private static func mergingBracketIdentifiers(_ parts: [Part]) -> [Part] { + guard parts.contains(where: { $0.text == "[" }) else { return parts } + + var out: [Part] = [] + out.reserveCapacity(parts.count) + var index = 0 + + while index < parts.count { + if parts[index].text == "[", index + 2 < parts.count, parts[index + 2].text == "]", + isPlainIdentifier(parts[index + 1].text) { + out.append(parts[index + 1]) + index += 3 + continue + } + out.append(parts[index]) + index += 1 + } + return out + } + + private static func isPlainIdentifier(_ value: String) -> Bool { + guard let first = value.first, first.isLetter || first == "_" else { return false } + return value.allSatisfy { $0.isLetter || $0.isNumber || $0 == "_" } + } + + static func hash(_ sql: String, databaseType: DatabaseType) -> Int64 { + digest(of: normalize(sql, databaseType: databaseType)) + } + + static func digest(of normalized: String) -> Int64 { + var result: UInt64 = 0 + for byte in SHA256.hash(data: Data(normalized.utf8)).prefix(8) { + result = (result << 8) | UInt64(byte) + } + return Int64(bitPattern: result) + } + + private static func treatsDoubleQuoteAsString(_ databaseType: DatabaseType) -> Bool { + SqlDialect.from(databaseTypeId: databaseType.rawValue) == .mysql + } + + private static func appendLiteral(to parts: inout [Part], previousWasLiteral: inout Bool) { + guard !previousWasLiteral else { return } + parts.append(Part(placeholder)) + previousWasLiteral = true + } + + private static func signIntroducesLiteral(previousType: SQLTokenType?, previousValue: String) -> Bool { + guard let previousType else { return true } + switch previousType { + case .keyword: + return true + case .operator: + // `]` reaches the tokenizer's unknown-character fallback, so it arrives typed as an + // operator while actually closing a subscript, which makes the next sign binary. + return previousValue != "]" + case .punctuation: + return signPrecedingPunctuation.contains(previousValue) + case .identifier, .number, .string, .placeholder, .comment, .whitespace, .directive: + return false + } + } + + private static func isDoubleQuoted(_ value: String) -> Bool { + value.hasPrefix("\"") + } + + private static func unquotedIdentifier(_ value: String) -> String { + guard let first = value.first else { return value } + let closing: Character + switch first { + case "`": closing = "`" + case "\"": closing = "\"" + case "[": closing = "]" + default: return value + } + var inner = value.dropFirst() + if inner.last == closing { + inner = inner.dropLast() + } + return String(inner) + } + + private static func truncated(_ sql: String) -> String { + let source = sql as NSString + guard source.length > maxSourceLength else { return sql } + return source.substring(to: maxSourceLength) + } + + private static func strippingDollarQuoted(_ sql: String) -> String { + guard sql.contains("$") else { return sql } + + let buffer = sql as NSString + let length = buffer.length + var out = "" + out.reserveCapacity(length) + var index = 0 + + while index < length { + let character = buffer.character(at: index) + guard character == SqlDollarQuote.dollar, + case .opener(let openerLength, let tag) = SqlDollarQuote.scanOpener( + at: index, + in: buffer, + bufLen: length + ) + else { + out += buffer.substring(with: NSRange(location: index, length: 1)) + index += 1 + continue + } + + var scan = index + openerLength + while scan < length, !SqlDollarQuote.matchesClose(at: scan, tag: tag, in: buffer, bufLen: length) { + scan += 1 + } + out += "''" + index = scan < length ? scan + (tag as NSString).length + 2 : length + } + return out + } + + private static func collapsingLists(_ parts: [Part]) -> [Part] { + var out: [Part] = [] + out.reserveCapacity(parts.count) + var index = 0 + + while index < parts.count { + let token = parts[index] + guard token.isKeyword, token.text == "IN" || token.text == "VALUES" else { + out.append(token) + index += 1 + continue + } + + guard let end = literalListEnd(in: parts, startingAfter: index) else { + out.append(token) + index += 1 + continue + } + + out.append(contentsOf: [token, Part(collapsedList)]) + index = end + 1 + } + return out + } + + private static func literalListEnd(in parts: [Part], startingAfter keywordIndex: Int) -> Int? { + var index = keywordIndex + 1 + var lastClosing: Int? + + while index < parts.count, parts[index].text == "(" { + var scan = index + 1 + var sawLiteral = false + while scan < parts.count, parts[scan].text != ")" { + guard parts[scan].text == placeholder || parts[scan].text == "," else { return lastClosing } + if parts[scan].text == placeholder { sawLiteral = true } + scan += 1 + } + // Truncation can cut the statement inside the list, which leaves no closing paren. + // The values seen so far are still values, so it collapses rather than surviving as + // several thousand placeholders. + guard sawLiteral else { return lastClosing } + guard scan < parts.count else { return parts.count - 1 } + + lastClosing = scan + index = scan + 1 + if index < parts.count, parts[index].text == "," { + index += 1 + } else { + break + } + } + return lastClosing + } + + private static let functionKeywords: Set = ["COUNT", "SUM", "AVG", "MIN", "MAX"] + + private static func join(_ parts: [Part]) -> String { + var out = "" + var previous: Part? + for part in parts { + let bindsToName = part.text == "(" + && previous.map { !$0.isKeyword || functionKeywords.contains($0.text) } == true + let binds = noSpaceBefore.contains(part.text) + || noSpaceAfter.contains(String(out.suffix(1))) + || bindsToName + if out.isEmpty || binds { + out += part.text + } else { + out += " " + part.text + } + previous = part + } + return out + } + + private static let keywords: Set = [ + // DML + "SELECT", "FROM", "WHERE", "AND", "OR", "NOT", "IN", "IS", "NULL", + "LIKE", "BETWEEN", "EXISTS", "AS", "ON", "USING", + "INSERT", "INTO", "VALUES", "UPDATE", "SET", "DELETE", + "JOIN", "INNER", "LEFT", "RIGHT", "FULL", "CROSS", "OUTER", "NATURAL", + "ORDER", "BY", "GROUP", "HAVING", "LIMIT", "OFFSET", "FETCH", "NEXT", "ROWS", "ONLY", + "UNION", "INTERSECT", "EXCEPT", "ALL", "DISTINCT", + // CASE + "CASE", "WHEN", "THEN", "ELSE", "END", + // DDL + "CREATE", "ALTER", "DROP", "TABLE", "INDEX", "VIEW", "DATABASE", "SCHEMA", + "ADD", "COLUMN", "CONSTRAINT", "PRIMARY", "KEY", "FOREIGN", "REFERENCES", + "UNIQUE", "CHECK", "DEFAULT", "AUTO_INCREMENT", "IDENTITY", + "IF", "TEMPORARY", "TEMP", "CASCADE", "RESTRICT", + // CTE + "WITH", "RECURSIVE", + // Data types + "INT", "INTEGER", "BIGINT", "SMALLINT", "TINYINT", + "VARCHAR", "CHAR", "TEXT", "NVARCHAR", "NCHAR", + "DECIMAL", "NUMERIC", "FLOAT", "DOUBLE", "REAL", + "BOOLEAN", "BOOL", "BIT", + "DATE", "TIME", "TIMESTAMP", "DATETIME", "INTERVAL", + "BLOB", "CLOB", "BINARY", "VARBINARY", + "JSON", "JSONB", "XML", "UUID", + "SERIAL", "BIGSERIAL", + // Aggregates + "COUNT", "SUM", "AVG", "MIN", "MAX", + // Transaction + "BEGIN", "COMMIT", "ROLLBACK", "SAVEPOINT", + // Other + "ASC", "DESC", "NULLS", "FIRST", "LAST", + "TRUE", "FALSE", "UNKNOWN", + "OVER", "PARTITION", "WINDOW", "FILTER", + "RETURNING", "CONFLICT", "DO", "NOTHING", + "EXPLAIN", "ANALYZE", "TRUNCATE", + "GRANT", "REVOKE", "DENY", + "TOP", "PERCENT", + ] + + private static func tokens(_ sql: String) -> [SQLToken] { + var tokens: [SQLToken] = [] + let chars = Array(sql) + let count = chars.count + var i = 0 + + while i < count { + let ch = chars[i] + + // Line comment: -- ... + if ch == "-" && i + 1 < count && chars[i + 1] == "-" { + let start = i + i += 2 + while i < count && chars[i] != "\n" { + i += 1 + } + tokens.append(SQLToken(type: .comment, value: String(chars[start..=, <=, <>, !=, ||, ::, ->>, -> + if i + 1 < count { + let twoChar = String([chars[i], chars[i + 1]]) + if [">=", "<=", "<>", "!=", "||", "::", "->"].contains(twoChar) { + if twoChar == "->" && i + 2 < count && chars[i + 2] == ">" { + tokens.append(SQLToken(type: .operator, value: "->>")) + i += 3 + } else { + tokens.append(SQLToken(type: .operator, value: twoChar)) + i += 2 + } + continue + } + } + + // Single-character operators + if "=<>+-*/%&|^~!".contains(ch) { + tokens.append(SQLToken(type: .operator, value: String(ch))) + i += 1 + continue + } + + // Punctuation: ( ) , ; . + if "(),;.".contains(ch) { + tokens.append(SQLToken(type: .punctuation, value: String(ch))) + i += 1 + continue + } + + // Words: keywords or identifiers + if ch.isLetter || ch == "_" { + let start = i + i += 1 + while i < count && (chars[i].isLetter || chars[i].isNumber || chars[i] == "_") { + i += 1 + } + let word = String(chars[start.. = ["AND", "RELEASE"] // A name after ROLLBACK TRAN may be a savepoint, and TO, PREPARED or FORCE target something else. private static let otherTarget = "?" + // `/*!` alone runs on every MySQL and MariaDB server; `/*!NNNNN` and `/*M!` only on some. + private static let unversionedOpenerLength = 3 static func rollsBackTransaction(_ sql: String, grammar: SQLLexicalGrammar, statusMessage: String? = nil) -> Bool { // PostgreSQL answers a COMMIT in a failed transaction by rolling it back. if let statusMessage, statusMessage == PostgreSQLSessionTransaction.failedTransactionRolledBackMessage { return true } - let words = leadingWords(of: sql, limit: 3, grammar: grammar) + // A versioned `/*!80000 ... */` runs only on a server that new, so either reading may be the one that ran. + return rollsBack(leadingWords(of: sql, limit: 3, grammar: grammar, readsVersionedBodies: true)) + || rollsBack(leadingWords(of: sql, limit: 3, grammar: grammar, readsVersionedBodies: false)) + } + + private static func rollsBack(_ words: [String]) -> Bool { switch words.first ?? "" { case "ABORT": return true @@ -30,7 +37,12 @@ nonisolated enum TransactionRollBackClassifier { } // Comments end where the engine ends them: a MySQL `#` line, a nested PostgreSQL `/* */`. - private static func leadingWords(of sql: String, limit: Int, grammar: SQLLexicalGrammar) -> [String] { + private static func leadingWords( + of sql: String, + limit: Int, + grammar: SQLLexicalGrammar, + readsVersionedBodies: Bool + ) -> [String] { let text = sql as NSString var words: [String] = [] var index = 0 @@ -49,11 +61,16 @@ nonisolated enum TransactionRollBackClassifier { } if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { if span.kind == .executableComment, - let opener = SqlLexer.executableCommentOpenerLength(text, at: index, length: text.length) { + let opener = SqlLexer.executableCommentOpenerLength(text, at: index, length: text.length), + readsVersionedBodies || opener == unversionedOpenerLength { executableBody = span index += opener continue } + if span.kind == .executableComment { + index = max(span.end, index + 1) + continue + } guard span.kind.isComment else { words.append(otherTarget) break diff --git a/apple/TableProMobile/TableProMobileTests/Helpers/TransactionRollBackClassifierTests.swift b/apple/TableProMobile/TableProMobileTests/Helpers/TransactionRollBackClassifierTests.swift index c03118bdd3..c4657f0e56 100644 --- a/apple/TableProMobile/TableProMobileTests/Helpers/TransactionRollBackClassifierTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Helpers/TransactionRollBackClassifierTests.swift @@ -63,5 +63,7 @@ struct TransactionRollBackClassifierTests { #expect(TransactionRollBackClassifier.rollsBackTransaction("ROLLBACK /*! AND CHAIN */", grammar: mysql)) #expect(TransactionRollBackClassifier.rollsBackTransaction("/*!40001 ROLLBACK */", grammar: mysql)) #expect(!TransactionRollBackClassifier.rollsBackTransaction("ROLLBACK /*! TO a */", grammar: mysql)) + #expect(TransactionRollBackClassifier.rollsBackTransaction("ROLLBACK /*!99999 TO s */", grammar: mysql)) + #expect(TransactionRollBackClassifier.rollsBackTransaction("/*!40001 ROLLBACK */", grammar: mysql)) } } diff --git a/apple/TableProTests/Core/Storage/QueryHistoryFingerprintMigrationTests.swift b/apple/TableProTests/Core/Storage/QueryHistoryFingerprintMigrationTests.swift index 4f810dc609..65ac38ecb9 100644 --- a/apple/TableProTests/Core/Storage/QueryHistoryFingerprintMigrationTests.swift +++ b/apple/TableProTests/Core/Storage/QueryHistoryFingerprintMigrationTests.swift @@ -163,7 +163,7 @@ struct QueryHistoryFingerprintMigrationTests { #expect(snapshot.mostRun.first?.callCount == 2) } - @Test("Rows the previous stable wrote get their shape, and this build leaves that release its own fingerprint") + @Test("Rows the previous stable wrote get their shape, and a new row keeps that release's fingerprint") func shapeColumnLeavesThePreviousStableItsFingerprint() async { let connectionId = UUID() let url = makeVersion3Database( @@ -190,7 +190,7 @@ struct QueryHistoryFingerprintMigrationTests { #expect(snapshot.mostRun.first?.callCount == 3) #expect(snapshot.mostRun.first?.normalizedQuery == "SELECT ? AS p, a FROM t") - // The previous stable fills a zero `fingerprint_hash` with its own reading when it opens the store. + // The previous stable groups and matches by `fingerprint_hash`, so a new row carries that release's reading. var db: OpaquePointer? guard sqlite3_open(url.path(percentEncoded: false), &db) == SQLITE_OK else { return } defer { sqlite3_close_v2(db) } @@ -199,7 +199,8 @@ struct QueryHistoryFingerprintMigrationTests { let sql = "SELECT fingerprint_hash, shape_hash FROM history WHERE query LIKE '%E:%';" #expect(sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK) #expect(sqlite3_step(statement) == SQLITE_ROW) - #expect(sqlite3_column_int64(statement, 0) == 0) - #expect(sqlite3_column_int64(statement, 1) != 0) + let query = "SELECT 'E:\\' AS p, a FROM t" + #expect(sqlite3_column_int64(statement, 0) == SQLLegacyFingerprint.hash(query, databaseType: .postgresql)) + #expect(sqlite3_column_int64(statement, 1) == SQLQueryFingerprint.hash(query, databaseType: .postgresql)) } } diff --git a/apple/TableProTests/Core/Storage/QueryPlanSnapshotStorageTests.swift b/apple/TableProTests/Core/Storage/QueryPlanSnapshotStorageTests.swift index 15571ae8cc..438856bde0 100644 --- a/apple/TableProTests/Core/Storage/QueryPlanSnapshotStorageTests.swift +++ b/apple/TableProTests/Core/Storage/QueryPlanSnapshotStorageTests.swift @@ -4,6 +4,7 @@ // import Foundation +import SQLite3 @testable import TablePro import TableProPluginKit import Testing @@ -23,6 +24,24 @@ struct QueryPlanSnapshotStorageTests { #expect(found.map(\.id) == [first]) } + @Test("A plan keeps the previous stable's fingerprint beside its shape") + func planKeepsThePreviousStablesFingerprint() async throws { + let store = TemporaryQueryHistoryStore() + let subject = "SELECT data #>> '{a}' FROM t WHERE tags @> ARRAY['x']" + _ = await store.record(subjectSQL: subject, rawPlan: "plan") + + var db: OpaquePointer? + try #require(sqlite3_open(store.url.path(percentEncoded: false), &db) == SQLITE_OK) + defer { sqlite3_close_v2(db) } + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + try #require(sqlite3_prepare_v2(db, "SELECT fingerprint_hash, shape_hash FROM plan_snapshots;", -1, &statement, nil) == SQLITE_OK) + try #require(sqlite3_step(statement) == SQLITE_ROW) + #expect(sqlite3_column_int64(statement, 0) == SQLLegacyFingerprint.hash(subject, databaseType: .postgresql)) + #expect(sqlite3_column_int64(statement, 1) == SQLQueryFingerprint.hash(subject, databaseType: .postgresql)) + #expect(sqlite3_column_int64(statement, 0) != sqlite3_column_int64(statement, 1)) + } + @Test("A different statement does not") func differentStatementDoesNotMatch() async { let store = TemporaryQueryHistoryStore() @@ -168,11 +187,12 @@ struct QueryPlanSnapshotStorageTests { /// developer's own history. private final class TemporaryQueryHistoryStore { let storage: QueryHistoryStorage + let url: URL private let connectionId = UUID() private(set) var lastHistoryId = UUID() init() { - let url = FileManager.default.temporaryDirectory + url = FileManager.default.temporaryDirectory .appendingPathComponent("plan-snapshots-\(UUID().uuidString).db") storage = QueryHistoryStorage(databaseURL: url, removeDatabaseOnDeinit: true) } diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLLegacyFingerprintTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLLegacyFingerprintTests.swift new file mode 100644 index 0000000000..f717a7eba9 --- /dev/null +++ b/apple/TableProTests/Core/Utilities/SQL/SQLLegacyFingerprintTests.swift @@ -0,0 +1,103 @@ +// +// SQLLegacyFingerprintTests.swift +// TableProTests +// +// Each shape was produced by v0.86.0's own SQLTokenizer and SQLQueryFingerprint, whose column this reading fills. +// + +import Foundation +import TableProPluginKit +@testable import TablePro +import Testing + +struct SQLLegacyFingerprintTests { + static let corpus: [(DatabaseType, String, String)] = [ + (.postgresql, "SELECT * FROM t WHERE id = 1", "SELECT * FROM t WHERE id = ?"), + (.postgresql, "select id, name from users where active = true and role = 'admin'", "SELECT id, name FROM users WHERE active = ? AND role = ?"), + (.mysql, "SELECT @@version", "SELECT @ ?"), + (.mysql, "SELECT @@sql_mode", "SELECT @ ?"), + (.mysql, "SELECT 1", "SELECT ?"), + (.mssql, "SELECT @@ROWCOUNT, @@ERROR", "SELECT @ ?, @ ?"), + (.sqlite, "SELECT @1, @1ab FROM t WHERE a = @2", "SELECT ?, ? FROM t WHERE a = ?"), + (.mssql, "SELECT @x, @y FROM t WHERE a = @p", "SELECT ?, ? FROM t WHERE a = ?"), + (.mysql, "SELECT apply - 1 FROM t", "SELECT apply - ? FROM t"), + (.mysql, "SELECT apply + 1 FROM t", "SELECT apply + ? FROM t"), + (.mysql, "SELECT * FROM apply a", "SELECT * FROM apply a"), + (.mysql, "SELECT 0bar FROM t", "SELECT ? bar FROM t"), + (.mysql, "SELECT 0baz FROM t", "SELECT ? baz FROM t"), + (.mysql, "SELECT 0xFF, 0x1A, 0b101 FROM t", "SELECT ? xFF, ? x1A, ? b101 FROM t"), + (.mysql, "SELECT 2fa_codes.id FROM 2fa_codes", "SELECT ? fa_codes.id FROM ? fa_codes"), + (.mssql, "SELECT N'x', N'longer text' FROM t", "SELECT N ?, N ? FROM t"), + (.mysql, "SELECT X'4D', B'101', _utf8mb4'a' FROM t", "SELECT X ?, B ?, _utf8mb4 ? FROM t"), + (.mysql, "SELECT a'alias', b'alias' FROM t", "SELECT a ?, b ? FROM t"), + (.postgresql, "SELECT U&'d\\0061t', E'it\\'s' FROM t", "SELECT U & ?, E ? FROM t"), + (.oracle, "SELECT q'[it's]' FROM dual", "SELECT q ? s] ?"), + (.mssql, "SELECT [order id], [group] FROM [my table] WHERE [select] = 1", "SELECT[ORDER id], GROUP FROM[my TABLE] WHERE SELECT = ?"), + (.mssql, "SELECT * FROM [users]", "SELECT * FROM users"), + (.mssql, "SELECT * FROM users", "SELECT * FROM users"), + (.mssql, "SELECT * INTO #tmp FROM t; SELECT * FROM ##global", "SELECT * INTO # tmp FROM t; SELECT * FROM # # global"), + (.mssql, "SELECT Part# FROM Parts", "SELECT Part # FROM Parts"), + (.mssql, "SET @x += 1", "SET ? + = ?"), + (.sqlite, "SELECT * FROM t WHERE a == b", "SELECT * FROM t WHERE a = = b"), + (.oracle, "SELECT y ** 2 FROM dual WHERE a ^= b", "SELECT y * * ? FROM dual WHERE a ^ = b"), + (.duckdb, "SELECT 7 // 2", "SELECT ? / / ?"), + (.mysql, "SELECT a <=> b, @a := 1 FROM t", "SELECT a <= > b, ? : = ? FROM t"), + (.postgresql, "SELECT $$secret payload$$ FROM t", "SELECT ? FROM t"), + (.postgresql, "SELECT $tag$body here$tag$ FROM t", "SELECT ? FROM t"), + (.postgresql, "CREATE FUNCTION f() RETURNS int AS $$ BEGIN RETURN 1; END; $$ LANGUAGE plpgsql", "CREATE FUNCTION f() RETURNS INT AS ? LANGUAGE plpgsql"), + (.postgresql, "SELECT a::int, b::text FROM t", "SELECT a :: INT, b :: TEXT FROM t"), + (.postgresql, "SELECT data->>'name', data->'a', data #>> '{a,b}' FROM t", "SELECT data ->> ?, data -> ?, data # > > ? FROM t"), + (.postgresql, "SELECT * FROM t WHERE tags @> ARRAY['x'] AND d ?| array['a']", "SELECT * FROM t WHERE tags @ > ARRAY[?] AND d ? | array[?]"), + (.postgresql, "SELECT * FROM t WHERE name ~* 'abc' AND name !~ 'x'", "SELECT * FROM t WHERE name ~ * ? AND name ! ~ ?"), + (.postgresql, "SELECT embedding <-> '[1,2,3]' FROM items", "SELECT embedding < -> ? FROM items"), + (.postgresql, "-- pick\nSELECT * FROM t /* hint */ WHERE id = 1", "SELECT * FROM t WHERE id = ?"), + (.mysql, "SELECT 1 # comment\nFROM t", "SELECT ? # comment FROM t"), + (.mysql, "SELECT 5--3", "SELECT ?"), + (.postgresql, "SELECT 1 /* a /* b */ c */ FROM t", "SELECT ? c * / FROM t"), + (.postgresql, "select a, b -- note\nfrom t where x = 1", "SELECT a, b FROM t WHERE x = ?"), + (.mysql, "SELECT /*!40001 SQL_NO_CACHE */ a FROM t", "SELECT a FROM t"), + (.postgresql, "SELECT * FROM t WHERE id IN (1, 2, 3)", "SELECT * FROM t WHERE id IN (...)"), + (.postgresql, "SELECT * FROM t WHERE id IN (SELECT id FROM u)", "SELECT * FROM t WHERE id IN (SELECT id FROM u)"), + (.postgresql, "INSERT INTO t VALUES (1, 'a'), (2, 'b')", "INSERT INTO t VALUES (...)"), + (.postgresql, "SELECT * FROM t WHERE x = -1 AND y = +2", "SELECT * FROM t WHERE x = ? AND y = ?"), + (.postgresql, "SELECT b - 1, f(a, -1) FROM t", "SELECT b - ?, f(a, ?) FROM t"), + (.postgresql, "SELECT tags[1] - 1 FROM t", "SELECT tags[?] - ? FROM t"), + (.postgresql, "SELECT 0.5, .5, 1e10, 1.5E-3 FROM t", "SELECT ?, ?, ?, ? FROM t"), + (.mysql, "SELECT * FROM t WHERE n = \"bob\"", "SELECT * FROM t WHERE n = ?"), + (.postgresql, "SELECT \"email\" FROM \"Users\" WHERE id = 1", "SELECT email FROM Users WHERE id = ?"), + (.mysql, "SELECT * FROM `users`", "SELECT * FROM users"), + (.oracle, "SELECT * FROM v$session WHERE sid = 1", "SELECT * FROM v ? WHERE sid = ?"), + (.oracle, "BEGIN x := 1; END;", "BEGIN x : = ?; END;"), + (.mssql, "SELECT 1\nGO\nSELECT 2", "SELECT ? GO SELECT ?"), + (.mysql, "DELIMITER //\nCREATE PROCEDURE p() BEGIN SELECT 1; END //\nDELIMITER ;", "DELIMITER / / CREATE PROCEDURE p() BEGIN SELECT ?; END / / DELIMITER;"), + (.mysql, "DELIMITER $$\nEND$$", "DELIMITER ?"), + (.clickhouse, "SELECT database, table FROM system.columns WHERE database = 'default'", "SELECT DATABASE, TABLE FROM system.columns WHERE DATABASE = ?"), + (.postgresql, "SELECT count(*), sum(total) FROM t GROUP BY 1", "SELECT COUNT(*), SUM(total) FROM t GROUP BY ?"), + (.postgresql, "SELECT * FROM t WHERE created > now() - interval '1 day'", "SELECT * FROM t WHERE created > now() - INTERVAL ?"), + (.postgresql, "SELECT 'C:\\', 'hello world' FROM t", "SELECT ? hello world ?"), + (.mysql, "SELECT 'it\\'s' FROM t", "SELECT ? FROM t"), + (.postgresql, "SELECT café, naïve FROM t", "SELECT café, naïve FROM t"), + (.postgresql, "SELECT * FROM t WHERE id = $1 AND name = :name", "SELECT * FROM t WHERE id = ? AND name = ?"), + (.postgresql, "SELECT * FROM t WHERE id = ?", "SELECT * FROM t WHERE id = ?"), + (.postgresql, "UPDATE t SET a = 1, b = 'x' WHERE id = 7", "UPDATE t SET a = ?, b = ? WHERE id = ?"), + (.postgresql, "DELETE FROM t WHERE id IN (1, 2)", "DELETE FROM t WHERE id IN (...)"), + (.postgresql, "SELECT CASE WHEN a = 1 THEN 'x' ELSE 'y' END FROM t", "SELECT CASE WHEN a = ? THEN ? ELSE ? END FROM t"), + (.postgresql, "SELECT * FROM t1 JOIN t2 ON t1.id = t2.id LEFT OUTER JOIN t3 ON t3.id = t1.id", "SELECT * FROM t1 JOIN t2 ON t1.id = t2.id LEFT OUTER JOIN t3 ON t3.id = t1.id"), + (.mssql, "SELECT * FROM a OUTER APPLY f(a.id) x CROSS APPLY g(a.id) y", "SELECT * FROM a OUTER APPLY f(a.id) x CROSS APPLY g(a.id) y"), + (.postgresql, "select 'foo'\n'bar' as s", "SELECT ? AS s"), + (.postgresql, "SELECT a FROM t WHERE b = 'x''y'", "SELECT a FROM t WHERE b = ?"), + ] + + @Test("Every query reads as the previous stable read it") + func readsAsThePreviousStable() { + for (type, sql, shape) in Self.corpus { + #expect(SQLLegacyFingerprint.normalize(sql, databaseType: type) == shape, "\(type.rawValue): \(sql)") + } + } + + @Test("A stored hash is the previous stable's for the same text") + func hashIsThePreviousStables() { + #expect(SQLLegacyFingerprint.hash("SELECT * FROM t WHERE id = 5", databaseType: .postgresql) == 1_936_778_350_161_199_288) + } +} +