Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,11 @@ public struct QueryScript: Sendable, Equatable {
return SQLStatementScanner.executableStatements(in: unit.sql, grammar: grammar).map(\.sql)
}

// The grammar the units were split by; standard SQL for a script built from units alone.
public var lexicalGrammar: SQLLexicalGrammar {
databaseType.map { Self.executionGrammar(for: $0, session: session) } ?? .ansi
}

// A bare Redis argument may end in a byte Foundation trims, such as a vertical tab or a no-break space.
public static func trimmed(_ text: String, databaseType: DatabaseType) -> String {
guard databaseType == .redis else { return text.trimmingCharacters(in: .whitespacesAndNewlines) }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ public enum SQLNonCodeSpan {
private static let capitalE = UInt16(UnicodeScalar("E").value)
private static let smallE = UInt16(UnicodeScalar("e").value)
private static let smallB = UInt16(UnicodeScalar("b").value)
private static let at = UInt16(UnicodeScalar("@").value)
private static let colon = UInt16(UnicodeScalar(":").value)

/// The comment, literal or quoted identifier starting at `index`, or nil when `index` starts code.
public static func span(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Span? {
Expand Down Expand Up @@ -119,6 +121,57 @@ public enum SQLNonCodeSpan {
return span.end
}

// A reader that walks every unit asks this first, and `span(at:)` only where the answer can be a span.
public static func mayStart(_ unit: UInt16) -> Bool {
switch unit {
case SqlLexer.dash, SqlLexer.hash, SqlLexer.slash, SqlLexer.singleQuote, SqlLexer.doubleQuote, SqlLexer.backtick,
openBracket, SqlLexer.openAngleQuote, SqlDollarQuote.dollar, at, colon, capitalE, smallE, smallB,
SqlLexer.smallQ, SqlLexer.capitalQ, SqlLexer.smallN, SqlLexer.capitalN:
return true
default:
return false
}
}

// Units past a span's start that decide its kind (`nq'` and a delimiter outside the BMP), and past its end that
// decide where it ends (a doubled quote or `]]` after the closer).
private static let openerLookahead = 8
private static let closerLookahead = 1

// Whether text appended after `text` could change what `span(at: index)` answered, which a chunked reader waits on.
public static func needsMoreText(_ span: Span?, at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Bool {
let length = text.length
guard length - index > openerLookahead else { return true }
guard let span else { return opensPastTheEnd(at: index, in: text, grammar: grammar) }
if !span.isTerminated || span.end + closerLookahead > length {
return true
}
guard grammar.contains(.escapeStringContinuations), span.kind == .quoted,
[capitalE, smallE, smallB].contains(text.character(at: span.start)),
case .runsOut = continuation(after: span.end, in: text, length: length, grammar: grammar)
else {
return false
}
return true
}

// A dollar tag or a Tcl-style parameter name can run past any fixed lookahead before it shows what it opens.
private static func opensPastTheEnd(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Bool {
let length = text.length
if let style = grammar.dollarQuoteStyle, text.character(at: index) == SqlDollarQuote.dollar,
case .needsMoreData = SqlDollarQuote.scanOpener(
at: index,
in: text,
bufLen: length,
style: style,
continuesName: { SQLNonCodeSpan.continuesName(beforeDollar: $0, grammar: grammar) }
) {
return true
}
guard grammar.contains(.parenthesizedParameterNames) else { return false }
return SqlLexer.parameterNameEnd(text, at: index, length: length) == length
}

public static func isWordUnit(_ unit: UInt16) -> Bool {
if unit < 0x80 {
return SqlDollarQuote.isIdentifierPart(unit)
Expand Down Expand Up @@ -320,7 +373,7 @@ public enum SQLNonCodeSpan {
if grammar.contains(.escapeStringContinuations) {
var newlines = span.newlines
while span.isClosed,
let continued = continuingQuote(after: span.next, in: text, length: length, grammar: grammar) {
case .quote(let continued) = continuation(after: span.next, in: text, length: length, grammar: grammar) {
span = SqlLexer.skipQuotedString(
text,
from: continued.quote,
Expand Down Expand Up @@ -360,12 +413,19 @@ public enum SQLNonCodeSpan {
/// blanks and `--` comments between, a line break among them. PostgreSQL 16 and later take a vertical tab for a
/// blank; an older server refuses the text. A non-ASCII space is a blank only where DuckDB 1.5 has blanked it
/// first: PostgreSQL 17.11 reads `E'a'` + U+00A0 + line break + `'b'` as a name between two strings.
private static func continuingQuote(
private enum Continuation {
case quote((quote: Int, newlines: Int))
case none
// Only blanks and comments up to the end of the text, so more text could still carry the string on.
case runsOut
}

private static func continuation(
after index: Int,
in text: NSString,
length: Int,
grammar: SQLLexicalGrammar
) -> (quote: Int, newlines: Int)? {
) -> Continuation {
var cursor = index
var lineBroke = false
var newlines = 0
Expand All @@ -390,13 +450,15 @@ public enum SQLNonCodeSpan {
carriageReturnEnds: true,
formFeedEnds: false
)
case SqlLexer.dash where cursor + 1 == length:
return .runsOut
case SqlLexer.singleQuote where lineBroke:
return (cursor, newlines)
return .quote((cursor, newlines))
default:
return nil
return .none
}
}
return nil
return .runsOut
}

private static func dollarQuotedSpan(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Span? {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -393,20 +393,11 @@ public enum SqlLexer {
/// `::` continues a name, as in `$a::b(...)`. A `$` inside a word starts no parameter: `a$b(` is the name `a$b`
/// and a `(`.
public static func skipParenthesizedParameterName(_ text: NSString, at offset: Int, length: Int) -> Span? {
guard parameterPrefixes.contains(text.character(at: offset)), !continuesSQLiteWord(text, at: offset) else {
guard var cursor = parameterNameEnd(text, at: offset, length: length),
cursor > offset + 1, cursor < length, text.character(at: cursor) == openParen
else {
return nil
}
var cursor = offset + 1
while cursor < length {
let character = text.character(at: cursor)
if isSQLiteIdentifierCharacter(character) {
cursor += 1
continue
}
guard character == colonUnit, cursor + 1 < length, text.character(at: cursor + 1) == colonUnit else { break }
cursor += 2
}
guard cursor > offset + 1, cursor < length, text.character(at: cursor) == openParen else { return nil }
cursor += 1
while cursor < length {
let character = text.character(at: cursor)
Expand All @@ -421,6 +412,24 @@ public enum SqlLexer {
return Span(next: length, newlines: 0, isClosed: false)
}

// Where a Tcl-style parameter's name ends, before its `(`; nil when no parameter starts at `offset`.
public static func parameterNameEnd(_ text: NSString, at offset: Int, length: Int) -> Int? {
guard parameterPrefixes.contains(text.character(at: offset)), !continuesSQLiteWord(text, at: offset) else {
return nil
}
var cursor = offset + 1
while cursor < length {
let character = text.character(at: cursor)
if isSQLiteIdentifierCharacter(character) {
cursor += 1
continue
}
guard character == colonUnit, cursor + 1 < length, text.character(at: cursor + 1) == colonUnit else { break }
cursor += 2
}
return cursor
}

/// Whether the prefix at `offset` is a `$` inside a word SQLite is already reading. `$` is one of its identifier
/// characters; `@`, `:` and `#` are not, and always start a token.
///
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -204,4 +204,11 @@ struct QueryScriptTests {
#expect(QueryScript.trimmed(" \n ; ", databaseType: .redis).isEmpty)
#expect(QueryScript.trimmed(" \nSELECT 1; \n", databaseType: .sqlite) == "SELECT 1;")
}

@Test("A script hands out the grammar it was split by, and standard SQL when it knows no engine")
func lexicalGrammarIsTheSplittingGrammar() {
#expect(plan("SELECT 1", .mysql).lexicalGrammar.contains(.hashLineComments))
#expect(plan("SELECT 1", .postgresql).lexicalGrammar.contains(.nestedBlockComments))
#expect(QueryScript(units: [.init(sql: "SELECT 1")], unitKind: .statement).lexicalGrammar == .ansi)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -402,4 +402,42 @@ struct SQLNonCodeSpanTests {
)
#expect(sqlite == "SELECT eval ( )")
}

private func needsMoreText(_ text: String, at index: Int = 0, _ grammar: SQLLexicalGrammar) -> Bool {
let buffer = text as NSString
let span = SQLNonCodeSpan.span(at: index, in: buffer, grammar: grammar)
return SQLNonCodeSpan.needsMoreText(span, at: index, in: buffer, grammar: grammar)
}

private static let continued: SQLLexicalGrammar = [.escapeStringPrefix, .escapeStringContinuations]
private static let longTag: String = "$" + String(repeating: "t", count: 40)
private static let longName: String = "$" + String(repeating: "p", count: 40)
private static let blanks = String(repeating: " ", count: 20)

private static let unsettled: [(String, SQLLexicalGrammar)] = [
(longTag, .taggedDollarQuotes),
("'a doubled quote may follow'", .ansi),
("-- a comment with no line break", .ansi),
("/* open comment ", .ansi),
(longName, .parenthesizedParameterNames),
("E'x'" + blanks + "-", continued),
("E'x'\n" + blanks + "-- c", continued),
]

private static let settled: [(String, SQLLexicalGrammar)] = [
("'abc' + 1 ", .ansi),
("E'x'" + blanks + "+ 1", continued),
("-- note\nSELECT 1 FROM t", .ansi),
("SELECT 1 FROM t WHERE a = 1", .ansi),
]

@Test("A reader of a buffer waits when more text could change the span", arguments: unsettled)
func waitsForMoreText(text: String, grammar: SQLLexicalGrammar) {
#expect(needsMoreText(text, grammar), "\(text)")
}

@Test("A span whose end no later text can move is settled", arguments: settled)
func settledSpan(text: String, grammar: SQLLexicalGrammar) {
#expect(!needsMoreText(text, grammar), "\(text)")
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import Foundation
@testable import TableProSQLGrammar
import Testing

// The core's script reader runs the same file in its own tests, so the two read each case where the engine does.
struct SQLScriptSharedCorpusTests {
struct Batch: Decodable, Equatable, CustomStringConvertible {
let text: String
let repeatCount: Int

enum CodingKeys: String, CodingKey {
case text
case repeatCount = "repeat"
}

var description: String { "\(repeatCount)x \(text)" }
}

struct Case: Decodable {
let name: String
let dialect: String
let sql: String
let statements: [String]
let batches: [Batch]?

var grammar: SQLLexicalGrammar? {
Self.databaseTypeIds[dialect].flatMap { SQLLexicalProfile.curated(forDatabaseTypeId: $0)?.grammar }
}

private static let databaseTypeIds = [
"postgresql": "PostgreSQL", "mysql": "MySQL", "sqlite": "SQLite", "mssql": "SQL Server",
"clickhouse": "ClickHouse", "duckdb": "DuckDB", "cql": "Cassandra",
]
}

private struct Corpus: Decodable {
let cases: [Case]
}

static let corpusURL = URL(fileURLWithPath: #filePath)
.deletingLastPathComponent()
.deletingLastPathComponent()
.deletingLastPathComponent()
.deletingLastPathComponent()
.deletingLastPathComponent()
.deletingLastPathComponent()
.appendingPathComponent("core/crates/core/tests/fixtures/sql_script_corpus.json")

static func cases() throws -> [Case] {
try JSONDecoder().decode(Corpus.self, from: Data(contentsOf: corpusURL)).cases
}

@Test("Every dialect the corpus names has a curated grammar")
func everyDialectIsKnown() throws {
let cases = try Self.cases()
#expect(!cases.isEmpty)
#expect(cases.filter { $0.grammar == nil }.map(\.dialect).isEmpty)
}

@Test("The scanner splits each case where the engine does")
func scannerSplitsEachCase() throws {
for testCase in try Self.cases() {
guard let grammar = testCase.grammar else { continue }
let statements = SQLStatementScanner.executableStatements(in: testCase.sql, grammar: grammar)
#expect(statements.map(\.sql) == testCase.statements, "\(testCase.dialect): \(testCase.name)")
guard let expected = testCase.batches else { continue }
let separators = SQLStatementScanner.batchSeparators(in: testCase.sql, grammar: grammar)
let batches = SQLStatementScanner.batches(in: testCase.sql, statements: statements, separators: separators)
let read = batches.map { Batch(text: $0.sql, repeatCount: $0.repeatCount) }
#expect(read == expected, "\(testCase.dialect): \(testCase.name)")
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ struct ConsoleRequestDiagnosticsProducer: QueryDiagnosticsProducing {

// The body highlights as JavaScript, so `//` and `/* */` read as comments on screen and the
// scan has to read them the same way.
let structure = QueryBracketScanner.scan(blankingRequestLine(source), comments: .javaScript)
let structure = QueryBracketScanner.scan(blankingRequestLine(source), language: .javaScript)
var results: [QueryDiagnostic] = []

if let range = structure.unmatchedClose {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ struct MongoDiagnosticsProducer: QueryDiagnosticsProducing {
// The bracket scanner has no regex state, so `db.c.find({x:/[)]/})` reads as an unmatched
// closer. The syntax checker below knows JavaScript properly, so the structural pass is
// only there to stay quiet while a query is still being typed.
let structure = QueryBracketScanner.scan(source, comments: .javaScript)
let structure = QueryBracketScanner.scan(source, language: .javaScript)
let hasRegexLiteral = source.range(of: "/", options: .literal).location != NSNotFound

if let range = structure.unmatchedClose, !hasRegexLiteral {
Expand Down
Loading
Loading