Repository navigation
Expand file tree
/
Copy pathSQLNonCodeSpan.swift
More file actions
432 lines (400 loc) · 18 KB
/
Copy pathSQLNonCodeSpan.swift
File metadata and controls
432 lines (400 loc) · 18 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
import Foundation
/// The one lexer: where a comment, a literal or a quoted identifier that starts at an offset ends, read by an engine's
/// ``SQLLexicalGrammar``.
///
/// Everything that has to know where code stops asks here: the statement scanner, the folding scanner, the token
/// cursor, the row-limit detector, the code-only projection the classifiers read, and the diagnostics. Two readers
/// that disagree about where a string ends can disagree about where a statement ends, and the gate that tiers one
/// statement would then let the engine run two.
public enum SQLNonCodeSpan {
public enum Kind: Sendable, Equatable {
case lineComment
case blockComment
/// MySQL's `/*! ... */`, whose body the server runs. It is code to a reader of statements, and one opaque
/// token to a reader of boundaries.
case executableComment
/// A string, a dollar-quoted body, or a quoted identifier: one token nothing inside can end early.
case quoted
/// SQLite's `$name(...)` parameter, which swallows quotes and semicolons up to its `)`.
case parameter
public var isComment: Bool {
self == .lineComment || self == .blockComment
}
}
public struct Span: Sendable, Equatable {
public let kind: Kind
public let start: Int
/// Past the closing delimiter, or the end of the text when the span never closes.
public let end: Int
/// Where the body stops, before the closing delimiter.
public let contentEnd: Int
/// Line feeds inside the span. A line comment ends at its line feed and does not count it.
public let newlines: Int
/// False when the text ended before the span's closing delimiter, so everything after its start is inside it.
public let isTerminated: Bool
public init(kind: Kind, start: Int, end: Int, contentEnd: Int, newlines: Int, isTerminated: Bool) {
self.kind = kind
self.start = start
self.end = end
self.contentEnd = contentEnd
self.newlines = newlines
self.isTerminated = isTerminated
}
}
private static let openBracket = UInt16(UnicodeScalar("[").value)
private static let closeBracket = UInt16(UnicodeScalar("]").value)
private static let capitalE = UInt16(UnicodeScalar("E").value)
private static let smallE = UInt16(UnicodeScalar("e").value)
private static let smallB = UInt16(UnicodeScalar("b").value)
/// The comment, literal or quoted identifier starting at `index`, or nil when `index` starts code.
public static func span(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Span? {
let length = text.length
guard index >= 0, index < length else { return nil }
let character = text.character(at: index)
if let comment = commentSpan(at: index, character: character, in: text, length: length, grammar: grammar) {
return comment
}
if grammar.isQuote(character) {
return quotedSpan(at: index, quote: character, in: text, length: length, grammar: grammar)
}
if grammar.contains(.bracketQuotedIdentifiers), character == openBracket {
let span = SqlLexer.skipBracketedIdentifier(
text,
from: index,
length: length,
doubledCloseEscapes: grammar.contains(.doubledClosingBracketEscapes)
)
return closedSpan(.quoted, at: index, span, closerLength: 1)
}
if grammar.contains(.angleQuotedIdentifiers), character == SqlLexer.openAngleQuote {
let span = SqlLexer.skipAngleQuotedIdentifier(
text,
from: index,
length: length,
backslashEscapes: grammar.contains(.backslashEscapesInAngleQuotes)
)
return closedSpan(.quoted, at: index, span, closerLength: 1)
}
if let prefixed = prefixedLiteralSpan(at: index, character: character, in: text, length: length, grammar: grammar) {
return prefixed
}
if character == SqlDollarQuote.dollar, let dollar = dollarQuotedSpan(at: index, in: text, grammar: grammar) {
return dollar
}
guard grammar.contains(.parenthesizedParameterNames),
let parameter = SqlLexer.skipParenthesizedParameterName(text, at: index, length: length)
else {
return nil
}
return Span(
kind: .parameter,
start: index,
end: parameter.next,
contentEnd: parameter.next,
newlines: 0,
isTerminated: parameter.isClosed
)
}
/// Where the span starting at `index` ends, or nil when `index` starts code. An executable comment is code, so it
/// answers nil: the reader walks into it and reads its body.
public static func end(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Int? {
guard let span = span(at: index, in: text, grammar: grammar), span.kind != .executableComment else {
return nil
}
return span.end
}
public static func isWordUnit(_ unit: UInt16) -> Bool {
if unit < 0x80 {
return SqlDollarQuote.isIdentifierPart(unit)
}
return !SQLSeparatingCharacter.isSeparating(unit)
}
/// Whether a literal prefix right after `unit` is part of the name `unit` belongs to, so `x$e'\'` is the name
/// `x$e` and a plain string. PostgreSQL 17.11 continues a name with `$` and every unit from 0x80 up, and DuckDB
/// 1.5.4 and 2.0.0 do too, less the spaces they blank (measured). CockroachDB's scanner reads bytes the same way, and
/// KingbaseES, a PostgreSQL fork, is taken to. Oracle and Dameng document `$` and `#` as name characters, and run
/// one statement per call.
public static func continuesName(beforePrefix unit: UInt16, grammar: SQLLexicalGrammar) -> Bool {
if unit >= 0x80 {
return !endsNameAsBlank(unit, grammar: grammar)
}
return SqlDollarQuote.isIdentifierPart(unit) || unit == SqlDollarQuote.dollar
|| (unit == SqlLexer.hash && grammar.contains(.dollarAndHashInIdentifiers))
}
/// Whether a `$` right after `unit` is part of the name `unit` belongs to, as in `a$$b`, rather than a dollar
/// quote's opener. On DuckDB `SELECT` + U+200B + `$$...$$` opens one, measured on 2.0.0.
public static func continuesName(beforeDollar unit: UInt16, grammar: SQLLexicalGrammar) -> Bool {
SqlDollarQuote.isIdentifierContinuation(unit) && !endsNameAsBlank(unit, grammar: grammar)
}
private static func endsNameAsBlank(_ unit: UInt16, grammar: SQLLexicalGrammar) -> Bool {
grammar.contains(.nonASCIISpacesEndNames) && spacesBeforeLexing.contains(unit)
}
// MARK: - Comments
private static func commentSpan(
at index: Int,
character: UInt16,
in text: NSString,
length: Int,
grammar: SQLLexicalGrammar
) -> Span? {
if startsLineComment(at: index, character: character, in: text, length: length, grammar: grammar) {
let end = SqlLexer.endOfLineComment(
text,
from: index,
length: length,
carriageReturnEnds: grammar.contains(.carriageReturnEndsLineComments),
formFeedEnds: grammar.contains(.formFeedEndsLineComments)
)
return Span(kind: .lineComment, start: index, end: end, contentEnd: end, newlines: 0, isTerminated: true)
}
guard SqlLexer.startsBlockComment(text, at: index, length: length) else { return nil }
if grammar.contains(.executableComments),
let opener = SqlLexer.executableCommentOpenerLength(text, at: index, length: length) {
return executableCommentSpan(at: index, openerLength: opener, in: text, length: length, grammar: grammar)
}
let comment = plainBlockComment(at: index, in: text, length: length, grammar: grammar)
return closedSpan(.blockComment, at: index, comment, closerLength: 2)
}
private static func plainBlockComment(
at index: Int,
in text: NSString,
length: Int,
grammar: SQLLexicalGrammar
) -> SqlLexer.Span {
grammar.contains(.nestedBlockComments)
? SqlLexer.skipNestedBlockComment(text, from: index, length: length)
: SqlLexer.skipBlockComment(text, from: index, length: length)
}
// The server lexes a run body as code, comments included, and a nested opener opens no second level: the first
// `*/` read as code closes the comment. Measured on MySQL 8.0 and 8.4 and MariaDB 10.11 and 11.4.
private static func executableCommentSpan(
at index: Int,
openerLength: Int,
in text: NSString,
length: Int,
grammar: SQLLexicalGrammar
) -> Span {
var cursor = index + openerLength
var newlines = 0
while cursor < length {
let character = text.character(at: cursor)
if character == SqlLexer.star, cursor + 1 < length, text.character(at: cursor + 1) == SqlLexer.slash {
return Span(
kind: .executableComment,
start: index,
end: cursor + 2,
contentEnd: cursor,
newlines: newlines,
isTerminated: true
)
}
if SqlLexer.startsBlockComment(text, at: cursor, length: length) {
if let nested = SqlLexer.executableCommentOpenerLength(text, at: cursor, length: length) {
cursor += nested
continue
}
let comment = plainBlockComment(at: cursor, in: text, length: length, grammar: grammar)
newlines += comment.newlines
cursor = comment.next
continue
}
if let inner = span(at: cursor, in: text, grammar: grammar) {
newlines += inner.newlines
cursor = max(inner.end, cursor + 1)
continue
}
if character == SqlLexer.newline {
newlines += 1
}
cursor += 1
}
return Span(
kind: .executableComment,
start: index,
end: length,
contentEnd: length,
newlines: newlines,
isTerminated: false
)
}
private static func startsLineComment(
at index: Int,
character: UInt16,
in text: NSString,
length: Int,
grammar: SQLLexicalGrammar
) -> Bool {
if character == SqlLexer.dash {
guard !grammar.contains(.withoutDashComments), SqlLexer.startsDashComment(
text,
at: index,
length: length,
needsWhitespace: grammar.contains(.dashCommentsNeedWhitespace)
) else { return false }
return !grammar.contains(.exponentSignsNeedNoDigits) || !TSQLTokens.signsExponent(at: index, in: text)
}
if character == SqlLexer.hash {
return grammar.contains(.hashLineComments)
}
guard character == SqlLexer.slash, grammar.contains(.doubleSlashLineComments) else { return false }
return SqlLexer.startsDoubleSlash(text, at: index, length: length)
}
// MARK: - Literals
private static func quotedSpan(
at index: Int,
quote: UInt16,
in text: NSString,
length: Int,
grammar: SQLLexicalGrammar
) -> Span {
let backslashEscapes = grammar.backslashEscapes(inQuote: quote)
if grammar.contains(.tripleQuotedStrings), SqlLexer.startsTripleQuote(text, at: index, length: length) {
let span = SqlLexer.skipTripleQuotedString(
text,
from: index,
length: length,
backslashEscapes: backslashEscapes
)
return closedSpan(.quoted, at: index, span, closerLength: 3)
}
let span = SqlLexer.skipQuotedString(
text,
from: index,
quote: quote,
length: length,
backslashEscapes: backslashEscapes
)
return closedSpan(.quoted, at: index, span, closerLength: 1)
}
/// `E'...'`, `b'...'` and `q'[...]'`, which only start where a name does not go on: `xE'` and `x$E'` are a name
/// followed by a string.
private static func prefixedLiteralSpan(
at index: Int,
character: UInt16,
in text: NSString,
length: Int,
grammar: SQLLexicalGrammar
) -> Span? {
guard index == 0 || !continuesName(beforePrefix: text.character(at: index - 1), grammar: grammar) else {
return nil
}
let startsEscapeString = grammar.contains(.escapeStringPrefix) && (character == capitalE || character == smallE)
let startsEscapeBytes = grammar.contains(.escapeBytesPrefix) && character == smallB
if startsEscapeString || startsEscapeBytes,
index + 1 < length,
text.character(at: index + 1) == SqlLexer.singleQuote {
var span = SqlLexer.skipQuotedString(
text,
from: index + 1,
quote: SqlLexer.singleQuote,
length: length,
backslashEscapes: true
)
if grammar.contains(.escapeStringContinuations) {
var newlines = span.newlines
while span.isClosed, let continued = continuingQuote(after: span.next, in: text, length: length) {
span = SqlLexer.skipQuotedString(
text,
from: continued.quote,
quote: SqlLexer.singleQuote,
length: length,
backslashEscapes: true
)
newlines += continued.newlines + span.newlines
}
span = SqlLexer.Span(next: span.next, newlines: newlines, isClosed: span.isClosed)
}
return closedSpan(.quoted, at: index, span, closerLength: 1)
}
guard grammar.contains(.alternativeQuoting),
let literal = SqlLexer.skipAlternativeQuotedString(text, at: index, length: length)
else {
return nil
}
return Span(
kind: .quoted,
start: index,
end: literal.span.next,
contentEnd: literal.bodyEnd,
newlines: literal.span.newlines,
isTerminated: literal.span.isClosed
)
}
private static let verticalTab: UInt16 = 0x0B
// DuckDB turns these into spaces before it lexes, outside what its pre-lexing pass reads as a quote, and no other
// non-ASCII unit (measured on 1.5.4 and 2.0.0).
// PostgreSQL and CockroachDB do not, and the text a join there leaves them does not parse, so they run none of it.
static let spacesBeforeLexing: Set<UInt16> = Set([0x00A0, 0x202F, 0x205F, 0x2060, 0x3000, 0xFEFF])
.union(0x2000...0x200B)
/// The quote that carries on the string closed just before `index`, as PostgreSQL's `quotecontinue` finds it: only
/// blanks and `--` comments between, a line break among them. PostgreSQL 16 and later take a vertical tab for a
/// blank; an older server refuses the text.
private static func continuingQuote(after index: Int, in text: NSString, length: Int) -> (quote: Int, newlines: Int)? {
var cursor = index
var lineBroke = false
var newlines = 0
while cursor < length {
switch text.character(at: cursor) {
case SqlLexer.newline:
lineBroke = true
newlines += 1
cursor += 1
case SqlLexer.carriageReturn:
lineBroke = true
cursor += 1
case SqlLexer.space, SqlLexer.tab, verticalTab, SqlLexer.formFeed:
cursor += 1
case let unit where spacesBeforeLexing.contains(unit):
cursor += 1
case SqlLexer.dash where SqlLexer.startsLineComment(text, at: cursor, length: length):
cursor = SqlLexer.endOfLineComment(
text,
from: cursor,
length: length,
carriageReturnEnds: true,
formFeedEnds: false
)
case SqlLexer.singleQuote where lineBroke:
return (cursor, newlines)
default:
return nil
}
}
return nil
}
private static func dollarQuotedSpan(at index: Int, in text: NSString, grammar: SQLLexicalGrammar) -> Span? {
let length = text.length
guard let style = grammar.dollarQuoteStyle,
case .opener(let openerLength, let tag) = SqlDollarQuote.scanOpener(
at: index,
in: text,
bufLen: length,
style: style,
continuesName: { SQLNonCodeSpan.continuesName(beforeDollar: $0, grammar: grammar) }
)
else {
return nil
}
let body = SqlLexer.skipDollarQuotedBody(text, from: index + openerLength, tag: tag, length: length)
return Span(
kind: .quoted,
start: index,
end: body.span.next,
contentEnd: body.bodyEnd,
newlines: body.span.newlines,
isTerminated: body.span.isClosed
)
}
/// A span whose closer is `closerLength` units long. One that ran to the end of the text without a closer runs its
/// content to the end of the text.
private static func closedSpan(_ kind: Kind, at start: Int, _ span: SqlLexer.Span, closerLength: Int) -> Span {
Span(
kind: kind,
start: start,
end: span.next,
contentEnd: span.isClosed ? max(start, span.next - closerLength) : span.next,
newlines: span.newlines,
isTerminated: span.isClosed
)
}
}