Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@ internal enum InvisibleCharacterRemover {
let end = min(NSMaxRange(scope), text.length)
var replacements: [TextReplacement] = []
var index = max(scope.location, 0)
// Literals are found in the text as cleaned so far, since that is what runs: PostgreSQL reads `SELECT<NBSP>E` as
// one name, but the cleaned `SELECT E'...'` holds an escape string whose characters stay.
let cleanedText = skippingLiteralsAndComments ? NSMutableString(string: text as String) : nil
while index < end {
if skippingLiteralsAndComments, let skipTo = SQLNonCodeSpan.end(at: index, in: text, grammar: grammar) {
if let cleanedText, let skipTo = SQLNonCodeSpan.end(at: index, in: cleanedText, grammar: grammar) {
index = skipTo
continue
}
Expand All @@ -32,6 +35,10 @@ internal enum InvisibleCharacterRemover {
}
let cleaned = replacement(for: classified, lineEnding: lineEnding)
replacements.append(TextReplacement(range: classified.range, string: cleaned))
if let cleanedText {
let filler = standIn(for: cleaned, replacing: classified.range, in: cleanedText)
cleanedText.replaceCharacters(in: classified.range, with: filler)
}
index = NSMaxRange(classified.range)
}
return replacements
Expand All @@ -51,6 +58,21 @@ internal enum InvisibleCharacterRemover {
return offset + delta
}

// Keeps offsets: the unit the cleaned text has before the next one, the replacement's last or, for a removed
// character, the one already before it.
private static func standIn(for cleaned: String, replacing range: NSRange, in cleanedText: NSString) -> String {
let replacement = cleaned as NSString
let unit: UInt16
if replacement.length > 0 {
unit = replacement.character(at: replacement.length - 1)
} else if range.location > 0 {
unit = cleanedText.character(at: range.location - 1)
} else {
unit = SqlLexer.space
}
return String(utf16CodeUnits: Array(repeating: unit, count: range.length), count: range.length)
}

private static func replacement(for classified: ClassifiedSpecialCharacter, lineEnding: String) -> String {
let value = classified.scalar.value
if lineBreaks.contains(value) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,22 @@ struct InvisibleCharacterRemoverTests {
#expect(clean(text, grammar: TestGrammar.postgres) == text)
}

/// Measured on PostgreSQL 17: `SELECT<NBSP>$$a b$$` is a syntax error, because every byte over 0x7F continues an
/// identifier there and a `$` glued to one opens no literal. So the text is code, and its blanks are cleaned.
@Test("A dollar sign glued to a non-ASCII blank opens no literal")
func dollarQuoteBehindABlankIsCode() {
#expect(clean("SELECT\u{A0}$$a\u{A0}b$$", grammar: TestGrammar.postgres) == "SELECT $$a b$$")
// PostgreSQL reads the raw NBSP into the name, but the cleaned text that runs opens a dollar quote there.
@Test("A dollar-quoted body after a non-ASCII blank keeps its characters")
func dollarQuoteBehindABlankKeepsItsBody() {
#expect(clean("SELECT\u{A0}$$a\u{A0}b$$", grammar: TestGrammar.postgres) == "SELECT $$a\u{A0}b$$")
#expect(clean("SELECT\u{1680}$$a\u{A0}b$$", grammar: TestGrammar.duckdb) == "SELECT $$a\u{A0}b$$")
}

@Test("A line separator before an escape string ends the name, as the line break it becomes does")
func lineSeparatorBeforeEscapeString() {
#expect(clean("SELECT\u{2028}E'a\\'\u{A0}b'", grammar: TestGrammar.postgres) == "SELECT\nE'a\\'\u{A0}b'")
}

@Test("A removed character joins its neighbours when deciding where an escape string starts")
func removedCharacterBeforeEscapeString() {
#expect(clean("SELECT \u{200B}E'a\\'\u{A0}b'", grammar: TestGrammar.postgres) == "SELECT E'a\\'\u{A0}b'")
#expect(clean("x\u{8}E'a\\'\u{A0}b'", grammar: TestGrammar.postgres) == "xE'a\\' b'")
}

@Test("A MySQL conditional comment runs, so it is cleaned like code")
Expand Down
1 change: 1 addition & 0 deletions apple/changelog.d/4240.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Remove Invisible Characters changed the text in a string that followed a non-breaking or zero-width space.
Loading