From 64bdcd1c795bcb5f472aa5f3167584b8d9a47551 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Fri, 13 Nov 2026 00:08:28 +0700 Subject: [PATCH 1/3] fix(editor): find literals in the cleaned text when removing invisible characters --- .../SQL/InvisibleCharacterRemover.swift | 24 ++++++++++++++++++- .../SQL/InvisibleCharacterRemoverTests.swift | 21 ++++++++++++---- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/apple/TablePro/Core/Utilities/SQL/InvisibleCharacterRemover.swift b/apple/TablePro/Core/Utilities/SQL/InvisibleCharacterRemover.swift index 949a2f53c7..e34247fef2 100644 --- a/apple/TablePro/Core/Utilities/SQL/InvisibleCharacterRemover.swift +++ b/apple/TablePro/Core/Utilities/SQL/InvisibleCharacterRemover.swift @@ -21,8 +21,11 @@ internal enum InvisibleCharacterRemover { let end = min(NSMaxRange(scope), text.length) var replacements: [TextReplacement] = [] var index = max(scope.location, 0) + // Literals are found in the text as cleaned so far, since that is what runs: PostgreSQL reads `SELECTE` as + // one name, but the cleaned `SELECT E'...'` holds an escape string whose characters stay. + let cleanedText = skippingLiteralsAndComments ? NSMutableString(string: text as String) : nil while index < end { - if skippingLiteralsAndComments, let skipTo = SQLNonCodeSpan.end(at: index, in: text, grammar: grammar) { + if let cleanedText, let skipTo = SQLNonCodeSpan.end(at: index, in: cleanedText, grammar: grammar) { index = skipTo continue } @@ -32,6 +35,10 @@ internal enum InvisibleCharacterRemover { } let cleaned = replacement(for: classified, lineEnding: lineEnding) replacements.append(TextReplacement(range: classified.range, string: cleaned)) + if let cleanedText { + let filler = standIn(for: cleaned, replacing: classified.range, in: cleanedText) + cleanedText.replaceCharacters(in: classified.range, with: filler) + } index = NSMaxRange(classified.range) } return replacements @@ -51,6 +58,21 @@ internal enum InvisibleCharacterRemover { return offset + delta } + // Keeps offsets: the unit the cleaned text has before the next one, the replacement's last or, for a removed + // character, the one already before it. + private static func standIn(for cleaned: String, replacing range: NSRange, in cleanedText: NSString) -> String { + let replacement = cleaned as NSString + let unit: UInt16 + if replacement.length > 0 { + unit = replacement.character(at: replacement.length - 1) + } else if range.location > 0 { + unit = cleanedText.character(at: range.location - 1) + } else { + unit = SqlLexer.space + } + return String(utf16CodeUnits: Array(repeating: unit, count: range.length), count: range.length) + } + private static func replacement(for classified: ClassifiedSpecialCharacter, lineEnding: String) -> String { let value = classified.scalar.value if lineBreaks.contains(value) { diff --git a/apple/TableProTests/Core/Utilities/SQL/InvisibleCharacterRemoverTests.swift b/apple/TableProTests/Core/Utilities/SQL/InvisibleCharacterRemoverTests.swift index a85513aba3..d3f128521f 100644 --- a/apple/TableProTests/Core/Utilities/SQL/InvisibleCharacterRemoverTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/InvisibleCharacterRemoverTests.swift @@ -71,11 +71,22 @@ struct InvisibleCharacterRemoverTests { #expect(clean(text, grammar: TestGrammar.postgres) == text) } - /// Measured on PostgreSQL 17: `SELECT$$a b$$` is a syntax error, because every byte over 0x7F continues an - /// identifier there and a `$` glued to one opens no literal. So the text is code, and its blanks are cleaned. - @Test("A dollar sign glued to a non-ASCII blank opens no literal") - func dollarQuoteBehindABlankIsCode() { - #expect(clean("SELECT\u{A0}$$a\u{A0}b$$", grammar: TestGrammar.postgres) == "SELECT $$a b$$") + // PostgreSQL reads the raw NBSP into the name, but the cleaned text that runs opens a dollar quote there. + @Test("A dollar-quoted body after a non-ASCII blank keeps its characters") + func dollarQuoteBehindABlankKeepsItsBody() { + #expect(clean("SELECT\u{A0}$$a\u{A0}b$$", grammar: TestGrammar.postgres) == "SELECT $$a\u{A0}b$$") + #expect(clean("SELECT\u{1680}$$a\u{A0}b$$", grammar: TestGrammar.duckdb) == "SELECT $$a\u{A0}b$$") + } + + @Test("A line separator before an escape string ends the name, as the line break it becomes does") + func lineSeparatorBeforeEscapeString() { + #expect(clean("SELECT\u{2028}E'a\\'\u{A0}b'", grammar: TestGrammar.postgres) == "SELECT\nE'a\\'\u{A0}b'") + } + + @Test("A removed character joins its neighbours when deciding where an escape string starts") + func removedCharacterBeforeEscapeString() { + #expect(clean("SELECT \u{200B}E'a\\'\u{A0}b'", grammar: TestGrammar.postgres) == "SELECT E'a\\'\u{A0}b'") + #expect(clean("x\u{8}E'a\\'\u{A0}b'", grammar: TestGrammar.postgres) == "xE'a\\' b'") } @Test("A MySQL conditional comment runs, so it is cleaned like code") From fcf0bf75bf010cef99da38d084a12e8f4136cae4 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Fri, 13 Nov 2026 00:08:57 +0700 Subject: [PATCH 2/3] docs(changelog): add the fragment for #4240 --- apple/changelog.d/4240.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 apple/changelog.d/4240.fixed.md diff --git a/apple/changelog.d/4240.fixed.md b/apple/changelog.d/4240.fixed.md new file mode 100644 index 0000000000..c24df66339 --- /dev/null +++ b/apple/changelog.d/4240.fixed.md @@ -0,0 +1 @@ +Remove Invisible Characters changed the text in a PostgreSQL `$$` or `E''` string after a non-breaking space. From f3cb31d9190f8099ee70b1acacd824d91bcba263 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Fri, 13 Nov 2026 00:10:20 +0700 Subject: [PATCH 3/3] docs(changelog): name only the shipped bug in #4240's fragment --- apple/changelog.d/4240.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apple/changelog.d/4240.fixed.md b/apple/changelog.d/4240.fixed.md index c24df66339..d3d6899722 100644 --- a/apple/changelog.d/4240.fixed.md +++ b/apple/changelog.d/4240.fixed.md @@ -1 +1 @@ -Remove Invisible Characters changed the text in a PostgreSQL `$$` or `E''` string after a non-breaking space. +Remove Invisible Characters changed the text in a string that followed a non-breaking or zero-width space.