diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift index 4cff78d896..187eb5dd2b 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift @@ -158,6 +158,12 @@ public struct SQLLexicalGrammar: OptionSet, Hashable, Sendable { /// See ``SQLPsqlScript``. Curated only. public static let psqlScripts = SQLLexicalGrammar(rawValue: 1 << 41) + /// The non-ASCII spaces ``SQLNonCodeSpan/spacesBeforeLexing`` lists end a name, as DuckDB turns them into spaces + /// before it lexes where its pre-lexing pass sees no quote, so `x` + U+200B + `E'...'` is a name and an escape + /// string. PostgreSQL reads every byte from 0x80 up as part of a name. Measured on DuckDB 1.5.4 and 2.0.0 and + /// PostgreSQL 17.11. That pass reads quotes its own way, so DuckDB's profile also doubts this fact. Curated only. + public static let nonASCIISpacesEndNames = SQLLexicalGrammar(rawValue: 1 << 42) + /// Standard SQL: quotes close only on a doubled quote, `--` and flat `/* */` are the only comments. public static let ansi: SQLLexicalGrammar = [] diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift index 819d1c7414..eb5701d9aa 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift @@ -127,10 +127,11 @@ public struct SQLLexicalProfile: Sendable, Hashable { /// quotes a name and a `'...'` after a line break is a string of its own, not part of an `E'...'`. 1.5.4 continues /// the `E'...'` as PostgreSQL does, a registry plugin built before 2.0 links 1.5, and `enable_legacy_parser` brings /// 1.5's parser back on 2.0, so a gate reads both. It has no `BEGIN ... END` body, so it takes PostgreSQL's - /// narrower fact. + /// narrower fact. Both versions turn the non-ASCII spaces into blanks before they lex, outside the quotes their + /// pre-lexing pass sees. static let duckDB: SQLLexicalGrammar = [ .taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix, .backtickQuotes, - .carriageReturnEndsLineComments, .atomicRoutineBodies, + .carriageReturnEndsLineComments, .atomicRoutineBodies, .nonASCIISpacesEndNames, ] /// Oracle 23.26, measured: a backslash is literal, block comments do not nest, `q'[...]'` is a literal, `$` and @@ -295,7 +296,13 @@ public struct SQLLexicalProfile: Sendable, Hashable { "libSQL": sqliteFamily, "Turso": sqliteFamily, "Cloudflare D1": sqliteFamily, - "DuckDB": SQLLexicalProfile(grammar: duckDB, undetermined: [.escapeStringContinuations]), + // DuckDB blanks a non-ASCII space only where its own pre-lexing pass is outside a quote, and that pass reads + // quotes its own way: after `a$$`, `E'\''` or `/* ' */` it keeps the space, a name character again + // (measured by hand on 1.5.4 and 2.0.0; the probe checks the execution reading only). So a gate reads both. + "DuckDB": SQLLexicalProfile( + grammar: duckDB, + undetermined: [.escapeStringContinuations, .nonASCIISpacesEndNames] + ), "Oracle": SQLLexicalProfile(grammar: oracle), "Dameng": SQLLexicalProfile( grammar: dameng, diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift index 452e7e4df0..be5805d4a9 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift @@ -196,6 +196,8 @@ enum SQLLexicalRelevance { if next == SqlLexer.singleQuote { facts.insert(.alternativeQuoting) } case capitalG, smallG: facts.insert(.batchSeparatorLines) + case let space where space >= 0xA0 && SQLNonCodeSpan.spacesBeforeLexing.contains(space): + facts.insert(.nonASCIISpacesEndNames) default: break } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift index 47e597afcc..81ad5eaf37 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLNonCodeSpan.swift @@ -119,8 +119,6 @@ public enum SQLNonCodeSpan { return span.end } - /// Whether `unit` continues a word, so a prefix like `E` or `q` glued to it is part of an identifier rather than - /// the start of a literal. public static func isWordUnit(_ unit: UInt16) -> Bool { if unit < 0x80 { return SqlDollarQuote.isIdentifierPart(unit) @@ -128,6 +126,29 @@ public enum SQLNonCodeSpan { return !SQLSeparatingCharacter.isSeparating(unit) } + /// Whether a literal prefix right after `unit` is part of the name `unit` belongs to, so `x$e'\'` is the name + /// `x$e` and a plain string. PostgreSQL 17.11 continues a name with `$` and every unit from 0x80 up, and DuckDB + /// 1.5.4 and 2.0.0 do too, less the spaces they blank (measured). CockroachDB's scanner reads bytes the same way, and + /// KingbaseES, a PostgreSQL fork, is taken to. Oracle and Dameng document `$` and `#` as name characters, and run + /// one statement per call. + public static func continuesName(beforePrefix unit: UInt16, grammar: SQLLexicalGrammar) -> Bool { + if unit >= 0x80 { + return !endsNameAsBlank(unit, grammar: grammar) + } + return SqlDollarQuote.isIdentifierPart(unit) || unit == SqlDollarQuote.dollar + || (unit == SqlLexer.hash && grammar.contains(.dollarAndHashInIdentifiers)) + } + + /// Whether a `$` right after `unit` is part of the name `unit` belongs to, as in `a$$b`, rather than a dollar + /// quote's opener. On DuckDB `SELECT` + U+200B + `$$...$$` opens one, measured on 2.0.0. + public static func continuesName(beforeDollar unit: UInt16, grammar: SQLLexicalGrammar) -> Bool { + SqlDollarQuote.isIdentifierContinuation(unit) && !endsNameAsBlank(unit, grammar: grammar) + } + + private static func endsNameAsBlank(_ unit: UInt16, grammar: SQLLexicalGrammar) -> Bool { + grammar.contains(.nonASCIISpacesEndNames) && spacesBeforeLexing.contains(unit) + } + // MARK: - Comments private static func commentSpan( @@ -272,8 +293,8 @@ public enum SQLNonCodeSpan { return closedSpan(.quoted, at: index, span, closerLength: 1) } - /// `E'...'`, `b'...'` and `q'[...]'`, which only start where a word could: `xE'` is an identifier followed by a - /// string. + /// `E'...'`, `b'...'` and `q'[...]'`, which only start where a name does not go on: `xE'` and `x$E'` are a name + /// followed by a string. private static func prefixedLiteralSpan( at index: Int, character: UInt16, @@ -281,7 +302,9 @@ public enum SQLNonCodeSpan { length: Int, grammar: SQLLexicalGrammar ) -> Span? { - guard index == 0 || !isWordUnit(text.character(at: index - 1)) else { return nil } + guard index == 0 || !continuesName(beforePrefix: text.character(at: index - 1), grammar: grammar) else { + return nil + } let startsEscapeString = grammar.contains(.escapeStringPrefix) && (character == capitalE || character == smallE) let startsEscapeBytes = grammar.contains(.escapeBytesPrefix) && character == smallB if startsEscapeString || startsEscapeBytes, @@ -327,9 +350,10 @@ public enum SQLNonCodeSpan { private static let verticalTab: UInt16 = 0x0B - // DuckDB 1.5 turns these into spaces before it lexes (measured on 1.5.4). PostgreSQL and CockroachDB do not, and - // the text a join there leaves them does not parse, so they run none of it. - private static let spacesBeforeLexing: Set = Set([0x00A0, 0x202F, 0x205F, 0x2060, 0x3000, 0xFEFF]) + // DuckDB turns these into spaces before it lexes, outside what its pre-lexing pass reads as a quote, and no other + // non-ASCII unit (measured on 1.5.4 and 2.0.0). + // PostgreSQL and CockroachDB do not, and the text a join there leaves them does not parse, so they run none of it. + static let spacesBeforeLexing: Set = Set([0x00A0, 0x202F, 0x205F, 0x2060, 0x3000, 0xFEFF]) .union(0x2000...0x200B) /// The quote that carries on the string closed just before `index`, as PostgreSQL's `quotecontinue` finds it: only @@ -376,7 +400,8 @@ public enum SQLNonCodeSpan { at: index, in: text, bufLen: length, - style: style + style: style, + continuesName: { SQLNonCodeSpan.continuesName(beforeDollar: $0, grammar: grammar) } ) else { return nil diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift index 1a874aa467..efc70b6c5c 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift @@ -23,6 +23,16 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { case list(depth: Int) } + // How a routine's body began outside a block. A statement never holds a block. After a `RETURN` a block opens + // only behind `AS` or `IS`, as the header `RETURN INT AS BEGIN` writes it. A control statement such as `IF` may + // hold blocks, and statement words inside it begin nothing. + private enum RoutineBody { + case unknown + case statement + case returnValue + case controlStatement + } + private enum Header { case opener /// After `CREATE`, `ALTER` or `REPLACE`. `accountWords` is what is left of an unquoted `DEFINER = user@host`. @@ -34,10 +44,23 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { private static let dot = UInt16(UnicodeScalar(".").value) private static let colon = UInt16(UnicodeScalar(":").value) + private static let comma = UInt16(UnicodeScalar(",").value) /// The words a statement inside a body can follow without a `;`, as in `IF x THEN BEGIN ... END; END IF`. private static let statementIntroducers: Set = ["THEN", "ELSE", "DO", "LOOP", "REPEAT"] + /// The words that open a routine body written as one statement, as in MySQL's + /// `CREATE PROCEDURE p() SELECT ... ORDER BY begin DESC`, where `begin` is a column. + private static let singleStatementBodyOpeners: Set = [ + "SELECT", "INSERT", "UPDATE", "DELETE", "REPLACE", "MERGE", + ] + + /// A trigger's event follows these, as in `BEFORE INSERT`, `INSTEAD OF DELETE` and `INSERT OR UPDATE`, so the + /// word after one is part of the header rather than the body. + private static let triggerEventLeaders: Set = ["BEFORE", "AFTER", "OF", "OR", "FOR"] + + private static let controlStatementOpeners: Set = ["IF", "WHILE", "LOOP", "REPEAT"] + private let grammar: SQLLexicalGrammar private let readsBlocksEverywhere: Bool // T-SQL outside a routine: where a block's `BEGIN` can stand, as the statement's first word, after `IF`, `WHILE` @@ -50,6 +73,14 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { private var header = Header.opener private var definesRoutine = false private var openedBlockBody = false + private var routineBody = RoutineBody.unknown + private var previousWord: String? + // A trigger's events stand before its table, as in SQLite's `CREATE TRIGGER tr DELETE ON t`. + private var readsTriggerEvents = false + // A trigger's event list goes on after a word in `triggerEventLeaders`, or after a `,` that follows an event, as + // in `FOR INSERT, UPDATE`. + private var expectsTriggerEvent = false + private var followsTriggerEvent = false private var depth = 0 /// The open `CASE`s that have held no `;`, which only a `CASE` statement can. The next `END` closes one of them, @@ -102,6 +133,11 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeWord(_ word: String) { merge?.observeWord(word) + let previous = previousWord + previousWord = word + let isTriggerEvent = expectsTriggerEvent && Self.singleStatementBodyOpeners.contains(word) + expectsTriggerEvent = Self.triggerEventLeaders.contains(word) + followsTriggerEvent = isTriggerEvent let isMember = followsDot followsName = true followsDot = false @@ -124,13 +160,22 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { observeRuleWord(word) } guard parenDepth == 0, readsBlocksEverywhere || (pastHeader && definesRoutine) else { return } + if !readsBlocksEverywhere, depth == 0 { + observeBodyStart(word, after: previous, isTriggerEvent: isTriggerEvent) + } switch word { case "BEGIN" where readsBlocksEverywhere: pendingBegin = true - pendingBeginMayOpen = opensNestedBlock(atStatementPosition: statementPosition, isFirstWord: isFirstWord) - case "BEGIN" where opensNestedBlock(atStatementPosition: statementPosition, isFirstWord: isFirstWord): - pendingBegin = true - pendingBeginMayOpen = true + pendingBeginMayOpen = opensNestedBlock( + atStatementPosition: statementPosition, + isFirstWord: isFirstWord, + after: previous + ) + case "BEGIN": + if opensNestedBlock(atStatementPosition: statementPosition, isFirstWord: isFirstWord, after: previous) { + pendingBegin = true + pendingBeginMayOpen = true + } case "IF" where readsBlocksEverywhere: followsControlFlow = true if depth == 0 { @@ -155,6 +200,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeSymbol(_ symbol: UInt16) { merge?.observeSymbol(symbol) + previousWord = nil + expectsTriggerEvent = symbol == Self.comma && followsTriggerEvent + followsTriggerEvent = false if ruleAction == .afterDo, symbol != SqlLexer.openParen { ruleAction = .none } @@ -184,6 +232,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeOpaqueToken() { merge?.observeOpaqueToken() + previousWord = nil + expectsTriggerEvent = false + followsTriggerEvent = false if ruleAction == .afterDo { ruleAction = .none } @@ -201,6 +252,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { public mutating func observeSemicolon() -> Bool { pendingBegin = false + previousWord = nil + expectsTriggerEvent = false + followsTriggerEvent = false if pendingEnd { pendingEnd = false closeBlock() @@ -273,6 +327,7 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { } if SqlBlockStructure.isRoutineKind(word, grammar: grammar) { definesRoutine = true + readsTriggerEvents = word == "TRIGGER" header = .name } else if word == "DEFINER" { header = .kind(accountWords: 2) @@ -303,12 +358,36 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { } } - private func opensNestedBlock(atStatementPosition: Bool, isFirstWord: Bool) -> Bool { + private func opensNestedBlock(atStatementPosition: Bool, isFirstWord: Bool, after previous: String?) -> Bool { if readsBlocksEverywhere { return depth > 0 || definesRoutine || isFirstWord || followsControlFlow || openedBlockBody } - guard depth > 0, !grammar.hasSingleBodyForm else { return true } - return atStatementPosition || inHandlerDeclaration + if grammar.hasSingleBodyForm { return true } + switch routineBody { + case .statement: + return false + case .returnValue: + return depth == 0 && (previous == "AS" || previous == "IS") + case .unknown, .controlStatement: + guard depth > 0 else { return true } + return atStatementPosition || inHandlerDeclaration + } + } + + // Where a routine's body is one statement, the word that starts it. + private mutating func observeBodyStart(_ word: String, after previous: String?, isTriggerEvent: Bool) { + if word == "ON" { + readsTriggerEvents = false + } + guard routineBody == .unknown else { return } + if Self.controlStatementOpeners.contains(word) { + routineBody = .controlStatement + } else if word == "RETURN", previous != "ON" { + // DB2's `COMMIT ON RETURN NO` is a header clause. + routineBody = .returnValue + } else if Self.singleStatementBodyOpeners.contains(word), !isTriggerEvent, !readsTriggerEvents { + routineBody = .statement + } } private mutating func observeRuleWord(_ word: String) { @@ -388,6 +467,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { // A trigger body holds no other; `BEGIN ATOMIC` is never a name, so a routine defined inside one nests. private mutating func openBlockBody() { guard !grammar.contains(.triggerBodies) || depth == 0 else { return } + if routineBody == .returnValue { + routineBody = .unknown + } depth += 1 openedBlockBody = true startsBodyStatement = true diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift index de2f991f56..bcc1e93a50 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SqlDollarQuote.swift @@ -41,13 +41,19 @@ public enum SqlDollarQuote { } /// Resolves a `$` at `pos` to a dollar-quote opener, a positional parameter like `$1`, or a non-tag dollar. A `$` - /// glued to a preceding identifier is not an opener. Returns `needsMoreData` when the buffer ends mid-tag; a - /// whole-string caller treats that as `notOpener`. + /// glued to a preceding identifier, as `continuesName` reads the unit before it, is not an opener. Returns + /// `needsMoreData` when the buffer ends mid-tag; a whole-string caller treats that as `notOpener`. /// /// A tag starts with a letter or an underscore and continues with letters, digits and underscores, where a letter /// is any non-ASCII character as well, which is how PostgreSQL accepts `$ü$`. - public static func scanOpener(at pos: Int, in buffer: NSString, bufLen: Int, style: Style) -> Opener { - if pos > 0, isIdentifierContinuation(buffer.character(at: pos - 1)) { + public static func scanOpener( + at pos: Int, + in buffer: NSString, + bufLen: Int, + style: Style, + continuesName: (unichar) -> Bool = SqlDollarQuote.isIdentifierContinuation + ) -> Opener { + if pos > 0, continuesName(buffer.character(at: pos - 1)) { return .notOpener } guard pos + 1 < bufLen else { return .needsMoreData } diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SQLStatementSplittingTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SQLStatementSplittingTests.swift index 3689fd6156..e13ba5a84a 100644 --- a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SQLStatementSplittingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/SQLStatementSplittingTests.swift @@ -31,6 +31,32 @@ struct SQLStatementSplittingTests { ).count == 1) } + // Measured on DuckDB 1.5.4 and 2.0.0, the engine whose plugin reads E'...' through the kit. + @Test("A prefix glued to a name is part of it, and the non-ASCII spaces DuckDB blanks end the name") + func prefixAfterAName() { + let duckDB: SQLLexicalFeatures = [.taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix, .backtickQuotes] + for glue in ["$", "\u{E9}", "\u{2019}", "\u{FF0C}"] { + #expect(SQLStatementSplitting.statements( + in: "SELECT x\(glue)e'\\'; DROP TABLE t; SELECT 'x'", + lexicalFeatures: duckDB + ).count == 3, "\(glue)") + } + for space in ["\u{00A0}", "\u{200B}", "\u{2060}", "\u{3000}", "\u{FEFF}"] { + #expect(SQLStatementSplitting.statements( + in: "SELECT varchar\(space)E'\\''; DROP TABLE t; SELECT 'x'", + lexicalFeatures: duckDB + ).count == 3, "\(space.unicodeScalars.first.map { String($0.value, radix: 16) } ?? "")") + } + #expect(SQLStatementSplitting.statements( + in: "SELECT 1 AS a,\u{200B}$$'$$; DROP TABLE t; SELECT 'x'", + lexicalFeatures: duckDB + ).count == 3) + let oracle: SQLLexicalFeatures = [.alternativeQuoting, .dollarAndHashInIdentifiers] + #expect(SQLStatementSplitting.statements(in: "SELECT x#q'['; SELECT 2 --]'", lexicalFeatures: oracle).count == 2) + #expect(SQLStatementSplitting.statements(in: "SELECT x#q'['; SELECT 2 --]'", lexicalFeatures: .alternativeQuoting) + .count == 1) + } + @Test("SQLite's $name(...) parameter keeps a semicolon, and a $ inside a name starts none") func featuresReadParenthesizedParameters() { #expect(SQLStatementSplitting.statements(in: "SELECT $a(;)", lexicalFeatures: .parenthesizedParameterNames) diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift index d855aeb433..5d492a2bae 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLNonCodeSpanTests.swift @@ -115,6 +115,79 @@ struct SQLNonCodeSpanTests { SQLLexicalReadings.resolve(databaseTypeId: engine, declared: nil, session: nil).execution } + private static func statements(_ sql: String, _ engine: String) -> Int { + SQLStatementScanner.executableStatements(in: sql, grammar: execution(engine)).count + } + + private static func tier(_ sql: String, _ engine: String) -> StatementTier { + let readings = SQLLexicalReadings.resolve(databaseTypeId: engine, declared: nil, session: nil) + return SQLStatementClassifier.classify(sql, databaseTypeId: engine, readings: readings).tier + } + + // Measured on PostgreSQL 17.11: with a domain named after each glued name, the server ran the DROP. + @Test( + "A prefix glued to a name by $ or by any non-ASCII unit is part of the name on PostgreSQL", + arguments: ["$", "\u{E9}", "\u{00A0}", "\u{3000}", "\u{200B}", "\u{2060}", "\u{FEFF}", "\u{2019}", "\u{FF0C}"] + ) + func prefixAfterANameOnPostgreSQL(glue: String) { + let sql = "SELECT x\(glue)e'\\'; DROP TABLE t; --'" + for engine in ["PostgreSQL", "CockroachDB", "Redshift"] { + #expect(Self.statements(sql, engine) == 2, "\(engine)") + #expect(Self.tier(sql, engine) == .destructive, "\(engine)") + } + #expect(Self.statements("SELECT x\(glue)b'\\'; DROP TABLE t; --'", "CockroachDB") == 2) + } + + // Measured on DuckDB 1.5.4 and 2.0.0: these spaces end `varchar` and the DROP ran; every other unit kept it a name. + @Test("DuckDB ends a name at the non-ASCII spaces it blanks before lexing and at no other non-ASCII unit") + func prefixAfterANameOnDuckDB() { + let blanks: [UInt16] = [0x00A0, 0x2000, 0x2005, 0x200A, 0x200B, 0x202F, 0x205F, 0x2060, 0x3000, 0xFEFF] + for unit in blanks { + let space = String(utf16CodeUnits: [unit], count: 1) + let sql = "SELECT varchar\(space)E'\\''; DROP TABLE t; --'" + #expect(Self.statements(sql, "DuckDB") == 2, "U+\(String(unit, radix: 16))") + #expect(Self.tier(sql, "DuckDB") == .destructive, "U+\(String(unit, radix: 16))") + #expect(Self.statements(sql, "PostgreSQL") == 1, "U+\(String(unit, radix: 16))") + } + for glue in ["$", "\u{E9}", "\u{200C}", "\u{2019}", "\u{FF0C}", "\u{2028}"] { + #expect(Self.statements("SELECT x\(glue)e'\\'; DROP TABLE t; --'", "DuckDB") == 2, "\(glue)") + #expect(Self.statements("SELECT varchar\(glue)E'\\''; DROP TABLE t; --'", "DuckDB") == 1, "\(glue)") + } + } + + // Measured on DuckDB 2.0.0: after these spaces `$$` opened a dollar quote and the DROP ran. + @Test("DuckDB opens a dollar quote after a space it blanks, and PostgreSQL keeps the $ in the name") + func dollarQuoteAfterABlankOnDuckDB() { + for space in ["\u{00A0}", "\u{200B}", "\u{3000}"] { + let sql = "SELECT 1 AS a,\(space)$$'$$; DROP TABLE t; SELECT 'x'" + #expect(Self.statements(sql, "DuckDB") == 3) + #expect(Self.tier(sql, "DuckDB") == .destructive) + #expect(Self.statements(sql, "PostgreSQL") == 1) + } + #expect(Self.statements("SELECT 1 AS a,\u{2019}$$'$$; DROP TABLE t; SELECT 'x'", "DuckDB") == 1) + } + + // Measured on DuckDB 1.5.4 and 2.0.0: its pre-lexing pass kept the space in each text, so the `$$` stayed in the + // name and the DROP ran. + @Test("A gate reads DuckDB both ways, since it keeps a non-ASCII space where its pre-lexing pass sees a quote", arguments: [ + "SELECT 1 AS a$$, 2 AS b\u{200B}$$; DROP TABLE t; --$$", + "/* ' */ SELECT 1 AS a\u{200B}$$; DROP TABLE t; --$$", + "SELECT E'\\'' AS c, 1 AS a\u{200B}$$; DROP TABLE t; --$$", + ]) + func duckDBKeepsSomeSpaces(sql: String) { + #expect(Self.tier(sql, "DuckDB") == .destructive) + } + + @Test("A q'...' glued to a name by $ or # is part of the name where both continue one") + func alternativeQuoteAfterAName() { + for engine in ["Oracle", "Dameng"] { + #expect(end("x$q'[it's]'", at: 2, Self.execution(engine)) == nil, "\(engine)") + #expect(end("x#q'[it's]'", at: 2, Self.execution(engine)) == nil, "\(engine)") + #expect(Self.statements("SELECT x$q'['; DROP TABLE t; --]' FROM dual", engine) == 2, "\(engine)") + } + #expect(end("x#e'\\'' y", at: 2, Self.execution("PostgreSQL")) == 7) + } + @Test("A string after a line break continues a PostgreSQL E'...' with its escapes, as quotecontinue reads it") func escapeStringContinuesAfterALineBreak() { let postgreSQL = Self.execution("PostgreSQL") diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLRoutineBodyNameTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLRoutineBodyNameTests.swift index aed32c07b6..2ade004a42 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLRoutineBodyNameTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLRoutineBodyNameTests.swift @@ -130,11 +130,82 @@ struct SQLRoutineBodyNameTests { "CREATE PROCEDURE p() BEGIN DECLARE CONTINUE HANDLER FOR SQLSTATE '23000' BEGIN SELECT 1; END; SELECT 2; END", "CREATE PROCEDURE p() BEGIN inner_block: BEGIN SELECT 1; END inner_block; SELECT 2; END", "CREATE PROCEDURE db.p() outer_block: BEGIN SELECT (CASE WHEN 1 THEN 2 END); SELECT 2; END outer_block", + "CREATE FUNCTION f() RETURNS VARCHAR(10) CHARACTER SET utf8mb4 DETERMINISTIC BEGIN SET @a = 1; RETURN 'x'; END", + "CREATE TRIGGER tr AFTER DELETE ON t FOR EACH ROW BEGIN DELETE FROM log; INSERT INTO log VALUES (1); END", + "CREATE TRIGGER tr BEFORE UPDATE ON t FOR EACH ROW FOLLOWS other BEGIN UPDATE log SET n = 1; SELECT 1; END", ]) func routineHeaders(routine: String) { #expect(sent(routine + ";\nSELECT 3", "MySQL") == [routine, "SELECT 3"]) } + @Test("A trigger's events are its header, so the BEGIN after them still opens its body", arguments: [ + "CREATE TRIGGER tr AFTER INSERT OR UPDATE ON t FOR EACH ROW BEGIN SELECT 1; SELECT 2; END", + "CREATE TRIGGER tr INSTEAD OF DELETE ON v FOR EACH ROW BEGIN SELECT 1; SELECT 2; END", + "CREATE TRIGGER tr ON t FOR INSERT, UPDATE AS BEGIN SELECT 1; SELECT 2; END", + "CREATE TRIGGER tr DELETE ON t BEGIN DELETE FROM log; INSERT INTO log VALUES (1); END", + "CREATE TRIGGER tr ON t AFTER INSERT AS IF UPDATE(c) BEGIN SELECT 1; SELECT 2; END", + ]) + func triggerEventsAreHeader(trigger: String) { + #expect(sent(trigger + ";\nSELECT 3", "Nonesuch") == [trigger, "SELECT 3"]) + } + + @Test("A begin column in a routine whose body is one statement opens no block", arguments: [ + "CREATE PROCEDURE latest_shift() SELECT id FROM shifts ORDER BY begin DESC LIMIT 1", + "CREATE EVENT purge ON SCHEDULE EVERY 1 DAY DO DELETE FROM sessions ORDER BY begin LIMIT 1000", + "CREATE PROCEDURE p() UPDATE shifts SET closed = 1 WHERE begin IS NULL", + "CREATE PROCEDURE p() INSERT INTO log SELECT begin FROM shifts", + "CREATE PROCEDURE p() REPLACE INTO log SELECT begin AS b FROM shifts", + "CREATE DEFINER=root@localhost PROCEDURE p() COMMENT 'x' READS SQL DATA SELECT begin end_at FROM shifts", + "CREATE TRIGGER tr AFTER INSERT ON shifts FOR EACH ROW INSERT INTO log SELECT begin FROM shifts ORDER BY begin DESC", + "CREATE FUNCTION open_since(begin DATETIME) RETURNS BOOL DETERMINISTIC RETURN begin IS NULL", + "CREATE PROCEDURE p() SELECT CASE WHEN closed THEN begin ELSE NULL END AS s FROM shifts", + "CREATE PROCEDURE p() UPDATE (shifts s JOIN x ON x.id = s.id) SET s.closed = 1 WHERE begin IS NULL", + ]) + func singleStatementBodies(routine: String) { + for typeId in ["MySQL", "MariaDB", "Nonesuch"] { + #expect( + sent(routine + ";\nCALL latest_shift();\nDROP TABLE users", typeId) + == [routine, "CALL latest_shift()", "DROP TABLE users"], + "\(typeId)" + ) + } + } + + @Test("A body that is a control statement keeps the blocks in it, and header clauses stay header", arguments: [ + "CREATE PROCEDURE p AS IF @x = 1 DELETE FROM t ELSE BEGIN SELECT 1; SELECT 2; END", + "CREATE PROCEDURE p() LANGUAGE SQL SET OPTION DBGVIEW = *SOURCE BEGIN SELECT 1; SELECT 2; END", + "CREATE PROCEDURE p() LANGUAGE SQL COMMIT ON RETURN NO BEGIN SELECT 1; SELECT 2; END", + "CREATE PROCEDURE p AS SET NOCOUNT ON BEGIN SELECT 1; SELECT 2; END", + ]) + func controlStatementBodies(routine: String) { + #expect(sent(routine + ";\nSELECT 3", "Nonesuch") == [routine, "SELECT 3"]) + } + + @Test("A RETURN body's CASE holds no block, so the statements after the function stay their own") + func returnBodyWithACase() { + let function = "CREATE FUNCTION f(begin int) RETURNS int RETURN CASE WHEN begin > 0 THEN begin ELSE 0 END" + #expect(sent(function + "; DROP TABLE users; END", "Nonesuch") == [function, "DROP TABLE users", "END"]) + } + + @Test("A RETURN that names the type before AS or IS still opens the block after it", arguments: [ + "CREATE FUNCTION f RETURN INT AS BEGIN SELECT 1; RETURN 2; END", + "CREATE OR REPLACE FUNCTION f(a INT) RETURN INT DETERMINISTIC IS BEGIN SELECT 1; RETURN a; END", + "CREATE FUNCTION f RETURN INT AS BEGIN IF a THEN BEGIN SELECT 1; END; END IF; RETURN 1; END", + ]) + func returnTypeHeaders(function: String) { + for typeId in ["MariaDB", "Nonesuch"] { + #expect(sent(function + ";\nSELECT 3", typeId) == [function, "SELECT 3"], "\(typeId)") + } + } + + @Test("A one-statement event body with a begin column no longer hides the DROP after it from the gate") + func singleStatementBodyTiersTheNextStatement() { + let sql = "CREATE EVENT purge ON SCHEDULE EVERY 1 DAY DO DELETE FROM sessions ORDER BY begin LIMIT 1000;\n" + + "DROP TABLE audit_tmp;" + let readings = SQLLexicalReadings.resolve(databaseTypeId: "MySQL", declared: nil, session: nil) + #expect(SQLStatementClassifier.classify(sql, databaseTypeId: "MySQL", readings: readings).tier == .destructive) + } + @Test("A block keyword glued to a name character is part of the name", arguments: [ "CREATE TABLE lookup (id INTEGER, upper$case TEXT)", "CREATE TABLE \u{E9}case (id int)", diff --git a/apple/Plugins/TableProPluginKit/SQLFeatureLexer.swift b/apple/Plugins/TableProPluginKit/SQLFeatureLexer.swift index b6e034b8e4..bfbd14b67f 100644 --- a/apple/Plugins/TableProPluginKit/SQLFeatureLexer.swift +++ b/apple/Plugins/TableProPluginKit/SQLFeatureLexer.swift @@ -249,7 +249,7 @@ struct SQLFeatureLexer { } private func prefixedLiteralEnd(at index: Int, unit: UInt16) -> Int? { - guard index == 0 || !isWordUnit(units[index - 1]) else { return nil } + guard index == 0 || !continuesNameBeforePrefix(units[index - 1]) else { return nil } if features.contains(.escapeStringPrefix), unit == 0x45 || unit == 0x65, unitAt(index + 1) == Self.singleQuote { return quotedEnd(from: index + 2, quote: Self.singleQuote, backslashEscapes: true) } @@ -371,8 +371,25 @@ struct SQLFeatureLexer { } } + // A prefix glued to a name is part of it: `$`, `#` where it continues a name, and every non-ASCII unit but the + // spaces DuckDB blanks before it lexes. DuckDB is the engine whose plugin reads `E'...'` through the kit, and this + // is its reading outside the quotes its pre-lexing pass sees; the app's gate also reads the other way. + private func continuesNameBeforePrefix(_ unit: UInt16) -> Bool { + if unit >= 0x80 { + return !Self.duckDBBlanks.contains(unit) + } + return isASCIIIdentifierPart(unit) || unit == Self.dollar + || (unit == Self.hash && features.contains(.dollarAndHashInIdentifiers)) + } + + private static let duckDBBlanks: Set = Set([0x00A0, 0x202F, 0x205F, 0x2060, 0x3000, 0xFEFF]) + .union(0x2000...0x200B) + private func continuesIdentifierBeforeDollar(_ unit: UInt16) -> Bool { - isASCIIIdentifierPart(unit) || unit == Self.dollar || unit >= 0x80 + if unit >= 0x80, features.contains(.escapeStringPrefix) { + return !Self.duckDBBlanks.contains(unit) + } + return isASCIIIdentifierPart(unit) || unit == Self.dollar || unit >= 0x80 } private func isTagStart(_ unit: UInt16) -> Bool { diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift index 24108fde77..0edc2c3d64 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift @@ -380,7 +380,7 @@ final class SQLFileParser: Sendable { // starts a word. let startsEscapeLiteral = (ctx.grammar.contains(.escapeStringPrefix) && (char == kCapitalE || char == kSmallE)) || (ctx.grammar.contains(.escapeBytesPrefix) && char == kSmallB) - if startsEscapeLiteral, !SQLNonCodeSpan.isWordUnit(ctx.previousUnit) { + if startsEscapeLiteral, !SQLNonCodeSpan.continuesName(beforePrefix: ctx.previousUnit, grammar: ctx.grammar) { if i + 1 < bufLen { if nsBuffer.character(at: i + 1) == kSingleQuote { return .none } } else if !ctx.atEndOfInput { @@ -388,7 +388,7 @@ final class SQLFileParser: Sendable { return .needsMoreData } } - if readsAlternativeQuotes, !SQLNonCodeSpan.isWordUnit(ctx.previousUnit), + if readsAlternativeQuotes, !SQLNonCodeSpan.continuesName(beforePrefix: ctx.previousUnit, grammar: ctx.grammar), let quoteLength = alternativeQuotePrefixLength(nsBuffer, at: i, bufLen: bufLen) { if quoteLength > 0 { return .opensAlternativeQuote(prefixLength: quoteLength) @@ -599,7 +599,7 @@ final class SQLFileParser: Sendable { let startsEscapeBytes = ctx.grammar.contains(.escapeBytesPrefix) && char == kSmallB if (startsEscapeString || startsEscapeBytes) && nextChar == kSingleQuote - && !SQLNonCodeSpan.isWordUnit(ctx.previousUnit) { + && !SQLNonCodeSpan.continuesName(beforePrefix: ctx.previousUnit, grammar: ctx.grammar) { settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) @@ -624,7 +624,14 @@ final class SQLFileParser: Sendable { } if let style = ctx.grammar.dollarQuoteStyle, char == kDollar { - switch SqlDollarQuote.scanOpener(at: i, in: nsBuffer, bufLen: bufLen, style: style) { + let grammar = ctx.grammar + switch SqlDollarQuote.scanOpener( + at: i, + in: nsBuffer, + bufLen: bufLen, + style: style, + continuesName: { SQLNonCodeSpan.continuesName(beforeDollar: $0, grammar: grammar) } + ) { case .opener(let length, let tag): settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( diff --git a/apple/TablePro/Core/Vim/VimEngine+Characters.swift b/apple/TablePro/Core/Vim/VimEngine+Characters.swift new file mode 100644 index 0000000000..186864c896 --- /dev/null +++ b/apple/TablePro/Core/Vim/VimEngine+Characters.swift @@ -0,0 +1,32 @@ +import Foundation + +extension VimEngine { + func characterStart(at offset: Int, in buffer: VimTextBuffer) -> Int { + buffer.characterRange(at: offset).location + } + + // At or past the end of the text this is the end of the text. + func characterEnd(at offset: Int, in buffer: VimTextBuffer) -> Int { + NSMaxRange(buffer.characterRange(at: offset)) + } + + // Stops at the last step that stays within `limit`; an offset already past it comes back to `limit`. + func characterOffset(after offset: Int, count: Int, limit: Int, in buffer: VimTextBuffer) -> Int { + var position = offset + for _ in 0.. position, next <= limit else { break } + position = next + } + return min(position, limit) + } + + func characterOffset(before offset: Int, count: Int, limit: Int, in buffer: VimTextBuffer) -> Int { + var position = offset + for _ in 0.. limit else { break } + position = characterStart(at: position - 1, in: buffer) + } + return max(position, limit) + } +} diff --git a/apple/TablePro/Core/Vim/VimEngine+InsertReplace.swift b/apple/TablePro/Core/Vim/VimEngine+InsertReplace.swift index ea721bd704..2a53ae73f8 100644 --- a/apple/TablePro/Core/Vim/VimEngine+InsertReplace.swift +++ b/apple/TablePro/Core/Vim/VimEngine+InsertReplace.swift @@ -24,7 +24,7 @@ extension VimEngine { let pos = buffer.selectedRange().location let lineRange = buffer.lineRange(forOffset: pos) if pos > lineRange.location { - buffer.setSelectedRange(NSRange(location: pos - 1, length: 0)) + buffer.setSelectedRange(NSRange(location: characterStart(at: pos - 1, in: buffer), length: 0)) } } return true @@ -41,7 +41,7 @@ extension VimEngine { let pos = buffer.selectedRange().location let lineRange = buffer.lineRange(forOffset: pos) if pos > lineRange.location { - buffer.setSelectedRange(NSRange(location: pos - 1, length: 0)) + buffer.setSelectedRange(NSRange(location: characterStart(at: pos - 1, in: buffer), length: 0)) } return true } @@ -64,27 +64,27 @@ extension VimEngine { let contentEnd = lineEnd > lineRange.location && lineEnd <= buffer.length && buffer.character(at: lineEnd - 1) == 0x0A ? lineEnd - 1 : lineEnd - if pos < contentEnd { - let overwritten = NSRange(location: pos, length: 1) - replaceModeEdits.append(VimReplaceModeEdit(offset: pos, original: buffer.string(in: overwritten))) - buffer.replaceCharacters(in: overwritten, with: String(char)) - } else { - replaceModeEdits.append(VimReplaceModeEdit(offset: pos, original: nil)) - buffer.replaceCharacters(in: NSRange(location: pos, length: 0), with: String(char)) - } + let typed = String(char) + let insertedLength = (typed as NSString).length + let overwritten = pos < contentEnd ? buffer.characterRange(at: pos) : NSRange(location: pos, length: 0) + let original = overwritten.length > 0 ? buffer.string(in: overwritten) : nil + replaceModeEdits.append(VimReplaceModeEdit(offset: pos, original: original, insertedLength: insertedLength)) + buffer.replaceCharacters(in: overwritten, with: typed) return true } func backspaceInReplace(in buffer: VimTextBuffer) { let pos = buffer.selectedRange().location guard pos > 0 else { return } - if let last = replaceModeEdits.last, last.offset == pos - 1 { + if let last = replaceModeEdits.last, last.offset + last.insertedLength == pos { replaceModeEdits.removeLast() - buffer.replaceCharacters(in: NSRange(location: pos - 1, length: 1), with: last.original ?? "") + let inserted = NSRange(location: last.offset, length: last.insertedLength) + buffer.replaceCharacters(in: inserted, with: last.original ?? "") + buffer.setSelectedRange(NSRange(location: last.offset, length: 0)) } else { replaceModeEdits.removeAll() + buffer.setSelectedRange(NSRange(location: characterStart(at: pos - 1, in: buffer), length: 0)) } - buffer.setSelectedRange(NSRange(location: pos - 1, length: 0)) } nonisolated static func isUnwritableControl(_ char: Character) -> Bool { @@ -140,7 +140,8 @@ extension VimEngine { let pos = buffer.selectedRange().location let lineStart = lineStartBeforeCaret(in: buffer) guard pos > lineStart else { return pos } - return max(lineStart, buffer.wordBoundary(forward: false, from: pos)) + // The word scan goes by code unit, so its boundary can fall inside a character. + return max(lineStart, characterStart(at: buffer.wordBoundary(forward: false, from: pos), in: buffer)) } func lineStartBeforeCaret(in buffer: VimTextBuffer) -> Int { diff --git a/apple/TablePro/Core/Vim/VimEngine+Motions.swift b/apple/TablePro/Core/Vim/VimEngine+Motions.swift index 8de33a21ff..10e46b619c 100644 --- a/apple/TablePro/Core/Vim/VimEngine+Motions.swift +++ b/apple/TablePro/Core/Vim/VimEngine+Motions.swift @@ -9,7 +9,7 @@ extension VimEngine { func moveLeft(_ count: Int, in buffer: VimTextBuffer) { let pos = buffer.selectedRange().location let lineRange = buffer.lineRange(forOffset: pos) - let newPos = max(lineRange.location, pos - count) + let newPos = characterOffset(before: pos, count: count, limit: lineRange.location, in: buffer) buffer.setSelectedRange(NSRange(location: newPos, length: 0)) goalColumn = nil } @@ -24,8 +24,11 @@ extension VimEngine { } else { contentEnd = lineEnd } - let maxPos = max(lineRange.location, contentEnd - 1) - let newPos = min(maxPos, pos + count) + // With an operator pending, `l` reaches past the last character, so `dl` there deletes it. + let maxPos = pendingOperator != nil || contentEnd == lineRange.location + ? contentEnd + : characterStart(at: contentEnd - 1, in: buffer) + let newPos = characterOffset(after: pos, count: count, limit: maxPos, in: buffer) buffer.setSelectedRange(NSRange(location: newPos, length: 0)) goalColumn = nil } @@ -90,7 +93,7 @@ extension VimEngine { } else { contentEnd = lineEnd } - let finalPos = contentEnd > lineRange.location ? contentEnd - 1 : lineRange.location + let finalPos = contentEnd > lineRange.location ? characterStart(at: contentEnd - 1, in: buffer) : lineRange.location buffer.setSelectedRange(NSRange(location: finalPos, length: 0)) } @@ -151,7 +154,7 @@ extension VimEngine { if pos >= contentEnd && contentEnd > lineRange.location { pos = contentEnd - 1 } - return pos + return characterStart(at: pos, in: buffer) } func wordBackward(_ count: Int, in buffer: VimTextBuffer) { @@ -235,8 +238,7 @@ extension VimEngine { func executeFindChar(_ char: Character, request: VimFindCharRequest, in buffer: VimTextBuffer) -> Bool { lastFindChar = VimLastFindChar(char: char, forward: request.forward, till: request.till) - guard let scalar = char.unicodeScalars.first else { return true } - let target = unichar(scalar.value) + let target = String(char) let count = consumeCount() let pos = buffer.selectedRange().location let lineRange = buffer.lineRange(forOffset: pos) @@ -244,37 +246,44 @@ extension VimEngine { let contentEnd = lineEnd > lineRange.location && lineEnd <= buffer.length && buffer.character(at: lineEnd - 1) == 0x0A ? lineEnd - 1 : lineEnd + let matches: (NSRange) -> Bool = { buffer.string(in: $0) == target } + // An offset below the line start ends the backward scan. + let previous: (Int) -> Int = { offset in + offset > lineRange.location ? self.characterStart(at: offset - 1, in: buffer) : offset - 1 + } var resolved: Int? if request.forward { - let initial = request.till ? pos + 2 : pos + 1 - var scanStart = min(initial, contentEnd) + var scanStart = characterEnd(at: pos, in: buffer) + if request.till { scanStart = characterEnd(at: scanStart, in: buffer) } + scanStart = min(scanStart, contentEnd) for _ in 0..= lineRange.location { - if buffer.character(at: idx) == target { + if matches(buffer.characterRange(at: idx)) { resolved = idx - scanStart = idx - 1 + scanStart = previous(idx) break } - idx -= 1 + idx = previous(idx) } if resolved == nil { break } } @@ -282,7 +291,7 @@ extension VimEngine { guard var finalPos = resolved else { return true } if request.till { - finalPos += request.forward ? -1 : 1 + finalPos = request.forward ? previous(finalPos) : characterEnd(at: finalPos, in: buffer) } executeMotion(in: buffer, inclusive: true) { buffer.setSelectedRange(NSRange(location: finalPos, length: 0)) diff --git a/apple/TablePro/Core/Vim/VimEngine+NormalMode.swift b/apple/TablePro/Core/Vim/VimEngine+NormalMode.swift index 52fd42aac5..1d689ecab8 100644 --- a/apple/TablePro/Core/Vim/VimEngine+NormalMode.swift +++ b/apple/TablePro/Core/Vim/VimEngine+NormalMode.swift @@ -85,7 +85,8 @@ extension VimEngine { switch char { case "h": - moveLeft(consumeCount(), in: buffer) + let count = consumeCount() + executeMotion(in: buffer) { self.moveLeft(count, in: buffer) } return true case "j": moveDown(consumeCount(), in: buffer) @@ -94,7 +95,8 @@ extension VimEngine { moveUp(consumeCount(), in: buffer) return true case "l": - moveRight(consumeCount(), in: buffer) + let count = consumeCount() + executeMotion(in: buffer) { self.moveRight(count, in: buffer) } return true case "w": let count = consumeCount() @@ -189,7 +191,7 @@ extension VimEngine { countPrefix = 0 let pos = buffer.selectedRange().location if pos < buffer.length { - buffer.setSelectedRange(NSRange(location: pos + 1, length: 0)) + buffer.setSelectedRange(NSRange(location: characterEnd(at: pos, in: buffer), length: 0)) } setMode(.insert) return true @@ -236,8 +238,7 @@ extension VimEngine { let pos = buffer.selectedRange().location visualAnchor = pos cursorOffset = pos - let initialLen = pos < buffer.length ? 1 : 0 - buffer.setSelectedRange(NSRange(location: pos, length: initialLen)) + buffer.setSelectedRange(buffer.characterRange(at: pos)) setMode(.visual(linewise: false)) return true case "V": diff --git a/apple/TablePro/Core/Vim/VimEngine+Operators.swift b/apple/TablePro/Core/Vim/VimEngine+Operators.swift index 9c03d4dbbe..582aa26f7a 100644 --- a/apple/TablePro/Core/Vim/VimEngine+Operators.swift +++ b/apple/TablePro/Core/Vim/VimEngine+Operators.swift @@ -20,7 +20,7 @@ extension VimEngine { let rangeStart = min(startPos, endPos) var rangeEnd = max(startPos, endPos) if inclusive && rangeEnd < buffer.length && buffer.character(at: rangeEnd) != 0x0A { - rangeEnd += 1 + rangeEnd = characterEnd(at: rangeEnd, in: buffer) } let range = NSRange(location: rangeStart, length: rangeEnd - rangeStart) @@ -28,8 +28,11 @@ extension VimEngine { pendingOperator = nil } - func executeOperatorOnRange(_ op: VimOperator, range: NSRange, linewise: Bool, in buffer: VimTextBuffer) { - guard range.length > 0 else { return } + func executeOperatorOnRange(_ op: VimOperator, range proposed: NSRange, linewise: Bool, in buffer: VimTextBuffer) { + guard proposed.length > 0 else { return } + // Text objects scan by code unit, so their ranges can end inside a character. + let start = characterStart(at: proposed.location, in: buffer) + let range = NSRange(location: start, length: characterEnd(at: NSMaxRange(proposed) - 1, in: buffer) - start) switch op { case .delete: @@ -37,8 +40,7 @@ extension VimEngine { writeToActiveRegister(text: text, linewise: linewise, asDelete: true) buffer.replaceCharacters(in: range, with: "") adjustMarksForEdit(in: range, replacementLength: 0) - let newPos = min(range.location, max(0, buffer.length - 1)) - buffer.setSelectedRange(NSRange(location: max(0, newPos), length: 0)) + buffer.setSelectedRange(NSRange(location: clampToContentPosition(range.location, in: buffer), length: 0)) case .yank: let text = buffer.string(in: range) writeToActiveRegister(text: text, linewise: linewise, asDelete: false) @@ -193,7 +195,7 @@ extension VimEngine { } } else { if after { - let insertPos = min(pos + 1, buffer.length) + let insertPos = characterEnd(at: pos, in: buffer) buffer.replaceCharacters(in: NSRange(location: insertPos, length: 0), with: source.text) let newPos = insertPos + (source.text as NSString).length - 1 buffer.setSelectedRange(NSRange(location: max(insertPos, newPos), length: 0)) @@ -212,16 +214,16 @@ extension VimEngine { let contentEnd = lineEnd > lineRange.location && lineEnd <= buffer.length && buffer.character(at: lineEnd - 1) == 0x0A ? lineEnd - 1 : lineEnd - let deleteCount = min(count, max(0, contentEnd - pos)) - guard deleteCount > 0 else { return } - let range = NSRange(location: pos, length: deleteCount) + let end = characterOffset(after: pos, count: count, limit: contentEnd, in: buffer) + guard end > pos else { return } + let range = NSRange(location: pos, length: end - pos) writeToActiveRegister(text: buffer.string(in: range), linewise: false, asDelete: true) adjustMarksForEdit(in: range, replacementLength: 0) noteEdit(at: pos, in: buffer) buffer.replaceCharacters(in: range, with: "") - let newContentEnd = contentEnd - deleteCount + let newContentEnd = contentEnd - range.length if pos >= newContentEnd && newContentEnd > lineRange.location { - buffer.setSelectedRange(NSRange(location: newContentEnd - 1, length: 0)) + buffer.setSelectedRange(NSRange(location: characterStart(at: newContentEnd - 1, in: buffer), length: 0)) } else { buffer.setSelectedRange(NSRange(location: pos, length: 0)) } @@ -230,10 +232,9 @@ extension VimEngine { func deleteCharBeforeCursor(_ count: Int, in buffer: VimTextBuffer) { let pos = buffer.selectedRange().location let lineRange = buffer.lineRange(forOffset: pos) - let deleteCount = min(count, pos - lineRange.location) - guard deleteCount > 0 else { return } - let start = pos - deleteCount - let range = NSRange(location: start, length: deleteCount) + let start = characterOffset(before: pos, count: count, limit: lineRange.location, in: buffer) + guard start < pos else { return } + let range = NSRange(location: start, length: pos - start) writeToActiveRegister(text: buffer.string(in: range), linewise: false, asDelete: true) buffer.replaceCharacters(in: range, with: "") buffer.setSelectedRange(NSRange(location: start, length: 0)) @@ -246,9 +247,9 @@ extension VimEngine { let contentEnd = lineEnd > lineRange.location && lineEnd <= buffer.length && buffer.character(at: lineEnd - 1) == 0x0A ? lineEnd - 1 : lineEnd - let deleteCount = min(count, max(0, contentEnd - pos)) - guard deleteCount > 0 else { setMode(.insert); return } - let range = NSRange(location: pos, length: deleteCount) + let end = characterOffset(after: pos, count: count, limit: contentEnd, in: buffer) + guard end > pos else { setMode(.insert); return } + let range = NSRange(location: pos, length: end - pos) writeToActiveRegister(text: buffer.string(in: range), linewise: false, asDelete: true) buffer.replaceCharacters(in: range, with: "") buffer.setSelectedRange(NSRange(location: pos, length: 0)) @@ -262,13 +263,16 @@ extension VimEngine { let contentEnd = lineEnd > lineRange.location && lineEnd <= buffer.length && buffer.character(at: lineEnd - 1) == 0x0A ? lineEnd - 1 : lineEnd - let toggleCount = min(count, max(0, contentEnd - pos)) - guard toggleCount > 0 else { return } - let range = NSRange(location: pos, length: toggleCount) + let end = characterOffset(after: pos, count: count, limit: contentEnd, in: buffer) + guard end > pos else { return } + let range = NSRange(location: pos, length: end - pos) let transformed = toggleCaseTransform(buffer.string(in: range)) buffer.replaceCharacters(in: range, with: transformed) - let newPos = min(pos + toggleCount, contentEnd > lineRange.location ? contentEnd - 1 : lineRange.location) - buffer.setSelectedRange(NSRange(location: newPos, length: 0)) + // A case change can change the length, as ß becomes SS. + let newEnd = pos + (transformed as NSString).length + let newContentEnd = contentEnd + newEnd - end + let lastCharacter = characterStart(at: newContentEnd - 1, in: buffer) + buffer.setSelectedRange(NSRange(location: min(newEnd, lastCharacter), length: 0)) } func applyCaseToLine(_ op: VimOperator, count: Int, in buffer: VimTextBuffer) { @@ -384,16 +388,16 @@ extension VimEngine { let contentEnd = lineEnd > lineRange.location && lineEnd <= buffer.length && buffer.character(at: lineEnd - 1) == 0x0A ? lineEnd - 1 : lineEnd - guard pos + count <= contentEnd else { return true } - let replacement: String - if char == "\r" || char == "\n" { - replacement = String(repeating: "\n", count: count) - } else { - replacement = String(repeating: char, count: count) + var end = pos + for _ in 0.. 0 else { return } lastVisualStart = sel.location - lastVisualEnd = sel.location + sel.length - 1 + lastVisualEnd = characterStart(at: NSMaxRange(sel) - 1, in: buffer) lastVisualLinewise = linewise } @@ -138,8 +138,8 @@ extension VimEngine { buffer.setSelectedRange(NSRange(location: lineStart, length: lineEnd - lineStart)) setMode(.visual(linewise: true)) } else { - let length = clampedEnd - clampedStart + (clampedEnd < buffer.length ? 1 : 0) - buffer.setSelectedRange(NSRange(location: clampedStart, length: length)) + let selectionEnd = clampedEnd < buffer.length ? characterEnd(at: clampedEnd, in: buffer) : clampedEnd + buffer.setSelectedRange(NSRange(location: clampedStart, length: selectionEnd - clampedStart)) setMode(.visual(linewise: false)) } } diff --git a/apple/TablePro/Core/Vim/VimEngine+VisualMode.swift b/apple/TablePro/Core/Vim/VimEngine+VisualMode.swift index 55c7a9aaae..1f3a12c37b 100644 --- a/apple/TablePro/Core/Vim/VimEngine+VisualMode.swift +++ b/apple/TablePro/Core/Vim/VimEngine+VisualMode.swift @@ -53,8 +53,8 @@ extension VimEngine { let cursorPos = visualCursorEnd(buffer: buffer) let newPos: Int switch char { - case "h": newPos = max(0, cursorPos - 1) - case "l": newPos = min(buffer.length, cursorPos + 1) + case "h": newPos = characterOffset(before: cursorPos, count: 1, limit: 0, in: buffer) + case "l": newPos = characterOffset(after: cursorPos, count: 1, limit: buffer.length, in: buffer) case "j": let (line, col) = buffer.lineAndColumn(forOffset: cursorPos) let targetLine = min(buffer.lineCount - 1, line + 1) @@ -205,12 +205,17 @@ extension VimEngine { func visualCursorEnd(buffer: VimTextBuffer) -> Int { let sel = buffer.selectedRange() if sel.location == visualAnchor { - return sel.location + max(sel.length, 1) - 1 + return lastSelectedCharacter(sel, in: buffer) } return sel.location } - func updateVisualSelection(cursorPos: Int, linewise: Bool, in buffer: VimTextBuffer) { + private func lastSelectedCharacter(_ sel: NSRange, in buffer: VimTextBuffer) -> Int { + sel.length > 0 ? characterStart(at: NSMaxRange(sel) - 1, in: buffer) : sel.location + } + + func updateVisualSelection(cursorPos proposedCursor: Int, linewise: Bool, in buffer: VimTextBuffer) { + let cursorPos = characterStart(at: proposedCursor, in: buffer) cursorOffset = cursorPos let start = min(visualAnchor, cursorPos) let end = max(visualAnchor, cursorPos) @@ -222,17 +227,15 @@ extension VimEngine { let lineEnd = endLineRange.location + endLineRange.length buffer.setSelectedRange(NSRange(location: lineStart, length: lineEnd - lineStart)) } else { - let length = end - start + (end < buffer.length ? 1 : 0) - buffer.setSelectedRange(NSRange(location: start, length: length)) + let selectionEnd = end < buffer.length ? characterEnd(at: end, in: buffer) : end + buffer.setSelectedRange(NSRange(location: start, length: selectionEnd - start)) } } func swapVisualAnchorAndCursor(in buffer: VimTextBuffer, linewise: Bool) { let sel = buffer.selectedRange() let cursor = visualCursorEnd(buffer: buffer) - let otherEnd = cursor == sel.location - ? sel.location + max(0, sel.length - 1) - : sel.location + let otherEnd = cursor == sel.location ? lastSelectedCharacter(sel, in: buffer) : sel.location visualAnchor = cursor cursorOffset = otherEnd updateVisualSelection(cursorPos: otherEnd, linewise: linewise, in: buffer) @@ -259,13 +262,14 @@ extension VimEngine { guard sel.length > 0 else { setMode(.normal); return } var replacement = "" replacement.reserveCapacity(sel.length) - for i in 0.. NSRange - /// Returns (0-based line index, 0-based column) for the given offset + // 0-based line and column. func lineAndColumn(forOffset offset: Int) -> (line: Int, column: Int) - /// Returns the offset for a given 0-based line and column func offset(forLine line: Int, column: Int) -> Int - /// Returns the UTF-16 code unit at the given offset — must be O(1) + // One UTF-16 code unit, in O(1). func character(at offset: Int) -> unichar - /// Returns the offset of the next/previous word boundary from the given offset + // An empty range at the clamped offset when `offset` is outside the text. + func characterRange(at offset: Int) -> NSRange + func wordBoundary(forward: Bool, from offset: Int) -> Int - /// Returns the offset of the end of the current word from the given offset func wordEnd(from offset: Int) -> Int - /// Returns the offset of the previous word-end (backward analog of wordEnd) — for ge + // The backward analog of `wordEnd`, for ge. func wordEndBackward(from offset: Int) -> Int - /// Returns the next/previous WORD boundary (whitespace-delimited, punctuation included) + // WORD motions split on whitespace only. func bigWordBoundary(forward: Bool, from offset: Int) -> Int - /// Returns the end of the current WORD (whitespace-delimited) func bigWordEnd(from offset: Int) -> Int - /// Returns the previous WORD-end (whitespace-delimited) func bigWordEndBackward(from offset: Int) -> Int - /// Returns the offset of the matching bracket pair at the given offset, or nil if none. - /// Supported pairs: () [] {}. Handles nested pairs correctly. + // Pairs () [] {}, counting nested pairs. func matchingBracket(at offset: Int) -> Int? - /// Returns the inclusive 0-based line range currently visible in the editor. - /// Test mocks return the whole buffer range. + // Inclusive 0-based lines. Test mocks return the whole buffer. func visibleLineRange() -> (firstLine: Int, lastLine: Int) - /// Returns the indent string (" " for 4-space, "\t" for tab) used by >> and <<. + // The unit >> and << add or strip. func indentString() -> String - /// Returns the indent width in columns. func indentWidth() -> Int - /// Returns the currently selected range func selectedRange() -> NSRange - /// Returns the string in the given range func string(in range: NSRange) -> String - /// Sets the selected range and scrolls to make it visible + // Widens the range to whole characters and scrolls it into view. func setSelectedRange(_ range: NSRange) - /// Replaces characters in the given range with new text func replaceCharacters(in range: NSRange, with string: String) - /// Undo the last change func undo() - /// Redo the last undone change func redo() } diff --git a/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift b/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift index fb47db43f7..53ee9d9c44 100644 --- a/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift +++ b/apple/TablePro/Core/Vim/VimTextBufferAdapter.swift @@ -83,6 +83,13 @@ final class VimTextBufferAdapter: VimTextBuffer { return text.character(at: offset) } + func characterRange(at offset: Int) -> NSRange { + guard let text, offset >= 0, offset < text.length else { + return NSRange(location: min(max(0, offset), length), length: 0) + } + return text.rangeOfComposedCharacterSequence(at: offset) + } + func wordBoundary(forward: Bool, from offset: Int) -> Int { guard let text else { return offset } guard text.length > 0 else { return 0 } @@ -153,13 +160,21 @@ final class VimTextBufferAdapter: VimTextBuffer { let documentLength = length let clampedLocation = max(0, min(range.location, documentLength)) let clampedLength = max(0, min(range.length, documentLength - clampedLocation)) - let clampedRange = NSRange(location: clampedLocation, length: clampedLength) + let wholeRange = wholeCharacters(NSRange(location: clampedLocation, length: clampedLength)) let currentRange = textView.selectedRange() - guard clampedRange != currentRange else { return } + guard wholeRange != currentRange else { return } - textView.selectionManager.setSelectedRange(clampedRange) - textView.scrollToRange(clampedRange) + textView.selectionManager.setSelectedRange(wholeRange) + textView.scrollToRange(wholeRange) + } + + // A caret or selection never splits a composed character sequence. + private func wholeCharacters(_ range: NSRange) -> NSRange { + guard range.length > 0, let text else { + return NSRange(location: characterRange(at: range.location).location, length: 0) + } + return text.rangeOfComposedCharacterSequences(for: range) } func replaceCharacters(in range: NSRange, with string: String) { diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift index 3ab0aa72b6..4bc0a135cf 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift @@ -74,6 +74,39 @@ struct SQLFileParserTests { #expect(stmts[1] == "SELECT 3") } + @Test("Postgres: an e glued to a name by $ or a non-ASCII unit starts a plain string, as the scanner reads it") + func postgresPrefixGluedToAName() async throws { + for glue in ["$", "\u{00A0}", "\u{2019}"] { + let sql = "SELECT x\(glue)e'\\'; DROP TABLE t; SELECT 'x';" + let stmts = try await Self.parse(sql, grammar: TestGrammar.postgres) + #expect(stmts == ["SELECT x\(glue)e'\\'", "DROP TABLE t", "SELECT 'x'"], "\(glue)") + } + } + + @Test("DuckDB: an E after a non-ASCII space DuckDB blanks starts an escape string") + func duckDBPrefixAfterABlank() async throws { + let sql = "SELECT varchar\u{200B}E'\\''; DROP TABLE t; SELECT 'x';" + let stmts = try await Self.parse(sql, grammar: TestGrammar.duckdb) + #expect(stmts == ["SELECT varchar\u{200B}E'\\''", "DROP TABLE t", "SELECT 'x'"]) + let dollar = try await Self.parse("SELECT 1 AS a,\u{200B}$$'$$; DROP TABLE t; SELECT 'x';", grammar: TestGrammar.duckdb) + #expect(dollar == ["SELECT 1 AS a,\u{200B}$$'$$", "DROP TABLE t", "SELECT 'x'"]) + } + + @Test("MySQL: a begin column in a one-statement procedure leaves the next statements their own") + func mysqlBeginColumnInAOneStatementBody() async throws { + let sql = """ + CREATE PROCEDURE latest_shift() SELECT id FROM shifts ORDER BY begin DESC LIMIT 1; + CALL latest_shift(); + UPDATE shifts SET closed = 1 WHERE id = 7; + """ + let stmts = try await Self.parse(sql, grammar: TestGrammar.mysql) + #expect(stmts == [ + "CREATE PROCEDURE latest_shift() SELECT id FROM shifts ORDER BY begin DESC LIMIT 1", + "CALL latest_shift()", + "UPDATE shifts SET closed = 1 WHERE id = 7", + ]) + } + @Test("CockroachDB byte literal: backslash escape active") func cockroach_byte_literal_backslash_escape() async throws { let sql = """ diff --git a/apple/TableProTests/Core/Vim/VimEngineComposedCharacterTests.swift b/apple/TableProTests/Core/Vim/VimEngineComposedCharacterTests.swift new file mode 100644 index 0000000000..c86aa02130 --- /dev/null +++ b/apple/TableProTests/Core/Vim/VimEngineComposedCharacterTests.swift @@ -0,0 +1,579 @@ +import AppKit +import Foundation +@testable import TablePro +import TableProTextEngine +import Testing + +private let thumbsUp = "\u{1F44D}" +private let family = "\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}" +private let accented = "e\u{301}" +private let composedCharacters = [thumbsUp, family, accented] + +private func units(_ text: String) -> Int { + (text as NSString).length +} + +extension GlobalStateSuites { + @Suite(.recordingClipboard) + @MainActor + struct VimEngineComposedCharacterTests { + @MainActor + private final class Session { + let buffer: VimTextBufferMock + let engine: VimEngine + + init(_ text: String, caret: Int) { + buffer = VimTextBufferMock(text: text, selectedRange: NSRange(location: caret, length: 0)) + engine = VimEngine(buffer: buffer) + } + + func press(_ keys: String) { + for key in keys { _ = engine.process(key, shift: false) } + } + + func escape() { _ = engine.process("\u{1B}", shift: false) } + + func backspace() { _ = engine.process("\u{7F}", shift: false) } + + // Insert mode leaves typing to the text view, which puts the text at the caret. + func typeInTextView(_ text: String) { + buffer.replaceCharacters(in: buffer.selectedRange(), with: text) + } + + var text: String { buffer.text } + var caret: Int { buffer.selectedRange().location } + var selection: NSRange { buffer.selectedRange() } + } + + // MARK: - x, X, s + + @Test(arguments: composedCharacters) + func xDeletesTheWholeCharacter(character: String) { + let session = Session("a\(character)b\n", caret: 1) + session.press("x") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + #expect(session.engine.register.text == character) + } + + @Test + func xOnAnEmojiInAStatement() { + let session = Session("INSERT INTO notes VALUES ('ok \(thumbsUp)')\n", caret: 30) + session.press("x") + #expect(session.text == "INSERT INTO notes VALUES ('ok ')\n") + #expect(session.caret == 30) + } + + @Test + func countedXDeletesWholeCharactersOfEachKind() { + let session = Session("a\(thumbsUp)\(family)\(accented)z\n", caret: 1) + session.press("3x") + #expect(session.text == "az\n") + #expect(session.caret == 1) + #expect(session.engine.register.text == thumbsUp + family + accented) + } + + @Test + func xOnTheLastCharacterLandsOnTheStartOfTheNewLastOne() { + let session = Session("a\(thumbsUp)\(thumbsUp)\n", caret: 3) + session.press("x") + #expect(session.text == "a\(thumbsUp)\n") + #expect(session.caret == 1) + } + + @Test(arguments: composedCharacters) + func capitalXDeletesTheWholeCharacterBeforeTheCaret(character: String) { + let session = Session("a\(character)b\n", caret: 1 + units(character)) + session.press("X") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + } + + @Test + func countedCapitalXDeletesWholeCharacters() { + let session = Session("a\(family)\(accented)b\n", caret: 1 + units(family) + units(accented)) + session.press("2X") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + } + + @Test(arguments: composedCharacters) + func sSubstitutesTheWholeCharacter(character: String) { + let session = Session("a\(character)b\n", caret: 1) + session.press("s") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + #expect(session.engine.mode == .insert) + } + + // MARK: - r + + @Test(arguments: composedCharacters) + func rReplacesTheWholeCharacter(character: String) { + let session = Session("a\(character)b\n", caret: 1) + session.press("r-") + #expect(session.text == "a-b\n") + #expect(session.caret == 1) + } + + @Test + func countedRReplacesEmojiByCharacter() { + let session = Session("\(thumbsUp)\(thumbsUp)z\n", caret: 0) + session.press("2r*") + #expect(session.text == "**z\n") + #expect(session.caret == 1) + } + + @Test + func rWithAnEmojiLeavesTheCaretOnTheLastOneWritten() { + let session = Session("abcd\n", caret: 0) + session.press("3r\(thumbsUp)") + #expect(session.text == "\(thumbsUp)\(thumbsUp)\(thumbsUp)d\n") + #expect(session.caret == 4) + } + + @Test + func rCountsCharactersLeftOnTheLine() { + let refused = Session("a\(thumbsUp)\n", caret: 0) + refused.press("3rx") + #expect(refused.text == "a\(thumbsUp)\n", "Two characters remain, so 3r is refused") + #expect(refused.caret == 0) + + let replaced = Session("a\(thumbsUp)\n", caret: 0) + replaced.press("2rx") + #expect(replaced.text == "xx\n") + #expect(replaced.caret == 1) + } + + // MARK: - ~ + + @Test + func tildeTogglesAcrossAnEmoji() { + let session = Session("a\(thumbsUp)b\n", caret: 0) + session.press("3~") + #expect(session.text == "A\(thumbsUp)B\n") + #expect(session.caret == 3) + } + + @Test + func tildeKeepsTheMarkOnItsLetter() { + let session = Session("\(accented)x\n", caret: 0) + session.press("~") + #expect(session.text == "E\u{301}x\n") + #expect(session.caret == 2) + } + + @Test + func tildeLandsAfterACaseChangeThatGrowsTheText() { + let session = Session("\u{DF}a\n", caret: 0) + session.press("~") + #expect(session.text == "SSa\n") + #expect(session.caret == 2) + } + + // MARK: - h, l, $, j, e + + @Test + func hAndLStepOneCharacterAtATime() { + // a 0, thumbs up 1, b 3, family 4, c 12, space 13, accented e 14, x 16 + let session = Session("a\(thumbsUp)b\(family)c \(accented)x\n", caret: 0) + var forward: [Int] = [] + for _ in 0..<8 { + session.press("l") + forward.append(session.caret) + } + #expect(forward == [1, 3, 4, 12, 13, 14, 16, 16]) + var backward: [Int] = [] + for _ in 0..<8 { + session.press("h") + backward.append(session.caret) + } + #expect(backward == [14, 13, 12, 4, 3, 1, 0, 0]) + } + + @Test + func countedLAndHCrossSeveralCharacters() { + let session = Session("a\(thumbsUp)b\(family)c \(accented)x\n", caret: 0) + session.press("3l") + #expect(session.caret == 4) + session.press("$") + session.press("3h") + #expect(session.caret == 12) + } + + @Test(arguments: composedCharacters) + func dollarLandsOnTheStartOfAFinalCharacter(character: String) { + let session = Session("ok \(character)\nnext\n", caret: 0) + session.press("$") + #expect(session.caret == 3) + } + + @Test + func jLandsOnACharacterStart() { + // Column 2 on the second line is the second half of the emoji. + let session = Session("abcdef\nx\(thumbsUp)yz\n", caret: 2) + session.press("j") + #expect(session.caret == 8) + session.press("k") + #expect(session.caret == 2) + } + + @Test + func eLandsOnTheStartOfAnEmojiWord() { + let session = Session("ab \(thumbsUp) cd\n", caret: 0) + session.press("ee") + #expect(session.caret == 3) + } + + // MARK: - Insert and append + + @Test(arguments: composedCharacters) + func aAppendsAfterTheWholeCharacter(character: String) { + let session = Session("x\(character)\n", caret: 1) + session.press("a") + #expect(session.engine.mode == .insert) + #expect(session.caret == 1 + units(character)) + session.typeInTextView("!") + #expect(session.text == "x\(character)!\n") + session.escape() + #expect(session.caret == 1 + units(character)) + } + + @Test(arguments: composedCharacters) + func escapeFromInsertStepsBackOverTheWholeCharacter(character: String) { + let session = Session("ab\(character)\n", caret: 2 + units(character)) + session.press("i") + session.escape() + #expect(session.engine.mode == .normal) + #expect(session.caret == 2) + } + + // MARK: - p + + @Test + func pPastesAfterTheEmojiUnderTheCaret() { + let session = Session("a\(thumbsUp)b\n", caret: 3) + session.press("x") + #expect(session.text == "a\(thumbsUp)\n") + #expect(session.caret == 1) + session.press("p") + #expect(session.text == "a\(thumbsUp)b\n") + #expect(session.caret == 3) + } + + @Test + func pLeavesTheCaretOnTheStartOfAPastedEmoji() { + let session = Session("\(thumbsUp)ab\n", caret: 0) + session.press("x") + session.press("p") + #expect(session.text == "a\(thumbsUp)b\n") + #expect(session.caret == 1) + } + + // MARK: - Operators with inclusive motions + + @Test + func deDeletesThroughAnEmojiWord() { + let session = Session("say \(thumbsUp) now\n", caret: 3) + session.press("de") + #expect(session.text == "say now\n") + #expect(session.caret == 3) + } + + @Test + func deDeletesThroughALetterAndItsMark() { + let session = Session("caf\(accented) ok\n", caret: 0) + session.press("de") + #expect(session.text == " ok\n") + #expect(session.caret == 0) + } + + @Test(arguments: composedCharacters) + func dfDeletesThroughTheTargetCharacter(character: String) { + let session = Session("go \(character) now\n", caret: 0) + session.press("df\(character)") + #expect(session.text == " now\n") + #expect(session.caret == 0) + } + + // MARK: - Operators with h and l + + @Test(arguments: composedCharacters + ["q"]) + func dlDeletesTheCharacterUnderTheCaret(character: String) { + let session = Session("a\(character)b\n", caret: 1) + session.press("dl") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + #expect(session.engine.register.text == character) + } + + @Test(arguments: composedCharacters + ["q"]) + func dlOnTheLastCharacterDeletesItAndStaysOnTheLine(character: String) { + let session = Session("a\(character)\nnext\n", caret: 1) + session.press("dl") + #expect(session.text == "a\nnext\n") + #expect(session.caret == 0) + } + + @Test(arguments: composedCharacters + ["q"]) + func dhDeletesTheCharacterBeforeTheCaret(character: String) { + let session = Session("a\(character)b\n", caret: 1 + units(character)) + session.press("dh") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + } + + @Test + func countedDlStopsAtTheLineEnd() { + let session = Session("a\(thumbsUp)\(family)\nnext\n", caret: 1) + session.press("5dl") + #expect(session.text == "a\nnext\n") + #expect(session.engine.mode == .normal) + } + + @Test + func clChangesTheWholeCharacter() { + let session = Session("a\(family)b\n", caret: 1) + session.press("cl") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + #expect(session.engine.mode == .insert) + } + + @Test + func dDollarLeavesTheCaretOnTheNewLastCharacter() { + let session = Session("ab\(thumbsUp)cd\nnext\n", caret: 4) + session.press("d$") + #expect(session.text == "ab\(thumbsUp)\nnext\n") + #expect(session.caret == 2) + } + + // MARK: - f, F, t, T + + @Test + func fFindsAnEmojiAndRepeats() { + // thumbs up at 3 and 10 + let session = Session("go \(thumbsUp) now \(thumbsUp)!\n", caret: 0) + session.press("f\(thumbsUp)") + #expect(session.caret == 3) + session.press(";") + #expect(session.caret == 10) + session.press(",") + #expect(session.caret == 3) + } + + @Test + func countedFFindsTheSecondEmoji() { + let session = Session("go \(thumbsUp) now \(thumbsUp)!\n", caret: 0) + session.press("2f\(thumbsUp)") + #expect(session.caret == 10) + } + + @Test + func fFindsAFamilyAsOneCharacter() { + let session = Session("a\(thumbsUp) \(family)b\n", caret: 0) + session.press("f\(family)") + #expect(session.caret == 4) + } + + @Test + func fWithAPrecomposedLetterFindsTheDecomposedOne() { + let session = Session("caf\(accented)!\n", caret: 0) + session.press("f\u{E9}") + #expect(session.caret == 3) + } + + @Test + func tStopsOnTheCharacterBeforeAnEmoji() { + let session = Session("go \(thumbsUp) now\n", caret: 0) + session.press("t\(thumbsUp)") + #expect(session.caret == 2) + } + + @Test + func tLandsOnAnEmojiBeforeTheTarget() { + let session = Session("a\(thumbsUp)x\n", caret: 0) + session.press("tx") + #expect(session.caret == 1) + } + + @Test + func capitalFAndTFindAnEmojiBackward() { + // thumbs up 3, space 5, w 8 + let find = Session("go \(thumbsUp) now\n", caret: 8) + find.press("F\(thumbsUp)") + #expect(find.caret == 3) + + let till = Session("go \(thumbsUp) now\n", caret: 8) + till.press("T\(thumbsUp)") + #expect(till.caret == 5) + } + + // MARK: - R + + @Test(arguments: composedCharacters) + func replaceModeOverwritesTheWholeCharacterAndBackspaceRestoresIt(character: String) { + let session = Session("a\(character)b\n", caret: 1) + session.press("R") + session.press("x") + #expect(session.text == "axb\n") + #expect(session.caret == 2) + session.backspace() + #expect(session.text == "a\(character)b\n") + #expect(session.caret == 1) + } + + @Test(arguments: composedCharacters) + func replaceModeTypesACharacterAndBackspaceRestoresTheLetter(character: String) { + let session = Session("abc\n", caret: 1) + session.press("R") + session.press(character) + #expect(session.text == "a\(character)c\n") + #expect(session.caret == 1 + units(character)) + session.backspace() + #expect(session.text == "abc\n") + #expect(session.caret == 1) + } + + @Test + func replaceModeBackspaceRemovesAnAppendedEmoji() { + let session = Session("ab\n", caret: 2) + session.press("R") + session.press(thumbsUp) + #expect(session.text == "ab\(thumbsUp)\n") + #expect(session.caret == 4) + session.backspace() + #expect(session.text == "ab\n") + #expect(session.caret == 2) + } + + @Test + func escapeFromReplaceModeStepsBackOverAnEmoji() { + let session = Session("abc\n", caret: 1) + session.press("R") + session.press(thumbsUp) + session.escape() + #expect(session.text == "a\(thumbsUp)c\n") + #expect(session.caret == 1) + } + + // MARK: - Visual + + @Test(arguments: composedCharacters) + func vSelectsTheWholeCharacter(character: String) { + let session = Session("a\(character)b\n", caret: 1) + session.press("v") + #expect(session.selection == NSRange(location: 1, length: units(character))) + } + + @Test + func visualLExtendsOverAnEmojiAndDeletes() { + let session = Session("a\(thumbsUp)b\n", caret: 0) + session.press("vl") + #expect(session.selection == NSRange(location: 0, length: 3)) + session.press("l") + #expect(session.selection == NSRange(location: 0, length: 4)) + session.press("d") + #expect(session.text == "\n") + #expect(session.engine.mode == .normal) + } + + @Test + func visualHExtendsBackOverAnEmoji() { + let session = Session("a\(thumbsUp)b\n", caret: 3) + session.press("vh") + #expect(session.selection == NSRange(location: 1, length: 3)) + session.press("h") + #expect(session.selection == NSRange(location: 0, length: 4)) + } + + @Test + func visualDeleteOfAnEmoji() { + let session = Session("a\(family)b\n", caret: 1) + session.press("vd") + #expect(session.text == "ab\n") + #expect(session.caret == 1) + } + + @Test + func visualReplaceWritesOneCharacterPerCharacter() { + let session = Session("a\(thumbsUp)b\n", caret: 0) + session.press("vllr-") + #expect(session.text == "---\n") + } + + @Test + func visualSwapKeepsTheEmojiWhole() { + let session = Session("a\(thumbsUp)b\n", caret: 0) + session.press("vlo") + #expect(session.selection == NSRange(location: 0, length: 3)) + session.press("l") + #expect(session.selection == NSRange(location: 1, length: 2)) + } + + // MARK: - The text view adapter + + private final class StubDelegate: TextViewDelegate {} + + private func makeTextView(_ string: String) -> TextView { + let textView = TextView( + string: string, + font: .monospacedSystemFont(ofSize: 12, weight: .regular), + textColor: .labelColor, + lineHeightMultiplier: 1.0, + wrapLines: false, + isEditable: true, + isSelectable: true, + letterSpacing: 1.0, + delegate: StubDelegate() + ) + textView.frame = NSRect(x: 0, y: 0, width: 500, height: 500) + textView.layout() + return textView + } + + @Test + func adapterCharacterRangeIsTheComposedSequence() { + let textView = makeTextView("a\(thumbsUp)b") + let adapter = VimTextBufferAdapter(textView: textView) + withExtendedLifetime(textView) { + #expect(adapter.characterRange(at: 2) == NSRange(location: 1, length: 2)) + #expect(adapter.characterRange(at: 3) == NSRange(location: 3, length: 1)) + #expect(adapter.characterRange(at: 4) == NSRange(location: 4, length: 0)) + #expect(adapter.characterRange(at: -1) == NSRange(location: 0, length: 0)) + } + } + + @Test + func adapterKeepsTheSelectionOnWholeCharacters() { + let textView = makeTextView("a\(thumbsUp)b") + let adapter = VimTextBufferAdapter(textView: textView) + withExtendedLifetime(textView) { + adapter.setSelectedRange(NSRange(location: 2, length: 0)) + #expect(textView.selectedRange() == NSRange(location: 1, length: 0)) + adapter.setSelectedRange(NSRange(location: 2, length: 1)) + #expect(textView.selectedRange() == NSRange(location: 1, length: 2)) + adapter.setSelectedRange(NSRange(location: 0, length: 2)) + #expect(textView.selectedRange() == NSRange(location: 0, length: 3)) + } + } + + @Test + func engineOverTheTextViewStepsAndDeletesWholeEmoji() { + let textView = makeTextView("INSERT INTO notes VALUES ('ok \(thumbsUp)')") + let adapter = VimTextBufferAdapter(textView: textView) + let engine = VimEngine(buffer: adapter) + withExtendedLifetime(textView) { + adapter.setSelectedRange(NSRange(location: 29, length: 0)) + _ = engine.process("l", shift: false) + #expect(adapter.selectedRange().location == 30) + _ = engine.process("l", shift: false) + #expect(adapter.selectedRange().location == 32) + _ = engine.process("h", shift: false) + _ = engine.process("x", shift: false) + #expect(textView.string == "INSERT INTO notes VALUES ('ok ')") + #expect(adapter.selectedRange().location == 30) + } + } + } +} diff --git a/apple/TableProTests/Core/Vim/VimTextBufferMock.swift b/apple/TableProTests/Core/Vim/VimTextBufferMock.swift index 441955690b..7beb32f778 100644 --- a/apple/TableProTests/Core/Vim/VimTextBufferMock.swift +++ b/apple/TableProTests/Core/Vim/VimTextBufferMock.swift @@ -36,9 +36,7 @@ final class VimTextBufferMock: VimTextBuffer { return max(1, count) } - func invalidateLineCache() { - // No-op in mock — lineCount is always computed from current text - } + func invalidateLineCache() {} func lineRange(forOffset offset: Int) -> NSRange { let nsString = text as NSString @@ -97,6 +95,14 @@ final class VimTextBufferMock: VimTextBuffer { return nsString.character(at: offset) } + func characterRange(at offset: Int) -> NSRange { + let nsString = text as NSString + guard offset >= 0 && offset < nsString.length else { + return NSRange(location: min(max(0, offset), nsString.length), length: 0) + } + return nsString.rangeOfComposedCharacterSequence(at: offset) + } + func wordBoundary(forward: Bool, from offset: Int) -> Int { let nsString = text as NSString guard nsString.length > 0 else { return 0 } @@ -285,14 +291,16 @@ final class VimTextBufferMock: VimTextBuffer { let clampedLocation = max(0, min(range.location, nsString.length)) let maxLength = nsString.length - clampedLocation let clampedLength = max(0, min(range.length, maxLength)) - _selectedRange = NSRange(location: clampedLocation, length: clampedLength) + let clamped = NSRange(location: clampedLocation, length: clampedLength) + _selectedRange = clamped.length > 0 + ? nsString.rangeOfComposedCharacterSequences(for: clamped) + : NSRange(location: characterRange(at: clampedLocation).location, length: 0) } func replaceCharacters(in range: NSRange, with string: String) { let mutable = NSMutableString(string: text) mutable.replaceCharacters(in: range, with: string) text = mutable as String - // Update selection to end of inserted text _selectedRange = NSRange(location: range.location + (string as NSString).length, length: 0) } diff --git a/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift b/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift index 3820e2e3d1..e469284b29 100644 --- a/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift +++ b/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift @@ -29,10 +29,10 @@ struct SQLLexicalFeatureMappingTests { .sqlScriptBlocks, .cqlBatches, .hashAndAtInIdentifiers, .angleQuotedIdentifiers, .backslashEscapesInAngleQuotes, .escapeBytesPrefix, .exponentSignsNeedNoDigits, .formFeedEndsLineComments, .atomicRoutineBodies, .triggerBodies, .statementsAsValues, .bracketedStatementSources, .pipeOperatorChains, - .withoutDashComments, .escapeStringContinuations, .databendTokens, .psqlScripts, + .withoutDashComments, .escapeStringContinuations, .databendTokens, .psqlScripts, .nonASCIISpacesEndNames, ] #expect(grammarBits.isDisjoint(with: curatedOnly)) - #expect(grammarBits.union(curatedOnly).rawValue == (1 << 42) - 1) + #expect(grammarBits.union(curatedOnly).rawValue == (1 << 43) - 1) } @Test("SAP HANA's SQLScript blocks stay in the app's curated grammar and never reach the kit") @@ -122,6 +122,9 @@ struct SQLLexicalFeatureMappingTests { "SELECT $$it's$$; DROP TABLE t; SELECT 'x'", "SELECT $ü$it's$ü$; DROP TABLE t; SELECT 'x'", "SELECT E'\\''; DROP TABLE t; --'", + "SELECT x$e'\\'; DROP TABLE t; --'", + "SELECT x$q'['; DROP TABLE t; --]' FROM dual", + "SELECT x#q'['; DROP TABLE t; --]' FROM dual", "SELECT 1 # '\n; DROP TABLE t; -- '", "SELECT 'a\\'; DROP TABLE t; -- '", "SELECT 1 AS [a]]'b]; DROP TABLE t; SELECT 'x'", diff --git a/apple/changelog.d/4236.fixed.1.md b/apple/changelog.d/4236.fixed.1.md new file mode 100644 index 0000000000..f80b087afc --- /dev/null +++ b/apple/changelog.d/4236.fixed.1.md @@ -0,0 +1 @@ +Vim mode commands such as `x`, `r`, `h` and `l` split emoji and accented letters, and `f` with an emoji closed the app. diff --git a/apple/changelog.d/4236.fixed.2.md b/apple/changelog.d/4236.fixed.2.md new file mode 100644 index 0000000000..30a31e7117 --- /dev/null +++ b/apple/changelog.d/4236.fixed.2.md @@ -0,0 +1 @@ +Vim mode `dl`, `dh` and `cl` did nothing and left the operator waiting for the next key. diff --git a/apple/changelog.d/4236.fixed.md b/apple/changelog.d/4236.fixed.md new file mode 100644 index 0000000000..cc7b33b07c --- /dev/null +++ b/apple/changelog.d/4236.fixed.md @@ -0,0 +1 @@ +An unquoted `begin` column in a one-statement MySQL procedure or event swallowed the rest of the script. diff --git a/apple/changelog.d/4236.security.md b/apple/changelog.d/4236.security.md new file mode 100644 index 0000000000..c1385bbfbe --- /dev/null +++ b/apple/changelog.d/4236.security.md @@ -0,0 +1 @@ +Safe Mode missed a statement after a string glued to a name, such as `x$e'\\'` on PostgreSQL. diff --git a/apple/scripts/probes/check-sql-lexical-grammar.sh b/apple/scripts/probes/check-sql-lexical-grammar.sh index 081d63ec2a..0feda2d90f 100755 --- a/apple/scripts/probes/check-sql-lexical-grammar.sh +++ b/apple/scripts/probes/check-sql-lexical-grammar.sh @@ -108,6 +108,8 @@ CASE_NAMES=( "dollars glued to an identifier" "dollars glued to a non-ASCII identifier" "canonically equivalent dollar tag" + "E'' after a zero width space" + "dollar quote after a zero width space" ) CASE_SQL=( "SELECT 'C:\\' AS p; DROP TABLE lexer_canary" @@ -125,6 +127,8 @@ CASE_SQL=( "SELECT 1 AS x\$\$; DROP TABLE lexer_canary; --\$\$" "SELECT 1 AS é\$\$; DROP TABLE lexer_canary; --\$\$" "SELECT \$K\$ x \$K\$ , \$K\$; DROP TABLE lexer_canary; -- \$K\$" # the second tag is U+212A KELVIN SIGN + $'SELECT varchar\xe2\x80\x8bE\'\\\'\'; DROP TABLE lexer_canary; SELECT \'x\'' # DuckDB blanks the U+200B, PostgreSQL keeps it in the name + $'SELECT 1 AS a,\xe2\x80\x8b$$\'$$; DROP TABLE lexer_canary; SELECT \'x\'' ) SQLITE_DB="$WORK/lexer.sqlite"