diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/DerivedTableParser.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/DerivedTableParser.swift index eb0a50ead9..2f6b38273a 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/DerivedTableParser.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/DerivedTableParser.swift @@ -446,6 +446,135 @@ struct DerivedTableParser { return target.reference } + // MARK: - Join scope + + struct JoinSource: Equatable { + let table: String + let schema: String? + let alias: String? + // Whether the name the statement exposes it by, the alias or else the table, was quoted. + let isQuoted: Bool + } + + private struct FromItemNames { + let sources: [JoinSource] + // One stored table named directly, not a join group, a derived table or a table function. + let isTable: Bool + let end: Int + } + + // Walking back from an ON, a clause word means it belongs to no FROM list, as in CREATE INDEX, GRANT or a trigger. + private static let joinScopeBreakers: Set = [ + "SELECT", "WHERE", "GROUP", "HAVING", "ORDER", "LIMIT", "OFFSET", "FETCH", "SET", "VALUES", "RETURNING", "INTO", + "UNION", "INTERSECT", "EXCEPT", "MINUS", "WINDOW", "QUALIFY", "INSERT", "DELETE", "MERGE", "CREATE", "ALTER", + "DROP", "GRANT", "REVOKE" + ] + + // Set only when the ON is the token right before `cursor` and follows a table joined by JOIN. The earlier tables + // are the ones before it since the last comma of its FROM list or join group, which is all such an ON may name. + func joinScope(in stream: SQLTokenStream, cursor: Int) -> (joined: JoinSource, earlier: [JoinSource])? { + guard let on = stream.tokens.lastIndex(where: { $0.range.location < cursor }), stream.isWord(on, "ON"), + stream.kind(on - 1) != .comma, !stream.isWord(on - 1, in: Self.joinWords), + let start = joinListStart(before: on, cursor: cursor, in: stream) else { return nil } + var earlier: [JoinSource] = [] + var isJoined = false + var index = start + while index < on { + let item = fromItemNames(at: index, to: on, in: stream) + let next = nextFromItem(after: item.end, to: on, in: stream).index + guard next < on else { + guard isJoined, item.isTable, let joined = item.sources.first, + firstTopLevel(["ON"], from: item.end, to: on, in: stream) == nil else { return nil } + return (joined, Array(earlier.reversed())) + } + if stream.kind(next - 1) == .comma { + earlier = [] + isJoined = false + } else { + earlier += item.sources + isJoined = stream.isWord(next - 1, "JOIN") || stream.isWord(next - 1, "STRAIGHT_JOIN") + } + index = next + } + return nil + } + + // The first item of the FROM list holding `on` at the cursor's query level, or of the join group around it. + private func joinListStart(before on: Int, cursor: Int, in stream: SQLTokenStream) -> Int? { + var floor = 0 + if let open = stream.enclosingParens(before: cursor).last { + guard isStatementBody(open, in: stream) else { + return opensFromItem(open, in: stream) ? open + 1 : nil + } + floor = open + 1 + } + var index = on - 1 + while index >= floor { + let token = stream.tokens[index] + if token.kind == .closeParen, token.partner >= floor { + index = token.partner - 1 + continue + } + // After a dot a word is a name, as in `a.set`. + if token.kind == .word, stream.kind(index - 1) != .dot { + switch token.upper { + case "FROM" where !stream.isWord(index - 1, "DISTINCT"), "UPDATE": + return index + 1 + case "USING" where stream.kind(index + 1) != .openParen: + return index + 1 + case let word where Self.joinScopeBreakers.contains(word): + return nil + default: + break + } + } + index -= 1 + } + return nil + } + + private func fromItemNames(at start: Int, to end: Int, in stream: SQLTokenStream) -> FromItemNames { + var index = start + while stream.isWord(index, "LATERAL") || stream.isWord(index, "ONLY") { index += 1 } + guard index < end else { return FromItemNames(sources: [], isTable: false, end: end) } + if stream.kind(index) == .openParen { + let close = stream.after(index) - 1 + guard !isQueryBody(index, in: stream) else { + return FromItemNames(sources: [], isTable: false, end: alias(after: close, in: stream)?.end ?? close + 1) + } + return FromItemNames(sources: groupNames(from: index + 1, to: min(close, end), in: stream), isTable: false, end: close + 1) + } + guard let path = path(at: index, to: end, in: stream) else { + return FromItemNames(sources: [], isTable: false, end: index + 1) + } + index = path.end + if stream.kind(index) == .openParen { + let close = stream.after(index) - 1 + return FromItemNames(sources: [], isTable: false, end: alias(after: close, in: stream)?.end ?? close + 1) + } + if stream.kind(index) == .star { index += 1 } + var source = JoinSource(table: path.table, schema: path.schema, alias: nil, isQuoted: path.isQuoted) + if let alias = alias(after: index - 1, in: stream) { + let aliasIndex = stream.isWord(index, "AS") ? index + 1 : index + source = JoinSource( + table: path.table, schema: path.schema, alias: alias.name, isQuoted: stream.kind(aliasIndex) == .quoted + ) + index = alias.end + } + return FromItemNames(sources: [source], isTable: true, end: index) + } + + private func groupNames(from start: Int, to end: Int, in stream: SQLTokenStream) -> [JoinSource] { + var names: [JoinSource] = [] + var index = start + while index < end { + let item = fromItemNames(at: index, to: end, in: stream) + names += item.sources + index = nextFromItem(after: item.end, to: end, in: stream).index + } + return names + } + // MARK: - Definitions // One query level. A CTE of its WITH list is in scope from the next definition on, and only a level that holds the diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionCatalog.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionCatalog.swift index 0cdf7d7599..e2a213f2df 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionCatalog.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionCatalog.swift @@ -14,10 +14,18 @@ public protocol SQLCompletionCatalog: Sendable { func allowedValues(forColumn column: String, in references: [TableReference]) async -> [String] func allColumnsInScope(for references: [TableReference]) async -> [SQLCompletionItem] func allColumnsFromCachedTables() async -> [SQLCompletionItem] + // The table's own keys, none pointing outside its schema. + func foreignKeys(of tableName: String, schema: String?) async -> [SQLCompletionForeignKey] + // For a surface that spells nothing after ranking; nil leaves the name as stored and written. + func writtenText(of identifier: SQLCompletionIdentifier) -> String? } public extension SQLCompletionCatalog { func tableCompletionItems(inSchema schema: String, isQuoted: Bool) async -> [SQLCompletionItem] { await tableCompletionItems(inSchema: schema) } + + func foreignKeys(of tableName: String, schema: String?) async -> [SQLCompletionForeignKey] { [] } + + func writtenText(of identifier: SQLCompletionIdentifier) -> String? { nil } } diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionItem.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionItem.swift index b5892646b0..9df5182b68 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionItem.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionItem.swift @@ -10,6 +10,7 @@ public enum SQLCompletionKind: String, CaseIterable, Sendable { case alias case `operator` case favorite // Saved SQL favorite (keyword expansion) + case joinCondition public var iconName: String { switch self { @@ -22,12 +23,14 @@ public enum SQLCompletionKind: String, CaseIterable, Sendable { case .alias: return "a.circle.fill" case .operator: return "equal.circle.fill" case .favorite: return "star.circle.fill" + case .joinCondition: return "link.circle.fill" } } /// Lower sorts first within the same context. public var basePriority: Int { switch self { + case .joinCondition: return 0 case .favorite: return 50 case .column: return 100 case .table: return 200 @@ -57,6 +60,8 @@ public struct SQLCompletionItem: Identifiable, Hashable, Sendable { public let caseFolding: SQLCompletionCaseFolding // Set on a name the catalog stores, so the connection's identifier rules can spell it. public let identifier: SQLCompletionIdentifier? + // The `a.x = b.y` pairs of a join condition, so each name is spelled like any other. + public let columnPairs: [SQLCompletionColumnPair] public var matchedRanges: [Range] = [] public var fuzzyPenalty: Int = 0 // `o.total` names a table and a column; a column literally named `order.total` is one name. @@ -71,7 +76,8 @@ public struct SQLCompletionItem: Identifiable, Hashable, Sendable { sortPriority: Int? = nil, filterText: String? = nil, caseFolding: SQLCompletionCaseFolding = .fixed, - identifier: SQLCompletionIdentifier? = nil + identifier: SQLCompletionIdentifier? = nil, + columnPairs: [SQLCompletionColumnPair] = [] ) { self.id = UUID() self.label = label @@ -83,6 +89,7 @@ public struct SQLCompletionItem: Identifiable, Hashable, Sendable { self.filterText = filterText ?? label.lowercased() self.caseFolding = caseFolding self.identifier = identifier + self.columnPairs = columnPairs } private init(recasing item: SQLCompletionItem, label: String, insertText: String) { @@ -96,6 +103,7 @@ public struct SQLCompletionItem: Identifiable, Hashable, Sendable { self.filterText = item.filterText self.caseFolding = item.caseFolding self.identifier = item.identifier + self.columnPairs = item.columnPairs self.matchedRanges = item.matchedRanges self.fuzzyPenalty = item.fuzzyPenalty self.isQualifiedReference = item.isQualifiedReference diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift index 4085547445..6c839f856a 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift @@ -258,27 +258,9 @@ public final class SQLCompletionProvider { } case .on: + items += await joinConditionItems(for: context.joinScope) items += await allowedValueItems(for: context) - items += await columnItems(for: context.tableReferences) - for ref in context.tableReferences { - let qualifier = ref.alias ?? ref.tableName - let cols = await catalog?.columnCompletionItems(for: ref.tableName, schema: ref.schema) ?? [] - for col in cols { - var item = SQLCompletionItem( - label: "\(qualifier).\(col.label)", - kind: .column, - insertText: "\(qualifier).\(col.label)", - detail: col.detail, - documentation: "Column from \(ref.tableName)", - sortPriority: 80, - identifier: col.identifier.map { - SQLCompletionIdentifier([Self.qualifierPart(for: ref)] + $0.parts) - } - ) - item.isQualifiedReference = true - items.append(item) - } - } + items += await qualifiedColumnItems(for: context.tableReferences) items += SQLKeywords.operatorItems() items += dialectOperatorItems() items += filterKeywords([ @@ -578,6 +560,85 @@ public final class SQLCompletionProvider { return await catalog?.allColumnsInScope(for: resolved) ?? [] } + // A catalog that already qualifies a column once two tables are in scope keeps its own item, so the name is + // offered once and with the catalog's spelling. + private func qualifiedColumnItems(for references: [TableReference]) async -> [SQLCompletionItem] { + var items = await columnItems(for: references) + var indexByLabel: [String: Int] = [:] + for (index, item) in items.enumerated() where item.kind == .column && indexByLabel[item.label] == nil { + indexByLabel[item.label] = index + } + for ref in references { + let cols = await catalog?.columnCompletionItems(for: ref.tableName, schema: ref.schema) ?? [] + for col in cols { + let label = "\(ref.identifier).\(col.label)" + if let index = indexByLabel[label] { + items[index].sortPriority = min(items[index].sortPriority, Self.qualifiedColumnPriority) + continue + } + let parts = [Self.qualifierPart(for: ref)] + (col.identifier?.parts ?? [.catalog(col.label)]) + var item = SQLCompletionItem( + label: label, + kind: .column, + insertText: catalog?.writtenText(of: SQLCompletionIdentifier(parts)) ?? "\(ref.identifier).\(col.insertText)", + detail: col.detail, + documentation: "Column from \(ref.tableName)", + sortPriority: Self.qualifiedColumnPriority, + identifier: col.identifier.map { SQLCompletionIdentifier([Self.qualifierPart(for: ref)] + $0.parts) } + ) + item.isQualifiedReference = true + indexByLabel[label] = items.count + items.append(item) + } + } + return items + } + + private static let qualifiedColumnPriority = 80 + + private func joinConditionItems(for scope: SQLJoinScope?) async -> [SQLCompletionItem] { + guard let scope, let catalog else { return [] } + let joined = await joinSide(scope.joined, in: catalog) + var earlier: [SQLJoinConditions.Side] = [] + for reference in scope.earlier { + earlier.append(await joinSide(reference, in: catalog)) + } + var seen = Set() + return SQLJoinConditions.conditions(joined: joined, earlier: earlier).compactMap { condition in + let item = joinConditionItem(condition, joined: scope.joined, catalog: catalog) + return seen.insert(item.label).inserted ? item : nil + } + } + + private func joinSide(_ reference: TableReference, in catalog: any SQLCompletionCatalog) async -> SQLJoinConditions.Side { + SQLJoinConditions.Side( + reference: reference, + keys: await catalog.foreignKeys(of: reference.tableName, schema: reference.schema), + columns: await catalog.columnCompletionItems(for: reference.tableName, schema: reference.schema).map(\.label) + ) + } + + private func joinConditionItem( + _ condition: SQLJoinConditions.Condition, + joined: TableReference, + catalog: any SQLCompletionCatalog + ) -> SQLCompletionItem { + let pairs = condition.columns.map { columns in + SQLCompletionColumnPair( + left: SQLCompletionIdentifier([Self.qualifierPart(for: condition.earlier), .catalog(columns.earlier)]), + right: SQLCompletionIdentifier([Self.qualifierPart(for: joined), .catalog(columns.joined)]) + ) + } + let written = { (name: SQLCompletionIdentifier) in catalog.writtenText(of: name) ?? name.plainText } + return SQLCompletionItem( + label: SQLCompletionColumnPair.condition(pairs.map { (left: $0.left.plainText, right: $0.right.plainText) }), + kind: .joinCondition, + insertText: SQLCompletionColumnPair.condition(pairs.map { (left: written($0.left), right: written($0.right)) }), + detail: condition.constraintName.isEmpty ? nil : condition.constraintName, + columnPairs: pairs + ) + } + // ORDER BY sorts by an output alias before a source column of the same name, so the alias leads. private func outputAliasItems( _ spellings: [String], besides columns: [SQLCompletionItem], insideQuotes: Bool @@ -843,7 +904,7 @@ public final class SQLCompletionProvider { } case .select, .where_, .and, .on, .having, .groupBy, .orderBy, .returning, .using, .window: - if item.kind == .column { + if item.kind == .column || item.kind == .joinCondition { score -= 200 } case .set, .insertColumns: diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift index 1266a5c3fd..e47d27274d 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift @@ -130,6 +130,7 @@ public struct SQLContext: Sendable { public private(set) var quotedName: SQLQuotedName? public let outputAliases: [String] // SELECT-list aliases an ORDER BY at the cursor may name, as written public let targetTable: TableReference? // The table the INSERT, UPDATE or MERGE at the cursor writes + public let joinScope: SQLJoinScope? // Right after the ON of a join public init( clauseType: SQLClauseType, @@ -148,7 +149,8 @@ public struct SQLContext: Sendable { comparisonColumn: String? = nil, quotedName: SQLQuotedName? = nil, outputAliases: [String] = [], - targetTable: TableReference? = nil + targetTable: TableReference? = nil, + joinScope: SQLJoinScope? = nil ) { self.clauseType = clauseType self.prefix = prefix @@ -167,6 +169,7 @@ public struct SQLContext: Sendable { self.quotedName = quotedName self.outputAliases = outputAliases self.targetTable = targetTable + self.joinScope = joinScope } public func replacingTableReferences(_ references: [TableReference]) -> SQLContext { @@ -390,6 +393,11 @@ public final class SQLContextAnalyzer { writtenTable($0, among: tableReferences) } : nil + let joinScope = resolution.clause == .on && dotPrefix == nil && quotedName == nil && !isCastTarget + ? Self.derivedTableParser.joinScope(in: tokens, cursor: clauseEnd).flatMap { + SQLJoinScope($0, references: tableReferences, cteNames: cteNames) + } + : nil return SQLContext( clauseType: isCastTarget ? .castTarget : resolution.clause, @@ -408,7 +416,8 @@ public final class SQLContextAnalyzer { comparisonColumn: comparisonColumn, quotedName: quotedName, outputAliases: outputAliases, - targetTable: targetTable + targetTable: targetTable, + joinScope: joinScope ) } diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift index 172e2763a5..2e394ac414 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift @@ -53,6 +53,31 @@ public struct SQLCompletionIdentifier: Sendable, Hashable { public static func catalog(_ name: String) -> SQLCompletionIdentifier { SQLCompletionIdentifier([.catalog(name)]) } + + // Each part as stored or as written, with no quotes. + public var plainText: String { + parts.map { part in + switch part { + case .catalog(let name): return name + case .statement(let text, _): return text + } + }.joined(separator: ".") + } +} + +public struct SQLCompletionColumnPair: Sendable, Hashable { + public let left: SQLCompletionIdentifier + public let right: SQLCompletionIdentifier + + public init(left: SQLCompletionIdentifier, right: SQLCompletionIdentifier) { + self.left = left + self.right = right + } + + // One equality per pair, in key order. + public static func condition(_ sides: [(left: String, right: String)]) -> String { + sides.map { "\($0.left) = \($0.right)" }.joined(separator: " AND ") + } } // Whether the token being completed already opened a quoted name, and whether that quote is closed. @@ -127,6 +152,17 @@ public struct SQLIdentifierSpelling: Sendable { typedPrefix: String, quoting: SQLIdentifierTokenQuoting ) -> SQLCompletionItem { + if !item.columnPairs.isEmpty { + // The typed text is a qualifier, which says nothing about how the columns are cased. + guard quoting == .none else { return item } + let sides = item.columnPairs.map { pair in + (left: spelled(pair.left.parts, typedPrefix: ""), right: spelled(pair.right.parts, typedPrefix: "")) + } + return item.recased( + label: SQLCompletionColumnPair.condition(sides.map { (left: $0.left.label, right: $0.right.label) }), + insertText: SQLCompletionColumnPair.condition(sides.map { (left: $0.left.text, right: $0.right.text) }) + ) + } guard let identifier = item.identifier else { return item } if quoting != .none, identifier.parts.count == 1, case .catalog(let name) = identifier.parts[0] { // The user's own quote asks for the exact name: inside a closed pair it goes in without the pair. @@ -136,9 +172,14 @@ public struct SQLIdentifierSpelling: Sendable { // A qualified reference replaces the user's quotes, so it is spelled as if none were typed. guard quoting != .enclosed || item.isQualifiedReference else { return item } + let name = spelled(identifier.parts, typedPrefix: typedPrefix) + return item.recased(label: name.label, insertText: name.text) + } + + private func spelled(_ parts: [SQLCompletionIdentifier.Part], typedPrefix: String) -> (label: String, text: String) { var labels: [String] = [] var texts: [String] = [] - for part in identifier.parts { + for part in parts { switch part { case .catalog(let name): switch form(of: name, typedPrefix: typedPrefix) { @@ -154,7 +195,7 @@ public struct SQLIdentifierSpelling: Sendable { texts.append(isQuoted ? quoted(text) : text) } } - return item.recased(label: labels.joined(separator: "."), insertText: texts.joined(separator: ".")) + return (labels.joined(separator: "."), texts.joined(separator: ".")) } private func canStayBare(_ name: String) -> Bool { diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLJoinConditions.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLJoinConditions.swift new file mode 100644 index 0000000000..47a49e09af --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLJoinConditions.swift @@ -0,0 +1,169 @@ +import Foundation + +// One foreign key, its column pairs in key order. +public struct SQLCompletionForeignKey: Sendable, Hashable { + public let name: String + public let columns: [String] + public let referencedTable: String + public let referencedSchema: String? + public let referencedColumns: [String] + + public init( + name: String, + columns: [String], + referencedTable: String, + referencedSchema: String? = nil, + referencedColumns: [String] + ) { + self.name = name + self.columns = columns + self.referencedTable = referencedTable + self.referencedSchema = referencedSchema + self.referencedColumns = referencedColumns + } + + // A driver reports one row per column pair. + public struct Row: Sendable { + public let name: String + public let column: String + public let referencedTable: String + public let referencedSchema: String? + public let referencedColumn: String + + public init(name: String, column: String, referencedTable: String, referencedSchema: String?, referencedColumn: String) { + self.name = name + self.column = column + self.referencedTable = referencedTable + self.referencedSchema = referencedSchema + self.referencedColumn = referencedColumn + } + } + + // A composite key's rows share its name and target. Keys keep the order the driver read them in. + public static func grouping(_ rows: [Row]) -> [SQLCompletionForeignKey] { + struct Identity: Hashable { + let name: String + let table: String + let schema: String? + } + var order: [Identity] = [] + var pairs: [Identity: [(column: String, referenced: String)]] = [:] + for row in rows { + let identity = Identity(name: row.name, table: row.referencedTable, schema: row.referencedSchema) + if pairs[identity] == nil { order.append(identity) } + let pair = (column: row.column, referenced: row.referencedColumn) + guard pairs[identity]?.contains(where: { $0 == pair }) != true else { continue } + pairs[identity, default: []].append(pair) + } + return order.map { identity in + let keyPairs = pairs[identity] ?? [] + return SQLCompletionForeignKey( + name: identity.name, + columns: keyPairs.map { $0.column }, + referencedTable: identity.table, + referencedSchema: identity.schema, + referencedColumns: keyPairs.map { $0.referenced } + ) + } + } +} + +// The table whose ON the cursor follows, and the tables that ON may name, nearest first. +public struct SQLJoinScope: Sendable, Equatable { + public let joined: TableReference + public let earlier: [TableReference] + + public init(joined: TableReference, earlier: [TableReference]) { + self.joined = joined + self.earlier = earlier + } + + // Only a stored table has foreign keys, so a CTE or a derived table on either side is left out. + init?( + _ found: (joined: DerivedTableParser.JoinSource, earlier: [DerivedTableParser.JoinSource]), + references: [TableReference], + cteNames: [String] + ) { + func stored(_ source: DerivedTableParser.JoinSource) -> TableReference? { + if source.schema == nil, cteNames.contains(where: { $0.caseInsensitiveCompare(source.table) == .orderedSame }) { + return nil + } + let reference = references.first { + $0.tableName == source.table && $0.alias == source.alias && $0.schema == source.schema + } ?? TableReference( + tableName: source.table, alias: source.alias, schema: source.schema, isIdentifierQuoted: source.isQuoted + ) + return reference.isDerived ? nil : reference + } + guard let joined = stored(found.joined) else { return nil } + let earlier = found.earlier.compactMap(stored) + guard !earlier.isEmpty else { return nil } + self.init(joined: joined, earlier: earlier) + } +} + +enum SQLJoinConditions { + struct Side { + let reference: TableReference + let keys: [SQLCompletionForeignKey] + let columns: [String] + } + + struct ColumnPair: Equatable { + let earlier: String + let joined: String + } + + struct Condition: Equatable { + let earlier: TableReference + let constraintName: String + let columns: [ColumnPair] + } + + // Per earlier table, nearest first: its keys to the joined table, then the joined table's keys to it. The earlier + // table's column goes first either way, and a self-join yields both orientations. + static func conditions(joined: Side, earlier: [Side]) -> [Condition] { + earlier.flatMap { side -> [Condition] in + let outgoing = side.keys.filter { targets($0, from: side.reference, joined.reference) }.map { key in + Condition(earlier: side.reference, constraintName: key.name, columns: zip(key.columns, key.referencedColumns).map { + ColumnPair(earlier: spelling($0.0, in: side.columns), joined: spelling($0.1, in: joined.columns)) + }) + } + let incoming = joined.keys.filter { targets($0, from: joined.reference, side.reference) }.map { key in + Condition(earlier: side.reference, constraintName: key.name, columns: zip(key.columns, key.referencedColumns).map { + ColumnPair(earlier: spelling($0.1, in: side.columns), joined: spelling($0.0, in: joined.columns)) + }) + } + return outgoing + incoming + } + } + + // A key with no target schema points into its own table's namespace. The catalog keeps only keys that stay in the + // namespace they were read in, so with the owner unqualified a key reaches the unqualified tables, and with the + // owner qualified it reaches only tables qualified the same way. + private static func targets( + _ key: SQLCompletionForeignKey, + from owner: TableReference, + _ reference: TableReference + ) -> Bool { + guard key.referencedTable.caseInsensitiveCompare(reference.tableName) == .orderedSame, + !key.columns.isEmpty, key.columns.count == key.referencedColumns.count else { return false } + switch (key.referencedSchema ?? owner.schema, reference.schema) { + case (nil, nil): + return true + case let (keySchema?, schema?): + return keySchema.caseInsensitiveCompare(schema) == .orderedSame + case (.some, nil): + return owner.schema == nil + case (nil, .some): + return false + } + } + + // SQLite reports a key's columns as its REFERENCES clause spells them, which may differ in case from the table's. + private static func spelling(_ column: String, in columns: [String]) -> String { + guard !columns.contains(column) else { return column } + let matches = columns.filter { $0.caseInsensitiveCompare(column) == .orderedSame } + return matches.count == 1 ? matches[0] : column + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQuery/QueryScript.swift b/apple/Packages/TableProCore/Sources/TableProQuery/QueryScript.swift index 7c760fe1e3..94c3c23d55 100644 --- a/apple/Packages/TableProCore/Sources/TableProQuery/QueryScript.swift +++ b/apple/Packages/TableProCore/Sources/TableProQuery/QueryScript.swift @@ -33,13 +33,13 @@ public struct QueryScript: Sendable, Equatable { /// How the session read literals when the text was split. public let session: SQLSessionLexicalFacts? - /// The text that was split, where each unit ends in it, and its engine, so the rest can be split again. + /// The text that was split, the statement each unit runs, and its engine, so the rest can be split again. private let source: String? - private let unitEnds: [Int] + private let statements: [SQLStatementScanner.ExecutableStatement] private let databaseType: DatabaseType? public init(units: [Unit], unitKind: UnitKind) { - self.init(units: units, unitKind: unitKind, session: nil, source: nil, unitEnds: [], databaseType: nil) + self.init(units: units, unitKind: unitKind, session: nil, source: nil, statements: [], databaseType: nil) } private init( @@ -47,14 +47,14 @@ public struct QueryScript: Sendable, Equatable { unitKind: UnitKind, session: SQLSessionLexicalFacts?, source: String?, - unitEnds: [Int], + statements: [SQLStatementScanner.ExecutableStatement], databaseType: DatabaseType? ) { self.units = units self.unitKind = unitKind self.session = session self.source = source - self.unitEnds = unitEnds + self.statements = statements self.databaseType = databaseType } @@ -93,7 +93,7 @@ public struct QueryScript: Sendable, Equatable { unitKind: .statement, session: session, source: text, - unitEnds: statements.map { $0.range.location + $0.range.length }, + statements: statements, databaseType: databaseType ) } @@ -107,7 +107,7 @@ public struct QueryScript: Sendable, Equatable { unitKind: .batch, session: session, source: nil, - unitEnds: [], + statements: [], databaseType: databaseType ) } @@ -136,12 +136,27 @@ public struct QueryScript: Sendable, Equatable { /// The text after unit `index`, split again under `session`, for a statement that changed how the session reads /// literals: after `SET sql_mode = 'NO_BACKSLASH_ESCAPES'` a backslash no longer escapes a quote, so the split /// made before it ran no longer matches what the server will read. Nil when the script cannot be split again. + /// + /// The rest starts where the scan went on after the unit, so a `COPY`'s rows stay rows and a client delimiter + /// stays in force. public func replanned(after index: Int, session: SQLSessionLexicalFacts?) -> QueryScript? { - guard unitKind == .statement, let source, let databaseType, unitEnds.indices.contains(index) else { + guard unitKind == .statement, let source, let databaseType, statements.indices.contains(index) else { return nil } - let rest = (source as NSString).substring(from: unitEnds[index]) - return QueryScript.plan(rest, databaseType: databaseType, sendsBatchesWhole: false, session: session) + let rest = SQLStatementScanner.executableStatements( + in: source, + after: statements[index], + through: (source as NSString).length, + grammar: Self.executionGrammar(for: databaseType, session: session) + ) + return QueryScript( + units: rest.map { Unit(sql: $0.sql) }, + unitKind: .statement, + session: session, + source: source, + statements: rest, + databaseType: databaseType + ) } } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLDelimiterDirective.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLDelimiterDirective.swift new file mode 100644 index 0000000000..6cae802465 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLDelimiterDirective.swift @@ -0,0 +1,153 @@ +import Foundation + +/// A `DELIMITER //` line, which changes the text that ends a statement the way the mysql client reads a script. +/// +/// The client reads it as the first word of a line, in any case, followed by a space, a tab or the line's end, and only +/// where no statement text is pending. The line goes to no server. The argument is the first word after it, up to a +/// space, or a string quoted with `'`, `"` or a backtick; the rest of the line is ignored. An empty argument, or one +/// holding a backslash, is refused and the delimiter stays as it was. The delimiter then ends a statement wherever it +/// stands in code, inside a word too, matched case for case. +public struct SQLDelimiterDirective: Sendable, Equatable { + /// The line from its `DELIMITER` to the end of the line, the line feed excluded. + public let range: NSRange + + /// The delimiter the line sets, nil when the client refuses the argument and keeps the one before. + public let delimiter: String? + + public init(range: NSRange, delimiter: String?) { + self.range = range + self.delimiter = delimiter + } + + private static let word = Array("delimiter".utf16) + + /// The directive whose `DELIMITER` starts at `offset`, or nil when the line there is not one. `offset` has to be + /// code where no statement text is pending. + public static func line(at offset: Int, in text: NSString, length: Int) -> SQLDelimiterDirective? { + var before = offset - 1 + while before >= 0, isLineBlank(text.character(at: before)) { + before -= 1 + } + guard before < 0 || text.character(at: before) == SqlLexer.newline else { return nil } + return line(startingAt: offset, in: text, length: length) + } + + /// The directive whose `DELIMITER` starts at `offset`, for a reader that already knows only spaces and tabs stand + /// before it on its line. `text` has to hold the whole line unless the line ends the text. + public static func line(startingAt offset: Int, in text: NSString, length: Int) -> SQLDelimiterDirective? { + let wordEnd = offset + word.count + guard spellsWord(in: text, at: offset, length: length) else { return nil } + if wordEnd < length { + let next = text.character(at: wordEnd) + guard isLineBlank(next) || next == SqlLexer.newline || next == SqlLexer.carriageReturn else { return nil } + } + var lineEnd = wordEnd + while lineEnd < length, text.character(at: lineEnd) != SqlLexer.newline { + lineEnd += 1 + } + // The client reads a line without the carriage return of a CRLF break. + var argumentEnd = lineEnd + if argumentEnd > wordEnd, text.character(at: argumentEnd - 1) == SqlLexer.carriageReturn { + argumentEnd -= 1 + } + var argument: [UInt16] = [] + for index in wordEnd.. Bool { + let wordEnd = offset + word.count + return wordEnd < length && spellsWord(in: text, at: offset, length: length) + && isLineBlank(text.character(at: wordEnd)) + } + + // ASCII only: the client compares bytes, so `delımiter` is no command. + private static func spellsWord(in text: NSString, at offset: Int, length: Int) -> Bool { + guard offset + word.count <= length else { return false } + for (index, unit) in word.enumerated() where text.character(at: offset + index) | 0x20 != unit { + return false + } + return true + } + + // As mysql's `get_arg` reads it: a backslash keeps the next unit in an unquoted argument, and a quoted one ends at + // its first matching quote, or at the line's end when it never closes. + private static func delimiter(fromArgument argument: [UInt16]) -> String? { + var index = 0 + while index < argument.count, isArgumentSpace(argument[index]) { + index += 1 + } + guard index < argument.count else { return nil } + var units: [UInt16] = [] + let opener = argument[index] + if opener == SqlLexer.singleQuote || opener == SqlLexer.doubleQuote || opener == SqlLexer.backtick { + index += 1 + while index < argument.count, argument[index] != opener { + units.append(argument[index]) + index += 1 + } + } else { + while index < argument.count, argument[index] != SqlLexer.space { + if argument[index] == SqlLexer.backslash, index + 1 < argument.count { + index += 1 + } + units.append(argument[index]) + index += 1 + } + } + guard !units.isEmpty, !units.contains(SqlLexer.backslash) else { return nil } + return String(utf16CodeUnits: units, count: units.count) + } + + private static func isLineBlank(_ unit: UInt16) -> Bool { + unit == SqlLexer.space || unit == SqlLexer.tab + } + + // `my_isspace`, which skips the blanks before the argument. + private static func isArgumentSpace(_ unit: UInt16) -> Bool { + unit == SqlLexer.space || (unit >= 0x09 && unit <= 0x0D) + } +} + +/// The delimiter a `DELIMITER` line set, which ends a statement wherever it stands in code, matched unit for unit. +public struct SQLClientDelimiter: Sendable, Equatable { + public let units: [UInt16] + + /// Nil for `;`, which the statement tracker reads. + public init?(_ text: String) { + guard text != ";" else { return nil } + units = Array(text.utf16) + } + + public func starts(in text: NSString, at offset: Int, length: Int) -> Bool { + guard offset + units.count <= length else { return false } + for (index, unit) in units.enumerated() where text.character(at: offset + index) != unit { + return false + } + return true + } + + /// `word` as read at `start`, cut where the delimiter starts inside it: `END$$` is the word `END`. + public func cut( + _ word: (text: String, end: Int), + readAt start: Int, + in text: NSString, + length: Int + ) -> (text: String, end: Int) { + let wordStart = word.end - (word.text as NSString).length + var cursor = max(wordStart, start) + 1 + while cursor < word.end, !starts(in: text, at: cursor, length: length) { + cursor += 1 + } + guard cursor < word.end else { return word } + let raw = text.substring(with: NSRange(location: wordStart, length: cursor - wordStart)) + return (raw.utf16.allSatisfy { $0 < 0x80 } ? raw.uppercased() : raw, cursor) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift index db77326018..4cff78d896 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalGrammar.swift @@ -153,6 +153,11 @@ public struct SQLLexicalGrammar: OptionSet, Hashable, Sendable { /// since no other fact here says that a hint's body is tokens or that a stage name keeps a quote. Curated only. public static let databendTokens = SQLLexicalGrammar(rawValue: 1 << 40) + /// A script reads as psql reads it: `COPY ... FROM STDIN` takes the lines after its own as rows, up to a line + /// holding only `\.`, and pg_dump's `\restrict` and `\unrestrict` lines run nothing, as no meta-command runs here. + /// See ``SQLPsqlScript``. Curated only. + public static let psqlScripts = SQLLexicalGrammar(rawValue: 1 << 41) + /// Standard SQL: quotes close only on a doubled quote, `--` and flat `/* */` are the only comments. public static let ansi: SQLLexicalGrammar = [] diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift index 73445d0e30..819d1c7414 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalProfile.swift @@ -70,7 +70,8 @@ public struct SQLLexicalProfile: Sendable, Hashable { /// which would read one statement where an unknown engine may run several. So are the two single body forms, which /// would cut the `BEGIN ... END` body of an engine that has the usual one. So is a statement inside another, which /// would make every `update` or `access` column an unknown engine names a write, and Snowflake's pipe operator, - /// which would cut a statement at the `->>` that is JSON's operator on an unknown engine. + /// which would cut a statement at the `->>` that is JSON's operator on an unknown engine. So are psql's `COPY` rows, + /// which would hide lines an unknown engine runs as statements. public static let everyKnownReading: [SQLLexicalGrammar] = { var seen: Set = [] var result: [SQLLexicalGrammar] = [] @@ -80,7 +81,7 @@ public struct SQLLexicalProfile: Sendable, Hashable { let unitless = reading.subtracting([ .plsqlBlocks, .delimiterDirective, .unterminatedStatements, .terminatedMergeStatements, .sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies, - .statementsAsValues, .bracketedStatementSources, .pipeOperatorChains, + .statementsAsValues, .bracketedStatementSources, .pipeOperatorChains, .psqlScripts, ]) if seen.insert(unitless).inserted { result.append(unitless) @@ -98,7 +99,7 @@ public struct SQLLexicalProfile: Sendable, Hashable { /// are operators. From its grammar: a routine's body is a literal or `BEGIN ATOMIC ... END`. static let postgreSQL: SQLLexicalGrammar = [ .taggedDollarQuotes, .nestedBlockComments, .escapeStringPrefix, .escapeStringContinuations, - .carriageReturnEndsLineComments, .atomicRoutineBodies, + .carriageReturnEndsLineComments, .atomicRoutineBodies, .psqlScripts, ] /// CockroachDB, from its scanner: PostgreSQL's grammar, and a `b'...'` byte literal read by the routine that reads diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift index 9fb7a0e504..452e7e4df0 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLLexicalReadings.swift @@ -138,7 +138,7 @@ enum SQLLexicalRelevance { var facts: SQLLexicalGrammar = [ .plsqlBlocks, .delimiterDirective, .unterminatedStatements, .terminatedMergeStatements, .sqlScriptBlocks, .cqlBatches, .atomicRoutineBodies, .triggerBodies, .statementsAsValues, - .pipeOperatorChains, .databendTokens, + .pipeOperatorChains, .databendTokens, .psqlScripts, ] var blockCommentOpeners = 0 var index = 0 diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLPsqlScript.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLPsqlScript.swift new file mode 100644 index 0000000000..60bbd7c47f --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLPsqlScript.swift @@ -0,0 +1,245 @@ +import Foundation + +/// The lines of a psql script that are not SQL, read the way psql 17 and 18 read them. +/// +/// `COPY ... FROM STDIN` takes its rows from the script itself: the lines after the line its `;` ends on, up to a line +/// holding only `\.`, or to the end of the script. psql skips them even when the `COPY` fails, since a row run as SQL +/// can be any statement. pg_dump writes a script that opens with `\restrict ` and closes with `\unrestrict `, +/// which only limit psql's own meta-commands; nothing here runs one, so both lines run nothing. +public enum SQLPsqlScript { + private static let backslash = SqlLexer.backslash + private static let period = UInt16(UnicodeScalar(".").value) + private static let restrict = Array("restrict".utf16) + private static let unrestrict = Array("unrestrict".utf16) + + /// Everything after the `;` at `semicolon` that belongs to a `COPY ... FROM STDIN`: the rest of the `;`'s line, then + /// each line up to and including the one holding only `\.`, its line break excluded, or up to the end of the text. + /// + /// psql runs code after the `;` on its own line once the rows are in. pg_dump never writes any, and here it never + /// runs, so a line break after it cannot decide whether it does. + public static func copyRows(afterSemicolonAt semicolon: Int, in text: NSString, length: Int) -> NSRange { + guard var lineStart = lineEnd(in: text, from: semicolon + 1, length: length).map({ $0 + 1 }) else { + return NSRange(location: semicolon + 1, length: length - semicolon - 1) + } + while lineStart < length { + if let markerEnd = endOfDataMarker(in: text, at: lineStart, length: length) { + return NSRange(location: semicolon + 1, length: markerEnd - semicolon - 1) + } + guard let next = lineEnd(in: text, from: lineStart, length: length) else { break } + lineStart = next + 1 + } + return NSRange(location: semicolon + 1, length: length - semicolon - 1) + } + + /// The `\restrict` or `\unrestrict` line whose backslash is at `offset`, its line break excluded, when only blanks + /// stand before it on its line. psql reads both names case for case. + public static func restrictLine(at offset: Int, in text: NSString, length: Int) -> NSRange? { + var before = offset - 1 + while before >= 0, isLineBlank(text.character(at: before)) { + before -= 1 + } + guard before < 0 || text.character(at: before) == SqlLexer.newline, + let nameEnd = restrictNameEnd(in: text, at: offset, length: length) + else { + return nil + } + let end = lineEnd(in: text, from: nameEnd, length: length) ?? length + return NSRange(location: offset, length: end - offset) + } + + /// Whether a `\restrict` or `\unrestrict` line starts at `offset`, for a reader that already knows only blanks + /// stand before it on its line. `text` has to hold the unit after the name unless the name ends the text. + public static func startsRestrictLine(in text: NSString, at offset: Int, length: Int) -> Bool { + restrictNameEnd(in: text, at: offset, length: length) != nil + } + + /// How many units a reader needs from a backslash on to answer ``startsRestrictLine(in:at:length:)``. + public static let restrictLineLookahead = 1 + 10 + 1 + + private static func restrictNameEnd(in text: NSString, at offset: Int, length: Int) -> Int? { + guard offset < length, text.character(at: offset) == backslash, + let nameEnd = [restrict, unrestrict].lazy.compactMap({ name in + matches(name, in: text, at: offset + 1, length: length) ? offset + 1 + name.count : nil + }).first + else { + return nil + } + if nameEnd < length { + let next = text.character(at: nameEnd) + guard isLineBlank(next) || next == SqlLexer.newline || next == SqlLexer.carriageReturn else { return nil } + } + return nameEnd + } + + // psql 18 matches `\.` + LF or CRLF at a line's start only; a last line `\.` with no line break ends the rows too. + // The unit after the returned end, when there is one, was read to decide. + private static func endOfDataMarker(in text: NSString, at lineStart: Int, length: Int) -> Int? { + guard lineStart + 1 < length, + text.character(at: lineStart) == backslash, + text.character(at: lineStart + 1) == period + else { + return nil + } + let after = lineStart + 2 + guard after < length else { return after } + switch text.character(at: after) { + case SqlLexer.newline: + return after + case SqlLexer.carriageReturn where after + 1 == length || text.character(at: after + 1) == SqlLexer.newline: + return after + default: + return nil + } + } + + private static func lineEnd(in text: NSString, from offset: Int, length: Int) -> Int? { + guard offset < length else { return nil } + let found = text.range( + of: "\n", + options: .literal, + range: NSRange(location: offset, length: length - offset) + ) + return found.location == NSNotFound ? nil : found.location + } + + private static func matches(_ name: [UInt16], in text: NSString, at offset: Int, length: Int) -> Bool { + guard offset + name.count <= length else { return false } + for (index, unit) in name.enumerated() where text.character(at: offset + index) != unit { + return false + } + return true + } + + private static func isLineBlank(_ unit: UInt16) -> Bool { + unit == SqlLexer.space || unit == SqlLexer.tab + } +} + +/// psql's test for a statement that takes rows from the script: its first word is `COPY`, and among its first eight +/// words outside parentheses a `FROM` comes right before `STDIN` or `STDOUT`, which the server's grammar does not tell +/// apart. A quoted name is no word, and a dotted name is one word per part. +struct SQLCopyFromStandardInput { + private static let wordsRead = 8 + + private var words = 0 + private var parenDepth = 0 + private var followsFrom = false + private var settled = false + private(set) var readsRows = false + + var needsWords: Bool { + !settled + } + + mutating func observeWord(_ word: String) { + guard !settled, parenDepth == 0 else { return } + if words == 0, word != "COPY" { + settled = true + return + } + if followsFrom, word == "STDIN" || word == "STDOUT" { + readsRows = true + settled = true + return + } + followsFrom = word == "FROM" + words += 1 + if words == Self.wordsRead { + settled = true + } + } + + mutating func observeSymbol(_ symbol: UInt16) { + guard !settled else { return } + switch symbol { + case SqlLexer.openParen: + parenDepth += 1 + case SqlLexer.closeParen: + parenDepth = max(0, parenDepth - 1) + default: + break + } + if words == 0 { + settled = true + } + } + + mutating func observeOpaqueToken() { + if words == 0 { + settled = true + } + } +} + +/// A statement tracker that also tells a `COPY ... FROM STDIN`, for a grammar with ``SQLLexicalGrammar/psqlScripts``. +struct SQLPsqlStatementTracker: SQLStatementBoundaryTracking { + private var base: any SQLStatementBoundaryTracking + private var copy = SQLCopyFromStandardInput() + + init(base: any SQLStatementBoundaryTracking) { + self.base = base + } + + var needsWords: Bool { + base.needsWords || copy.needsWords + } + + var terminator: SQLStatementTerminator { + base.terminator + } + + var acceptsBindParameters: Bool { + base.acceptsBindParameters + } + + var holdsOpenBlock: Bool { + base.holdsOpenBlock + } + + var awaitsContinuation: Bool { + base.awaitsContinuation + } + + var copiesFromStandardInput: Bool { + copy.readsRows + } + + mutating func observeWord(_ word: String) { + copy.observeWord(word) + if base.needsWords { + base.observeWord(word) + } + } + + mutating func observeSymbol(_ symbol: UInt16) { + copy.observeSymbol(symbol) + base.observeSymbol(symbol) + } + + mutating func observeOpaqueToken() { + copy.observeOpaqueToken() + base.observeOpaqueToken() + } + + mutating func observeQuotedToken(opener: UInt16) { + copy.observeOpaqueToken() + base.observeQuotedToken(opener: opener) + } + + mutating func observeGap() { + base.observeGap() + } + + mutating func observeSemicolon() -> Bool { + base.observeSemicolon() + } + + mutating func continuesStatement(with word: String?) -> Bool { + base.continuesStatement(with: word) + } + + mutating func reset() { + base.reset() + copy = SQLCopyFromStandardInput() + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift index d4abc7cc10..1a874aa467 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLRoutineBodyTracker.swift @@ -9,7 +9,8 @@ import Foundation /// `BEGIN`, `CASE` and `END` count only where a keyword can stand: an engine that does not reserve them takes them as /// names, and a block a name opened would never close. T-SQL reserves all three, so there a block also opens outside /// a routine, as in `IF @x = 1 BEGIN ... END` and `BEGIN TRY ... END TRY BEGIN CATCH ... END CATCH`. A text left -/// inside one is refused by the external gate. +/// inside one is refused by the external gate. T-SQL also lets a `;` end an `IF`'s body before its `ELSE`, as in +/// `IF @x = 1 SELECT 1; ELSE SELECT 2;`, so that `;` ends the statement only once the next token is not the `ELSE`. /// /// A `;` inside parentheses ends the statement, so a stray `(` cannot swallow what follows it, except in a PostgreSQL /// rule's action list, `DO ALSO (INSERT ...; INSERT ...)`, which holds its commands together. @@ -42,6 +43,9 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { // T-SQL outside a routine: where a block's `BEGIN` can stand, as the statement's first word, after `IF`, `WHILE` // or `ELSE`, or after a block the statement already opened. Anywhere else `begin` is in a name's place. private var followsControlFlow = false + // The statement's own `IF`s still waiting for an `ELSE`, which may follow the `;` that ends the `IF`'s body. + private var conditionalsAwaitingElse = 0 + private var endedBeforeElse = false private var pendingBeginMayOpen = false private var header = Header.opener private var definesRoutine = false @@ -127,10 +131,18 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { case "BEGIN" where opensNestedBlock(atStatementPosition: statementPosition, isFirstWord: isFirstWord): pendingBegin = true pendingBeginMayOpen = true - case "IF" where readsBlocksEverywhere, "WHILE" where readsBlocksEverywhere: + case "IF" where readsBlocksEverywhere: + followsControlFlow = true + if depth == 0 { + conditionalsAwaitingElse += 1 + } + case "WHILE" where readsBlocksEverywhere: followsControlFlow = true case "ELSE" where readsBlocksEverywhere && openCaseExpressions == 0: followsControlFlow = true + if depth == 0 { + conditionalsAwaitingElse = max(0, conditionalsAwaitingElse - 1) + } case "CASE" where !grammar.hasSingleBodyForm: depth += 1 openCaseExpressions += 1 @@ -218,9 +230,23 @@ public struct SQLRoutineBodyTracker: SQLStatementBoundaryTracking { inDeclaration = false inHandlerDeclaration = false merge?.observeSemicolon(endsStatement: endsStatement) + endedBeforeElse = endsStatement && readsBlocksEverywhere && conditionalsAwaitingElse > 0 return endsStatement } + public var awaitsContinuation: Bool { + endedBeforeElse + } + + // T-SQL writes `IF c SELECT 1; ELSE SELECT 2;`, so the `;` ended the `IF`'s body rather than the `IF`. + public mutating func continuesStatement(with word: String?) -> Bool { + guard endedBeforeElse else { return false } + endedBeforeElse = false + guard word == "ELSE" else { return false } + merge?.observeSemicolon(endsStatement: false) + return true + } + public mutating func reset() { self = SQLRoutineBodyTracker(grammar: grammar) } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementBoundaryTracking.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementBoundaryTracking.swift index 3c9ec3fb97..59181a6a75 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementBoundaryTracking.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementBoundaryTracking.swift @@ -48,10 +48,24 @@ public protocol SQLStatementBoundaryTracking { /// Returns whether this `;` ends the statement. mutating func observeSemicolon() -> Bool + /// Whether the `;` that just ended the statement can still be taken back by the token after it, as T-SQL takes + /// back the one in `IF @x = 1 SELECT 1; ELSE SELECT 2;` at its `ELSE`. A reader holds the end until the next token + /// that is not whitespace or a comment, then asks ``continuesStatement(with:)``. + var awaitsContinuation: Bool { get } + + /// Whether the first token after a `;` that left ``awaitsContinuation`` set carries the statement on. `word` is + /// that token uppercased, or nil for a token that is not a word. On true the reader keeps the statement open and + /// observes the token as usual; on false it ends the statement at the `;` and resets. + mutating func continuesStatement(with word: String?) -> Bool + /// Whether a block this statement opened is still open, which at the end of a text means everything after it was /// swallowed by a body that never closed. var holdsOpenBlock: Bool { get } + /// Whether the statement is a `COPY ... FROM STDIN` whose rows a psql script holds on the lines after it, which a + /// reader then takes as ``SQLPsqlScript/copyRows(afterSemicolonAt:in:length:)`` rather than as SQL. + var copiesFromStandardInput: Bool { get } + mutating func reset() } @@ -60,17 +74,35 @@ public extension SQLStatementBoundaryTracking { observeOpaqueToken() } + var awaitsContinuation: Bool { + false + } + + mutating func continuesStatement(with word: String?) -> Bool { + false + } + var holdsOpenBlock: Bool { false } + + var copiesFromStandardInput: Bool { + false + } } public enum SQLStatementBoundaries { /// The one place a grammar is matched to its statement boundaries, so no reader can pick a different one. public static func makeTracker(for grammar: SQLLexicalGrammar) -> any SQLStatementBoundaryTracking { - if grammar.contains(.plsqlBlocks) { return PLSQLUnitTracker() } - if grammar.contains(.cqlBatches) { return CQLBatchTracker() } - return SQLRoutineBodyTracker(grammar: grammar) + let tracker: any SQLStatementBoundaryTracking + if grammar.contains(.plsqlBlocks) { + tracker = PLSQLUnitTracker() + } else if grammar.contains(.cqlBatches) { + tracker = CQLBatchTracker() + } else { + tracker = SQLRoutineBodyTracker(grammar: grammar) + } + return grammar.contains(.psqlScripts) ? SQLPsqlStatementTracker(base: tracker) : tracker } /// Whether a statement `grammar` reads can own a `;`: the one that ends a PL/SQL unit or a T-SQL `MERGE`, or one diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScan.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScan.swift new file mode 100644 index 0000000000..4596a59245 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScan.swift @@ -0,0 +1,357 @@ +import Foundation + +/// One pass of ``SQLStatementScanner`` over a text. +/// +/// Where a statement ends is the tracker's decision, except at a `GO` line, which ends the batch whatever the tracker +/// thinks and is a segment of its own that runs nothing, and at a client delimiter, which ends the statement wherever +/// it stands in code. A `DELIMITER` line and a psql `\restrict` line are segments that run nothing, and a +/// `COPY ... FROM STDIN` runs on through the rows after it. +/// +/// A custom delimiter does not stop a `;` from ending a statement where the tracker says it does: the server still +/// splits the text it gets there, so reading it as one statement would let a gate miss the second. +struct SQLStatementScan { + struct Events { + let checkpoint: (SQLStatementScanner.Checkpoint) -> Void + let batchSeparator: (SQLBatchSeparator) -> Void + let scriptText: (NSRange) -> Void + let statement: (SQLStatementScanner.LocatedStatement) -> Bool + } + + private enum Step { + case declined + case consumed + case finished + } + + private let text: NSString + private let length: Int + private let grammar: SQLLexicalGrammar + private let cursor: Int? + private let events: Events + + private var tracker: any SQLStatementBoundaryTracking + private var currentStart: Int + private var index: Int + private var hasStatementContent = false + private var lastEndedStatement: SQLStatementScanner.EndedStatement? + private var delimiter: SQLClientDelimiter? + + // Where a `;` ended the statement while the tracker could still take it back at the next token, and the further + // `;`s after it, which T-SQL also reads as part of an `IF`'s body before its `ELSE`. + private var heldEnd: Int? + private var heldSemicolons: [Int] = [] + + private static let smallD = UInt16(UnicodeScalar("d").value) + + init( + text: NSString, + cursorPosition: Int?, + grammar: SQLLexicalGrammar, + resume: SQLStatementScanner.Checkpoint?, + events: Events + ) { + self.text = text + self.length = text.length + self.grammar = grammar + self.cursor = cursorPosition.map { min(max(0, $0), text.length) } + self.events = events + self.tracker = SQLStatementBoundaries.makeTracker(for: grammar) + self.currentStart = min(max(0, resume?.offset ?? 0), text.length) + self.index = currentStart + self.lastEndedStatement = resume?.endedStatement + self.delimiter = resume?.delimiter + } + + var holdsOpenBlock: Bool { + tracker.holdsOpenBlock + } + + /// Returns false when the scan stopped early, because the cursor's segment was found or a reader asked to stop. + mutating func run() -> Bool { + while index < length { + let unit = text.character(at: index) + var step = readClientLine(unit) + if step == .declined { step = readDelimiter(unit) } + if step == .declined { step = readNonCode(unit) } + if step == .declined { step = readBatchSeparator(unit) } + if step == .declined { step = readWord(unit) } + if step == .declined { step = readSemicolon(unit) } + if step == .declined { step = readSlashLine(unit) } + switch step { + case .finished: + return false + case .consumed: + continue + case .declined: + guard readBlankOrSymbol(unit) else { return false } + } + } + guard settleHeldEnd(word: nil, decidedThrough: length) else { return false } + if currentStart < length { + _ = events.statement(SQLStatementScanner.LocatedStatement( + sql: text.substring(with: NSRange(location: currentStart, length: length - currentStart)), + offset: currentStart, + hasContent: hasStatementContent, + terminator: tracker.terminator, + acceptsBindParameters: tracker.acceptsBindParameters, + clientDelimiter: delimiter + )) + } + return true + } + + // MARK: - Tokens + + private mutating func readClientLine(_ unit: UInt16) -> Step { + guard !hasStatementContent else { return .declined } + if grammar.contains(.delimiterDirective), unit | 0x20 == Self.smallD, + let directive = SQLDelimiterDirective.line(at: index, in: text, length: length) { + if let set = directive.delimiter { + delimiter = SQLClientDelimiter(set) + } + return finishClientLine(at: NSMaxRange(directive.range)) + } + if grammar.contains(.psqlScripts), unit == SqlLexer.backslash, + let line = SQLPsqlScript.restrictLine(at: index, in: text, length: length) { + events.scriptText(line) + return finishClientLine(at: NSMaxRange(line)) + } + return .declined + } + + private mutating func readDelimiter(_ unit: UInt16) -> Step { + guard let delimiter, delimiter.starts(in: text, at: index, length: length) else { return .declined } + guard settleHeldEnd(word: nil, decidedThrough: index) else { return .finished } + let end = index + delimiter.units.count + guard finishSegment(at: end, hasContent: hasStatementContent, delimiterLength: delimiter.units.count) else { + return .finished + } + endStatement(at: end, decidedThrough: end - 1) + return .consumed + } + + private mutating func readNonCode(_ unit: UInt16) -> Step { + guard let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) else { return .declined } + if !span.kind.isComment { + guard settleHeldEnd(word: nil, decidedThrough: index) else { return .finished } + } + switch span.kind { + case .lineComment, .blockComment: + tracker.observeGap() + case .executableComment: + hasStatementContent = true + case .quoted: + hasStatementContent = true + tracker.observeQuotedToken(opener: text.character(at: span.start)) + case .parameter: + hasStatementContent = true + tracker.observeOpaqueToken() + } + index = max(span.end, index + 1) + return .consumed + } + + private mutating func readBatchSeparator(_ unit: UInt16) -> Step { + guard grammar.contains(.batchSeparatorLines), + let separator = SQLBatchSeparator.line(at: index, in: text, length: length, grammar: grammar) + else { + return .declined + } + guard settleHeldEnd(word: nil, decidedThrough: index) else { return .finished } + events.batchSeparator(separator) + if hasStatementContent { + guard finishSegment(at: index, hasContent: true) else { return .finished } + } + let separatorEnd = NSMaxRange(separator.range) + guard finishSegment(at: separatorEnd, hasContent: false, standsForEndedStatement: true) else { + return .finished + } + lastEndedStatement = nil + endStatement(at: separatorEnd, decidedThrough: separatorEnd) + return .consumed + } + + private mutating func readWord(_ unit: UInt16) -> Step { + guard SqlBlockStructure.startsWord(text, at: index, length: length, grammar: grammar) else { return .declined } + guard heldEnd != nil || tracker.needsWords else { + hasStatementContent = true + index += 1 + while index < length, SqlBlockStructure.continuesWord(text.character(at: index), grammar: grammar), + delimiter?.starts(in: text, at: index, length: length) != true { + index += 1 + } + return .consumed + } + let read = SqlBlockStructure.readKeyword(text, at: index, length: length, grammar: grammar) + let word = delimiter?.cut(read, readAt: index, in: text, length: length) ?? read + guard settleHeldEnd(word: word.text, decidedThrough: word.end) else { return .finished } + hasStatementContent = true + if tracker.needsWords { + tracker.observeWord(word.text) + } + index = word.end + return .consumed + } + + private mutating func readSemicolon(_ unit: UInt16) -> Step { + guard unit == SqlLexer.semicolon else { return .declined } + if heldEnd != nil { + _ = tracker.observeSemicolon() + heldSemicolons.append(index) + index += 1 + return .consumed + } + guard tracker.observeSemicolon() else { + hasStatementContent = true + index += 1 + return .consumed + } + if tracker.awaitsContinuation { + heldEnd = index + 1 + index += 1 + return .consumed + } + if tracker.copiesFromStandardInput { + let rows = SQLPsqlScript.copyRows(afterSemicolonAt: index, in: text, length: length) + let end = NSMaxRange(rows) + let segment = NSRange(location: currentStart, length: end - currentStart) + // The rows run to `end` as written: a last row's trailing tab is a column. + let start = currentStart + StatementBlank.contentRange(of: text.substring(with: segment)).location + events.scriptText(NSRange(location: start, length: end - start)) + guard finishSegment(at: end, hasContent: hasStatementContent, copyRows: rows) else { return .finished } + endStatement(at: end, decidedThrough: end + 1) + return .consumed + } + guard finishSegment(at: index + 1, hasContent: hasStatementContent) else { return .finished } + // A delimiter such as `;;` was looked for past this `;` before the `;` alone was read. + endStatement(at: index + 1, decidedThrough: index + max(0, (delimiter?.units.count ?? 1) - 1)) + return .consumed + } + + private mutating func readSlashLine(_ unit: UInt16) -> Step { + guard grammar.contains(.slashLineTerminators), unit == SqlLexer.slash, + SQLStatementScanner.isSlashLine(text, at: index, length: length) + else { + return .declined + } + guard settleHeldEnd(word: nil, decidedThrough: index) else { return .finished } + if hasStatementContent { + guard finishSegment(at: index, hasContent: true) else { return .finished } + } + guard finishSegment(at: index + 1, hasContent: false, standsForEndedStatement: true) else { return .finished } + endStatement( + at: index + 1, + decidedThrough: SQLStatementScanner.endOfSlashLine(text, at: index, length: length) + ) + return .consumed + } + + /// Returns false when the scan is over. + private mutating func readBlankOrSymbol(_ unit: UInt16) -> Bool { + let blankLength = StatementBlank.blankLength(in: text, at: index) + if blankLength > 0 { + tracker.observeGap() + index += blankLength + return true + } + guard settleHeldEnd(word: nil, decidedThrough: index) else { return false } + hasStatementContent = true + tracker.observeSymbol(unit) + index += 1 + return true + } + + // MARK: - Segments + + // A caret on a SQL*Plus `/` line stands for the statement the slash ends, where `Cmd+Enter` is expected to act. + // A caret on a `GO` line stands for the last statement of its batch, and for nothing when the batch is empty. + // Returns false once the scan has reported what it was asked for. + private mutating func finishSegment( + at end: Int, + hasContent: Bool, + standsForEndedStatement: Bool = false, + delimiterLength: Int = 0, + copyRows: NSRange? = nil, + isClientLine: Bool = false + ) -> Bool { + let range = NSRange(location: currentStart, length: end - currentStart) + if hasContent { + lastEndedStatement = SQLStatementScanner.EndedStatement( + range: range, + terminator: tracker.terminator, + acceptsBindParameters: tracker.acceptsBindParameters, + delimiterLength: delimiterLength, + copyRows: copyRows, + clientDelimiter: delimiter + ) + } + if let cursor { + guard cursor >= currentStart, cursor <= end else { + currentStart = end + return true + } + if standsForEndedStatement, let ended = lastEndedStatement { + _ = events.statement(ended.located(in: text)) + return false + } + } + let statement = SQLStatementScanner.LocatedStatement( + sql: text.substring(with: range), + offset: currentStart, + hasContent: hasContent, + terminator: tracker.terminator, + acceptsBindParameters: tracker.acceptsBindParameters, + delimiterLength: delimiterLength, + copyRows: copyRows, + clientDelimiter: delimiter, + isClientLine: isClientLine + ) + if cursor != nil { + _ = events.statement(statement) + return false + } + currentStart = end + return events.statement(statement) + } + + private mutating func endStatement(at end: Int, decidedThrough: Int) { + tracker.reset() + hasStatementContent = false + checkpoint(at: end, decidedThrough: decidedThrough) + index = end + } + + private func checkpoint(at offset: Int, decidedThrough: Int) { + events.checkpoint(SQLStatementScanner.Checkpoint( + offset: offset, + decidedThrough: decidedThrough, + endedStatement: lastEndedStatement, + delimiter: delimiter + )) + } + + // A `DELIMITER` or `\restrict` line ends a segment that runs nothing. + private mutating func finishClientLine(at end: Int) -> Step { + guard finishSegment(at: end, hasContent: false, isClientLine: true) else { return .finished } + endStatement(at: end, decidedThrough: end) + return .consumed + } + + // Settles a held end at the first token after it; returns false once the scan has reported its statement. + private mutating func settleHeldEnd(word: String?, decidedThrough: Int) -> Bool { + guard let end = heldEnd else { return true } + heldEnd = nil + let semicolons = heldSemicolons + heldSemicolons.removeAll() + if tracker.continuesStatement(with: word) { return true } + guard finishSegment(at: end, hasContent: hasStatementContent) else { return false } + tracker.reset() + hasStatementContent = false + checkpoint(at: end, decidedThrough: decidedThrough) + for semicolon in semicolons { + guard finishSegment(at: semicolon + 1, hasContent: false) else { return false } + checkpoint(at: semicolon + 1, decidedThrough: decidedThrough) + } + return true + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift index 7e70ec3f98..44584b5e4f 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLStatementScanner.swift @@ -7,6 +7,20 @@ public enum SQLStatementScanner { public let hasContent: Bool public let terminator: SQLStatementTerminator public let acceptsBindParameters: Bool + + /// How many units at the end of ``sql`` are a client delimiter such as `//`, which no server receives. Zero when + /// a `;` or nothing ends the statement. + public let delimiterLength: Int + + /// For a `COPY ... FROM STDIN`, everything after its `;` that a psql script holds as rows, in the document's + /// coordinates. None of it is SQL. + public let copyRows: NSRange? + + /// The client delimiter in force once the statement ends. + public let clientDelimiter: SQLClientDelimiter? + + /// A `DELIMITER` or psql `\restrict` line, which the client reads and no server receives. + public let isClientLine: Bool private let isAllContent: Bool public init( @@ -14,13 +28,21 @@ public enum SQLStatementScanner { offset: Int, hasContent: Bool = true, terminator: SQLStatementTerminator = .separator, - acceptsBindParameters: Bool = true + acceptsBindParameters: Bool = true, + delimiterLength: Int = 0, + copyRows: NSRange? = nil, + clientDelimiter: SQLClientDelimiter? = nil, + isClientLine: Bool = false ) { self.sql = sql self.offset = offset self.hasContent = hasContent self.terminator = terminator self.acceptsBindParameters = acceptsBindParameters + self.delimiterLength = delimiterLength + self.copyRows = copyRows + self.clientDelimiter = clientDelimiter + self.isClientLine = isClientLine isAllContent = false } @@ -31,6 +53,10 @@ public enum SQLStatementScanner { hasContent = true terminator = .separator acceptsBindParameters = true + delimiterLength = 0 + copyRows = nil + clientDelimiter = nil + isClientLine = false isAllContent = true } @@ -44,9 +70,13 @@ public enum SQLStatementScanner { /// `offset` is the index just past the previous semicolon, so in a script written one statement per line it /// lands on the newline that ended the previous line. A decoration or a gutter anchor placed from ``range`` /// therefore starts a line early, and uses this instead. + /// + /// A client delimiter is left out: Run Current scans the text again on its own, where no `DELIMITER` line + /// precedes it, so the delimiter would reach the server. public var contentRange: NSRange { guard !isAllContent else { return range } - let content = StatementBlank.contentRange(of: sql) + let text = sql as NSString + let content = StatementBlank.contentRange(of: text.substring(to: max(0, text.length - delimiterLength))) return NSRange(location: offset + content.location, length: content.length) } } @@ -60,7 +90,7 @@ public enum SQLStatementScanner { /// The range is relative to the text the scan was given. A run started from a selection or from a single /// statement scans a fragment, so those callers shift the range onto the tab's whole query with ``offset(by:)`` /// before it travels any further. Everything downstream may then assume tab coordinates. - public struct ExecutableStatement: Sendable { + public struct ExecutableStatement: Sendable, Equatable { public let sql: String public let range: NSRange @@ -68,10 +98,23 @@ public enum SQLStatementScanner { /// ``SQLStatementBoundaryTracking/acceptsBindParameters``. public let acceptsBindParameters: Bool - public init(sql: String, range: NSRange, acceptsBindParameters: Bool = true) { + /// Where the scan went on after this statement: past its `;`, a client delimiter or a `COPY`'s rows. A run + /// that splits the rest of a script again starts there, with ``clientDelimiter`` in force. + public let segmentEnd: Int + public let clientDelimiter: SQLClientDelimiter? + + public init( + sql: String, + range: NSRange, + acceptsBindParameters: Bool = true, + segmentEnd: Int? = nil, + clientDelimiter: SQLClientDelimiter? = nil + ) { self.sql = sql self.range = range self.acceptsBindParameters = acceptsBindParameters + self.segmentEnd = segmentEnd ?? NSMaxRange(range) + self.clientDelimiter = clientDelimiter } public func offset(by delta: Int) -> ExecutableStatement { @@ -79,7 +122,9 @@ public enum SQLStatementScanner { return ExecutableStatement( sql: sql, range: NSRange(location: range.location + delta, length: range.length), - acceptsBindParameters: acceptsBindParameters + acceptsBindParameters: acceptsBindParameters, + segmentEnd: segmentEnd + delta, + clientDelimiter: clientDelimiter ) } } @@ -179,6 +224,16 @@ public enum SQLStatementScanner { return separators } + /// The text of a psql script a formatter keeps as written, in document order: each `COPY ... FROM STDIN` with the + /// rows after it through the `\.` line, and pg_dump's `\restrict` and `\unrestrict` lines. Empty for an engine + /// whose scripts are not psql's. + public static func psqlScriptText(in sql: String, grammar: SQLLexicalGrammar) -> [NSRange] { + guard grammar.contains(.psqlScripts) else { return [] } + var ranges: [NSRange] = [] + scan(sql: sql, cursorPosition: nil, grammar: grammar, onScriptText: { ranges.append($0) }) { _ in true } + return ranges + } + /// Returns statements as the driver receives them, for driver execution. public static func allStatements(in sql: String, grammar: SQLLexicalGrammar) -> [String] { executableStatements(in: sql, grammar: grammar).map(\.sql) @@ -215,10 +270,13 @@ public enum SQLStatementScanner { /// /// A `;` that belongs to the statement stays, but only when there is something before it: a unit reduced to its /// terminator is as empty as any other. + /// + /// A client delimiter never reaches the server, and a `COPY ... FROM STDIN` is sent without its rows. public static func executableStatement(from located: LocatedStatement) -> ExecutableStatement? { + guard !located.isClientLine else { return nil } let rawSQL = located.sql - var content = StatementBlank.trimming(rawSQL[...]) - if content.last == ";" { + var content = StatementBlank.trimming(rawSQL[.. [ExecutableStatement] { + let text = sql as NSString + let limit = min(end, text.length) + guard ran.segmentEnd < limit else { return [] } + let resume = Checkpoint( + offset: ran.segmentEnd, + decidedThrough: ran.segmentEnd, + endedStatement: nil, + delimiter: ran.clientDelimiter + ) + var results: [ExecutableStatement] = [] + scan(sql: text.substring(to: limit), cursorPosition: nil, grammar: grammar, resumingAt: resume) { located in + if located.hasContent, let statement = executableStatement(from: located) { + results.append(statement) + } + return true + } + return results + } + + private static func sentEnd(of located: LocatedStatement) -> String.Index { + let length = located.sql.utf16.count + let end = located.copyRows.map { $0.location - located.offset } ?? length - located.delimiterLength + return String.Index(utf16Offset: min(max(0, end), length), in: located.sql) + } + /// Whether the text ends inside a block it opened, so the last statement swallowed everything after its opener. /// An engine that reads that opener as a name runs the rest as statements of their own. public static func endsInsideOpenBlock(_ sql: String, grammar: SQLLexicalGrammar) -> Bool { @@ -248,7 +343,7 @@ public enum SQLStatementScanner { var results: [String] = [] scan(sql: sql, cursorPosition: nil, grammar: grammar) { statement in guard statement.hasContent else { return true } - let trimmed = StatementBlank.trimming(statement.sql) + let trimmed = StatementBlank.trimming(String(statement.sql[.. LocatedStatement { LocatedStatement( @@ -288,187 +386,56 @@ public enum SQLStatementScanner { offset: range.location, hasContent: true, terminator: terminator, - acceptsBindParameters: acceptsBindParameters + acceptsBindParameters: acceptsBindParameters, + delimiterLength: delimiterLength, + copyRows: copyRows, + clientDelimiter: clientDelimiter ) } } - // A point where the scan holds no state but the last statement with content, so it can start again there. + // A point where the scan holds no state but the last statement with content and the client delimiter in force, + // so it can start again there. struct Checkpoint: Sendable, Equatable { let offset: Int // The last offset whose text decided this boundary: an edit there or earlier moves it. let decidedThrough: Int let endedStatement: EndedStatement? + var delimiter: SQLClientDelimiter? } // MARK: - Scanning // Segments tile the document: each runs from the end of the previous one to its terminator, whitespace included. // With a cursor only the segment holding it is reported, or the last one when it sits past every terminator. - // Where a statement ends is the tracker's decision, except at a `GO` line, which ends the batch whatever the - // tracker thinks and is a segment of its own that runs nothing. + // See ``SQLStatementScan`` for where a segment ends. static func scan( sql: String, cursorPosition: Int?, grammar: SQLLexicalGrammar, resumingAt resume: Checkpoint? = nil, - onCheckpoint: (Checkpoint) -> Void = { _ in }, - onBatchSeparator: (SQLBatchSeparator) -> Void = { _ in }, + onCheckpoint: @escaping (Checkpoint) -> Void = { _ in }, + onBatchSeparator: @escaping (SQLBatchSeparator) -> Void = { _ in }, + onScriptText: @escaping (NSRange) -> Void = { _ in }, onFinish: (_ holdsOpenBlock: Bool) -> Void = { _ in }, - onStatement: (LocatedStatement) -> Bool + onStatement: @escaping (LocatedStatement) -> Bool ) { - let nsQuery = sql as NSString - let length = nsQuery.length - guard length > 0 else { return } - - let safePosition = cursorPosition.map { min(max(0, $0), length) } - - var tracker = SQLStatementBoundaries.makeTracker(for: grammar) - var currentStart = min(max(0, resume?.offset ?? 0), length) - var hasStatementContent = false - var i = currentStart - - var lastEndedStatement = resume?.endedStatement - - // A caret on a SQL*Plus `/` line stands for the statement the slash ends, where `Cmd+Enter` is expected to act. - // A caret on a `GO` line stands for the last statement of its batch, and for nothing when the batch is empty. - func finishSegment(at end: Int, hasContent: Bool, standsForEndedStatement: Bool = false) -> Bool { - let range = NSRange(location: currentStart, length: end - currentStart) - if hasContent { - lastEndedStatement = EndedStatement( - range: range, - terminator: tracker.terminator, - acceptsBindParameters: tracker.acceptsBindParameters - ) - } - if let cursor = safePosition { - guard cursor >= currentStart, cursor <= end else { - currentStart = end - return true - } - if standsForEndedStatement, let ended = lastEndedStatement { - _ = onStatement(ended.located(in: nsQuery)) - return false - } - } - let statement = LocatedStatement( - sql: nsQuery.substring(with: range), - offset: currentStart, - hasContent: hasContent, - terminator: tracker.terminator, - acceptsBindParameters: tracker.acceptsBindParameters - ) - if safePosition != nil { - _ = onStatement(statement) - return false - } - currentStart = end - return onStatement(statement) - } - - while i < length { - let ch = nsQuery.character(at: i) - - if let span = SQLNonCodeSpan.span(at: i, in: nsQuery, grammar: grammar) { - switch span.kind { - case .lineComment, .blockComment: - tracker.observeGap() - case .executableComment: - hasStatementContent = true - case .quoted: - hasStatementContent = true - tracker.observeQuotedToken(opener: nsQuery.character(at: span.start)) - case .parameter: - hasStatementContent = true - tracker.observeOpaqueToken() - } - i = max(span.end, i + 1) - continue - } - - if grammar.contains(.batchSeparatorLines), - let separator = SQLBatchSeparator.line(at: i, in: nsQuery, length: length, grammar: grammar) { - onBatchSeparator(separator) - if hasStatementContent { - guard finishSegment(at: i, hasContent: true) else { return } - } - let separatorEnd = NSMaxRange(separator.range) - guard finishSegment(at: separatorEnd, hasContent: false, standsForEndedStatement: true) else { return } - lastEndedStatement = nil - tracker.reset() - hasStatementContent = false - onCheckpoint(Checkpoint(offset: separatorEnd, decidedThrough: separatorEnd, endedStatement: nil)) - i = separatorEnd - continue - } - - if SqlBlockStructure.startsWord(nsQuery, at: i, length: length, grammar: grammar) { - hasStatementContent = true - if tracker.needsWords { - let word = SqlBlockStructure.readKeyword(nsQuery, at: i, length: length, grammar: grammar) - tracker.observeWord(word.text) - i = word.end - } else { - i += 1 - while i < length, SqlBlockStructure.continuesWord(nsQuery.character(at: i), grammar: grammar) { - i += 1 - } - } - continue - } - - if ch == SqlLexer.semicolon { - guard tracker.observeSemicolon() else { - hasStatementContent = true - i += 1 - continue - } - guard finishSegment(at: i + 1, hasContent: hasStatementContent) else { return } - tracker.reset() - hasStatementContent = false - onCheckpoint(Checkpoint(offset: i + 1, decidedThrough: i, endedStatement: lastEndedStatement)) - i += 1 - continue - } - - if grammar.contains(.slashLineTerminators), ch == SqlLexer.slash, isSlashLine(nsQuery, at: i, length: length) { - if hasStatementContent { - guard finishSegment(at: i, hasContent: true) else { return } - } - guard finishSegment(at: i + 1, hasContent: false, standsForEndedStatement: true) else { return } - tracker.reset() - hasStatementContent = false - onCheckpoint(Checkpoint( - offset: i + 1, - decidedThrough: endOfSlashLine(nsQuery, at: i, length: length), - endedStatement: lastEndedStatement - )) - i += 1 - continue - } - - let blankLength = StatementBlank.blankLength(in: nsQuery, at: i) - if blankLength > 0 { - tracker.observeGap() - i += blankLength - continue - } - hasStatementContent = true - tracker.observeSymbol(ch) - i += 1 - } - - if currentStart < length { - let statement = LocatedStatement( - sql: nsQuery.substring(with: NSRange(location: currentStart, length: length - currentStart)), - offset: currentStart, - hasContent: hasStatementContent, - terminator: tracker.terminator, - acceptsBindParameters: tracker.acceptsBindParameters + let text = sql as NSString + guard text.length > 0 else { return } + var scan = SQLStatementScan( + text: text, + cursorPosition: cursorPosition, + grammar: grammar, + resume: resume, + events: SQLStatementScan.Events( + checkpoint: onCheckpoint, + batchSeparator: onBatchSeparator, + scriptText: onScriptText, + statement: onStatement ) - _ = onStatement(statement) - } - onFinish(tracker.holdsOpenBlock) + ) + guard scan.run() else { return } + onFinish(scan.holdsOpenBlock) } /// Whether the `/` at `offset` stands alone on its line, which is what makes it SQL*Plus's terminator rather than @@ -484,7 +451,7 @@ public enum SQLStatementScanner { } // The last offset `isSlashLine` reads after the slash: its line break, or the end of the text. - private static func endOfSlashLine(_ text: NSString, at offset: Int, length: Int) -> Int { + static func endOfSlashLine(_ text: NSString, at offset: Int, length: Int) -> Int { var after = offset + 1 while after < length, isLineBlank(text.character(at: after)) { after += 1 diff --git a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLCompletionProviderCatalogTests.swift b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLCompletionProviderCatalogTests.swift index b2d62173dd..e03d45832a 100644 --- a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLCompletionProviderCatalogTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLCompletionProviderCatalogTests.swift @@ -3,13 +3,47 @@ import Foundation import TableProSQLGrammar import Testing -private struct StubCatalog: SQLCompletionCatalog { +struct StubCatalog: SQLCompletionCatalog { let tables: [String] let columns: [String: [String]] var values: [String] = [] var databases: [String] = [] var schemas: [String] = [] var schemaTables: [String: [String]] = [:] + var keys: [String: [SQLCompletionForeignKey]] = [:] + // Like the apps' catalogs: two tables in scope qualify every column. + var qualifiesInScope = false + // Like the iOS catalog, which quotes a mixed-case name itself because nothing spells it later. + var writesNames = false + let keyReads = KeyReads() + + final class KeyReads: @unchecked Sendable { + private let lock = NSLock() + private var tables: [String] = [] + + func record(_ table: String) { + lock.withLock { tables.append(table) } + } + + var all: [String] { + lock.withLock { tables } + } + } + + func foreignKeys(of tableName: String, schema: String?) -> [SQLCompletionForeignKey] { + keyReads.record(tableName) + return keys[tableName.lowercased()] ?? [] + } + + func writtenText(of identifier: SQLCompletionIdentifier) -> String? { + guard writesNames else { return nil } + return identifier.parts.map { part in + switch part { + case .catalog(let name): return name == name.lowercased() ? name : "\"\(name)\"" + case .statement(let text, let isQuoted): return isQuoted ? "\"\(text)\"" : text + } + }.joined(separator: ".") + } func resolveAlias(_ aliasOrName: String, in references: [TableReference]) -> String? { let name = aliasOrName.lowercased() @@ -41,9 +75,23 @@ private struct StubCatalog: SQLCompletionCatalog { func allowedValues(forColumn column: String, in references: [TableReference]) -> [String] { values } func allColumnsInScope(for references: [TableReference]) -> [SQLCompletionItem] { - references.flatMap { reference in + references.flatMap { reference -> [SQLCompletionItem] in guard let derived = reference.derivedColumns else { - return columnCompletionItems(for: reference.tableName, schema: reference.schema) + let items = columnCompletionItems(for: reference.tableName, schema: reference.schema) + guard qualifiesInScope, references.count > 1 else { return items } + return items.map { column in + let parts: [SQLCompletionIdentifier.Part] = [ + .statement(reference.identifier, quoted: reference.isIdentifierQuoted), .catalog(column.label), + ] + var item = SQLCompletionItem( + label: "\(reference.identifier).\(column.label)", + kind: .column, + insertText: writtenText(of: SQLCompletionIdentifier(parts)), + identifier: SQLCompletionIdentifier(parts) + ) + item.isQualifiedReference = true + return item + } } return derived.map { SQLCompletionItem.column($0, dataType: nil, tableName: reference.identifier) } } diff --git a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLContextAnalyzerJoinScopeTests.swift b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLContextAnalyzerJoinScopeTests.swift new file mode 100644 index 0000000000..44e9356f49 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLContextAnalyzerJoinScopeTests.swift @@ -0,0 +1,117 @@ +import Foundation +@testable import TableProAutocomplete +import TableProSQLGrammar +import Testing + +struct SQLContextAnalyzerJoinScopeTests { + private let analyzer = SQLContextAnalyzer() + + // `|` marks the cursor. + private func scope(_ marked: String, grammar: SQLLexicalGrammar = .ansi) -> SQLJoinScope? { + let marker = (marked as NSString).range(of: "|") + let text = (marked as NSString).replacingCharacters(in: marker, with: "") + return analyzer.analyze(query: text, cursorPosition: marker.location, grammar: grammar).joinScope + } + + private func names(_ marked: String, grammar: SQLLexicalGrammar = .ansi) -> [String]? { + guard let scope = scope(marked, grammar: grammar) else { return nil } + return [scope.joined.identifier] + scope.earlier.map(\.identifier) + } + + @Test("Right after a join's ON, the joined table and the earlier ones, nearest first", arguments: [ + ("SELECT * FROM orders o JOIN customers c ON |", ["c", "o"]), + ("SELECT * FROM orders o JOIN customers c ON cu|", ["c", "o"]), + ("SELECT * FROM orders JOIN customers ON |", ["customers", "orders"]), + ("SELECT * FROM orders o LEFT OUTER JOIN customers AS c ON |", ["c", "o"]), + ("SELECT * FROM customers c JOIN orders o ON o.customer_id = c.id JOIN payments p ON |", ["p", "o", "c"]), + ("SELECT * FROM a JOIN b USING (id) JOIN c ON |", ["c", "b", "a"]), + ("SELECT * FROM a JOIN b ON a.x IS DISTINCT FROM b.x JOIN c ON |", ["c", "b", "a"]), + ("SELECT * FROM a STRAIGHT_JOIN b ON |", ["b", "a"]), + ("UPDATE a JOIN b ON | SET a.x = 1", ["b", "a"]), + ("DELETE FROM t USING a JOIN b ON |", ["b", "a"]), + ("UPDATE t SET x = 1 FROM a JOIN b ON |", ["b", "a"]), + ]) + func joinScope(_ marked: String, _ expected: [String]) { + #expect(names(marked) == expected) + } + + @Test("A table joined after the cursor is not in scope") + func laterJoinIsOutOfScope() { + #expect(names("SELECT * FROM orders o JOIN customers c ON | JOIN payments p ON p.order_id = o.id") == ["c", "o"]) + } + + @Test("Only the tables since the last comma may be named, as the server reads the ON") + func commaEndsTheScope() { + #expect(names("SELECT * FROM orders o, customers c JOIN payments p ON |") == ["p", "c"]) + } + + @Test("A subquery reads its own FROM list") + func subquery() { + #expect(names("SELECT * FROM x WHERE id IN (SELECT 1 FROM orders o JOIN customers c ON |)") == ["c", "o"]) + #expect(names("WITH r AS (SELECT * FROM orders o JOIN customers c ON |) SELECT * FROM r") == ["c", "o"]) + } + + @Test("Inside a join group the ON names the group's own tables") + func joinGroup() { + #expect(names("SELECT * FROM a JOIN (b JOIN c ON |) ON a.id = b.id") == ["c", "b"]) + #expect(names("SELECT * FROM (a JOIN b ON a.id = b.id) JOIN c ON |") == ["c", "b", "a"]) + } + + @Test("A table hint between the table and its ON is skipped", arguments: [ + ("SELECT * FROM a WITH (NOLOCK) JOIN b WITH (NOLOCK) ON |", "SQL Server"), + ("SELECT * FROM a JOIN b USE INDEX (ix_b) ON |", "MySQL"), + ]) + func tableHint(_ marked: String, _ engine: String) { + #expect(names(marked, grammar: .engine(engine)) == ["b", "a"]) + } + + @Test("A schema and a quoted name are kept") + func schemaAndQuotes() throws { + let found = try #require(scope("SELECT * FROM sales.orders \"O\" JOIN \"Customers\" ON |", grammar: .engine("PostgreSQL"))) + #expect(found.joined.tableName == "Customers") + #expect(found.joined.alias == nil) + #expect(found.joined.isIdentifierQuoted) + #expect(found.earlier.first?.tableName == "orders") + #expect(found.earlier.first?.schema == "sales") + #expect(found.earlier.first?.alias == "O") + #expect(found.earlier.first?.isIdentifierQuoted == true) + } + + @Test("No scope for an ON that belongs to no join, or once the condition has started", arguments: [ + "CREATE INDEX idx ON |", + "CREATE UNIQUE INDEX idx ON |", + "GRANT SELECT ON |", + "CREATE TRIGGER trg AFTER INSERT ON |", + "CREATE TRIGGER trg AFTER UPDATE ON |", + "MERGE INTO orders o USING customers c ON |", + "SELECT * FROM orders o ON |", + "SELECT * FROM orders o, customers c ON |", + "SELECT * FROM orders o JOIN ON |", + "SELECT * FROM orders o JOIN customers c ON o.|", + "SELECT * FROM orders o JOIN customers c ON \"|", + "SELECT * FROM orders o JOIN customers c ON o.customer_id = |", + "SELECT * FROM orders o JOIN customers c ON o.customer_id = c.id |", + "SELECT * FROM orders o JOIN customers c ON o.customer_id = c.id AND |", + "SELECT * FROM orders o JOIN customers c ON|", + "SELECT * FROM orders o CROSS APPLY f(o.id) x ON |", + ]) + func noScope(_ marked: String) { + #expect(scope(marked) == nil) + } + + @Test("A derived table or a CTE has no keys, on either side", arguments: [ + "SELECT * FROM orders o JOIN (SELECT * FROM customers) c ON |", + "SELECT * FROM (SELECT * FROM orders) o JOIN customers c ON |", + "WITH recent AS (SELECT * FROM orders) SELECT * FROM recent r JOIN customers c ON |", + "WITH customers AS (SELECT 1 AS id) SELECT * FROM orders o JOIN customers c ON |", + "SELECT * FROM orders o JOIN generate_series(1, 3) g ON |", + ]) + func derivedSides(_ marked: String) { + #expect(scope(marked) == nil) + } + + @Test("A derived table before a stored one leaves the stored ones in scope") + func derivedEarlierTable() { + #expect(names("SELECT * FROM customers c JOIN (SELECT * FROM x) d ON d.id = c.id JOIN orders o ON |") == ["o", "c"]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLIdentifierSpellingTests.swift b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLIdentifierSpellingTests.swift index 12c08eee63..203d93370d 100644 --- a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLIdentifierSpellingTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLIdentifierSpellingTests.swift @@ -155,6 +155,46 @@ struct SQLIdentifierSpellingTests { #expect(applied([item], .lower, quoting: .enclosed)[0].insertText == "o.\"MixedCase\"") } + @Test("A join condition spells each column and keeps each qualifier as written") + func joinConditionSides() { + let item = SQLCompletionItem( + label: "O.CUSTOMER_ID = c.ID AND O.Region = c.REGION", + kind: .joinCondition, + columnPairs: [ + SQLCompletionColumnPair( + left: SQLCompletionIdentifier([.statement("O", quoted: true), .catalog("CUSTOMER_ID")]), + right: SQLCompletionIdentifier([.statement("c", quoted: false), .catalog("ID")]) + ), + SQLCompletionColumnPair( + left: SQLCompletionIdentifier([.statement("O", quoted: true), .catalog("Region")]), + right: SQLCompletionIdentifier([.statement("c", quoted: false), .catalog("REGION")]) + ), + ] + ) + let spelled = applied([item], .lower, prefix: "O")[0] + #expect(spelled.label == "O.customer_id = c.id AND O.Region = c.region") + #expect(spelled.insertText == "\"O\".customer_id = c.id AND \"O\".\"Region\" = c.region") + #expect(spelled.filterText == item.filterText) + + let opened = applied([item], .lower, quoting: .opened)[0] + #expect(opened.insertText == item.insertText) + } + + @Test("On a lower-folding engine a mixed-case key column is quoted and a quoted alias keeps its quotes") + func joinConditionOnLowerFolding() { + let item = SQLCompletionItem( + label: "O.CustomerId = c.id", + kind: .joinCondition, + columnPairs: [SQLCompletionColumnPair( + left: SQLCompletionIdentifier([.statement("O", quoted: true), .catalog("CustomerId")]), + right: SQLCompletionIdentifier([.statement("c", quoted: false), .catalog("id")]) + )] + ) + let spelled = spelling(.asStored, rules: lowerFolding).applied(to: [item], typedPrefix: "", quoting: .none)[0] + #expect(spelled.insertText == "\"O\".\"CustomerId\" = c.id") + #expect(spelled.label == "O.CustomerId = c.id") + } + @Test("Keywords and items without a catalog name are untouched") func otherItemsUntouched() { let keyword = SQLCompletionItem.keyword("SELECT") diff --git a/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLJoinConditionCompletionTests.swift b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLJoinConditionCompletionTests.swift new file mode 100644 index 0000000000..9a969331e1 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProAutocompleteTests/SQLJoinConditionCompletionTests.swift @@ -0,0 +1,228 @@ +import Foundation +@testable import TableProAutocomplete +import TableProSQLGrammar +import Testing + +struct SQLJoinConditionCompletionTests { + private static func key( + _ name: String, _ columns: [String], _ table: String, _ referenced: [String], schema: String? = nil + ) -> SQLCompletionForeignKey { + SQLCompletionForeignKey( + name: name, columns: columns, referencedTable: table, referencedSchema: schema, referencedColumns: referenced + ) + } + + // A fresh catalog per use, so one test's key reads never show in another's. + private static var shop: StubCatalog { + StubCatalog( + tables: ["orders", "customers", "order_items", "payments", "flights", "airports", "employees"], + columns: [ + "orders": ["id", "customer_id", "tenant_id", "total"], + "customers": ["id", "name"], + "order_items": ["order_id", "tenant_id", "qty"], + "payments": ["id", "order_id", "customer_id"], + "flights": ["id", "origin", "destination"], + "airports": ["code", "name"], + "employees": ["id", "manager_id"], + ], + keys: [ + "orders": [key("orders_customer_fk", ["customer_id"], "customers", ["id"])], + "order_items": [key("items_order_fk", ["order_id", "tenant_id"], "orders", ["id", "tenant_id"])], + "payments": [ + key("payments_order_fk", ["order_id"], "orders", ["id"]), + key("payments_customer_fk", ["customer_id"], "customers", ["id"]), + ], + "flights": [ + key("flights_origin_fk", ["origin"], "airports", ["code"]), + key("flights_destination_fk", ["destination"], "airports", ["code"]), + ], + "employees": [key("employees_manager_fk", ["manager_id"], "employees", ["id"])], + ] + ) + } + + // `|` marks the cursor. + private func items(_ marked: String, catalog: StubCatalog? = nil) async -> [SQLCompletionItem] { + let provider = SQLCompletionProvider(catalog: catalog ?? Self.shop, grammar: .engine("SQLite"), vocabulary: .generic) + let marker = (marked as NSString).range(of: "|") + let text = (marked as NSString).replacingCharacters(in: marker, with: "") + return await provider.completionSession(text: text, cursorPosition: marker.location).items + } + + private func conditions(_ marked: String, catalog: StubCatalog? = nil) async -> [String] { + await items(marked, catalog: catalog).filter { $0.kind == .joinCondition }.map(\.label) + } + + @Test("Right after a join's ON the condition from the foreign key leads, the earlier table first") + func conditionLeads() async { + let first = await items("SELECT * FROM orders o JOIN customers c ON |").first + #expect(first?.label == "o.customer_id = c.id") + #expect(first?.insertText == "o.customer_id = c.id") + #expect(first?.kind == .joinCondition) + #expect(first?.detail == "orders_customer_fk") + } + + @Test("A key from the joined table to an earlier one is written with the earlier table first too") + func reverseDirection() async { + #expect(await items("SELECT * FROM customers c JOIN orders o ON |").first?.label == "c.id = o.customer_id") + } + + @Test("A table with no alias is named as written") + func noAlias() async { + #expect(await items("SELECT * FROM orders JOIN customers ON |").first?.label == "orders.customer_id = customers.id") + } + + @Test("A composite key joins its pairs with AND in key order") + func compositeKey() async { + #expect(await items("SELECT * FROM orders o JOIN order_items i ON |").first?.label + == "o.id = i.order_id AND o.tenant_id = i.tenant_id") + } + + @Test("Two keys between one pair give one condition each") + func twoKeys() async { + #expect(await conditions("SELECT * FROM flights f JOIN airports a ON |") + == ["f.origin = a.code", "f.destination = a.code"]) + } + + @Test("A self-join offers the key both ways") + func selfJoin() async { + #expect(await conditions("SELECT * FROM employees e JOIN employees m ON |") + == ["e.manager_id = m.id", "e.id = m.manager_id"]) + } + + @Test("Earlier tables are paired nearest first") + func nearestFirst() async { + let marked = "SELECT * FROM customers c JOIN orders o ON o.customer_id = c.id JOIN payments p ON |" + #expect(await conditions(marked) == ["o.id = p.order_id", "c.id = p.customer_id"]) + } + + @Test("A table joined after the cursor is neither paired nor read") + func laterTableIsNotRead() async { + let catalog = Self.shop + let marked = "SELECT * FROM orders o JOIN customers c ON | JOIN payments p ON p.order_id = o.id" + #expect(await conditions(marked, catalog: catalog) == ["o.customer_id = c.id"]) + #expect(catalog.keyReads.all.sorted() == ["customers", "orders"]) + } + + @Test("Typing the earlier alias keeps the condition first") + func qualifierPrefix() async { + #expect(await items("SELECT * FROM orders o JOIN customers c ON o|").first?.label == "o.customer_id = c.id") + } + + @Test("No condition and no key read where the ON belongs to no join or the condition has started", arguments: [ + "CREATE INDEX idx ON |", + "SELECT * FROM orders o JOIN customers c ON o.|", + "SELECT * FROM orders o JOIN customers c ON \"|", + "SELECT * FROM orders o JOIN customers c ON o.customer_id = c.id AND |", + "SELECT * FROM orders o JOIN customers c ON o.customer_id = c.id |", + "SELECT * FROM orders o JOIN (SELECT * FROM customers) c ON |", + "WITH customers AS (SELECT 1 AS id) SELECT * FROM orders o JOIN customers c ON |", + "SELECT * FROM orders o WHERE |", + ]) + func noCondition(_ marked: String) async { + let catalog = Self.shop + #expect(await conditions(marked, catalog: catalog).isEmpty) + #expect(catalog.keyReads.all.isEmpty) + } + + @Test("A catalog that writes its own names quotes each side, and the label stays plain") + func catalogWrittenCondition() async throws { + var catalog = StubCatalog( + tables: ["orders", "customers"], + columns: ["orders": ["Id", "CustomerId"], "customers": ["Id"]], + keys: ["orders": [Self.key("fk", ["CustomerId"], "customers", ["Id"])]] + ) + catalog.writesNames = true + let first = try #require(await items("SELECT * FROM orders \"O\" JOIN customers c ON |", catalog: catalog).first) + #expect(first.label == "O.CustomerId = c.Id") + #expect(first.insertText == "\"O\".\"CustomerId\" = c.\"Id\"") + } + + @Test("A key column spelled in another case takes the table's spelling") + func keyColumnTakesTableSpelling() async { + let catalog = StubCatalog( + tables: ["orders", "customers"], + columns: ["orders": ["id", "customer_id"], "customers": ["id"]], + keys: ["orders": [Self.key("fk", ["CUSTOMER_ID"], "Customers", ["ID"])]] + ) + #expect(await conditions("SELECT * FROM orders o JOIN customers c ON |", catalog: catalog) == ["o.customer_id = c.id"]) + } + + // MARK: - The qualified columns of the same ON + + @Test("A column the catalog already qualifies is offered once, with the catalog's spelling") + func qualifiedColumnOnce() async throws { + var catalog = StubCatalog(tables: ["orders", "customers"], columns: ["orders": ["Id", "total"], "customers": ["Id"]]) + catalog.qualifiesInScope = true + catalog.writesNames = true + let items = await items("SELECT * FROM orders o JOIN customers c ON |", catalog: catalog) + + #expect(items.filter { $0.label == "o.Id" }.count == 1) + let column = try #require(items.first { $0.label == "o.Id" }) + #expect(column.insertText == "o.\"Id\"") + #expect(column.isQualifiedReference) + #expect(column.sortPriority == 80) + } + + @Test("A qualified column the catalog writes keeps a quoted alias and the column's quotes") + func writtenQualifiedColumn() async { + var catalog = StubCatalog(tables: ["orders"], columns: ["orders": ["Id"]]) + catalog.writesNames = true + let items = await items("SELECT * FROM orders \"O\" ON |", catalog: catalog) + #expect(items.first { $0.label == "O.Id" }?.insertText == "\"O\".\"Id\"") + } + + // MARK: - Pairing + + private func side(_ table: String, alias: String, schema: String? = nil, keys: [SQLCompletionForeignKey] = []) -> SQLJoinConditions.Side { + SQLJoinConditions.Side( + reference: TableReference(tableName: table, alias: alias, schema: schema), + keys: keys, + columns: [] + ) + } + + @Test("Keys keep the driver's order, an earlier table's own keys before the joined table's") + func pairingOrder() { + let joined = side("airports", alias: "a", keys: [Self.key("hub_fk", ["hub_flight"], "flights", ["id"])]) + let earlier = side("flights", alias: "f", keys: [ + Self.key("destination_fk", ["destination"], "airports", ["code"]), + Self.key("origin_fk", ["origin"], "airports", ["code"]), + ]) + let names = SQLJoinConditions.conditions(joined: joined, earlier: [earlier]).map(\.constraintName) + #expect(names == ["destination_fk", "origin_fk", "hub_fk"]) + } + + @Test("A key into another schema pairs only with a table of that schema") + func pairingSchema() { + let toSales = Self.key("fk", ["customer_id"], "customers", ["id"], schema: "sales") + let orders = side("orders", alias: "o", keys: [toSales]) + #expect(SQLJoinConditions.conditions(joined: side("customers", alias: "c", schema: "sales"), earlier: [orders]).count == 1) + #expect(SQLJoinConditions.conditions(joined: side("customers", alias: "c", schema: "public"), earlier: [orders]).isEmpty) + #expect(SQLJoinConditions.conditions(joined: side("customers", alias: "c"), earlier: [orders]).count == 1) + + let salesOrders = side("orders", alias: "o", schema: "sales", keys: [toSales]) + #expect(SQLJoinConditions.conditions(joined: side("customers", alias: "c"), earlier: [salesOrders]).isEmpty) + let ownNamespace = Self.key("fk", ["user_id"], "users", ["id"]) + let otherOrders = side("orders", alias: "o", schema: "otherdb", keys: [ownNamespace]) + #expect(SQLJoinConditions.conditions(joined: side("users", alias: "u"), earlier: [otherOrders]).isEmpty) + #expect(SQLJoinConditions.conditions(joined: side("users", alias: "u", schema: "otherdb"), earlier: [otherOrders]).count == 1) + let localOrders = side("orders", alias: "o", keys: [ownNamespace]) + #expect(SQLJoinConditions.conditions(joined: side("users", alias: "u", schema: "otherdb"), earlier: [localOrders]).isEmpty) + } + + @Test("Driver rows group into one key per name and target, pairs in key order") + func grouping() { + let rows = [ + SQLCompletionForeignKey.Row(name: "items_fk", column: "order_id", referencedTable: "orders", referencedSchema: nil, referencedColumn: "id"), + SQLCompletionForeignKey.Row(name: "origin_fk", column: "origin", referencedTable: "airports", referencedSchema: nil, referencedColumn: "code"), + SQLCompletionForeignKey.Row(name: "items_fk", column: "tenant_id", referencedTable: "orders", referencedSchema: nil, referencedColumn: "tenant_id"), + SQLCompletionForeignKey.Row(name: "items_fk", column: "tenant_id", referencedTable: "orders", referencedSchema: nil, referencedColumn: "tenant_id"), + SQLCompletionForeignKey.Row(name: "destination_fk", column: "destination", referencedTable: "airports", referencedSchema: nil, referencedColumn: "code"), + ] + let keys = SQLCompletionForeignKey.grouping(rows) + #expect(keys.map(\.name) == ["items_fk", "origin_fk", "destination_fk"]) + #expect(keys.first?.columns == ["order_id", "tenant_id"]) + #expect(keys.first?.referencedColumns == ["id", "tenant_id"]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProQueryTests/QueryScriptTests.swift b/apple/Packages/TableProCore/Tests/TableProQueryTests/QueryScriptTests.swift index 4a85284b43..fa0eeb768c 100644 --- a/apple/Packages/TableProCore/Tests/TableProQueryTests/QueryScriptTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProQueryTests/QueryScriptTests.swift @@ -163,6 +163,16 @@ struct QueryScriptTests { #expect(QueryScript(units: script.units, unitKind: .statement).replanned(after: 0, session: noEscapes) == nil) } + @Test("A replan after a COPY starts past its rows") + func replanAfterACopy() throws { + let text = "COPY t FROM stdin;\nDROP TABLE users; COMMIT;\n\\.\nSELECT 1;" + let script = QueryScript.plan(text, databaseType: .postgresql, sendsBatchesWhole: false) + #expect(script.units.map(\.sql) == ["COPY t FROM stdin", "SELECT 1"]) + let standard = SQLSessionLexicalFacts(determined: .backslashEscapesInSingleQuotes, enabled: []) + let rest = try #require(script.replanned(after: 0, session: standard)) + #expect(rest.units.map(\.sql) == ["SELECT 1"]) + } + @Test("Redis text runs one command per line or per semicolon, a quoted value kept whole") func redisSplitsIntoCommands() { #expect(plan("RPUSH jobs a\nRPUSH jobs b", .redis).units == [ diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLDelimiterDirectiveTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLDelimiterDirectiveTests.swift new file mode 100644 index 0000000000..4495d5c4c9 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLDelimiterDirectiveTests.swift @@ -0,0 +1,132 @@ +import Foundation +import TableProSQLGrammar +import Testing + +// The editor reads a DELIMITER line as the mysql client does: it runs nothing and changes what ends a statement. +struct SQLDelimiterDirectiveTests { + private static let mySQL = SQLLexicalReadings.resolve(databaseTypeId: "MySQL", declared: nil, session: nil).execution + + private func sent(_ sql: String) -> [String] { + SQLStatementScanner.executableStatements(in: sql, grammar: Self.mySQL).map(\.sql) + } + + private func atCursor(_ sql: String, on marker: String) -> String { + let offset = (sql as NSString).range(of: marker).location + return SQLStatementScanner.statementAtCursor(in: sql, cursorPosition: offset, grammar: Self.mySQL) + } + + private func delimiter(of line: String) -> String? { + let text = line as NSString + return SQLDelimiterDirective.line(startingAt: 0, in: text, length: text.length)?.delimiter + } + + @Test("A routine between DELIMITER lines is one statement, and the lines run nothing") + func routineBetweenDelimiterLines() { + let sql = "DELIMITER //\nCREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END//\nDELIMITER ;\nSELECT 3;" + #expect(sent(sql) == ["CREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END", "SELECT 3"]) + #expect(atCursor(sql, on: "SELECT 2") == "CREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END") + #expect(atCursor(sql, on: "DELIMITER ;").isEmpty) + #expect(SQLStatementScanner.navigableStatements(in: sql, grammar: Self.mySQL).count == 2) + } + + @Test("Run Current sends a delimited routine on its own, without the delimiter") + func runCurrentLeavesTheDelimiterOut() { + let sql = "DELIMITER //\nCREATE PROCEDURE p() BEGIN SELECT 1; END//\nDELIMITER ;" + let statements = SQLStatementScanner.navigableStatements(in: sql, grammar: Self.mySQL) + #expect(statements.count == 1) + let fragment = (sql as NSString).substring(with: statements[0].contentRange) + #expect(fragment == "CREATE PROCEDURE p() BEGIN SELECT 1; END") + #expect(sent(fragment) == [fragment]) + } + + @Test("mysqldump's routine, trigger and event forms each arrive whole", arguments: [ + ( + "DELIMITER ;;\nCREATE DEFINER=`root`@`localhost` PROCEDURE `proc1`(IN p INT)\nBEGIN SELECT p; SELECT 'a;;b'; END\n;;\nDELIMITER ;\n", + "CREATE DEFINER=`root`@`localhost` PROCEDURE `proc1`(IN p INT)\nBEGIN SELECT p; SELECT 'a;;b'; END" + ), + ( + "DELIMITER ;;\nCREATE DEFINER=`root`@`localhost` PROCEDURE `p`()\nBEGIN\n SELECT 1;\nEND ;;\nDELIMITER ;\n", + "CREATE DEFINER=`root`@`localhost` PROCEDURE `p`()\nBEGIN\n SELECT 1;\nEND" + ), + ( + "DELIMITER ;;\n/*!50003 CREATE*/ /*!50017 DEFINER=`root`@`localhost`*/ /*!50003 TRIGGER trg1 BEFORE INSERT ON t1 FOR EACH ROW BEGIN SET NEW.note = 'x;y'; INSERT INTO t2 VALUES (NEW.id); END \n*/;;\nDELIMITER ;\n", + "/*!50003 CREATE*/ /*!50017 DEFINER=`root`@`localhost`*/ /*!50003 TRIGGER trg1 BEFORE INSERT ON t1 FOR EACH ROW BEGIN SET NEW.note = 'x;y'; INSERT INTO t2 VALUES (NEW.id); END \n*/" + ), + ( + "DELIMITER ;;\n/*!50106 CREATE*/ /*!50106 EVENT `ev1` ON SCHEDULE EVERY 1 DAY DO BEGIN INSERT INTO t2 VALUES (1); INSERT INTO t2 VALUES (2); END \n*/ ;;\nDELIMITER ;\n", + "/*!50106 CREATE*/ /*!50106 EVENT `ev1` ON SCHEDULE EVERY 1 DAY DO BEGIN INSERT INTO t2 VALUES (1); INSERT INTO t2 VALUES (2); END \n*/" + ), + ]) + func mysqldumpForms(sql: String, expected: String) { + #expect(sent(sql) == [expected]) + } + + @Test("The delimiter ends a statement anywhere in code, inside a word too, matched case for case") + func delimiterMatching() { + #expect(sent("DELIMITER $$\nCREATE FUNCTION f() RETURNS INT BEGIN RETURN 1; END$$\nDELIMITER ;\nSELECT f();") == [ + "CREATE FUNCTION f() RETURNS INT BEGIN RETURN 1; END", "SELECT f()", + ]) + #expect(sent("DELIMITER X\nCREATE TABLE ix (a INT, INDEX(a))X\nDELIMITER ;\nSELECT 1;") == [ + "CREATE TABLE ix (a INT, INDE", "(a))", "SELECT 1", + ]) + #expect(sent("DELIMITER GO\nSELECT 'GO' go\nGO") == ["SELECT 'GO' go"]) + #expect(sent("DELIMITER //\nSELECT '//', `a//b` -- //\n# //\n/* // */ FROM t//") == [ + "SELECT '//', `a//b` -- //\n# //\n/* // */ FROM t", + ]) + } + + @Test("A ; the server would split at still ends a statement under a custom delimiter") + func semicolonStillSplitsForTheGate() { + #expect(sent("DELIMITER //\nSELECT 1; DROP TABLE t//") == ["SELECT 1", "DROP TABLE t"]) + let readings = SQLLexicalReadings.resolve(databaseTypeId: "MySQL", declared: nil, session: nil) + #expect(SQLStatementClassifier.classify( + "DELIMITER //\nSELECT 1; DROP TABLE t//", databaseTypeId: "MySQL", readings: readings + ).tier == .destructive) + } + + @Test("A DELIMITER line counts only as the first code on a line where no statement is pending") + func whereTheDirectiveCounts() { + #expect(sent(" delimiter //\nSELECT 1//\n-- note\nDeLiMiTeR ;\nSELECT 2;") == ["SELECT 1", "SELECT 2"]) + #expect(sent("SELECT 1\nDELIMITER //\nSELECT 2;") == ["SELECT 1\nDELIMITER //\nSELECT 2"]) + #expect(sent("SELECT 1; DELIMITER //\nSELECT 2;") == ["SELECT 1", "DELIMITER //\nSELECT 2"]) + #expect(sent("/* c */ DELIMITER //\nSELECT 2;") == ["/* c */ DELIMITER //\nSELECT 2"]) + #expect(sent("DELIMITER//\nSELECT 2;") == ["DELIMITER//\nSELECT 2"]) + #expect(sent("SELECT 'a\nDELIMITER //\n';") == ["SELECT 'a\nDELIMITER //\n'"]) + #expect(sent("DELIMITER //\r\nSELECT 1//\r\nDELIMITER ;\r\nSELECT 2;") == ["SELECT 1", "SELECT 2"]) + } + + @Test("The argument is read as mysql's get_arg reads it", arguments: [ + ("DELIMITER //", "//"), + ("delimiter $$ ", "$$"), + ("DELIMITER //\t", "//\t"), + ("DELIMITER //;", "//;"), + ("DELIMITER // ;", "//"), + ("DELIMITER ;;", ";;"), + ("DELIMITER // SELECT 99", "//"), + ("DELIMITER // -- set", "//"), + ("DELIMITER 'a b'", "a b"), + ("DELIMITER \"$$\"", "$$"), + ("DELIMITER `|`", "|"), + ("DELIMITER '$$", "$$"), + ("DELIMITER a\\b", "ab"), + ("DELIMITER //\r", "//"), + ]) + func argument(line: String, expected: String) { + #expect(delimiter(of: line) == expected) + } + + @Test("An empty argument or one holding a backslash is refused and the delimiter stays", arguments: [ + "DELIMITER", "DELIMITER ", "DELIMITER ''", "DELIMITER \\\\", "DELIMITER 'a\\b'", + ]) + func refusedArgument(line: String) { + #expect(delimiter(of: line) == nil) + #expect(sent("DELIMITER //\n\(line)\nSELECT 1//") == ["SELECT 1"]) + } + + @Test("Engines without the mysql client read DELIMITER as a word", arguments: ["PostgreSQL", "SQL Server", "SQLite"]) + func otherEngines(typeId: String) { + let grammar = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil).execution + let statements = SQLStatementScanner.executableStatements(in: "DELIMITER //\nSELECT 1;", grammar: grammar) + #expect(statements.map(\.sql) == ["DELIMITER //\nSELECT 1"]) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLPsqlScriptTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLPsqlScriptTests.swift new file mode 100644 index 0000000000..b5d127c657 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLPsqlScriptTests.swift @@ -0,0 +1,113 @@ +import Foundation +import TableProSQLGrammar +import Testing + +// A psql script's COPY rows and pg_dump's \restrict lines are never SQL to the editor. +struct SQLPsqlScriptTests { + private static let postgreSQL = SQLLexicalReadings.resolve(databaseTypeId: "PostgreSQL", declared: nil, session: nil) + + private func sent(_ sql: String, typeId: String = "PostgreSQL") -> [String] { + let grammar = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil).execution + return SQLStatementScanner.executableStatements(in: sql, grammar: grammar).map(\.sql) + } + + private func atCursor(_ sql: String, on marker: String) -> String { + let offset = (sql as NSString).range(of: marker).location + return SQLStatementScanner.statementAtCursor(in: sql, cursorPosition: offset, grammar: Self.postgreSQL.execution) + } + + private static let dump = """ + COPY public.t (a, b) FROM stdin; + 1\tit's; here + 2\t-- not a comment + 3\t$$ and /* nothing */ + \\. + + + SELECT 2; + """ + + @Test("The rows after COPY FROM STDIN are no statements, and the COPY is sent without them") + func rowsAreNotStatements() { + #expect(sent(Self.dump) == ["COPY public.t (a, b) FROM stdin", "SELECT 2"]) + #expect(sent("COPY t FROM stdin;\nx\tDROP TABLE users;\n") == ["COPY t FROM stdin"]) + #expect(sent("COPY t FROM stdin;", typeId: "CockroachDB") == ["COPY t FROM stdin"]) + } + + @Test("Run Current anywhere in the rows stands for the COPY") + func caretInTheRows() { + let sql = "SELECT 0;\n\(Self.dump)" + #expect(atCursor(sql, on: "2\t--") == "COPY public.t (a, b) FROM stdin") + #expect(atCursor(sql, on: "\\.") == "COPY public.t (a, b) FROM stdin") + #expect(atCursor(sql, on: "SELECT 2") == "SELECT 2") + #expect(SQLStatementScanner.navigableStatements(in: sql, grammar: Self.postgreSQL.execution).count == 3) + } + + @Test("Run Current sends the COPY's whole text, which again reads as the COPY alone", arguments: [ + Self.dump, "COPY t FROM stdin; DROP TABLE users;\n", + ]) + func runCurrentFragment(sql: String) { + let statements = SQLStatementScanner.navigableStatements(in: sql, grammar: Self.postgreSQL.execution) + let fragment = (sql as NSString).substring(with: statements[0].contentRange) + #expect(sent(fragment) == sent(sql).prefix(1).map { $0 }) + #expect(sent(fragment).count == 1) + } + + @Test("The rows end at the first line holding only \\.", arguments: [ + ("COPY t FROM stdin;\n\\. \n\\.\nSELECT 7;", ["COPY t FROM stdin", "SELECT 7"]), + ("COPY t FROM stdin;\n x\\.\n\\.\r\nSELECT 7;", ["COPY t FROM stdin", "SELECT 7"]), + ("COPY t FROM STDIN WITH (FORMAT csv);\n1,\"a;b\"\n\\.", ["COPY t FROM STDIN WITH (FORMAT csv)"]), + ("COPY t FROM stdin;\n1\ta\nSELECT 7;", ["COPY t FROM stdin"]), + ("COPY t FROM stdin; SELECT 8;\n1\ta\n\\.\nSELECT 7;", ["COPY t FROM stdin", "SELECT 7"]), + ("COPY t FROM stdin; DROP TABLE users;", ["COPY t FROM stdin"]), + ("COPY t FROM stdin; DROP TABLE users;\n", ["COPY t FROM stdin"]), + ]) + func endOfRows(sql: String, expected: [String]) { + #expect(sent(sql) == expected) + } + + @Test("Only psql's COPY FROM STDIN takes rows from the script", arguments: [ + "COPY t FROM 'stdin';\nSELECT 1;", + "COPY (SELECT 1) TO STDOUT;\nSELECT 1;", + "COPY t TO STDOUT;\nSELECT 1;", + "SELECT 1 FROM stdin;\nSELECT 1;", + ]) + func noRows(sql: String) { + #expect(sent(sql).count == 2) + } + + @Test("Every form psql reads rows for takes them", arguments: [ + "COPY t FROM STDIN;", "copy t from stdin;", "COPY BINARY t FROM STDIN;", "COPY db.sch.t (a) FROM STDOUT;", + "COPY t FROM stdin WHERE a > 1;", "COPY \"t\" FROM stdin;", "COPY t (a, b, c, d, e, f, g) FROM STDIN;", + ]) + func rowForms(header: String) { + #expect(sent("\(header)\nSELECT 1;\n\\.\nSELECT 2;").count == 2) + } + + @Test("pg_dump's \\restrict and \\unrestrict lines run nothing") + func restrictLines() { + let sql = "\\restrict abc123\n\nSET statement_timeout = 0;\nSELECT 6;\n\\unrestrict abc123\n" + #expect(sent(sql) == ["SET statement_timeout = 0", "SELECT 6"]) + #expect(sent("SELECT 1\n\\restrict abc;") == ["SELECT 1\n\\restrict abc"]) + #expect(sent("\\connect db\nSELECT 1;") == ["\\connect db\nSELECT 1"]) + #expect(sent("\\RESTRICT abc\nSELECT 1;") == ["\\RESTRICT abc\nSELECT 1"]) + } + + @Test("The gate reads the COPY alone, and an engine TablePro does not know reads the rows as text") + func gate() { + #expect(SQLStatementClassifier.classify(Self.dump, databaseTypeId: "PostgreSQL", readings: Self.postgreSQL).tier + != .safe) + let unknown = SQLLexicalReadings.resolve(databaseTypeId: "Unknown", declared: nil, session: nil) + #expect(!unknown.all.contains { $0.contains(.psqlScripts) }) + let rows = "COPY t FROM stdin;\nx\tDROP TABLE users;\n" + #expect(SQLStatementScanner.executableStatements(in: rows, grammar: unknown.execution).count == 2) + } + + @Test("The text psql holds as no SQL is reported for the formatter") + func scriptText() { + let sql = "\\restrict k\nCOPY t FROM stdin;\n1\ta\n\\.\nSELECT 1;" + let ranges = SQLStatementScanner.psqlScriptText(in: sql, grammar: Self.postgreSQL.execution) + #expect(ranges.map { (sql as NSString).substring(with: $0) } == ["\\restrict k", "COPY t FROM stdin;\n1\ta\n\\."]) + #expect(SQLStatementScanner.psqlScriptText(in: sql, grammar: SQLLexicalProfile.everyKnownReading[0]).isEmpty) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift index 79bcaf44ed..270d9f37f8 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/SQLStatementCursorIndexTests.swift @@ -21,6 +21,11 @@ private let cursorIndexFixtures = [ /* c; */ SELECT E'x' 'y';; CREATE RULE r AS ON INSERT TO t DO ALSO (INSERT INTO a VALUES (1); INSERT INTO b VALUES (2)); + \\restrict k1 + COPY t (a, b) FROM stdin; + 1\tit's; here + 2\t$$ -- x + \\. SELECT 2 """), CursorIndexFixture(typeId: "MySQL", text: """ @@ -32,6 +37,12 @@ private let cursorIndexFixtures = [ SELECT 3;; # hash; comment SELECT `a;b` FROM t; + DELIMITER // + CREATE PROCEDURE q() BEGIN SELECT 1; END// + SELECT 'x//'; SELECT 4// + DELIMITER ;; + CREATE TRIGGER g BEFORE INSERT ON t FOR EACH ROW BEGIN SET NEW.a = 1; END ;; + DELIMITER ; SELECT /*! 1; */ 2 """), CursorIndexFixture(typeId: "SQL Server", text: """ @@ -54,6 +65,10 @@ private let cursorIndexFixtures = [ SELECT (5; 6); SET XACT_ABORT ON BEGIN TRY SELECT 7; END TRY BEGIN CATCH THROW; END CATCH + IF @y = 1 SELECT 8; -- then + ELSE IF @z = 1 SELECT 9; ELSE SELECT 10; + DROP TABLE IF EXISTS t;; SELECT 11 + IF @w = 1 SELECT 12;; ELSE SELECT 13 """), CursorIndexFixture(typeId: "Oracle", text: """ BEGIN @@ -81,7 +96,7 @@ private let cursorIndexFixtures = [ private let cursorIndexInsertions = [ ";", "'", "\nGO\n", "\n/\n", " ", "x", "$$", "BEGIN ", "END;", "--", "/*", "*/", "\n", "GO", "/", "(", ")", - "CASE ", "END ", + "CASE ", "END ", "ELSE ", "IF ", "DELIMITER ", "//", "\\.", "COPY t FROM stdin;\n", "\\restrict x\n", ] private let longScriptRow = "SELECT id, name FROM users WHERE active = true;\n" diff --git a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift index 271bc67937..cd56a2f527 100644 --- a/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProSQLGrammarTests/TSQLControlFlowBlockTests.swift @@ -144,6 +144,63 @@ struct TSQLControlFlowBlockTests { #expect(!SQLUnterminatedStatements.blockHoldsSeveralStatements(sql, grammar: postgreSQL)) } + @Test("A ; that ends an IF's body before its ELSE stays inside the IF", arguments: [ + ( + "IF DATENAME(weekday, GETDATE()) IN (N'Saturday', N'Sunday')\n SELECT 'Weekend';\nELSE\n SELECT 'Weekday';", + ["IF DATENAME(weekday, GETDATE()) IN (N'Saturday', N'Sunday')\n SELECT 'Weekend';\nELSE\n SELECT 'Weekday'"] + ), + ("IF @a = 1 SELECT 1; -- one\nELSE SELECT 2; SELECT 3", ["IF @a = 1 SELECT 1; -- one\nELSE SELECT 2", "SELECT 3"]), + ( + "IF @a = 1 IF @b = 1 SELECT 1; ELSE SELECT 2; ELSE SELECT 3; SELECT 4", + ["IF @a = 1 IF @b = 1 SELECT 1; ELSE SELECT 2; ELSE SELECT 3", "SELECT 4"] + ), + ( + "IF @a = 1 SELECT 1; ELSE IF @b = 1 SELECT 2; ELSE SELECT 3; SELECT 4", + ["IF @a = 1 SELECT 1; ELSE IF @b = 1 SELECT 2; ELSE SELECT 3", "SELECT 4"] + ), + ("WHILE @i < 3 IF @i = 1 BREAK; ELSE SET @i += 1; SELECT @i", ["WHILE @i < 3 IF @i = 1 BREAK; ELSE SET @i += 1", "SELECT @i"]), + ("IF @a = 1 BEGIN SELECT 1; END; ELSE SELECT 2; SELECT 3", ["IF @a = 1 BEGIN SELECT 1; END; ELSE SELECT 2", "SELECT 3"]), + ("IF @a = 1 SELECT 1;; ELSE SELECT 2", ["IF @a = 1 SELECT 1;; ELSE SELECT 2"]), + ("IF @a = 1 \(Self.merge); ELSE SELECT 2;", ["IF @a = 1 \(Self.merge); ELSE SELECT 2"]), + ("IF @a = 1 SELECT 1; ELSE \(Self.merge);", ["IF @a = 1 SELECT 1; ELSE \(Self.merge);"]), + ]) + func semicolonBeforeElse(sql: String, expected: [String]) { + #expect(sent(sql) == expected) + #expect(!SQLUnterminatedStatements.blockHoldsSeveralStatements(expected[0], grammar: Self.sqlServer)) + } + + @Test("A ; after an IF's body ends the IF when no ELSE follows", arguments: [ + ("IF @a = 1 SELECT 1; SELECT 2;", ["IF @a = 1 SELECT 1", "SELECT 2"]), + ("IF @a = 1 SELECT 1;\n", ["IF @a = 1 SELECT 1"]), + ("IF @a = 1 SELECT 1;; SELECT 2", ["IF @a = 1 SELECT 1", "SELECT 2"]), + ("IF @a = 1 SELECT 1; [ELSE]", ["IF @a = 1 SELECT 1", "[ELSE]"]), + ("IF @a = 1 SELECT 1; ELSEWHERE", ["IF @a = 1 SELECT 1", "ELSEWHERE"]), + ("IF @a = 1 SELECT 1 ELSE SELECT 2; ELSE SELECT 3", ["IF @a = 1 SELECT 1 ELSE SELECT 2", "ELSE SELECT 3"]), + ("SELECT 1; ELSE SELECT 2", ["SELECT 1", "ELSE SELECT 2"]), + ("DROP TABLE IF EXISTS t; SELECT 1", ["DROP TABLE IF EXISTS t", "SELECT 1"]), + ( + "IF @a = 1 SELECT CASE WHEN @b = 1 THEN 1 ELSE 2 END; SELECT 3", + ["IF @a = 1 SELECT CASE WHEN @b = 1 THEN 1 ELSE 2 END", "SELECT 3"] + ), + ]) + func semicolonWithoutElse(sql: String, expected: [String]) { + #expect(sent(sql) == expected) + } + + @Test("Run Current anywhere in an IF ... ELSE runs the whole IF, and the gate tiers both branches") + func caretOnTheElseRunsTheIf() { + let statement = "IF @env = 'dev'\n SELECT 1;\nELSE\n DROP TABLE dbo.t" + let sql = "SELECT 0;\n\(statement);\nSELECT 9;" + + #expect(atCursor(sql, on: "ELSE") == statement) + #expect(atCursor(sql, on: "DROP") == statement) + #expect(atCursor(sql, on: "SELECT 9") == "SELECT 9") + #expect(SQLStatementScanner.navigableStatements(in: sql, grammar: Self.sqlServer).count == 3) + + let readings = SQLLexicalReadings.resolve(databaseTypeId: "SQL Server", declared: nil, session: nil) + #expect(SQLStatementClassifier.classify(statement, databaseTypeId: "SQL Server", readings: readings).tier == .destructive) + } + @Test("Other engines still read BEGIN outside a routine as a word", arguments: ["PostgreSQL", "MySQL", "SQLite"]) func otherEnginesAreUnchanged(typeId: String) { let grammar = SQLLexicalReadings.resolve(databaseTypeId: typeId, declared: nil, session: nil).execution diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/Folding/SQLFoldScanner.swift b/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/Folding/SQLFoldScanner.swift index 30a7ce249d..4806926b3f 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/Folding/SQLFoldScanner.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProUtilities/SQL/Folding/SQLFoldScanner.swift @@ -40,6 +40,12 @@ private struct Scan { /// in the same gutter agree about where a statement stops. private var boundaries: any SQLStatementBoundaryTracking + // A `;` that ended the statement while the tracker could still take it back at the next token, and its line. + private var heldEnd: (index: Int, line: Int)? + + // The client delimiter a `DELIMITER` line set, which ends a statement wherever it stands in code. + private var delimiter: SQLClientDelimiter? + init(text: NSString, grammar: SQLLexicalGrammar) { self.text = text self.length = text.length @@ -53,16 +59,33 @@ private struct Scan { while index < length { step() } + settleHeldEnd(word: nil) closeFramesAtEndOfDocument() } + private mutating func settleHeldEnd(word: String?) { + guard let held = heldEnd else { return } + heldEnd = nil + guard !boundaries.continuesStatement(with: word) else { return } + boundaries.reset() + if frames.last?.kind == .statement { + popFrame(.statement, end: held.index, endLine: held.line) + } + } + private mutating func step() { let character = text.character(at: index) if consumeTrivia(character) { return } + if consumeClientLine(character) { return } + if consumeDelimiter() { return } if consumeNonCode() { return } if consumeSlashLine(character) { return } + if heldEnd != nil, character != SqlLexer.semicolon { + let word = SqlBlockStructure.readKeyword(text, at: index, length: length, grammar: grammar).text + settleHeldEnd(word: word.isEmpty ? nil : word) + } openStatementIfNeeded() consumeStructure(character) } @@ -161,6 +184,22 @@ private struct Scan { /// frame of their own: when it ends the unit, whatever the fold still holds open ends with it. private mutating func consumeSemicolon() { let endsStatement = boundaries.observeSemicolon() + if heldEnd != nil || (endsStatement && boundaries.awaitsContinuation) { + heldEnd = heldEnd ?? (index, line) + index += 1 + return + } + // The rows after a `COPY ... FROM STDIN` belong to it, so its fold runs through them. + if endsStatement, boundaries.copiesFromStandardInput { + let rows = SQLPsqlScript.copyRows(afterSemicolonAt: index, in: text, length: length) + boundaries.reset() + line += newlines(in: rows) + index = NSMaxRange(rows) + if frames.last?.kind == .statement { + popFrame(.statement, end: index) + } + return + } if endsStatement { boundaries.reset() if grammar.contains(.plsqlBlocks) { @@ -182,7 +221,8 @@ private struct Scan { } private mutating func consumeKeyword(_ character: UInt16) { - let word = SqlBlockStructure.readKeyword(text, at: index, length: length, grammar: grammar) + let read = SqlBlockStructure.readKeyword(text, at: index, length: length, grammar: grammar) + let word = read.text.isEmpty ? read : delimiter?.cut(read, readAt: index, in: text, length: length) ?? read guard !word.text.isEmpty else { if !SqlLexer.isWhitespace(character) { boundaries.observeSymbol(character) @@ -211,6 +251,50 @@ private struct Scan { } } + // MARK: - Client lines + + /// A `DELIMITER` line or a psql `\restrict` line, where no statement is open, runs nothing and folds nothing. + private mutating func consumeClientLine(_ character: UInt16) -> Bool { + guard frames.isEmpty else { return false } + if grammar.contains(.delimiterDirective), character | 0x20 == Self.smallD, + let directive = SQLDelimiterDirective.line(at: index, in: text, length: length) { + if let set = directive.delimiter { + delimiter = SQLClientDelimiter(set) + } + index = NSMaxRange(directive.range) + return true + } + if grammar.contains(.psqlScripts), character == SqlLexer.backslash, + let restrict = SQLPsqlScript.restrictLine(at: index, in: text, length: length) { + index = NSMaxRange(restrict) + return true + } + return false + } + + /// A client delimiter ends the statement whatever the fold still holds open, as it does for the mysql client. + private mutating func consumeDelimiter() -> Bool { + guard let delimiter, delimiter.starts(in: text, at: index, length: length) else { return false } + settleHeldEnd(word: nil) + closeFramesThroughStatement(end: index) + if frames.last?.kind == .statement { + popFrame(.statement, end: index) + } + boundaries.reset() + index += delimiter.units.count + return true + } + + private func newlines(in range: NSRange) -> Int { + var count = 0 + for offset in range.location.. Bool { @@ -234,6 +318,8 @@ private struct Scan { let startLine = line if span.kind.isComment { boundaries.observeGap() + } else { + settleHeldEnd(word: nil) } switch span.kind { case .lineComment: diff --git a/apple/TablePro/Core/Autocomplete/SQLSchemaProvider.swift b/apple/TablePro/Core/Autocomplete/SQLSchemaProvider.swift index a256b5de58..398bcff33c 100644 --- a/apple/TablePro/Core/Autocomplete/SQLSchemaProvider.swift +++ b/apple/TablePro/Core/Autocomplete/SQLSchemaProvider.swift @@ -1,10 +1,3 @@ -// -// SQLSchemaProvider.swift -// TablePro -// -// Cached database schema provider for autocomplete -// - import Foundation import os import TableProAutocomplete @@ -63,20 +56,27 @@ actor SQLSchemaProvider { let fetchAllColumns: @Sendable () async throws -> [String: [ColumnInfo]] let fetchSchemaTables: (@Sendable (_ schema: String) async throws -> [TableInfo])? let sampleFieldPaths: (@Sendable (_ table: String, _ limit: Int) async throws -> [PluginFieldPath])? + let fetchForeignKeys: (@Sendable (_ table: String, _ schema: String?) async throws -> [ForeignKeyInfo])? init( fetchColumns: @escaping @Sendable (_ table: String, _ schema: String?) async throws -> [ColumnInfo], fetchAllColumns: @escaping @Sendable () async throws -> [String: [ColumnInfo]], fetchSchemaTables: (@Sendable (_ schema: String) async throws -> [TableInfo])? = nil, - sampleFieldPaths: (@Sendable (_ table: String, _ limit: Int) async throws -> [PluginFieldPath])? = nil + sampleFieldPaths: (@Sendable (_ table: String, _ limit: Int) async throws -> [PluginFieldPath])? = nil, + fetchForeignKeys: (@Sendable (_ table: String, _ schema: String?) async throws -> [ForeignKeyInfo])? = nil ) { self.fetchColumns = fetchColumns self.fetchAllColumns = fetchAllColumns self.fetchSchemaTables = fetchSchemaTables self.sampleFieldPaths = sampleFieldPaths + self.fetchForeignKeys = fetchForeignKeys } } + // Keyed like the column cache and cleared with it, so a table's keys are read once per table list. + private var foreignKeyCache: [ColumnCacheKey: [SQLCompletionForeignKey]] = [:] + private var foreignKeyTasks: [ColumnCacheKey: Task<[SQLCompletionForeignKey]?, Never>] = [:] + private var fieldPathCache: [String: [PluginFieldPath]] = [:] private var fieldPathTasks: [String: Task<[PluginFieldPath], Never>] = [:] @@ -116,7 +116,6 @@ actor SQLSchemaProvider { // MARK: - Public API - /// Load schema from the database (driver should already be connected). /// Concurrent callers await the same in-flight Task instead of firing duplicate queries. func loadSchema(using driver: DatabaseDriver, connection: DatabaseConnection? = nil) async { if let existing = loadTask { @@ -155,17 +154,14 @@ actor SQLSchemaProvider { noteTableListWritten() } - /// Get the current connection info func getConnectionInfo() -> DatabaseConnection? { connectionInfo } - /// Get all tables func getTables() -> [TableInfo] { tables } - /// Get columns for a specific table (with LRU caching) func getColumns(for tableName: String, schema: String? = nil) async -> [ColumnInfo] { let key = cacheKey(table: tableName, schema: schema) @@ -210,7 +206,60 @@ actor SQLSchemaProvider { while columnAccessOrder.count > Self.maxCachedTables { let evicted = columnAccessOrder.removeFirst() columnCache.removeValue(forKey: evicted) + foreignKeyCache.removeValue(forKey: evicted) + } + } + + // Read after the ON of a join, never per keystroke: concurrent callers await the read in flight, and a failed read + // is kept as no keys until the table list is read again, so a catalog that refuses it is not asked on every key. + func foreignKeys(of tableName: String, schema: String? = nil) async -> [SQLCompletionForeignKey] { + let key = cacheKey(table: tableName, schema: schema) + if let cached = foreignKeyCache[key] { return cached } + if let inFlight = foreignKeyTasks[key] { return await inFlight.value ?? [] } + guard connectionInfo?.type.supportsForeignKeys != false else { return [] } + + let readSchema = schema.map { catalogSpelling(ofSchema: $0, isQuoted: false) ?? $0 } + let read: @Sendable () async throws -> [ForeignKeyInfo] + if let source = metadataSource { + guard let fetch = source.fetchForeignKeys else { return [] } + read = { try await fetch(tableName, readSchema) } + } else if let driver = cachedDriver { + guard driver.connection.type.supportsForeignKeys else { return [] } + read = { try await driver.fetchForeignKeys(table: tableName, schema: readSchema) } + } else { + return [] + } + + let task = Task<[SQLCompletionForeignKey]?, Never> { + do { + let rows = try await read() + return Self.completionForeignKeys(rows) + } catch { + Self.logger.debug( + "[schema] foreign key read failed for completion: \(error.publicLogShape, privacy: .public)" + ) + return nil + } } + foreignKeyTasks[key] = task + let fetched = await task.value + guard foreignKeyTasks[key] == task else { return fetched ?? [] } + foreignKeyTasks[key] = nil + let keys = fetched ?? [] + foreignKeyCache[key] = keys + return keys + } + + private static func completionForeignKeys(_ rows: [ForeignKeyInfo]) -> [SQLCompletionForeignKey] { + SQLCompletionForeignKey.grouping(rows.map { + SQLCompletionForeignKey.Row( + name: $0.name, + column: $0.column, + referencedTable: $0.referencedTable, + referencedSchema: $0.referencedSchema, + referencedColumn: $0.referencedColumn + ) + }) } /// A lookup that names no schema reads the schema an unqualified fetch targets, which is the one @@ -255,6 +304,8 @@ actor SQLSchemaProvider { self.columnCache.removeAll() self.columnAccessOrder.removeAll() self.columnTasks.removeAll() + self.foreignKeyCache.removeAll() + self.foreignKeyTasks.removeAll() self.fieldPathCache.removeAll() self.fieldPathTasks.removeAll() self.onDemandSchemaTables.removeAll() @@ -272,6 +323,8 @@ actor SQLSchemaProvider { columnCache.removeAll() columnAccessOrder.removeAll() columnTasks.removeAll() + foreignKeyCache.removeAll() + foreignKeyTasks.removeAll() fieldPathCache.removeAll() fieldPathTasks.removeAll() } @@ -410,7 +463,6 @@ actor SQLSchemaProvider { await task.value } - /// Find table name from alias func resolveAlias(_ aliasOrName: String, in references: [TableReference]) -> String? { let lowerName = aliasOrName.lowercased() @@ -663,7 +715,6 @@ actor SQLSchemaProvider { return [] } - /// Get completion items for all columns of tables in scope func allColumnsInScope(for references: [TableReference]) async -> [SQLCompletionItem] { // swiftlint:disable:next large_tuple var itemDataBuilder: [( diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift index 7006e2f157..f0d0f2a9e8 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+MultiStatement.swift @@ -39,15 +39,12 @@ extension QueryExecutionCoordinator { // The rest of a run, cut again once a statement changed how the session reads literals, in tab coordinates. nonisolated static func remainingStatements( in text: String, - after ran: NSRange, + after ran: SQLStatementScanner.ExecutableStatement, through end: Int, grammar: SQLLexicalGrammar ) -> [SQLStatementScanner.ExecutableStatement] { - let source = text as NSString - let start = NSMaxRange(ran) - guard start < end, end <= source.length else { return [] } - let rest = source.substring(with: NSRange(location: start, length: end - start)) - return SQLStatementScanner.executableStatements(in: rest, grammar: grammar).map { $0.offset(by: start) } + guard end <= (text as NSString).length else { return [] } + return SQLStatementScanner.executableStatements(in: text, after: ran, through: end, grammar: grammar) } // False once the editor's text no longer holds each statement where the run found it. diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift index 0520265783..1fe3f0132e 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift @@ -44,6 +44,8 @@ private struct PreparedStatement: @unchecked Sendable { let parameterValues: [Any?]? let rowCap: Int? let anchor: StatementAnchor? + /// Where the script was cut after this statement, which a replan of the rest starts from. + let statement: SQLStatementScanner.ExecutableStatement? /// Whether this is the script's own `COMMIT`, read from the text before the lease is taken so /// the run never lexes inside it. let isCommitPoint: Bool @@ -74,10 +76,10 @@ private final class RunningStatements { func replanned(after ran: Int, session: SQLLexicalGrammar) -> [PreparedStatement]? { guard session != grammar, let script, prepared.indices.contains(ran - 1), - let last = prepared[ran - 1].anchor + let last = prepared[ran - 1].statement else { return nil } let rest = QueryExecutionCoordinator.remainingStatements( - in: script, after: last.range, through: end, grammar: session + in: script, after: last, through: end, grammar: session ).map { prepare($0, session) } grammar = session prepared = Array(prepared.prefix(ran)) + rest @@ -521,6 +523,7 @@ extension QueryExecutionCoordinator { parameterValues: conversion?.values, rowCap: bounded.rowCap, anchor: StatementAnchor(statement), + statement: statement, isCommitPoint: BatchCommitStatement.matches(sql, grammar: grammar) ) } diff --git a/apple/TablePro/Core/Services/Formatting/SQLFormatterService.swift b/apple/TablePro/Core/Services/Formatting/SQLFormatterService.swift index deddba5282..ebf788e835 100644 --- a/apple/TablePro/Core/Services/Formatting/SQLFormatterService.swift +++ b/apple/TablePro/Core/Services/Formatting/SQLFormatterService.swift @@ -261,7 +261,14 @@ internal struct SQLTokenFormatter { directiveLines: directiveLines ) } - return text.trimmingCharacters(in: .whitespacesAndNewlines) + guard stream.last?.type == .directive else { return text.trimmingCharacters(in: .whitespacesAndNewlines) } + // A directive's text stays as written to its end: a COPY's last row can end in a tab that is a column. + let start = text.firstIndex { !$0.isWhitespace } ?? text.endIndex + var end = text.endIndex + while end > start, text[text.index(before: end)] == "\n" { + end = text.index(before: end) + } + return String(text[start.. = Set("(),;.".utf16) private static let operatorUnits: Set = Set("=<>+-*/%&|^~!@#?:$".utf16) - private static let delimiterWord = "DELIMITER" static func isOperatorUnit(_ unit: UInt16) -> Bool { operatorUnits.contains(unit) @@ -329,6 +328,7 @@ private struct Lexer { let grammar: SQLLexicalGrammar let tokenizer: SQLTokenizer private let batchSeparatorEnds: [Int: Int] + private let psqlTextEnds: [Int: Int] private var delimiter = ";" private var atStatementStart = true private var tokens: [SQLToken] = [] @@ -344,6 +344,11 @@ private struct Lexer { ends[separator.range.location] = NSMaxRange(separator.range) } self.batchSeparatorEnds = ends + var psqlEnds: [Int: Int] = [:] + for range in SQLStatementScanner.psqlScriptText(in: text as String, grammar: grammar) { + psqlEnds[range.location] = NSMaxRange(range) + } + self.psqlTextEnds = psqlEnds } mutating func run() -> [SQLToken] { @@ -356,20 +361,21 @@ private struct Lexer { private mutating func step() { let unit = text.character(at: index) + // A `GO` line, a `COPY ... FROM STDIN` with its rows and a psql `\restrict` line keep their text. + if let end = batchSeparatorEnds[index] ?? psqlTextEnds[index] { + let start = index + index = end + emit(.directive, from: start) + return + } if isBlank(unit) { let start = index - while index < length, isBlank(text.character(at: index)) { + while index < length, isBlank(text.character(at: index)), psqlTextEnds[index] == nil { index += 1 } emit(.whitespace, from: start) return } - if let end = batchSeparatorEnds[index] { - let start = index - index = end - emit(.directive, from: start) - return - } if unit == Self.slash, grammar.contains(.slashLineTerminators), SQLStatementScanner.isSlashLine(text, at: index, length: length) { let start = index @@ -440,23 +446,16 @@ private struct Lexer { // `DELIMITER //` at the start of a statement, on a line of its own, changes what ends a statement, as the mysql // client and the import read a script. private mutating func readDelimiterDirective() -> Bool { - guard grammar.contains(.delimiterDirective), atStatementStart, isAtLineStart(index) else { return false } - let wordLength = (Self.delimiterWord as NSString).length - guard index + wordLength < length, - substring(from: index, to: index + wordLength).uppercased() == Self.delimiterWord, - isLineBlank(text.character(at: index + wordLength)) + guard grammar.contains(.delimiterDirective), atStatementStart, isAtLineStart(index), + let directive = SQLDelimiterDirective.line(startingAt: index, in: text, length: length) else { return false } - var end = index + wordLength - while end < length, text.character(at: end) != Self.newline { - end += 1 + let end = NSMaxRange(directive.range) + if let newDelimiter = directive.delimiter { + delimiter = newDelimiter } - let line = substring(from: index, to: end).trimmingCharacters(in: .whitespaces) - let newDelimiter = String(line.dropFirst(Self.delimiterWord.count)).trimmingCharacters(in: .whitespaces) - guard !newDelimiter.isEmpty else { return false } - delimiter = newDelimiter - append(SQLToken(type: .directive, value: line)) + append(SQLToken(type: .directive, value: substring(from: index, to: end).trimmingCharacters(in: .whitespaces))) index = end return true } diff --git a/apple/TablePro/Core/Services/Query/SchemaProviderRegistry.swift b/apple/TablePro/Core/Services/Query/SchemaProviderRegistry.swift index 1b86aeb4b8..fbe27536cb 100644 --- a/apple/TablePro/Core/Services/Query/SchemaProviderRegistry.swift +++ b/apple/TablePro/Core/Services/Query/SchemaProviderRegistry.swift @@ -1,11 +1,3 @@ -// -// SchemaProviderRegistry.swift -// TablePro -// -// Manages shared SQLSchemaProvider instances, one per DatabaseScope. -// Ref-counted per connection with grace period removal to avoid redundant schema loads. -// - import Foundation import os import TableProSQLGrammar @@ -106,6 +98,32 @@ final class SchemaProviderRegistry: CatalogChangeTarget { try await metadataProvider.withMetadataDriver(scope: scope) { driver in try await driver.sampleFieldPaths(table: table, limit: limit) } + }, + fetchForeignKeys: { table, schema in + try await metadataProvider.withMetadataDriver(scope: scope) { driver -> [ForeignKeyInfo] in + let type = driver.connection.type + guard type.supportsForeignKeys else { return [] } + let keys = try await driver.fetchForeignKeys(table: table, schema: schema) + // A key into another schema would pair with a same-named table of this one. On an engine with no + // schema layer a qualifier names a database. With no schema named, the keys come from the + // driver's current one, and a scope with no schema would keep keys into every schema. + let slot = await MainActor.run { EngineNamespaceSlot(databaseType: type) } + let origin = slot == .database + ? DatabaseScope(connectionId: scope.connectionId, database: schema ?? scope.database, schema: nil) + : DatabaseScope( + connectionId: scope.connectionId, + database: scope.database, + schema: schema ?? scope.schema ?? (driver as? SchemaSwitchable)?.currentSchema + ) + return keys.filter { key in + ForeignKeyTargetScope.pointsWithin( + origin, + referencedDatabase: key.referencedDatabase, + referencedSchema: key.referencedSchema, + slot: slot + ) + } + } } ) let provider = SQLSchemaProvider(metadataSource: source) diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser+Literals.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser+Literals.swift new file mode 100644 index 0000000000..bdf0e0b269 --- /dev/null +++ b/apple/TablePro/Core/Utilities/SQL/SQLFileParser+Literals.swift @@ -0,0 +1,235 @@ +import Foundation +import TableProSQLGrammar + +// A literal or quoted name runs to its closer here, and stops at the end of a chunk when the next one decides it. +extension SQLFileParser { + static func processAlternativeQuote( + _ ctx: inout ParserContext, + i: inout Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult { + let start = i + var pos = i + while pos < bufLen { + let ch = nsBuffer.character(at: pos) + if pos > start && ch == kNewline { + ctx.currentLine += 1 + } + if ch == ctx.alternativeQuoteCloser { + let quoteAt = pos + (ctx.alternativeQuoteCloserTrail == nil ? 1 : 2) + if quoteAt >= bufLen { + if ctx.atEndOfInput { + pos += 1 + continue + } + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + let trailMatches = ctx.alternativeQuoteCloserTrail.map { nsBuffer.character(at: pos + 1) == $0 } ?? true + if trailMatches, nsBuffer.character(at: quoteAt) == kSingleQuote { + pos = quoteAt + 1 + ctx.state = .normal + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + } + pos += 1 + } + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + + /// Runs a triple-quoted literal to the three quotes that close it, holding back a quote or a backslash the buffer + /// ends on until the next chunk shows what follows. + static func processTripleQuotedString( + _ ctx: inout ParserContext, + i: inout Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult { + let start = i + var pos = i + while pos < bufLen { + let ch = nsBuffer.character(at: pos) + if pos > start && ch == kNewline { + ctx.currentLine += 1 + } + if ctx.backslashEscapesActive && ch == kBackslash { + guard pos + 1 < bufLen || ctx.atEndOfInput else { + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + pos += 2 + continue + } + if ch == ctx.quoteChar { + guard pos + 2 < bufLen || ctx.atEndOfInput else { + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + if SqlLexer.startsTripleQuote(nsBuffer, at: pos, length: bufLen) { + pos += 3 + ctx.state = .normal + ctx.backslashEscapesActive = false + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + } + pos += 1 + } + appendRange(&ctx, from: start, to: min(pos, bufLen), in: nsBuffer) + i = min(pos, bufLen) + return StepResult(advanced: true, deferred: false) + } + + /// Runs a `[...]` identifier to its `]`, which `]]` does not close where the grammar escapes it. + static func processBracketedIdentifier( + _ ctx: inout ParserContext, + i: inout Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult { + let start = i + var pos = i + let doubledEscapes = ctx.grammar.contains(.doubledClosingBracketEscapes) + while pos < bufLen { + let ch = nsBuffer.character(at: pos) + if pos > start && ch == kNewline { + ctx.currentLine += 1 + } + guard ch == kCloseBracket else { + pos += 1 + continue + } + if doubledEscapes { + guard pos + 1 < bufLen || ctx.atEndOfInput else { + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + if pos + 1 < bufLen, nsBuffer.character(at: pos + 1) == kCloseBracket { + pos += 2 + continue + } + } + pos += 1 + ctx.state = .normal + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + + static func processQuotedString( + _ ctx: inout ParserContext, + quoteChar: unichar, + i: inout Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult { + let start = i + var pos = i + let escapesActive = ctx.backslashEscapesActive + + while pos < bufLen { + let ch = nsBuffer.character(at: pos) + if pos > start && ch == kNewline { + ctx.currentLine += 1 + } + + if escapesActive && ch == kBackslash { + if pos + 1 >= bufLen { + if ctx.atEndOfInput { + pos += 1 + continue + } + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + let next = nsBuffer.character(at: pos + 1) + if next == kNewline { ctx.currentLine += 1 } + pos += 2 + continue + } + + if ch == quoteChar { + if pos + 1 >= bufLen && !ctx.atEndOfInput { + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + let next: unichar? = pos + 1 < bufLen ? nsBuffer.character(at: pos + 1) : nil + if next == quoteChar { + pos += 2 + continue + } + pos += 1 + ctx.state = .normal + ctx.backslashEscapesActive = false + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + + pos += 1 + } + + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + + static func processDollarQuote( + _ ctx: inout ParserContext, + i: inout Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult { + let start = i + var pos = i + let closeLen = (ctx.dollarTag as NSString).length + 2 + + while pos < bufLen { + let ch = nsBuffer.character(at: pos) + if pos > start && ch == kNewline { + ctx.currentLine += 1 + } + + if ch == kDollar { + if pos + closeLen > bufLen { + if ctx.atEndOfInput { + pos += 1 + continue + } + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: true) + } + if SqlDollarQuote.matchesClose(at: pos, tag: ctx.dollarTag, in: nsBuffer, bufLen: bufLen) { + pos += closeLen + ctx.state = .normal + ctx.dollarTag = "" + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } + } + pos += 1 + } + + appendRange(&ctx, from: start, to: pos, in: nsBuffer) + i = pos + return StepResult(advanced: true, deferred: false) + } +} diff --git a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift index 4a4445a695..24108fde77 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLFileParser.swift @@ -9,6 +9,23 @@ import TableProPluginKit import TableProSQLGrammar import TableProUtilities +/// A file the import refuses before it sends the statement that needs what it cannot give. +enum SQLFileParseError: LocalizedError, Equatable { + case copyFromStandardInput(line: Int) + + var errorDescription: String? { + switch self { + case .copyFromStandardInput(let line): + return String( + format: String( + localized: "Line %lld runs COPY FROM STDIN, whose rows TablePro cannot send. Run the file with psql, or dump it again with --inserts." + ), + line + ) + } + } +} + final class SQLFileParser: Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "SQLFileParser") @@ -27,15 +44,17 @@ final class SQLFileParser: Sendable { /// One statement the file holds, or on a grammar that cuts scripts into batches, one batch, with how many times /// the script runs it. - private struct ParsedStatement { + struct ParsedStatement { let statement: String let lineNumber: Int let repeatCount: Int } - private enum ParserState { + enum ParserState { case normal case inSingleLineComment + /// A psql `\restrict` line, which runs to its line feed whatever ends a comment. + case inClientLine case inMultiLineComment case inSingleQuotedString case inDoubleQuotedString @@ -47,24 +66,24 @@ final class SQLFileParser: Sendable { } private static let kSemicolon: unichar = 0x3B - private static let kSingleQuote: unichar = 0x27 + static let kSingleQuote: unichar = 0x27 private static let kDoubleQuote: unichar = 0x22 private static let kBacktick: unichar = 0x60 - private static let kBackslash: unichar = 0x5C + static let kBackslash: unichar = 0x5C private static let kDash: unichar = 0x2D private static let kSlash: unichar = 0x2F private static let kStar: unichar = 0x2A private static let kHash: unichar = 0x23 - private static let kNewline: unichar = 0x0A + static let kNewline: unichar = 0x0A private static let kSpace: unichar = 0x20 private static let kTab: unichar = 0x09 private static let kCarriageReturn: unichar = 0x0D - private static let kDollar: unichar = 0x24 + static let kDollar: unichar = 0x24 private static let kCapitalE: unichar = 0x45 private static let kSmallE: unichar = 0x65 private static let kSmallB: unichar = 0x62 private static let kOpenBracket: unichar = 0x5B - private static let kCloseBracket: unichar = 0x5D + static let kCloseBracket: unichar = 0x5D private static let kCapitalG: unichar = 0x47 private static let kSmallG: unichar = 0x67 @@ -101,7 +120,7 @@ final class SQLFileParser: Sendable { return char == kCloseBracket case .inMultiLineComment: return char == kStar || char == kSlash - case .inSingleLineComment: + case .inSingleLineComment, .inClientLine: return false case .inDollarQuote: return char == kDollar @@ -137,34 +156,37 @@ final class SQLFileParser: Sendable { return true } - private static let delimiterPrefix = "DELIMITER " - private static let delimiterPrefixLength = 10 - - // mysqldump writes `DELIMITER ;;`, whose new delimiter is the rest of the line, so a `;` in it ends nothing. + // mysqldump writes `DELIMITER ;;`, so a `;` on the directive's line ends nothing. private static func readsDelimiterDirective(_ ctx: ParserContext) -> Bool { - guard ctx.grammar.contains(.delimiterDirective), let statement = ctx.currentStatement else { return false } + guard ctx.grammar.contains(.delimiterDirective), ctx.statementOpensLine, + let statement = ctx.currentStatement + else { return false } let length = statement.length var index = 0 while index < length, isWhitespace(statement.character(at: index)) { index += 1 } - let prefix = delimiterPrefix as NSString - guard length - index >= prefix.length, - statement.substring(with: NSRange(location: index, length: prefix.length)).uppercased() == delimiterPrefix - else { return false } + guard SQLDelimiterDirective.startsLine(in: statement, at: index, length: length) else { return false } let rest = NSRange(location: index, length: length - index) return statement.range(of: "\n", options: [], range: rest).location == NSNotFound } - private static func extractDelimiterChange(_ text: String) -> String? { - let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) - guard trimmed.uppercased().hasPrefix(delimiterPrefix) else { return nil } - let newDelim = String(trimmed.dropFirst(delimiterPrefixLength)) - .trimmingCharacters(in: .whitespacesAndNewlines) - return newDelim.isEmpty ? nil : newDelim + /// The `DELIMITER` line the statement read so far is, where it opened its line as the mysql client requires. + private static func delimiterDirective(_ ctx: ParserContext) -> SQLDelimiterDirective? { + guard ctx.grammar.contains(.delimiterDirective), ctx.statementOpensLine, + let statement = ctx.currentStatement + else { return nil } + let length = statement.length + var start = 0 + while start < length, isWhitespace(statement.character(at: start)) { + start += 1 + } + let rest = NSRange(location: start, length: length - start) + guard statement.range(of: "\n", options: [], range: rest).location == NSNotFound else { return nil } + return SQLDelimiterDirective.line(startingAt: start, in: statement, length: length) } - private struct ParserContext { + struct ParserContext { var grammar: SQLLexicalGrammar var state: ParserState = .normal let currentStatement: NSMutableString? @@ -215,11 +237,21 @@ final class SQLFileParser: Sendable { /// other dialect has always split an import at each `;` and relies on `DELIMITER` or dollar quoting for a /// routine body, and keeps doing so. var boundaries: (any SQLStatementBoundaryTracking)? + + /// A `;` that ended the statement while the tracker could still take it back at the next token. + var heldEnd: HeldEnd? var word: [unichar] = [] + /// Where the word being read starts in the statement text, and on which line. + var wordStart = 0 + var wordStartLine = 1 var alternativeQuoteCloser: unichar = 0 // A delimiter outside the Basic Multilingual Plane is two units, and the literal closes at both. var alternativeQuoteCloserTrail: unichar? var lineHasCode = false + /// Whether the statement's first unit was the first code on its line, where a `DELIMITER` can stand. + var statementOpensLine = false + /// The line of a `COPY ... FROM STDIN` the file holds, which no driver here can send rows to. + var copyFromStandardInputLine: Int? var pendingSlashLine = false var pendingSlashTrailing: [unichar] = [] @@ -240,7 +272,7 @@ final class SQLFileParser: Sendable { self.currentStatement = currentStatement self.readsBatches = readsBatches self.batchCutLength = batchCutLength - self.boundaries = !readsBatches && SQLStatementBoundaries.statementsCanOwnSemicolons(in: grammar) + self.boundaries = !readsBatches && Self.readsStatementsWithTracker(grammar) ? SQLStatementBoundaries.makeTracker(for: grammar) : nil } @@ -249,17 +281,50 @@ final class SQLFileParser: Sendable { mutating func adopt(_ next: SQLLexicalGrammar) { guard state == .normal, !hasStatementContent, next != grammar else { return } grammar = next - boundaries = !readsBatches && SQLStatementBoundaries.statementsCanOwnSemicolons(in: next) + boundaries = !readsBatches && Self.readsStatementsWithTracker(next) ? SQLStatementBoundaries.makeTracker(for: next) : nil } + // A psql script needs the tracker to find a `COPY ... FROM STDIN`, and it then holds a `BEGIN ATOMIC` body + // and a rule's action list together as the editor does. + private static func readsStatementsWithTracker(_ grammar: SQLLexicalGrammar) -> Bool { + SQLStatementBoundaries.statementsCanOwnSemicolons(in: grammar) || grammar.contains(.psqlScripts) + } + // A SQL Server statement is a batch of its own and keeps the comments inside it, not the ones before it. var keepsComments: Bool { readsBatches || (grammar.contains(.batchSeparatorLines) && hasStatementContent) || insideConditionalComment } } + /// The statement's text before the held `;`, and whether the `;` belongs to it. + struct HeldEnd { + let textLength: Int + let keepsSemicolon: Bool + } + + /// Settles a held end at the first token after it, `word` being that token when it is a word. When the `;` ended + /// the statement, the text up to it is yielded and the word, if any, starts the next statement. + private static func settleHeldEnd(_ ctx: inout ParserContext, word: String?) { + guard let held = ctx.heldEnd else { return } + ctx.heldEnd = nil + guard ctx.boundaries?.continuesStatement(with: word) != true else { return } + var next = "" + if let statement = ctx.currentStatement { + if word != nil, ctx.wordStart < statement.length { + next = statement.substring(from: ctx.wordStart) + } + let kept = held.textLength + (held.keepsSemicolon ? 1 : 0) + statement.deleteCharacters(in: NSRange(location: kept, length: statement.length - kept)) + } + yieldAndReset(&ctx) + guard word != nil else { return } + ctx.currentStatement?.setString(next) + ctx.hasStatementContent = true + ctx.statementStartLine = ctx.wordStartLine + } + private static func trimmedStatement(_ ctx: ParserContext) -> String { (ctx.currentStatement as NSString?)? .trimmingCharacters(in: .whitespacesAndNewlines) ?? "" @@ -270,6 +335,8 @@ final class SQLFileParser: Sendable { ctx.hasStatementContent = false ctx.insideConditionalComment = false ctx.boundaries?.reset() + ctx.heldEnd = nil + ctx.statementOpensLine = false ctx.word.removeAll(keepingCapacity: true) } @@ -283,6 +350,7 @@ final class SQLFileParser: Sendable { guard !ctx.word.isEmpty else { return } let word = String(utf16CodeUnits: ctx.word, count: ctx.word.count).uppercased() ctx.word.removeAll(keepingCapacity: true) + settleHeldEnd(&ctx, word: word) ctx.boundaries?.observeWord(word) } @@ -308,9 +376,11 @@ final class SQLFileParser: Sendable { flushWord(&ctx) } guard SqlBlockStructure.startsWord(nsBuffer, at: i, length: bufLen, grammar: ctx.grammar) else { return .none } - // `E'...'` is a literal, and the escape-string branch that reads it never runs once `E` starts a word. - if ctx.grammar.contains(.escapeStringPrefix), char == kCapitalE || char == kSmallE, - !SQLNonCodeSpan.isWordUnit(ctx.previousUnit) { + // `E'...'` and CockroachDB's `b'...'` are literals, and the branch that reads them never runs once the prefix + // starts a word. + let startsEscapeLiteral = (ctx.grammar.contains(.escapeStringPrefix) && (char == kCapitalE || char == kSmallE)) + || (ctx.grammar.contains(.escapeBytesPrefix) && char == kSmallB) + if startsEscapeLiteral, !SQLNonCodeSpan.isWordUnit(ctx.previousUnit) { if i + 1 < bufLen { if nsBuffer.character(at: i + 1) == kSingleQuote { return .none } } else if !ctx.atEndOfInput { @@ -330,7 +400,9 @@ final class SQLFileParser: Sendable { } guard !followsWord else { return .none } ctx.previousUnitInWord = true - if ctx.boundaries?.needsWords == true { + if ctx.heldEnd != nil || ctx.boundaries?.needsWords == true { + ctx.wordStart = ctx.currentStatement?.length ?? 0 + ctx.wordStartLine = ctx.currentLine ctx.word.append(char) } return .continues @@ -400,19 +472,36 @@ final class SQLFileParser: Sendable { return false } + // A refused argument leaves the delimiter as it was, and the line still runs nothing. private static func processDelimiterChange(_ ctx: inout ParserContext, char: unichar) { - guard ctx.grammar.contains(.delimiterDirective) else { return } - guard char == kNewline && ctx.hasStatementContent else { return } - let text = trimmedStatement(ctx) - if let newDelim = extractDelimiterChange(text) { + guard char == kNewline, ctx.hasStatementContent, let directive = delimiterDirective(ctx) else { return } + if let newDelim = directive.delimiter { ctx.currentDelimiter = newDelim as NSString ctx.isSingleCharDelimiter = ctx.currentDelimiter.length == 1 && ctx.currentDelimiter.character(at: 0) == kSemicolon - resetStatement(&ctx) } + resetStatement(&ctx) } - private struct StepResult { + // pg_dump's `\restrict` and `\unrestrict` limit psql's meta-commands, and none runs here, so the line is read as a + // comment. + private static func skipRestrictLine( + _ ctx: inout ParserContext, + at i: Int, + nsBuffer: NSString, + bufLen: Int + ) -> StepResult? { + guard ctx.grammar.contains(.psqlScripts), !ctx.hasStatementContent, !ctx.lineHasCode else { return nil } + if i + SQLPsqlScript.restrictLineLookahead > bufLen, !ctx.atEndOfInput { + return StepResult(advanced: false, deferred: true) + } + guard SQLPsqlScript.startsRestrictLine(in: nsBuffer, at: i, length: bufLen) else { return nil } + ctx.state = .inClientLine + ctx.boundaries?.observeGap() + return StepResult(advanced: false, deferred: false) + } + + struct StepResult { var advanced: Bool var deferred: Bool } @@ -434,8 +523,15 @@ final class SQLFileParser: Sendable { return step } + if char == kBackslash, let step = skipRestrictLine(&ctx, at: i, nsBuffer: nsBuffer, bufLen: bufLen) { + return step + } + switch observeWordCharacter(&ctx, char: char, i: i, nsBuffer: nsBuffer, bufLen: bufLen) { case .continues: + if !ctx.hasStatementContent { + ctx.statementOpensLine = !ctx.lineHasCode + } (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) appendChar(char, to: ctx.currentStatement) @@ -443,6 +539,7 @@ final class SQLFileParser: Sendable { case .needsMoreData: return StepResult(advanced: false, deferred: true) case let .opensAlternativeQuote(prefixLength): + settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) appendRange(&ctx, from: i, to: i + prefixLength, in: nsBuffer) @@ -503,6 +600,7 @@ final class SQLFileParser: Sendable { if (startsEscapeString || startsEscapeBytes) && nextChar == kSingleQuote && !SQLNonCodeSpan.isWordUnit(ctx.previousUnit) { + settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) appendChar(char, to: ctx.currentStatement) @@ -516,6 +614,7 @@ final class SQLFileParser: Sendable { } if char == kOpenBracket, ctx.grammar.contains(.bracketQuotedIdentifiers) { + settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) appendChar(char, to: ctx.currentStatement) @@ -527,6 +626,7 @@ final class SQLFileParser: Sendable { if let style = ctx.grammar.dollarQuoteStyle, char == kDollar { switch SqlDollarQuote.scanOpener(at: i, in: nsBuffer, bufLen: bufLen, style: style) { case .opener(let length, let tag): + settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) if let target = ctx.currentStatement { @@ -579,17 +679,24 @@ final class SQLFileParser: Sendable { return StepResult(advanced: true, deferred: false) } - if !ctx.hasStatementContent && !isWhitespace(char) { - ctx.statementStartLine = ctx.currentLine - ctx.hasStatementContent = true - } + appendPlainUnit(&ctx, char: char) + return StepResult(advanced: false, deferred: false) + } + + // Whitespace, or a symbol no rule above claimed. + private static func appendPlainUnit(_ ctx: inout ParserContext, char: unichar) { if isWhitespace(char) { ctx.boundaries?.observeGap() } else { + settleHeldEnd(&ctx, word: nil) + if !ctx.hasStatementContent { + ctx.statementStartLine = ctx.currentLine + ctx.hasStatementContent = true + ctx.statementOpensLine = !ctx.lineHasCode + } ctx.boundaries?.observeSymbol(char) } appendChar(char, to: ctx.currentStatement) - return StepResult(advanced: false, deferred: false) } /// A `/*` opens a comment, or the code of a conditional comment the server runs, which a `*/` read as code closes. @@ -614,6 +721,7 @@ final class SQLFileParser: Sendable { return StepResult(advanced: false, deferred: true) } if SqlLexer.executableCommentOpenerLength(nsBuffer, at: i, length: bufLen) != nil { + settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) ctx.insideConditionalComment = true @@ -654,57 +762,35 @@ final class SQLFileParser: Sendable { return } flushWord(&ctx) + // T-SQL also reads further `;`s after an `IF`'s body as part of it, up to its `ELSE`. + if ctx.heldEnd != nil { + _ = ctx.boundaries?.observeSemicolon() + appendChar(kSemicolon, to: ctx.currentStatement) + return + } let endsStatement = ctx.boundaries?.observeSemicolon() ?? true guard endsStatement else { appendChar(kSemicolon, to: ctx.currentStatement) return } + if ctx.boundaries?.copiesFromStandardInput == true { + ctx.copyFromStandardInputLine = ctx.statementStartLine + return + } + if ctx.boundaries?.awaitsContinuation == true, ctx.hasStatementContent { + ctx.heldEnd = HeldEnd( + textLength: ctx.currentStatement?.length ?? 0, + keepsSemicolon: ctx.boundaries?.terminator == .partOfStatement + ) + appendChar(kSemicolon, to: ctx.currentStatement) + return + } if ctx.boundaries?.terminator == .partOfStatement, ctx.hasStatementContent { appendChar(kSemicolon, to: ctx.currentStatement) } yieldAndReset(&ctx) } - private static func processAlternativeQuote( - _ ctx: inout ParserContext, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - if ch == ctx.alternativeQuoteCloser { - let quoteAt = pos + (ctx.alternativeQuoteCloserTrail == nil ? 1 : 2) - if quoteAt >= bufLen { - if ctx.atEndOfInput { - pos += 1 - continue - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - let trailMatches = ctx.alternativeQuoteCloserTrail.map { nsBuffer.character(at: pos + 1) == $0 } ?? true - if trailMatches, nsBuffer.character(at: quoteAt) == kSingleQuote { - pos = quoteAt + 1 - ctx.state = .normal - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - } - pos += 1 - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - private static func isQuote(_ char: unichar, grammar: SQLLexicalGrammar) -> Bool { grammar.isQuote(char) } @@ -715,6 +801,7 @@ final class SQLFileParser: Sendable { nextChar: unichar? ) -> Bool? { guard isQuote(char, grammar: ctx.grammar) else { return nil } + settleHeldEnd(&ctx, word: nil) if let next = nextChar, next == char { (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) @@ -736,6 +823,7 @@ final class SQLFileParser: Sendable { } private static func openTripleQuote(_ ctx: inout ParserContext, quote: unichar, at i: Int, in buffer: NSString) { + settleHeldEnd(&ctx, word: nil) (ctx.hasStatementContent, ctx.statementStartLine) = markContent( ctx.hasStatementContent, ctx.statementStartLine, ctx.currentLine) appendRange(&ctx, from: i, to: i + 3, in: buffer) @@ -745,93 +833,6 @@ final class SQLFileParser: Sendable { ctx.boundaries?.observeQuotedToken(opener: quote) } - /// Runs a triple-quoted literal to the three quotes that close it, holding back a quote or a backslash the buffer - /// ends on until the next chunk shows what follows. - private static func processTripleQuotedString( - _ ctx: inout ParserContext, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - if ctx.backslashEscapesActive && ch == kBackslash { - guard pos + 1 < bufLen || ctx.atEndOfInput else { - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - pos += 2 - continue - } - if ch == ctx.quoteChar { - guard pos + 2 < bufLen || ctx.atEndOfInput else { - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - if SqlLexer.startsTripleQuote(nsBuffer, at: pos, length: bufLen) { - pos += 3 - ctx.state = .normal - ctx.backslashEscapesActive = false - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - } - pos += 1 - } - appendRange(&ctx, from: start, to: min(pos, bufLen), in: nsBuffer) - i = min(pos, bufLen) - return StepResult(advanced: true, deferred: false) - } - - /// Runs a `[...]` identifier to its `]`, which `]]` does not close where the grammar escapes it. - private static func processBracketedIdentifier( - _ ctx: inout ParserContext, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - let doubledEscapes = ctx.grammar.contains(.doubledClosingBracketEscapes) - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - guard ch == kCloseBracket else { - pos += 1 - continue - } - if doubledEscapes { - guard pos + 1 < bufLen || ctx.atEndOfInput else { - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - if pos + 1 < bufLen, nsBuffer.character(at: pos + 1) == kCloseBracket { - pos += 2 - continue - } - } - pos += 1 - ctx.state = .normal - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - private static func yieldAndReset(_ ctx: inout ParserContext) { if ctx.hasStatementContent { let text = trimmedStatement(ctx) @@ -935,7 +936,7 @@ final class SQLFileParser: Sendable { return false } - private static func appendRange( + static func appendRange( _ ctx: inout ParserContext, from start: Int, to end: Int, @@ -945,109 +946,6 @@ final class SQLFileParser: Sendable { target.append(buffer.substring(with: NSRange(location: start, length: end - start))) } - private static func processQuotedString( - _ ctx: inout ParserContext, - quoteChar: unichar, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - let escapesActive = ctx.backslashEscapesActive - - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - - if escapesActive && ch == kBackslash { - if pos + 1 >= bufLen { - if ctx.atEndOfInput { - pos += 1 - continue - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - let next = nsBuffer.character(at: pos + 1) - if next == kNewline { ctx.currentLine += 1 } - pos += 2 - continue - } - - if ch == quoteChar { - if pos + 1 >= bufLen && !ctx.atEndOfInput { - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - let next: unichar? = pos + 1 < bufLen ? nsBuffer.character(at: pos + 1) : nil - if next == quoteChar { - pos += 2 - continue - } - pos += 1 - ctx.state = .normal - ctx.backslashEscapesActive = false - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - - pos += 1 - } - - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - - private static func processDollarQuote( - _ ctx: inout ParserContext, - i: inout Int, - nsBuffer: NSString, - bufLen: Int - ) -> StepResult { - let start = i - var pos = i - let closeLen = (ctx.dollarTag as NSString).length + 2 - - while pos < bufLen { - let ch = nsBuffer.character(at: pos) - if pos > start && ch == kNewline { - ctx.currentLine += 1 - } - - if ch == kDollar { - if pos + closeLen > bufLen { - if ctx.atEndOfInput { - pos += 1 - continue - } - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: true) - } - if SqlDollarQuote.matchesClose(at: pos, tag: ctx.dollarTag, in: nsBuffer, bufLen: bufLen) { - pos += closeLen - ctx.state = .normal - ctx.dollarTag = "" - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - } - pos += 1 - } - - appendRange(&ctx, from: start, to: pos, in: nsBuffer) - i = pos - return StepResult(advanced: true, deferred: false) - } - func parseFile(url: URL, encoding: String.Encoding, grammar: SQLLexicalGrammar) -> AsyncThrowingStream<(statement: String, lineNumber: Int), Error> { parseFile(url: url, encoding: encoding, grammar: { grammar }) } @@ -1129,9 +1027,12 @@ extension SQLFileParser { self.batchCutLength = batchCutLength self.decoder = SQLChunkDecoder(encoding: encoding) self.readingSettled = reading == .batches + let initialGrammar = grammar() + // A `DELIMITER` line is read from the statement's text, so a count needs the text too. + let keepsText = !countOnly || initialGrammar.contains(.delimiterDirective) self.ctx = ParserContext( - grammar: grammar(), - currentStatement: countOnly ? nil : NSMutableString(), + grammar: initialGrammar, + currentStatement: keepsText ? NSMutableString() : nil, batchCutLength: batchCutLength, readsBatches: reading == .batches ) @@ -1217,8 +1118,10 @@ extension SQLFileParser { } private func readNextStatement() throws { + try refuseCopyFromStandardInput() processBuffer() guard ctx.collected.isEmpty else { return } + try refuseCopyFromStandardInput() guard !ctx.atEndOfInput else { emitTrailingStatement() finished = true @@ -1292,6 +1195,16 @@ extension SQLFileParser { || (char == SqlLexer.formFeed && ctx.grammar.contains(.formFeedEndsLineComments)) { ctx.state = .normal } + // `DELIMITER // -- note` ends at this line feed, which the comment took. + if char == SQLFileParser.kNewline { + SQLFileParser.processDelimiterChange(&ctx, char: char) + } + step = StepResult(advanced: false, deferred: false) + + case .inClientLine: + if char == SQLFileParser.kNewline { + ctx.state = .normal + } step = StepResult(advanced: false, deferred: false) case .inMultiLineComment: @@ -1341,12 +1254,18 @@ extension SQLFileParser { nsBuffer, from: stepStart, to: min(i, bufLen), before: ctx.lineHoldsOnlyBlanks ) } - if !ctx.collected.isEmpty { break } + if !ctx.collected.isEmpty || ctx.copyFromStandardInputLine != nil { break } } readOffset = min(i, bufLen) } + // The rows after the `COPY` would otherwise reach the server as statements. + private func refuseCopyFromStandardInput() throws { + guard let line = ctx.copyFromStandardInputLine else { return } + throw SQLFileParseError.copyFromStandardInput(line: line) + } + private func emitTrailingStatement() { ctx.pendingSlashLine = false ctx.pendingSlashTrailing.removeAll() @@ -1354,9 +1273,11 @@ extension SQLFileParser { SQLFileParser.yieldBatch(&ctx, repeatCount: 1, nextBatchStart: ctx.unitsBeforeBuffer + readOffset) return } + SQLFileParser.flushWord(&ctx) + SQLFileParser.settleHeldEnd(&ctx, word: nil) guard ctx.hasStatementContent else { return } let text = SQLFileParser.trimmedStatement(ctx) - if SQLFileParser.extractDelimiterChange(text) == nil { + if SQLFileParser.delimiterDirective(ctx) == nil { ctx.collected.append( ParsedStatement(statement: text, lineNumber: ctx.statementStartLine, repeatCount: 1) ) diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 59ea7fab79..f252c3182f 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -103462,6 +103462,9 @@ } } } + }, + "Line %lld runs COPY FROM STDIN, whose rows TablePro cannot send. Run the file with psql, or dump it again with --inserts." : { + }, "Line break" : { "extractionState" : "manual", diff --git a/apple/TablePro/Views/Editor/SQLCompletionKind+IconColor.swift b/apple/TablePro/Views/Editor/SQLCompletionKind+IconColor.swift index bbafd7d64d..98727fa462 100644 --- a/apple/TablePro/Views/Editor/SQLCompletionKind+IconColor.swift +++ b/apple/TablePro/Views/Editor/SQLCompletionKind+IconColor.swift @@ -1,8 +1,3 @@ -// -// SQLCompletionKind+IconColor.swift -// TablePro -// - import AppKit import TableProAutocomplete @@ -19,6 +14,7 @@ extension SQLCompletionKind { case .alias: return .systemGray case .operator: return .systemIndigo case .favorite: return .systemYellow + case .joinCondition: return .systemBrown } } } diff --git a/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift b/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift index 9b87a2bce5..aaee312be2 100644 --- a/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift +++ b/apple/TableProMobile/TableProMobile/Completion/QueryCompletionCatalog.swift @@ -7,6 +7,7 @@ import TableProPluginKit // Nil when there is no session to read from: nothing is read and nothing is kept. typealias QueryColumnReader = @Sendable (_ table: String, _ schema: String?) async throws -> [ColumnInfo]? typealias QuerySchemaTableReader = @Sendable (_ schema: String) async throws -> [TableInfo]? +typealias QueryForeignKeyReader = @Sendable (_ table: String, _ schema: String?) async throws -> [ForeignKeyInfo]? actor QueryCompletionCatalog: SQLCompletionCatalog { private static let logger = Logger(subsystem: "com.TablePro", category: "QueryCompletionCatalog") @@ -21,6 +22,11 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { let columns: [ColumnInfo] } + nonisolated private struct CachedForeignKeys: Sendable { + let generation: Int + let keys: [SQLCompletionForeignKey] + } + private var tables: [TableInfo] { didSet { cachedTableItems = nil @@ -56,9 +62,12 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { } private var schemaTables: [String: (generation: Int, tables: [TableInfo])] = [:] private var schemaTableReads: [String: Task<[TableInfo]?, any Error>] = [:] + private let readCatalogForeignKeys: QueryForeignKeyReader? + private var cachedForeignKeys: [ColumnKey: CachedForeignKeys] = [:] + private var foreignKeyReads: [ColumnKey: Task<[SQLCompletionForeignKey]?, any Error>] = [:] // `readCatalogColumns` reads on a connection beside the session and answers nil when there is none, - // so only `readColumns`, the session's own, waits for runs. + // so only `readColumns`, the session's own, waits for runs. Foreign keys are read on that connection only. init( tables: [TableInfo], databaseType: DatabaseType, @@ -67,6 +76,7 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { readCatalogColumns: QueryColumnReader? = nil, readSchemaTables: QuerySchemaTableReader? = nil, readCatalogSchemaTables: QuerySchemaTableReader? = nil, + readCatalogForeignKeys: QueryForeignKeyReader? = nil, schemas: [String] = [], databases: [String] = [] ) { @@ -77,6 +87,7 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { self.readCatalogColumns = readCatalogColumns self.readSchemaTables = readSchemaTables self.readCatalogSchemaTables = readCatalogSchemaTables + self.readCatalogForeignKeys = readCatalogForeignKeys self.schemas = schemas self.databases = databases } @@ -370,6 +381,76 @@ actor QueryCompletionCatalog: SQLCompletionCatalog { } } + // Kept and marked stale with the columns. A failed read keeps the last answer for this generation, so a catalog + // that refuses it is not asked on every key. + func foreignKeys(of tableName: String, schema: String?) async -> [SQLCompletionForeignKey] { + guard let readCatalogForeignKeys else { return [] } + let key = columnKey(tableName, schema: schema) + let known = cachedForeignKeys[key] + if let known, known.generation == columnGeneration { return known.keys } + if let pending = foreignKeyReads[key] { + return (try? await pending.value).flatMap { $0 } ?? known?.keys ?? [] + } + + let generation = columnGeneration + let namespace = schema ?? self.schema + let read = Task<[SQLCompletionForeignKey]?, any Error> { + try await readCatalogForeignKeys(key.table, key.schema).map { Self.completionForeignKeys($0, within: namespace) } + } + foreignKeyReads[key] = read + defer { + if foreignKeyReads[key] == read { foreignKeyReads[key] = nil } + } + do { + guard let keys = try await read.value else { return known?.keys ?? [] } + if generation == columnGeneration { + cachedForeignKeys[key] = CachedForeignKeys(generation: generation, keys: keys) + } + return keys + } catch { + Self.logger.warning("Reading foreign keys for completion failed: \(error.localizedDescription, privacy: .private)") + if generation == columnGeneration { + cachedForeignKeys[key] = CachedForeignKeys(generation: generation, keys: known?.keys ?? []) + } + return known?.keys ?? [] + } + } + + // A key into another schema or database would pair with a same-named table of this one. + nonisolated private static func completionForeignKeys( + _ rows: [ForeignKeyInfo], + within namespace: String? + ) -> [SQLCompletionForeignKey] { + let local = rows.filter { row in + guard let namespace, let target = row.referencedSchema, !target.isEmpty else { return true } + return target.caseInsensitiveCompare(namespace) == .orderedSame + } + return SQLCompletionForeignKey.grouping(local.map { + SQLCompletionForeignKey.Row( + name: $0.name, + column: $0.column, + referencedTable: $0.referencedTable, + referencedSchema: $0.referencedSchema, + referencedColumn: $0.referencedColumn + ) + }) + } + + // Nothing spells the items after ranking here, so a condition's names are quoted as the other items are. + nonisolated func writtenText(of identifier: SQLCompletionIdentifier) -> String? { + let type = databaseType + return identifier.parts.map { part -> String in + switch part { + case .catalog(let name): + return CompletionIdentifierQuoting.quoted(name, for: type) + case .statement(let text, let isQuoted): + return isQuoted + ? SQLBuilder.quoteIdentifier(text, for: type) + : CompletionIdentifierQuoting.quotedAsWritten(text, for: type) + } + }.joined(separator: ".") + } + private func columns(of tableName: String, schema: String?) async -> [ColumnInfo] { let key = columnKey(tableName, schema: schema) let known = columns[key] diff --git a/apple/TableProMobile/TableProMobile/Completion/QueryCompletionController.swift b/apple/TableProMobile/TableProMobile/Completion/QueryCompletionController.swift index a2cf6873c8..26993d44cf 100644 --- a/apple/TableProMobile/TableProMobile/Completion/QueryCompletionController.swift +++ b/apple/TableProMobile/TableProMobile/Completion/QueryCompletionController.swift @@ -80,7 +80,8 @@ final class QueryCompletionController { readColumns: @escaping QueryColumnReader, readCatalogColumns: QueryColumnReader? = nil, readSchemaTables: QuerySchemaTableReader? = nil, - readCatalogSchemaTables: QuerySchemaTableReader? = nil + readCatalogSchemaTables: QuerySchemaTableReader? = nil, + readCatalogForeignKeys: QueryForeignKeyReader? = nil ) { let previous = scope guard previous != newScope else { return } @@ -145,6 +146,7 @@ final class QueryCompletionController { readCatalogColumns: readCatalogColumns, readSchemaTables: readSchemaTables, readCatalogSchemaTables: readCatalogSchemaTables, + readCatalogForeignKeys: readCatalogForeignKeys, schemas: newScope.schemas, databases: newScope.databases ) @@ -216,7 +218,7 @@ final class QueryCompletionController { ) show([], answer: nil) return edit - case .keyword, .operator: + case .keyword, .operator, .joinCondition: return Self.keywordEdit(for: item, replacing: answer.range, in: source) case .function, .favorite: let insertion = SQLCompletionInsertion.resolve(for: item) diff --git a/apple/TableProMobile/TableProMobile/Localizable.xcstrings b/apple/TableProMobile/TableProMobile/Localizable.xcstrings index 2588102bc1..da37df0bb5 100644 --- a/apple/TableProMobile/TableProMobile/Localizable.xcstrings +++ b/apple/TableProMobile/TableProMobile/Localizable.xcstrings @@ -12799,6 +12799,9 @@ }, "Its uncommitted changes are discarded." : { + }, + "Join Condition" : { + }, "JSON" : { "extractionState" : "manual", diff --git a/apple/TableProMobile/TableProMobile/Views/Components/QuerySuggestionBar.swift b/apple/TableProMobile/TableProMobile/Views/Components/QuerySuggestionBar.swift index 3e6b56d507..08b338c458 100644 --- a/apple/TableProMobile/TableProMobile/Views/Components/QuerySuggestionBar.swift +++ b/apple/TableProMobile/TableProMobile/Views/Components/QuerySuggestionBar.swift @@ -103,6 +103,7 @@ final class QuerySuggestionBar: UIView { case .alias: String(localized: "Alias") case .operator: String(localized: "Operator") case .favorite: String(localized: "Favorite") + case .joinCondition: String(localized: "Join Condition") } } } diff --git a/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift b/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift index 10c61e5e8e..85ab904dd9 100644 --- a/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift +++ b/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift @@ -112,6 +112,28 @@ struct QueryEditorView: View { } } + private var catalogForeignKeyReader: QueryForeignKeyReader { + let manager = appState.connectionManager + let connectionId = connectionId + let database = coordinator.supportsDatabaseSwitching ? coordinator.activeDatabase : "" + let activeSchema = activeSchema + let coordinator = coordinator + return { table, schema in + guard await coordinator.catalogMatchesSession else { return nil } + let namespace = schema ?? activeSchema + let keys = try await manager.readCatalog(for: connectionId, database: database) { driver in + try await driver.fetchForeignKeys(table: table, schema: namespace) + } + // MySQL, which has no schemas, names a key's database there. A key into another namespace would pair with + // a same-named table of this one. + let home = namespace.flatMap { $0.isEmpty ? nil : $0 } ?? database + return keys?.filter { key in + guard let target = key.referencedSchema, !target.isEmpty else { return true } + return target.caseInsensitiveCompare(home) == .orderedSame + } + } + } + private var catalogTableReader: QuerySchemaTableReader { let manager = appState.connectionManager let connectionId = connectionId @@ -199,7 +221,8 @@ struct QueryEditorView: View { readColumns: sessionColumnReader, readCatalogColumns: catalogColumnReader, readSchemaTables: sessionTableReader, - readCatalogSchemaTables: catalogTableReader + readCatalogSchemaTables: catalogTableReader, + readCatalogForeignKeys: catalogForeignKeyReader ) } .alert(String(localized: "Write Query Blocked"), isPresented: $showWriteBlockedAlert) { diff --git a/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift b/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift index 5dd5104935..ac042a81c5 100644 --- a/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionCatalogTests.swift @@ -242,7 +242,7 @@ struct QueryCompletionCatalogTests { #expect(labels(first) == ["invoices", "Quarterly Totals"]) #expect(first.map(\.insertText) == ["invoices", "\"Quarterly Totals\""]) #expect(labels(again) == labels(first)) - #expect(await reads.count == 1) + #expect(await reads.readCount == 1) } @Test("another schema's tables come from the catalog connection first, from the session only while reads are open") @@ -266,7 +266,7 @@ struct QueryCompletionCatalogTests { #expect(labels(sales) == ["from_catalog"]) #expect(closed.isEmpty) #expect(labels(open) == ["log"]) - #expect(await sessionReads.count == 1) + #expect(await sessionReads.readCount == 1) } @Test("closing session reads waits for a schema read already sent on the session") @@ -304,7 +304,7 @@ struct QueryCompletionCatalogTests { #expect(items.map(\.insertText) == ["public", "sales", "\"Sales\""]) #expect(labels(lower) == ["lower_orders"]) #expect(labels(upper) == ["upper_orders"]) - #expect(await reads.count == 2) + #expect(await reads.readCount == 2) } @Test("a schema that differs from the active one only in case lists its own tables") @@ -331,7 +331,7 @@ struct QueryCompletionCatalogTests { #expect(await catalog.isKnownSchema("ARCHIVE")) #expect(labels(await catalog.tableCompletionItems(inSchema: "archive")) == ["orders_2019"]) - #expect(await reads.count == 1) + #expect(await reads.readCount == 1) } @Test("an enum column offers its values: the driver's labels, or the ENUM type MySQL and DuckDB spell out") @@ -537,7 +537,7 @@ private actor SchemaTableReads { private var answersNothing = false private var heldReads: [CheckedContinuation] = [] private var readWaiters: [CheckedContinuation] = [] - private(set) var count = 0 + private(set) var readCount = 0 private(set) var finished = 0 init(_ tables: [String: [TableInfo]], held: Bool = false) { @@ -554,7 +554,7 @@ private actor SchemaTableReads { } func waitForRead() async { - guard count == 0 else { return } + guard readCount == 0 else { return } await withCheckedContinuation { readWaiters.append($0) } } @@ -566,7 +566,7 @@ private actor SchemaTableReads { private func read(_ schema: String) async -> [TableInfo]? { guard !answersNothing else { return nil } - count += 1 + readCount += 1 readWaiters.forEach { $0.resume() } readWaiters = [] if isHeld { @@ -642,3 +642,83 @@ private actor CatalogColumnSource { return columns[table] ?? [] } } + +extension QueryCompletionCatalogTests { + private func foreignKey(_ column: String, to table: String, _ referenced: String, schema: String? = nil) -> ForeignKeyInfo { + ForeignKeyInfo( + name: "\(column)_fk", column: column, referencedTable: table, referencedColumn: referenced, referencedSchema: schema + ) + } + + @Test("a table's foreign keys are read once on the catalog connection, and again once marked stale") + func foreignKeysReadOncePerGeneration() async { + let keys = ForeignKeySource(["orders": [foreignKey("customer_id", to: "customers", "id")]]) + let catalog = QueryCompletionCatalog( + tables: [table("orders"), table("customers")], databaseType: .postgresql, schema: "public", + readColumns: CatalogColumnSource([:]).reader, readCatalogForeignKeys: keys.reader + ) + + let first = await catalog.foreignKeys(of: "orders", schema: nil) + _ = await catalog.foreignKeys(of: "orders", schema: nil) + await catalog.markColumnsStale() + _ = await catalog.foreignKeys(of: "orders", schema: nil) + + #expect(first == [SQLCompletionForeignKey( + name: "customer_id_fk", columns: ["customer_id"], referencedTable: "customers", referencedColumns: ["id"] + )]) + #expect(await keys.reads == ["orders", "orders"]) + } + + @Test("a key into another schema is left out, and the session is never asked for keys") + func foreignKeysStayInTheSchema() async { + let keys = ForeignKeySource(["orders": [ + foreignKey("customer_id", to: "customers", "id", schema: "public"), + foreignKey("auditor_id", to: "auditors", "id", schema: "audit"), + ]]) + let session = CatalogColumnSource([:]) + let catalog = QueryCompletionCatalog( + tables: [table("orders")], databaseType: .postgresql, schema: "public", + readColumns: session.reader, readCatalogForeignKeys: keys.reader + ) + + let read = await catalog.foreignKeys(of: "orders", schema: nil) + + #expect(read.map { $0.referencedTable } == ["customers"]) + #expect(await session.reads.isEmpty) + } + + @Test("without a catalog connection no keys are offered") + func noForeignKeysWithoutCatalogReader() async { + let catalog = catalog(CatalogColumnSource([:]), tables: [table("orders")], type: .postgresql, schema: "public") + #expect(await catalog.foreignKeys(of: "orders", schema: nil).isEmpty) + } + + @Test("a name is written with the engine's quotes, and a statement name only where it was quoted") + func writtenNames() { + let postgres = catalog(CatalogColumnSource([:]), tables: [], type: .postgresql, schema: "public") + #expect(postgres.writtenText(of: SQLCompletionIdentifier([.statement("O", quoted: true), .catalog("CustomerId")])) + == "\"O\".\"CustomerId\"") + #expect(postgres.writtenText(of: SQLCompletionIdentifier([.statement("Ord", quoted: false), .catalog("id")])) + == "Ord.id") + let mysql = catalog(CatalogColumnSource([:]), tables: [], type: .mysql) + #expect(mysql.writtenText(of: SQLCompletionIdentifier([.statement("o", quoted: false), .catalog("order")])) == "o.`order`") + } +} + +private actor ForeignKeySource { + private let keys: [String: [ForeignKeyInfo]] + private(set) var reads: [String] = [] + + init(_ keys: [String: [ForeignKeyInfo]]) { + self.keys = keys + } + + nonisolated var reader: QueryForeignKeyReader { + { table, _ in await self.read(table) } + } + + private func read(_ table: String) -> [ForeignKeyInfo]? { + reads.append(table) + return keys[table] ?? [] + } +} diff --git a/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionControllerTests.swift b/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionControllerTests.swift index 7d2745486f..3f68ab827e 100644 --- a/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionControllerTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Completion/QueryCompletionControllerTests.swift @@ -621,3 +621,49 @@ extension QueryCompletionControllerTests { #expect(await !suggest(controller, "/*\n" + notes + "it's done */\nSELECT * FROM or").isEmpty) } } + +extension QueryCompletionControllerTests { + @Test("right after a join's ON the foreign key's condition leads and goes in quoted for the engine") + func joinConditionFromForeignKey() async throws { + let source = ControllerColumnSource([ + "orders": [ColumnInfo(name: "Id", typeName: "integer"), ColumnInfo(name: "CustomerId", typeName: "integer")], + "customers": [ColumnInfo(name: "Id", typeName: "integer")], + ]) + let keys: QueryForeignKeyReader = { table, _ in + guard table == "orders" else { return [] } + return [ForeignKeyInfo(name: "orders_customer_fk", column: "CustomerId", referencedTable: "customers", referencedColumn: "Id")] + } + let controller = QueryCompletionController() + controller.update( + scope: scope(.postgresql, tables: ["orders", "customers"], database: "shop", schema: "public"), + readColumns: source.reader, + readCatalogForeignKeys: keys + ) + let text = "SELECT * FROM orders o JOIN customers c ON " + + let suggestions = await suggest(controller, text) + #expect(suggestions.first?.label == "o.CustomerId = c.Id") + #expect(suggestions.first?.kind == .joinCondition) + let edit = try accept("o.CustomerId = c.Id", in: text, with: controller) + #expect(applying(edit, to: text) == text + "o.\"CustomerId\" = c.\"Id\" ") + } + + // Two copies of `o.Id` were offered, and the first went in bare, which PostgreSQL folds to `o.id`. + @Test("a qualified column after a join's ON is offered once and keeps its quotes") + func qualifiedColumnOnceAfterOn() async throws { + let source = ControllerColumnSource([ + "orders": [ColumnInfo(name: "Id", typeName: "integer")], + "customers": [ColumnInfo(name: "Id", typeName: "integer")], + ]) + let controller = controller( + scope(.postgresql, tables: ["orders", "customers"], database: "shop", schema: "public"), + source: source + ) + let text = "SELECT * FROM orders o JOIN customers c ON " + + let suggestions = await suggest(controller, text) + #expect(suggestions.filter { $0.label == "o.Id" }.count == 1) + let edit = try accept("o.Id", in: text, with: controller) + #expect(applying(edit, to: text) == text + "o.\"Id\"") + } +} diff --git a/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift b/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift index f7bd3cfbb5..0908d2f496 100644 --- a/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift +++ b/apple/TableProTests/Core/Autocomplete/SQLSchemaProviderTests.swift @@ -1257,3 +1257,127 @@ private final class CallRecorder: @unchecked Sendable { return storage } } + +private struct ForeignKeyReadFailure: Error {} + +extension SQLSchemaProviderTests { + nonisolated private static let orderItemKey = [ + ForeignKeyInfo(name: "items_order_fk", column: "order_id", referencedTable: "orders", referencedColumn: "id"), + ForeignKeyInfo(name: "items_order_fk", column: "tenant_id", referencedTable: "orders", referencedColumn: "tenant_id"), + ] + + private static func keySource( + recorder: CallRecorder, + read: (@Sendable () async throws -> [ForeignKeyInfo])? = nil + ) -> SQLSchemaProvider.ColumnMetadataSource { + SQLSchemaProvider.ColumnMetadataSource( + fetchColumns: { _, _ in [] }, + fetchAllColumns: { [:] }, + fetchForeignKeys: { table, _ in + recorder.record(table) + return try await read?() ?? Self.orderItemKey + } + ) + } + + @Test("A table's foreign keys are read once across keystrokes, a composite key as one") + func foreignKeysReadOnce() async { + let recorder = CallRecorder() + let provider = SQLSchemaProvider(metadataSource: Self.keySource(recorder: recorder)) + + let first = await provider.foreignKeys(of: "order_items") + _ = await provider.foreignKeys(of: "order_items") + _ = await provider.foreignKeys(of: "ORDER_ITEMS") + + #expect(first.count == 1) + #expect(first.first?.columns == ["order_id", "tenant_id"]) + #expect(first.first?.referencedColumns == ["id", "tenant_id"]) + #expect(recorder.calls == ["order_items"]) + } + + @Test("Concurrent requests for one table's foreign keys share one read") + func foreignKeysShareTheReadInFlight() async { + let recorder = CallRecorder() + let gate = AsyncGate() + let provider = SQLSchemaProvider(metadataSource: Self.keySource(recorder: recorder) { + await gate.wait() + return Self.orderItemKey + }) + + let first = Task { await provider.foreignKeys(of: "order_items") } + await gate.arrived(1) + let second = Task { await provider.foreignKeys(of: "order_items") } + await gate.open() + + #expect(await first.value.count == 1) + #expect(await second.value.count == 1) + #expect(recorder.calls == ["order_items"]) + } + + @Test("resetForDatabase drops the foreign keys, so the next request reads again") + func resetClearsForeignKeys() async { + let recorder = CallRecorder() + let provider = SQLSchemaProvider(metadataSource: Self.keySource(recorder: recorder)) + + _ = await provider.foreignKeys(of: "order_items") + await provider.resetForDatabase( + "db", tables: [TestFixtures.makeTableInfo(name: "order_items")], driver: MockDatabaseDriver() + ) + _ = await provider.foreignKeys(of: "order_items") + + #expect(recorder.calls == ["order_items", "order_items"]) + } + + @Test("A failed foreign key read is kept as no keys until the table list is read again") + func failedForeignKeyReadIsKeptUntilTheTablesAreReadAgain() async { + let recorder = CallRecorder() + let driver = MockDatabaseDriver(connection: TestFixtures.makeConnection(type: .postgresql)) + let provider = SQLSchemaProvider(metadataSource: Self.keySource(recorder: recorder) { + guard recorder.calls.count > 1 else { throw ForeignKeyReadFailure() } + return Self.orderItemKey + }) + + let failed = await provider.foreignKeys(of: "order_items") + let kept = await provider.foreignKeys(of: "order_items") + await provider.resetForDatabase("db", tables: [], driver: driver, connection: driver.connection) + let reread = await provider.foreignKeys(of: "order_items") + + #expect(failed.isEmpty) + #expect(kept.isEmpty) + #expect(reread.count == 1) + #expect(recorder.calls == ["order_items", "order_items"]) + } + + @Test("An engine without foreign keys is never asked for them") + func noForeignKeyReadWithoutSupport() async { + let recorder = CallRecorder() + let driver = MockDatabaseDriver(connection: TestFixtures.makeConnection(type: .mongodb)) + let provider = SQLSchemaProvider(metadataSource: Self.keySource(recorder: recorder)) + await provider.resetForDatabase("db", tables: [], driver: driver, connection: driver.connection) + + #expect(await provider.foreignKeys(of: "orders").isEmpty) + #expect(recorder.calls.isEmpty) + } + + @Test("A join's ON offers the condition from the cached keys") + func joinConditionThroughTheProvider() async { + let source = SQLSchemaProvider.ColumnMetadataSource( + fetchColumns: { table, _ in + table == "orders" + ? [TestFixtures.makeColumnInfo(name: "id"), TestFixtures.makeColumnInfo(name: "tenant_id")] + : [TestFixtures.makeColumnInfo(name: "order_id"), TestFixtures.makeColumnInfo(name: "tenant_id")] + }, + fetchAllColumns: { [:] }, + fetchForeignKeys: { table, _ in table == "order_items" ? Self.orderItemKey : [] } + ) + let schemaProvider = SQLSchemaProvider(metadataSource: source) + let provider = SQLCompletionProvider(schemaProvider: schemaProvider) + let text = "SELECT * FROM orders o JOIN order_items i ON " + + let (items, _) = await provider.getCompletions(text: text, cursorPosition: (text as NSString).length) + + #expect(items.first?.label == "o.id = i.order_id AND o.tenant_id = i.tenant_id") + #expect(items.first?.kind == .joinCondition) + #expect(items.filter { $0.label == "o.id" }.count == 1) + } +} diff --git a/apple/TableProTests/Core/Coordinators/MultiStatementReplanTests.swift b/apple/TableProTests/Core/Coordinators/MultiStatementReplanTests.swift index a2dc916f67..1f5f9164fc 100644 --- a/apple/TableProTests/Core/Coordinators/MultiStatementReplanTests.swift +++ b/apple/TableProTests/Core/Coordinators/MultiStatementReplanTests.swift @@ -28,7 +28,7 @@ struct MultiStatementReplanTests { let last = try #require(planned.last) let rest = QueryExecutionCoordinator.remainingStatements( - in: Self.script, after: first.range, through: NSMaxRange(last.range), grammar: Self.noEscapes + in: Self.script, after: first, through: NSMaxRange(last.range), grammar: Self.noEscapes ) let restSQL = rest.map { $0.sql } let restText = rest.map { (Self.script as NSString).substring(with: $0.range) } @@ -39,6 +39,23 @@ struct MultiStatementReplanTests { #expect(!QueryExecutionCoordinator.scriptHolds(planned, in: "SELECT 1")) } + @Test("A replan starts past a COPY's rows and keeps the DELIMITER in force") + func replanStartsPastTheSegment() throws { + let postgres = "COPY t FROM stdin;\nDROP TABLE users; COMMIT;\n\\.\nSELECT 1;" + let copy = try #require(SQLStatementScanner.executableStatements(in: postgres, grammar: TestGrammar.postgres).first) + let afterCopy = QueryExecutionCoordinator.remainingStatements( + in: postgres, after: copy, through: (postgres as NSString).length, grammar: TestGrammar.postgres + ) + #expect(afterCopy.map { $0.sql } == ["SELECT 1"]) + + let mysql = "DELIMITER //\nSET sql_mode = 'NO_BACKSLASH_ESCAPES'//\nSELECT 'a\\'//" + let set = try #require(SQLStatementScanner.executableStatements(in: mysql, grammar: TestGrammar.mysql).first) + let afterSet = QueryExecutionCoordinator.remainingStatements( + in: mysql, after: set, through: (mysql as NSString).length, grammar: Self.noEscapes + ) + #expect(afterSet.map { $0.sql } == ["SELECT 'a\\'"]) + } + @Test("A run sends the replanned rest in place of what it planned") func runSendsTheReplannedRest() async { let driver = ScriptAnsweringDriver(connection: TestFixtures.makeConnection(type: .mysql), sendsBatchesWhole: false) diff --git a/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift b/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift index 7de7e4234b..305a631b6c 100644 --- a/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift +++ b/apple/TableProTests/Core/Services/SQLFormatterServiceTests.swift @@ -1235,6 +1235,25 @@ struct SQLFormatterServiceTests { ) } + @Test("A pg_dump COPY block keeps its rows, its \\. line and the statements after it") + func copyBlockStaysAsWritten() throws { + let copy = "COPY public.t (a, b) FROM stdin;\n\tit's\n2\tselect -- x\n\\." + let sql = "\\restrict k\n\(copy)\nselect 1;" + let formatted = try formatter.format(sql, dialect: .postgresql, options: .default).formattedSQL + #expect(formatted.contains(copy + "\n")) + #expect(formatted.hasPrefix("\\restrict k\n")) + let grammar = DatabaseType.postgresql.lexicalGrammar + #expect(SQLStatementScanner.executableStatements(in: formatted, grammar: grammar).map { $0.sql.uppercased() } == [ + "COPY PUBLIC.T (A, B) FROM STDIN", "SELECT 1", + ]) + } + + @Test("COPY rows that end the text keep a trailing tab") + func copyRowsAtTheEndKeepTheirTab() throws { + let sql = "COPY t (a, b) FROM stdin;\n1\t" + #expect(try formatter.format(sql, dialect: .postgresql, options: .default).formattedSQL == sql) + } + @Test("PostgreSQL operators stay whole") func postgresOperators() throws { try expectFormat( diff --git a/apple/TableProTests/Core/Services/SQLTokenizerTests.swift b/apple/TableProTests/Core/Services/SQLTokenizerTests.swift index 7cb207c89e..13e35076d3 100644 --- a/apple/TableProTests/Core/Services/SQLTokenizerTests.swift +++ b/apple/TableProTests/Core/Services/SQLTokenizerTests.swift @@ -305,6 +305,22 @@ struct SQLTokenizerTests { #expect(code("select 1;\ndelimiter $$", .mysql).last?.type == .directive) } + @Test("A DELIMITER line sets the first word after it, as the mysql client reads it") + func delimiterArgumentIsTheFirstWord() { + let tokens = code("DELIMITER // -- set\nselect 1//\nDELIMITER\nselect 2;", .mysql) + #expect(tokens.map(\.value) == ["DELIMITER // -- set", "select", "1", "//", "DELIMITER", "select", "2", ";"]) + #expect(tokens.first?.type == .directive) + #expect(tokens[4].type == .directive) + } + + @Test("A COPY with its rows and a psql \\restrict line keep their text") + func psqlScriptText() { + let tokens = code("\\restrict k\nCOPY t FROM stdin;\n\tselect 'x\n\\.\nselect 2;", .postgresql) + #expect(tokens.map(\.value) == ["\\restrict k", "COPY t FROM stdin;\n\tselect 'x\n\\.", "select", "2", ";"]) + #expect(tokens[0].type == .directive) + #expect(tokens[1].type == .directive) + } + @Test("A MySQL # comment runs to the end of its line") func hashComment() { #expect(code("a # b and\nc", .mysql).map(\.value) == ["a", "# b and", "c"]) diff --git a/apple/TableProTests/Core/Utilities/SQL/QueryClassifierClientScriptTests.swift b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierClientScriptTests.swift new file mode 100644 index 0000000000..8b95fdee77 --- /dev/null +++ b/apple/TableProTests/Core/Utilities/SQL/QueryClassifierClientScriptTests.swift @@ -0,0 +1,35 @@ +import Foundation +@testable import TablePro +import TableProSQLGrammar +import Testing + +// A script line the client reads, a DELIMITER or a COPY's rows, never lets a second statement past the gate. +struct QueryClassifierClientScriptTests { + @Test("A routine between DELIMITER lines is one statement for the gate") + func delimitedRoutineIsOneStatement() { + let sql = "DELIMITER //\nCREATE PROCEDURE p() BEGIN UPDATE a SET c = 1; UPDATE b SET c = 1; END//\nDELIMITER ;" + #expect(!QueryClassifier.isMultiStatement(sql, databaseType: .mysql)) + } + + @Test("A ; the server splits at is still a second statement under a custom delimiter") + func customDelimiterHidesNoStatement() { + let sql = "DELIMITER //\nSELECT 1; DROP TABLE users//" + #expect(QueryClassifier.isMultiStatement(sql, databaseType: .mysql)) + #expect(QueryClassifier.isDangerousQuery(sql, databaseType: .mysql)) + } + + @Test("A COPY's rows are never statements, and the COPY is never safe") + func copyRowsAreNotStatements() { + let sql = "COPY t FROM stdin;\nx\tDROP TABLE users;\n\\.\n" + #expect(!QueryClassifier.isMultiStatement(sql, databaseType: .postgresql)) + #expect(QueryClassifier.classifyTier(sql, databaseType: .postgresql) != .safe) + #expect(QueryClassifier.isMultiStatement(sql + "SELECT 1;", databaseType: .postgresql)) + } + + @Test("An IF ... ELSE with a ; before the ELSE is one statement, and both branches are tiered") + func ifElseIsOneStatement() { + let sql = "IF @env = 'dev' SELECT 1; ELSE DROP TABLE dbo.t;" + #expect(!QueryClassifier.isMultiStatement(sql, databaseType: .mssql)) + #expect(QueryClassifier.classifyTier(sql, databaseType: .mssql) == .destructive) + } +} diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift index 70208995de..3ab0aa72b6 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFileParserTests.swift @@ -519,4 +519,140 @@ struct SQLFileParserTests { #expect(statements == Self.cqlStatements, "boundary at \(boundary)") } } + + private static func parseWithLines(_ sql: String, grammar: SQLLexicalGrammar) async throws -> [ParsedLine] { + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try sql.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + var statements: [ParsedLine] = [] + for try await (statement, line) in SQLFileParser().parseFile(url: url, encoding: .utf8, grammar: grammar) { + statements.append(ParsedLine(statement: statement, line: line)) + } + return statements + } + + private struct ParsedLine: Equatable { + let statement: String + let line: Int + } + + @Test("MySQL: a DELIMITER line is read as the mysql client reads it", arguments: [ + ("DELIMITER\t//\nSELECT 1//\nDELIMITER ;\nSELECT 2;\n", ["SELECT 1", "SELECT 2"]), + ("DELIMITER // -- set\nSELECT 1//\nDELIMITER ;\nSELECT 2;\n", ["SELECT 1", "SELECT 2"]), + ("DELIMITER // SELECT 99\nSELECT 1//\n", ["SELECT 1"]), + ("DELIMITER\nSELECT 1;\n", ["SELECT 1"]), + (" delimiter $$\nCREATE FUNCTION f() RETURNS INT BEGIN RETURN 1; END$$\ndelimiter ;\nSELECT f();\n", [ + "CREATE FUNCTION f() RETURNS INT BEGIN RETURN 1; END", "SELECT f()", + ]), + ("SELECT 1; DELIMITER //\nSELECT 2;\n", ["SELECT 1", "DELIMITER //\nSELECT 2"]), + ]) + func mysqlDelimiterLines(sql: String, expected: [String]) async throws { + #expect(try await Self.parse(sql, grammar: TestGrammar.mysql) == expected) + } + + @Test("Import reads a line ending in a comment, a CockroachDB byte literal and a \\restrict line as the editor does") + func linesTheEditorReadsTheSameWay() async throws { + #expect(try await Self.parse("DELIMITER // -- set\nSELECT 1//\nSELECT 2//\n", grammar: TestGrammar.mysql) == [ + "SELECT 1", "SELECT 2", + ]) + let cockroach = DatabaseType.cockroachdb.lexicalGrammar + #expect(try await Self.parse("INSERT INTO t VALUES (b'\\''); DELETE FROM x; SELECT 'a';", grammar: cockroach) == [ + "INSERT INTO t VALUES (b'\\'')", "DELETE FROM x", "SELECT 'a'", + ]) + #expect(try await Self.parse("\\restrict k\rDROP TABLE t;\nSELECT 1;\n", grammar: TestGrammar.postgres) == ["SELECT 1"]) + } + + @Test("MySQL: the statement count reads DELIMITER lines as the import does") + func mysqlCountReadsDelimiterLines() async throws { + let sql = "DELIMITER //\nCREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END//\nDELIMITER ;\nSELECT 3;\n" + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try sql.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + let count = try await SQLFileParser().countStatements(url: url, encoding: .utf8, grammar: TestGrammar.mysql) + #expect(count == 2) + #expect(try await Self.parse(sql, grammar: TestGrammar.mysql).count == count) + } + + @Test("Postgres: a plain pg_dump opens with \\restrict, which runs nothing") + func postgresRestrictLinesRunNothing() async throws { + let sql = "\\restrict abc123\n\nSET statement_timeout = 0;\nSELECT 1;\n\n\\unrestrict abc123\n" + #expect(try await Self.parseWithLines(sql, grammar: TestGrammar.postgres) == [ + ParsedLine(statement: "SET statement_timeout = 0", line: 3), + ParsedLine(statement: "SELECT 1", line: 4), + ]) + } + + @Test("Postgres: COPY FROM STDIN stops the file at its line before the COPY or its rows are sent") + func postgresCopyFromStandardInputIsRefused() async throws { + let sql = """ + CREATE TABLE t (a int, b text); + COPY public.t (a, b) FROM stdin; + 1\tit's + 2\tDROP TABLE users; + \\. + SELECT 1; + """ + let url = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString + ".sql") + try sql.write(to: url, atomically: true, encoding: .utf8) + defer { try? FileManager.default.removeItem(at: url) } + + var statements: [String] = [] + await #expect(throws: SQLFileParseError.copyFromStandardInput(line: 2)) { + for try await (statement, _) in SQLFileParser().parseFile(url: url, encoding: .utf8, grammar: TestGrammar.postgres) { + statements.append(statement) + } + } + #expect(statements == ["CREATE TABLE t (a int, b text)"]) + await #expect(throws: SQLFileParseError.copyFromStandardInput(line: 2)) { + _ = try await SQLFileParser().countStatements(url: url, encoding: .utf8, grammar: TestGrammar.postgres) + } + } + + @Test("Postgres: a COPY that reads a server file or writes rows out still runs", arguments: [ + "COPY t FROM '/tmp/stdin';\nSELECT 1;", + "COPY (SELECT 1) TO STDOUT;\nSELECT 1;", + ]) + func postgresCopyWithoutScriptRowsRuns(sql: String) async throws { + #expect(try await Self.parse(sql, grammar: TestGrammar.postgres).count == 2) + } + + @Test("Postgres: a BEGIN ATOMIC body and a rule's action list arrive whole, as the editor reads them") + func postgresBodiesArriveWhole() async throws { + let function = "CREATE FUNCTION f() RETURNS int LANGUAGE sql BEGIN ATOMIC SELECT 1; SELECT 2; END" + let rule = "CREATE RULE r AS ON INSERT TO t DO ALSO (INSERT INTO a VALUES (1); INSERT INTO b VALUES (2))" + let sql = "\(function);\n\(rule);\nSELECT 3;\n" + #expect(try await Self.parse(sql, grammar: TestGrammar.postgres) == [function, rule, "SELECT 3"]) + } + + @Test("SQL Server: a ; before an IF's ELSE stays inside the IF, and the next statement keeps its line") + func sqlServerSemicolonBeforeElse() async throws { + let sql = """ + IF DATENAME(weekday, GETDATE()) IN (N'Saturday', N'Sunday') + SELECT 'Weekend'; + -- weekday + ELSE + SELECT 'Weekday'; + IF @a = 1 SELECT 1; + SELECT 2; + """ + #expect(try await Self.parseWithLines(sql, grammar: TestGrammar.sqlServer) == [ + ParsedLine( + statement: "IF DATENAME(weekday, GETDATE()) IN (N'Saturday', N'Sunday')\n SELECT 'Weekend';\n-- weekday\nELSE\n SELECT 'Weekday'", + line: 1 + ), + ParsedLine(statement: "IF @a = 1 SELECT 1", line: 6), + ParsedLine(statement: "SELECT 2", line: 7), + ]) + } + + @Test("SQL Server: a chunk boundary anywhere around a held ; changes nothing") + func sqlServerHeldSemicolonAcrossChunks() async throws { + let script = "IF @a = 1 SELECT 1;; ELSE SELECT 2;\nIF @b = 1 SELECT 3; SELECT 4;" + let expected = ["IF @a = 1 SELECT 1;; ELSE SELECT 2", "IF @b = 1 SELECT 3", "SELECT 4"] + let chunkSize = 65_536 + for boundary in 0..<(script as NSString).length { + let padding = "--" + String(repeating: "x", count: chunkSize - boundary - 3) + "\n" + #expect(try await Self.parse(padding + script, grammar: TestGrammar.sqlServer) == expected, "boundary at \(boundary)") + } + } } diff --git a/apple/TableProTests/Core/Utilities/SQL/SQLFoldScannerTests.swift b/apple/TableProTests/Core/Utilities/SQL/SQLFoldScannerTests.swift index 3e44d78a03..677d512ffb 100644 --- a/apple/TableProTests/Core/Utilities/SQL/SQLFoldScannerTests.swift +++ b/apple/TableProTests/Core/Utilities/SQL/SQLFoldScannerTests.swift @@ -335,6 +335,55 @@ struct SQLFoldScannerTests { #expect(blocks.map { $0.startLine } == [0]) #expect(blocks.map { $0.endLine } == [3]) } + + // MARK: - Client lines + + private func statementLines(_ sql: String, grammar: SQLLexicalGrammar) -> [ClosedRange] { + regions(sql, grammar: grammar).filter { $0.kind == .statement }.map { $0.startLine...$0.endLine } + } + + @Test("A DELIMITER line folds nothing, and the delimiter ends the routine's fold") + func delimiterLines() { + let sql = """ + DELIMITER // + CREATE PROCEDURE p() + BEGIN + SELECT 1; + END// + DELIMITER ; + SELECT 2, + 3; + """ + #expect(statementLines(sql, grammar: TestGrammar.mysql) == [1...4, 6...7]) + } + + @Test("A COPY's fold runs through its rows, which open no fold of their own") + func copyRows() { + let sql = """ + \\restrict k + COPY t (a, b) FROM stdin; + 1\t(it's + 2\tBEGIN /* + \\. + SELECT 2, + 3; + """ + #expect(statementLines(sql, grammar: TestGrammar.postgres) == [1...4, 5...6]) + #expect(regions(sql, grammar: TestGrammar.postgres).allSatisfy { $0.kind == .statement }) + } + + @Test("A ; before an IF's ELSE does not end the IF's fold") + func semicolonBeforeElse() { + let sql = """ + IF @a = 1 + SELECT 1; + ELSE + SELECT 2; + SELECT 3, + 4; + """ + #expect(statementLines(sql, grammar: TestGrammar.sqlServer) == [0...3, 4...5]) + } } struct SQLFoldEventOrderingTests { diff --git a/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift b/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift index fdb985927d..3820e2e3d1 100644 --- a/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift +++ b/apple/TableProTests/Plugins/SQLLexicalFeatureMappingTests.swift @@ -29,10 +29,10 @@ struct SQLLexicalFeatureMappingTests { .sqlScriptBlocks, .cqlBatches, .hashAndAtInIdentifiers, .angleQuotedIdentifiers, .backslashEscapesInAngleQuotes, .escapeBytesPrefix, .exponentSignsNeedNoDigits, .formFeedEndsLineComments, .atomicRoutineBodies, .triggerBodies, .statementsAsValues, .bracketedStatementSources, .pipeOperatorChains, - .withoutDashComments, .escapeStringContinuations, .databendTokens, + .withoutDashComments, .escapeStringContinuations, .databendTokens, .psqlScripts, ] #expect(grammarBits.isDisjoint(with: curatedOnly)) - #expect(grammarBits.union(curatedOnly).rawValue == (1 << 41) - 1) + #expect(grammarBits.union(curatedOnly).rawValue == (1 << 42) - 1) } @Test("SAP HANA's SQLScript blocks stay in the app's curated grammar and never reach the kit") @@ -78,6 +78,7 @@ struct SQLLexicalFeatureMappingTests { SQLLexicalGrammar(pluginFeatures: cockroach.pluginFeatures) == cockroach.subtracting([ .escapeBytesPrefix, .escapeStringContinuations, .atomicRoutineBodies, .bracketedStatementSources, + .psqlScripts, ]) ) } @@ -93,7 +94,7 @@ struct SQLLexicalFeatureMappingTests { @Test("A grammar survives the trip through the kit's features, less the facts only the curated table sets") func roundTrip() { let grammar = DatabaseType.postgresql.lexicalGrammar - let curatedOnly: SQLLexicalGrammar = [.atomicRoutineBodies, .escapeStringContinuations] + let curatedOnly: SQLLexicalGrammar = [.atomicRoutineBodies, .escapeStringContinuations, .psqlScripts] #expect(grammar.isSuperset(of: curatedOnly)) #expect(SQLLexicalGrammar(pluginFeatures: grammar.pluginFeatures) == grammar.subtracting(curatedOnly)) #expect(SQLLexicalGrammar(pluginFeatures: DatabaseType.mysql.lexicalGrammar.pluginFeatures) == DatabaseType.mysql.lexicalGrammar) diff --git a/apple/changelog.d/4210.added.md b/apple/changelog.d/4210.added.md new file mode 100644 index 0000000000..242c829ee8 --- /dev/null +++ b/apple/changelog.d/4210.added.md @@ -0,0 +1 @@ +Autocomplete offers join conditions from foreign keys right after the ON of a join. diff --git a/apple/changelog.d/4210.fixed.1.md b/apple/changelog.d/4210.fixed.1.md new file mode 100644 index 0000000000..8585969013 --- /dev/null +++ b/apple/changelog.d/4210.fixed.1.md @@ -0,0 +1 @@ +The editor read a MySQL DELIMITER line as SQL, so a routine between DELIMITER lines was cut at each semicolon. diff --git a/apple/changelog.d/4210.fixed.2.md b/apple/changelog.d/4210.fixed.2.md new file mode 100644 index 0000000000..c7e6a59385 --- /dev/null +++ b/apple/changelog.d/4210.fixed.2.md @@ -0,0 +1 @@ +The rows after COPY FROM STDIN in a PostgreSQL script ran as SQL in the editor and in SQL import. diff --git a/apple/changelog.d/4210.fixed.3.md b/apple/changelog.d/4210.fixed.3.md new file mode 100644 index 0000000000..7583794837 --- /dev/null +++ b/apple/changelog.d/4210.fixed.3.md @@ -0,0 +1 @@ +SQL import of a plain pg_dump file failed at its start, on the `\restrict` line current pg_dump versions write. diff --git a/apple/changelog.d/4210.fixed.md b/apple/changelog.d/4210.fixed.md new file mode 100644 index 0000000000..08b9f34840 --- /dev/null +++ b/apple/changelog.d/4210.fixed.md @@ -0,0 +1 @@ +Run Current, the gutter, folding and SQL import split a SQL Server IF ... ELSE at a semicolon before the ELSE.