diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDefinitionText.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDefinitionText.swift index c9051e99ed..2a75a10868 100644 --- a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDefinitionText.swift +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLDefinitionText.swift @@ -19,6 +19,12 @@ public enum MSSQLDefinitionText { ["NATIONAL", "TEXT"], ["BINARY", "VARYING"], ] private static let longestMultiWordType = 3 + // ALTER COLUMN takes COLLATE on these and their ISO synonyms only, never on text, ntext or an alias type. + private static let characterTypes: Set<[String]> = [ + ["char"], ["varchar"], ["nchar"], ["nvarchar"], ["character"], ["char", "varying"], ["character", "varying"], + ["national", "char"], ["national", "character"], ["national", "char", "varying"], + ["national", "character", "varying"], + ] public static func isParenthesizedExpression(_ text: String) -> Bool { SQLExpressionFragment.isSingleExpression(text, databaseTypeId: databaseTypeId) @@ -54,6 +60,11 @@ public enum MSSQLDefinitionText { return SQLExpressionFragment.isSingleType(source.substring(from: index), databaseTypeId: databaseTypeId) } + public static func isCharacterType(_ text: String) -> Bool { + let name = text.prefix { $0 != "(" }.lowercased().filter { $0 != "[" && $0 != "]" } + return characterTypes.contains(name.split(whereSeparator: \.isWhitespace).map(String.init)) + } + public static func isCollationName(_ text: String) -> Bool { !text.isEmpty && text.unicodeScalars.allSatisfy { scalar in ("A"..."Z").contains(scalar) || ("a"..."z").contains(scalar) || ("0"..."9").contains(scalar) || scalar == "_" @@ -205,6 +216,14 @@ public struct MSSQLDefinitionRefusal: Error, LocalizedError, Sendable, Equatable )) } + public static func foreignKeyAction(_ constraint: String, _ action: String) -> MSSQLDefinitionRefusal { + MSSQLDefinitionRefusal(reason: String( + format: String(localized: "The referential action of foreign key %@ is not one SQL Server has: %@"), + constraint, + action + )) + } + public static func indexFilter(_ index: String) -> MSSQLDefinitionRefusal { MSSQLDefinitionRefusal(reason: String( format: String(localized: "The filter of index %@ is not one SQL expression, so it cannot be written."), diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift index 8984ccce8a..64913d299a 100644 --- a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLSchemaQueries.swift @@ -96,42 +96,171 @@ public enum MSSQLSchemaQueries { } } + // A column keeps COLLATE only where it differs from `databaseCollation`; nil writes every column's. public static func createTableStatement( schema: String, table: String, columns: [MSSQLColumnRow], - primaryKeyColumns: [String], - foreignKeys: [MSSQLForeignKeyRow] + keyConstraints: [MSSQLKeyConstraintRow] = [], + checkConstraints: [MSSQLCheckConstraintRow] = [], + foreignKeys: [MSSQLForeignKeyRow] = [], + databaseCollation: String? ) throws -> String { - var parts = try columns.map { column -> String in - let type = column.displayType.uppercased() - guard MSSQLDefinitionText.isColumnType(type) else { - throw MSSQLDefinitionRefusal.columnType(column.name, type) + var parts = try columns.map { try columnDefinition($0, databaseCollation: databaseCollation) } + parts += keyConstraintDefinitions(keyConstraints) + parts += try foreignKeyDefinitions(foreignKeys, schema: schema) + parts += try checkConstraintDefinitions(checkConstraints, table: table) + let body = parts.map { " " + $0 }.joined(separator: ",\n") + let created = "CREATE TABLE \(bracketed(schema: schema, table: table)) (\n\(body)\n);" + // CREATE TABLE has no way to declare a check disabled or untrusted, and rows the source let past one would + // fail the restore, so it is switched off before any row goes in. + let unchecked = checkConstraints.filter { $0.definition != nil && ($0.isDisabled || $0.isNotTrusted) } + .map { "ALTER TABLE \(bracketed(schema: schema, table: table)) NOCHECK CONSTRAINT \(SQLQuoting.identifier($0.name, quote: .bracket));" } + return ([created] + unchecked).joined(separator: "\n") + } + + // After the rows: a check the source enforced without having checked its rows is turned back on the same way. + public static func checkConstraintReenableStatements( + schema: String, + table: String, + checkConstraints: [MSSQLCheckConstraintRow] + ) -> [String] { + checkConstraints.filter { $0.definition != nil && $0.isNotTrusted && !$0.isDisabled }.map { + "ALTER TABLE \(bracketed(schema: schema, table: table)) WITH NOCHECK CHECK CONSTRAINT " + + "\(SQLQuoting.identifier($0.name, quote: .bracket));" + } + } + + private static func columnDefinition(_ column: MSSQLColumnRow, databaseCollation: String?) throws -> String { + let name = SQLQuoting.identifier(column.name, quote: .bracket) + if column.isComputed { + guard let expression = column.computedDefinition.flatMap(MSSQLDefinitionText.expression) else { + throw MSSQLDefinitionRefusal.computedColumn(column.name) } - var definition = " \(SQLQuoting.identifier(column.name, quote: .bracket)) \(type)" - if column.isIdentity { definition += " IDENTITY(1,1)" } - definition += column.isNullable ? " NULL" : " NOT NULL" - if let value = column.defaultValue { - guard let expression = MSSQLDefinitionText.expression(value) else { - throw MSSQLDefinitionRefusal.columnDefault(column.name) - } - definition += " DEFAULT \(expression)" + // NOT NULL is allowed on a computed column only with PERSISTED. + guard column.isPersisted else { return "\(name) AS \(expression)" } + return "\(name) AS \(expression) PERSISTED" + (column.isNullable ? "" : " NOT NULL") + } + let type = column.displayType.uppercased() + guard MSSQLDefinitionText.isColumnType(type) else { + throw MSSQLDefinitionRefusal.columnType(column.name, type) + } + var definition = "\(name) \(type)" + if let collation = column.collation, collation != databaseCollation { + guard MSSQLDefinitionText.isCollationName(collation) else { + throw MSSQLDefinitionRefusal.collation(column.name, collation) + } + definition += " COLLATE \(collation)" + } + if column.isIdentity { + let identity = "\(column.identitySeed ?? ""),\(column.identityIncrement ?? "")" + guard MSSQLDefinitionText.isIdentitySpecification(identity) else { + throw MSSQLDefinitionRefusal.identity(column.name, identity) + } + definition += " IDENTITY(\(identity))" + } + definition += column.isNullable ? " NULL" : " NOT NULL" + if let value = column.defaultValue { + guard let expression = MSSQLDefinitionText.expression(value) else { + throw MSSQLDefinitionRefusal.columnDefault(column.name) + } + definition += " DEFAULT \(expression)" + } + return definition + } + + private static func keyConstraintDefinitions(_ rows: [MSSQLKeyConstraintRow]) -> [String] { + let constraints = grouped(rows, by: { $0.name }) + let primaryFirst = constraints.filter { $0.first?.isPrimaryKey == true } + + constraints.filter { $0.first?.isPrimaryKey != true } + return primaryFirst.compactMap { keyRows -> String? in + guard let first = keyRows.first else { return nil } + var definition = "CONSTRAINT \(SQLQuoting.identifier(first.name, quote: .bracket))" + definition += first.isPrimaryKey ? " PRIMARY KEY" : " UNIQUE" + if let kind = MSSQLDefinitionText.indexKind(first.indexKind) { + definition += " \(kind)" + } + let keys = keyRows.map { key in + SQLQuoting.identifier(key.columnName, quote: .bracket) + (key.isDescending ? " DESC" : "") + } + return definition + " (\(keys.joined(separator: ", ")))" + } + } + + // A referenced table without a schema resolves against the login's default schema, so it is always qualified. + private static func foreignKeyDefinitions(_ rows: [MSSQLForeignKeyRow], schema: String) throws -> [String] { + try grouped(rows, by: { $0.constraintName }).compactMap { keyRows -> String? in + guard let first = keyRows.first else { return nil } + guard let onDelete = MSSQLDefinitionText.referentialAction(first.onDelete) else { + throw MSSQLDefinitionRefusal.foreignKeyAction(first.constraintName, first.onDelete) + } + guard let onUpdate = MSSQLDefinitionText.referentialAction(first.onUpdate) else { + throw MSSQLDefinitionRefusal.foreignKeyAction(first.constraintName, first.onUpdate) } + let columns = keyRows.map { SQLQuoting.identifier($0.columnName, quote: .bracket) } + let referencedColumns = keyRows.map { SQLQuoting.identifier($0.referencedColumn, quote: .bracket) } + let referencedSchema = first.referencedSchema.flatMap { $0.isEmpty ? nil : $0 } ?? schema + var definition: String = "CONSTRAINT \(SQLQuoting.identifier(first.constraintName, quote: .bracket))" + + " FOREIGN KEY (\(columns.joined(separator: ", ")))" + + " REFERENCES \(bracketed(schema: referencedSchema, table: first.referencedTable))" + + " (\(referencedColumns.joined(separator: ", ")))" + if onDelete != "NO ACTION" { definition += " ON DELETE \(onDelete)" } + if onUpdate != "NO ACTION" { definition += " ON UPDATE \(onUpdate)" } return definition } - if !primaryKeyColumns.isEmpty { - let keys = primaryKeyColumns.map { SQLQuoting.identifier($0, quote: .bracket) }.joined(separator: ", ") - parts.append(" CONSTRAINT \(SQLQuoting.identifier("PK_" + table, quote: .bracket)) PRIMARY KEY (\(keys))") + } + + // An unreadable definition is left out, as an unreadable default is. + private static func checkConstraintDefinitions(_ rows: [MSSQLCheckConstraintRow], table: String) throws -> [String] { + try rows.compactMap { row -> String? in + guard let definition = row.definition else { return nil } + guard let expression = MSSQLDefinitionText.expression(definition) else { + throw MSSQLDefinitionRefusal.checkConstraint(table) + } + return "CONSTRAINT \(SQLQuoting.identifier(row.name, quote: .bracket)) CHECK \(expression)" } - for key in foreignKeys { - parts.append( - " CONSTRAINT \(SQLQuoting.identifier(key.constraintName, quote: .bracket))" - + " FOREIGN KEY (\(SQLQuoting.identifier(key.columnName, quote: .bracket)))" - + " REFERENCES \(SQLQuoting.identifier(key.referencedTable, quote: .bracket))" - + " (\(SQLQuoting.identifier(key.referencedColumn, quote: .bracket)))" - ) + } + + private static func grouped(_ rows: [Row], by key: (Row) -> String) -> [[Row]] { + var order: [String] = [] + var groups: [String: [Row]] = [:] + for row in rows { + let name = key(row) + if groups[name] == nil { order.append(name) } + groups[name, default: []].append(row) } - return "CREATE TABLE \(bracketed(schema: schema, table: table)) (\n" + parts.joined(separator: ",\n") + "\n);" + return order.compactMap { groups[$0] } + } + + // SQL Server gives a column altered without COLLATE the database default, so a character column restates the + // collation the new definition carries, or else the one it had. + public static func alterColumnCollation(type: String, collation: String?, previous: String?) -> String? { + guard MSSQLDefinitionText.isCharacterType(type) else { return nil } + return collation.flatMap(nonEmpty) ?? previous.flatMap(nonEmpty) + } + + public static func alterColumnStatement( + qualifiedTable: String, + column: String, + type: String, + collation: String?, + isNullable: Bool + ) throws -> String { + guard MSSQLDefinitionText.isColumnType(type) else { + throw MSSQLDefinitionRefusal.columnType(column, type) + } + var statement = "ALTER TABLE \(qualifiedTable) ALTER COLUMN \(SQLQuoting.identifier(column, quote: .bracket)) \(type)" + if let collation { + guard MSSQLDefinitionText.isCollationName(collation) else { + throw MSSQLDefinitionRefusal.collation(column, collation) + } + statement += " COLLATE \(collation)" + } + return statement + (isNullable ? " NULL" : " NOT NULL") + } + + private static func nonEmpty(_ text: String) -> String? { + text.isEmpty ? nil : text } public static func qualifiedName(schema: String?, table: String) -> String { @@ -237,7 +366,7 @@ public enum MSSQLSchemaQueries { SELECT \(columnSelectList) FROM INFORMATION_SCHEMA.COLUMNS c - \(columnDefaultJoin) + \(columnCatalogJoin) LEFT JOIN ( SELECT kcu.COLUMN_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc @@ -262,7 +391,7 @@ public enum MSSQLSchemaQueries { \(columnSelectList), c.TABLE_NAME FROM INFORMATION_SCHEMA.COLUMNS c - \(columnDefaultJoin) + \(columnCatalogJoin) LEFT JOIN ( SELECT kcu.TABLE_NAME, kcu.COLUMN_NAME FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc @@ -277,11 +406,12 @@ public enum MSSQLSchemaQueries { """ } - public static let allColumnsTableNameIndex = 11 + public static let allColumnsTableNameIndex = 16 - /// A `rowversion` column reports its type as `timestamp`, and a system-versioned period or ledger column reports a - /// `GeneratedAlwaysType` above zero. Neither is identity or computed, and the server refuses a written value for - /// both. Before SQL Server 2016 the property does not exist and reads NULL, which is not above zero. + // A `rowversion` column reports its type as `timestamp`, and a system-versioned period or ledger column reports a + // `GeneratedAlwaysType` above zero. Neither is identity or computed, and the server refuses a written value for + // both. Before SQL Server 2016 the property does not exist and reads NULL, which is not above zero. + // `sys.identity_columns` keeps seed and increment as `sql_variant`, read here as text. private static let columnSelectList = """ c.COLUMN_NAME, c.DATA_TYPE, @@ -297,15 +427,27 @@ public enum MSSQLSchemaQueries { WHEN c.DATA_TYPE = 'timestamp' THEN 1 WHEN COLUMNPROPERTY(\(columnObjectID), c.COLUMN_NAME, 'GeneratedAlwaysType') > 0 THEN 1 ELSE 0 - END AS IS_SERVER_GENERATED + END AS IS_SERVER_GENERATED, + \(columnDefinitionSelectList) + """ + + // The parts a CREATE TABLE restates, after the flags every reader takes, read the same way for a `#name` table. + private static let columnDefinitionSelectList = """ + c.COLLATION_NAME, + cc.definition AS COMPUTED_DEFINITION, + cc.is_persisted AS IS_PERSISTED, + CONVERT(varchar(40), idc.seed_value) AS IDENTITY_SEED, + CONVERT(varchar(40), idc.increment_value) AS IDENTITY_INCREMENT """ private static let columnObjectID = "OBJECT_ID(QUOTENAME(c.TABLE_SCHEMA) + '.' + QUOTENAME(c.TABLE_NAME))" // `INFORMATION_SCHEMA.COLUMNS.COLUMN_DEFAULT` stops at 4,000 characters, which can leave a literal open. - private static let columnDefaultJoin = """ + private static let columnCatalogJoin = """ LEFT JOIN sys.columns sc ON sc.object_id = \(columnObjectID) AND sc.name = c.COLUMN_NAME LEFT JOIN sys.default_constraints dc ON dc.object_id = sc.default_object_id + LEFT JOIN sys.computed_columns cc ON cc.object_id = sc.object_id AND cc.column_id = sc.column_id + LEFT JOIN sys.identity_columns idc ON idc.object_id = sc.object_id AND idc.column_id = sc.column_id """ // A `#name` table lives in `tempdb` whatever database the session is on. @@ -363,10 +505,13 @@ public enum MSSQLSchemaQueries { WHEN c.DATA_TYPE = 'timestamp' THEN 1 WHEN sc.generated_always_type > 0 THEN 1 ELSE 0 - END AS IS_SERVER_GENERATED + END AS IS_SERVER_GENERATED, + \(columnDefinitionSelectList) FROM tempdb.INFORMATION_SCHEMA.COLUMNS c JOIN tempdb.sys.columns sc ON sc.object_id = \(object) AND sc.name = c.COLUMN_NAME LEFT JOIN tempdb.sys.default_constraints dc ON dc.object_id = sc.default_object_id + LEFT JOIN tempdb.sys.computed_columns cc ON cc.object_id = sc.object_id AND cc.column_id = sc.column_id + LEFT JOIN tempdb.sys.identity_columns idc ON idc.object_id = sc.object_id AND idc.column_id = sc.column_id WHERE c.TABLE_NAME = OBJECT_NAME(\(object), DB_ID(N'tempdb')) ORDER BY c.ORDINAL_POSITION """ @@ -401,6 +546,69 @@ public enum MSSQLSchemaQueries { """ } + // Rows are what `indexStatements` reads. A primary key's or UNIQUE constraint's index is left out: + // `createTableStatement` declares the constraint, which builds it. + public static func indexDefinitions(schema: String, table: String) -> String { + let object = MSSQLStringLiteral.quoted(bracketed(schema: schema, table: table)) + return """ + SELECT i.name, i.type_desc, i.is_unique, i.filter_definition, + c.name AS column_name, ic.is_included_column, ic.is_descending_key + FROM sys.indexes i + JOIN sys.index_columns ic + ON i.object_id = ic.object_id AND i.index_id = ic.index_id + JOIN sys.columns c + ON ic.object_id = c.object_id AND ic.column_id = c.column_id + WHERE i.object_id = OBJECT_ID(\(object)) + AND i.name IS NOT NULL + AND i.is_primary_key = 0 + AND i.is_unique_constraint = 0 + AND i.type IN (1, 2) + ORDER BY i.index_id, ic.is_included_column, ic.key_ordinal + """ + } + + // Rows are `name, type, type_desc, column_name, is_descending_key`, one per key column in key order. + public static func keyConstraints(schema: String, table: String) -> String { + let (catalog, object) = catalogObject(schema: schema, table: table) + return """ + SELECT kc.name, kc.type, i.type_desc, c.name AS column_name, ic.is_descending_key + FROM \(catalog)sys.key_constraints kc + JOIN \(catalog)sys.indexes i + ON i.object_id = kc.parent_object_id AND i.index_id = kc.unique_index_id + JOIN \(catalog)sys.index_columns ic + ON ic.object_id = i.object_id AND ic.index_id = i.index_id + JOIN \(catalog)sys.columns c + ON c.object_id = ic.object_id AND c.column_id = ic.column_id + WHERE kc.parent_object_id = \(object) + AND ic.key_ordinal > 0 + ORDER BY CASE kc.type WHEN 'PK' THEN 0 ELSE 1 END, kc.name, ic.key_ordinal + """ + } + + // Rows are `name, definition, is_disabled, is_not_trusted`. + public static func checkConstraints(schema: String, table: String) -> String { + let (catalog, object) = catalogObject(schema: schema, table: table) + return """ + SELECT cc.name, cc.definition, cc.is_disabled, cc.is_not_trusted + FROM \(catalog)sys.check_constraints cc + WHERE cc.parent_object_id = \(object) + ORDER BY cc.name + """ + } + + // A `#name` table's columns take tempdb's default, not the current database's. + public static func databaseCollation(forTable table: String) -> String { + let database = namesTemporaryTable(table) ? "N'tempdb'" : "DB_NAME()" + return "SELECT CONVERT(nvarchar(128), DATABASEPROPERTYEX(\(database), 'Collation'))" + } + + private static func catalogObject(schema: String, table: String) -> (catalog: String, object: String) { + if namesTemporaryTable(table) { + return ("tempdb.", temporaryObjectID(table)) + } + return ("", "OBJECT_ID(\(MSSQLStringLiteral.quoted(bracketed(schema: schema, table: table))))") + } + // One statement for a table's keys and a schema's, so the two reads cannot drift apart. public static func foreignKeys(schema: String, table: String?) -> String { let s = MSSQLStringLiteral.quoted(schema) @@ -455,6 +663,11 @@ public struct MSSQLColumnRow: Sendable, Equatable { public let isPrimaryKey: Bool public let isComputed: Bool public let isServerGenerated: Bool + public let collation: String? + public let computedDefinition: String? + public let isPersisted: Bool + public let identitySeed: String? + public let identityIncrement: String? public init( name: String, @@ -467,7 +680,12 @@ public struct MSSQLColumnRow: Sendable, Equatable { isIdentity: Bool, isPrimaryKey: Bool, isComputed: Bool = false, - isServerGenerated: Bool = false + isServerGenerated: Bool = false, + collation: String? = nil, + computedDefinition: String? = nil, + isPersisted: Bool = false, + identitySeed: String? = nil, + identityIncrement: String? = nil ) { self.name = name self.dataType = dataType @@ -480,6 +698,11 @@ public struct MSSQLColumnRow: Sendable, Equatable { self.isPrimaryKey = isPrimaryKey self.isComputed = isComputed self.isServerGenerated = isServerGenerated + self.collation = collation + self.computedDefinition = computedDefinition + self.isPersisted = isPersisted + self.identitySeed = identitySeed + self.identityIncrement = identityIncrement } /// Whether the server refuses any value written to the column, which is what the host treats as generated. @@ -523,6 +746,36 @@ public struct MSSQLIndexRow: Sendable, Equatable { } } +public struct MSSQLKeyConstraintRow: Sendable, Equatable { + public let name: String + public let isPrimaryKey: Bool + public let indexKind: String? + public let columnName: String + public let isDescending: Bool + + public init(name: String, isPrimaryKey: Bool, indexKind: String?, columnName: String, isDescending: Bool = false) { + self.name = name + self.isPrimaryKey = isPrimaryKey + self.indexKind = indexKind + self.columnName = columnName + self.isDescending = isDescending + } +} + +public struct MSSQLCheckConstraintRow: Sendable, Equatable { + public let name: String + public let definition: String? + public let isDisabled: Bool + public let isNotTrusted: Bool + + public init(name: String, definition: String?, isDisabled: Bool = false, isNotTrusted: Bool = false) { + self.name = name + self.definition = definition + self.isDisabled = isDisabled + self.isNotTrusted = isNotTrusted + } +} + public struct MSSQLForeignKeyRow: Sendable, Equatable { public let constraintName: String public let columnName: String @@ -574,7 +827,33 @@ public extension MSSQLSchemaQueries { isIdentity: (row[safe: 7] ?? nil) == "1", isPrimaryKey: (row[safe: 8] ?? nil) == "1", isComputed: (row[safe: 9] ?? nil) == "1", - isServerGenerated: (row[safe: 10] ?? nil) == "1" + isServerGenerated: (row[safe: 10] ?? nil) == "1", + collation: (row[safe: 11] ?? nil).flatMap { $0.isEmpty ? nil : $0 }, + computedDefinition: row[safe: 12] ?? nil, + isPersisted: (row[safe: 13] ?? nil) == "1", + identitySeed: row[safe: 14] ?? nil, + identityIncrement: row[safe: 15] ?? nil + ) + } + + static func parseKeyConstraintRow(_ row: [String?]) -> MSSQLKeyConstraintRow? { + guard let name = row[safe: 0] ?? nil, let column = row[safe: 3] ?? nil else { return nil } + return MSSQLKeyConstraintRow( + name: name, + isPrimaryKey: (row[safe: 1] ?? nil)?.trimmingCharacters(in: .whitespaces) == "PK", + indexKind: row[safe: 2] ?? nil, + columnName: column, + isDescending: (row[safe: 4] ?? nil) == "1" + ) + } + + static func parseCheckConstraintRow(_ row: [String?]) -> MSSQLCheckConstraintRow? { + guard let name = row[safe: 0] ?? nil else { return nil } + return MSSQLCheckConstraintRow( + name: name, + definition: row[safe: 1] ?? nil, + isDisabled: (row[safe: 2] ?? nil) == "1", + isNotTrusted: (row[safe: 3] ?? nil) == "1" ) } diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/RecordDecoder.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/RecordDecoder.swift index 157d694a8c..27b0827601 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/RecordDecoder.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/RecordDecoder.swift @@ -41,7 +41,8 @@ enum RecordDecoder { case TeradataType.decimal: return try reader.take(decimalByteWidth(precision: column.dataLength >> 8)) default: - throw TeradataWireError.unsupported("record field type \(column.baseCode)") + // NUMBER's length byte is not documented well enough to read, and LOBs need the multipart request. + throw TeradataWireError.unreadableColumn(name: column.name, typeCode: column.typeCode) } } @@ -115,7 +116,7 @@ enum RecordDecoder { let day = Int(packed % 100) return .text(String(format: "%04d-%02d-%02d", year, month, day)) default: - throw TeradataWireError.unsupported("record field type \(column.baseCode)") + throw TeradataWireError.unreadableColumn(name: column.name, typeCode: column.typeCode) } } } diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataColumnType.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataColumnType.swift index 155fe14060..e6b8c0ca40 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataColumnType.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataColumnType.swift @@ -58,6 +58,11 @@ public enum TeradataColumnType { } public static func wireTypeName(_ code: UInt16) -> String { + knownWireTypeName(code) ?? "VARCHAR" + } + + // Codes from the CLIv2 data type table; JSON's from the embedded SQL SQLDA table, which CLIv2 does not list. + public static func knownWireTypeName(_ code: UInt16) -> String? { switch code & 0xFFFE { case TeradataType.byteint: return "BYTEINT" case TeradataType.smallint: return "SMALLINT" @@ -65,7 +70,7 @@ public enum TeradataColumnType { case TeradataType.bigint: return "BIGINT" case TeradataType.float: return "FLOAT" case TeradataType.decimal: return "DECIMAL" - case 604: return "NUMBER" + case TeradataType.number: return "NUMBER" case TeradataType.char: return "CHAR" case TeradataType.varchar: return "VARCHAR" case TeradataType.longVarchar: return "LONG VARCHAR" @@ -76,10 +81,19 @@ public enum TeradataColumnType { case 764: return "TIMESTAMP" case 768: return "TIME WITH TIME ZONE" case 772: return "TIMESTAMP WITH TIME ZONE" - case 400: return "BLOB" - case 416: return "CLOB" - case 880: return "JSON" - default: return "VARCHAR" + case 776, 780, 784, 788, 792, 796, 800, 804, 808, 812, 816, 820, 824: return "INTERVAL" + case 832: return "PERIOD(DATE)" + case 836: return "PERIOD(TIME)" + case 840: return "PERIOD(TIME WITH TIME ZONE)" + case 844: return "PERIOD(TIMESTAMP)" + case 848: return "PERIOD(TIMESTAMP WITH TIME ZONE)" + case 400, 404, 408, 412: return "BLOB" + case 416, 420, 424: return "CLOB" + case 432, 436, 440: return "UDT" + case 504, 508: return "ARRAY" + case 852, 856, 860: return "XML" + case 880, 884, 888: return "JSON" + default: return nil } } diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift index 8f8d3c83fd..6549494aff 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataConnection.swift @@ -29,6 +29,10 @@ public final class TeradataConnection: @unchecked Sendable { private var _operationActive = false // The request a stop is waiting on the server for, so the stop's deadline closes nothing that came after it. private var _stoppingRequest: UInt32? + // Under socketLock: the semantics the session runs with, and what its requests have left open. + private var _sessionMode: TeradataTransactionMode + private var _sessionModeSource: TeradataSessionModeSource + private var _transaction: TeradataTransactionTracker // 3110: "The transaction was aborted by the user." public static let userAbortCode = 3_110 @@ -63,6 +67,10 @@ public final class TeradataConnection: @unchecked Sendable { public init(config: TeradataConnectionConfig) { self.config = config stopGrace = Self.defaultStopGrace + let configured = Self.configuredMode(config.transactionMode) + _sessionMode = configured.mode + _sessionModeSource = configured.source + _transaction = TeradataTransactionTracker(mode: configured.mode) } init( @@ -74,10 +82,27 @@ public final class TeradataConnection: @unchecked Sendable { self._socket = transport self.stopGrace = stopGrace confidentialityBypassed = true + let configured = Self.configuredMode(config.transactionMode) + _sessionMode = configured.mode + _sessionModeSource = configured.source + _transaction = TeradataTransactionTracker(mode: configured.mode) + } + + private static func configuredMode( + _ mode: TeradataTransactionMode + ) -> (mode: TeradataTransactionMode, source: TeradataSessionModeSource) { + mode == .default ? (.default, .unresolved) : (mode, .configured) } public var isConnected: Bool { socket != nil } + // `.default` when the session could not say which semantics it runs with; it then gets Teradata's. + public var sessionTransactionMode: TeradataTransactionMode { socketLock.withLock { _sessionMode } } + + public var sessionModeSource: TeradataSessionModeSource { socketLock.withLock { _sessionModeSource } } + + public var transactionState: TeradataTransactionState { socketLock.withLock { _transaction.state } } + // Set when a request failed partway, so the session it ran on was given up rather than ended. public var hasLostConnection: Bool { socketLock.withLock { _hasLostConnection } } @@ -114,10 +139,7 @@ public final class TeradataConnection: @unchecked Sendable { let params = try Td2Token.parseServerParams(serverToken) try runKeyExchange(params) try runConnect() - - if let database = config.database, !database.isEmpty { - _ = try execute(TeradataSchemaQueries.setDatabase(database)) - } + try prepareSession() _ = try deadline.remainingMilliseconds() established = true } @@ -132,6 +154,84 @@ public final class TeradataConnection: @unchecked Sendable { self.connectDeadline = nil } + // DEFAULT is the site's default, so the session is asked which one it got. An ANSI session opens a transaction + // with its first request and counts DATABASE as DDL, after which only COMMIT is legal, so the setup is committed. + func prepareSession() throws { + let (mode, source) = try resolveSessionMode() + var ranRequest = config.transactionMode == .default + if let database = config.database, !database.isEmpty { + _ = try execute(TeradataSchemaQueries.setDatabase(database)) + ranRequest = true + } + if mode == .ansi, ranRequest { + _ = try execute(TeradataTransactionSQL.commit(.ansi)) + } + socketLock.withLock { + _sessionMode = mode + _sessionModeSource = source + _transaction = TeradataTransactionTracker(mode: mode) + } + } + + private func resolveSessionMode() throws -> (TeradataTransactionMode, TeradataSessionModeSource) { + guard config.transactionMode == .default else { return (config.transactionMode, .configured) } + if let mode = try probeSessionMode( + TeradataSessionModeProbe.helpSession, read: TeradataSessionModeProbe.mode(fromHelpSession:) + ) { + return (mode, .helpSession) + } + if let mode = try probeSessionMode( + TeradataSessionModeProbe.sessionInfo, read: TeradataSessionModeProbe.mode(fromSessionInfo:) + ) { + return (mode, .sessionInfoView) + } + return (.default, .unresolved) + } + + // A refusal, or a reply this decoder cannot read, leaves the session usable for the next way of asking. + private func probeSessionMode( + _ sql: String, + read: (TeradataResultSet) -> TeradataTransactionMode? + ) throws -> TeradataTransactionMode? { + do { + return read(try execute(sql)) + } catch TeradataWireError.cancelled { + throw TeradataWireError.cancelled + } catch { + guard isConnected else { throw error } + return nil + } + } + + public func beginTransaction() throws { + guard let statement = TeradataTransactionSQL.begin(sessionTransactionMode) else { return } + _ = try execute(statement) + } + + public func commitTransaction() throws { + _ = try execute(TeradataTransactionSQL.commit(sessionTransactionMode)) + } + + public func rollbackTransaction() throws { + do { + _ = try execute(TeradataTransactionSQL.rollback(sessionTransactionMode)) + } catch TeradataWireError.server(let code, _) where TeradataTransactionSQL.rollbackAnswerCodes.contains(code) { + // The failure parcel a rollback answers with is its success. + } + socketLock.withLock { _transaction.rolledBack() } + } + + // After DATABASE inside a transaction only the closing statement is legal (ANSI counts it as DDL; Teradata mode + // is not documented either way), so a switch never joins an open one, and in ANSI it is committed at once. + public func switchDatabase(to database: String) throws { + let (mode, state) = socketLock.withLock { (_sessionMode, _transaction.state) } + guard state != .open else { throw TeradataWireError.openTransactionBlocksDatabaseSwitch } + _ = try execute(TeradataSchemaQueries.setDatabase(database)) + if mode == .ansi { + _ = try execute(TeradataTransactionSQL.commit(.ansi)) + } + } + public func execute(_ sql: String) throws -> TeradataResultSet { var columns: [TeradataColumn] = [] var rows: [[TeradataValue]] = [] @@ -179,6 +279,7 @@ public final class TeradataConnection: @unchecked Sendable { onRows: ([[TeradataValue]]) throws -> Bool ) throws -> Int { guard socket != nil else { throw TeradataWireError.connectionFailed("not connected") } + let kinds = TeradataStatementKind.kinds(in: sql) let requestNumber = nextRequest() let body = TeradataMessages.optionsParcel(returnStatementInfo: false).encoded() + TeradataMessages.requestParcel(sql: sql).encoded() @@ -200,14 +301,25 @@ public final class TeradataConnection: @unchecked Sendable { case .notSent: throw TeradataWireError.cancelled case .refused(let error): - if stopped, case .server(let code, _)? = error as? TeradataWireError, code == Self.userAbortCode { - // In Teradata mode a request outside BT is its own transaction, so the abort undid that request alone. - throw config.transactionMode == .ansi ? TeradataWireError.rolledBackByStop : TeradataWireError.cancelled + guard case .server(let code, _)? = error as? TeradataWireError else { + // Refused by its reader rather than the server, so the request ran. + socketLock.withLock { _transaction.ran(kinds) } + throw error + } + let abortRolledBack = socketLock.withLock { () -> Bool in + let heldTransaction = _transaction.abortRollsBackTransaction + _transaction.failed(kinds, code: code) + return heldTransaction + } + if stopped, code == Self.userAbortCode { + throw abortRolledBack ? TeradataWireError.rolledBackByStop : TeradataWireError.cancelled } throw error case .outputEnded where stopped: + socketLock.withLock { _transaction.ran(kinds) } throw TeradataWireError.cancelled case .answered, .outputEnded: + socketLock.withLock { _transaction.ran(kinds) } try reader.finish(onColumns: onColumns) return reader.activityCount } diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift index af9f40875e..af2f79984c 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataSocket.swift @@ -10,6 +10,8 @@ public enum TeradataWireError: Error, CustomStringConvertible, LocalizedError { case cancelled // Stop aborted the request, and the server rolled back the transaction it ran in. case rolledBackByStop + case unreadableColumn(name: String, typeCode: UInt16) + case openTransactionBlocksDatabaseSwitch public var description: String { switch self { @@ -20,9 +22,20 @@ public enum TeradataWireError: Error, CustomStringConvertible, LocalizedError { case .unsupported(let what): return "unsupported: \(what)" case .cancelled: return "cancelled" case .rolledBackByStop: return "stopped, and the server rolled back the transaction (\(TeradataConnection.userAbortCode))" + case .unreadableColumn(let name, let typeCode): return Self.unreadableColumnMessage(name: name, typeCode: typeCode) + case .openTransactionBlocksDatabaseSwitch: + return String(localized: "Commit or roll back the open transaction before switching database.") } } + private static func unreadableColumnMessage(name: String, typeCode: UInt16) -> String { + let typeName = TeradataColumnType.knownWireTypeName(typeCode) ?? String(typeCode) + let format = typeCode & 0xFFFE == TeradataType.number + ? String(localized: "Column \"%1$@\" has Teradata type %2$@, which TablePro cannot read yet. CAST it to DECIMAL or VARCHAR in the query.") + : String(localized: "Column \"%1$@\" has Teradata type %2$@, which TablePro cannot read yet. CAST it to VARCHAR in the query.") + return String(format: format, name, typeName) + } + public var errorDescription: String? { description } } diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataStatementKind.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataStatementKind.swift new file mode 100644 index 0000000000..989c8dbc2a --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataStatementKind.swift @@ -0,0 +1,160 @@ +import Foundation + +// What a statement does to the session's transaction, read from its first keyword. Anything not known to be a read or +// a transaction statement counts as a change, so a misread keeps a transaction reported open rather than hidden. +enum TeradataStatementKind: Equatable { + case read + case change + case begin + case end + case commit + case rollback + + static func kinds(in request: String) -> [TeradataStatementKind] { + TeradataRequestTokens.statements(in: request).map { kind(of: $0[...]) } + } + + private static let readKeywords: Set = ["SELECT", "SEL", "HELP", "SHOW", "EXPLAIN"] + private static let writeKeywords: Set = ["INSERT", "INS", "UPDATE", "UPD", "DELETE", "DEL", "MERGE"] + private static let lockTypes: Set = ["ACCESS", "READ", "WRITE", "EXCLUSIVE", "SHARE", "CHECKSUM", "LOAD"] + private static let lockTypeModifiers: Set = ["MODE", "NOWAIT", "OVERRIDE"] + + private static func kind(of tokens: ArraySlice) -> TeradataStatementKind { + let statement = tokens.drop { $0 == .symbol(UInt8(ascii: "(")) } + guard case .word(let keyword)? = statement.first else { return .change } + let rest = statement.dropFirst() + // `SELECT AND CONSUME` removes the row it reads from a queue table. + if keyword == "SELECT" || keyword == "SEL", rest.first?.word == "AND", rest.dropFirst().first?.word == "CONSUME" { + return .change + } + if readKeywords.contains(keyword) { return .read } + switch keyword { + case "WITH": + return rest.contains { token in token.word.map { writeKeywords.contains($0) } ?? false } ? .change : .read + case "LOCKING", "LOCK": + return lockedStatementKind(rest) + case "BT": + return .begin + case "ET": + return .end + case "BEGIN": + return rest.first?.word == "TRANSACTION" ? .begin : .change + case "END": + return rest.first?.word == "TRANSACTION" ? .end : .change + case "COMMIT": + return .commit + case "ROLLBACK", "ABORT": + return .rollback + default: + return .change + } + } + + // `LOCKING ROW FOR ACCESS SELECT ...`: the statement the modifier applies to follows the lock type. + private static func lockedStatementKind(_ tokens: ArraySlice) -> TeradataStatementKind { + var sawLockType = false + for index in tokens.indices { + guard let word = tokens[index].word else { continue } + if sawLockType { + if lockTypeModifiers.contains(word) { continue } + return kind(of: tokens[index...]) + } + sawLockType = lockTypes.contains(word) + } + return .change + } +} + +// Splits a request at each `;` outside quotes and comments, keeping each statement's words uppercased. +enum TeradataRequestTokens { + enum Token: Equatable { + case word(String) + case quoted + case symbol(UInt8) + + var word: String? { + guard case .word(let word) = self else { return nil } + return word + } + } + + static func statements(in request: String) -> [[Token]] { + let bytes = Array(request.utf8) + var statements: [[Token]] = [] + var current: [Token] = [] + var index = 0 + while index < bytes.count { + let byte = bytes[index] + let following = index + 1 < bytes.count ? bytes[index + 1] : 0 + switch byte { + case UInt8(ascii: ";"): + if !current.isEmpty { statements.append(current) } + current = [] + index += 1 + case UInt8(ascii: "'"), UInt8(ascii: "\""): + current.append(.quoted) + index = endOfQuoted(bytes, from: index) + case UInt8(ascii: "-") where following == UInt8(ascii: "-"): + index = endOfLineComment(bytes, from: index) + case UInt8(ascii: "/") where following == UInt8(ascii: "*"): + index = endOfBlockComment(bytes, from: index) + case UInt8(ascii: " "), UInt8(ascii: "\t"), UInt8(ascii: "\n"), UInt8(ascii: "\r"), 0x0B, 0x0C: + index += 1 + default: + guard isWordByte(byte) else { + current.append(.symbol(byte)) + index += 1 + continue + } + let start = index + while index < bytes.count, isWordByte(bytes[index]) { index += 1 } + current.append(.word(String(bytes: bytes[start.. Bool { + switch byte { + case UInt8(ascii: "a")...UInt8(ascii: "z"), UInt8(ascii: "A")...UInt8(ascii: "Z"), + UInt8(ascii: "0")...UInt8(ascii: "9"), UInt8(ascii: "_"), UInt8(ascii: "$"), UInt8(ascii: "#"): + return true + default: + return byte >= 0x80 + } + } + + // A doubled quote stands for itself; an unclosed one runs to the end. + private static func endOfQuoted(_ bytes: [UInt8], from start: Int) -> Int { + let quote = bytes[start] + var index = start + 1 + while index < bytes.count { + guard bytes[index] == quote else { + index += 1 + continue + } + guard index + 1 < bytes.count, bytes[index + 1] == quote else { return index + 1 } + index += 2 + } + return index + } + + // A carriage return ends the comment too, which leans toward reading more of the request as code. + private static func endOfLineComment(_ bytes: [UInt8], from start: Int) -> Int { + var index = start + 2 + while index < bytes.count, bytes[index] != UInt8(ascii: "\n"), bytes[index] != UInt8(ascii: "\r") { + index += 1 + } + return index + } + + private static func endOfBlockComment(_ bytes: [UInt8], from start: Int) -> Int { + var index = start + 2 + while index + 1 < bytes.count { + if bytes[index] == UInt8(ascii: "*"), bytes[index + 1] == UInt8(ascii: "/") { return index + 2 } + index += 1 + } + return bytes.count + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataTransaction.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataTransaction.swift new file mode 100644 index 0000000000..ca89d7403c --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataTransaction.swift @@ -0,0 +1,129 @@ +import Foundation + +public enum TeradataTransactionState: Sendable, Equatable { + case idle + case open + case unknown +} + +public enum TeradataSessionModeSource: Sendable, Equatable { + case configured + case helpSession + case sessionInfoView + // Neither read answered, so the session runs with Teradata semantics as before and its state is unknown. + case unresolved +} + +// BT and ET are invalid in an ANSI session, and COMMIT in a Teradata one. ANSI opens a transaction with the first +// request, so there is nothing to send to begin one. +enum TeradataTransactionSQL { + // ROLLBACK and ABORT answer with a failure parcel even when they worked: 3514, or 3513 with a message. + static let rollbackAnswerCodes: Set = [3_513, 3_514] + + static func begin(_ mode: TeradataTransactionMode) -> String? { + mode == .ansi ? nil : "BEGIN TRANSACTION" + } + + static func commit(_ mode: TeradataTransactionMode) -> String { + mode == .ansi ? "COMMIT WORK" : "END TRANSACTION" + } + + static func rollback(_ mode: TeradataTransactionMode) -> String { + mode == .ansi ? "ROLLBACK WORK" : "ROLLBACK" + } +} + +enum TeradataSessionModeProbe { + static let helpSession = "HELP SESSION" + static let sessionInfo = "SELECT Transaction_Mode FROM DBC.SessionInfoV WHERE SessionNo = SESSION" + + static func mode(fromHelpSession result: TeradataResultSet) -> TeradataTransactionMode? { + switch value(of: "Transaction Semantics", in: result) { + case "ANSI"?: return .ansi + case "TERADATA"?: return .tera + default: return nil + } + } + + static func mode(fromSessionInfo result: TeradataResultSet) -> TeradataTransactionMode? { + switch value(of: "Transaction_Mode", in: result) { + case "A"?: return .ansi + case "T"?: return .tera + default: return nil + } + } + + // Matched by name, ignoring case, spaces and underscores, since the column's place in HELP SESSION is not fixed. + private static func value(of column: String, in result: TeradataResultSet) -> String? { + let wanted = normalized(column) + guard let index = result.columns.firstIndex(where: { normalized($0.name) == wanted }), + let row = result.rows.first, index < row.count, + case .text(let text) = row[index] else { return nil } + return text.trimmingCharacters(in: .whitespaces).uppercased() + } + + private static func normalized(_ name: String) -> String { + name.uppercased().filter { $0 != " " && $0 != "_" } + } +} + +// Tracks the session's transaction from the requests that ran on it and how each ended. +struct TeradataTransactionTracker { + let mode: TeradataTransactionMode + private var explicitDepth = 0 + private var holdsChanges = false + + init(mode: TeradataTransactionMode) { + self.mode = mode + } + + // An ANSI session that only read holds read locks, which a commit releases harmlessly. + var state: TeradataTransactionState { + switch mode { + case .ansi: return holdsChanges ? .open : .idle + case .tera: return explicitDepth > 0 ? .open : .idle + case .default: return .unknown + } + } + + // An abort rolls back the whole transaction; outside BT a Teradata-mode request is a transaction of its own. + var abortRollsBackTransaction: Bool { + mode == .ansi || explicitDepth > 0 + } + + mutating func ran(_ kinds: [TeradataStatementKind]) { + for kind in kinds { + switch kind { + case .read: + continue + case .change: + holdsChanges = true + case .begin: + if mode == .ansi { holdsChanges = true } else { explicitDepth += 1 } + case .end: + if mode == .ansi { holdsChanges = true } else { explicitDepth = max(explicitDepth - 1, 0) } + case .commit: + if mode == .ansi { holdsChanges = false } + case .rollback: + // Only `ROLLBACK ... WHERE` with a false condition answers success, and it rolled nothing back. + continue + } + } + } + + mutating func failed(_ kinds: [TeradataStatementKind], code: Int) { + if code == TeradataConnection.userAbortCode + || (kinds.contains(.rollback) && TeradataTransactionSQL.rollbackAnswerCodes.contains(code)) { + rolledBack() + return + } + // In Teradata semantics a failure rolls the whole transaction back. In ANSI the docs disagree on whether it + // ends the transaction, so what it held is kept. + if mode != .ansi { explicitDepth = 0 } + } + + mutating func rolledBack() { + explicitDepth = 0 + holdsChanges = false + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataType.swift b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataType.swift index 44c90928d9..7b41f5257e 100644 --- a/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataType.swift +++ b/apple/Packages/TableProCore/Sources/TableProTeradataCore/TeradataType.swift @@ -21,6 +21,7 @@ enum TeradataType { static let integer: UInt16 = 496 static let smallint: UInt16 = 500 static let bigint: UInt16 = 600 + static let number: UInt16 = 604 static let byteint: UInt16 = 756 static let float: UInt16 = 480 static let decimal: UInt16 = 484 diff --git a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLAlterColumnStatementTests.swift b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLAlterColumnStatementTests.swift new file mode 100644 index 0000000000..94b157d804 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLAlterColumnStatementTests.swift @@ -0,0 +1,73 @@ +import Foundation +import TableProMSSQLCore +import TableProSQLGrammar +import Testing + +// SQL Server's ALTER TABLE docs: a column altered without COLLATE gets the database's default collation. +struct MSSQLAlterColumnStatementTests { + @Test("Widening a character column keeps the collation it was read with") + func wideningKeepsTheCollation() throws { + let collation = MSSQLSchemaQueries.alterColumnCollation( + type: "varchar(40)", collation: "Latin1_General_CS_AS", previous: "Latin1_General_CS_AS" + ) + let sql = try MSSQLSchemaQueries.alterColumnStatement( + qualifiedTable: "[dbo].[items]", column: "code", type: "varchar(40)", collation: collation, isNullable: true + ) + #expect(sql == "ALTER TABLE [dbo].[items] ALTER COLUMN [code] varchar(40) COLLATE Latin1_General_CS_AS NULL") + } + + @Test("The new definition's collation wins, and the old one stays when the new one is not known") + func collationPrecedence() { + #expect(MSSQLSchemaQueries.alterColumnCollation(type: "nvarchar(10)", collation: "Latin1_General_BIN2", previous: "Latin1_General_CS_AS") + == "Latin1_General_BIN2") + #expect(MSSQLSchemaQueries.alterColumnCollation(type: "nvarchar(10)", collation: nil, previous: "Latin1_General_CS_AS") + == "Latin1_General_CS_AS") + #expect(MSSQLSchemaQueries.alterColumnCollation(type: "nvarchar(10)", collation: "", previous: "Latin1_General_CS_AS") + == "Latin1_General_CS_AS") + #expect(MSSQLSchemaQueries.alterColumnCollation(type: "nvarchar(10)", collation: nil, previous: nil) == nil) + } + + // ALTER COLUMN refuses COLLATE on any other type, text and ntext included. + @Test("A type other than char, varchar, nchar or nvarchar takes no collation", arguments: [ + "int", "text", "ntext", "xml", "dbo.Code", "varbinary(20)", + ]) + func nonCharacterTypeTakesNoCollation(type: String) { + #expect(MSSQLSchemaQueries.alterColumnCollation(type: type, collation: "Latin1_General_CS_AS", previous: "Latin1_General_CS_AS") == nil) + } + + @Test("A column without a collation is altered as before") + func noCollation() throws { + let sql = try MSSQLSchemaQueries.alterColumnStatement( + qualifiedTable: "[dbo].[items]", column: "qty", type: "bigint", collation: nil, isNullable: false + ) + #expect(sql == "ALTER TABLE [dbo].[items] ALTER COLUMN [qty] bigint NOT NULL") + } + + @Test("A collation that is not a collation name fails the statement and names the column") + func collationThatIsNotANameIsRefused() { + let text = "Latin1_General_CS_AS NULL; DROP TABLE t; --" + #expect(throws: MSSQLDefinitionRefusal.collation("code", text)) { + try MSSQLSchemaQueries.alterColumnStatement( + qualifiedTable: "[dbo].[items]", column: "code", type: "varchar(40)", collation: text, isNullable: true + ) + } + } + + @Test("A type that would start a statement fails the statement and names the column") + func typeThatStartsAStatementIsRefused() { + #expect(throws: MSSQLDefinitionRefusal.columnType("code", "varchar(40) DROP TABLE t")) { + try MSSQLSchemaQueries.alterColumnStatement( + qualifiedTable: "[dbo].[items]", column: "code", type: "varchar(40) DROP TABLE t", collation: nil, isNullable: true + ) + } + } + + @Test("A bracket in the column name stays inside the name") + func bracketInColumnName() throws { + let sql = try MSSQLSchemaQueries.alterColumnStatement( + qualifiedTable: "[dbo].[items]", column: "a] int; DROP TABLE t; --", type: "int", collation: nil, isNullable: true + ) + #expect(sql == "ALTER TABLE [dbo].[items] ALTER COLUMN [a]] int; DROP TABLE t; --] int NULL") + #expect(SQLStatementScanner.allStatements(in: sql, grammar: SQLLexicalProfile.sqlServer).count == 1) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLCreateTableStatementTests.swift b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLCreateTableStatementTests.swift index e95081a1ea..6c8a02c047 100644 --- a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLCreateTableStatementTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLCreateTableStatementTests.swift @@ -4,60 +4,122 @@ import TableProSQLGrammar import Testing struct MSSQLCreateTableStatementTests { - private static func column(_ name: String, type: String = "int", default value: String? = nil) -> MSSQLColumnRow { + private static let serverDefault = "SQL_Latin1_General_CP1_CI_AS" + + private static func column( + _ name: String, + type: String = "int", + length: Int? = nil, + nullable: Bool = true, + default value: String? = nil, + collation: String? = nil + ) -> MSSQLColumnRow { MSSQLColumnRow( name: name, dataType: type, - characterMaxLength: nil, + characterMaxLength: length, numericPrecision: nil, numericScale: nil, - isNullable: true, + isNullable: nullable, defaultValue: value, isIdentity: false, - isPrimaryKey: false + isPrimaryKey: false, + collation: collation + ) + } + + private static func computed( + _ name: String, + definition: String?, + persisted: Bool = false, + nullable: Bool = true + ) -> MSSQLColumnRow { + MSSQLColumnRow( + name: name, + dataType: "numeric", + characterMaxLength: nil, + numericPrecision: 21, + numericScale: 2, + isNullable: nullable, + defaultValue: nil, + isIdentity: false, + isPrimaryKey: false, + isComputed: true, + computedDefinition: definition, + isPersisted: persisted + ) + } + + private static func identity(_ name: String, seed: String?, increment: String?) -> MSSQLColumnRow { + MSSQLColumnRow( + name: name, + dataType: "int", + characterMaxLength: nil, + numericPrecision: 10, + numericScale: 0, + isNullable: false, + defaultValue: nil, + isIdentity: true, + isPrimaryKey: true, + identitySeed: seed, + identityIncrement: increment ) } private static func statement( table: String = "t", columns: [MSSQLColumnRow], - primaryKeyColumns: [String] = [], - foreignKeys: [MSSQLForeignKeyRow] = [] + keyConstraints: [MSSQLKeyConstraintRow] = [], + checkConstraints: [MSSQLCheckConstraintRow] = [], + foreignKeys: [MSSQLForeignKeyRow] = [], + databaseCollation: String? = serverDefault ) throws -> String { try MSSQLSchemaQueries.createTableStatement( schema: "dbo", table: table, columns: columns, - primaryKeyColumns: primaryKeyColumns, - foreignKeys: foreignKeys + keyConstraints: keyConstraints, + checkConstraints: checkConstraints, + foreignKeys: foreignKeys, + databaseCollation: databaseCollation ) } + private static func statementCount(_ ddl: String) -> Int { + SQLStatementScanner.allStatements(in: ddl, grammar: SQLLexicalProfile.sqlServer).count + } + @Test("A bracket in a column name stays inside the name") func bracketInColumnNamesStaysInsideTheName() throws { let ddl = try Self.statement(columns: [ Self.column("a] INT); DROP TABLE [victims]; /*"), Self.column("*/ CREATE TABLE [z] ([y"), ]) - #expect(SQLStatementScanner.allStatements(in: ddl, grammar: SQLLexicalProfile.sqlServer).count == 1) + #expect(Self.statementCount(ddl) == 1) #expect(ddl.contains(" [a]] INT); DROP TABLE [victims]]; /*] INT NULL,\n")) #expect(ddl.contains(" [*/ CREATE TABLE [z]] ([y] INT NULL\n);")) } - @Test("A bracket in the table, key and foreign key names is doubled") + @Test("A bracket in the table, key, constraint and referenced names is doubled") func bracketInKeyNamesIsDoubled() throws { let ddl = try Self.statement( table: "t]x", columns: [Self.column("id]")], - primaryKeyColumns: ["id]"], + keyConstraints: [ + MSSQLKeyConstraintRow(name: "pk]", isPrimaryKey: true, indexKind: "CLUSTERED", columnName: "id]"), + MSSQLKeyConstraintRow(name: "uq]", isPrimaryKey: false, indexKind: "NONCLUSTERED", columnName: "id]"), + ], + checkConstraints: [MSSQLCheckConstraintRow(name: "ck]", definition: "([id]]]>(0))")], foreignKeys: [MSSQLForeignKeyRow( - constraintName: "fk]", columnName: "id]", referencedTable: "r]", referencedColumn: "rc]" + constraintName: "fk]", columnName: "id]", referencedTable: "r]", referencedColumn: "rc]", referencedSchema: "s]" )] ) #expect(ddl.hasPrefix("CREATE TABLE [dbo].[t]]x] (\n")) - #expect(ddl.contains(" CONSTRAINT [PK_t]]x] PRIMARY KEY ([id]]])")) - #expect(ddl.contains(" CONSTRAINT [fk]]] FOREIGN KEY ([id]]]) REFERENCES [r]]] ([rc]]])")) - #expect(SQLStatementScanner.allStatements(in: ddl, grammar: SQLLexicalProfile.sqlServer).count == 1) + #expect(ddl.contains(" CONSTRAINT [pk]]] PRIMARY KEY CLUSTERED ([id]]])")) + #expect(ddl.contains(" CONSTRAINT [uq]]] UNIQUE NONCLUSTERED ([id]]])")) + #expect(ddl.contains(" CONSTRAINT [fk]]] FOREIGN KEY ([id]]]) REFERENCES [s]]].[r]]] ([rc]]])")) + #expect(ddl.contains(" CONSTRAINT [ck]]] CHECK ([id]]]>(0))")) + #expect(Self.statementCount(ddl) == 1) } @Test("A default as the catalog stores it is written as it is") @@ -83,4 +145,286 @@ struct MSSQLCreateTableStatementTests { try Self.statement(columns: [Self.column("x", type: "int drop table t")]) } } + + @Test("Every clause lands in one statement: columns, then the primary key, UNIQUE, FOREIGN KEY and CHECK") + func clauseOrder() throws { + let ddl = try Self.statement( + table: "order_lines", + columns: [Self.column("order_id", nullable: false), Self.column("line_no", nullable: false)], + keyConstraints: [ + MSSQLKeyConstraintRow(name: "UQ_line", isPrimaryKey: false, indexKind: "NONCLUSTERED", columnName: "line_no"), + MSSQLKeyConstraintRow(name: "PK_order_lines", isPrimaryKey: true, indexKind: "CLUSTERED", columnName: "order_id"), + MSSQLKeyConstraintRow(name: "PK_order_lines", isPrimaryKey: true, indexKind: "CLUSTERED", columnName: "line_no"), + ], + checkConstraints: [MSSQLCheckConstraintRow(name: "CK_line", definition: "([line_no]>(0))")], + foreignKeys: [MSSQLForeignKeyRow( + constraintName: "FK_order", columnName: "order_id", referencedTable: "orders", referencedColumn: "id", referencedSchema: "dbo" + )] + ) + #expect(ddl == """ + CREATE TABLE [dbo].[order_lines] ( + [order_id] INT NOT NULL, + [line_no] INT NOT NULL, + CONSTRAINT [PK_order_lines] PRIMARY KEY CLUSTERED ([order_id], [line_no]), + CONSTRAINT [UQ_line] UNIQUE NONCLUSTERED ([line_no]), + CONSTRAINT [FK_order] FOREIGN KEY ([order_id]) REFERENCES [dbo].[orders] ([id]), + CONSTRAINT [CK_line] CHECK ([line_no]>(0)) + ); + """) + #expect(Self.statementCount(ddl) == 1) + } + + // Each catalog row is one key column, and writing one constraint per row declared FK_x twice: error 2714 on restore. + @Test("A composite foreign key is one constraint with its columns in key order") + func compositeForeignKey() throws { + let ddl = try Self.statement( + table: "shipments", + columns: [Self.column("order_id"), Self.column("line_no")], + foreignKeys: [ + MSSQLForeignKeyRow( + constraintName: "FK_x", columnName: "order_id", referencedTable: "order_lines", + referencedColumn: "order_id", referencedSchema: "dbo" + ), + MSSQLForeignKeyRow( + constraintName: "FK_x", columnName: "line_no", referencedTable: "order_lines", + referencedColumn: "line_no", referencedSchema: "dbo" + ), + ] + ) + #expect(ddl == """ + CREATE TABLE [dbo].[shipments] ( + [order_id] INT NULL, + [line_no] INT NULL, + CONSTRAINT [FK_x] FOREIGN KEY ([order_id], [line_no]) REFERENCES [dbo].[order_lines] ([order_id], [line_no]) + ); + """) + } + + @Test("A key into another schema names that schema and keeps its ON DELETE action") + func crossSchemaForeignKeyWithCascade() throws { + let ddl = try Self.statement( + columns: [Self.column("order_id")], + foreignKeys: [MSSQLForeignKeyRow( + constraintName: "FK_orders", columnName: "order_id", referencedTable: "orders", referencedColumn: "id", + referencedSchema: "sales", onDelete: "CASCADE", onUpdate: "NO ACTION" + )] + ) + #expect(ddl.contains(" CONSTRAINT [FK_orders] FOREIGN KEY ([order_id]) REFERENCES [sales].[orders] ([id]) ON DELETE CASCADE\n")) + #expect(!ddl.contains("ON UPDATE")) + } + + @Test("Both actions read from the catalog are written after the reference") + func catalogActionsAreWritten() throws { + let row = try #require(MSSQLSchemaQueries.parseForeignKeyRow( + ["FK_a", "a_id", "a", "id", "sales", "SET_NULL", "CASCADE", "t"] + )) + let ddl = try Self.statement(columns: [Self.column("a_id")], foreignKeys: [row]) + #expect(ddl.contains("REFERENCES [sales].[a] ([id]) ON DELETE SET NULL ON UPDATE CASCADE\n")) + } + + @Test("A key with no referenced schema is qualified with the table's own") + func foreignKeyWithoutSchemaUsesTheTablesSchema() throws { + let ddl = try Self.statement( + columns: [Self.column("a_id")], + foreignKeys: [MSSQLForeignKeyRow(constraintName: "FK_a", columnName: "a_id", referencedTable: "a", referencedColumn: "id")] + ) + #expect(ddl.contains("REFERENCES [dbo].[a] ([id])")) + } + + @Test("An action SQL Server does not have fails the statement and names the key") + func unknownActionIsRefused() { + #expect(throws: MSSQLDefinitionRefusal.foreignKeyAction("FK_a", "CASCADE; DROP TABLE t")) { + try Self.statement( + columns: [Self.column("a_id")], + foreignKeys: [MSSQLForeignKeyRow( + constraintName: "FK_a", columnName: "a_id", referencedTable: "a", referencedColumn: "id", + onDelete: "CASCADE; DROP TABLE t" + )] + ) + } + } + + @Test("The primary key keeps its catalog name, its kind and each column's direction") + func primaryKeyKeepsItsName() throws { + let ddl = try Self.statement( + columns: [Self.column("a", nullable: false), Self.column("b", nullable: false)], + keyConstraints: [ + MSSQLKeyConstraintRow(name: "PK__t__3213E83F", isPrimaryKey: true, indexKind: "NONCLUSTERED", columnName: "a"), + MSSQLKeyConstraintRow( + name: "PK__t__3213E83F", isPrimaryKey: true, indexKind: "NONCLUSTERED", columnName: "b", isDescending: true + ), + ] + ) + #expect(ddl.contains(" CONSTRAINT [PK__t__3213E83F] PRIMARY KEY NONCLUSTERED ([a], [b] DESC)\n")) + #expect(!ddl.contains("PK_t]")) + } + + @Test("A UNIQUE constraint is declared with its kind and its columns in key order") + func uniqueConstraint() throws { + let ddl = try Self.statement( + columns: [Self.column("code", type: "varchar", length: 20), Self.column("region")], + keyConstraints: [ + MSSQLKeyConstraintRow(name: "UQ_code", isPrimaryKey: false, indexKind: "CLUSTERED", columnName: "code"), + MSSQLKeyConstraintRow( + name: "UQ_code", isPrimaryKey: false, indexKind: "CLUSTERED", columnName: "region", isDescending: true + ), + ] + ) + #expect(ddl.contains(" CONSTRAINT [UQ_code] UNIQUE CLUSTERED ([code], [region] DESC)\n")) + } + + @Test("A kind other than the two rowstore kinds is left to the server's default") + func unknownKeyKindIsOmitted() throws { + let ddl = try Self.statement( + columns: [Self.column("code")], + keyConstraints: [MSSQLKeyConstraintRow( + name: "UQ_code", isPrimaryKey: false, indexKind: "CLUSTERED; DROP TABLE t; --", columnName: "code" + )] + ) + #expect(ddl.contains(" CONSTRAINT [UQ_code] UNIQUE ([code])\n")) + #expect(Self.statementCount(ddl) == 1) + } + + @Test("A CHECK constraint is written with its name and the expression the catalog stores") + func checkConstraint() throws { + let ddl = try Self.statement( + columns: [Self.column("qty")], + checkConstraints: [MSSQLCheckConstraintRow(name: "CK_qty", definition: "([qty]>(0))")] + ) + #expect(ddl == "CREATE TABLE [dbo].[t] (\n [qty] INT NULL,\n CONSTRAINT [CK_qty] CHECK ([qty]>(0))\n);") + } + + @Test("A CHECK whose text reaches past one expression fails the statement") + func escapingCheckIsRefused() { + #expect(throws: MSSQLDefinitionRefusal.checkConstraint("t")) { + try Self.statement( + columns: [Self.column("qty")], + checkConstraints: [MSSQLCheckConstraintRow(name: "CK_qty", definition: "([qty]>(0))) DROP TABLE t; --")] + ) + } + } + + @Test("A disabled or untrusted CHECK is switched off before the rows, and an untrusted one back on after them") + func uncheckedCheckConstraints() throws { + let checks = [ + MSSQLCheckConstraintRow(name: "CK_off", definition: "([qty]>(0))", isDisabled: true, isNotTrusted: true), + MSSQLCheckConstraintRow(name: "CK_loose", definition: "([qty]<(9))", isNotTrusted: true), + MSSQLCheckConstraintRow(name: "CK_on", definition: "([qty]<>(5))"), + ] + let ddl = try Self.statement(columns: [Self.column("qty")], checkConstraints: checks) + #expect(ddl.hasSuffix(""" + ); + ALTER TABLE [dbo].[t] NOCHECK CONSTRAINT [CK_off]; + ALTER TABLE [dbo].[t] NOCHECK CONSTRAINT [CK_loose]; + """)) + #expect(Self.statementCount(ddl) == 3) + #expect(MSSQLSchemaQueries.checkConstraintReenableStatements(schema: "dbo", table: "t", checkConstraints: checks) + == ["ALTER TABLE [dbo].[t] WITH NOCHECK CHECK CONSTRAINT [CK_loose];"]) + } + + @Test("A CHECK with no readable definition is left out") + func unreadableCheckIsLeftOut() throws { + let ddl = try Self.statement( + columns: [Self.column("qty")], + checkConstraints: [MSSQLCheckConstraintRow(name: "CK_qty", definition: nil)] + ) + #expect(!ddl.contains("CHECK")) + } + + @Test("A computed column is written as its expression, not as a plain column of its result type") + func computedColumn() throws { + let ddl = try Self.statement(columns: [ + Self.column("qty"), + Self.computed("total", definition: "([qty]*[price])", nullable: false), + ]) + #expect(ddl.contains(" [total] AS ([qty]*[price])\n")) + #expect(!ddl.contains("NUMERIC")) + } + + @Test("A persisted computed column keeps PERSISTED, and NOT NULL only with it") + func persistedComputedColumn() throws { + let notNull = try Self.statement(columns: [Self.computed("total", definition: "([qty]*[price])", persisted: true, nullable: false)]) + #expect(notNull.contains(" [total] AS ([qty]*[price]) PERSISTED NOT NULL\n")) + + let nullable = try Self.statement(columns: [Self.computed("total", definition: "([qty]*[price])", persisted: true)]) + #expect(nullable.contains(" [total] AS ([qty]*[price]) PERSISTED\n")) + } + + @Test("A computed expression that is not one expression, or cannot be read, fails the statement", arguments: [ + "([qty]*[price])) DROP TABLE t; --", nil, + ]) + func unwritableComputedColumnIsRefused(definition: String?) { + #expect(throws: MSSQLDefinitionRefusal.computedColumn("total")) { + try Self.statement(columns: [Self.computed("total", definition: definition)]) + } + } + + @Test("An identity keeps its seed and increment") + func identitySeedAndIncrement() throws { + let ddl = try Self.statement(columns: [Self.identity("id", seed: "1000", increment: "5")]) + #expect(ddl == "CREATE TABLE [dbo].[t] (\n [id] INT IDENTITY(1000,5) NOT NULL\n);") + + let descending = try Self.statement(columns: [Self.identity("id", seed: "-1", increment: "-1")]) + #expect(descending.contains("[id] INT IDENTITY(-1,-1) NOT NULL")) + } + + @Test("An identity without a whole-number seed and increment fails the statement") + func unreadableIdentityIsRefused() { + #expect(throws: MSSQLDefinitionRefusal.identity("id", ",")) { + try Self.statement(columns: [Self.identity("id", seed: nil, increment: nil)]) + } + #expect(throws: MSSQLDefinitionRefusal.identity("id", "1) NOT NULL, [x] INT --,1")) { + try Self.statement(columns: [Self.identity("id", seed: "1) NOT NULL, [x] INT --", increment: "1")]) + } + } + + @Test("A collation other than the database's is written after the type") + func differingCollationIsWritten() throws { + let ddl = try Self.statement(columns: [ + Self.column("code", type: "varchar", length: 20, collation: "Latin1_General_CS_AS"), + ]) + #expect(ddl == "CREATE TABLE [dbo].[t] (\n [code] VARCHAR(20) COLLATE Latin1_General_CS_AS NULL\n);") + } + + @Test("The database's own collation is not written") + func databaseCollationIsNotWritten() throws { + let ddl = try Self.statement(columns: [ + Self.column("code", type: "varchar", length: 20, collation: Self.serverDefault), + ]) + #expect(ddl == "CREATE TABLE [dbo].[t] (\n [code] VARCHAR(20) NULL\n);") + } + + @Test("Without the database's collation every column states its own") + func unknownDatabaseCollationWritesEveryCollation() throws { + let ddl = try Self.statement( + columns: [Self.column("code", type: "varchar", length: 20, collation: Self.serverDefault)], + databaseCollation: nil + ) + #expect(ddl.contains("[code] VARCHAR(20) COLLATE SQL_Latin1_General_CP1_CI_AS NULL")) + } + + @Test("A collation that is not a collation name fails the statement") + func collationThatIsNotANameIsRefused() { + let text = "Latin1_General_CS_AS NOT NULL, [x] INT" + #expect(throws: MSSQLDefinitionRefusal.collation("code", text)) { + try Self.statement(columns: [Self.column("code", type: "varchar", length: 20, collation: text)]) + } + } + + @Test("A column read from the catalog writes the collation, computed expression and identity it was read with") + func catalogRowsRoundTrip() throws { + let rows: [[String?]] = [ + ["id", "bigint", nil, "19", "0", "NO", nil, "1", "1", "0", "0", nil, nil, nil, "100", "10"], + ["code", "nvarchar", "40", nil, nil, "YES", nil, "0", "0", "0", "0", "Latin1_General_CS_AS", nil, nil, nil, nil], + ["twice", "bigint", nil, "19", "0", "NO", nil, "0", "0", "1", "0", nil, "([id]*(2))", "1", nil, nil], + ] + let ddl = try Self.statement(columns: rows.compactMap(MSSQLSchemaQueries.parseColumnRow)) + #expect(ddl == """ + CREATE TABLE [dbo].[t] ( + [id] BIGINT IDENTITY(100,10) NOT NULL, + [code] NVARCHAR(40) COLLATE Latin1_General_CS_AS NULL, + [twice] AS ([id]*(2)) PERSISTED NOT NULL + ); + """) + } } diff --git a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDefinitionTextTests.swift b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDefinitionTextTests.swift index f2e40d5d1c..fe64211a32 100644 --- a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDefinitionTextTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLDefinitionTextTests.swift @@ -105,6 +105,21 @@ struct MSSQLDefinitionTextTests { #expect(!MSSQLDefinitionText.isCollationName("")) } + @Test("A character type is char, varchar, nchar or nvarchar, or an ISO synonym of one", arguments: [ + "varchar(40)", "NVARCHAR(MAX)", "char", "nchar(10)", "nvarchar (50)", "[nvarchar](50)", "character varying(255)", + "char varying(10)", "national character varying(max)", "national char(10)", "character", + ]) + func characterTypeIsRecognised(type: String) { + #expect(MSSQLDefinitionText.isCharacterType(type)) + } + + @Test("Any other type is not a character type", arguments: [ + "text", "ntext", "national text", "int", "varbinary(20)", "dbo.Code", "[dbo].[varchar]", "xml", "sysname", "", + ]) + func otherTypeIsNotACharacterType(type: String) { + #expect(!MSSQLDefinitionText.isCharacterType(type)) + } + @Test("Only the four referential actions are written") func referentialActions() { #expect(MSSQLDefinitionText.referentialAction("CASCADE") == "CASCADE") diff --git a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift index 5ee9d1e2dd..f005fd9221 100644 --- a/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProMSSQLCoreTests/MSSQLSchemaQueriesTests.swift @@ -145,9 +145,122 @@ struct MSSQLSchemaQueriesTests { @Test func testAllColumnsQueryCarriesTheTableNameAfterTheParsedColumns() { let sql = MSSQLSchemaQueries.allColumns(schema: "dbo") - #expect(sql.contains("AS IS_SERVER_GENERATED,\n c.TABLE_NAME")) + #expect(sql.contains("AS IDENTITY_INCREMENT,\n c.TABLE_NAME")) #expect(sql.contains("c.TABLE_SCHEMA = N'dbo'")) - #expect(MSSQLSchemaQueries.allColumnsTableNameIndex == 11) + #expect(MSSQLSchemaQueries.allColumnsTableNameIndex == 16) + } + + // Without COLLATION_NAME the Structure editor never knew a column's collation, and ALTER COLUMN reset it. + @Test func testColumnReadsSelectWhatACreateTableRestates() { + let reads = [ + MSSQLSchemaQueries.columns(schema: "dbo", table: "orders"), + MSSQLSchemaQueries.allColumns(schema: "dbo"), + MSSQLSchemaQueries.temporaryColumns(table: "#orders"), + ] + for sql in reads { + #expect(sql.contains("AS IS_SERVER_GENERATED,\n")) + #expect(sql.contains("c.COLLATION_NAME,")) + #expect(sql.contains("cc.definition AS COMPUTED_DEFINITION,")) + #expect(sql.contains("cc.is_persisted AS IS_PERSISTED,")) + #expect(sql.contains("CONVERT(varchar(40), idc.seed_value) AS IDENTITY_SEED,")) + #expect(sql.contains("CONVERT(varchar(40), idc.increment_value) AS IDENTITY_INCREMENT")) + #expect(sql.contains("computed_columns cc ON cc.object_id = sc.object_id AND cc.column_id = sc.column_id")) + #expect(sql.contains("identity_columns idc ON idc.object_id = sc.object_id AND idc.column_id = sc.column_id")) + } + let temporary = MSSQLSchemaQueries.temporaryColumns(table: "#orders") + #expect(temporary.contains("LEFT JOIN tempdb.sys.computed_columns cc")) + #expect(temporary.contains("LEFT JOIN tempdb.sys.identity_columns idc")) + } + + @Test func testParseColumnRowReadsCollationComputedAndIdentity() throws { + let identity = try #require(MSSQLSchemaQueries.parseColumnRow( + ["id", "int", nil, "10", "0", "NO", nil, "1", "1", "0", "0", nil, nil, "0", "1000", "5"] + )) + #expect(identity.identitySeed == "1000") + #expect(identity.identityIncrement == "5") + #expect(identity.collation == nil) + + let text = try #require(MSSQLSchemaQueries.parseColumnRow( + ["code", "varchar", "20", nil, nil, "YES", nil, "0", "0", "0", "0", "Latin1_General_CS_AS", nil, nil, nil, nil] + )) + #expect(text.collation == "Latin1_General_CS_AS") + + let computed = try #require(MSSQLSchemaQueries.parseColumnRow( + ["total", "numeric", nil, "21", "2", "YES", nil, "0", "0", "1", "0", nil, "([qty]*[price])", "1", nil, nil] + )) + #expect(computed.computedDefinition == "([qty]*[price])") + #expect(computed.isPersisted) + } + + @Test func testParseColumnRowOfTheShorterShapeHasNoDefinitionParts() throws { + let row = try #require(MSSQLSchemaQueries.parseColumnRow(["code", "varchar", "20", nil, nil, "YES", nil, "0", "0"])) + #expect(row.collation == nil) + #expect(row.computedDefinition == nil) + #expect(!row.isPersisted) + #expect(row.identitySeed == nil) + } + + @Test func testKeyConstraintsQueryReadsPrimaryAndUniqueKeysInKeyOrder() { + let sql = MSSQLSchemaQueries.keyConstraints(schema: "sales", table: "o'rders") + #expect(sql.contains("FROM sys.key_constraints kc")) + #expect(sql.contains("ON i.object_id = kc.parent_object_id AND i.index_id = kc.unique_index_id")) + #expect(sql.contains("WHERE kc.parent_object_id = OBJECT_ID(N'[sales].[o''rders]')")) + #expect(sql.contains("ORDER BY CASE kc.type WHEN 'PK' THEN 0 ELSE 1 END, kc.name, ic.key_ordinal")) + #expect(!sql.contains("tempdb")) + } + + @Test func testConstraintReadsOfATemporaryTableUseTempdb() { + let keys = MSSQLSchemaQueries.keyConstraints(schema: "dbo", table: "#t") + #expect(keys.contains("FROM tempdb.sys.key_constraints kc")) + #expect(keys.contains("JOIN tempdb.sys.index_columns ic")) + #expect(keys.contains("WHERE kc.parent_object_id = OBJECT_ID(N'tempdb..' + QUOTENAME(N'#t'))")) + + let checks = MSSQLSchemaQueries.checkConstraints(schema: "dbo", table: "#t") + #expect(checks.contains("FROM tempdb.sys.check_constraints cc")) + #expect(checks.contains("WHERE cc.parent_object_id = OBJECT_ID(N'tempdb..' + QUOTENAME(N'#t'))")) + } + + @Test func testCheckConstraintsQueryReadsTheStoredDefinition() { + let sql = MSSQLSchemaQueries.checkConstraints(schema: "dbo", table: "orders") + #expect(sql.contains("SELECT cc.name, cc.definition, cc.is_disabled, cc.is_not_trusted\n")) + #expect(sql.contains("FROM sys.check_constraints cc")) + #expect(sql.contains("WHERE cc.parent_object_id = OBJECT_ID(N'[dbo].[orders]')")) + } + + @Test func testDatabaseCollationIsTempdbsForATemporaryTable() { + #expect(MSSQLSchemaQueries.databaseCollation(forTable: "orders") + == "SELECT CONVERT(nvarchar(128), DATABASEPROPERTYEX(DB_NAME(), 'Collation'))") + #expect(MSSQLSchemaQueries.databaseCollation(forTable: "#orders") + == "SELECT CONVERT(nvarchar(128), DATABASEPROPERTYEX(N'tempdb', 'Collation'))") + } + + // The CREATE TABLE declares both constraints, so their indexes written again would fail on restore. + @Test func testIndexDefinitionsLeaveOutConstraintIndexes() { + let sql = MSSQLSchemaQueries.indexDefinitions(schema: "dbo", table: "orders") + #expect(sql.contains("AND i.is_primary_key = 0")) + #expect(sql.contains("AND i.is_unique_constraint = 0")) + #expect(sql.contains("WHERE i.object_id = OBJECT_ID(N'[dbo].[orders]')")) + #expect(sql.contains("ORDER BY i.index_id, ic.is_included_column, ic.key_ordinal")) + } + + @Test func testParseKeyConstraintRow() throws { + let primary = try #require(MSSQLSchemaQueries.parseKeyConstraintRow(["PK_orders", "PK", "CLUSTERED", "id", "0"])) + #expect(primary == MSSQLKeyConstraintRow(name: "PK_orders", isPrimaryKey: true, indexKind: "CLUSTERED", columnName: "id")) + + let unique = try #require(MSSQLSchemaQueries.parseKeyConstraintRow(["UQ_code", "UQ", "NONCLUSTERED", "code", "1"])) + #expect(!unique.isPrimaryKey) + #expect(unique.isDescending) + + #expect(MSSQLSchemaQueries.parseKeyConstraintRow(["PK_orders", "PK", "CLUSTERED", nil, "0"]) == nil) + } + + @Test func testParseCheckConstraintRow() { + #expect(MSSQLSchemaQueries.parseCheckConstraintRow(["CK_qty", "([qty]>(0))"]) + == MSSQLCheckConstraintRow(name: "CK_qty", definition: "([qty]>(0))")) + #expect(MSSQLSchemaQueries.parseCheckConstraintRow(["CK_qty", nil])?.definition == nil) + #expect(MSSQLSchemaQueries.parseCheckConstraintRow(["CK_qty", "([qty]>(0))", "1", "0"])?.isDisabled == true) + #expect(MSSQLSchemaQueries.parseCheckConstraintRow(["CK_qty", "([qty]>(0))", "0", "1"])?.isNotTrusted == true) + #expect(MSSQLSchemaQueries.parseCheckConstraintRow([nil, "(1)"]) == nil) } @Test func testParseColumnRowReadsServerGeneratedColumns() { diff --git a/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginRowStreamDeliveryTests.swift b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginRowStreamDeliveryTests.swift new file mode 100644 index 0000000000..bf8ad65eaf --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProPluginKitTests/PluginRowStreamDeliveryTests.swift @@ -0,0 +1,87 @@ +// +// PluginRowStreamDeliveryTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +struct PluginRowStreamDeliveryTests { + private static func rows(_ value: Int) -> PluginStreamElement { + .rows([[.text(String(value))]]) + } + + private static func values(of element: PluginStreamElement) -> [String] { + guard case .rows(let rows) = element else { return [] } + return rows.compactMap { $0.first?.asText } + } + + @Test("A producer on its own thread waits for room and loses no batch") + func blockingDeliveryKeepsEveryBatch() async throws { + let total = PluginRowStream.boundedBufferBatches * 4 + let stream = PluginRowStream.makeBounded { continuation, _ in + Thread.detachNewThread { + for value in 0...makeStream() + var stream: AsyncThrowingStream? = PluginRowStream.makeBounded { continuation, _ in + Thread.detachNewThread { + var delivered = 0 + while PluginRowStream.deliverBlocking(Self.rows(delivered), to: continuation) { delivered += 1 } + ended.continuation.yield(delivered) + ended.continuation.finish() + } + } + var iterator = stream?.makeAsyncIterator() + _ = try await iterator?.next() + iterator = nil + stream = nil + var delivered = Int.max + for await count in ended.stream { delivered = count } + #expect(delivered <= PluginRowStream.boundedBufferBatches + 1) + } + + @Test("A blocked producer gives up when asked to stop, and the batch is not sent") + func blockingDeliveryHonoursStop() { + let (stream, continuation) = AsyncThrowingStream.makeStream( + bufferingPolicy: .bufferingOldest(1) + ) + #expect(PluginRowStream.deliverBlocking(Self.rows(0), to: continuation, stops: { true })) + #expect(!PluginRowStream.deliverBlocking(Self.rows(1), to: continuation, stops: { true })) + _ = stream + } + + @Test("A producer running as a task suspends for room and loses no batch") + func asyncDeliveryKeepsEveryBatch() async throws { + let total = PluginRowStream.boundedBufferBatches * 3 + let stream = PluginRowStream.makeBounded { continuation, _ in + Task { + for value in 0.. [UInt8] { + LanMessage(kind: .start, body: parcels.flatMap { $0.encoded() }).encoded() + } + + static func u16(_ value: Int) -> [UInt8] { + [UInt8((value >> 8) & 0xFF), UInt8(value & 0xFF)] + } + + static func prepInfoX(_ columns: [(type: UInt16, length: Int, name: String)]) -> Parcel { + var body = [UInt8](repeating: 0, count: 8) + u16(0) + u16(columns.count) + for column in columns { + let name = Array(column.name.utf8) + body += u16(Int(column.type)) + body += (0..<8).map { UInt8(truncatingIfNeeded: column.length >> (8 * (7 - $0))) } + body += [0, 0] + body += u16(name.count) + name + body += u16(0) + u16(0) + } + return Parcel(.prepInfoX, body: body) + } + + // One row of VARCHAR columns, each named and holding the text given. + static func textRow(_ cells: [(name: String, value: String)]) -> [UInt8] { + let header = prepInfoX(cells.map { (type: varchar, length: 64, name: $0.name) }) + var record = [UInt8](repeating: 0, count: (cells.count + 7) / 8) + for cell in cells { + let text = Array(cell.value.utf8) + record += u16(text.count) + text + } + return reply([Parcel(.success), header, Parcel(.record, body: record), Parcel(.endRequest)]) + } + + static func failure(code: Int, _ message: String = "refused") -> [UInt8] { + let text = Array(message.utf8) + return reply([Parcel(.failure, body: [UInt8](repeating: 0, count: 8) + u16(code) + u16(text.count) + text)]) + } + + static func connection( + _ replies: [[UInt8]], + mode: TeradataTransactionMode, + database: String? = nil + ) -> (TeradataConnection, StoppableTransport) { + let transport = StoppableTransport(replies: replies) + let config = TeradataConnectionConfig( + host: "td.example", username: "dbc", password: "dbc", database: database, transactionMode: mode + ) + return (TeradataConnection(config: config, transport: transport), transport) + } + + static func sentSQL(_ transport: StoppableTransport) throws -> [String] { + try transport.sent.compactMap { message in + try Parcel.decodeAll(message.body) + .first { $0.flavor == ParcelFlavor.indicRequest.rawValue } + .flatMap { String(bytes: $0.body, encoding: .utf8) } + } + } + + static func serverCode(of body: () throws -> Void) -> Int? { + do { + try body() + return nil + } catch TeradataWireError.server(let code, _) { + return code + } catch { + return nil + } + } +} + +struct TeradataStatementKindTests { + @Test( + "Reads leave the transaction as it was", + arguments: [ + "SELECT 1", + "sel * from sales.t", + "HELP SESSION", + "SHOW TABLE sales.t", + "EXPLAIN DELETE FROM sales.t", + "(SELECT 1) UNION (SELECT 2)", + "/* nightly */ SELECT 1", + "-- nightly\nSELECT 1", + "LOCKING ROW FOR ACCESS SELECT * FROM sales.t", + "LOCK TABLE sales.t FOR READ OVERRIDE SELECT * FROM sales.t", + "LOCKING TABLE a FOR ACCESS LOCKING TABLE b FOR ACCESS SELECT * FROM a, b", + "WITH c (n) AS (SELECT 1) SELECT * FROM c", + ] + ) + func reads(statement: String) { + #expect(TeradataStatementKind.kinds(in: statement) == [.read]) + } + + @Test( + "Anything that is not a known read or transaction statement counts as a change", + arguments: [ + "INSERT INTO sales.t VALUES (1)", + "ins sales.t (1)", + "UPDATE sales.t SET a = 1", + "DELETE sales.t ALL", + "MERGE INTO sales.t USING s ON t.id = s.id WHEN MATCHED THEN UPDATE SET a = s.a", + "CREATE TABLE sales.t (a INTEGER)", + "DATABASE sales", + "SET SESSION DATEFORM = ANSIDATE", + "CALL sales.p()", + "EXEC sales.m", + "BEGIN QUERY LOGGING ON ALL", + "END LOGGING ON ALL ON TABLE sales.t", + "USING (a INTEGER) INSERT INTO sales.t VALUES (:a)", + "LOCKING TABLE sales.t FOR WRITE UPDATE sales.t SET a = 1", + "LOCKING TABLE sales.t FOR WRITE", + "WITH c (n) AS (SELECT 1) INSERT INTO sales.t SELECT n FROM c", + "'not a statement'", + "COLLECT STATISTICS COLUMN (a) ON sales.t", + "SELECT AND CONSUME TOP 1 * FROM sales.q", + "sel and consume top 1 * from sales.q", + ] + ) + func changes(statement: String) { + #expect(TeradataStatementKind.kinds(in: statement) == [.change]) + } + + @Test("Transaction statements are told apart, in either spelling") + func transactionStatements() { + #expect(TeradataStatementKind.kinds(in: "BT") == [.begin]) + #expect(TeradataStatementKind.kinds(in: "begin transaction") == [.begin]) + #expect(TeradataStatementKind.kinds(in: "ET") == [.end]) + #expect(TeradataStatementKind.kinds(in: "END TRANSACTION") == [.end]) + #expect(TeradataStatementKind.kinds(in: "COMMIT") == [.commit]) + #expect(TeradataStatementKind.kinds(in: "COMMIT WORK") == [.commit]) + #expect(TeradataStatementKind.kinds(in: "ROLLBACK WORK") == [.rollback]) + #expect(TeradataStatementKind.kinds(in: "ABORT 'stopped by hand'") == [.rollback]) + } + + @Test("A request is split at each semicolon outside quotes and comments") + func requestSplitting() { + #expect(TeradataStatementKind.kinds(in: "INSERT INTO t VALUES (1); COMMIT;") == [.change, .commit]) + #expect(TeradataStatementKind.kinds(in: "SELECT 'a; COMMIT'") == [.read]) + #expect(TeradataStatementKind.kinds(in: "SELECT 'it''s; COMMIT' FROM t") == [.read]) + #expect(TeradataStatementKind.kinds(in: "SELECT \"COMMIT;\" FROM t") == [.read]) + #expect(TeradataStatementKind.kinds(in: "SELECT 1 -- ; COMMIT\n") == [.read]) + #expect(TeradataStatementKind.kinds(in: "/* ; COMMIT */ SELECT 1") == [.read]) + #expect(TeradataStatementKind.kinds(in: " ; ; ").isEmpty) + } +} + +struct TeradataTransactionTrackerTests { + @Test("In ANSI a change opens the transaction, a read does not, and COMMIT closes it") + func ansiChangesOpenTheTransaction() { + var tracker = TeradataTransactionTracker(mode: .ansi) + tracker.ran([.read]) + #expect(tracker.state == .idle) + tracker.ran([.change]) + #expect(tracker.state == .open) + tracker.ran([.read]) + #expect(tracker.state == .open) + tracker.ran([.commit]) + #expect(tracker.state == .idle) + } + + @Test("In ANSI a rollback's own failure answer closes the transaction and any other failure keeps it") + func ansiFailures() { + var tracker = TeradataTransactionTracker(mode: .ansi) + tracker.ran([.change]) + tracker.failed([.change], code: 3_807) + #expect(tracker.state == .open) + tracker.failed([.change], code: 3_514) + #expect(tracker.state == .open) + tracker.failed([.rollback], code: 3_514) + #expect(tracker.state == .idle) + + tracker.ran([.change]) + tracker.failed([.rollback], code: 3_513) + #expect(tracker.state == .idle) + + tracker.ran([.change]) + tracker.failed([.read], code: TeradataConnection.userAbortCode) + #expect(tracker.state == .idle) + } + + @Test("A rollback that answered success had a condition that held nothing, so the transaction stays open") + func conditionalRollbackKeepsTheTransaction() { + var ansi = TeradataTransactionTracker(mode: .ansi) + ansi.ran([.change, .rollback]) + #expect(ansi.state == .open) + + var tera = TeradataTransactionTracker(mode: .tera) + tera.ran([.begin, .rollback]) + #expect(tera.state == .open) + } + + @Test("In Teradata mode only an explicit transaction is open, and the outermost pair decides") + func teradataModeTracksTheOutermostPair() { + var tracker = TeradataTransactionTracker(mode: .tera) + tracker.ran([.change]) + #expect(tracker.state == .idle) + #expect(!tracker.abortRollsBackTransaction) + tracker.ran([.begin]) + #expect(tracker.state == .open) + #expect(tracker.abortRollsBackTransaction) + tracker.ran([.begin]) + tracker.ran([.end]) + #expect(tracker.state == .open) + tracker.ran([.end]) + #expect(tracker.state == .idle) + tracker.ran([.end]) + #expect(tracker.state == .idle) + tracker.ran([.begin]) + #expect(tracker.state == .open) + } + + @Test("In Teradata mode any failure rolls the explicit transaction back") + func teradataFailureEndsTheTransaction() { + var tracker = TeradataTransactionTracker(mode: .tera) + tracker.ran([.begin, .begin, .change]) + tracker.failed([.change], code: 3_706) + #expect(tracker.state == .idle) + } + + @Test("A session whose mode could not be read reports nothing it cannot know") + func unresolvedModeIsUnknown() { + var tracker = TeradataTransactionTracker(mode: .default) + #expect(tracker.state == .unknown) + tracker.ran([.begin]) + #expect(tracker.state == .unknown) + #expect(tracker.abortRollsBackTransaction) + } +} + +struct TeradataSessionModeProbeTests { + private static func result(_ cells: [(name: String, value: String)]) -> TeradataResultSet { + TeradataResultSet( + columns: cells.map { TeradataColumn(name: $0.name, typeCode: TeradataWire.varchar, dataLength: 64) }, + rows: [cells.map { .text($0.value) }], + activityCount: 1 + ) + } + + @Test("HELP SESSION's Transaction Semantics column is found by its name, wherever it is") + func helpSessionColumnByName() { + let ansi = Self.result([("User Name", "DBC"), ("Transaction Semantics", "ANSI"), ("Current DateForm", "IntegerDate")]) + #expect(TeradataSessionModeProbe.mode(fromHelpSession: ansi) == .ansi) + + let teradata = Self.result([("Transaction Semantics", "Teradata"), ("User Name", "ANSI")]) + #expect(TeradataSessionModeProbe.mode(fromHelpSession: teradata) == .tera) + + let spelled = Self.result([("TRANSACTION_SEMANTICS", "teradata ")]) + #expect(TeradataSessionModeProbe.mode(fromHelpSession: spelled) == .tera) + } + + @Test("A HELP SESSION answer without a known value says nothing") + func helpSessionWithoutTheAnswer() { + #expect(TeradataSessionModeProbe.mode(fromHelpSession: Self.result([("User Name", "ANSI")])) == nil) + #expect(TeradataSessionModeProbe.mode(fromHelpSession: Self.result([("Transaction Semantics", "Other")])) == nil) + let empty = TeradataResultSet( + columns: [TeradataColumn(name: "Transaction Semantics", typeCode: TeradataWire.varchar, dataLength: 8)], + rows: [], + activityCount: 0 + ) + #expect(TeradataSessionModeProbe.mode(fromHelpSession: empty) == nil) + } + + @Test("DBC.SessionInfoV codes the mode as A or T") + func sessionInfoCodes() { + #expect(TeradataSessionModeProbe.mode(fromSessionInfo: Self.result([("Transaction_Mode", "A")])) == .ansi) + #expect(TeradataSessionModeProbe.mode(fromSessionInfo: Self.result([("Transaction_Mode", "T")])) == .tera) + #expect(TeradataSessionModeProbe.mode(fromSessionInfo: Self.result([("Transaction_Mode", "D")])) == nil) + } +} + +struct TeradataSessionSetupTests { + private static let ansiSession = TeradataWire.textRow([("User Name", "DBC"), ("Transaction Semantics", "ANSI")]) + + @Test("DEFAULT asks HELP SESSION, and an ANSI session commits what its setup opened") + func defaultReadsHelpSession() throws { + let (connection, transport) = TeradataWire.connection( + [Self.ansiSession, TeradataWire.finished, TeradataWire.finished], mode: .default, database: "sales" + ) + + try connection.prepareSession() + + #expect(try TeradataWire.sentSQL(transport) == [ + "HELP SESSION", TeradataSchemaQueries.setDatabase("sales"), "COMMIT WORK", + ]) + #expect(connection.sessionTransactionMode == .ansi) + #expect(connection.sessionModeSource == .helpSession) + #expect(connection.transactionState == .idle) + } + + @Test("A HELP SESSION the server refuses falls back to DBC.SessionInfoV") + func refusedHelpSessionFallsBackToTheView() throws { + let (connection, transport) = TeradataWire.connection( + [TeradataWire.failure(code: 3_523), TeradataWire.textRow([("Transaction_Mode", "T")])], mode: .default + ) + + try connection.prepareSession() + + #expect(try TeradataWire.sentSQL(transport) == ["HELP SESSION", TeradataSessionModeProbe.sessionInfo]) + #expect(connection.sessionTransactionMode == .tera) + #expect(connection.sessionModeSource == .sessionInfoView) + #expect(connection.transactionState == .idle) + } + + @Test("A HELP SESSION reply the decoder cannot read falls back to DBC.SessionInfoV") + func unreadableHelpSessionFallsBackToTheView() throws { + let unreadable = TeradataWire.reply([ + TeradataWire.prepInfoX([(605, 20, "Spool")]), Parcel(.record, body: [0x00, 0x01]), Parcel(.endRequest), + ]) + let (connection, _) = TeradataWire.connection( + [unreadable, TeradataWire.textRow([("Transaction_Mode", "A")]), TeradataWire.finished], mode: .default + ) + + try connection.prepareSession() + + #expect(connection.sessionTransactionMode == .ansi) + #expect(connection.sessionModeSource == .sessionInfoView) + } + + @Test("When neither read answers, the session keeps Teradata semantics and its state is unknown") + func unresolvedModeKeepsTodaysBehaviour() throws { + let (connection, transport) = TeradataWire.connection( + [TeradataWire.failure(code: 3_523), TeradataWire.failure(code: 3_523), TeradataWire.finished], mode: .default + ) + + try connection.prepareSession() + #expect(connection.sessionTransactionMode == .default) + #expect(connection.sessionModeSource == .unresolved) + #expect(connection.transactionState == .unknown) + + try connection.beginTransaction() + #expect(try TeradataWire.sentSQL(transport).last == "BEGIN TRANSACTION") + } + + @Test("An explicit mode is trusted without asking") + func explicitModeIsTrusted() throws { + let (ansi, ansiTransport) = TeradataWire.connection( + [TeradataWire.finished, TeradataWire.finished], mode: .ansi, database: "sales" + ) + try ansi.prepareSession() + #expect(try TeradataWire.sentSQL(ansiTransport) == [TeradataSchemaQueries.setDatabase("sales"), "COMMIT WORK"]) + #expect(ansi.sessionModeSource == .configured) + + let (tera, teraTransport) = TeradataWire.connection([], mode: .tera) + try tera.prepareSession() + #expect(teraTransport.sent.isEmpty) + #expect(tera.sessionTransactionMode == .tera) + #expect(tera.transactionState == .idle) + } +} + +struct TeradataTransactionStatementTests { + @Test("ANSI begins with nothing, commits with COMMIT WORK, and reports a change until then") + func ansiTransaction() throws { + let (connection, transport) = TeradataWire.connection( + [TeradataWire.finished, TeradataWire.finished, TeradataWire.finished], mode: .ansi + ) + + try connection.beginTransaction() + #expect(transport.sent.isEmpty) + _ = try connection.execute("SELECT * FROM sales.t") + #expect(connection.transactionState == .idle) + _ = try connection.execute("UPDATE sales.t SET a = 1") + #expect(connection.transactionState == .open) + try connection.commitTransaction() + + #expect(try TeradataWire.sentSQL(transport) == ["SELECT * FROM sales.t", "UPDATE sales.t SET a = 1", "COMMIT WORK"]) + #expect(connection.transactionState == .idle) + } + + @Test("Teradata mode begins with BEGIN TRANSACTION and commits with END TRANSACTION") + func teradataTransaction() throws { + let (connection, transport) = TeradataWire.connection([TeradataWire.finished, TeradataWire.finished], mode: .tera) + + try connection.beginTransaction() + #expect(connection.transactionState == .open) + try connection.commitTransaction() + + #expect(try TeradataWire.sentSQL(transport) == ["BEGIN TRANSACTION", "END TRANSACTION"]) + #expect(connection.transactionState == .idle) + } + + @Test( + "A rollback answered with 3513 or 3514 succeeded", + arguments: [TeradataTransactionMode.ansi, .tera], + [3_513, 3_514] + ) + func rollbackAnswerIsSuccess(mode: TeradataTransactionMode, code: Int) throws { + let (connection, transport) = TeradataWire.connection( + [TeradataWire.finished, TeradataWire.failure(code: code, "User-generated transaction ABORT.")], mode: mode + ) + _ = try connection.execute(mode == .ansi ? "DELETE FROM sales.t" : "BT") + #expect(connection.transactionState == .open) + + try connection.rollbackTransaction() + + #expect(try TeradataWire.sentSQL(transport).last == (mode == .ansi ? "ROLLBACK WORK" : "ROLLBACK")) + #expect(connection.transactionState == .idle) + } + + @Test("Any other failure from a rollback still throws") + func otherRollbackFailureThrows() { + let (connection, _) = TeradataWire.connection([TeradataWire.failure(code: 2_631, "Transaction ABORTed due to deadlock.")], mode: .ansi) + #expect(TeradataWire.serverCode { try connection.rollbackTransaction() } == 2_631) + } + + @Test("The rollback answer code is success only for a rollback") + func rollbackCodeFromAnotherStatementThrows() { + let (connection, _) = TeradataWire.connection([TeradataWire.failure(code: 3_514)], mode: .ansi) + #expect(TeradataWire.serverCode { _ = try connection.execute("UPDATE sales.t SET a = 1") } == 3_514) + } + + @Test("In Teradata mode a failure inside BT ends the transaction") + func teradataFailureInsideTransaction() throws { + let (connection, _) = TeradataWire.connection( + [TeradataWire.finished, TeradataWire.failure(code: 3_706, "Syntax error")], mode: .tera + ) + try connection.beginTransaction() + #expect(TeradataWire.serverCode { _ = try connection.execute("INSRT INTO sales.t VALUES (1)") } == 3_706) + #expect(connection.transactionState == .idle) + } + + @Test("Stop inside BT reports that the server rolled the transaction back") + func stopInsideExplicitTransactionRollsBack() throws { + let aborted = TeradataWire.failure(code: TeradataConnection.userAbortCode, "The transaction was aborted by the user.") + let (connection, transport) = TeradataWire.connection([TeradataWire.finished, aborted], mode: .tera) + try connection.beginTransaction() + transport.beforeReply[1] = { connection.cancel() } + + do { + _ = try connection.execute("SELECT * FROM sales.orders CROSS JOIN sales.items") + Issue.record("expected the Stop to throw") + } catch TeradataWireError.rolledBackByStop { + #expect(connection.transactionState == .idle) + } + } + + @Test("A database switch commits at once in ANSI and never joins an open transaction") + func databaseSwitch() throws { + let (ansi, ansiTransport) = TeradataWire.connection( + [TeradataWire.finished, TeradataWire.finished, TeradataWire.finished], mode: .ansi + ) + try ansi.switchDatabase(to: "sales") + #expect(try TeradataWire.sentSQL(ansiTransport) == [TeradataSchemaQueries.setDatabase("sales"), "COMMIT WORK"]) + #expect(ansi.transactionState == .idle) + + _ = try ansi.execute("UPDATE sales.t SET a = 1") + #expect(throws: TeradataWireError.self) { try ansi.switchDatabase(to: "hr") } + #expect(try TeradataWire.sentSQL(ansiTransport).count == 3) + + let (tera, teraTransport) = TeradataWire.connection([TeradataWire.finished, TeradataWire.finished], mode: .tera) + try tera.switchDatabase(to: "sales") + try tera.beginTransaction() + #expect(throws: TeradataWireError.self) { try tera.switchDatabase(to: "hr") } + #expect(try TeradataWire.sentSQL(teraTransport) == [TeradataSchemaQueries.setDatabase("sales"), "BEGIN TRANSACTION"]) + } +} + +struct TeradataUnreadableColumnTests { + private static func decodingError(type: UInt16, name: String) -> TeradataWireError? { + let reply = TeradataWire.reply([ + TeradataWire.prepInfoX([(type, 20, name)]), Parcel(.record, body: [0x00, 0x01]), Parcel(.endRequest), + ]) + let (connection, _) = TeradataWire.connection([reply], mode: .tera) + do { + _ = try connection.execute("SELECT \(name) FROM sales.t") + return nil + } catch { + return error as? TeradataWireError + } + } + + @Test("A NUMBER column names itself and says to cast it to DECIMAL or VARCHAR") + func numberColumn() throws { + let error = try #require(Self.decodingError(type: 605, name: "amount")) + guard case .unreadableColumn(let name, let typeCode) = error else { + Issue.record("expected an unreadable column, got \(error)") + return + } + #expect(name == "amount") + #expect(typeCode == 605) + #expect(error.description.contains("\"amount\"")) + #expect(error.description.contains("NUMBER")) + #expect(error.description.contains("CAST it to DECIMAL or VARCHAR")) + } + + @Test("Another unreadable type names itself and says to cast it to VARCHAR") + func periodColumn() throws { + let error = try #require(Self.decodingError(type: 833, name: "span")) + #expect(error.description.contains("\"span\"")) + #expect(error.description.contains("PERIOD(DATE)")) + #expect(error.description.contains("CAST it to VARCHAR")) + #expect(!error.description.contains("DECIMAL")) + } + + @Test("A type code with no known name is given by its number") + func unknownTypeCode() throws { + let error = try #require(Self.decodingError(type: 1_234, name: "odd")) + #expect(error.description.contains("type 1234")) + } +} diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleServerRelease.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleServerRelease.swift index 66d1fcceb5..e7732e76a0 100644 --- a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleServerRelease.swift +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleServerRelease.swift @@ -32,4 +32,14 @@ public struct OracleServerRelease: Sendable, Equatable { public var hasVectorInfo: Bool { major > 23 || (major == 23 && update >= 4) } + + // 12.1 added `ALL_CONSTRAINTS.SEARCH_CONDITION_VC`. Before it a check's text is only in the LONG `SEARCH_CONDITION`. + public var hasSearchConditionText: Bool { + major >= 12 + } + + // 23ai brought SQL's BOOLEAN type. Before it `TRUE` and `FALSE` are names, and a statement using one fails ORA-00904. + public var hasBooleanType: Bool { + major >= 23 + } } diff --git a/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTableConstraint.swift b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTableConstraint.swift new file mode 100644 index 0000000000..47ddaa87db --- /dev/null +++ b/apple/Packages/TableProOracle/Sources/TableProOracleCore/OracleTableConstraint.swift @@ -0,0 +1,192 @@ +import Foundation + +// A PRIMARY KEY, UNIQUE or CHECK constraint, as `CREATE TABLE` writes it after the columns. A dump needs the keys as +// constraints, not as unique indexes: a foreign key added later fails with ORA-02270 without one on the columns it +// references. +public struct OracleTableConstraint: Sendable, Equatable { + public enum Kind: Sendable, Equatable { + case primaryKey + case unique + case check + + init?(code: String?) { + switch code { + case "P"?: self = .primaryKey + case "U"?: self = .unique + case "C"?: self = .check + default: return nil + } + } + + var rank: Int { + switch self { + case .primaryKey: return 0 + case .unique: return 1 + case .check: return 2 + } + } + } + + public let name: String + public let kind: Kind + // In key order for a key. For a check, the columns its condition names. + public let columns: [String] + // Nil for a key. + public var condition: String? + // `SEARCH_CONDITION_VC` is a VARCHAR2(4000) that cuts a longer condition, so one of 4,000 bytes may be cut short. + public let conditionMayBeCut: Bool + public let isDeferrable: Bool + public let isInitiallyDeferred: Bool + public let isEnabled: Bool + // An enabled NOVALIDATE constraint was never checked against the rows it already had, so they may break it. + public let isValidated: Bool + // A `SYS_C` name is numbered across the source database, so on another one it may already be taken. + public let hasGeneratedName: Bool + + public init( + name: String, + kind: Kind, + columns: [String], + condition: String? = nil, + conditionMayBeCut: Bool = false, + isDeferrable: Bool = false, + isInitiallyDeferred: Bool = false, + isEnabled: Bool = true, + isValidated: Bool = true, + hasGeneratedName: Bool = false + ) { + self.name = name + self.kind = kind + self.columns = columns + self.condition = condition + self.conditionMayBeCut = conditionMayBeCut + self.isDeferrable = isDeferrable + self.isInitiallyDeferred = isInitiallyDeferred + self.isEnabled = isEnabled + self.isValidated = isValidated + self.hasGeneratedName = hasGeneratedName + } + + // Written disabled so the rows that break it still restore, then enabled after them without a check. + public var enablesAfterRows: Bool { + isEnabled && !isValidated + } + + // A check with a generated name has no name to enable it by, so it is added after the rows instead of written in + // the CREATE TABLE. + public var isAddedAfterRows: Bool { + enablesAfterRows && kind == .check && hasGeneratedName + } + + // A key with a generated name is enabled by what it is, which needs no name. Nil for a check added after the rows. + public func enableAfterRowsStatement(qualifiedTable: String, quote: (String) -> String) -> String? { + guard enablesAfterRows else { return nil } + guard hasGeneratedName else { + return "ALTER TABLE \(qualifiedTable) ENABLE NOVALIDATE CONSTRAINT \(quote(name))" + } + switch kind { + case .primaryKey: + return "ALTER TABLE \(qualifiedTable) ENABLE NOVALIDATE PRIMARY KEY" + case .unique: + return "ALTER TABLE \(qualifiedTable) ENABLE NOVALIDATE UNIQUE (\(columns.map(quote).joined(separator: ", ")))" + case .check: + return nil + } + } + + // Oracle keeps a NOT NULL as a check reading `"COL" IS NOT NULL`, which the column's own definition already + // writes when the column is NOT NULL. + public func restatesNotNull(of notNullColumns: Set) -> Bool { + guard kind == .check, isEnabled, isValidated, columns.count == 1, let column = columns.first, + notNullColumns.contains(column), let condition else { return false } + return condition.trimmingCharacters(in: .whitespacesAndNewlines) + == "\(OracleSchemaQueries.quoteIdentifier(column)) IS NOT NULL" + } + + // A generated name is left for the server to choose again. + public func clause(quote: (String) -> String) -> String { + var parts: [String] = [] + if !hasGeneratedName { + parts.append("CONSTRAINT \(quote(name))") + } + let columnList = columns.map(quote).joined(separator: ", ") + switch kind { + case .primaryKey: + parts.append("PRIMARY KEY (\(columnList))") + case .unique: + parts.append("UNIQUE (\(columnList))") + case .check: + let text = (condition ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + parts.append("CHECK (\(text))") + } + if isDeferrable { + parts.append(isInitiallyDeferred ? "DEFERRABLE INITIALLY DEFERRED" : "DEFERRABLE") + } + if !isEnabled || !isValidated { + parts.append("DISABLE") + } + return parts.joined(separator: " ") + } +} + +extension OracleSchemaQueries { + // One row per constraint column. Before 12.1 a check's text is only in a LONG oracle-nio cannot decode, so only + // the keys are read there. + public static func tableConstraints(schema: String, table: String, release: OracleServerRelease) -> String { + let readsChecks = release.hasSearchConditionText + let condition = readsChecks ? "c.SEARCH_CONDITION_VC" : "NULL" + let conditionBytes = readsChecks ? "LENGTHB(c.SEARCH_CONDITION_VC)" : "NULL" + let types = readsChecks ? "'P', 'U', 'C'" : "'P', 'U'" + return """ + SELECT c.CONSTRAINT_NAME, c.CONSTRAINT_TYPE, c.DEFERRABLE, c.DEFERRED, c.STATUS, c.GENERATED, + \(condition) AS SEARCH_CONDITION_VC, \(conditionBytes) AS CONDITION_BYTES, cc.COLUMN_NAME, + c.VALIDATED + FROM \(OracleDictionary.allConstraints) c + LEFT JOIN \(OracleDictionary.allConsColumns) cc ON cc.OWNER = c.OWNER + AND cc.CONSTRAINT_NAME = c.CONSTRAINT_NAME + WHERE c.OWNER = '\(escapeLiteral(schema))' + AND c.TABLE_NAME = '\(escapeLiteral(table))' + AND c.CONSTRAINT_TYPE IN (\(types)) + ORDER BY c.CONSTRAINT_NAME, cc.POSITION, cc.COLUMN_NAME + """ + } + + // Keys first, then checks, each by name, so a dump reads the same every time. + public static func parseTableConstraints(_ rows: [[OracleRawCell]]) -> [OracleTableConstraint] { + var order: [String] = [] + var firstRows: [String: (row: [OracleRawCell], kind: OracleTableConstraint.Kind)] = [:] + var columns: [String: [String]] = [:] + for row in rows { + guard let name = row[safe: 0]?.stringValue, + let kind = OracleTableConstraint.Kind(code: row[safe: 1]?.stringValue) else { continue } + if firstRows[name] == nil { + order.append(name) + firstRows[name] = (row, kind) + } + if let column = row[safe: 8]?.stringValue { + columns[name, default: []].append(column) + } + } + let constraints = order.compactMap { name -> OracleTableConstraint? in + guard let entry = firstRows[name] else { return nil } + let row = entry.row + let kind = entry.kind + let conditionBytes = row[safe: 7]?.stringValue.flatMap(OracleSchemaQueries.integer) + return OracleTableConstraint( + name: name, + kind: kind, + columns: columns[name] ?? [], + condition: kind == .check ? row[safe: 6]?.stringValue : nil, + conditionMayBeCut: kind == .check && (conditionBytes ?? 0) >= maxSearchConditionBytes, + isDeferrable: row[safe: 2]?.stringValue == "DEFERRABLE", + isInitiallyDeferred: row[safe: 3]?.stringValue == "DEFERRED", + isEnabled: row[safe: 4]?.stringValue != "DISABLED", + isValidated: row[safe: 9]?.stringValue != "NOT VALIDATED", + hasGeneratedName: row[safe: 5]?.stringValue == "GENERATED NAME" + ) + } + return constraints.sorted { ($0.kind.rank, $0.name) < ($1.kind.rank, $1.name) } + } + + static let maxSearchConditionBytes = 4_000 +} diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleDictionaryQualificationTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleDictionaryQualificationTests.swift index 3ae756dc8a..1b9ab70987 100644 --- a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleDictionaryQualificationTests.swift +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleDictionaryQualificationTests.swift @@ -40,6 +40,8 @@ struct OracleDictionaryQualificationTests { ("columns 23ai", OracleSchemaQueries.columns(schema: "HR", table: "T", release: OracleServerRelease(major: 23))), ("indexes", OracleSchemaQueries.indexes(schema: "HR", table: "T")), ("foreignKeys", OracleSchemaQueries.foreignKeys(schema: "HR", table: "T")), + ("tableConstraints 11g", OracleSchemaQueries.tableConstraints(schema: "HR", table: "T", release: OracleServerRelease(major: 11))), + ("tableConstraints 23ai", OracleSchemaQueries.tableConstraints(schema: "HR", table: "T", release: OracleServerRelease(major: 23))), ("allColumns 11g", OracleSchemaQueries.allColumns(schema: "HR", release: OracleServerRelease(major: 11))), ("allColumns 23ai", OracleSchemaQueries.allColumns(schema: "HR", release: OracleServerRelease(major: 23))), ("allForeignKeys", OracleSchemaQueries.allForeignKeys(schema: "HR")), diff --git a/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleTableConstraintTests.swift b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleTableConstraintTests.swift new file mode 100644 index 0000000000..0e89b608f3 --- /dev/null +++ b/apple/Packages/TableProOracle/Tests/TableProOracleCoreTests/OracleTableConstraintTests.swift @@ -0,0 +1,195 @@ +import Foundation +@testable import TableProOracleCore +import Testing + +// An Oracle dump wrote no PRIMARY KEY, UNIQUE or CHECK constraint, so every foreign key it added later failed with +// ORA-02270. Rows are in the column order of `OracleSchemaQueries.tableConstraints`. +struct OracleTableConstraintTests { + private func row( + _ name: String, + _ type: String, + column: String?, + deferrable: String = "NOT DEFERRABLE", + deferred: String = "IMMEDIATE", + status: String = "ENABLED", + generated: String = "USER NAME", + condition: String? = nil, + conditionBytes: Int? = nil, + validated: String = "VALIDATED" + ) -> [OracleRawCell] { + [ + .string(name), .string(type), .string(deferrable), .string(deferred), .string(status), .string(generated), + condition.map(OracleRawCell.string) ?? .null, + conditionBytes.map { OracleRawCell.string(String($0)) } ?? .null, + column.map(OracleRawCell.string) ?? .null, + .string(validated) + ] + } + + // Rows that break a NOVALIDATE constraint are in the source, so it goes in disabled and is enabled after them. + @Test func testANovalidateConstraintIsWrittenDisabledAndEnabledAfterTheRows() { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("PK_LOG", "P", column: "ID", validated: "NOT VALIDATED"), + row("SYS_C0042", "U", column: "CODE", generated: "GENERATED NAME", validated: "NOT VALIDATED"), + row("UQ_REF", "U", column: "REF") + ]) + let quote = OracleSchemaQueries.quoteIdentifier + + #expect(constraints.map { $0.clause(quote: quote) } == [ + "CONSTRAINT \"PK_LOG\" PRIMARY KEY (\"ID\") DISABLE", + "UNIQUE (\"CODE\") DISABLE", + "CONSTRAINT \"UQ_REF\" UNIQUE (\"REF\")" + ]) + #expect(constraints.compactMap { $0.enableAfterRowsStatement(qualifiedTable: "\"S\".\"LOG\"", quote: quote) } == [ + "ALTER TABLE \"S\".\"LOG\" ENABLE NOVALIDATE CONSTRAINT \"PK_LOG\"", + "ALTER TABLE \"S\".\"LOG\" ENABLE NOVALIDATE UNIQUE (\"CODE\")" + ]) + } + + @Test func testOnlyAValidatedNotNullCheckCountsAsRestated() { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("NN_ID", "C", column: "ID", condition: "\"ID\" IS NOT NULL", conditionBytes: 16), + row("NN_ID2", "C", column: "ID", condition: "\"ID\" IS NOT NULL", conditionBytes: 16, validated: "NOT VALIDATED") + ]) + #expect(constraints.map { $0.restatesNotNull(of: ["ID"]) } == [true, false]) + } + + // A generated name was not written, so a key is enabled by what it is and a check is added after the rows. + @Test func testAGeneratedNameNovalidateConstraintNeedsNoName() { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("SYS_C0001", "P", column: "ID", generated: "GENERATED NAME", validated: "NOT VALIDATED"), + row("SYS_C0002", "C", column: "QTY", generated: "GENERATED NAME", condition: "qty > 0", + conditionBytes: 7, validated: "NOT VALIDATED") + ]) + let quote = OracleSchemaQueries.quoteIdentifier + + #expect(constraints.map { $0.isAddedAfterRows } == [false, true]) + #expect(constraints.compactMap { $0.enableAfterRowsStatement(qualifiedTable: "T", quote: quote) } + == ["ALTER TABLE T ENABLE NOVALIDATE PRIMARY KEY"]) + } + + @Test func testRowsGroupIntoConstraintsKeysFirst() { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("CK_QTY", "C", column: "QTY", condition: "qty > 0", conditionBytes: 7), + row("PK_ORDERS", "P", column: "TENANT"), + row("PK_ORDERS", "P", column: "ID"), + row("UQ_CODE", "U", column: "CODE") + ]) + + let names = constraints.map(\.name) + #expect(names == ["PK_ORDERS", "UQ_CODE", "CK_QTY"]) + #expect(constraints.first?.columns == ["TENANT", "ID"]) + #expect(constraints.last?.condition == "qty > 0") + } + + @Test func testAKeyIsWrittenAsAConstraintOnItsColumns() throws { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("PK_ORDERS", "P", column: "TENANT"), + row("PK_ORDERS", "P", column: "ID"), + row("UQ_CODE", "U", column: "CODE") + ]) + + #expect(constraints.map { $0.clause(quote: OracleSchemaQueries.quoteIdentifier) } == [ + "CONSTRAINT \"PK_ORDERS\" PRIMARY KEY (\"TENANT\", \"ID\")", + "CONSTRAINT \"UQ_CODE\" UNIQUE (\"CODE\")" + ]) + } + + @Test func testAGeneratedNameIsLeftForTheServer() throws { + let constraint = try #require(OracleSchemaQueries.parseTableConstraints([ + row("SYS_C008123", "P", column: "ID", generated: "GENERATED NAME") + ]).first) + + #expect(constraint.clause(quote: OracleSchemaQueries.quoteIdentifier) == "PRIMARY KEY (\"ID\")") + } + + @Test func testDeferralAndADisabledStateAreKept() { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("UQ_A", "U", column: "A", deferrable: "DEFERRABLE", deferred: "DEFERRED"), + row("UQ_B", "U", column: "B", deferrable: "DEFERRABLE"), + row("UQ_C", "U", column: "C", status: "DISABLED") + ]) + + #expect(constraints.map { $0.clause(quote: OracleSchemaQueries.quoteIdentifier) } == [ + "CONSTRAINT \"UQ_A\" UNIQUE (\"A\") DEFERRABLE INITIALLY DEFERRED", + "CONSTRAINT \"UQ_B\" UNIQUE (\"B\") DEFERRABLE", + "CONSTRAINT \"UQ_C\" UNIQUE (\"C\") DISABLE" + ]) + } + + @Test func testACheckKeepsItsConditionAsTheCatalogStoresIt() throws { + let constraint = try #require(OracleSchemaQueries.parseTableConstraints([ + row("CK_STATUS", "C", column: "STATUS", condition: "status IN ('A', 'B')\n", conditionBytes: 21) + ]).first) + + #expect(constraint.clause(quote: OracleSchemaQueries.quoteIdentifier) == "CONSTRAINT \"CK_STATUS\" CHECK (status IN ('A', 'B'))") + #expect(!constraint.conditionMayBeCut) + } + + // `SEARCH_CONDITION_VC` holds 4,000 bytes, so a condition that fills it may have been cut. + @Test func testAConditionThatFillsTheColumnMayBeCut() throws { + let constraint = try #require(OracleSchemaQueries.parseTableConstraints([ + row("CK_LONG", "C", column: "A", condition: String(repeating: "a", count: 4_000), conditionBytes: 4_000) + ]).first) + + #expect(constraint.conditionMayBeCut) + } + + @Test func testTheNotNullCheckOfANotNullColumnIsTheColumnsOwn() throws { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("SYS_C0081", "C", column: "ID", generated: "GENERATED NAME", condition: "\"ID\" IS NOT NULL"), + row("NN_NAME", "C", column: "Na\"me", condition: "\"Na\"\"me\" IS NOT NULL"), + row("CK_NOTE", "C", column: "NOTE", condition: "\"NOTE\" IS NOT NULL"), + row("CK_QTY", "C", column: "QTY", condition: "QTY > 0") + ]) + + let restated = constraints.filter { $0.restatesNotNull(of: ["ID", "Na\"me", "QTY"]) }.map(\.name) + #expect(restated == ["NN_NAME", "SYS_C0081"]) + } + + @Test func testAKeyNeverRestatesANotNull() throws { + let constraint = try #require(OracleSchemaQueries.parseTableConstraints([row("PK_T", "P", column: "ID")]).first) + + #expect(!constraint.restatesNotNull(of: ["ID"])) + } + + @Test func testOtherConstraintTypesAreLeftOut() { + let constraints = OracleSchemaQueries.parseTableConstraints([ + row("FK_T", "R", column: "PARENT_ID"), + row("V_T", "V", column: nil) + ]) + + #expect(constraints.isEmpty) + } + + @Test func testTheQueryReadsChecksFrom12On() { + let sql = OracleSchemaQueries.tableConstraints(schema: "HR", table: "ORDERS", release: OracleServerRelease(major: 19)) + + #expect(sql.contains("c.SEARCH_CONDITION_VC AS SEARCH_CONDITION_VC")) + #expect(sql.contains("LENGTHB(c.SEARCH_CONDITION_VC) AS CONDITION_BYTES")) + #expect(sql.contains("c.CONSTRAINT_TYPE IN ('P', 'U', 'C')")) + } + + // 11g keeps a check's text only in the LONG `SEARCH_CONDITION`, which oracle-nio cannot decode. + @Test func testTheQueryReadsOnlyKeysOn11g() { + let sql = OracleSchemaQueries.tableConstraints(schema: "HR", table: "ORDERS", release: OracleServerRelease(major: 11)) + + #expect(!sql.contains("SEARCH_CONDITION_VC)")) + #expect(sql.contains("NULL AS SEARCH_CONDITION_VC")) + #expect(sql.contains("c.CONSTRAINT_TYPE IN ('P', 'U')")) + } + + @Test func testTheQueryEscapesTheOwnerAndTable() { + let sql = OracleSchemaQueries.tableConstraints( + schema: "O'BRIEN", table: "IT'S", release: OracleServerRelease(major: 23) + ) + + #expect(sql.contains("c.OWNER = 'O''BRIEN'")) + #expect(sql.contains("c.TABLE_NAME = 'IT''S'")) + } + + @Test func testBooleanLiteralsArriveWith23ai() { + #expect(!OracleServerRelease(major: 21).hasBooleanType) + #expect(OracleServerRelease(major: 23).hasBooleanType) + } +} diff --git a/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift b/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift index 321389ab22..017eed05f2 100644 --- a/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift +++ b/apple/Plugins/DuckDBDriverPlugin/DuckDBConnection.swift @@ -214,7 +214,7 @@ actor DuckDBConnectionActor { } noteActivity(query) - var opened = try openResult(query: query, parameters: [], connection: conn, unread: .abandon) + var opened = try openResult(query: query, parameters: [], connection: conn, unread: .abandonWhenIdle) defer { duckdb_destroy_result(&opened.result) } try streamResultRows(opened, continuation: continuation) } @@ -667,8 +667,8 @@ actor DuckDBConnectionActor { rows.append(contentsOf: kept == batch.count ? batch[...] : batch.prefix(kept)) return !budget.isExceeded } - if !reachedEnd, opened.isStreaming, !opened.abandonsUnreadRows { - try Self.drain(opened.result) + if !reachedEnd { + try finishUnreadRows(of: opened) } return DuckDBRawResult( @@ -692,10 +692,19 @@ actor DuckDBConnectionActor { ))) var cancelled = false - try readRows(of: opened) { batch in - cancelled = Task.isCancelled - || !DuckDBBoundedStream.deliver(PluginStreamElement.rows(batch), to: continuation) - return !cancelled + let reachedEnd: Bool + do { + reachedEnd = try readRows(of: opened) { batch in + cancelled = Task.isCancelled + || !DuckDBBoundedStream.deliver(PluginStreamElement.rows(batch), to: continuation) + return !cancelled + } + } catch is CancellationError { + try finishUnreadRows(of: opened) + throw CancellationError() + } + if !reachedEnd { + try finishUnreadRows(of: opened) } if cancelled { continuation.finish(throwing: CancellationError()) @@ -743,6 +752,13 @@ actor DuckDBConnectionActor { } } + // A consumer that stops reading inside a transaction must not abort it: a SELECT calling nextval left unfinished + // fails the transaction (scripts/probes/check-duckdb-streaming-results.sh). + private func finishUnreadRows(of opened: OpenedResult) throws { + guard opened.isStreaming, !opened.abandonsUnreadRows else { return } + try Self.drain(opened.result) + } + // Completes a statement whose rows are no longer wanted, so that what it changed is kept. private static func drain(_ result: duckdb_result) throws { var source = result diff --git a/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift b/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift index 571d6669e2..7ea6ea662c 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaPluginDriver.swift @@ -105,7 +105,7 @@ final class HanaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { func executeBoundedQuery(query: String, rowCap: Int) async throws -> PluginQueryResult? { let report = HanaStreamReport() let collected = try await PluginBoundedStream.collect( - stream(query, report: report), + stream(query, report: report, refusesCutValues: false), rowCap: rowCap, startedAt: Date() ) @@ -113,7 +113,7 @@ final class HanaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } func streamRows(query: String) -> AsyncThrowingStream { - stream(query, report: HanaStreamReport()) + stream(query, report: HanaStreamReport(), refusesCutValues: true) } func switchSchema(to schema: String) async throws { @@ -175,12 +175,19 @@ final class HanaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } } - private func stream(_ sql: String, report: HanaStreamReport) -> AsyncThrowingStream { - PluginRowStream.make { continuation, abort in + // A copy, export or compare reads the stream, and would write a value cut at 64 MiB as if it were whole. + private func stream( + _ sql: String, + report: HanaStreamReport, + refusesCutValues: Bool + ) -> AsyncThrowingStream { + PluginRowStream.makeBounded { continuation, abort in let driver = self let producer = Task { do { - try await driver.produce(sql, into: continuation, report: report) + try await driver.produce( + sql, into: continuation, report: report, refusesCutValues: refusesCutValues + ) continuation.finish() } catch { continuation.finish(throwing: error) @@ -195,16 +202,17 @@ final class HanaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private func produce( _ sql: String, into continuation: AsyncThrowingStream.Continuation, - report: HanaStreamReport + report: HanaStreamReport, + refusesCutValues: Bool ) async throws { if HanaExplainStatement.explainedStatement(in: sql) != nil { let result = try await run(sql, parameters: nil, rowCap: nil) - continuation.yield(.header(PluginStreamHeader( + _ = await PluginRowStream.deliver(.header(PluginStreamHeader( columns: result.columns, columnTypeNames: result.columnTypeNames - ))) + )), to: continuation) if !result.rows.isEmpty { - continuation.yield(.rows(result.rows)) + _ = await PluginRowStream.deliver(.rows(result.rows), to: continuation) } return } @@ -216,20 +224,37 @@ final class HanaPluginDriver: PluginDatabaseDriver, @unchecked Sendable { cancellation: query.cancellation, onHeader: { header in report.record(header) - continuation.yield(.header(PluginStreamHeader( + _ = PluginRowStream.deliverBlocking(.header(PluginStreamHeader( columns: header.columns, columnTypeNames: header.columnTypeNames - ))) + )), to: continuation, stops: { query.cancellation.isCancelled }) }, onBatch: { batch in report.record(truncatedLargeObjects: batch.truncatedLobCount) - continuation.yield(.rows(HanaResultMapping.pluginRows(batch.rows))) + // The cursor loop sees the slot cancelled after this batch and closes the cursor. + if refusesCutValues, batch.truncatedLobCount > 0 { + _ = query.cancellation.cancel() + return + } + // Runs on the session's own queue, which waits here while the reader is behind. + let delivered = PluginRowStream.deliverBlocking( + .rows(HanaResultMapping.pluginRows(batch.rows)), + to: continuation, + stops: { query.cancellation.isCancelled } + ) + if !delivered { _ = query.cancellation.cancel() } } ) } catch let failure as HanaBridgeFailure { + if refusesCutValues, report.hasCutLargeObjects { + throw HanaResultMapping.cutLargeObjectError + } let requested = Task.isCancelled || cancellationGate.isCancelled(query.generation) throw HanaFailureMapping.error(for: failure, cancellationRequested: requested) } + if refusesCutValues, report.hasCutLargeObjects { + throw HanaResultMapping.cutLargeObjectError + } } private func beginQuery() -> RunningQuery { diff --git a/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift b/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift index 0b2131cc46..8d1f4f83f9 100644 --- a/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift +++ b/apple/Plugins/HanaDriverPlugin/HanaResultMapping.swift @@ -32,6 +32,15 @@ enum HanaResultMapping { ) } + static var cutLargeObjectError: HanaError { + HanaError( + kind: .invalidResult, + message: String( + localized: "A large object value is longer than the 64 MiB TablePro reads from SAP HANA, so it cannot be copied or exported whole." + ) + ) + } + static func columnMeta( columns: [String], columnTypeNames: [String], @@ -84,7 +93,10 @@ final class HanaStreamReport: @unchecked Sendable { lock.withLock { truncatedLargeObjects += count } } - // A stream header has no field for the warning a result carries in statusMessage, so an export only logs it. + var hasCutLargeObjects: Bool { + lock.withLock { truncatedLargeObjects > 0 } + } + func logTruncatedLargeObjects() { let count = lock.withLock { truncatedLargeObjects } guard count > 0 else { return } diff --git a/apple/Plugins/MSSQLDriverPlugin/FreeTDSConnection.swift b/apple/Plugins/MSSQLDriverPlugin/FreeTDSConnection.swift index c2a30356ce..1f982adca7 100644 --- a/apple/Plugins/MSSQLDriverPlugin/FreeTDSConnection.swift +++ b/apple/Plugins/MSSQLDriverPlugin/FreeTDSConnection.swift @@ -1000,7 +1000,7 @@ nonisolated private struct FreeTDSBatchReader { private mutating func readResultSet(columnCount: Int, errorsBefore: Int) throws -> ResultSetEnd { let descriptors = (1...columnCount).map(describeColumn) if case .stream(let continuation) = sink { - continuation.yield(.header(columns: descriptors)) + _ = PluginRowStream.deliverBlocking(.header(columns: descriptors), to: continuation, stops: readerStopped) headerStreamed = true } @@ -1123,21 +1123,26 @@ nonisolated private struct FreeTDSBatchReader { } } + // A reader that left or a Stop is handled at the top of the row loop. private func streamIfFull(_ rows: inout [[MSSQLRawCell]]) { guard case .stream(let continuation) = sink, rows.count >= MSSQLRowLimits.streamBatchSize else { return } - continuation.yield(.rows(rows)) + guard PluginRowStream.deliverBlocking(.rows(rows), to: continuation, stops: readerStopped) else { return } rows.removeAll(keepingCapacity: true) } private func streamRemaining(_ rows: inout [[MSSQLRawCell]]) { guard case .stream(let continuation) = sink, !rows.isEmpty else { return } - continuation.yield(.rows(rows)) + _ = PluginRowStream.deliverBlocking(.rows(rows), to: continuation, stops: readerStopped) rows.removeAll() } private func finishCountOnlyStream() { guard case .stream(let continuation) = sink, !headerStreamed, errorTotal == 0 else { return } - continuation.yield(.affectedRows(rowsAffected)) + _ = PluginRowStream.deliverBlocking(.affectedRows(rowsAffected), to: continuation, stops: readerStopped) + } + + private func readerStopped() -> Bool { + userCancelled() || consumerStopped() } private mutating func absorbMessages() { diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift index 60fb612887..daab01e389 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLPlugin.swift @@ -700,9 +700,11 @@ final class MSSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { guard let conn = freeTDSConn else { return AsyncThrowingStream { $0.finish(throwing: MSSQLPluginError.notConnected) } } - return PluginRowStream.make { continuation, abort in + return PluginRowStream.makeBounded { continuation, abort in let streamTask = Task { - let coreStream = AsyncThrowingStream { coreContinuation in + let coreStream = AsyncThrowingStream( + bufferingPolicy: .bufferingOldest(PluginRowStream.boundedBufferBatches) + ) { coreContinuation in /// A plain `Task {}` inherits context but is not a child, so cancelling the /// outer task never reaches this one. The abort closure is what does. Task { @@ -721,21 +723,23 @@ final class MSSQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } do { for try await element in coreStream { + let delivered: Bool switch element { case .header(let columns): - continuation.yield(.header(PluginStreamHeader( + delivered = await PluginRowStream.deliver(.header(PluginStreamHeader( columns: columns.map { $0.name }, columnTypeNames: columns.map { $0.type.canonicalName }, estimatedRowCount: nil - ))) + )), to: continuation) case .rows(let batch): let pluginRows: [PluginRow] = batch.map { row in row.map { $0.asPluginCell } } - continuation.yield(.rows(pluginRows)) + delivered = await PluginRowStream.deliver(.rows(pluginRows), to: continuation) case .affectedRows: - break + delivered = true } + guard delivered else { break } } continuation.finish() } catch let error as MSSQLCoreError { diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+DDL.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+DDL.swift index 2eec458c6d..04ee0192ab 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+DDL.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+DDL.swift @@ -36,11 +36,12 @@ extension MSSQLPluginDriver { case .addColumn(let column): return MSSQLTableDefinitionSQL.refusal(for: column) case .modifyColumn(let old, let new): - let writesType = old.dataType != new.dataType || old.isNullable != new.isNullable + let alteration = Self.columnAlteration(from: old, to: new) return MSSQLTableDefinitionSQL.refusal( for: new, - writesType: writesType, - writesDefault: writesType || old.defaultValue != new.defaultValue + writesType: alteration.writesType, + writesDefault: alteration.writesType || old.defaultValue != new.defaultValue, + collation: alteration.writesType ? alteration.collation : nil ) default: return nil @@ -60,7 +61,8 @@ extension MSSQLPluginDriver { func generateModifyColumnSQL(table: String, oldColumn: PluginColumnDefinition, newColumn: PluginColumnDefinition) -> String? { let qt = mssqlQualifiedTable(table) var stmts: [String] = [] - let needsTypeChange = oldColumn.dataType != newColumn.dataType || oldColumn.isNullable != newColumn.isNullable + let alteration = Self.columnAlteration(from: oldColumn, to: newColumn) + let needsTypeChange = alteration.writesType let defaultChanged = oldColumn.defaultValue != newColumn.defaultValue // Rename column first so subsequent statements reference the correct name @@ -88,9 +90,14 @@ extension MSSQLPluginDriver { } if needsTypeChange { - guard MSSQLDefinitionText.isColumnType(newColumn.dataType) else { return nil } - let nullable = newColumn.isNullable ? "NULL" : "NOT NULL" - stmts.append("ALTER TABLE \(qt) ALTER COLUMN \(colName) \(newColumn.dataType) \(nullable)") + guard let alter = try? MSSQLSchemaQueries.alterColumnStatement( + qualifiedTable: qt, + column: newColumn.name, + type: newColumn.dataType, + collation: alteration.collation, + isNullable: newColumn.isNullable + ) else { return nil } + stmts.append(alter) } // The re-add mirrors the drop above. A type or nullability change drops the constraint too, @@ -104,6 +111,18 @@ extension MSSQLPluginDriver { return stmts.isEmpty ? nil : stmts.joined(separator: ";\n") } + // ALTER COLUMN also runs when only the collation changes, since it is the one statement that sets it. + private static func columnAlteration( + from old: PluginColumnDefinition, + to new: PluginColumnDefinition + ) -> (writesType: Bool, collation: String?) { + let collation = MSSQLSchemaQueries.alterColumnCollation( + type: new.dataType, collation: new.collation, previous: old.collation + ) + let changesCollation = collation.map { $0 != old.collation } ?? false + return (old.dataType != new.dataType || old.isNullable != new.isNullable || changesCollation, collation) + } + func generateDropColumnSQL(table: String, columnName: String) -> String? { "ALTER TABLE \(mssqlQualifiedTable(table)) DROP COLUMN \(quoteIdentifier(columnName))" } diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift index 034fe2b2ff..98e731ebcd 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLPluginDriver+Schema.swift @@ -64,7 +64,7 @@ extension MSSQLPluginDriver { func fetchViewRowEditing(view: String, schema: String?) async throws -> PluginViewRowEditing? { let schema = effectiveSchema(schema) - let factRows = try await viewCatalogRows(MSSQLViewSources.viewFactsQuery(schema: schema, view: view)) + let factRows = try await catalogRows(MSSQLViewSources.viewFactsQuery(schema: schema, view: view)) guard let facts = MSSQLViewSources.ViewFacts(rows: factRows) else { return nil } if let refusal = facts.refusal { return .readOnly(PluginViewReadOnlyReason(refusal)) } // Once the relation proved to be a view, a failed read must not let it pass for a table. @@ -77,7 +77,7 @@ extension MSSQLPluginDriver { private func rowEditing(throughView view: String, schema: String) async throws -> PluginViewRowEditing { let browse = MSSQLViewSources.BrowseFacts( - rows: try await viewCatalogRows(MSSQLViewSources.browseQuery(schema: schema, view: view)) + rows: try await catalogRows(MSSQLViewSources.browseQuery(schema: schema, view: view)) ) let baseTable: MSSQLViewSources.BaseTable switch browse.source { @@ -87,7 +87,7 @@ extension MSSQLPluginDriver { baseTable = table } - let probeRows = try await viewCatalogRows(MSSQLViewSources.probesQuery( + let probeRows = try await catalogRows(MSSQLViewSources.probesQuery( schema: schema, view: view, writableColumns: browse.writableColumns @@ -100,7 +100,7 @@ extension MSSQLPluginDriver { probeRows: probeRows ) let base = MSSQLViewSources.BaseFacts( - rows: try await viewCatalogRows(MSSQLViewSources.baseColumnsQuery(baseTable)) + rows: try await catalogRows(MSSQLViewSources.baseColumnsQuery(baseTable)) ) return PluginViewRowEditing.resolve( @@ -126,7 +126,7 @@ extension MSSQLPluginDriver { ) } - private func viewCatalogRows(_ query: String) async throws -> [[String?]] { + private func catalogRows(_ query: String) async throws -> [[String?]] { try await execute(query: query).rows.map { $0.map(\.asText) } } @@ -175,31 +175,18 @@ extension MSSQLPluginDriver { }.sorted { $0.name < $1.name } } - // Only the primary key's index is left out: `fetchTableDDL` never declares a `UNIQUE` constraint, so its index comes here. + // The primary key and UNIQUE constraints are declared in `fetchTableDDL`, which builds their indexes. What runs + // after the rows also turns back on the checks the source enforced without having checked its rows. func fetchIndexDDL(table: String, schema: String?) async throws -> [String] { - // SQL Server allows both `]` and `'` in a name, so it is bracket-quoted and then quoted as a literal. - let objectLiteral = MSSQLStringLiteral.quoted( - MSSQLSchemaQueries.bracketed(schema: effectiveSchema(schema), table: table)) - let sql = """ - SELECT i.name, i.type_desc, i.is_unique, i.filter_definition, - c.name AS column_name, ic.is_included_column, ic.is_descending_key - FROM sys.indexes i - JOIN sys.index_columns ic - ON i.object_id = ic.object_id AND i.index_id = ic.index_id - JOIN sys.columns c - ON ic.object_id = c.object_id AND ic.column_id = c.column_id - WHERE i.object_id = OBJECT_ID(\(objectLiteral)) - AND i.name IS NOT NULL - AND i.is_primary_key = 0 - AND i.type IN (1, 2) - ORDER BY i.index_id, ic.is_included_column, ic.key_ordinal - """ - let result = try await execute(query: sql) + let resolvedSchema = effectiveSchema(schema) + let rows = try await catalogRows(MSSQLSchemaQueries.indexDefinitions(schema: resolvedSchema, table: table)) + let checks = try await catalogRows(MSSQLSchemaQueries.checkConstraints(schema: resolvedSchema, table: table)) + .compactMap(MSSQLSchemaQueries.parseCheckConstraintRow) do { - return try MSSQLSchemaQueries.indexStatements( - rows: result.rows.map { row in row.map { $0.asText } }, - schema: effectiveSchema(schema), - table: table) + return try MSSQLSchemaQueries.indexStatements(rows: rows, schema: resolvedSchema, table: table) + + MSSQLSchemaQueries.checkConstraintReenableStatements( + schema: resolvedSchema, table: table, checkConstraints: checks + ) } catch let refusal as MSSQLDefinitionRefusal { throw MSSQLPluginError(refusal) } @@ -347,21 +334,35 @@ extension MSSQLPluginDriver { } let columns = try await columnRows(table: table, schema: schema) - let indexes = try await fetchIndexes(table: table, schema: schema) + let keyConstraints = try await catalogRows(MSSQLSchemaQueries.keyConstraints(schema: resolvedSchema, table: table)) + .compactMap(MSSQLSchemaQueries.parseKeyConstraintRow) + let checkConstraints = try await catalogRows(MSSQLSchemaQueries.checkConstraints(schema: resolvedSchema, table: table)) + .compactMap(MSSQLSchemaQueries.parseCheckConstraintRow) let foreignKeys = try await foreignKeyRows(table: table, schema: schema) + let collation = await databaseCollation(forTable: table) do { return try MSSQLSchemaQueries.createTableStatement( schema: resolvedSchema, table: table, columns: columns, - primaryKeyColumns: indexes.filter(\.isPrimary).flatMap(\.columns), - foreignKeys: foreignKeys + keyConstraints: keyConstraints, + checkConstraints: checkConstraints, + foreignKeys: foreignKeys, + databaseCollation: collation ) } catch let refusal as MSSQLDefinitionRefusal { throw MSSQLPluginError(refusal) } } + // Unread, every column states its collation, which keeps it rather than changing it. + private func databaseCollation(forTable table: String) async -> String? { + guard let result = try? await execute(query: MSSQLSchemaQueries.databaseCollation(forTable: table)) else { + return nil + } + return result.rows.first?[safe: 0]?.asText + } + func fetchViewDefinition(view: String, schema: String?) async throws -> String { let resolvedSchema = effectiveSchema(schema) let module = try await storedModule(schema: resolvedSchema, name: view) @@ -495,6 +496,7 @@ extension MSSQLColumnRow { isPrimaryKey: isPrimaryKey, defaultValue: defaultValue, extra: isIdentity ? "IDENTITY" : nil, + collation: collation, identityKind: isIdentity ? .always : nil, isGenerated: isGenerated ) diff --git a/apple/Plugins/MSSQLDriverPlugin/MSSQLTableDefinitionSQL.swift b/apple/Plugins/MSSQLDriverPlugin/MSSQLTableDefinitionSQL.swift index bfb50d77a9..945552275e 100644 --- a/apple/Plugins/MSSQLDriverPlugin/MSSQLTableDefinitionSQL.swift +++ b/apple/Plugins/MSSQLDriverPlugin/MSSQLTableDefinitionSQL.swift @@ -144,10 +144,18 @@ enum MSSQLTableDefinitionSQL { return definition } - static func refusal(for column: PluginColumnDefinition, writesType: Bool = true, writesDefault: Bool = true) -> String? { + static func refusal( + for column: PluginColumnDefinition, + writesType: Bool = true, + writesDefault: Bool = true, + collation: String? = nil + ) -> String? { if writesType, !MSSQLDefinitionText.isColumnType(column.dataType) { return MSSQLDefinitionRefusal.columnType(column.name, column.dataType).reason } + if let collation, !MSSQLDefinitionText.isCollationName(collation) { + return MSSQLDefinitionRefusal.collation(column.name, collation).reason + } if writesDefault, let value = column.defaultValue, MSSQLDefinitionText.expression(value) == nil { return MSSQLDefinitionRefusal.columnDefault(column.name).reason } diff --git a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift index ba69a63dc3..4c893cd5ee 100644 --- a/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift +++ b/apple/Plugins/MySQLDriverPlugin/MariaDBPluginConnection.swift @@ -148,6 +148,9 @@ final class MariaDBPluginConnection: @unchecked Sendable { private var baselineSelectLimit: UInt64? private var hasCapturedBaselineSelectLimit = false + // Whether this session already waits long enough for a streaming reader. Only touched from `queue`. + private var hasStreamingWriteTimeout = false + /// Whether the connection that actually succeeded negotiated TLS. `.preferred` falls back to /// plaintext, so the configured mode does not say what the transport ended up being, and the /// `KILL` connection has to repeat what worked rather than what was asked for. @@ -401,6 +404,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { self.appliedSelectLimit = nil self.baselineSelectLimit = nil self.hasCapturedBaselineSelectLimit = false + self.hasStreamingWriteTimeout = false let threadId = mysql_thread_id(handle) self.stateLock.lock() @@ -1451,7 +1455,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { /// The drain belongs to the producer. Enqueued from `onTermination` it would sit behind the /// producer on this one serial queue and run only after the whole result had been read, /// which is why the abort is a polled flag instead. - return PluginRowStream.make { continuation, abort in + return PluginRowStream.makeBounded { continuation, abort in self.queue.async { [self] in do { let stream = { @@ -1494,6 +1498,7 @@ final class MariaDBPluginConnection: @unchecked Sendable { guard !abort.isAborted else { return } try reconcileSelectLimit(rowCap: nil, statement: queryToRun, on: mysql) + raiseStreamingWriteTimeout(on: mysql) let queryStatus = queryToRun.withCString { queryPtr in mysql_real_query(mysql, queryPtr, UInt(queryToRun.utf8.count)) @@ -1520,11 +1525,12 @@ final class MariaDBPluginConnection: @unchecked Sendable { binaryStringsAreText: binaryStringsAreText(in: queryToRun, flavor: sessionFlavor) ) - continuation.yield(.header(PluginStreamHeader( + let stops = { self.cancellationGate.isCancelled(generation) } + _ = PluginRowStream.deliverBlocking(.header(PluginStreamHeader( columns: columns.names, columnTypeNames: columns.typeNames, estimatedRowCount: nil - ))) + )), to: continuation, stops: stops) let batchSize = 5_000 var batch: [PluginRow] = [] @@ -1541,14 +1547,13 @@ final class MariaDBPluginConnection: @unchecked Sendable { } batch.append(textProtocolRow(rowPtr, lengths: mysql_fetch_lengths(resultPtr), columns: columns)) - if batch.count >= batchSize { - continuation.yield(.rows(batch)) + // A reader that left or a Stop is handled at the top of the loop. + if batch.count >= batchSize, + PluginRowStream.deliverBlocking(.rows(batch), to: continuation, stops: stops) { batch.removeAll(keepingCapacity: true) } } - if !batch.isEmpty { - continuation.yield(.rows(batch)) - } + let delivered = batch.isEmpty || PluginRowStream.deliverBlocking(.rows(batch), to: continuation, stops: stops) if mysql_errno(mysql) != 0 { let error = noting(getError(), generation: generation) @@ -1558,6 +1563,24 @@ final class MariaDBPluginConnection: @unchecked Sendable { mysql_free_result(resultPtr) try consumeTrailingResults(on: mysql, generation: generation) + guard delivered else { throw CancellationError() } + } + + // While the stream waits for its reader the server waits on the socket, and gives up after `net_write_timeout`, + // 60 seconds by default. 600 is the value MySQL Connector/J sets for a streaming result + // (`netTimeoutForStreamingResults`). Asked once per session: a server that refuses it keeps its own. + private func raiseStreamingWriteTimeout(on mysql: UnsafeMutablePointer) { + guard !hasStreamingWriteTimeout else { return } + hasStreamingWriteTimeout = true + let statement = "SET SESSION net_write_timeout = GREATEST(@@SESSION.net_write_timeout, 600)" + let status = statement.withCString { mysql_real_query(mysql, $0, UInt(statement.utf8.count)) } + if let discarded = mysql_store_result(mysql) { + mysql_free_result(discarded) + } + guard status == 0 else { + logger.warning("net_write_timeout not raised for a stream: \(self.errorMessage(from: mysql))") + return + } } // MARK: - Trailing Results diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLCheckConstraints.swift b/apple/Plugins/MySQLDriverPlugin/MySQLCheckConstraints.swift index 0d9061812f..aaf9fdf55d 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLCheckConstraints.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLCheckConstraints.swift @@ -78,6 +78,21 @@ internal enum MySQLCheckConstraints { return "ALTER TABLE \(quotedTable) \(keyword) \(quotedName)" } + // CHECK_CONSTRAINTS has no TABLE_NAME here. TABLE_CONSTRAINTS also lists keys, whose names are per table, so + // without the type another table's key named like this table's check would match. + static func joinedCatalogQuery(databaseLiteral: String, tableLiteral: String) -> String { + """ + SELECT cc.CONSTRAINT_NAME, cc.CHECK_CLAUSE + FROM INFORMATION_SCHEMA.CHECK_CONSTRAINTS cc + JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc + ON tc.CONSTRAINT_SCHEMA = cc.CONSTRAINT_SCHEMA + AND tc.CONSTRAINT_NAME = cc.CONSTRAINT_NAME + AND tc.CONSTRAINT_TYPE = 'CHECK' + WHERE cc.CONSTRAINT_SCHEMA = '\(databaseLiteral)' AND tc.TABLE_NAME = '\(tableLiteral)' + ORDER BY cc.CONSTRAINT_NAME + """ + } + static func parse(createTable sql: String) -> [PluginCheckConstraintInfo] { guard let body = MySQLCreateTableScanner.firstGroup(in: Substring(sql)) else { return [] } return MySQLCreateTableScanner.topLevelElements(of: body).compactMap(checkConstraint(in:)) diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Schema.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Schema.swift index 092b8fff7b..6ba43b5c93 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Schema.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver+Schema.swift @@ -311,15 +311,7 @@ internal extension MySQLPluginDriver { ORDER BY CONSTRAINT_NAME """ } else { - query = """ - SELECT cc.CONSTRAINT_NAME, cc.CHECK_CLAUSE - FROM INFORMATION_SCHEMA.CHECK_CONSTRAINTS cc - JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc - ON tc.CONSTRAINT_SCHEMA = cc.CONSTRAINT_SCHEMA - AND tc.CONSTRAINT_NAME = cc.CONSTRAINT_NAME - WHERE cc.CONSTRAINT_SCHEMA = \'\(database)\' AND tc.TABLE_NAME = \'\(safeTable)\' - ORDER BY cc.CONSTRAINT_NAME - """ + query = MySQLCheckConstraints.joinedCatalogQuery(databaseLiteral: database, tableLiteral: safeTable) } let result = try await execute(ownStatement: query) return result.rows.compactMap { row in diff --git a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift index 6778d15221..2328b5082d 100644 --- a/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift +++ b/apple/Plugins/MySQLDriverPlugin/MySQLPluginDriver.swift @@ -1100,7 +1100,8 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { query: String, readOnlySessionState: (@Sendable (Bool) -> PluginSessionTransactionState)? ) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in + // Bounded like the connection's own stream, or this relay would read it dry into an unbounded buffer. + AsyncThrowingStream(bufferingPolicy: .bufferingOldest(PluginRowStream.boundedBufferBatches)) { continuation in let task = Task { do { try await retryingAfterStop { @@ -1108,7 +1109,7 @@ final class MySQLPluginDriver: PluginDatabaseDriver, @unchecked Sendable { defer { self.endOperation(on: conn) } noteActivity(query) for try await element in conn.streamQuery(query, readOnlySessionState: readOnlySessionState) { - continuation.yield(element) + guard await PluginRowStream.deliver(element, to: continuation) else { break } } } continuation.finish() diff --git a/apple/Plugins/MySQLDriverTests/MySQLCheckConstraintsTests.swift b/apple/Plugins/MySQLDriverTests/MySQLCheckConstraintsTests.swift index 1830f44e54..cb6820610a 100644 --- a/apple/Plugins/MySQLDriverTests/MySQLCheckConstraintsTests.swift +++ b/apple/Plugins/MySQLDriverTests/MySQLCheckConstraintsTests.swift @@ -156,4 +156,12 @@ struct MySQLCheckConstraintsTests { quotedTable: "`t`", quotedName: "`c`", banner: "10.6.28-MariaDB", flavor: .mariadb ) == "ALTER TABLE `t` DROP CONSTRAINT `c`") } + + // Measured on MySQL 8.4.13: without the type, table b's UNIQUE key named like table a's check listed a's check. + @Test("The joined catalog read matches only check constraints") + func joinedCatalogQueryMatchesChecksOnly() { + let query = MySQLCheckConstraints.joinedCatalogQuery(databaseLiteral: "shop", tableLiteral: "b") + #expect(query.contains("AND tc.CONSTRAINT_TYPE = 'CHECK'\n")) + #expect(query.contains("WHERE cc.CONSTRAINT_SCHEMA = 'shop' AND tc.TABLE_NAME = 'b'\n")) + } } diff --git a/apple/Plugins/OracleDriverPlugin/OracleBindPlaceholders.swift b/apple/Plugins/OracleDriverPlugin/OracleBindPlaceholders.swift index 37d91df4fa..92029328e5 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleBindPlaceholders.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleBindPlaceholders.swift @@ -4,59 +4,120 @@ // import Foundation +import TableProOracleCore import TableProPluginKit import TableProSQLQuoting -/// Writes the values for the host's `?` placeholders into an Oracle statement. -/// -/// A value a literal can carry is written as the literal the PluginKit default has always written, so a statement the -/// host builds (an import, a copy, an MCP write, a query parameter) reads exactly as it did: numeric text unquoted, -/// other text quoted, NULL as `NULL`. What no literal can carry is bound as a positional `:1 ... :n`, numbered in -/// order: bytes, because Oracle has no `X'..'` literal (ORA-03046), and text over the literal limit (ORA-01704). -/// -/// Only a `?` that is a placeholder is replaced. One inside a string literal, an alternative-quoted `q'[...]'` -/// literal (any delimiter, with or without the national `n` prefix), a double-quoted name such as `"Paid?"`, or a -/// `--` or `/* */` comment is text, and replacing it would change the statement and shift every value after it. +// Writes the values for the host's `?` placeholders into an Oracle statement. Each value is written for the type of +// the column it goes into when the host names one, the same way a grid save writes it; else for the kind the user +// gave the parameter; else as the PluginKit default writes untyped text. What no literal can carry is bound as a +// positional `:1 ... :n`, numbered in order: bytes, because Oracle has no `X'..'` literal (ORA-03046), and text over +// the literal limit (ORA-01704). No session NLS setting is changed: a quoted decimal fails with ORA-01722 under +// `NLS_NUMERIC_CHARACTERS = ',.'`, so a number goes in as a validated literal. +// +// Only a `?` that is a placeholder is replaced. One inside a string literal, an alternative-quoted `q'[...]'` literal +// (any delimiter, with or without the national `n` prefix), a double-quoted name such as `"Paid?"`, or a `--` or +// `/* */` comment is text, and replacing it would change the statement and shift every value after it. internal enum OracleBindPlaceholders { /// The longest string literal Oracle parses, in bytes of the value it holds rather than of its doubled quotes /// (measured on 23ai with `MAX_STRING_SIZE = STANDARD`: 4,000 bytes parse, 4,001 fail with ORA-01704). static let maxLiteralBytes = 4_000 - /// The statement with every placeholder replaced, and the values left to bind, in `:n` order. A placeholder with - /// no parameter left stays `?`, as the PluginKit default leaves it. - static func substituting(_ sql: String, parameters: [PluginCellValue]) -> (sql: String, binds: [PluginCellValue]) { + // A placeholder with no parameter left stays `?`, as the PluginKit default leaves it. Throws for a BFILE value that + // is not a `BFILENAME` call, which no SQL writes. + static func substituting( + _ sql: String, + parameters: [PluginCellValue], + kinds: [PluginColumnKind?]? = nil, + typeNames: [String?]? = nil, + writesBooleans: Bool = false + ) throws -> (sql: String, binds: [PluginCellValue]) { var binds: [PluginCellValue] = [] - let rewritten = replacingPlaceholders(in: sql) { index in + let rewritten = try replacingPlaceholders(in: sql) { index in guard index < parameters.count else { return nil } let value = parameters[index] - if let literal = literal(for: value) { return literal } - binds.append(value) - return ":\(binds.count)" + guard let written = Self.operand( + for: value, + kind: kinds.flatMap { $0.indices.contains(index) ? $0[index] : nil }, + typeName: typeNames.flatMap { $0.indices.contains(index) ? $0[index] : nil }, + writesBooleans: writesBooleans, + bind: { + binds.append(value) + return ":\(binds.count)" + } + ) else { + throw OraclePluginError(core: .queryFailed(OracleValueKind.unwritableBFileReason)) + } + return written } return (rewritten, binds) } - /// The literal for a value, or nil when it has to be bound. NUL characters are dropped as the PluginKit default - /// drops them. A quote is doubled scalar by scalar, so one carrying a combining mark cannot end the literal. - static func literal(for value: PluginCellValue) -> String? { + static func operand( + for value: PluginCellValue, + kind: PluginColumnKind?, + typeName: String?, + writesBooleans: Bool, + bind: () -> String + ) -> String? { + if let typeName { + return OracleValueKind(typeName: typeName).operand(for: value, bind: bind) + } switch value { case .null: return "NULL" case .bytes: - return nil + return bind() case .text(let text): - return PluginNumericLiteral.isValid(text) ? text : quotedLiteral(text) + return literal(for: text, kind: kind, writesBooleans: writesBooleans) ?? bind() + } + } + + // Nil when the text is over the literal limit and has to be bound. Text the kind does not fit is quoted, for the + // server to refuse. `TRUE` and `FALSE` parse only where SQL has a BOOLEAN type, 23ai on. + static func literal(for text: String, kind: PluginColumnKind?, writesBooleans: Bool) -> String? { + guard let kind else { + // No numeric column reads as `00123` or `-007`, so that text is a code or a zip and stays a string. + if PluginNumericLiteral.isValid(text), !opensWithPaddingZero(text) { return text } + return quotedLiteral(text) + } + switch kind { + case .integer, .decimal: + if OracleNumericLiteral.isValid(text) { return text } + case .boolean: + guard writesBooleans else { break } + switch PluginSQLLiteral.booleanSynonym(for: text, kind: kind) { + case .isTrue?: return "TRUE" + case .isFalse?: return "FALSE" + default: break + } + case .text, .other: + break + @unknown default: + break } + return quotedLiteral(text) } - // Nil when the text is over the literal limit and has to be bound. + // Nil when the text is over the literal limit and has to be bound. NUL characters are dropped as the PluginKit + // default drops them. A quote is doubled scalar by scalar, so one carrying a combining mark cannot end the literal. static func quotedLiteral(_ text: String) -> String? { let kept = text.unicodeScalars.filter { $0 != "\0" } guard String(kept).utf8.count <= maxLiteralBytes else { return nil } return SQLQuoting.stringLiteral(text) } - private static func replacingPlaceholders(in sql: String, with replacement: (Int) -> String?) -> String { + // `007`, `-0042`: an integer part that opens with 0 and goes on with another digit. + private static func opensWithPaddingZero(_ text: String) -> Bool { + let unsigned = text.unicodeScalars.drop { $0 == "-" || $0 == "+" } + guard unsigned.first == "0", let next = unsigned.dropFirst().first else { return false } + return ("0"..."9").contains(next) + } + + private static func replacingPlaceholders( + in sql: String, + with replacement: (Int) throws -> String? + ) rethrows -> String { let units = Array(sql.utf16) var output = "" var placeholderIndex = 0 @@ -67,7 +128,7 @@ internal enum OracleBindPlaceholders { index = skippedEnd(in: units, from: index) continue } - let replaced = replacement(placeholderIndex) + let replaced = try replacement(placeholderIndex) placeholderIndex += 1 guard let text = replaced else { index += 1 diff --git a/apple/Plugins/OracleDriverPlugin/OracleIndexStatements.swift b/apple/Plugins/OracleDriverPlugin/OracleIndexStatements.swift index 07a77e1231..df7ff44f6b 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleIndexStatements.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleIndexStatements.swift @@ -4,24 +4,32 @@ // import Foundation +import TableProOracleCore import TableProSQLQuoting -/// The indexes an Oracle dump has to recreate, and the `CREATE INDEX` that recreates them. -/// -/// Every index is taken, constraint-backed ones included, because this driver's `fetchTableDDL` -/// writes column definitions and nothing else: no primary key, no unique constraint. Excluding -/// them would have left an Oracle dump with no uniqueness anywhere, so a restore would accept -/// duplicate rows the source rejected. +// The indexes an Oracle dump recreates after the rows, and the `CREATE INDEX` that recreates each. An index an enabled +// PRIMARY KEY or UNIQUE constraint owns is left out: `fetchTableDDL` writes the constraint, which makes its own index, +// and a second index on the same columns fails with ORA-01408. A disabled or NOVALIDATE constraint is written disabled +// and makes none, so an index it kept is written, and a NOVALIDATE one is enabled on it after the rows. internal enum OracleIndexStatements { - // NORMAL and BITMAP only: a function-based index keeps its key in a LONG column OracleNIO cannot decode. + // NORMAL and BITMAP only: a function-based index keeps its key in a LONG column OracleNIO cannot decode. A + // constraint row with no `INDEX_OWNER` is read as naming an index of its own schema. internal static func query(schema: String, table: String) -> String { """ SELECT i.INDEX_NAME, i.UNIQUENESS, i.INDEX_TYPE, ic.COLUMN_NAME, ic.DESCEND - FROM SYS.ALL_INDEXES i - JOIN SYS.ALL_IND_COLUMNS ic ON i.INDEX_NAME = ic.INDEX_NAME AND i.OWNER = ic.INDEX_OWNER + FROM \(OracleDictionary.allIndexes) i + JOIN \(OracleDictionary.allIndColumns) ic ON i.INDEX_NAME = ic.INDEX_NAME AND i.OWNER = ic.INDEX_OWNER WHERE i.TABLE_NAME = \(SQLQuoting.stringLiteral(table)) AND i.OWNER = \(SQLQuoting.stringLiteral(schema)) AND i.INDEX_TYPE IN ('NORMAL', 'BITMAP') + AND NOT EXISTS ( + SELECT 1 FROM \(OracleDictionary.allConstraints) c + WHERE c.CONSTRAINT_TYPE IN ('P', 'U') + AND c.STATUS = 'ENABLED' + AND c.VALIDATED = 'VALIDATED' + AND c.INDEX_NAME = i.INDEX_NAME + AND NVL(c.INDEX_OWNER, c.OWNER) = i.OWNER + ) ORDER BY i.INDEX_NAME, ic.COLUMN_POSITION """ } diff --git a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift index 9f364ae3d4..c083766d07 100644 --- a/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift +++ b/apple/Plugins/OracleDriverPlugin/OraclePlugin.swift @@ -378,11 +378,25 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { return pluginResult(of: query, result, executionTime: Date().timeIntervalSince(startTime)) } - /// Values a literal can carry stay literals, so the host's statements read as they always have; only bytes and - /// text over the literal limit are bound. See ``OracleBindPlaceholders``. func executeParameterized(query: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult { + try await executeParameterized(query: query, parameters: parameters, context: PluginStatementContext()) + } + + // Values a literal can carry stay literals; only bytes and text over the literal limit are bound. The context + // says the type of each value's column, or the kind the user gave it (`OracleBindPlaceholders`). + func executeParameterized( + query: String, + parameters: [PluginCellValue], + context: PluginStatementContext + ) async throws -> PluginQueryResult { guard !parameters.isEmpty else { return try await execute(query: query) } - let statement = OracleBindPlaceholders.substituting(query, parameters: parameters) + let statement = try OracleBindPlaceholders.substituting( + query, + parameters: parameters, + kinds: context.parameterKinds, + typeNames: context.parameterTypeNames, + writesBooleans: core?.serverRelease?.hasBooleanType ?? false + ) guard !statement.binds.isEmpty else { return try await execute(query: statement.sql) } guard let core else { throw OraclePluginError(core: .notConnected) } let startTime = Date() @@ -396,6 +410,37 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { return pluginResult(of: query, result, executionTime: Date().timeIntervalSince(startTime)) } + func executeUserQuery(query: String, rowCap: Int?, parameters: [PluginCellValue]?) async throws -> PluginQueryResult { + try await executeUserQuery(query: query, rowCap: rowCap, parameters: parameters, context: PluginStatementContext()) + } + + // Overridden so the kinds the user gave the parameters reach the binder; the PluginKit default drops the context. + // A capped result keeps its column hints, so a DATE column still reads with its time. + func executeUserQuery( + query: String, + rowCap: Int?, + parameters: [PluginCellValue]?, + context: PluginStatementContext + ) async throws -> PluginQueryResult { + let raw: PluginQueryResult + if let parameters { + raw = try await executeParameterized(query: query, parameters: parameters, context: context) + } else { + raw = try await execute(query: query, context: context) + } + guard let cap = rowCap, cap > 0, raw.rows.count > cap else { return raw } + return PluginQueryResult( + columns: raw.columns, + columnTypeNames: raw.columnTypeNames, + rows: Array(raw.rows.prefix(cap)), + rowsAffected: raw.rowsAffected, + timing: raw.timing, + isTruncated: true, + statusMessage: raw.statusMessage, + columnMeta: raw.columnMeta + ) + } + /// An anonymous block reports no row count of its own; whatever oracle-nio counted belongs to the statements /// inside it. private func pluginResult( @@ -586,11 +631,19 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { func fetchIndexDDL(table: String, schema: String?) async throws -> [String] { let owner = effectiveSchema(schema) let result = try await rawQuery(OracleIndexStatements.query(schema: owner, table: table)) - return OracleIndexStatements.render( + let indexes = OracleIndexStatements.render( rows: result.rows.map { row in row.map { $0.stringValue } }, schema: owner, table: table, quote: quoteIdentifier) + let constraints = try await rawQuery( + OracleSchemaQueries.tableConstraints(schema: owner, table: table, release: serverRelease()) + ) + return indexes + (try OracleTableDDL.afterRowsStatements( + qualifiedTable: "\(quoteIdentifier(owner)).\(quoteIdentifier(table))", + constraints: OracleSchemaQueries.parseTableConstraints(constraints.rows), + quote: quoteIdentifier + )) } func fetchForeignKeys(table: String, schema: String?) async throws -> [PluginForeignKeyInfo] { @@ -708,9 +761,14 @@ final class OraclePluginDriver: PluginDatabaseDriver, @unchecked Sendable { if !view.rows.isEmpty { return try await viewStatement(from: view, schema: owner, view: table) } + let columns = try await fetchColumns(table: table, schema: schema) + let constraints = try await rawQuery( + OracleSchemaQueries.tableConstraints(schema: owner, table: table, release: serverRelease()) + ) return try OracleTableDDL.createTable( qualifiedTable: "\(quoteIdentifier(owner)).\(quoteIdentifier(table))", - columns: try await fetchColumns(table: table, schema: schema), + columns: columns, + constraints: OracleSchemaQueries.parseTableConstraints(constraints.rows), quote: quoteIdentifier ) } diff --git a/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift b/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift index 5d9984ebe2..6b82456d53 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleRowWriter.swift @@ -240,10 +240,7 @@ internal struct OracleRowWriter { // MARK: - Values - /// Text for a column of a known type that is not a number is quoted here: left as `?`, it would reach the - /// placeholder writer, which writes numeric-looking text unquoted, so `00123` would be stored in a VARCHAR2 as - /// `123` and a keyless match on it would compare the column as a number. Text over the literal limit stays `?` and - /// is bound. A column of no known type keeps the plain `?`. + // A column of no known type keeps the plain `?`, which the placeholder writer fills in. private func sql( for value: PluginCellValue, column: String, @@ -255,37 +252,13 @@ internal struct OracleRowWriter { parameters.append(value) return "?" } - let kind = OracleValueKind(typeName: typeName) - if kind == .bfile, !value.isNull { - return try bfileSQL(for: value, rowIndex: rowIndex) - } - switch value { - case .null: - return "NULL" - case .bytes: + guard let operand = OracleValueKind(typeName: typeName).operand(for: value, bind: { parameters.append(value) return "?" - case .text(let text): - if let literal = kind.inlineSQL(for: text) { return literal } - if !kind.isNumeric, let quoted = OracleBindPlaceholders.quotedLiteral(text) { - return kind.converting(quoted, text: text) - } - parameters.append(value) - return kind.converting("?", text: text) - } - } - - /// A BFILE reads as the `BFILENAME('DIR', 'file')` call that makes it, and only that call writes one back: the - /// text quoted or bound fails with ORA-00932. Anything else in a BFILE cell, such as the `` placeholder, - /// is refused rather than sent. - private func bfileSQL(for value: PluginCellValue, rowIndex: Int) throws -> String { - guard let text = value.asText, let call = OracleBFileText.call(from: text) else { - throw PluginRowWriteRefusal( - rowIndex: rowIndex, - reason: String(format: String(localized: "Values of type %@ can't be written."), "BFILE") - ) + }) else { + throw PluginRowWriteRefusal(rowIndex: rowIndex, reason: OracleValueKind.unwritableBFileReason) } - return call + return operand } // MARK: - Helpers @@ -353,6 +326,32 @@ internal enum OracleValueKind: Equatable { } } + static var unwritableBFileReason: String { + String(format: String(localized: "Values of type %@ can't be written."), "BFILE") + } + + // A literal where one carries the value, else `bind()`'s placeholder in the conversion the type needs. Text that + // is not a number is quoted here: the placeholder writer would write `00123` bare, and a VARCHAR2 would store 123. + // Nil for a BFILE value that is not the `BFILENAME('DIR', 'file')` call it reads as, the only text that writes + // one back (anything else fails with ORA-00932). + func operand(for value: PluginCellValue, bind: () -> String) -> String? { + if self == .bfile, !value.isNull { + return value.asText.flatMap(OracleBFileText.call(from:)) + } + switch value { + case .null: + return "NULL" + case .bytes: + return bind() + case .text(let text): + if let literal = inlineSQL(for: text) { return literal } + if !isNumeric, let quoted = OracleBindPlaceholders.quotedLiteral(text) { + return converting(quoted, text: text) + } + return converting(bind(), text: text) + } + } + /// A text operand, a literal or `?`, as the column's type takes it. /// /// A TIMESTAMP WITH LOCAL TIME ZONE value reads as the session's wall clock with that instant's offset, which diff --git a/apple/Plugins/OracleDriverPlugin/OracleTableDDL.swift b/apple/Plugins/OracleDriverPlugin/OracleTableDDL.swift index 036c2ad204..215d9e4ee4 100644 --- a/apple/Plugins/OracleDriverPlugin/OracleTableDDL.swift +++ b/apple/Plugins/OracleDriverPlugin/OracleTableDDL.swift @@ -17,15 +17,60 @@ import TableProSQLGrammar /// `GENERATED ALWAYS AS (...) VIRTUAL` or `DEFAULT`, then `NOT NULL`. `NOT NULL` before `DEFAULT` fails with /// ORA-03076, and a type folded to upper case names no type at all when it is a quoted mixed-case object type. internal enum OracleTableDDL { + // The keys and checks follow the columns. A NOT NULL the column definition writes is not written again as a check. static func createTable( qualifiedTable: String, columns: [PluginColumnInfo], + constraints: [OracleTableConstraint] = [], quote: (String) -> String ) throws -> String { - let definitions = try columns.map { " " + (try columnDefinition($0, quote: quote)) } + let notNullColumns = Set(columns.filter { !$0.isNullable }.map(\.name)) + let columnDefinitions = try columns.map { try columnDefinition($0, quote: quote) } + let written = constraints.filter { !$0.restatesNotNull(of: notNullColumns) } + // A check added after the rows is refused here, with the others, rather than in the index phase. + for check in written where check.isAddedAfterRows { + _ = try checkCondition(of: check) + } + let constraintClauses = try written.filter { !$0.isAddedAfterRows }.map { try constraintClause($0, quote: quote) } + let definitions = (columnDefinitions + constraintClauses).map { " " + $0 } return "CREATE TABLE \(qualifiedTable) (\n" + definitions.joined(separator: ",\n") + "\n);" } + // A check's catalog text goes in only as one expression, as a default does. + private static func constraintClause(_ constraint: OracleTableConstraint, quote: (String) -> String) throws -> String { + guard constraint.kind == .check else { return constraint.clause(quote: quote) } + var written = constraint + written.condition = try checkCondition(of: constraint) + return written.clause(quote: quote) + } + + private static func checkCondition(of constraint: OracleTableConstraint) throws -> String { + let condition = SQLExpressionFragment.strippingComments(constraint.condition ?? "", databaseTypeId: "Oracle") + guard !constraint.conditionMayBeCut, !condition.isEmpty, isSingleExpression(condition) else { + throw OraclePluginError(core: .queryFailed(String( + format: String(localized: "The condition of check constraint %@ cannot be written as SQL."), + constraint.name + ))) + } + return condition + } + + // What runs after the rows: a NOVALIDATE constraint is enabled again without checking them, and one with a + // generated name is enabled by what it is, or added if it is a check. + static func afterRowsStatements( + qualifiedTable: String, + constraints: [OracleTableConstraint], + quote: (String) -> String + ) throws -> [String] { + try constraints.compactMap { constraint in + if let statement = constraint.enableAfterRowsStatement(qualifiedTable: qualifiedTable, quote: quote) { + return statement + } + guard constraint.isAddedAfterRows else { return nil } + return "ALTER TABLE \(qualifiedTable) ADD CHECK (\(try checkCondition(of: constraint))) ENABLE NOVALIDATE" + } + } + /// `TEXT_VC` is the view's query alone, so its column names go in the header: a view declared /// `v (public_id) AS SELECT internal_id ...` replayed without them exposes `INTERNAL_ID`, and one /// over an unaliased expression cannot be recreated at all. `DBMS_METADATA` writes the list too. diff --git a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift index 88679f7f66..18e30f256f 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/LibPQPluginConnection.swift @@ -1585,7 +1585,7 @@ final class LibPQPluginConnection: @unchecked Sendable { func streamQuery(_ query: String, readOnly: Bool = false) -> AsyncThrowingStream { let queryToRun = String(query) - return PluginRowStream.make { continuation, abort in + return PluginRowStream.makeBounded { continuation, abort in self.queue.async { [self] in let handle = connectionHandle @@ -1706,6 +1706,7 @@ final class LibPQPluginConnection: @unchecked Sendable { let batchSize = 5_000 var batch: [PluginRow] = [] batch.reserveCapacity(batchSize) + let stops = { self.cancellationGate.isCancelled(generation) } while let result = PQgetResult(conn) { let status = PQresultStatus(result) @@ -1713,7 +1714,7 @@ final class LibPQPluginConnection: @unchecked Sendable { if !headerSent, Self.carriesRowDescription(status) { let described = streamHeader(from: result) columnOids = described.columnOids - continuation.yield(.header(described.header)) + _ = PluginRowStream.deliverBlocking(.header(described.header), to: continuation, stops: stops) headerSent = true } @@ -1733,15 +1734,13 @@ final class LibPQPluginConnection: @unchecked Sendable { PQclear(result) batch.append(row) - if batch.count >= batchSize { - continuation.yield(.rows(batch)) + // A reader that left or a Stop is handled just below. + if batch.count >= batchSize, + PluginRowStream.deliverBlocking(.rows(batch), to: continuation, stops: stops) { batch.removeAll(keepingCapacity: true) } if run.abort.isAborted || cancellationGate.isCancelled(generation) { - if !batch.isEmpty { - continuation.yield(.rows(batch)) - } _ = cancelAndDrain(conn, suppressCancel: suppressCancel) // A Stop leaves the block aborted, as it does for any statement; only a reader that left is undone. let undoing = !cancellationGate.isCancelled(generation) @@ -1770,9 +1769,7 @@ final class LibPQPluginConnection: @unchecked Sendable { } } - if !batch.isEmpty { - continuation.yield(.rows(batch)) - } + let delivered = batch.isEmpty || PluginRowStream.deliverBlocking(.rows(batch), to: continuation, stops: stops) let outcome = finishPendingResults(conn, cancellingOutput: cancelsOutput) recordTransactionState(of: conn) @@ -1787,6 +1784,7 @@ final class LibPQPluginConnection: @unchecked Sendable { if let abandoned = abandonedCopyError(outcome, generation: generation) { throw abandoned } + guard delivered else { throw CancellationError() } } private static let streamSavepoint = "SAVEPOINT tablepro_stream_read" diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCommentStatements.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCommentStatements.swift index 08c52f7978..71c957c861 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCommentStatements.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLCommentStatements.swift @@ -74,7 +74,8 @@ public enum PostgreSQLCommentStatements { return rows.compactMap { statement(target: target, row: $0) } } - private static func statement(target: String, row: [String?]) -> String? { + // `target` is already qualified and quoted. + static func statement(target: String, row: [String?]) -> String? { guard row.count >= 3, let relkind = row[0], let keyword = PostgreSQLRelationSQL.commentKeyword(forRelkind: relkind), diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift index 41d4b5ef1e..6daf40166b 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLPluginDriver.swift @@ -452,16 +452,44 @@ class PostgreSQLPluginDriver: LibPQBackedDriver, @unchecked Sendable { var parts = columnDefs parts.append(contentsOf: constraints) - let shape = PostgreSQLTableDDL.shape(fromRows: shapeRows?.rows.map { $0.map(\.asText) }) + var shape = PostgreSQLTableDDL.shape(fromRows: shapeRows?.rows.map { $0.map(\.asText) }) var partitions: [PostgreSQLTableDDL.Partition] = [] + var partitionIndexes: [String] = [] + var partitionComments: [String] = [] if shape.partitionKey != nil { let tree = try await catalogRead(PostgreSQLTableDDL.partitionTreeQuery(schema: resolvedSchema, table: table)) partitions = tree.rows.compactMap { PostgreSQLTableDDL.partition(from: $0.map(\.asText)) } + if !partitions.isEmpty { + async let elementRows = catalogRead(PostgreSQLTableDDL.partitionElementsQuery( + schema: resolvedSchema, table: table, scope: .below, capabilities: versionedCapabilities + )) + async let indexRows = catalogRead(PostgreSQLTableDDL.partitionIndexesQuery(schema: resolvedSchema, table: table)) + async let commentRows = catalogRead(PostgreSQLTableDDL.partitionCommentsQuery(schema: resolvedSchema, table: table)) + let elements = PostgreSQLTableDDL.elementsByPartition(try await elementRows.rows.map { $0.map(\.asText) }) + partitions = partitions.map { partition in + var partition = partition + partition.elements = elements[partition.name] ?? [] + return partition + } + partitionIndexes = try await indexRows.rows.compactMap { $0.first?.asText } + partitionComments = PostgreSQLTableDDL.partitionComments(try await commentRows.rows.map { $0.map(\.asText) }) + } + } else if shape.partitionOf != nil { + let elementRows = try await catalogRead(PostgreSQLTableDDL.partitionElementsQuery( + schema: resolvedSchema, table: table, scope: .itself, capabilities: versionedCapabilities + )) + shape.partitionElements = PostgreSQLTableDDL.elementsByPartition(elementRows.rows.map { $0.map(\.asText) }) + .values.first ?? [] } let quotedSchema = quoteIdentifier(resolvedSchema) return PostgreSQLTableDDL.script( - name: "\(quotedSchema).\(quotedTable)", definitions: parts, shape: shape, partitions: partitions + name: "\(quotedSchema).\(quotedTable)", + definitions: parts, + shape: shape, + partitions: partitions, + partitionIndexes: partitionIndexes, + partitionComments: partitionComments ) } diff --git a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift index e4728a714a..347c088d4d 100644 --- a/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift +++ b/apple/Plugins/PostgreSQLDriverPlugin/PostgreSQLTableDDL.swift @@ -7,13 +7,15 @@ import Foundation import TableProPostgreSQLCore // What a table's CREATE statement says outside its column list. The table listing hides the partitions of a -// partitioned table, so the parent's script creates them: without them a restore has nowhere to put a row. +// partitioned table, so the parent's script creates them, with what each holds of its own: without them a restore has +// nowhere to put a row, and a partition's own check, key, default and index would be lost. enum PostgreSQLTableDDL { struct Shape: Equatable { var isUnlogged = false var partitionKey: String? var partitionOf: String? var partitionBound: String? + var partitionElements: [String] = [] } struct Partition: Equatable { @@ -22,6 +24,14 @@ enum PostgreSQLTableDDL { let bound: String let partitionKey: String? let isUnlogged: Bool + var foreignServer: String? + var foreignOptions: String? + var elements: [String] = [] + } + + enum PartitionScope { + case below + case itself } // `relispartition`, `relpartbound` and `pg_get_partkeydef` are PostgreSQL 10. @@ -47,39 +57,183 @@ enum PostgreSQLTableDDL { """ } - // Every partition below the table, a parent before its own partitions. A foreign table attached as a partition - // is left out: CREATE TABLE cannot make one. + // Every partition below the table, a parent before its own partitions, foreign table partitions with their server + // and options. static func partitionTreeQuery(schema: String, table: String) -> String { """ - WITH RECURSIVE parts AS ( - SELECT i.inhrelid AS relid, i.inhparent AS parent, 1 AS depth - FROM pg_catalog.pg_inherits i - JOIN pg_catalog.pg_class root ON root.oid = i.inhparent - JOIN pg_catalog.pg_namespace rn ON rn.oid = root.relnamespace - WHERE root.relname = \(PostgreSQLObjectQueries.quoteLiteral(table)) - AND rn.nspname = \(PostgreSQLObjectQueries.quoteLiteral(schema)) - AND root.relkind = 'p' - UNION ALL - SELECT i.inhrelid, i.inhparent, parts.depth + 1 - FROM pg_catalog.pg_inherits i - JOIN parts ON parts.relid = i.inhparent - ) + \(partitionRelations(schema: schema, table: table, scope: .below)) SELECT pg_catalog.quote_ident(cn.nspname) || '.' || pg_catalog.quote_ident(c.relname), pg_catalog.quote_ident(pn.nspname) || '.' || pg_catalog.quote_ident(p.relname), pg_catalog.pg_get_expr(c.relpartbound, c.oid), CASE WHEN c.relkind = 'p' THEN pg_catalog.pg_get_partkeydef(c.oid) END, - c.relpersistence::text + c.relpersistence::text, + pg_catalog.quote_ident(fs.srvname), + (SELECT pg_catalog.string_agg( + pg_catalog.quote_ident(o.option_name) || ' ' || pg_catalog.quote_literal(o.option_value), + ', ') + FROM pg_catalog.pg_options_to_table(ft.ftoptions) o) FROM parts JOIN pg_catalog.pg_class c ON c.oid = parts.relid JOIN pg_catalog.pg_namespace cn ON cn.oid = c.relnamespace JOIN pg_catalog.pg_class p ON p.oid = parts.parent JOIN pg_catalog.pg_namespace pn ON pn.oid = p.relnamespace + LEFT JOIN pg_catalog.pg_foreign_table ft ON ft.ftrelid = c.oid AND c.relkind = 'f' + LEFT JOIN pg_catalog.pg_foreign_server fs ON fs.oid = ft.ftserver WHERE c.relispartition - AND c.relkind IN ('r', 'p') + AND c.relkind IN ('r', 'p', 'f') ORDER BY parts.depth, cn.nspname, c.relname """ } + // What a partition holds of its own, as `PARTITION OF` element list entries: a default or NOT NULL its parent's + // column lacks, and a key or check it declares itself. The server copies the parent's into every new partition. + static func partitionElementsQuery( + schema: String, + table: String, + scope: PartitionScope, + capabilities: PostgreSQLCapabilities + ) -> String { + let notGenerated = capabilities.hasGeneratedColumns ? "a.attgenerated = ''" : "true" + let ownDefault = """ + d.oid IS NOT NULL AND (pd.oid IS NULL + OR pg_catalog.pg_get_expr(d.adbin, d.adrelid) IS DISTINCT FROM pg_catalog.pg_get_expr(pd.adbin, pd.adrelid)) + """ + return """ + \(partitionRelations(schema: schema, table: table, scope: scope)) + SELECT pg_catalog.quote_ident(cn.nspname) || '.' || pg_catalog.quote_ident(c.relname), elements.element + FROM ( + SELECT parts.relid, parts.depth, 0 AS kind, a.attnum::int AS position, + pg_catalog.quote_ident(a.attname) || ' WITH OPTIONS' + || CASE WHEN \(ownDefault) THEN ' DEFAULT ' || pg_catalog.pg_get_expr(d.adbin, d.adrelid) ELSE '' END + || CASE WHEN a.attnotnull AND NOT pa.attnotnull THEN ' NOT NULL' ELSE '' END AS element + FROM parts + JOIN pg_catalog.pg_attribute a ON a.attrelid = parts.relid AND a.attnum > 0 AND NOT a.attisdropped + JOIN pg_catalog.pg_attribute pa + ON pa.attrelid = parts.parent AND pa.attname = a.attname AND NOT pa.attisdropped + LEFT JOIN pg_catalog.pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum + LEFT JOIN pg_catalog.pg_attrdef pd ON pd.adrelid = pa.attrelid AND pd.adnum = pa.attnum + WHERE \(notGenerated) + AND ((\(ownDefault)) OR (a.attnotnull AND NOT pa.attnotnull)) + UNION ALL + SELECT parts.relid, parts.depth, 1, + CASE con.contype WHEN 'p' THEN 0 WHEN 'u' THEN 1 WHEN 'c' THEN 2 ELSE 3 END, + 'CONSTRAINT ' || pg_catalog.quote_ident(con.conname) || ' ' + || pg_catalog.pg_get_constraintdef(con.oid, true) + FROM parts + JOIN pg_catalog.pg_constraint con ON con.conrelid = parts.relid + WHERE con.contype IN ('p', 'u', 'c', 'x') + AND con.conislocal + AND con.coninhcount = 0 + ) AS elements + JOIN pg_catalog.pg_class c ON c.oid = elements.relid + JOIN pg_catalog.pg_namespace cn ON cn.oid = c.relnamespace + ORDER BY elements.depth, cn.nspname, c.relname, elements.kind, elements.position, elements.element + """ + } + + // An index a partition has of its own: no parent index it is attached to, and no constraint of the partition's + // behind it, which the element list already declares. + static func partitionIndexesQuery(schema: String, table: String) -> String { + """ + \(partitionRelations(schema: schema, table: table, scope: .below)) + SELECT pg_catalog.pg_get_indexdef(i.indexrelid) + FROM parts + JOIN pg_catalog.pg_index i ON i.indrelid = parts.relid + JOIN pg_catalog.pg_class ic ON ic.oid = i.indexrelid + JOIN pg_catalog.pg_class c ON c.oid = parts.relid + JOIN pg_catalog.pg_namespace cn ON cn.oid = c.relnamespace + WHERE i.indisvalid + AND NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_inherits ih WHERE ih.inhrelid = i.indexrelid) + AND NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_constraint con + WHERE con.conrelid = parts.relid AND con.conindid = i.indexrelid AND con.contype IN ('p', 'u', 'x')) + ORDER BY parts.depth, cn.nspname, c.relname, ic.relname + """ + } + + // Rows are the partition, then what `PostgreSQLCommentStatements.catalogQuery` projects. A foreign table partition + // is not created, so its comments are left out too. + static func partitionCommentsQuery(schema: String, table: String) -> String { + """ + \(partitionRelations(schema: schema, table: table, scope: .below)) + SELECT target, relkind, attname, description + FROM ( + SELECT pg_catalog.quote_ident(cn.nspname) || '.' || pg_catalog.quote_ident(c.relname) AS target, + parts.depth, c.relkind::text AS relkind, NULL::text AS attname, + pg_catalog.obj_description(c.oid, 'pg_class') AS description, 0 AS attnum + FROM parts + JOIN pg_catalog.pg_class c ON c.oid = parts.relid + JOIN pg_catalog.pg_namespace cn ON cn.oid = c.relnamespace + WHERE c.relkind <> 'f' + AND pg_catalog.obj_description(c.oid, 'pg_class') IS NOT NULL + UNION ALL + SELECT pg_catalog.quote_ident(cn.nspname) || '.' || pg_catalog.quote_ident(c.relname), + parts.depth, c.relkind::text, a.attname::text, + pg_catalog.col_description(c.oid, a.attnum), a.attnum::int + FROM parts + JOIN pg_catalog.pg_class c ON c.oid = parts.relid + JOIN pg_catalog.pg_namespace cn ON cn.oid = c.relnamespace + JOIN pg_catalog.pg_attribute a ON a.attrelid = c.oid AND a.attnum > 0 AND NOT a.attisdropped + WHERE c.relkind <> 'f' + AND pg_catalog.col_description(c.oid, a.attnum) IS NOT NULL + ) AS partition_comments + ORDER BY depth, target, attnum + """ + } + + private static func partitionRelations(schema: String, table: String, scope: PartitionScope) -> String { + let nameLiteral = PostgreSQLObjectQueries.quoteLiteral(table) + let schemaLiteral = PostgreSQLObjectQueries.quoteLiteral(schema) + switch scope { + case .below: + return """ + WITH RECURSIVE parts AS ( + SELECT i.inhrelid AS relid, i.inhparent AS parent, 1 AS depth + FROM pg_catalog.pg_inherits i + JOIN pg_catalog.pg_class root ON root.oid = i.inhparent + JOIN pg_catalog.pg_namespace rn ON rn.oid = root.relnamespace + WHERE root.relname = \(nameLiteral) + AND rn.nspname = \(schemaLiteral) + AND root.relkind = 'p' + UNION ALL + SELECT i.inhrelid, i.inhparent, parts.depth + 1 + FROM pg_catalog.pg_inherits i + JOIN parts ON parts.relid = i.inhparent + ) + """ + case .itself: + return """ + WITH parts AS ( + SELECT c.oid AS relid, i.inhparent AS parent, 0 AS depth + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + JOIN pg_catalog.pg_inherits i ON i.inhrelid = c.oid + WHERE c.relname = \(nameLiteral) + AND n.nspname = \(schemaLiteral) + AND c.relispartition + ) + """ + } + } + + // Rows from `partitionElementsQuery`, by the partition's qualified name. + static func elementsByPartition(_ rows: [[String?]]) -> [String: [String]] { + var grouped: [String: [String]] = [:] + for row in rows { + guard row.count >= 2, let name = row[0], let element = row[1], !element.isEmpty else { continue } + grouped[name, default: []].append(element) + } + return grouped + } + + static func partitionComments(_ rows: [[String?]]) -> [String] { + rows.compactMap { row in + guard let target = row.first ?? nil, !target.isEmpty else { return nil } + return PostgreSQLCommentStatements.statement(target: target, row: Array(row.dropFirst())) + } + } + // A server that reports version 10 without these catalog functions fails the read; the table is then plain. static func shape(fromRows rows: [[String?]]?) -> Shape { rows?.first.map(shape(from:)) ?? Shape() @@ -105,14 +259,27 @@ enum PostgreSQLTableDDL { let bound = row[2], !bound.isEmpty else { return nil } let key = row[3].flatMap { $0.isEmpty ? nil : $0 } - return Partition(name: name, parent: parent, bound: bound, partitionKey: key, isUnlogged: row[4] == "u") + func text(_ index: Int) -> String? { + guard row.indices.contains(index), let value = row[index], !value.isEmpty else { return nil } + return value + } + return Partition( + name: name, + parent: parent, + bound: bound, + partitionKey: key, + isUnlogged: row[4] == "u", + foreignServer: text(5), + foreignOptions: text(6) + ) } static func createTable(name: String, definitions: [String], shape: Shape) -> String { let partitionBy = shape.partitionKey.map { " PARTITION BY \($0)" } ?? "" if let parent = shape.partitionOf, let bound = shape.partitionBound { let isUnlogged = shape.isUnlogged && shape.partitionKey == nil - return "\(keyword(isUnlogged: isUnlogged)) \(name) PARTITION OF \(parent) \(bound)\(partitionBy);" + let elements = elementList(shape.partitionElements) + return "\(keyword(isUnlogged: isUnlogged)) \(name) PARTITION OF \(parent)\(elements) \(bound)\(partitionBy);" } // PostgreSQL 18 refuses an unlogged partitioned table, and before it the flag did nothing on the parent. let isUnlogged = shape.isUnlogged && shape.partitionKey == nil @@ -121,24 +288,51 @@ enum PostgreSQLTableDDL { + "\n)\(partitionBy);" } + // A foreign table partition is written as a comment. The export reads its rows through the parent, and replayed + // into a partition on that server they would be written to the remote table a second time. static func createPartition(_ partition: Partition) -> String { - createTable( + if let server = partition.foreignServer { + let options = partition.foreignOptions.map { " OPTIONS (\($0))" } ?? "" + let statement = "CREATE FOREIGN TABLE \(partition.name) PARTITION OF \(partition.parent)" + + "\(elementList(partition.elements)) \(partition.bound) SERVER \(server)\(options);" + let note = "\(partition.name) is a foreign table partition on server \(server), left out so a restore writes" + + " nothing\nto that server. Rows exported for it have no partition until it is created:\n" + // Every line break the server's lexer ends a comment at, a carriage return included, starts a new comment. + return (note + statement).components(separatedBy: .newlines).map { "-- " + $0 }.joined(separator: "\n") + } + return createTable( name: partition.name, definitions: [], shape: Shape( isUnlogged: partition.isUnlogged, partitionKey: partition.partitionKey, partitionOf: partition.parent, - partitionBound: partition.bound + partitionBound: partition.bound, + partitionElements: partition.elements ) ) } - static func script(name: String, definitions: [String], shape: Shape, partitions: [Partition]) -> String { - ([createTable(name: name, definitions: definitions, shape: shape)] + partitions.map(createPartition)) + // A partition's own indexes and comments follow every partition, since one may name a partition created later. + static func script( + name: String, + definitions: [String], + shape: Shape, + partitions: [Partition], + partitionIndexes: [String] = [], + partitionComments: [String] = [] + ) -> String { + let indexes = partitionIndexes.map { indexCoveringPartitions($0) + ";" } + let comments = partitionComments.map { $0 + ";" } + return ([createTable(name: name, definitions: definitions, shape: shape)] + partitions.map(createPartition) + + indexes + comments) .joined(separator: "\n\n") } + private static func elementList(_ elements: [String]) -> String { + elements.isEmpty ? "" : " (\n " + elements.joined(separator: ",\n ") + "\n)" + } + // `pg_get_indexdef` writes ON ONLY for an index over a partitioned table, and replayed that builds nothing on the // partitions and leaves the index invalid. Without it the server builds each partition's index itself. static func indexCoveringPartitions(_ definition: String) -> String { diff --git a/apple/Plugins/PostgreSQLDriverTests/PostgreSQLTableDDLTests.swift b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLTableDDLTests.swift index 2664e5b315..9556e9c2b3 100644 --- a/apple/Plugins/PostgreSQLDriverTests/PostgreSQLTableDDLTests.swift +++ b/apple/Plugins/PostgreSQLDriverTests/PostgreSQLTableDDLTests.swift @@ -125,7 +125,15 @@ struct PostgreSQLTableDDLTests { let hostile = "o'rders" let statements = [ PostgreSQLTableDDL.shapeQuery(schema: hostile, table: hostile, capabilities: Self.modern), - PostgreSQLTableDDL.partitionTreeQuery(schema: hostile, table: hostile) + PostgreSQLTableDDL.partitionTreeQuery(schema: hostile, table: hostile), + PostgreSQLTableDDL.partitionElementsQuery( + schema: hostile, table: hostile, scope: .below, capabilities: Self.modern + ), + PostgreSQLTableDDL.partitionElementsQuery( + schema: hostile, table: hostile, scope: .itself, capabilities: Self.modern + ), + PostgreSQLTableDDL.partitionIndexesQuery(schema: hostile, table: hostile), + PostgreSQLTableDDL.partitionCommentsQuery(schema: hostile, table: hostile) ] for sql in statements { #expect(PostgreSQLCatalogQualificationTests.unqualifiedCalls(in: sql).isEmpty, "\(sql)") @@ -134,12 +142,114 @@ struct PostgreSQLTableDDLTests { } } - @Test("The partition tree lists a parent before its own partitions") + @Test("The partition tree lists a parent before its own partitions, foreign ones included") func partitionTreeOrder() { let sql = PostgreSQLTableDDL.partitionTreeQuery(schema: "public", table: "events") #expect(sql.contains("WITH RECURSIVE")) #expect(sql.contains("ORDER BY parts.depth")) - #expect(sql.contains("c.relkind IN ('r', 'p')")) + #expect(sql.contains("c.relkind IN ('r', 'p', 'f')")) + } + + @Test("A foreign table partition's comments are left out with it") + func foreignPartitionCommentsAreLeftOut() { + let sql = PostgreSQLTableDDL.partitionCommentsQuery(schema: "public", table: "events") + #expect(sql.components(separatedBy: "WHERE c.relkind <> 'f'").count == 3) + } + + // Each statement below was replayed on PostgreSQL 18 into an empty database and recreated the partition as read. + @Test("A partition's own defaults, NOT NULLs and constraints go in its element list") + func partitionElements() throws { + var partition = try #require(PostgreSQLTableDDL.partition( + from: ["public.events_2025", "public.events", "FOR VALUES FROM ('2025-01-01') TO ('2026-01-01')", "HASH (id)", "p"] + )) + partition.elements = PostgreSQLTableDDL.elementsByPartition([ + ["public.events_2025", "user_id WITH OPTIONS NOT NULL"], + ["public.events_2025", "note WITH OPTIONS DEFAULT 'y'::text"], + ["public.events_2024", "CONSTRAINT ck_2024 CHECK (user_id > 0)"] + ])["public.events_2025"] ?? [] + #expect(PostgreSQLTableDDL.createPartition(partition) == """ + CREATE TABLE public.events_2025 PARTITION OF public.events ( + user_id WITH OPTIONS NOT NULL, + note WITH OPTIONS DEFAULT 'y'::text + ) FOR VALUES FROM ('2025-01-01') TO ('2026-01-01') PARTITION BY HASH (id); + """) + } + + @Test("A partition asked for by name keeps its own element list") + func partitionByNameKeepsElements() { + var shape = PostgreSQLTableDDL.shape(from: ["p", nil, "FOR VALUES FROM (1) TO (100)", "public.events"]) + shape.partitionElements = ["CONSTRAINT ck CHECK (id > 0)"] + #expect(PostgreSQLTableDDL.createTable(name: "public.events_1", definitions: Self.columns, shape: shape) + == "CREATE TABLE public.events_1 PARTITION OF public.events (\n CONSTRAINT ck CHECK (id > 0)\n) FOR VALUES FROM (1) TO (100);") + } + + // The export reads a foreign partition's rows through the parent; replayed into it, they would go to the server. + @Test("A foreign table partition is written as a comment that names its server and options") + func foreignPartition() throws { + var partition = try #require(PostgreSQLTableDDL.partition(from: [ + "public.logs_old", "public.logs", "FOR VALUES FROM (MINVALUE) TO ('2025-01-01')", nil, "p", + "remote_srv", "schema_name 'public', table_name 'old''logs'" + ])) + partition.elements = ["at WITH OPTIONS DEFAULT '2000-01-01'::date"] + let ddl = PostgreSQLTableDDL.createPartition(partition) + #expect(ddl == """ + -- public.logs_old is a foreign table partition on server remote_srv, left out so a restore writes nothing + -- to that server. Rows exported for it have no partition until it is created: + -- CREATE FOREIGN TABLE public.logs_old PARTITION OF public.logs ( + -- at WITH OPTIONS DEFAULT '2000-01-01'::date + -- ) FOR VALUES FROM (MINVALUE) TO ('2025-01-01') SERVER remote_srv OPTIONS (schema_name 'public', table_name 'old''logs'); + """) + #expect(ddl.split(separator: "\n").allSatisfy { $0.hasPrefix("-- ") }) + + let bare = try #require(PostgreSQLTableDDL.partition( + from: ["public.l", "public.logs", "DEFAULT", nil, "p", "\"Srv\"", nil] + )) + #expect(PostgreSQLTableDDL.createPartition(bare).hasSuffix( + "-- CREATE FOREIGN TABLE public.l PARTITION OF public.logs DEFAULT SERVER \"Srv\";" + )) + + let hostile = try #require(PostgreSQLTableDDL.partition( + from: ["public.\"a\rDROP TABLE t; --\"", "public.logs", "DEFAULT", nil, "p", "s", nil] + )) + let lines = PostgreSQLTableDDL.createPartition(hostile).components(separatedBy: .newlines) + #expect(lines.allSatisfy { $0.hasPrefix("-- ") }) + } + + @Test("A partition's own indexes and comments follow every partition") + func partitionIndexesAndComments() { + let shape = PostgreSQLTableDDL.shape(from: ["p", "RANGE (created_at)", nil, nil]) + let partitions = [ + ["public.events_2024", "public.events", "FOR VALUES FROM ('2024-01-01') TO ('2025-01-01')", nil, "p"] + ].compactMap { PostgreSQLTableDDL.partition(from: $0) } + let comments = PostgreSQLTableDDL.partitionComments([ + ["public.events_2024", "r", nil, "year 2024"], + ["public.events_2024", "r", "note", "it's a note"], + [nil, "r", nil, "dropped"] + ]) + + let ddl = PostgreSQLTableDDL.script( + name: "\"public\".\"events\"", + definitions: Self.columns, + shape: shape, + partitions: partitions, + partitionIndexes: [ + "CREATE INDEX events_2024_user_id_idx ON public.events_2024 USING btree (user_id)", + "CREATE INDEX events_2025_user_id_idx ON ONLY public.events_2025 USING btree (user_id)" + ], + partitionComments: comments + ) + + #expect(ddl.hasSuffix(""" + CREATE TABLE public.events_2024 PARTITION OF public.events FOR VALUES FROM ('2024-01-01') TO ('2025-01-01'); + + CREATE INDEX events_2024_user_id_idx ON public.events_2024 USING btree (user_id); + + CREATE INDEX events_2025_user_id_idx ON public.events_2025 USING btree (user_id); + + COMMENT ON TABLE public.events_2024 IS 'year 2024'; + + COMMENT ON COLUMN public.events_2024."note" IS 'it''s a note'; + """)) } @Test("An index over a partitioned table is written so the server builds it on every partition") diff --git a/apple/Plugins/TableProPluginKit/PluginStatementContext.swift b/apple/Plugins/TableProPluginKit/PluginStatementContext.swift index ef39c06a8c..11b0524808 100644 --- a/apple/Plugins/TableProPluginKit/PluginStatementContext.swift +++ b/apple/Plugins/TableProPluginKit/PluginStatementContext.swift @@ -35,6 +35,10 @@ public struct PluginStatementContext: Sendable, Equatable { // `.decimal` as numbers, `.boolean` as TRUE or FALSE. Nil, or a nil entry, where none was given. public var parameterKinds: [PluginColumnKind?]? + // The type of the column each parameter is written to or compared with, by position, as the target's driver names + // it. Nil, or a nil entry, where the host does not know it. + public var parameterTypeNames: [String?]? + public init() {} } diff --git a/apple/Plugins/TableProPluginKit/PluginStreamAbort.swift b/apple/Plugins/TableProPluginKit/PluginStreamAbort.swift index 7ed43ac87e..66f7dd8d42 100644 --- a/apple/Plugins/TableProPluginKit/PluginStreamAbort.swift +++ b/apple/Plugins/TableProPluginKit/PluginStreamAbort.swift @@ -71,4 +71,69 @@ public enum PluginRowStream { build(continuation, abort) } } + + // A batch is a few thousand rows, so this caps what a reader that falls behind leaves in memory. + public static let boundedBufferBatches = 16 + + // Every element the producer sends goes through `deliverBlocking` or `deliver`: a full buffer drops a plain yield. + public static func makeBounded( + _ build: @escaping @Sendable ( + AsyncThrowingStream.Continuation, + PluginStreamAbort + ) -> Void + ) -> AsyncThrowingStream { + make(bufferingPolicy: .bufferingOldest(boundedBufferBatches), build) + } + + // Blocks the calling thread while the buffer is full, so it is only for a producer on its own serial queue or + // thread, never the cooperative pool. False, with the element not delivered, once the reader has gone or `stops` + // answers true. + public static func deliverBlocking( + _ element: Element, + to continuation: AsyncThrowingStream.Continuation, + stops: () -> Bool = { false } + ) -> Bool { + var pending = element + while true { + switch continuation.yield(pending) { + case .enqueued: + return true + case .dropped(let refused): + guard !stops() else { return false } + pending = refused + usleep(pollMicroseconds) + case .terminated: + return false + @unknown default: + return false + } + } + } + + // `deliverBlocking` for a producer running as a task: it suspends while the buffer is full. + public static func deliver( + _ element: Element, + to continuation: AsyncThrowingStream.Continuation + ) async -> Bool { + var pending = element + while true { + switch continuation.yield(pending) { + case .enqueued: + return true + case .dropped(let refused): + pending = refused + do { + try await Task.sleep(nanoseconds: UInt64(pollMicroseconds) * 1_000) + } catch { + return false + } + case .terminated: + return false + @unknown default: + return false + } + } + } + + private static let pollMicroseconds: UInt32 = 2_000 } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift index 7d851354e3..e8b6bbd7a0 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataAsyncConnection.swift @@ -19,6 +19,12 @@ final class TeradataAsyncConnection: @unchecked Sendable { // Read outside the queue, so asking never waits behind a running statement. var hasLostConnection: Bool { connection.hasLostConnection } + var transactionState: TeradataTransactionState { connection.transactionState } + + var sessionTransactionMode: TeradataTransactionMode { connection.sessionTransactionMode } + + var sessionModeSource: TeradataSessionModeSource { connection.sessionModeSource } + func connect() async throws { try await run { try $0.connect() } } @@ -44,6 +50,22 @@ final class TeradataAsyncConnection: @unchecked Sendable { try await run { try $0.stream(sql, onColumns: onColumns, onRows: onRows) } } + func beginTransaction() async throws { + try await run { try $0.beginTransaction() } + } + + func commitTransaction() async throws { + try await run { try $0.commitTransaction() } + } + + func rollbackTransaction() async throws { + try await run { try $0.rollbackTransaction() } + } + + func switchDatabase(to database: String) async throws { + try await run { try $0.switchDatabase(to: database) } + } + func disconnect() { queue.sync { connection.disconnect() } } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift index c9dd03b96e..d285ae54af 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPlugin.swift @@ -197,6 +197,7 @@ final class TeradataPluginDriver: PluginDatabaseDriver, @unchecked Sendable { connection.disconnect() throw error } + logSessionMode(connection) self.connection = connection } @@ -282,7 +283,43 @@ final class TeradataPluginDriver: PluginDatabaseDriver, @unchecked Sendable { connection?.cancel() } - func beginTransaction() async throws { _ = try await execute(query: "BEGIN TRANSACTION") } - func commitTransaction() async throws { _ = try await execute(query: "END TRANSACTION") } - func rollbackTransaction() async throws { _ = try await execute(query: "ROLLBACK") } + func beginTransaction() async throws { + try await liveConnection().beginTransaction() + } + + func commitTransaction() async throws { + try await liveConnection().commitTransaction() + } + + func rollbackTransaction() async throws { + try await liveConnection().rollbackTransaction() + } + + func sessionTransactionState() async -> PluginSessionTransactionState { + guard let connection, !connection.hasLostConnection else { return .unknown } + switch connection.transactionState { + case .idle: return .idle + case .open: return .inTransaction + case .unknown: return .unknown + } + } + + func liveConnection() throws -> TeradataAsyncConnection { + guard let connection else { throw TeradataWireError.connectionFailed("not connected") } + return connection + } + + private func logSessionMode(_ connection: TeradataAsyncConnection) { + let mode = connection.sessionTransactionMode.rawValue + switch connection.sessionModeSource { + case .configured: + Self.logger.debug("Transaction mode \(mode, privacy: .public) as configured") + case .helpSession: + Self.logger.info("Transaction mode \(mode, privacy: .public) read from HELP SESSION") + case .sessionInfoView: + Self.logger.info("Transaction mode \(mode, privacy: .public) read from DBC.SessionInfoV") + case .unresolved: + Self.logger.warning("Neither HELP SESSION nor DBC.SessionInfoV gave the transaction mode; using Teradata semantics") + } + } } diff --git a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Schema.swift b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Schema.swift index f6c68eb7a3..0a3483f3fb 100644 --- a/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Schema.swift +++ b/apple/Plugins/TeradataDriverPlugin/TeradataPluginDriver+Schema.swift @@ -131,7 +131,7 @@ extension TeradataPluginDriver { } func switchDatabase(to database: String) async throws { - _ = try await execute(query: TeradataSchemaQueries.setDatabase(database)) + try await liveConnection().switchDatabase(to: database) currentDatabaseName = database } diff --git a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift index 5583026125..435e92693c 100644 --- a/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift +++ b/apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift @@ -11,10 +11,18 @@ import os import TableProPluginKit import TableProSQLGrammar -/// A parameterized SQL statement with placeholders and bound values struct ParameterizedStatement: @unchecked Sendable { let sql: String let parameters: [Any?] + // By position, the type of the column each parameter is written to or compared with. Nil when none is known. + var parameterTypeNames: [String?]? + + // What a driver needs to write each value for its column; everything else stays the host's default. + var context: PluginStatementContext { + var context = PluginStatementContext() + context.parameterTypeNames = parameterTypeNames + return context + } } // The rows it should touch hold the server to a count; the changes it writes tell a written change from a dropped one. @@ -49,6 +57,7 @@ struct SQLStatementGenerator { /// Set only by a restore that writes a row back with the identity it had: PostgreSQL refuses an explicit value /// for a `GENERATED ALWAYS` column unless the INSERT says so. let insertOverridesSystemValue: Bool + private let columnTypeNames: [String: String] // BIT columns of a family that reads them as a number. Text bound to one is stored and compared as its bytes. private let numberBitColumns: Set private let quoteIdentifierFn: (String) -> String @@ -82,6 +91,7 @@ struct SQLStatementGenerator { self.databaseType = databaseType self.parameterStyle = parameterStyle ?? Self.defaultParameterStyle(for: databaseType) self.insertOverridesSystemValue = insertOverridesSystemValue + self.columnTypeNames = columnTypeNames self.numberBitColumns = Self.numberBitColumns(in: columnTypeNames, databaseType: databaseType) if let quoteIdentifier { self.quoteIdentifierFn = quoteIdentifier @@ -191,18 +201,43 @@ struct SQLStatementGenerator { return Set(bitColumns.keys) } + private struct Bindings { + var values: [Any?] = [] + var typeNames: [String?] = [] + + var count: Int { values.count } + + mutating func append(_ value: Any?, typeName: String?) { + values.append(value) + typeNames.append(typeName) + } + } + + private func statement(_ sql: String, _ bindings: Bindings) -> ParameterizedStatement { + ParameterizedStatement( + sql: sql, + parameters: bindings.values, + parameterTypeNames: bindings.typeNames.contains { $0 != nil } ? bindings.typeNames : nil + ) + } + + private func typeName(of column: String?) -> String? { + column.flatMap { columnTypeNames[$0] } + } + // Blank text goes as '', which the server stores as 0: cast, strict mode refuses it as a truncated integer, and a // space bound as it is would be its byte, 32. - private func bind(_ value: PluginCellValue, toColumn column: String?, into parameters: inout [Any?]) -> String { + private func bind(_ value: PluginCellValue, toColumn column: String?, into parameters: inout Bindings) -> String { guard let column, numberBitColumns.contains(column), case .text(let text) = value else { - parameters.append(value.asAny) + parameters.append(value.asAny, typeName: typeName(of: column)) return placeholder(at: parameters.count - 1) } guard !text.trimmingCharacters(in: .whitespaces).isEmpty else { - parameters.append("") + parameters.append("", typeName: typeName(of: column)) return placeholder(at: parameters.count - 1) } - parameters.append(Self.decimalBitText(text)) + // The value is the CAST's operand, not the column's. + parameters.append(Self.decimalBitText(text), typeName: nil) return "CAST(\(placeholder(at: parameters.count - 1)) AS UNSIGNED)" } @@ -249,7 +284,7 @@ struct SQLStatementGenerator { { var nonDefaultColumns: [String] = [] var placeholderParts: [String] = [] - var bindParameters: [Any?] = [] + var bindParameters = Bindings() for (index, value) in values.enumerated() { // Left out rather than written as DEFAULT, which SQLite does not take in VALUES. @@ -278,7 +313,7 @@ struct SQLStatementGenerator { let sql = "INSERT INTO \(qualifiedTableName) (\(columnList))\(override) VALUES (\(placeholders))" - return ParameterizedStatement(sql: sql, parameters: bindParameters) + return statement(sql, bindParameters) } private func allDefaultsInsertStatement() -> ParameterizedStatement? { @@ -293,7 +328,7 @@ struct SQLStatementGenerator { { guard !insertColumns.isEmpty, insertColumns.count == values.count else { return nil } - var bindParameters: [Any?] = [] + var bindParameters = Bindings() let columnList = insertColumns.map(quoteIdentifierFn).joined(separator: ", ") let placeholders = zip(insertColumns, values).map { column, value in bind(value, toColumn: column, into: &bindParameters) @@ -302,7 +337,7 @@ struct SQLStatementGenerator { let sql = "INSERT INTO \(qualifiedTableName) (\(columnList)) VALUES (\(placeholders))" - return ParameterizedStatement(sql: sql, parameters: bindParameters) + return statement(sql, bindParameters) } func insertStatement(columns insertColumns: [String], rows: [[PluginCellValue]]) @@ -311,7 +346,7 @@ struct SQLStatementGenerator { guard !insertColumns.isEmpty, !rows.isEmpty, rows.allSatisfy({ $0.count == insertColumns.count }) else { return nil } - var bindParameters: [Any?] = [] + var bindParameters = Bindings() let columnList = insertColumns.map(quoteIdentifierFn).joined(separator: ", ") let rowTuples = rows.map { values -> String in let placeholders = zip(insertColumns, values).map { column, value in @@ -323,14 +358,14 @@ struct SQLStatementGenerator { let sql = "INSERT INTO \(qualifiedTableName) (\(columnList)) VALUES \(rowTuples)" - return ParameterizedStatement(sql: sql, parameters: bindParameters) + return statement(sql, bindParameters) } // The server compares, so its casting and collation decide a match, as they do for the UPDATE that follows. func keyMatchCountStatement(keyColumns: [String], keys: [[PluginCellValue]]) -> ParameterizedStatement? { guard !keyColumns.isEmpty, !keys.isEmpty, keys.allSatisfy({ $0.count == keyColumns.count }) else { return nil } - var bindParameters: [Any?] = [] + var bindParameters = Bindings() let selects = keys.enumerated().map { position, values -> String in let conditions = zip(keyColumns, values).map { column, value -> String in "\(quoteIdentifierFn(column)) = \(bind(value, toColumn: column, into: &bindParameters))" @@ -338,7 +373,7 @@ struct SQLStatementGenerator { return "SELECT \(position), COUNT(*) FROM \(qualifiedTableName) WHERE \(conditions)" } - return ParameterizedStatement(sql: selects.joined(separator: " UNION ALL "), parameters: bindParameters) + return statement(selects.joined(separator: " UNION ALL "), bindParameters) } func keyedUpdateStatement( @@ -350,7 +385,7 @@ struct SQLStatementGenerator { guard !setColumns.isEmpty, setColumns.count == values.count, !keyColumns.isEmpty, keyColumns.count == keyValues.count else { return nil } - var bindParameters: [Any?] = [] + var bindParameters = Bindings() func bound(_ column: String, _ value: PluginCellValue) -> String { "\(quoteIdentifierFn(column)) = \(bind(value, toColumn: column, into: &bindParameters))" } @@ -359,7 +394,7 @@ struct SQLStatementGenerator { let sql = rowUpdateFn?(qualifiedTableName, assignments, conditions) ?? "UPDATE \(qualifiedTableName) SET \(assignments) WHERE \(conditions)" - return ParameterizedStatement(sql: sql, parameters: bindParameters) + return statement(sql, bindParameters) } var maxBindParameters: Int { @@ -388,7 +423,7 @@ struct SQLStatementGenerator { quoteIdentifierFn($0.columnName) }.joined(separator: ", ") - var parameters: [Any?] = [] + var parameters = Bindings() let placeholders = nonDefaultChanges.map { cellChange -> String in switch cellChange.newValue { case .text(let expression) where cellChange.newIsExpression: @@ -401,7 +436,7 @@ struct SQLStatementGenerator { let sql = "INSERT INTO \(qualifiedTableName) (\(columnNames)) VALUES (\(placeholders))" - return ParameterizedStatement(sql: sql, parameters: parameters) + return statement(sql, parameters) } // MARK: - UPDATE Generation @@ -412,7 +447,7 @@ struct SQLStatementGenerator { let writableChanges = change.cellChanges.filter { !generatedColumns.contains($0.columnName) } guard !writableChanges.isEmpty else { return nil } - var parameters: [Any?] = [] + var parameters = Bindings() let setClauses = writableChanges.map { cellChange -> String in switch cellChange.newValue { case .text(let expression) where cellChange.newIsExpression: @@ -451,7 +486,7 @@ struct SQLStatementGenerator { let whereClause = rowCondition(terms, parameters: ¶meters) let sql = "UPDATE \(qualifiedTableName) SET \(setClauses) WHERE \(whereClause)" - return ParameterizedStatement(sql: sql, parameters: parameters) + return statement(sql, parameters) } else { guard let originalRow = change.originalRow else { Self.logger.warning( @@ -482,7 +517,7 @@ struct SQLStatementGenerator { let sql = "UPDATE \(qualifiedTableName) SET \(setClauses) WHERE \(whereClause)" - return ParameterizedStatement(sql: sql, parameters: parameters) + return statement(sql, parameters) } } @@ -501,22 +536,23 @@ struct SQLStatementGenerator { /// affinity, and the grid read either one as `5`, so a number the grid read matches both, held by CAST to the /// text SQLite reads it back as so a REAL 5.0 is not matched too; a typed column converts either literal to its /// own type, so the form is exact there too. When both rows exist the statement itself refuses them. - private func rowCondition(_ terms: [RowMatchTerm], parameters: inout [Any?]) -> String { + private func rowCondition(_ terms: [RowMatchTerm], parameters: inout Bindings) -> String { let clause = conditions(terms, parameters: ¶meters) guard terms.contains(where: { matchesEitherWay($0.value) }) else { return clause } let again = conditions(terms, parameters: ¶meters) return "\(clause) AND (SELECT COUNT(*) FROM \(qualifiedTableName) WHERE \(again)) = 1" } - private func conditions(_ terms: [RowMatchTerm], parameters: inout [Any?]) -> String { + private func conditions(_ terms: [RowMatchTerm], parameters: inout Bindings) -> String { terms.map { term in guard let value = term.value else { return "\(term.matched) IS NULL" } guard matchesEitherWay(value), case .text(let text) = value else { return "\(term.matched) = \(bind(value, toColumn: term.column, into: ¶meters))" } - parameters.append(value.asAny) + parameters.append(value.asAny, typeName: typeName(of: term.column)) let bound = placeholder(at: parameters.count - 1) - parameters.append(value.asAny) + // Compared with the column's text, not the column. + parameters.append(value.asAny, typeName: nil) let read = placeholder(at: parameters.count - 1) return "(\(term.matched) IN (\(text), \(bound)) AND CAST(\(term.matched) AS TEXT) = \(read))" }.joined(separator: " AND ") @@ -622,7 +658,7 @@ struct SQLStatementGenerator { } private func deleteStatement(for rows: [[DeleteColumnMatch]]) -> ParameterizedStatement { - var parameters: [Any?] = [] + var parameters = Bindings() let rowClauses = rows.map { matches -> String in let terms = matches.map { match in RowMatchTerm( @@ -643,6 +679,6 @@ struct SQLStatementGenerator { let whereClause = rowClauses.joined(separator: " OR ") let sql = "DELETE FROM \(qualifiedTableName) WHERE \(whereClause)" - return ParameterizedStatement(sql: sql, parameters: parameters) + return statement(sql, parameters) } } diff --git a/apple/TablePro/Core/Compare/CompareRowService.swift b/apple/TablePro/Core/Compare/CompareRowService.swift index 909a199607..fde3b89964 100644 --- a/apple/TablePro/Core/Compare/CompareRowService.swift +++ b/apple/TablePro/Core/Compare/CompareRowService.swift @@ -197,7 +197,9 @@ internal struct CompareRowService { ), targetDriver: targetPlugin ) - return try await body(engine, sides) + return try await BulkReadTimeout.lifted(on: [sourceDriver, targetDriver]) { + try await body(engine, sides) + } } } } diff --git a/apple/TablePro/Core/Compare/StreamingRowProvider.swift b/apple/TablePro/Core/Compare/StreamingRowProvider.swift index 54a4ac107f..55c04befd1 100644 --- a/apple/TablePro/Core/Compare/StreamingRowProvider.swift +++ b/apple/TablePro/Core/Compare/StreamingRowProvider.swift @@ -5,11 +5,11 @@ // Feeds the merge join from a driver's row stream. It holds one batch of rows // at a time, and hands them out one row at a time. // -// It is not what bounds a comparison's memory. A driver's stream buffers -// `.unbounded` and its producer starts at the moment the stream is built, not -// at the first read, so a side the walk consumes slowly runs ahead inside the +// It is not what bounds a comparison's memory. PostgreSQL, MySQL, SQL Server, +// SAP HANA and DuckDB hold a few batches and wait for the walk; the other +// drivers' streams run ahead of a side the walk consumes slowly, inside the // stream's own buffer. The row limit, the filter, and the key order the two -// sides share are what keep a comparison's footprint down. +// sides share are what keep a comparison's footprint down there. // import Foundation diff --git a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift index 641a46bbc1..480e650ecb 100644 --- a/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift +++ b/apple/TablePro/Core/Coordinators/QueryExecutionCoordinator+Parameters.swift @@ -606,10 +606,14 @@ extension QueryExecutionCoordinator { let endState: PluginSessionTransactionState = asksEndState ? await driver.heldSessionTransactionState() : .unknown + // An unwrapped run can still leave a transaction open: a Teradata ANSI session holds its DDL in one. + let failedState: PluginSessionTransactionState = sessionPlan == .autocommit && !outcome.isCompleted + ? (asksEndState ? endState : await driver.heldSessionTransactionState()) + : .idle return MultiStatementRun( outcome: outcome, plan: sessionPlan, - sessionState: .idle, + sessionState: failedState, startState: startState, endState: endState ) diff --git a/apple/TablePro/Core/Database/SessionStateLedger.swift b/apple/TablePro/Core/Database/SessionStateLedger.swift index f248ee458c..8ed93187c8 100644 --- a/apple/TablePro/Core/Database/SessionStateLedger.swift +++ b/apple/TablePro/Core/Database/SessionStateLedger.swift @@ -135,7 +135,7 @@ extension SQLSessionDialect { self = .sqlite case .duckdb: self = .duckDB - case .redis, .other: + case .redis, .teradata, .other: return nil } } diff --git a/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift b/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift index 5b6028255f..2eb427e9b1 100644 --- a/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift +++ b/apple/TablePro/Core/ObjectCopy/ObjectCopyRowCopier.swift @@ -188,13 +188,13 @@ internal struct ObjectCopyRowCopier: Sendable { format: String(localized: "Could not build an INSERT for %@."), step.qualifiedTargetName )) } - /// The generator's own parameter array rather than the batch flattened again: the two - /// agree today, and taking the statement's own leaves no second place for the placeholder - /// order to be decided. The round trip is lossless because `PluginCellValue` is text, - /// bytes or null and `asAny` maps each to itself. + // The generator's own parameter array rather than the batch flattened again, so the placeholder order is + // decided in one place. `PluginCellValue` is text, bytes or null and `asAny` maps each to itself. The context + // carries each value's target column type: Oracle wrote `00123` into a VARCHAR2 as 123 without it. _ = try await driver.executeParameterized( query: statement.sql, - parameters: statement.parameters.map(PluginDriverAdapter.cellValue(for:)) + parameters: statement.parameters.map(PluginDriverAdapter.cellValue(for:)), + context: statement.context ) return batch.count } diff --git a/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift b/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift index f8c04fd656..336bd5ba68 100644 --- a/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift +++ b/apple/TablePro/Core/ObjectCopy/ObjectCopyRunner.swift @@ -702,8 +702,10 @@ internal struct ObjectCopyRunner { guard let sourcePlugin = CompareMetadataService.pluginDriver(from: sourceDriver) else { throw ObjectCopyError.refused(noSourceDriver) } - return try await copier.copy(from: sourcePlugin, to: targetPlugin) { rows in - progress.setRowsForCurrentObject(rows, completedBefore: completedBefore) + return try await BulkReadTimeout.lifted(on: [sourceDriver]) { + try await copier.copy(from: sourcePlugin, to: targetPlugin) { rows in + progress.setRowsForCurrentObject(rows, completedBefore: completedBefore) + } } } guard !outcome.cancelled else { @@ -804,8 +806,10 @@ internal struct ObjectCopyRunner { guard let sourcePlugin = CompareMetadataService.pluginDriver(from: sourceDriver) else { throw ObjectCopyError.refused(Self.noSourceDriver) } - let outcome = try await copier.copy(from: sourcePlugin, to: targetPlugin) { rows in - progress.setRowsForCurrentObject(rows, completedBefore: completedBefore) + let outcome = try await BulkReadTimeout.lifted(on: [sourceDriver]) { + try await copier.copy(from: sourcePlugin, to: targetPlugin) { rows in + progress.setRowsForCurrentObject(rows, completedBefore: completedBefore) + } } return CopiedRows(outcome: outcome, joinedSessionTransaction: owner == .session) } diff --git a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift index cb350fb44f..e8c2c62d93 100644 --- a/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift +++ b/apple/TablePro/Core/Plugins/ImportDataSinkAdapter.swift @@ -125,14 +125,28 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { ) } - // MySQL stores text bound to a BIT column as its bytes, so a 5 lands as 53 unless the INSERT knows the column is BIT. + // The driver gets each parameter's column type: Oracle stored `00123` bound to a VARCHAR2 as 123 without it, and + // MySQL stores text bound to a BIT column as its bytes, so a 5 lands as 53. private func writingGenerator() async throws -> SQLStatementGenerator? { - guard let rowGenerator, let targetTable, SQLTypeFamily.of(databaseType).readsBitsAsDecimal else { - return rowGenerator - } + guard let rowGenerator, let targetTable else { return rowGenerator } if let typed = typedRowGenerator.withLockUnchecked({ $0 }) { return typed } - let columns = try await importedData { - try await self.driver.fetchColumns(table: targetTable, schema: self.targetSchema) + // A BIT family and Oracle write wrong values without the types; elsewhere the import goes on as it did before + // the types were read, and the error stays out of the run's list. + let family = SQLTypeFamily.of(databaseType) + let needsTypes = family.readsBitsAsDecimal || family == .oracle + let read: [ColumnInfo]? = try await importedData { + do { + return try await self.driver.fetchColumns(table: targetTable, schema: self.targetSchema) + } catch where !needsTypes && !(error is CancellationError) { + Self.logger.warning( + "Import writes untyped values: the target's columns could not be read: \(error.localizedDescription, privacy: .private)" + ) + return nil + } + } + guard let columns = read else { + typedRowGenerator.withLockUnchecked { $0 = rowGenerator } + return rowGenerator } var typeNames: [String: String] = [:] for column in columns where typeNames[column.name] == nil { @@ -246,7 +260,9 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { } try await importedData(rows: 1, writes: true) { - _ = try await self.driver.executeParameterized(query: statement.sql, parameters: statement.parameters) + _ = try await self.driver.executeParameterized( + query: statement.sql, parameters: statement.parameters, context: statement.context + ) self.tally.withLock { $0.inserted += 1 } } } @@ -343,7 +359,9 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { ) } try await importedData(rows: batch.count, writes: true) { - _ = try await self.driver.executeParameterized(query: statement.sql, parameters: statement.parameters) + _ = try await self.driver.executeParameterized( + query: statement.sql, parameters: statement.parameters, context: statement.context + ) self.tally.withLock { $0.inserted += batch.count } } } @@ -456,7 +474,9 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { throw PluginImportError.importFailed(String(localized: "Could not build an UPDATE for the mapped columns")) } try await importedData(rows: 1, writes: true) { - _ = try await self.driver.executeParameterized(query: statement.sql, parameters: statement.parameters) + _ = try await self.driver.executeParameterized( + query: statement.sql, parameters: statement.parameters, context: statement.context + ) self.tally.withLock { $0.updated += 1 } } } @@ -486,7 +506,9 @@ final class ImportDataSinkAdapter: PluginImportDataSink, @unchecked Sendable { throw PluginImportError.importFailed(String(localized: "Could not build the query that matches rows")) } let result = try await importedData { - try await self.driver.executeParameterized(query: statement.sql, parameters: statement.parameters) + try await self.driver.executeParameterized( + query: statement.sql, parameters: statement.parameters, context: statement.context + ) } counts += try Self.counts(in: result, expected: chunk.count) start += chunk.count diff --git a/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement+Teradata.swift b/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement+Teradata.swift new file mode 100644 index 0000000000..680bb09c20 --- /dev/null +++ b/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement+Teradata.swift @@ -0,0 +1,49 @@ +// +// AutocommitOnlyStatement+Teradata.swift +// TablePro +// + +import Foundation +import TableProSQLGrammar +import TableProUtilities + +internal extension AutocommitOnlyStatement { + // Inside a transaction Teradata takes nothing after DDL but its end: failure 3932 in Teradata mode, error 3722 in + // ANSI. DDL is what the SQL Data Definition Language manual lists, less HELP and SHOW; the ANSI transaction rules + // name DATABASE and SET SESSION, and the Data Control Language manual puts GRANT, REVOKE and GIVE under the same + // rule. Unmeasured: there is no Teradata server here. + static func matchesTeradata(_ statement: NSString, grammar: SQLLexicalGrammar) -> Bool { + var cursor = SQLTokenCursor(statement, grammar: grammar) + guard let keyword = cursor.next()?.word else { return false } + if teradataDefinitionKeywords.contains(keyword) { return true } + let following = cursor.next()?.word ?? "" + switch keyword { + case "BEGIN", "END": + return teradataLoggingTargets.contains(following) + case "CHECKPOINT": + return following == "ISOLATED" + case "COLLECT": + return teradataStatisticsWords.contains(following) + case "DELETE": + return following == "DATABASE" || following == "USER" + case "SET": + return teradataSessionSettings.contains(following) + default: + return false + } + } +} + +private extension AutocommitOnlyStatement { + // CT is the CREATE TABLE abbreviation the CREATE TABLE syntax diagram gives. + static let teradataDefinitionKeywords: Set = [ + "CREATE", "CT", "ALTER", "DROP", "RENAME", "REPLACE", "COMMENT", "MODIFY", "DATABASE", "FLUSH", "LOGGING", + "INCREMENTAL", "GRANT", "REVOKE", "GIVE" + ] + + static let teradataLoggingTargets: Set = ["LOGGING", "QUERY", "ISOLATED"] + + static let teradataStatisticsWords: Set = ["STATISTICS", "STATS", "STAT", "SUMMARY"] + + static let teradataSessionSettings: Set = ["SESSION", "ROLE", "QUERY_BAND", "TIME", "TRANSFORM"] +} diff --git a/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement.swift b/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement.swift index 5266aed1b5..dad6484b4e 100644 --- a/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement.swift +++ b/apple/TablePro/Core/Services/Execution/AutocommitOnlyStatement.swift @@ -45,6 +45,8 @@ internal enum AutocommitOnlyStatement { return matchesDuckDB(statement, grammar: grammar) case .sqlServer: return matchesSQLServer(statement, grammar: grammar) + case .teradata: + return matchesTeradata(statement, grammar: grammar) case .oracle, .redis, .other: return false } diff --git a/apple/TablePro/Core/Services/Execution/MultiStatementFailure.swift b/apple/TablePro/Core/Services/Execution/MultiStatementFailure.swift index d4b7700a1e..06c8c9b015 100644 --- a/apple/TablePro/Core/Services/Execution/MultiStatementFailure.swift +++ b/apple/TablePro/Core/Services/Execution/MultiStatementFailure.swift @@ -157,7 +157,7 @@ internal struct MultiStatementFailureContext: Equatable, Sendable { private func appliedWorkNote(position: Int) -> String? { switch unit { case .statement: - return appliedStatementsNote() + return sessionState.openTransactionNotice ?? appliedStatementsNote() case .batch: return sessionState.openTransactionNotice ?? appliedBatchesNote(position: position) } diff --git a/apple/TablePro/Core/Services/Execution/TransactionEngineFamily.swift b/apple/TablePro/Core/Services/Execution/TransactionEngineFamily.swift index bc761bd61d..05065ae969 100644 --- a/apple/TablePro/Core/Services/Execution/TransactionEngineFamily.swift +++ b/apple/TablePro/Core/Services/Execution/TransactionEngineFamily.swift @@ -27,6 +27,7 @@ internal enum TransactionEngineFamily: String, Hashable, Sendable, CaseIterable case sqlServer case oracle case redis + case teradata case other internal static func of(_ type: DatabaseType) -> TransactionEngineFamily { @@ -83,6 +84,7 @@ internal enum TransactionEngineFamily: String, Hashable, Sendable, CaseIterable "DuckDB": .duckdb, "SQL Server": .sqlServer, "Oracle": .oracle, - "Redis": .redis + "Redis": .redis, + "Teradata": .teradata ] } diff --git a/apple/TablePro/Core/Services/Export/BulkReadTimeout.swift b/apple/TablePro/Core/Services/Export/BulkReadTimeout.swift new file mode 100644 index 0000000000..64245263cb --- /dev/null +++ b/apple/TablePro/Core/Services/Export/BulkReadTimeout.swift @@ -0,0 +1,58 @@ +// +// BulkReadTimeout.swift +// TablePro +// + +import Foundation +import os + +// A driver's row stream waits for its reader, so a bulk read lasts as long as the copy, transfer or compare consuming +// it, and the session's interactive query timeout would end it part way. The export lifts the timeout the same way. +enum BulkReadTimeout { + private static let logger = Logger(subsystem: "com.TablePro", category: "BulkReadTimeout") + + static func lifted( + on drivers: [DatabaseDriver], + _ operation: () async throws -> T + ) async throws -> T { + var restored: [(DatabaseDriver, Int)] = [] + for driver in drivers { + restored.append((driver, await restoredSeconds(for: driver))) + await apply(0, on: driver) + } + do { + let value = try await operation() + await restore(restored) + return value + } catch { + await restore(restored) + throw error + } + } + + // In a task of its own, so a Stop that cancelled the read still puts the timeout back on the pooled connection. + private static func restore(_ timeouts: [(DatabaseDriver, Int)]) async { + await Task { + for (driver, seconds) in timeouts { await apply(seconds, on: driver) } + }.value + } + + static func restoredSeconds(for driver: DatabaseDriver) async -> Int { + if let adapter = driver as? PluginDriverAdapter { return adapter.effectiveQueryTimeoutSeconds } + let configured = driver.connection.queryTimeoutSeconds + return await MainActor.run { + ConnectionTimeoutPolicy.effectiveQueryTimeoutSeconds( + configuredSeconds: configured, + globalSeconds: AppSettingsManager.shared.general.queryTimeoutSeconds + ) + } + } + + private static func apply(_ seconds: Int, on driver: DatabaseDriver) async { + do { + try await driver.applyQueryTimeout(seconds) + } catch { + logger.warning("Query timeout \(seconds) not applied for a bulk read: \(error.localizedDescription, privacy: .private)") + } + } +} diff --git a/apple/TablePro/Core/Services/Export/TableTransferService.swift b/apple/TablePro/Core/Services/Export/TableTransferService.swift index 8806cac233..63555de315 100644 --- a/apple/TablePro/Core/Services/Export/TableTransferService.swift +++ b/apple/TablePro/Core/Services/Export/TableTransferService.swift @@ -177,11 +177,13 @@ final class TableTransferService: ObservableObject { workload: .bulk, cancellation: .protectedWrite ) { destinationDriver in - try await self.transfer( - request: request, - sourceDriver: sourceDriver, - destinationDriver: destinationDriver - ) + try await BulkReadTimeout.lifted(on: [sourceDriver]) { + try await self.transfer( + request: request, + sourceDriver: sourceDriver, + destinationDriver: destinationDriver + ) + } } } } diff --git a/apple/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift b/apple/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift index 9672323deb..a501f6cbb9 100644 --- a/apple/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift +++ b/apple/TablePro/Core/Utilities/SQL/SQLClientEncodingRewriter.swift @@ -36,7 +36,7 @@ internal enum SQLClientEncodingRewriter { replacements = mysqlClientCharacterSets(&cursor, in: text) case .postgres, .redshift, .cockroach: replacements = postgresClientEncoding(&cursor, in: text).map { [Replacement(range: $0, text: postgresEncoding)] } ?? [] - case .sqlite, .duckdb, .sqlServer, .oracle, .redis, .other: + case .sqlite, .duckdb, .sqlServer, .oracle, .redis, .teradata, .other: return nil } guard !replacements.isEmpty else { return nil } diff --git a/apple/TablePro/Core/Utilities/SQL/TableEditDialect.swift b/apple/TablePro/Core/Utilities/SQL/TableEditDialect.swift index 214686aad0..1edd37db8c 100644 --- a/apple/TablePro/Core/Utilities/SQL/TableEditDialect.swift +++ b/apple/TablePro/Core/Utilities/SQL/TableEditDialect.swift @@ -78,7 +78,7 @@ struct TableEditDialect: Sendable, Equatable { return .sqlServer case .oracle: return .oracle - case .cockroach, .redis, .other: + case .cockroach, .redis, .teradata, .other: return nil } } diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index a3f7909720..63f0a55377 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -16249,6 +16249,10 @@ }, "A label cannot be removed or moved once it exists." : { + }, + "A large object value is longer than the 64 MiB TablePro reads from SAP HANA, so it cannot be copied or exported whole." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "A larger binary value stays on the server until you open, copy or export it. Applies to tables with a primary key." : { @@ -44073,6 +44077,14 @@ } } }, + "Column \"%1$@\" has Teradata type %2$@, which TablePro cannot read yet. CAST it to DECIMAL or VARCHAR in the query." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "Column \"%1$@\" has Teradata type %2$@, which TablePro cannot read yet. CAST it to VARCHAR in the query." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "column %@ first appears in a later chunk" : { "extractionState" : "manual", "generatesSymbol" : false @@ -45569,6 +45581,10 @@ }, "Commit Message" : { + }, + "Commit or roll back the open transaction before switching database." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "Commit the transaction" : { @@ -187156,6 +187172,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The condition of check constraint %@ cannot be written as SQL." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The condition of index %@ cannot be written as SQL." : { "extractionState" : "manual", "generatesSymbol" : false @@ -192081,6 +192101,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The referential action of foreign key %@ is not one SQL Server has: %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "the replica is not available on this broker" : { "extractionState" : "manual", "generatesSymbol" : false, diff --git a/apple/TableProMobile/TableProMobile/Localizable.xcstrings b/apple/TableProMobile/TableProMobile/Localizable.xcstrings index bafad10cd7..0975eda411 100644 --- a/apple/TableProMobile/TableProMobile/Localizable.xcstrings +++ b/apple/TableProMobile/TableProMobile/Localizable.xcstrings @@ -25816,6 +25816,10 @@ "extractionState" : "manual", "generatesSymbol" : false }, + "The referential action of foreign key %@ is not one SQL Server has: %@" : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The report contains:" : { "localizations" : { "ko" : { diff --git a/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorParameterTypeTests.swift b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorParameterTypeTests.swift new file mode 100644 index 0000000000..2f9aed77eb --- /dev/null +++ b/apple/TableProTests/Core/ChangeTracking/SQLStatementGeneratorParameterTypeTests.swift @@ -0,0 +1,119 @@ +// +// SQLStatementGeneratorParameterTypeTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +// Oracle stored an imported `00123` in a VARCHAR2 as 123: the driver was never told which column each value was for. +struct SQLStatementGeneratorParameterTypeTests { + private static let columns = ["ID", "CODE", "BORN"] + private static let typeNames = ["ID": "NUMBER(10)", "CODE": "VARCHAR2(20 BYTE)", "BORN": "DATE"] + + private func makeGenerator( + primaryKeyColumns: [String] = ["ID"], + databaseType: DatabaseType = .postgresql, + typeNames: [String: String] = SQLStatementGeneratorParameterTypeTests.typeNames + ) throws -> SQLStatementGenerator { + try SQLStatementGenerator( + tableName: "people", + columns: Self.columns, + primaryKeyColumns: primaryKeyColumns, + databaseType: databaseType, + columnTypeNames: typeNames + ) + } + + @Test + func anInsertNamesTheColumnTypeOfEachValue() throws { + let statement = try #require(try makeGenerator().insertStatement(columns: ["CODE", "ID"], values: ["00123", "7"])) + + #expect(statement.parameterTypeNames == ["VARCHAR2(20 BYTE)", "NUMBER(10)"]) + } + + @Test + func aMultiRowInsertNamesTheTypesForEveryRow() throws { + let statement = try #require(try makeGenerator().insertStatement( + columns: ["ID", "CODE"], + rows: [["1", "00123"], ["2", "0.50"]] + )) + + #expect(statement.parameterTypeNames == ["NUMBER(10)", "VARCHAR2(20 BYTE)", "NUMBER(10)", "VARCHAR2(20 BYTE)"]) + } + + @Test + func aColumnOfNoKnownTypeLeavesItsEntryEmpty() throws { + let statement = try #require(try makeGenerator().insertStatement(columns: ["CODE", "NOTE"], values: ["a", "b"])) + + #expect(statement.parameterTypeNames == ["VARCHAR2(20 BYTE)", nil]) + } + + @Test + func noKnownTypeLeavesTheListOut() throws { + let generator = try makeGenerator(typeNames: [:]) + + let insert = try #require(generator.insertStatement(columns: ["CODE"], values: ["00123"])) + let match = try #require(generator.keyMatchCountStatement(keyColumns: ["ID"], keys: [["7"]])) + + #expect(insert.parameterTypeNames == nil) + #expect(match.parameterTypeNames == nil) + } + + @Test + func aKeyedUpdateNamesItsAssignmentsThenItsKey() throws { + let statement = try #require(try makeGenerator().keyedUpdateStatement( + columns: ["CODE", "BORN"], + values: ["00123", "2024-01-15"], + keyColumns: ["ID"], + keyValues: ["7"] + )) + + #expect(statement.parameterTypeNames == ["VARCHAR2(20 BYTE)", "DATE", "NUMBER(10)"]) + } + + @Test + func aKeyMatchNamesTheKeyTypeForEveryKey() throws { + let statement = try #require(try makeGenerator().keyMatchCountStatement( + keyColumns: ["CODE", "ID"], + keys: [["00123", "1"], ["00456", "2"]] + )) + + #expect(statement.parameterTypeNames == ["VARCHAR2(20 BYTE)", "NUMBER(10)", "VARCHAR2(20 BYTE)", "NUMBER(10)"]) + } + + @Test + func aGridEditNamesItsNewValueAndItsKey() throws { + let change = RowChange( + rowID: .existing(0), + type: .update, + cellChanges: [CellChange(columnIndex: 1, columnName: "CODE", oldValue: "00100", newValue: "00123")], + originalRow: ["7", "00100", .null] + ) + + let statement = try #require(try makeGenerator().generateUpdateSQL(for: change)) + + #expect(statement.parameterTypeNames == ["VARCHAR2(20 BYTE)", "NUMBER(10)"]) + } + + @Test + func aBitValueCastToANumberHasNoColumnType() throws { + let generator = try makeGenerator(databaseType: .mysql, typeNames: ["ID": "INT", "CODE": "BIT(8)"]) + + let statement = try #require(generator.insertStatement(columns: ["CODE", "ID"], values: ["5", "1"])) + + #expect(statement.sql == "INSERT INTO `people` (`CODE`, `ID`) VALUES (CAST(? AS UNSIGNED), ?)") + #expect(statement.parameterTypeNames == [nil, "INT"]) + } + + @Test + func theStatementContextCarriesTheTypesAndNothingElse() throws { + let statement = try #require(try makeGenerator().insertStatement(columns: ["CODE"], values: ["00123"])) + var expected = PluginStatementContext() + expected.parameterTypeNames = ["VARCHAR2(20 BYTE)"] + + #expect(statement.context == expected) + } +} diff --git a/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift b/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift index 772fdf6ba5..f04ead34ba 100644 --- a/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift +++ b/apple/TableProTests/Core/ObjectCopy/ObjectCopyRowCopierTests.swift @@ -17,6 +17,7 @@ private final class CopyDriver: PluginDriverStub, @unchecked Sendable { var batchSize = 2 var executedQueries: [String] = [] var executedParameters: [[PluginCellValue]] = [] + var executedTypeNames: [[String?]?] = [] var quote: (String) -> String = { "`\($0)`" } var declaredParameterStyle: ParameterStyle = .questionMark var transactionEvents: [String] = [] @@ -34,6 +35,15 @@ private final class CopyDriver: PluginDriverStub, @unchecked Sendable { return PluginQueryResult(columns: [], columnTypeNames: [], rows: [], rowsAffected: 0, executionTime: 0) } + func executeParameterized( + query: String, + parameters: [PluginCellValue], + context: PluginStatementContext + ) async throws -> PluginQueryResult { + executedTypeNames.append(context.parameterTypeNames) + return try await executeParameterized(query: query, parameters: parameters) + } + func streamRows(query: String) -> AsyncThrowingStream { let rows = streamed let size = batchSize @@ -361,6 +371,38 @@ struct ObjectCopyRowCopierTests { #expect(target.executedParameters.first?.map(\.sortKey) == ["1", "00000101"]) } + // MARK: - Column types + + // Oracle stored a copied `00123` in a VARCHAR2 as 123: the driver was never told the column was text. + @Test + func eachValueTellsTheTargetItsColumnType() async throws { + let source = CopyDriver() + source.streamed = [[.text("1"), .text("00123")], [.text("2"), .text("0.50")]] + let target = CopyDriver() + + _ = try await ObjectCopyRowCopier( + step: step(columnTypeNames: ["id": "NUMBER", "name": "VARCHAR2(20 BYTE)"]), + sourceDatabaseType: .postgresql, + targetDatabaseType: .oracle + ).copy(from: source, to: target) { _ in } + + // Oracle takes one row per INSERT. + #expect(target.executedTypeNames == [["NUMBER", "VARCHAR2(20 BYTE)"], ["NUMBER", "VARCHAR2(20 BYTE)"]]) + #expect(target.executedParameters.map { row in row.map { $0.sortKey } } == [["1", "00123"], ["2", "0.50"]]) + } + + @Test + func aTargetOfNoKnownColumnTypesIsToldNone() async throws { + let source = CopyDriver() + source.streamed = rows(1) + let target = CopyDriver() + + _ = try await ObjectCopyRowCopier(step: step(), sourceDatabaseType: .mysql, targetDatabaseType: .mysql) + .copy(from: source, to: target) { _ in } + + #expect(target.executedTypeNames == [nil]) + } + private static func draft( columns: [PluginColumnInfo], from source: DatabaseType, diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift index 6af9834a0a..013f2fcdd4 100644 --- a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterBitColumnTests.swift @@ -101,13 +101,14 @@ struct ImportDataSinkAdapterBitColumnTests { #expect(sql.last == "UPDATE `devices` SET `name` = ? WHERE `flags` = CAST(? AS UNSIGNED)") } + // Every engine is asked once, for the column types its driver writes each value by. @Test - func anEngineThatReadsBitsAsDigitsIsNotAsked() async throws { + func anEngineThatReadsBitsAsDigitsGetsNoCast() async throws { let driver = BitTableDriver() try await sink(on: driver, type: .postgresql).insertRows([["id": .text("1"), "flags": .text("101")]]) - #expect(driver.columnFetches == 0) + #expect(driver.columnFetches == 1) #expect(driver.statements.first?.sql.contains("CAST") == false) } } diff --git a/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterParameterTypeTests.swift b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterParameterTypeTests.swift new file mode 100644 index 0000000000..8f34cb63b3 --- /dev/null +++ b/apple/TableProTests/Core/Plugins/ImportDataSinkAdapterParameterTypeTests.swift @@ -0,0 +1,119 @@ +// +// ImportDataSinkAdapterParameterTypeTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +private struct SentStatement { + let sql: String + let typeNames: [String?]? +} + +private struct CatalogRefused: Error {} + +private final class TypedTableDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var sent: [SentStatement] = [] + private let refusesCatalog: Bool + + init(refusesCatalog: Bool = false) { + self.refusesCatalog = refusesCatalog + } + + var statements: [SentStatement] { lock.withLock { sent } } + + func quoteIdentifier(_ name: String) -> String { "\"\(name)\"" } + + func fetchColumns(table: String, schema: String?) async throws -> [PluginColumnInfo] { + if refusesCatalog { throw CatalogRefused() } + return [ + PluginColumnInfo(name: "ID", dataType: "NUMBER(10)", isNullable: false, isPrimaryKey: true), + PluginColumnInfo(name: "CODE", dataType: "VARCHAR2(20 BYTE)") + ] + } + + func executeParameterized( + query: String, + parameters: [PluginCellValue], + context: PluginStatementContext + ) async throws -> PluginQueryResult { + lock.withLock { sent.append(SentStatement(sql: query, typeNames: context.parameterTypeNames)) } + guard query.hasPrefix("SELECT") else { return .empty } + return PluginQueryResult( + columns: ["position", "count"], columnTypeNames: [], rows: [[.text("0"), .text("1")]], + rowsAffected: 0, executionTime: 0 + ) + } +} + +// Oracle stored an imported `00123` in a VARCHAR2 as 123: the driver was never told the column was text. +struct ImportDataSinkAdapterParameterTypeTests { + private static let mapping = ["ID": "ID", "CODE": "CODE"] + + private func sink(on driver: TypedTableDriver, rowWriting: ImportRowWriting = .insert) -> ImportDataSinkAdapter { + ImportDataSinkAdapter( + driver: PluginDriverAdapter(connection: TestFixtures.makeConnection(type: .postgresql), pluginDriver: driver), + databaseType: .postgresql, + targetTable: "PEOPLE", + columnMapping: Self.mapping, + sourceFields: Set(Self.mapping.keys), + rowWriting: rowWriting + ) + } + + @Test + func importedRowsTellTheDriverEachValuesColumnType() async throws { + let driver = TypedTableDriver() + + try await sink(on: driver).insertRows([["ID": .text("1"), "CODE": .text("00123")]]) + + let insert = try #require(driver.statements.first) + #expect(insert.sql.hasPrefix("INSERT INTO \"PEOPLE\" (\"CODE\", \"ID\")")) + #expect(insert.typeNames == ["VARCHAR2(20 BYTE)", "NUMBER(10)"]) + } + + @Test + func aSingleRowAndOrderedRowsTellTheDriverTheTypesToo() async throws { + let driver = TypedTableDriver() + let target = sink(on: driver) + + try await target.insertRow(["ID": .text("1"), "CODE": .text("00123")]) + try await target.insertRows(columns: ["ID", "CODE"], values: [[.text("2"), .text("00456")]]) + + let typeNames = driver.statements.map { $0.typeNames } + #expect(typeNames == [ + ["VARCHAR2(20 BYTE)", "NUMBER(10)"], + ["NUMBER(10)", "VARCHAR2(20 BYTE)"] + ]) + } + + @Test + func matchingTellsTheDriverTheKeyAndAssignmentTypes() async throws { + let driver = TypedTableDriver() + let target = sink(on: driver, rowWriting: ImportRowWriting(mode: .update, keyColumns: ["ID"])) + + try await target.insertRows([["ID": .text("7"), "CODE": .text("00123")]]) + + let match = try #require(driver.statements.first) + let update = try #require(driver.statements.last) + #expect(match.sql.hasPrefix("SELECT 0, COUNT(*)")) + #expect(match.typeNames == ["NUMBER(10)"]) + #expect(update.sql.hasPrefix("UPDATE \"PEOPLE\" SET \"CODE\"")) + #expect(update.typeNames == ["VARCHAR2(20 BYTE)", "NUMBER(10)"]) + } + + @Test + func aTargetWhoseColumnsCannotBeReadStillImportsUntyped() async throws { + let driver = TypedTableDriver(refusesCatalog: true) + + try await sink(on: driver).insertRows([["ID": .text("1"), "CODE": .text("00123")]]) + + let insert = try #require(driver.statements.first) + #expect(insert.sql.hasPrefix("INSERT INTO \"PEOPLE\"")) + #expect(insert.typeNames == nil) + } +} diff --git a/apple/TableProTests/Core/Services/BulkReadTimeoutTests.swift b/apple/TableProTests/Core/Services/BulkReadTimeoutTests.swift new file mode 100644 index 0000000000..2109f9ee73 --- /dev/null +++ b/apple/TableProTests/Core/Services/BulkReadTimeoutTests.swift @@ -0,0 +1,40 @@ +// +// BulkReadTimeoutTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +@MainActor +struct BulkReadTimeoutTests { + private struct Failure: Error {} + + @Test("A bulk read runs with the timeout off and puts the connection's own back") + func liftsAndRestores() async throws { + var connection = TestFixtures.makeConnection() + connection.queryTimeoutSeconds = 17 + let source = MockDatabaseDriver(connection: connection) + let target = MockDatabaseDriver() + let configured = AppSettingsManager.shared.general.queryTimeoutSeconds + + let value = try await BulkReadTimeout.lifted(on: [source, target]) { 42 } + + #expect(value == 42) + #expect(source.applyQueryTimeoutValues == [0, 17]) + #expect(target.applyQueryTimeoutValues == [0, configured]) + } + + @Test("A bulk read that fails still puts the timeout back") + func restoresAfterFailure() async { + var connection = TestFixtures.makeConnection() + connection.queryTimeoutSeconds = 9 + let source = MockDatabaseDriver(connection: connection) + + await #expect(throws: Failure.self) { + try await BulkReadTimeout.lifted(on: [source]) { () async throws -> Int in throw Failure() } + } + #expect(source.applyQueryTimeoutValues == [0, 9]) + } +} diff --git a/apple/TableProTests/Core/Services/Execution/AutocommitOnlyStatementTests.swift b/apple/TableProTests/Core/Services/Execution/AutocommitOnlyStatementTests.swift index 0cd0f79c01..2322218c1e 100644 --- a/apple/TableProTests/Core/Services/Execution/AutocommitOnlyStatementTests.swift +++ b/apple/TableProTests/Core/Services/Execution/AutocommitOnlyStatementTests.swift @@ -28,6 +28,8 @@ private enum AutocommitOnlyFixture { return sqlServer case .oracle: return TestGrammar.oracle + case .teradata: + return DatabaseType.teradata.lexicalGrammar case .redis, .other: return TestGrammar.standard } @@ -392,3 +394,68 @@ struct AutocommitOnlyStatementSQLServerTests { #expect(!AutocommitOnlyFixture.matches(statement, .oracle)) } } + +struct AutocommitOnlyStatementTeradataTests { + @Test( + "Teradata takes nothing but the end of a transaction after these", + arguments: [ + "CREATE MULTISET TABLE sales.t (id INTEGER) PRIMARY INDEX (id)", + "create volatile table v (id integer) on commit preserve rows", + "CT sales.t (id INTEGER)", + "ALTER TABLE sales.t ADD note VARCHAR(10)", + "DROP TABLE sales.t", + "RENAME TABLE sales.t TO t2", + "REPLACE VIEW sales.v AS SELECT 1 AS a", + "COMMENT ON TABLE sales.t AS 'orders'", + "COLLECT STATISTICS COLUMN (id) ON sales.t", + "COLLECT STATS ON sales.t COLUMN id", + "DATABASE sales", + "SET SESSION DATABASE sales", + "SET SESSION DATEFORM = ANSIDATE", + "SET QUERY_BAND = 'app=tp;' FOR SESSION", + "SET ROLE analyst", + "SET TIME ZONE LOCAL", + "MODIFY USER u AS PERM = 1e9", + "DELETE DATABASE sales", + "DELETE USER u", + "GRANT SELECT ON sales TO u", + "REVOKE SELECT ON sales FROM u", + "BEGIN QUERY LOGGING ON ALL", + "END QUERY LOGGING ON ALL", + "BEGIN LOGGING ON EACH ALL ON TABLE sales.t", + "FLUSH QUERY LOGGING WITH ALL", + "-- nightly\nDROP TABLE sales.t", + "/* rebuild */ CREATE TABLE t (a INTEGER)", + ] + ) + func teradataRefusals(statement: String) { + #expect(AutocommitOnlyFixture.matches(statement, .teradata)) + } + + @Test( + "Data and transaction statements keep the wrap", + arguments: [ + "SELECT * FROM sales.t", + "SEL * FROM sales.t", + "INSERT INTO sales.t VALUES (1)", + "UPDATE sales.t SET id = 2", + "DELETE FROM sales.t", + "DELETE sales.t ALL", + "MERGE INTO sales.t USING s ON t.id = s.id WHEN MATCHED THEN UPDATE SET id = s.id", + "HELP SESSION", + "SHOW TABLE sales.t", + "BT", + "ET", + "BEGIN TRANSACTION", + "END TRANSACTION", + "COMMIT WORK", + "ROLLBACK", + "CHECKPOINT sales.j", + "LOCKING ROW FOR ACCESS SELECT * FROM sales.t", + "SELECT 'CREATE TABLE t (a INTEGER)'", + ] + ) + func teradataAcceptances(statement: String) { + #expect(!AutocommitOnlyFixture.matches(statement, .teradata)) + } +} diff --git a/apple/TableProTests/Core/Services/Execution/BatchTransactionPolicyTests.swift b/apple/TableProTests/Core/Services/Execution/BatchTransactionPolicyTests.swift index f3e3df23bf..73ad2f68cf 100644 --- a/apple/TableProTests/Core/Services/Execution/BatchTransactionPolicyTests.swift +++ b/apple/TableProTests/Core/Services/Execution/BatchTransactionPolicyTests.swift @@ -238,6 +238,33 @@ struct BatchTransactionPolicyTests { #expect(Self.plan(["INSERT INTO t VALUES (1)", "CHECKPOINT"], .postgresql) == .appTransaction) } + // Inside BT a statement after DDL fails with 3932 and rolls the batch back, so a Teradata batch holding DDL runs + // each statement as its own request. + @Test( + "A Teradata batch holding DDL runs in autocommit", + arguments: [ + ["CREATE MULTISET TABLE t (id INTEGER) PRIMARY INDEX (id)", "INSERT INTO t VALUES (1)"], + ["DATABASE sales", "SELECT * FROM t"], + ["INSERT INTO t VALUES (1)", "COLLECT STATISTICS COLUMN (id) ON t"], + ["SET SESSION DATEFORM = ANSIDATE", "SELECT CURRENT_DATE"], + ["GRANT SELECT ON sales TO u", "UPDATE t SET id = 2"], + ] + ) + func teradataDefinitionBatchIsNotWrapped(statements: [String]) { + #expect(Self.plan(statements, .teradata) == .autocommit) + } + + @Test("A Teradata batch of data statements stays wrapped") + func teradataDataBatchIsWrapped() { + #expect(Self.plan(["INSERT INTO t VALUES (1)", "UPDATE t SET id = 2", "SEL * FROM t"], .teradata) == .appTransaction) + #expect(Self.plan(["DELETE FROM t", "DELETE t ALL"], .teradata) == .appTransaction) + } + + @Test("A Teradata script that begins its own transaction runs as written") + func teradataBeginTransactionIsNotWrapped() { + #expect(Self.plan(["BEGIN TRANSACTION", "INSERT INTO t VALUES (1)", "END TRANSACTION"], .teradata) == .scriptTransaction) + } + @Test("A SQLite savepoint opens a transaction, so the script owns it after all") func savepointAfterAnUnwrappingStatement() { #expect(Self.plan(["VACUUM", "SAVEPOINT a", "INSERT INTO t VALUES (1)"], .sqlite) == .scriptTransaction) diff --git a/apple/TableProTests/Core/Services/Execution/MultiStatementFailureTests.swift b/apple/TableProTests/Core/Services/Execution/MultiStatementFailureTests.swift index 6b31814b93..afa228a19e 100644 --- a/apple/TableProTests/Core/Services/Execution/MultiStatementFailureTests.swift +++ b/apple/TableProTests/Core/Services/Execution/MultiStatementFailureTests.swift @@ -308,6 +308,22 @@ struct MultiStatementFailureTests { #expect(!report.message.contains("stays applied")) } + // A Teradata ANSI session holds an unwrapped DDL statement in a transaction it leaves open. + @Test("An unwrapped statement failure over a transaction left open says so") + func statementFailureReportsOpenTransaction() { + let report = Self.report( + .statement(sql: "INSERT INTO t VALUES (1)"), + executed: 1, + total: 2, + error: "Only a COMMIT WORK or null statement is legal after a DDL Statement.", + plan: .autocommit, + sessionState: .inTransaction, + unit: .statement + ) + #expect(report.message.contains("still open")) + #expect(!report.message.contains("stays applied")) + } + /// Work done inside a transaction the user already had open is pending in it, or went with it when the error /// ended it, and the session cannot say which, so the note claims neither. @Test("A batch failure inside the user's own transaction never says the work stays applied") diff --git a/apple/TableProTests/Core/Services/Execution/TransactionEngineFamilyTests.swift b/apple/TableProTests/Core/Services/Execution/TransactionEngineFamilyTests.swift index b1184108b9..181e681d0b 100644 --- a/apple/TableProTests/Core/Services/Execution/TransactionEngineFamilyTests.swift +++ b/apple/TableProTests/Core/Services/Execution/TransactionEngineFamilyTests.swift @@ -79,6 +79,13 @@ struct TransactionEngineFamilyTests { #expect(TransactionEngineFamily.of(.redis) == .redis) } + @Test("Teradata, keyed by the id its plugin registers, reads its own rules") + func teradataIsItsOwnFamily() { + #expect(TransactionEngineFamily.of(.teradata) == .teradata) + #expect(TransactionEngineFamily.of(DatabaseType(rawValue: "Teradata")) == .teradata) + #expect(TransactionEngineFamily.teradata.wrapsBatchInTransaction) + } + /// Redis is the only engine the app cannot open a transaction on: `MULTI` queues commands until /// `EXEC` and answers `+QUEUED` in place of every reply, and `DISCARD` can only drop a queue /// nothing has applied. diff --git a/apple/changelog.d/4223.fixed.1.md b/apple/changelog.d/4223.fixed.1.md new file mode 100644 index 0000000000..e232389f4f --- /dev/null +++ b/apple/changelog.d/4223.fixed.1.md @@ -0,0 +1 @@ +Saving edits on a Teradata connection in ANSI mode failed and discarded the editor's uncommitted changes. diff --git a/apple/changelog.d/4223.fixed.10.md b/apple/changelog.d/4223.fixed.10.md new file mode 100644 index 0000000000..a7e9ae7a4c --- /dev/null +++ b/apple/changelog.d/4223.fixed.10.md @@ -0,0 +1 @@ +Run All of a Teradata script that creates a table and then uses it failed with error 3932. diff --git a/apple/changelog.d/4223.fixed.2.md b/apple/changelog.d/4223.fixed.2.md new file mode 100644 index 0000000000..9969846c4f --- /dev/null +++ b/apple/changelog.d/4223.fixed.2.md @@ -0,0 +1 @@ +SQL Server table DDL and SQL exports dropped computed columns, identity seeds, checks and composite foreign keys. diff --git a/apple/changelog.d/4223.fixed.3.md b/apple/changelog.d/4223.fixed.3.md new file mode 100644 index 0000000000..d0e8b00414 --- /dev/null +++ b/apple/changelog.d/4223.fixed.3.md @@ -0,0 +1 @@ +Changing a SQL Server column's type in the structure editor reset its collation to the database default. diff --git a/apple/changelog.d/4223.fixed.4.md b/apple/changelog.d/4223.fixed.4.md new file mode 100644 index 0000000000..2e74fe2f31 --- /dev/null +++ b/apple/changelog.d/4223.fixed.4.md @@ -0,0 +1 @@ +Oracle SQL exports wrote no primary keys, so every foreign key in the dump failed on restore. diff --git a/apple/changelog.d/4223.fixed.5.md b/apple/changelog.d/4223.fixed.5.md new file mode 100644 index 0000000000..54783e88e2 --- /dev/null +++ b/apple/changelog.d/4223.fixed.5.md @@ -0,0 +1 @@ +Copying, transferring or comparing a large PostgreSQL, MySQL or SQL Server table held all of it in memory. diff --git a/apple/changelog.d/4223.fixed.6.md b/apple/changelog.d/4223.fixed.6.md new file mode 100644 index 0000000000..6723e860f5 --- /dev/null +++ b/apple/changelog.d/4223.fixed.6.md @@ -0,0 +1 @@ +PostgreSQL DDL of a partitioned table dropped each partition's own indexes, checks and defaults. diff --git a/apple/changelog.d/4223.fixed.7.md b/apple/changelog.d/4223.fixed.7.md new file mode 100644 index 0000000000..cb84f549f0 --- /dev/null +++ b/apple/changelog.d/4223.fixed.7.md @@ -0,0 +1 @@ +Cancelling a DuckDB export inside an open transaction could abort that transaction. diff --git a/apple/changelog.d/4223.fixed.8.md b/apple/changelog.d/4223.fixed.8.md new file mode 100644 index 0000000000..fe3dc4fe16 --- /dev/null +++ b/apple/changelog.d/4223.fixed.8.md @@ -0,0 +1 @@ +SAP HANA exports wrote large objects over 64 MiB cut short without reporting it. diff --git a/apple/changelog.d/4223.fixed.9.md b/apple/changelog.d/4223.fixed.9.md new file mode 100644 index 0000000000..acf6a54d6c --- /dev/null +++ b/apple/changelog.d/4223.fixed.9.md @@ -0,0 +1 @@ +The MySQL structure view could show another table's check constraint. diff --git a/apple/changelog.d/4223.fixed.md b/apple/changelog.d/4223.fixed.md new file mode 100644 index 0000000000..28c3c6b397 --- /dev/null +++ b/apple/changelog.d/4223.fixed.md @@ -0,0 +1 @@ +Importing or copying into Oracle stored text such as 00123 as 123, and national text as question marks.