diff --git a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift index 6aef52ba87..4d706359dd 100644 --- a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift +++ b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRowWrites.swift @@ -38,6 +38,15 @@ public enum BigtableRowWrites { BigtableRequest(verb: .delete, table: table, key: key) } + // The locator is the base64 of the key the row was read with; the _key text is only the fallback. + public static func originalKey(locator: String?, keyCell: BigtableRowSet.Cell?) throws -> Data? { + guard let locator else { return try keyCell.flatMap { try key($0) } } + guard let decoded = Data(base64Encoded: locator), !decoded.isEmpty else { + throw BigtableError.malformedResponse(String(localized: "This row cannot be found again from what was read. Reload the table.")) + } + return decoded + } + public static func key(_ cell: BigtableRowSet.Cell) throws -> Data? { switch cell { case .absent: return nil diff --git a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift index 63240c11ed..17e221ff48 100644 --- a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift +++ b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableBytes.swift @@ -15,12 +15,18 @@ public enum BigtableBytes { /// The bytes as text when `display` would show them as themselves. public static func printableText(_ data: Data) -> String? { - guard let text = String(data: data, encoding: .utf8), isPrintable(text), !text.hasPrefix(base64Prefix) else { + guard let text = exactText(data), isPrintable(text), !text.hasPrefix(base64Prefix) else { return nil } return text } + // String(data:encoding:) drops a leading byte order mark, which names a different row or column. + static func exactText(_ data: Data) -> String? { + let text = String(decoding: data, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + return text.utf8.elementsEqual(data) ? text : nil + } + public static func parse(_ text: String) throws -> Data { guard text.hasPrefix(base64Prefix) else { return Data(text.utf8) } guard let data = Data(base64Encoded: String(text.dropFirst(base64Prefix.count))) else { diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraCellLifetime.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraCellLifetime.swift new file mode 100644 index 0000000000..a742825116 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraCellLifetime.swift @@ -0,0 +1,78 @@ +// +// CassandraCellLifetime.swift +// TableProCassandraCore +// + +import Foundation + +// A write resets the TTL of every cell it writes to its own USING TTL, or to the table's default without one, so an +// edit has to carry what each cell had left when it was read. +public enum CassandraCellLifetime: Hashable, Sendable { + case expiring(seconds: Int) + case permanent + + // Nil for a cell read as null, whose edit is written without a TTL as before. + public init?(valueIsNull: Bool, timeToLive: String?) { + guard !valueIsNull else { return nil } + guard let timeToLive else { + self = .permanent + return + } + guard let seconds = Int(timeToLive), seconds > 0 else { return nil } + self = .expiring(seconds: seconds) + } + + public static func remaining(seconds: Int, after elapsed: Duration) -> Int { + max(1, seconds - Int(max(elapsed.components.seconds, 0))) + } +} + +// The lifetimes of one row's cells as a browse read them, with what the table says about naming the row. +public struct CassandraRowLifetimes: Sendable, Equatable { + public let columns: [String] + public let lifetimes: [CassandraCellLifetime?] + public let readAt: ContinuousClock.Instant + public let layout: CassandraTableLayout + + // Nil when every cell would be written the way a write without USING writes it, so nothing is kept for the row. + public init?( + columns: [String], + lifetimes: [CassandraCellLifetime?], + readAt: ContinuousClock.Instant, + layout: CassandraTableLayout + ) { + guard columns.count == lifetimes.count else { return nil } + let changesAWrite = lifetimes.contains { lifetime in + switch lifetime { + case .expiring: return true + case .permanent: return layout.defaultTimeToLive != 0 + case nil: return false + } + } + guard changesAWrite else { return nil } + self.columns = columns + self.lifetimes = lifetimes + self.readAt = readAt + self.layout = layout + } + + // A cell whose TTL ran out since the read is gone from the server, so an edit of it would live for one second. + public func hasExpired(_ column: String, at now: ContinuousClock.Instant) -> Bool { + guard let index = columns.firstIndex(of: column), case .expiring(let seconds) = lifetimes[index] else { + return false + } + return readAt.duration(to: now) >= .seconds(seconds) + } + + // The USING TTL a write of `column` keeps the cell's lifetime with, or nil for none. A cell that never expired + // is written with 0 unless the table's default TTL is 0, where a write without USING keeps it the same. + public func timeToLive(of column: String, at now: ContinuousClock.Instant) -> Int? { + guard let index = columns.firstIndex(of: column), let lifetime = lifetimes[index] else { return nil } + switch lifetime { + case .expiring(let seconds): + return CassandraCellLifetime.remaining(seconds: seconds, after: readAt.duration(to: now)) + case .permanent: + return layout.defaultTimeToLive == 0 ? nil : 0 + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift index a4772dda72..771217b554 100644 --- a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraConditionalWrite.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProSQLGrammar // A grid write names its row by key and is conditioned on it: an INSERT is an upsert, and an UPDATE writes its cells // whether or not the row is there, so either would otherwise replace or recreate a row the grid never showed. @@ -13,6 +14,9 @@ public enum CassandraConditionalWrite { public static let insertCondition = "IF NOT EXISTS" public static let updateCondition = "IF EXISTS" + private static let grammar = SQLLexicalReadings.resolve(databaseTypeId: "Cassandra", declared: nil, session: nil) + .execution + public enum Refusal: Sendable, Equatable { case keyTaken case rowMissing @@ -23,12 +27,29 @@ public enum CassandraConditionalWrite { + insertCondition } - public static func updateStatement(target: String, assignments: [String], keyCondition: String) -> String { - "UPDATE \(target) SET \(assignments.joined(separator: ", ")) WHERE \(keyCondition) \(updateCondition)" + public static func updateStatement( + target: String, + assignments: [String], + keyCondition: String, + timeToLive: Int? = nil + ) -> String { + "UPDATE \(target)\(usingClause(timeToLive: timeToLive)) SET \(assignments.joined(separator: ", ")) " + + "WHERE \(keyCondition) \(updateCondition)" + } + + public static func usingClause(timeToLive: Int?) -> String { + timeToLive.map { " USING TTL \($0)" } ?? "" + } + + public static func batch(_ statements: [String]) -> String { + "BEGIN BATCH " + statements.map { $0 + "; " }.joined() + "APPLY BATCH" } public static func refusal(statement: String, firstColumn: String?, firstValue: String?) -> Refusal? { guard firstColumn == appliedColumn, firstValue?.lowercased() == "false" else { return nil } + if let statements = CQLBatch.statements(in: statement, grammar: grammar) { + return batchRefusal(statements) + } let normalized = statement.uppercased() if normalized.hasPrefix("INSERT"), normalized.hasSuffix(" " + insertCondition) { return .keyTaken @@ -38,4 +59,25 @@ public enum CassandraConditionalWrite { } return nil } + + // An edit whose cells keep different TTLs is a batch of UPDATEs on the row, each IF EXISTS or, for static + // cells, unconditioned. A batch with any other statement or condition is left to its result. + private static func batchRefusal(_ statements: [String]) -> Refusal? { + var holdsRow = false + for statement in statements { + let words = codeWords(statement) + guard words.first == "UPDATE" else { return nil } + guard let condition = words.firstIndex(of: "IF") else { continue } + guard words[condition...].elementsEqual(["IF", "EXISTS"]) else { return nil } + holdsRow = true + } + return holdsRow ? .rowMissing : nil + } + + private static func codeWords(_ statement: String) -> [String] { + SQLCodeProjection.code(of: statement, grammar: grammar) + .uppercased() + .split { !($0.isLetter || $0.isNumber || $0 == "_") } + .map(String.init) + } } diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraLifetimeRead.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraLifetimeRead.swift new file mode 100644 index 0000000000..d7d9646b92 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraLifetimeRead.swift @@ -0,0 +1,95 @@ +// +// CassandraLifetimeRead.swift +// TableProCassandraCore +// + +import Foundation + +// The browse the plugin sends with a TTL() selector after the shown columns for each one that can carry its own +// TTL. `SELECT *` takes no other selector, so it is spelled out in the order the server returns it. +public enum CassandraLifetimeRead { + public struct Plan: Equatable, Sendable { + public let cql: String + public let shownColumns: [String] + // The column of each TTL() selector, in the order the selectors follow the shown columns. + public let lifetimeColumns: [String] + public let lifetimeValueIndices: [Int] + public let keyIndices: [String: Int] + } + + // Nil unless `cql` is a browse of `target` as the renderer writes it, every key column is shown, and at least one + // shown column takes a TTL() selector. + public static func plan(cql: String, target: String, layout: CassandraTableLayout) -> Plan? { + guard let select = select(of: cql, target: target) else { return nil } + let shown = select.columns ?? layout.selectOrder + var lifetimeColumns: [String] = [] + var lifetimeValueIndices: [Int] = [] + for (index, column) in shown.enumerated() + where layout.acceptsTimeToLive(column) && !lifetimeColumns.contains(column) { + lifetimeColumns.append(column) + lifetimeValueIndices.append(index) + } + guard !lifetimeColumns.isEmpty else { return nil } + var keyIndices: [String: Int] = [:] + for key in layout.primaryKey { + guard let index = shown.firstIndex(of: key) else { return nil } + keyIndices[key] = index + } + let selectors = shown.map(CassandraObjectQueries.quote) + + lifetimeColumns.map { "TTL(\(CassandraObjectQueries.quote($0)))" } + return Plan( + cql: "SELECT \(selectors.joined(separator: ", ")) FROM \(target)\(select.tail)", + shownColumns: shown, + lifetimeColumns: lifetimeColumns, + lifetimeValueIndices: lifetimeValueIndices, + keyIndices: keyIndices + ) + } + + private struct Select { + // Nil for `*`. + let columns: [String]? + let tail: String + } + + private static func select(of cql: String, target: String) -> Select? { + var rest = Substring(cql).unicodeScalars + guard consume("SELECT ", from: &rest) else { return nil } + var columns: [String]? + if !consume("* ", from: &rest) { + var names: [String] = [] + repeat { + guard let name = quotedName(from: &rest) else { return nil } + names.append(name) + } while consume(", ", from: &rest) + guard consume(" ", from: &rest) else { return nil } + columns = names + } + guard consume("FROM " + target, from: &rest) else { return nil } + let tail = String(Substring(rest)) + guard tail.isEmpty || tail.hasPrefix(" WHERE ") else { return nil } + return Select(columns: columns, tail: tail) + } + + private static func consume(_ text: String, from rest: inout Substring.UnicodeScalarView) -> Bool { + guard rest.starts(with: text.unicodeScalars) else { return false } + rest = rest.dropFirst(text.unicodeScalars.count) + return true + } + + // A double-quoted identifier, `""` standing for one quote, as SQLQuoting.identifier writes it. + private static func quotedName(from rest: inout Substring.UnicodeScalarView) -> String? { + guard consume("\"", from: &rest) else { return nil } + var name = String.UnicodeScalarView() + while let scalar = rest.first { + rest = rest.dropFirst() + guard scalar == "\"" else { + name.append(scalar) + continue + } + guard consume("\"", from: &rest) else { return String(name) } + name.append(scalar) + } + return nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraLifetimeWrite.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraLifetimeWrite.swift new file mode 100644 index 0000000000..f2b3319274 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraLifetimeWrite.swift @@ -0,0 +1,181 @@ +// +// CassandraLifetimeWrite.swift +// TableProCassandraCore +// + +import Foundation + +// A CQL write carries one TTL for all the cells it writes, so cells that keep different TTLs go in statements of +// their own, joined in one batch. The batch names one partition, so Cassandra and ScyllaDB apply it atomically. +// A cell with no TTL to keep is written without USING, as before. +public enum CassandraLifetimeWrite { + public struct KeyCondition { + public let cql: String + public let values: [Value] + + public init(cql: String, values: [Value]) { + self.cql = cql + self.values = values + } + } + + public struct RowKeys { + public let row: KeyCondition + public let partition: KeyCondition + + public init(row: KeyCondition, partition: KeyCondition) { + self.row = row + self.partition = partition + } + } + + public struct Assignment { + public let cql: String + public let parameters: [Value] + public let timeToLive: Int? + public let isStatic: Bool + + public init(cql: String, parameters: [Value], timeToLive: Int?, isStatic: Bool) { + self.cql = cql + self.parameters = parameters + self.timeToLive = timeToLive + self.isStatic = isStatic + } + } + + public struct RestoredCell { + public let column: String + public let value: Value + public let isKey: Bool + public let isNull: Bool + public let isStatic: Bool + public let timeToLive: Int? + + public init(column: String, value: Value, isKey: Bool, isNull: Bool, isStatic: Bool, timeToLive: Int?) { + self.column = column + self.value = value + self.isKey = isKey + self.isNull = isNull + self.isStatic = isStatic + self.timeToLive = timeToLive + } + } + + public struct Statement { + public let cql: String + public let parameters: [Value] + } + + // Every statement of the batch is held to the row with IF EXISTS, except one that writes only static cells + // beside others that do not: CQL refuses clustering columns in an UPDATE of static columns alone, so that one + // names the partition and the batch's other conditions hold it. + public static func update( + target: String, + assignments: [Assignment], + row: KeyCondition, + partition: KeyCondition? + ) -> Statement { + let writesClusteringRow = assignments.contains { !$0.isStatic } + let statements = grouped(assignments, by: \.timeToLive).map { group -> Statement in + let set = group.items.map(\.cql) + let parameters = group.items.flatMap(\.parameters) + if let partition, writesClusteringRow, group.items.allSatisfy(\.isStatic) { + return Statement( + cql: plainUpdate(target: target, timeToLive: group.timeToLive, set: set, key: partition.cql), + parameters: parameters + partition.values + ) + } + return Statement( + cql: CassandraConditionalWrite.updateStatement( + target: target, assignments: set, keyCondition: row.cql, timeToLive: group.timeToLive + ), + parameters: parameters + row.values + ) + } + return batched(statements) + } + + // The INSERT carries the key and also writes the row marker, so it takes the cells that live longest: those + // that never expire, then those left to the table's default, then the longest TTL. The rest are UPDATEs. + public static func restore( + target: String, + cells: [RestoredCell], + keys: RowKeys? + ) -> Statement { + let valued = cells.filter { !$0.isKey && !$0.isNull } + guard let keys, valued.contains(where: { $0.timeToLive != nil }) else { + return insert(target: target, cells: cells, timeToLive: nil) + } + let groups = grouped(valued, by: \.timeToLive) + let inserted = longestLiving(groups.map { $0.timeToLive }) + let insertion = insert( + target: target, + cells: cells.filter { $0.isKey || $0.isNull || $0.timeToLive == inserted }, + timeToLive: inserted + ) + let updates = groups.filter { $0.timeToLive != inserted }.map { group -> Statement in + let key = group.items.allSatisfy(\.isStatic) ? keys.partition : keys.row + let set = group.items.map { "\(CassandraObjectQueries.quote($0.column)) = ?" } + return Statement( + cql: plainUpdate(target: target, timeToLive: group.timeToLive, set: set, key: key.cql), + parameters: group.items.map(\.value) + key.values + ) + } + return batched([insertion] + updates) + } + + static func longestLiving(_ timesToLive: [Int?]) -> Int? { + if timesToLive.contains(0) { return 0 } + if timesToLive.contains(where: { $0 == nil }) { return nil } + return timesToLive.compactMap { $0 }.max() + } + + private static func grouped( + _ items: [Item], + by timeToLive: (Item) -> Int? + ) -> [(timeToLive: Int?, items: [Item])] { + var groups: [(timeToLive: Int?, items: [Item])] = [] + for item in items { + let key = timeToLive(item) + if let index = groups.firstIndex(where: { $0.timeToLive == key }) { + groups[index].items.append(item) + } else { + groups.append((key, [item])) + } + } + return groups + } + + private static func insert( + target: String, + cells: [RestoredCell], + timeToLive: Int? + ) -> Statement { + let names = cells.map { CassandraObjectQueries.quote($0.column) }.joined(separator: ", ") + let markers = Array(repeating: "?", count: cells.count).joined(separator: ", ") + return Statement( + cql: "INSERT INTO \(target) (\(names)) VALUES (\(markers))" + + CassandraConditionalWrite.usingClause(timeToLive: timeToLive), + parameters: cells.map(\.value) + ) + } + + private static func plainUpdate(target: String, timeToLive: Int?, set: [String], key: String) -> String { + "UPDATE \(target)\(CassandraConditionalWrite.usingClause(timeToLive: timeToLive)) SET " + + "\(set.joined(separator: ", ")) WHERE \(key)" + } + + private static func batched(_ statements: [Statement]) -> Statement { + if statements.count == 1, let only = statements.first { return only } + return Statement( + cql: CassandraConditionalWrite.batch(statements.map(\.cql)), + parameters: statements.flatMap(\.parameters) + ) + } +} + +extension CassandraLifetimeWrite.KeyCondition: Sendable where Value: Sendable {} +extension CassandraLifetimeWrite.RowKeys: Sendable where Value: Sendable {} +extension CassandraLifetimeWrite.Assignment: Sendable where Value: Sendable {} +extension CassandraLifetimeWrite.RestoredCell: Sendable where Value: Sendable {} +extension CassandraLifetimeWrite.Statement: Sendable where Value: Sendable {} diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraRecentEntries.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraRecentEntries.swift new file mode 100644 index 0000000000..dfdbb46820 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraRecentEntries.swift @@ -0,0 +1,47 @@ +// +// CassandraRecentEntries.swift +// TableProCassandraCore +// + +import Foundation + +// At most `capacity` entries in two halves. When the newer half is full the older half is dropped whole, so the +// entries written longest ago go first and no write costs more than a dictionary insert. +public struct CassandraRecentEntries { + public let capacity: Int + private var newer: [Key: Value] = [:] + private var older: [Key: Value] = [:] + + public init(capacity: Int) { + self.capacity = max(capacity, 2) + } + + public subscript(key: Key) -> Value? { + newer[key] ?? older[key] + } + + public var count: Int { + newer.count + older.count + } + + public mutating func set(_ value: Value, for key: Key) { + older.removeValue(forKey: key) + if newer[key] == nil, newer.count >= capacity / 2 { + older = newer + newer = [:] + } + newer[key] = value + } + + public mutating func removeValue(for key: Key) { + newer.removeValue(forKey: key) + older.removeValue(forKey: key) + } + + public mutating func removeAll() { + newer = [:] + older = [:] + } +} + +extension CassandraRecentEntries: Sendable where Key: Sendable, Value: Sendable {} diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraTableLayout.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraTableLayout.swift new file mode 100644 index 0000000000..59bf99ab45 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraTableLayout.swift @@ -0,0 +1,107 @@ +// +// CassandraTableLayout.swift +// TableProCassandraCore +// + +import Foundation + +// What system_schema says about a table's columns: which ones a browse can read a TTL() for, and how an edit names +// the row or, for static columns, the partition. +public struct CassandraTableLayout: Sendable, Equatable { + public enum Kind: String, Sendable { + case partitionKey = "partition_key" + case clustering + case staticColumn = "static" + case regular + } + + public struct Column: Sendable, Equatable { + public let name: String + public let type: String + public let kind: Kind + public let position: Int + + public init(name: String, type: String, kind: Kind, position: Int) { + self.name = name + self.type = type + self.kind = kind + self.position = position + } + } + + public let partitionKey: [String] + public let clustering: [String] + public let staticColumns: Set + public let selectOrder: [String] + // Nil when the table's options could not be read. + public let defaultTimeToLive: Int? + private let lifetimeColumns: Set + + // CQL3Type.Native without counter and empty: each holds one cell. + private static let singleCellTypes: Set = [ + "ascii", "bigint", "blob", "boolean", "date", "decimal", "double", "duration", "float", "inet", "int", + "smallint", "text", "time", "timestamp", "timeuuid", "tinyint", "uuid", "varchar", "varint", + ] + private static let singleCellPrefixes = ["frozen<", "tuple<", "vector<"] + + // Nil for a table system_schema lists no columns for, such as a virtual table, and for a COMPACT STORAGE table, + // whose `empty` value column `SELECT *` leaves out. + public init?(columns: [Column], defaultTimeToLive: Int?) { + guard !columns.isEmpty, !columns.contains(where: { Self.normalized($0.type) == "empty" }) else { return nil } + let byPosition = { (kind: Kind) in + columns.filter { $0.kind == kind }.sorted { $0.position < $1.position }.map(\.name) + } + let byName = { (kind: Kind) in + columns.filter { $0.kind == kind }.map(\.name).sorted { $0.utf8.lexicographicallyPrecedes($1.utf8) } + } + partitionKey = byPosition(.partitionKey) + clustering = byPosition(.clustering) + let statics = byName(.staticColumn) + staticColumns = Set(statics) + // Cassandra's allColumnsInSelectOrder: the key in order, then static and regular columns by name. + selectOrder = partitionKey + clustering + statics + byName(.regular) + self.defaultTimeToLive = defaultTimeToLive + lifetimeColumns = Set(columns.filter { column in + (column.kind == .regular || column.kind == .staticColumn) && Self.holdsOneCell(column.type) + }.map(\.name)) + } + + public var primaryKey: [String] { partitionKey + clustering } + + // TTL() refuses a key column, and before Cassandra 5.0 a non-frozen collection or user type. A counter has no + // TTL, and an UPDATE of one refuses USING TTL. + public func acceptsTimeToLive(_ column: String) -> Bool { + lifetimeColumns.contains(column) + } + + public static func columnsQuery(keyspace: String, table: String) -> String { + """ + SELECT column_name, type, kind, position FROM system_schema.columns + WHERE keyspace_name = '\(CassandraObjectQueries.escapeLiteral(keyspace))' + AND table_name = '\(CassandraObjectQueries.escapeLiteral(table))' + """ + } + + public static func defaultTimeToLiveQuery(keyspace: String, table: String) -> String { + """ + SELECT default_time_to_live FROM system_schema.tables + WHERE keyspace_name = '\(CassandraObjectQueries.escapeLiteral(keyspace))' + AND table_name = '\(CassandraObjectQueries.escapeLiteral(table))' + """ + } + + // One row of `columnsQuery`, nil when a part is missing or the kind is not one CQL has. + public static func column(name: String?, type: String?, kind: String?, position: String?) -> Column? { + guard let name, let type, let kind = kind.flatMap(Kind.init(rawValue:)) else { return nil } + return Column(name: name, type: type, kind: kind, position: position.flatMap { Int($0) } ?? 0) + } + + private static func holdsOneCell(_ type: String) -> Bool { + let type = normalized(type) + return singleCellTypes.contains(type) || singleCellPrefixes.contains { type.hasPrefix($0) } + } + + private static func normalized(_ type: String) -> String { + type.trimmingCharacters(in: .whitespaces).lowercased() + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraWireText.swift b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraWireText.swift new file mode 100644 index 0000000000..f38c31f1f2 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProCassandraCore/CassandraWireText.swift @@ -0,0 +1,19 @@ +// +// CassandraWireText.swift +// TableProCassandraCore +// + +import Foundation + +public enum CassandraWireText { + // Foundation's UTF-8 initializers drop a leading byte order mark, and a key that starts with one names another row. + public static func text(_ bytes: UnsafeRawBufferPointer) -> String? { + let text = String(decoding: bytes, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + return text.utf8.elementsEqual(bytes) ? text : nil + } + + public static func text(_ bytes: Data) -> String? { + let text = String(decoding: bytes, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + return text.utf8.elementsEqual(bytes) ? text : nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaOffsetMath.swift b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaOffsetMath.swift index cb7d49a806..91ea59b7e3 100644 --- a/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaOffsetMath.swift +++ b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaOffsetMath.swift @@ -23,4 +23,69 @@ public enum KafkaOffsetMath { let (start, overflows) = end.subtractingReportingOverflow(step) return overflows ? floor : max(floor, start) } + + // A v2 batch header keeps the last offset it covers after compaction empties it or its records are aborted. + public static func offset(afterBatchAt baseOffset: Int64, lastOffsetDelta: Int32) -> Int64? { + let (last, overflows) = baseOffset.addingReportingOverflow(Int64(max(0, lastOffsetDelta))) + return overflows ? nil : offset(after: last) + } + + // As far back as the last range's density puts the missing records, a quarter more for an uneven spread, so a + // sparse stretch is crossed in a few reads and a dense one is not read past the page. At most `maximumGrowth` + // times the last range, which is also the step after a range that held nothing. + public static func nextWidth( + lastWidth: Int64, + lastRecords: Int, + needed: Int, + maximumGrowth: Int64 + ) -> Int64 { + let ceiling = max(1, lastWidth).multipliedReportingOverflow(by: max(2, maximumGrowth)) + let cap = ceiling.overflow ? Int64.max : ceiling.partialValue + guard lastRecords > 0 else { return cap } + let estimate = Double(max(0, needed)) * Double(max(1, lastWidth)) / Double(lastRecords) * 1.25 + guard estimate < Double(cap) else { return cap } + return max(1, Int64(estimate.rounded(.up))) + } +} + +// Offsets are not one per record: transaction markers, aborted records and compaction leave offsets with no +// record, so a tail window is widened by the records it holds, not by the offsets it spans. +public struct KafkaTailWalk { + public private(set) var records: [Record] = [] + // Nil once the walk holds `wanted` records or has read down to the floor. + public private(set) var nextRange: (start: Int64, end: Int64)? + public private(set) var start: Int64 + private let wanted: Int + private let floor: Int64 + private let maximumGrowth: Int64 + + public init(wanted: Int, end: Int64, firstStart: Int64, floor: Int64, maximumGrowth: Int64 = 4) { + self.wanted = wanted + self.floor = floor + self.maximumGrowth = maximumGrowth + start = firstStart + nextRange = wanted > 0 ? (firstStart, end) : nil + } + + // How many more records the walk takes; a read of `nextRange` keeps only its newest this many. + public var needed: Int { max(0, wanted - records.count) } + + // `earlier` holds the newest records of `nextRange` in offset order, all older than the ones already held. + public mutating func absorb(_ earlier: [Record]) { + guard let range = nextRange else { return } + start = range.start + records = Array((earlier + records).suffix(wanted)) + guard records.count < wanted, range.start > floor else { + nextRange = nil + return + } + let (span, overflows) = range.end.subtractingReportingOverflow(range.start) + let width = KafkaOffsetMath.nextWidth( + lastWidth: overflows ? Int64.max : span, + lastRecords: earlier.count, + needed: needed, + maximumGrowth: maximumGrowth + ) + nextRange = (KafkaOffsetMath.stepBack(from: range.start, by: width, floor: floor), range.start) + } } diff --git a/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaRecordBatch.swift b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaRecordBatch.swift index bb11e7e97d..8ba659cab5 100644 --- a/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaRecordBatch.swift +++ b/apple/Packages/TableProCore/Sources/TableProKafkaCore/KafkaRecordBatch.swift @@ -73,17 +73,20 @@ public enum KafkaRecordBatchDecoder { // The blob is a sequence of batches cut at partition_max_bytes, so a trailing partial batch // is normal. Control batches hold transaction markers, never rows for the grid. // Every batch charges `budget`, so the records kept across the whole blob stay within it. + // `nextOffset` follows the last whole batch of any kind, nil when none was whole: a reply of only + // markers, aborted or compacted batches still moves the read on. public static func decode( blob: Data, partition: Int32, abortedTransactions: [KafkaAbortedTransaction] = [], skipAborted: Bool, budget: inout DecompressionBudget - ) throws -> (records: [KafkaRecord], truncated: Bool) { + ) throws -> (records: [KafkaRecord], truncated: Bool, nextOffset: Int64?) { let bytes = [UInt8](blob) var cursor = 0 var records: [KafkaRecord] = [] var truncated = false + var nextOffset: Int64? // A partition can carry several producers at once, so an aborted run belongs to its // producer. Keying this on the offset alone let one producer's commit marker end // another's aborted run, and its rolled-back records then reached the grid. @@ -120,6 +123,7 @@ public enum KafkaRecordBatchDecoder { ) } let attributes = Self.int16(bytes, at: cursor + 21) + let lastOffsetDelta = Self.int32(bytes, at: cursor + 23) let baseTimestamp = Self.int64(bytes, at: cursor + 27) let producerId = Self.int64(bytes, at: cursor + 43) let count = Int(Self.int32(bytes, at: cursor + 57)) @@ -140,6 +144,7 @@ public enum KafkaRecordBatchDecoder { // A control batch is its producer's commit or abort marker, and ends only that // producer's run. abortedProducers.remove(producerId) + nextOffset = KafkaOffsetMath.offset(afterBatchAt: baseOffset, lastOffsetDelta: lastOffsetDelta) cursor = end continue } @@ -157,11 +162,12 @@ public enum KafkaRecordBatchDecoder { partition: partition )) } + nextOffset = KafkaOffsetMath.offset(afterBatchAt: baseOffset, lastOffsetDelta: lastOffsetDelta) cursor = end } if cursor < bytes.count { truncated = true } - return (records, truncated) + return (records, truncated, nextOffset) } /// The records inside a batch use ZIG-ZAG signed varints, unlike the unsigned LEB128 the diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedItemEdit.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedItemEdit.swift new file mode 100644 index 0000000000..54b5bba162 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedItemEdit.swift @@ -0,0 +1,34 @@ +import Foundation + +// How an edit of an item the grid read is written. A write that keeps the loaded Value or Flags is a `cas` on the +// unique `gets` returned, so a change made since the read fails instead of being undone. +public enum MemcachedItemEdit: Equatable, Sendable { + case touch + case set + case compareAndSwap(unique: UInt64) + case needsRefresh + + // Nil when nothing the item stores was edited. + public static func write(editsValue: Bool, editsFlags: Bool, editsTTL: Bool, unique: UInt64?) -> MemcachedItemEdit? { + guard editsValue || editsFlags else { return editsTTL ? .touch : nil } + if editsValue, editsFlags { return .set } + guard let unique else { return .needsRefresh } + return .compareAndSwap(unique: unique) + } +} + +// protocol.txt, Storage commands: a `cas` answers `EXISTS` when the item was modified since it was fetched and +// `NOT_FOUND` when it does not exist. Either way nothing was stored. +public enum MemcachedCASRefusal: Equatable, Sendable { + case changed + case notFound + + public init?(replyTo command: MemcachedCommand, status: String) { + guard case .compareAndSwap = command else { return nil } + switch status { + case "EXISTS": self = .changed + case "NOT_FOUND": self = .notFound + default: return nil + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift index dd91bb1722..5aee29a27b 100644 --- a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift @@ -25,6 +25,14 @@ public enum MemcachedStatements { "set \(MemcachedCommandParser.argument(key)) \(flags) \(exptime) \(MemcachedCommandParser.quoted(bytes: bytes))" } + public static func cas(key: String, flags: UInt32, exptime: Int64, unique: UInt64, value: String) -> String { + "cas \(MemcachedCommandParser.argument(key)) \(flags) \(exptime) \(unique) \(MemcachedCommandParser.quoted(value))" + } + + public static func cas(key: String, flags: UInt32, exptime: Int64, unique: UInt64, bytes: Data) -> String { + "cas \(MemcachedCommandParser.argument(key)) \(flags) \(exptime) \(unique) \(MemcachedCommandParser.quoted(bytes: bytes))" + } + public static func touch(key: String, exptime: Int64) -> String { "touch \(MemcachedCommandParser.argument(key)) \(exptime)" } diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisArgumentCodec.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisArgumentCodec.swift index d849a7244e..5c6e02855c 100644 --- a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisArgumentCodec.swift +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisArgumentCodec.swift @@ -47,7 +47,7 @@ public enum RedisArgumentCodec { } public static func quote(_ bytes: Data) -> String { - if let text = String(data: bytes, encoding: .utf8) { + if let text = RedisWireText.decode(bytes) { return isBare(text) ? text : quotedText(text) } return quotedBytes(bytes) diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisKeySummary.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisKeySummary.swift index 32ac2c4114..1eb803d10b 100644 --- a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisKeySummary.swift +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisKeySummary.swift @@ -34,10 +34,11 @@ nonisolated public enum RedisKeySummary { } } - public static func previewCommand(for kind: RedisKeyKind, key: String) -> [String] { + public static func previewCommand(for kind: RedisKeyKind, key: String, stringByteLimit: Int? = nil) -> [String] { switch kind { case .string: - return ["GET", key] + guard let stringByteLimit, stringByteLimit > 0 else { return ["GET", key] } + return ["GETRANGE", key, "0", String(stringByteLimit - 1)] case .hash: return ["HSCAN", key, "0", "COUNT", String(collectionPreviewLimit)] case .list: @@ -99,6 +100,6 @@ nonisolated public enum RedisKeySummary { ) else { return nil } - return String(data: data, encoding: .utf8) + return RedisWireText.decode(data) } } diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisStringPreview.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisStringPreview.swift new file mode 100644 index 0000000000..c3398560a4 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisStringPreview.swift @@ -0,0 +1,46 @@ +import Foundation + +nonisolated public enum RedisStringPreview { + // Holds a grid page to its row count times 64 KiB, where reading each string whole had no bound. + public static let byteLimit = 65_536 + + // The mark JSONTruncation puts after a cut value. + public static let marker = "..." + + public enum Shown: Equatable, Sendable { + case text(String) + case bytes(Data) + } + + // STRLEN declined leaves only the read itself, and a read that filled the range may have been cut. + public static func isShortened(length: Int?, shownByteCount: Int, byteLimit: Int = RedisStringPreview.byteLimit) -> Bool { + guard let length else { return shownByteCount >= byteLimit } + return length > byteLimit + } + + public static func shown(_ bytes: Data, length: Int?, byteLimit: Int = RedisStringPreview.byteLimit) -> Shown { + guard isShortened(length: length, shownByteCount: bytes.count, byteLimit: byteLimit) else { + return RedisWireText.decode(bytes).map(Shown.text) ?? .bytes(bytes) + } + guard let text = RedisWireText.decode(wholeSequencePrefix(bytes)) else { return .bytes(bytes) } + return .text(text + marker) + } + + // Drops only a multi-byte sequence the range cut through, so a binary value never passes as text. + static func wholeSequencePrefix(_ bytes: Data) -> Data { + let tail = Array(bytes.suffix(3)) + for offset in stride(from: 1, through: tail.count, by: 1) { + let byte = tail[tail.count - offset] + if byte & 0xC0 == 0x80 { continue } + let sequenceLength: Int + switch byte { + case 0xC0...0xDF: sequenceLength = 2 + case 0xE0...0xEF: sequenceLength = 3 + case 0xF0...0xF7: sequenceLength = 4 + default: return bytes + } + return offset < sequenceLength ? Data(bytes.prefix(bytes.count - offset)) : bytes + } + return bytes + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisWireText.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisWireText.swift new file mode 100644 index 0000000000..3e00789e3a --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisWireText.swift @@ -0,0 +1,9 @@ +import Foundation + +nonisolated public enum RedisWireText { + // String(data:encoding:) drops a leading byte order mark, which names a different key. + public static func decode(_ bytes: Data) -> String? { + let text = String(decoding: bytes, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + return text.utf8.elementsEqual(bytes) ? text : nil + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift b/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift index 7442a5c272..57d0d8e57f 100644 --- a/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableRowTests.swift @@ -98,6 +98,27 @@ struct BigtableBytesTests { #expect(throws: BigtableError.self) { try BigtableBytes.parse("b64:***") } } + @Test("Text that starts with a byte order mark shows with it, and reads back to the same bytes") + func byteOrderMarkKept() throws { + let marked = Data([0xEF, 0xBB, 0xBF] + Array("abc".utf8)) + let text = try #require(BigtableBytes.printableText(marked)) + #expect(Data(text.utf8) == marked) + #expect(try BigtableBytes.parse(BigtableBytes.display(marked)) == marked) + #expect(BigtableBytes.display(marked) != BigtableBytes.display(Data("abc".utf8))) + #expect(BigtableBytes.printableText(Data([0x61, 0xFF])) == nil) + } + + @Test("A key and a qualifier that start with a byte order mark read back from a request's own text") + func byteOrderMarkRequestRoundTrip() throws { + let marked = Data([0xEF, 0xBB, 0xBF] + Array("abc".utf8)) + let column = BigtableColumn(family: "cf", qualifier: marked) + let request = BigtableRequest(verb: .delete, table: "events", key: marked, column: column.name) + let parsed = try BigtableRequest.parse(request.text) + #expect(parsed.key == marked) + let columnName = try #require(parsed.column) + #expect(try BigtableColumn(name: columnName).qualifier == marked) + } + @Test("Prefix end increments the last byte that can grow") func prefixEnd() { #expect(BigtableBytes.prefixEnd(Data("ab".utf8)) == Data("ac".utf8)) diff --git a/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift b/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift index e9bf7ce67b..2a89954929 100644 --- a/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProBigtableCoreTests/BigtableStatementTests.swift @@ -149,4 +149,31 @@ struct BigtableRowWritesTests { func delete() { #expect(BigtableRowWrites.delete(table: "events", key: Data("a b".utf8)).text == #"DELETE events/"a b""#) } + + @Test("A row's locator names its key over the _key text") + func locatorNamesTheKey() throws { + let real = Data([0xEF, 0xBB, 0xBF] + Array("abc".utf8)) + let key = try #require(try BigtableRowWrites.originalKey(locator: real.base64EncodedString(), keyCell: .text("abc"))) + #expect(key == real) + let written = try BigtableRequest.parse(BigtableRowWrites.delete(table: "events", key: key).text) + #expect(written.key == real) + } + + @Test("Without a locator the key comes from the _key text") + func keyTextWithoutLocator() throws { + #expect(try BigtableRowWrites.originalKey(locator: nil, keyCell: .text("b64:/wA=")) == Data([0xFF, 0x00])) + #expect(try BigtableRowWrites.originalKey(locator: nil, keyCell: .text("k")) == Data("k".utf8)) + #expect(try BigtableRowWrites.originalKey(locator: nil, keyCell: nil) == nil) + #expect(try BigtableRowWrites.originalKey(locator: nil, keyCell: .absent) == nil) + } + + @Test("A locator that is not base64 is refused rather than read as some other key") + func damagedLocatorRefused() { + #expect(throws: BigtableError.self) { + try BigtableRowWrites.originalKey(locator: "***", keyCell: .text("k")) + } + #expect(throws: BigtableError.self) { + try BigtableRowWrites.originalKey(locator: "", keyCell: .text("k")) + } + } } diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraCellLifetimeTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraCellLifetimeTests.swift new file mode 100644 index 0000000000..430c0cb539 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraCellLifetimeTests.swift @@ -0,0 +1,81 @@ +// +// CassandraCellLifetimeTests.swift +// TableProCassandraCoreTests +// + +import TableProCassandraCore +import Testing + +struct CassandraCellLifetimeTests { + @Test("A cell read with its TTL() is expiring, without one permanent, and as null unknown") + func readLifetimes() { + #expect(CassandraCellLifetime(valueIsNull: false, timeToLive: "86400") == .expiring(seconds: 86_400)) + #expect(CassandraCellLifetime(valueIsNull: false, timeToLive: nil) == .permanent) + #expect(CassandraCellLifetime(valueIsNull: true, timeToLive: nil) == nil) + #expect(CassandraCellLifetime(valueIsNull: true, timeToLive: "30") == nil) + #expect(CassandraCellLifetime(valueIsNull: false, timeToLive: "soon") == nil) + } + + @Test("What is left is the TTL less the whole seconds since the read, and at least one", arguments: [ + (100, Duration.seconds(10), 90), + (100, Duration.milliseconds(10_900), 90), + (100, Duration.zero, 100), + (5, Duration.seconds(30), 1), + (5, Duration.seconds(5), 1), + (100, Duration.seconds(-3), 100), + ]) + func remaining(seconds: Int, elapsed: Duration, expected: Int) { + #expect(CassandraCellLifetime.remaining(seconds: seconds, after: elapsed) == expected) + } + + @Test("An edit keeps what an expiring cell had left, and a permanent one only when the table has a default TTL") + func rowTimesToLive() throws { + let readAt = ContinuousClock.now + let later = readAt.advanced(by: .seconds(30)) + let lifetimes: [CassandraCellLifetime?] = [.expiring(seconds: 100), .permanent, nil] + let columns = ["note", "address", "owner"] + + func row(defaultTimeToLive: Int?) throws -> CassandraRowLifetimes { + let layout = try #require(CassandraTableLayoutTests.visits(defaultTimeToLive: defaultTimeToLive)) + return try #require(CassandraRowLifetimes( + columns: columns, lifetimes: lifetimes, readAt: readAt, layout: layout + )) + } + + let noDefault = try row(defaultTimeToLive: 0) + #expect(noDefault.timeToLive(of: "note", at: later) == 70) + #expect(noDefault.timeToLive(of: "address", at: later) == nil) + #expect(noDefault.timeToLive(of: "owner", at: later) == nil) + #expect(noDefault.timeToLive(of: "Count", at: later) == nil) + #expect(try row(defaultTimeToLive: 3_600).timeToLive(of: "address", at: later) == 0) + #expect(try row(defaultTimeToLive: nil).timeToLive(of: "address", at: later) == 0) + } + + @Test("A cell whose TTL ran out since the read has expired, and no other cell has") + func expiredCells() throws { + let readAt = ContinuousClock.now + let layout = try #require(CassandraTableLayoutTests.visits(defaultTimeToLive: 0)) + let row = try #require(CassandraRowLifetimes( + columns: ["note", "address"], lifetimes: [.expiring(seconds: 100), .permanent], readAt: readAt, layout: layout + )) + #expect(!row.hasExpired("note", at: readAt.advanced(by: .seconds(99)))) + #expect(row.hasExpired("note", at: readAt.advanced(by: .seconds(100)))) + #expect(!row.hasExpired("address", at: readAt.advanced(by: .seconds(1_000)))) + #expect(!row.hasExpired("owner", at: readAt.advanced(by: .seconds(1_000)))) + } + + @Test("Nothing is kept for a row every cell of which a plain write keeps the same") + func rowsWithNothingToKeep() throws { + let noDefault = try #require(CassandraTableLayoutTests.visits(defaultTimeToLive: 0)) + #expect(CassandraRowLifetimes( + columns: ["note", "owner"], lifetimes: [.permanent, nil], readAt: .now, layout: noDefault + ) == nil) + #expect(CassandraRowLifetimes( + columns: ["note"], lifetimes: [.permanent, .permanent], readAt: .now, layout: noDefault + ) == nil) + let withDefault = try #require(CassandraTableLayoutTests.visits(defaultTimeToLive: 60)) + #expect(CassandraRowLifetimes( + columns: ["note", "owner"], lifetimes: [.permanent, nil], readAt: .now, layout: withDefault + ) != nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift index b02a53181e..8b0a3fa3f1 100644 --- a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraConditionalWriteTests.swift @@ -58,4 +58,40 @@ struct CassandraConditionalWriteTests { statement: "UPDATE t SET a = 1 WHERE k = 1", firstColumn: "a", firstValue: "false" ) == nil) } + + @Test("An edit that keeps a TTL names it before SET and is refused the same way") + func updateWithTimeToLive() { + let update = CassandraConditionalWrite.updateStatement( + target: "\"shop\".\"orders\"", assignments: ["\"note\" = ?"], keyCondition: "\"id\" = ?", timeToLive: 300 + ) + #expect(update == "UPDATE \"shop\".\"orders\" USING TTL 300 SET \"note\" = ? WHERE \"id\" = ? IF EXISTS") + #expect(CassandraConditionalWrite.refusal(statement: update, firstColumn: "[applied]", firstValue: "false") + == .rowMissing) + } + + @Test("An unapplied batch of the row's UPDATEs, static ones unconditioned, is refused as a missing row") + func batchOfRowUpdatesIsRefused() { + let batch = CassandraConditionalWrite.batch([ + "UPDATE \"t\" USING TTL 60 SET \"a\" = ? WHERE \"k\" = ? AND \"c\" = ? IF EXISTS", + "UPDATE \"t\" USING TTL 0 SET \"b\" = ? WHERE \"k\" = ? AND \"c\" = ? IF EXISTS", + "UPDATE \"t\" USING TTL 30 SET \"s\" = ? WHERE \"k\" = ?", + ]) + #expect(batch.hasPrefix("BEGIN BATCH UPDATE")) + #expect(batch.hasSuffix("; APPLY BATCH")) + #expect(CassandraConditionalWrite.refusal(statement: batch, firstColumn: "[applied]", firstValue: "false") + == .rowMissing) + #expect(CassandraConditionalWrite.refusal(statement: batch, firstColumn: "[applied]", firstValue: "true") + == nil) + } + + @Test("A batch with a condition of its own, an INSERT or no condition is left to the result", arguments: [ + "BEGIN BATCH UPDATE t SET a = 1 WHERE k = 1 IF a = 0; UPDATE t SET b = 1 WHERE k = 1 IF EXISTS; APPLY BATCH", + "BEGIN BATCH INSERT INTO t (k) VALUES (1) IF NOT EXISTS; UPDATE t SET b = 1 WHERE k = 1 IF EXISTS APPLY BATCH", + "BEGIN BATCH INSERT INTO t (k, a) VALUES (1, 2) USING TTL 5; UPDATE t SET b = 1 WHERE k = 1; APPLY BATCH", + "BEGIN BATCH UPDATE t SET a = 'IF EXISTS' WHERE k = 1; APPLY BATCH", + ]) + func otherBatchesAreLeftToTheResult(batch: String) { + #expect(CassandraConditionalWrite.refusal(statement: batch, firstColumn: "[applied]", firstValue: "false") + == nil) + } } diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraLifetimeReadTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraLifetimeReadTests.swift new file mode 100644 index 0000000000..5ef19efe07 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraLifetimeReadTests.swift @@ -0,0 +1,63 @@ +// +// CassandraLifetimeReadTests.swift +// TableProCassandraCoreTests +// + +import TableProCassandraCore +import Testing + +struct CassandraLifetimeReadTests { + private static let target = "\"shop\".\"visits\"" + + @Test("A browse of named columns reads TTL() after them for each one that takes it") + func namedColumns() throws { + let layout = try #require(CassandraTableLayoutTests.visits()) + let plan = try #require(CassandraLifetimeRead.plan( + cql: "SELECT \"id\", \"day\", \"note\", \"tags\" FROM \"shop\".\"visits\" WHERE \"note\" = ? ALLOW FILTERING", + target: Self.target, + layout: layout + )) + #expect(plan.cql + == "SELECT \"id\", \"day\", \"note\", \"tags\", TTL(\"note\") FROM \"shop\".\"visits\" WHERE \"note\" = ? ALLOW FILTERING") + #expect(plan.shownColumns == ["id", "day", "note", "tags"]) + #expect(plan.lifetimeColumns == ["note"]) + #expect(plan.lifetimeValueIndices == [2]) + #expect(plan.keyIndices == ["id": 0, "day": 1]) + } + + @Test("SELECT * is spelled out in the order the server returns it, so the TTL() selectors can follow") + func wildcard() throws { + let layout = try #require(CassandraTableLayoutTests.visits()) + let plan = try #require(CassandraLifetimeRead.plan(cql: "SELECT * FROM \"visits\"", target: "\"visits\"", layout: layout)) + #expect(plan.shownColumns == ["id", "day", "owner", "Count", "address", "note", "tags"]) + #expect(plan.cql == "SELECT \"id\", \"day\", \"owner\", \"Count\", \"address\", \"note\", \"tags\", " + + "TTL(\"owner\"), TTL(\"Count\"), TTL(\"address\"), TTL(\"note\") FROM \"visits\"") + #expect(plan.lifetimeValueIndices == [2, 3, 4, 5]) + } + + @Test("A name holding a quote and a comma is read back whole") + func quotedNames() throws { + let layout = try #require(CassandraTableLayoutTests.visits( + extra: [.init(name: "a\"b, c", type: "text", kind: .regular, position: -1)] + )) + let plan = try #require(CassandraLifetimeRead.plan( + cql: "SELECT \"id\", \"day\", \"a\"\"b, c\" FROM \"shop\".\"visits\"", target: Self.target, layout: layout + )) + #expect(plan.shownColumns == ["id", "day", "a\"b, c"]) + #expect(plan.cql == "SELECT \"id\", \"day\", \"a\"\"b, c\", TTL(\"a\"\"b, c\") FROM \"shop\".\"visits\"") + } + + @Test("Nothing is added to another table, another shape of statement, a read without the key or without a TTL", arguments: [ + "SELECT * FROM \"shop\".\"other\"", + "SELECT * FROM \"shop\".\"visits\" LIMIT 5", + "SELECT id, day, note FROM \"shop\".\"visits\"", + "SELECT \"note\" FROM \"shop\".\"visits\"", + "SELECT \"id\", \"day\", \"tags\" FROM \"shop\".\"visits\"", + "SELECT \"id\", \"day\", \"note FROM \"shop\".\"visits\"", + "select * from \"shop\".\"visits\"", + ]) + func untouchedStatements(cql: String) throws { + let layout = try #require(CassandraTableLayoutTests.visits()) + #expect(CassandraLifetimeRead.plan(cql: cql, target: Self.target, layout: layout) == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraLifetimeWriteTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraLifetimeWriteTests.swift new file mode 100644 index 0000000000..282485c749 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraLifetimeWriteTests.swift @@ -0,0 +1,150 @@ +// +// CassandraLifetimeWriteTests.swift +// TableProCassandraCoreTests +// + +import TableProCassandraCore +import Testing + +struct CassandraLifetimeWriteTests { + private typealias Write = CassandraLifetimeWrite + private static let target = "\"shop\".\"visits\"" + private static let row = Write.KeyCondition(cql: "\"id\" = ? AND \"day\" = ?", values: ["k", "d"]) + private static let partition = Write.KeyCondition(cql: "\"id\" = ?", values: ["k"]) + + private static func set(_ column: String, _ value: String, ttl: Int?, isStatic: Bool = false) -> Write.Assignment { + Write.Assignment(cql: "\"\(column)\" = ?", parameters: [value], timeToLive: ttl, isStatic: isStatic) + } + + @Test("An edit with no TTL to keep is the statement it always was") + func noTimeToLive() { + let write = Write.update( + target: Self.target, assignments: [Self.set("note", "n", ttl: nil)], row: Self.row, partition: Self.partition + ) + #expect(write.cql == CassandraConditionalWrite.updateStatement( + target: Self.target, assignments: ["\"note\" = ?"], keyCondition: Self.row.cql + )) + #expect(write.parameters == ["n", "k", "d"]) + } + + @Test("Cells that keep one TTL are one UPDATE with it") + func oneTimeToLive() { + let write = Write.update( + target: Self.target, + assignments: [Self.set("note", "n", ttl: 120), Self.set("address", "a", ttl: 120)], + row: Self.row, + partition: Self.partition + ) + #expect(write.cql == "UPDATE \"shop\".\"visits\" USING TTL 120 SET \"note\" = ?, \"address\" = ? " + + "WHERE \"id\" = ? AND \"day\" = ? IF EXISTS") + #expect(write.parameters == ["n", "a", "k", "d"]) + } + + @Test("Cells that keep different TTLs are one batch on the row, a static-only UPDATE naming the partition") + func severalTimesToLive() { + let write = Write.update( + target: Self.target, + assignments: [ + Self.set("note", "n", ttl: 120), + Self.set("address", "a", ttl: 0), + Self.set("Count", "c", ttl: nil), + Self.set("owner", "o", ttl: 50, isStatic: true), + Self.set("extra", "e", ttl: 120), + ], + row: Self.row, + partition: Self.partition + ) + #expect(write.cql == "BEGIN BATCH " + + "UPDATE \"shop\".\"visits\" USING TTL 120 SET \"note\" = ?, \"extra\" = ? WHERE \"id\" = ? AND \"day\" = ? IF EXISTS; " + + "UPDATE \"shop\".\"visits\" USING TTL 0 SET \"address\" = ? WHERE \"id\" = ? AND \"day\" = ? IF EXISTS; " + + "UPDATE \"shop\".\"visits\" SET \"Count\" = ? WHERE \"id\" = ? AND \"day\" = ? IF EXISTS; " + + "UPDATE \"shop\".\"visits\" USING TTL 50 SET \"owner\" = ? WHERE \"id\" = ?; " + + "APPLY BATCH") + #expect(write.parameters == ["n", "e", "k", "d", "a", "k", "d", "c", "k", "d", "o", "k"]) + #expect(CassandraConditionalWrite.refusal(statement: write.cql, firstColumn: "[applied]", firstValue: "false") + == .rowMissing) + } + + private static func cell( + _ column: String, + ttl: Int? = nil, + isKey: Bool = false, + isNull: Bool = false, + isStatic: Bool = false + ) -> Write.RestoredCell { + Write.RestoredCell( + column: column, value: column, isKey: isKey, isNull: isNull, isStatic: isStatic, timeToLive: ttl + ) + } + + private static let keys = Write.RowKeys(row: row, partition: partition) + + @Test("A restored row with no TTL to keep is the INSERT it always was") + func restoreWithoutTimesToLive() { + let cells = [Self.cell("id", isKey: true), Self.cell("day", isKey: true), Self.cell("note"), Self.cell("Count", isNull: true)] + let expected = "INSERT INTO \"shop\".\"visits\" (\"id\", \"day\", \"note\", \"Count\") VALUES (?, ?, ?, ?)" + for keys in [nil, Self.keys] { + let write = Write.restore(target: Self.target, cells: cells, keys: keys) + #expect(write.cql == expected) + #expect(write.parameters == ["id", "day", "note", "Count"]) + } + let unkeyed = Write.restore( + target: Self.target, cells: [Self.cell("id", isKey: true), Self.cell("note", ttl: 40)], keys: nil + ) + #expect(unkeyed.cql == "INSERT INTO \"shop\".\"visits\" (\"id\", \"note\") VALUES (?, ?)") + } + + @Test("A restored row whose cells share a TTL is one INSERT with it") + func restoreOneTimeToLive() { + let write = Write.restore( + target: Self.target, + cells: [Self.cell("id", isKey: true), Self.cell("day", isKey: true), Self.cell("note", ttl: 90), + Self.cell("Count", isNull: true), Self.cell("owner", ttl: 90, isStatic: true)], + keys: Self.keys + ) + #expect(write.cql == "INSERT INTO \"shop\".\"visits\" (\"id\", \"day\", \"note\", \"Count\", \"owner\") " + + "VALUES (?, ?, ?, ?, ?) USING TTL 90") + #expect(write.parameters == ["id", "day", "note", "Count", "owner"]) + } + + // The INSERT also writes the row marker, so it takes the cells that live longest. + @Test("A restored row with several TTLs is a batch whose INSERT carries the longest-living cells") + func restoreSeveralTimesToLive() { + let write = Write.restore( + target: Self.target, + cells: [ + Self.cell("id", isKey: true), Self.cell("day", isKey: true), + Self.cell("owner", ttl: 50, isStatic: true), Self.cell("note", ttl: 120), + Self.cell("address", ttl: 0), Self.cell("Count", isNull: true), Self.cell("tags"), + ], + keys: Self.keys + ) + #expect(write.cql == "BEGIN BATCH " + + "INSERT INTO \"shop\".\"visits\" (\"id\", \"day\", \"address\", \"Count\") VALUES (?, ?, ?, ?) USING TTL 0; " + + "UPDATE \"shop\".\"visits\" USING TTL 50 SET \"owner\" = ? WHERE \"id\" = ?; " + + "UPDATE \"shop\".\"visits\" USING TTL 120 SET \"note\" = ? WHERE \"id\" = ? AND \"day\" = ?; " + + "UPDATE \"shop\".\"visits\" SET \"tags\" = ? WHERE \"id\" = ? AND \"day\" = ?; " + + "APPLY BATCH") + #expect(write.parameters == [ + "id", "day", "address", "Count", "owner", "k", "note", "k", "d", "tags", "k", "d", + ]) + #expect(CassandraConditionalWrite.refusal(statement: write.cql, firstColumn: "[applied]", firstValue: "false") + == nil) + } + + @Test("Without a cell that never expires, the INSERT takes the table's default, else the longest TTL") + func restoreInsertChoice() { + let base = [Self.cell("id", isKey: true), Self.cell("day", isKey: true)] + let toDefault = Write.restore( + target: Self.target, cells: base + [Self.cell("note", ttl: 30), Self.cell("tags")], keys: Self.keys + ) + #expect(toDefault.cql.hasPrefix("BEGIN BATCH INSERT INTO \"shop\".\"visits\" (\"id\", \"day\", \"tags\") VALUES (?, ?, ?); ")) + let longest = Write.restore( + target: Self.target, cells: base + [Self.cell("note", ttl: 30), Self.cell("address", ttl: 900)], keys: Self.keys + ) + #expect(longest.cql.hasPrefix( + "BEGIN BATCH INSERT INTO \"shop\".\"visits\" (\"id\", \"day\", \"address\") VALUES (?, ?, ?) USING TTL 900; " + )) + #expect(longest.cql.contains("UPDATE \"shop\".\"visits\" USING TTL 30 SET \"note\" = ? WHERE \"id\" = ? AND \"day\" = ?; ")) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraRecentEntriesTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraRecentEntriesTests.swift new file mode 100644 index 0000000000..ba4c1d4cb8 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraRecentEntriesTests.swift @@ -0,0 +1,49 @@ +// +// CassandraRecentEntriesTests.swift +// TableProCassandraCoreTests +// + +import TableProCassandraCore +import Testing + +struct CassandraRecentEntriesTests { + @Test("Past the capacity the entries written longest ago are dropped first") + func dropsOldestPastCapacity() { + var entries = CassandraRecentEntries(capacity: 4) + for key in 1...5 { + entries.set("v\(key)", for: key) + } + #expect(entries.count <= 4) + #expect(entries[1] == nil) + #expect(entries[2] == nil) + #expect(entries[3] == "v3") + #expect(entries[5] == "v5") + + for key in 6...100 { + entries.set("v\(key)", for: key) + #expect(entries.count <= 4) + } + #expect(entries[100] == "v100") + #expect(entries[99] == "v99") + } + + @Test("Writing a key again replaces its value and keeps it among the newest") + func rewriteRefreshes() { + var entries = CassandraRecentEntries(capacity: 4) + entries.set(1, for: "a") + entries.set(2, for: "b") + entries.set(3, for: "c") + entries.set(10, for: "a") + entries.set(4, for: "d") + #expect(entries["a"] == 10) + #expect(entries["b"] == nil) + #expect(entries["c"] == 3) + #expect(entries["d"] == 4) + #expect(entries.count == 3) + entries.removeValue(for: "a") + #expect(entries["a"] == nil) + entries.removeAll() + #expect(entries["c"] == nil) + #expect(entries["d"] == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraTableLayoutTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraTableLayoutTests.swift new file mode 100644 index 0000000000..57d6a0e8b3 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraTableLayoutTests.swift @@ -0,0 +1,66 @@ +// +// CassandraTableLayoutTests.swift +// TableProCassandraCoreTests +// + +import TableProCassandraCore +import Testing + +struct CassandraTableLayoutTests { + static func visits(defaultTimeToLive: Int? = 0, extra: [CassandraTableLayout.Column] = []) -> CassandraTableLayout? { + CassandraTableLayout(columns: [ + .init(name: "note", type: "text", kind: .regular, position: -1), + .init(name: "day", type: "date", kind: .clustering, position: 0), + .init(name: "tags", type: "set", kind: .regular, position: -1), + .init(name: "owner", type: "text", kind: .staticColumn, position: -1), + .init(name: "id", type: "uuid", kind: .partitionKey, position: 0), + .init(name: "Count", type: "int", kind: .regular, position: -1), + .init(name: "address", type: "frozen
", kind: .regular, position: -1), + ] + extra, defaultTimeToLive: defaultTimeToLive) + } + + @Test("SELECT * order: the key in order, then static and regular columns by name") + func selectOrder() throws { + let layout = try #require(Self.visits()) + #expect(layout.selectOrder == ["id", "day", "owner", "Count", "address", "note", "tags"]) + #expect(layout.partitionKey == ["id"]) + #expect(layout.primaryKey == ["id", "day"]) + #expect(layout.staticColumns == ["owner"]) + } + + @Test("TTL() is read for regular and static cells that hold one value", arguments: [ + ("note", true), ("owner", true), ("Count", true), ("address", true), + ("tags", false), ("id", false), ("day", false), ("missing", false), + ]) + func acceptedColumns(column: String, accepted: Bool) throws { + let layout = try #require(Self.visits()) + #expect(layout.acceptsTimeToLive(column) == accepted) + } + + @Test("Counters, non-frozen collections and user types, and custom types take no TTL()", arguments: [ + ("counter", false), ("list", false), ("map", false), ("address", false), + ("'org.apache.cassandra.db.marshal.DynamicCompositeType'", false), + ("frozen>", true), ("tuple", true), ("vector", true), ("duration", true), + (" TIMESTAMP ", true), + ]) + func typesThatHoldOneCell(type: String, accepted: Bool) throws { + let layout = try #require(Self.visits(extra: [.init(name: "probe", type: type, kind: .regular, position: -1)])) + #expect(layout.acceptsTimeToLive("probe") == accepted) + } + + @Test("No layout for a table without columns or with a COMPACT STORAGE value column") + func unusableTables() { + #expect(CassandraTableLayout(columns: [], defaultTimeToLive: 0) == nil) + #expect(Self.visits(extra: [.init(name: "value", type: "empty", kind: .regular, position: -1)]) == nil) + } + + @Test("A system_schema row needs a name, a type and a kind CQL has") + func columnRows() { + #expect(CassandraTableLayout.column(name: "note", type: "text", kind: "regular", position: "-1") + == .init(name: "note", type: "text", kind: .regular, position: -1)) + #expect(CassandraTableLayout.column(name: "id", type: "int", kind: "partition_key", position: "0")?.kind + == .partitionKey) + #expect(CassandraTableLayout.column(name: "note", type: "text", kind: "other", position: "0") == nil) + #expect(CassandraTableLayout.column(name: nil, type: "text", kind: "regular", position: "0") == nil) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraWireTextTests.swift b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraWireTextTests.swift new file mode 100644 index 0000000000..49956b11f8 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProCassandraCoreTests/CassandraWireTextTests.swift @@ -0,0 +1,32 @@ +// +// CassandraWireTextTests.swift +// TableProCassandraCoreTests +// + +import Foundation +import TableProCassandraCore +import Testing + +struct CassandraWireTextTests { + // Foundation drops this mark, and the key without it names another row. + @Test("A leading byte order mark stays part of the text") + func byteOrderMarkIsKept() { + let bytes: [UInt8] = [0xEF, 0xBB, 0xBF, 0x61, 0x62, 0x63] + let fromData = CassandraWireText.text(Data(bytes)) + let fromBuffer = bytes.withUnsafeBytes { CassandraWireText.text($0) } + #expect(fromData == "\u{FEFF}abc") + #expect(fromBuffer == "\u{FEFF}abc") + #expect(fromData.map { Array($0.utf8) } == bytes) + } + + @Test("Text that is not UTF-8 is not read as text", arguments: [[0xFF], [0x61, 0xC3], [0xED, 0xA0, 0x80]] as [[UInt8]]) + func invalidText(bytes: [UInt8]) { + #expect(CassandraWireText.text(Data(bytes)) == nil) + } + + @Test("Empty and plain text read as written") + func plainText() { + #expect(CassandraWireText.text(Data()) == "") + #expect(CassandraWireText.text(Data("héllo".utf8)) == "héllo") + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaRecordOffsetTests.swift b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaRecordOffsetTests.swift index bbfe76b995..9c67c9d975 100644 --- a/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaRecordOffsetTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaRecordOffsetTests.swift @@ -85,4 +85,81 @@ private func decodeOne(baseOffset: Int64, baseTimestamp: Int64, offsetDelta: Int budget: &budget ) #expect(decoded.records.map { $0.offset } == [4, 11, 13]) + #expect(decoded.nextOffset == 14) +} + +@Test func offsetAfterABatchComesFromItsHeader() { + #expect(KafkaOffsetMath.offset(afterBatchAt: 100, lastOffsetDelta: 4) == 105) + #expect(KafkaOffsetMath.offset(afterBatchAt: 100, lastOffsetDelta: 0) == 101) + #expect(KafkaOffsetMath.offset(afterBatchAt: 100, lastOffsetDelta: -1) == 101) + #expect(KafkaOffsetMath.offset(afterBatchAt: .max, lastOffsetDelta: 0) == nil) + #expect(KafkaOffsetMath.offset(afterBatchAt: .max - 1, lastOffsetDelta: 2) == nil) +} + +private func decodeBlob(_ blob: Data) throws -> (records: [KafkaRecord], truncated: Bool, nextOffset: Int64?) { + var budget = DecompressionBudget.standard + return try KafkaRecordBatchDecoder.decode(blob: blob, partition: 0, skipAborted: true, budget: &budget) +} + +private func marker(at offset: Int64, producerId: Int64 = 7) -> Data { + KafkaBatchFixture.batch( + baseOffset: offset, + flags: KafkaBatchFixture.transactional | KafkaBatchFixture.control, + producerId: producerId, + payload: KafkaBatchFixture.record(value: nil) + ) +} + +@Test func aReplyOfOnlyACommitMarkerStillMovesTheReadOn() throws { + let decoded = try decodeBlob(marker(at: 41)) + #expect(decoded.records.isEmpty) + #expect(decoded.nextOffset == 42) +} + +@Test func aTrailingMarkerMovesTheReadPastIt() throws { + let blob = KafkaBatchFixture.batch( + baseOffset: 40, flags: KafkaBatchFixture.transactional, producerId: 7, + payload: KafkaBatchFixture.record(value: Data("row".utf8)) + ) + marker(at: 41) + let decoded = try decodeBlob(blob) + #expect(decoded.records.map { $0.offset } == [40]) + #expect(decoded.nextOffset == 42) +} + +@Test func anAbortedRunMovesTheReadPastIt() throws { + let blob = KafkaBatchFixture.batch( + baseOffset: 10, flags: KafkaBatchFixture.transactional, producerId: 7, recordCount: 3, + payload: KafkaBatchFixture.record(offsetDelta: 0, value: Data("a".utf8)) + + KafkaBatchFixture.record(offsetDelta: 1, value: Data("b".utf8)) + + KafkaBatchFixture.record(offsetDelta: 2, value: Data("c".utf8)) + ) + marker(at: 13) + var budget = DecompressionBudget.standard + let decoded = try KafkaRecordBatchDecoder.decode( + blob: blob, + partition: 0, + abortedTransactions: [KafkaAbortedTransaction(producerId: 7, firstOffset: 10)], + skipAborted: true, + budget: &budget + ) + #expect(decoded.records.isEmpty) + #expect(decoded.nextOffset == 14) +} + +// Compaction keeps an emptied batch's header, last offset included (Kafka protocol docs, Record Batch). +@Test func aBatchCompactionEmptiedMovesTheReadPastItsWholeRange() throws { + let decoded = try decodeBlob(KafkaBatchFixture.batch(baseOffset: 200, recordCount: 5, payload: Data())) + #expect(decoded.records.isEmpty) + #expect(decoded.nextOffset == 205) +} + +@Test func onlyWholeBatchesMoveTheReadOn() throws { + let whole = KafkaBatchFixture.batch(baseOffset: 7, payload: KafkaBatchFixture.record(value: Data("x".utf8))) + let cut = try decodeBlob(whole + whole.prefix(20)) + #expect(cut.records.map { $0.offset } == [7]) + #expect(cut.truncated) + #expect(cut.nextOffset == 8) + + let nothingWhole = try decodeBlob(whole.prefix(whole.count - 1)) + #expect(nothingWhole.records.isEmpty) + #expect(nothingWhole.nextOffset == nil) } diff --git a/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaTailWalkTests.swift b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaTailWalkTests.swift new file mode 100644 index 0000000000..d58225b0fa --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProKafkaCoreTests/KafkaTailWalkTests.swift @@ -0,0 +1,117 @@ +import Foundation +import TableProKafkaCore +import Testing + +private struct WalkOutcome { + let records: [Int64] + let reads: [[Int64]] + let start: Int64 +} + +// A partition log given by the offsets that hold a record, read the way the browse engine reads it. +private func walk( + log: [Int64], + end: Int64, + floor: Int64, + wanted: Int +) -> WalkOutcome { + let firstStart = KafkaOffsetMath.stepBack(from: end, by: Int64(wanted), floor: floor) + var tail = KafkaTailWalk(wanted: wanted, end: end, firstStart: firstStart, floor: floor) + var reads: [[Int64]] = [] + while let range = tail.nextRange { + reads.append([range.start, range.end]) + tail.absorb(Array(log.filter { $0 >= range.start && $0 < range.end }.suffix(tail.needed))) + } + return WalkOutcome(records: tail.records, reads: reads, start: tail.start) +} + +// What KafkaRecordOrdering.page takes from a tail window on one partition. +private func page(_ records: [Int64], skip: Int, limit: Int) -> [Int64] { + Array(records.dropLast(skip).suffix(limit)) +} + +struct KafkaTailWalkTests { + // One commit marker after every record: half the offsets hold no record. + private static let markedLog = stride(from: Int64(0), to: 2_000, by: 2).map { $0 } + + @Test("A window with fewer records than offsets steps back until it holds the page") + func stepsBackPastMarkers() { + let outcome = walk(log: Self.markedLog, end: 2_000, floor: 0, wanted: 100) + #expect(outcome.records == Array(stride(from: Int64(1_800), through: 1_998, by: 2))) + #expect(outcome.reads == [[1_900, 2_000], [1_775, 1_900]]) + #expect(outcome.start == 1_775) + } + + @Test("Page two follows page one with no record skipped or repeated") + func pagesAreContiguous() { + let first = page(walk(log: Self.markedLog, end: 2_000, floor: 0, wanted: 100).records, skip: 0, limit: 100) + let second = page(walk(log: Self.markedLog, end: 2_000, floor: 0, wanted: 200).records, skip: 100, limit: 100) + let third = page(walk(log: Self.markedLog, end: 2_000, floor: 0, wanted: 300).records, skip: 200, limit: 100) + #expect(first == Array(stride(from: Int64(1_800), through: 1_998, by: 2))) + #expect(second == Array(stride(from: Int64(1_600), through: 1_798, by: 2))) + #expect(third == Array(stride(from: Int64(1_400), through: 1_598, by: 2))) + } + + @Test("A deep page of a sparse log is not empty") + func deepPageOfSparseLog() { + let sparse = stride(from: Int64(0), to: 100_000, by: 10).map { $0 } + let tenth = page(walk(log: sparse, end: 100_000, floor: 0, wanted: 1_000).records, skip: 900, limit: 100) + #expect(tenth.count == 100) + let expectedLast: Int64 = 99_990 - 900 * 10 + #expect(tenth.last == expectedLast) + } + + @Test("The walk stops at the earliest offset with what the log holds") + func stopsAtTheFloor() { + let outcome = walk(log: [503, 507, 990], end: 1_000, floor: 500, wanted: 100) + #expect(outcome.records == [503, 507, 990]) + #expect(outcome.start == 500) + #expect(outcome.reads.last?.first == 500) + } + + @Test("A window that already holds the page reads once") + func denseWindowReadsOnce() { + let dense = (0 ..< 1_000).map { Int64($0) } + let outcome = walk(log: dense, end: 1_000, floor: 0, wanted: 50) + #expect(outcome.reads == [[950, 1_000]]) + #expect(outcome.records == Array(950 ..< 1_000).map { Int64($0) }) + } + + @Test("A walk that finds nothing grows each step by the most it may") + func emptyStretchGrowsFast() { + let outcome = walk(log: [0], end: 10_000, floor: 0, wanted: 100) + #expect(outcome.reads == [[9_900, 10_000], [9_500, 9_900], [7_900, 9_500], [1_500, 7_900], [0, 1_500]]) + #expect(outcome.records == [0]) + } + + @Test("An empty partition and an empty page read nothing past the end") + func emptyCases() { + let empty = walk(log: [], end: 40, floor: 40, wanted: 10) + #expect(empty.reads == [[40, 40]]) + #expect(empty.records.isEmpty) + + var none = KafkaTailWalk(wanted: 0, end: 10, firstStart: 0, floor: 0) + let pending = none.nextRange != nil + #expect(!pending) + none.absorb([1]) + #expect(none.records.isEmpty) + } + + @Test("The next width reaches as far as the last range's density says, and grows by the cap after an empty range") + func nextWidth() { + #expect(KafkaOffsetMath.nextWidth(lastWidth: 100, lastRecords: 50, needed: 50, maximumGrowth: 4) == 125) + #expect(KafkaOffsetMath.nextWidth(lastWidth: 1_000, lastRecords: 1_000, needed: 1, maximumGrowth: 4) == 2) + #expect(KafkaOffsetMath.nextWidth(lastWidth: 1_000, lastRecords: 100, needed: 900, maximumGrowth: 4) == 4_000) + #expect(KafkaOffsetMath.nextWidth(lastWidth: 100, lastRecords: 0, needed: 10, maximumGrowth: 4) == 400) + #expect(KafkaOffsetMath.nextWidth(lastWidth: .max, lastRecords: 0, needed: 1, maximumGrowth: 4) == .max) + } + + @Test("An empty stretch below a dense range is crossed by growing steps, not a few offsets at a time") + func denseThenEmpty() { + let log = [Int64(0)] + (99_001 ..< 100_000).map { Int64($0) } + let outcome = walk(log: log, end: 100_000, floor: 0, wanted: 1_000) + #expect(outcome.reads.map { $0[1] - $0[0] } == [1_000, 2, 8, 32, 128, 512, 2_048, 8_192, 32_768, 55_310]) + #expect(outcome.records.count == 1_000) + #expect(outcome.records.first == 0) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedItemEditTests.swift b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedItemEditTests.swift new file mode 100644 index 0000000000..7a0207a0be --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProMemcachedCoreTests/MemcachedItemEditTests.swift @@ -0,0 +1,59 @@ +import Foundation +import TableProMemcachedCore +import Testing + +struct MemcachedItemEditTests { + private func write(value: Bool = false, flags: Bool = false, ttl: Bool = false, unique: UInt64?) -> MemcachedItemEdit? { + MemcachedItemEdit.write(editsValue: value, editsFlags: flags, editsTTL: ttl, unique: unique) + } + + @Test("A Flags edit carries the loaded Value and a Value edit the loaded Flags, so each is a cas on the read's unique") + func keepingALoadedFieldIsACompareAndSwap() { + #expect(write(flags: true, unique: 42) == .compareAndSwap(unique: 42)) + #expect(write(flags: true, ttl: true, unique: 42) == .compareAndSwap(unique: 42)) + #expect(write(value: true, unique: 42) == .compareAndSwap(unique: 42)) + #expect(write(value: true, ttl: true, unique: 42) == .compareAndSwap(unique: 42)) + } + + @Test("Without a unique from the read, an edit that keeps a loaded field asks for a refresh") + func keepingALoadedFieldWithoutAUniqueNeedsARefresh() { + #expect(write(flags: true, unique: nil) == .needsRefresh) + #expect(write(value: true, ttl: true, unique: nil) == .needsRefresh) + } + + @Test("An edit that writes both Value and Flags is a set, a TTL alone a touch, and nothing stored is no write") + func writesThatKeepNothingLoaded() { + #expect(write(value: true, flags: true, unique: nil) == .set) + #expect(write(value: true, flags: true, ttl: true, unique: 42) == .set) + #expect(write(ttl: true, unique: nil) == .touch) + #expect(write(ttl: true, unique: 42) == .touch) + #expect(write(unique: 42) == nil) + } + + @Test("A cas answered EXISTS changed since the read and NOT_FOUND is gone; STORED and other commands' replies pass") + func compareAndSwapReplies() { + let cas = MemcachedCommand.compareAndSwap(key: "k", flags: 0, exptime: 0, unique: 7, value: Data("v".utf8)) + #expect(MemcachedCASRefusal(replyTo: cas, status: "EXISTS") == .changed) + #expect(MemcachedCASRefusal(replyTo: cas, status: "NOT_FOUND") == .notFound) + #expect(MemcachedCASRefusal(replyTo: cas, status: "STORED") == nil) + #expect(MemcachedCASRefusal(replyTo: .touch(key: "k", exptime: 0), status: "NOT_FOUND") == nil) + #expect(MemcachedCASRefusal(replyTo: .delete(key: "k"), status: "NOT_FOUND") == nil) + #expect(MemcachedCASRefusal( + replyTo: .store(.add, key: "k", flags: 0, exptime: 0, value: Data()), status: "EXISTS" + ) == nil) + } + + @Test("A cas statement parses back into its command and sends the unique after the byte count") + func compareAndSwapStatement() throws { + let command = try MemcachedCommandParser.parse( + MemcachedStatements.cas(key: "k", flags: 7, exptime: 60, unique: 9_000_000_000, value: "hello") + ) + #expect(command == .compareAndSwap(key: "k", flags: 7, exptime: 60, unique: 9_000_000_000, value: Data("hello".utf8))) + #expect(String(bytes: command.wireBytes, encoding: .utf8) == "cas k 7 60 5 9000000000\r\nhello\r\n") + + let bytes = Data([0x00, 0xFF, 0x22, 0x0A]) + #expect(try MemcachedCommandParser.parse( + MemcachedStatements.cas(key: "user'1", flags: 0, exptime: 0, unique: 1, bytes: bytes) + ) == .compareAndSwap(key: "user'1", flags: 0, exptime: 0, unique: 1, value: bytes)) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisKeySummaryTests.swift b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisKeySummaryTests.swift index 264c3f6009..227c42183c 100644 --- a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisKeySummaryTests.swift +++ b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisKeySummaryTests.swift @@ -12,6 +12,19 @@ struct RedisKeySummaryCommandTests { @Test("a string key is read with GET so the whole value arrives") func stringPreviewReadsWholeValue() { #expect(RedisKeySummary.previewCommand(for: .string, key: "session:1") == ["GET", "session:1"]) + #expect(RedisKeySummary.previewCommand(for: .string, key: "session:1", stringByteLimit: nil) == ["GET", "session:1"]) + } + + @Test("a byte limit reads a string with GETRANGE over that many bytes") + func stringPreviewWithLimitReadsRange() { + #expect(RedisKeySummary.previewCommand(for: .string, key: "page:1", stringByteLimit: 65_536) + == ["GETRANGE", "page:1", "0", "65535"]) + } + + @Test("a byte limit leaves collection previews as they were") + func byteLimitIgnoredForCollections() { + #expect(RedisKeySummary.previewCommand(for: .list, key: "k", stringByteLimit: 10) + == RedisKeySummary.previewCommand(for: .list, key: "k")) } @Test("length command matches the Redis command for each kind") diff --git a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisStringPreviewTests.swift b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisStringPreviewTests.swift new file mode 100644 index 0000000000..0e25632dd6 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisStringPreviewTests.swift @@ -0,0 +1,77 @@ +import Foundation +import Testing + +@testable import TableProRedisCore + +struct RedisStringPreviewTests { + @Test("A value within the limit shows whole and unmarked") + func shortValueIsWhole() { + let shown = RedisStringPreview.shown(Data("hello".utf8), length: 5) + #expect(shown == .text("hello")) + } + + @Test("A value exactly at the limit is not shortened") + func valueAtTheLimitIsWhole() { + let bytes = Data(repeating: 0x61, count: 8) + #expect(RedisStringPreview.shown(bytes, length: 8, byteLimit: 8) == .text("aaaaaaaa")) + } + + @Test("A value past the limit shows its start followed by the marker") + func longValueIsMarked() { + let bytes = Data(repeating: 0x61, count: 8) + #expect(RedisStringPreview.shown(bytes, length: 500_000, byteLimit: 8) == .text("aaaaaaaa...")) + } + + @Test("A range that ends inside a multi-byte character drops that character only") + func cutCharacterIsDropped() { + // "aé€" is 61 C3 A9 E2 82 AC; a range of 5 bytes ends inside the euro sign. + let full = Array("aé€".utf8) + let shown = RedisStringPreview.shown(Data(full.prefix(5)), length: full.count, byteLimit: 5) + #expect(shown == .text("aé...")) + } + + @Test("A range that ends inside a four-byte character drops all of it") + func cutFourByteCharacterIsDropped() { + let full = Array("ab😀".utf8) + let shown = RedisStringPreview.shown(Data(full.prefix(4)), length: full.count, byteLimit: 4) + #expect(shown == .text("ab...")) + } + + @Test("A binary value stays bytes, whole range, without a marker") + func binaryValueStaysBytes() { + let bytes = Data([0x1F, 0x8B, 0x08, 0x00, 0xFF, 0xFE, 0x00, 0x03]) + #expect(RedisStringPreview.shown(bytes, length: 4_096, byteLimit: 8) == .bytes(bytes)) + #expect(RedisStringPreview.shown(bytes, length: 8, byteLimit: 8) == .bytes(bytes)) + } + + @Test("A value that starts with a byte order mark keeps it") + func byteOrderMarkSurvives() { + let bytes = Data([0xEF, 0xBB, 0xBF] + Array("abc".utf8)) + #expect(RedisStringPreview.shown(bytes, length: 6) == .text("\u{FEFF}abc")) + } + + @Test("Without a length, a read that filled the range counts as shortened") + func unknownLengthFilledRangeIsShortened() { + #expect(RedisStringPreview.isShortened(length: nil, shownByteCount: 8, byteLimit: 8)) + #expect(!RedisStringPreview.isShortened(length: nil, shownByteCount: 7, byteLimit: 8)) + #expect(RedisStringPreview.shown(Data(repeating: 0x61, count: 8), length: nil, byteLimit: 8) == .text("aaaaaaaa...")) + } + + @Test("The length alone decides when the server gave one") + func lengthDecides() { + #expect(RedisStringPreview.isShortened(length: 9, shownByteCount: 0, byteLimit: 8)) + #expect(!RedisStringPreview.isShortened(length: 8, shownByteCount: 8, byteLimit: 8)) + } + + @Test("The marked text of a cut value is at least as long as the limit") + func markedTextReachesTheLimit() throws { + let full = Array(String(repeating: "€", count: 30).utf8) + let limit = 64 + guard case .text(let text) = RedisStringPreview.shown(Data(full.prefix(limit)), length: full.count, byteLimit: limit) else { + Issue.record("expected text") + return + } + #expect(text.hasSuffix(RedisStringPreview.marker)) + #expect(text.utf8.count >= limit) + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisWireTextTests.swift b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisWireTextTests.swift new file mode 100644 index 0000000000..07990daf06 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProRedisCoreTests/RedisWireTextTests.swift @@ -0,0 +1,38 @@ +import Foundation +import Testing + +@testable import TableProRedisCore + +private let byteOrderMark: [UInt8] = [0xEF, 0xBB, 0xBF] + +struct RedisWireTextTests { + @Test("A leading byte order mark stays part of the text") + func keepsLeadingByteOrderMark() throws { + let bytes = Data(byteOrderMark + Array("user:1".utf8)) + let text = try #require(RedisWireText.decode(bytes)) + #expect(text.unicodeScalars.first == "\u{FEFF}") + #expect(Data(text.utf8) == bytes) + } + + @Test("Plain and non-ASCII text decode as they are", arguments: ["", "user:1", "café ☕", "a\u{0}b"]) + func decodesText(_ value: String) { + #expect(RedisWireText.decode(Data(value.utf8)) == value) + } + + @Test("Bytes that are not UTF-8 are not text", arguments: [ + Data([0xFF, 0xFE]), + Data([0x61, 0xE2, 0x82]), + Data([0xC0, 0x80]), + ]) + func rejectsInvalidBytes(_ bytes: Data) { + #expect(RedisWireText.decode(bytes) == nil) + } + + @Test("A quoted argument keeps a leading byte order mark") + func quoteKeepsByteOrderMark() throws { + let bytes = Data(byteOrderMark + Array("user:1".utf8)) + let quoted = RedisArgumentCodec.quote(bytes) + let parsed = try #require(RedisArgumentCodec.split(quoted)) + #expect(parsed == [bytes]) + } +} diff --git a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift index 78becc1c1e..9c08165f7f 100644 --- a/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift +++ b/apple/Plugins/BigtableDriverPlugin/BigtablePluginDriver+Editing.swift @@ -12,18 +12,37 @@ extension BigtablePluginDriver { insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set + ) throws -> [PluginRowWrite]? { + try generateRowWrites( + table: table, schema: schema, columns: columns, primaryKeyColumns: primaryKeyColumns, changes: changes, + insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, + insertedRowIndices: insertedRowIndices, context: PluginRowWriteContext() + ) + } + + func generateRowWrites( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set, + context: PluginRowWriteContext ) throws -> [PluginRowWrite]? { try changes.compactMap { change in do { - return try request(for: change, table: table, columns: columns, insertedRowData: insertedRowData) - .map { PluginRowWrite(statement: $0.text, rowIndices: [change.rowIndex]) } + return try request( + for: change, table: table, columns: columns, insertedRowData: insertedRowData, + locator: context.rowLocators[change.rowIndex] + ).map { PluginRowWrite(statement: $0.text, rowIndices: [change.rowIndex]) } } catch { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: error.localizedDescription) } } } - /// Restoring a deleted row writes it back under its own key, refused if that key is taken again. func generateIdentityPreservingInsert( table: String, schema: String?, @@ -32,8 +51,32 @@ extension BigtablePluginDriver { rows: [[PluginCellValue]], absentCells: [Int: Set] ) -> [(statement: String, parameters: [PluginCellValue])]? { - let statements = rows.enumerated().compactMap { index, row in - try? BigtableRowWrites.insert(table: table, columns: columns, values: cells(row, absent: absentCells[index] ?? [])).text + generateIdentityPreservingInsert( + table: table, schema: schema, columns: columns, primaryKeyColumns: primaryKeyColumns, rows: rows, + absentCells: absentCells, context: PluginRowWriteContext() + ) + } + + // Restoring a deleted row writes it back under its own key, refused if that key is taken again. + func generateIdentityPreservingInsert( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + rows: [[PluginCellValue]], + absentCells: [Int: Set], + context: PluginRowWriteContext + ) -> [(statement: String, parameters: [PluginCellValue])]? { + let keyIndex = columns.firstIndex(of: BigtableRowSet.keyColumn) + let statements = rows.enumerated().compactMap { index, row -> String? in + var values = cells(row, absent: absentCells[index] ?? []) + if let keyIndex, keyIndex < values.count { + guard let key = try? BigtableRowWrites.originalKey(locator: context.rowLocators[index], keyCell: values[keyIndex]) else { + return nil + } + values[keyIndex] = .bytes(key) + } + return try? BigtableRowWrites.insert(table: table, columns: columns, values: values).text } return statements.count == rows.count ? statements.map { ($0, []) } : nil } @@ -42,7 +85,8 @@ extension BigtablePluginDriver { for change: PluginRowChange, table: String, columns: [String], - insertedRowData: [Int: [PluginCellValue]] + insertedRowData: [Int: [PluginCellValue]], + locator: String? ) throws -> BigtableRequest? { let absent = change.absentColumns ?? [] switch change.type { @@ -59,16 +103,20 @@ extension BigtablePluginDriver { } return (column: cell.columnName, value: absent.contains(cell.columnIndex) ? BigtableRowSet.Cell.absent : Self.cell(cell.newValue)) } - return try BigtableRowWrites.update(table: table, key: try originalKey(change, columns: columns), changes: changes) + return try BigtableRowWrites.update( + table: table, key: try originalKey(change, columns: columns, locator: locator), changes: changes + ) case .delete: - return BigtableRowWrites.delete(table: table, key: try originalKey(change, columns: columns)) + return BigtableRowWrites.delete(table: table, key: try originalKey(change, columns: columns, locator: locator)) } } - private func originalKey(_ change: PluginRowChange, columns: [String]) throws -> Data { - guard let index = columns.firstIndex(of: BigtableRowSet.keyColumn), - let row = change.originalRow, index < row.count, - let key = try BigtableRowWrites.key(Self.cell(row[index])) else { + private func originalKey(_ change: PluginRowChange, columns: [String], locator: String?) throws -> Data { + var keyCell: BigtableRowSet.Cell? + if let index = columns.firstIndex(of: BigtableRowSet.keyColumn), let row = change.originalRow, index < row.count { + keyCell = Self.cell(row[index]) + } + guard let key = try BigtableRowWrites.originalKey(locator: locator, keyCell: keyCell) else { throw BigtableError.configuration(String(localized: "The row has no _key, so the row it came from is unknown. Reload the table.")) } return key diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraBoundValue.swift b/apple/Plugins/CassandraDriverPlugin/CassandraBoundValue.swift index cd8c0b396a..73d1b395c2 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraBoundValue.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraBoundValue.swift @@ -141,7 +141,7 @@ enum CassandraValueParser { case .blob: return .bytes(data) case .text: - guard let text = String(data: data, encoding: .utf8) else { + guard let text = CassandraWireText.text(data) else { throw CassandraValueRefusal.invalid("0x" + data.map { String(format: "%02x", $0) }.joined(), typeName: type.name) } return .string(text) diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraBrowseStatement.swift b/apple/Plugins/CassandraDriverPlugin/CassandraBrowseStatement.swift index 310d047813..cadd7debd4 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraBrowseStatement.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraBrowseStatement.swift @@ -19,6 +19,9 @@ struct CassandraBrowseStatement: Equatable, Sendable { let limit: Int let values: [String] let refusal: String? + // The table the CQL reads, for the TTL() selectors the run adds. A header written before them has neither. + var keyspace: String? + var table: String? } static let opening = "/* TablePro browse " @@ -34,15 +37,23 @@ struct CassandraBrowseStatement: Equatable, Sendable { return Self.opening + header + Self.closing + cql } - /// The same browse read one row past a query tab's cap, so a result the cap trims is still reported as trimmed. + // The same browse read one row past a query tab's cap, so a result the cap trims is still reported as trimmed. func cappedAt(rowCap: Int) -> CassandraBrowseStatement { let limit = min(window.limit, rowCap + 1) return CassandraBrowseStatement( - window: Window(offset: window.offset, limit: limit, values: window.values, refusal: window.refusal), + window: Window( + offset: window.offset, limit: limit, values: window.values, refusal: window.refusal, + keyspace: window.keyspace, table: window.table + ), cql: cql ) } + // The run keys its resume points on the CQL it sends. + func sending(_ cql: String) -> CassandraBrowseStatement { + CassandraBrowseStatement(window: window, cql: cql) + } + static func parse(_ text: String) -> CassandraBrowseStatement? { let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) guard trimmed.hasPrefix(opening), @@ -111,7 +122,10 @@ enum CassandraBrowseRenderer { if sorted { throw CassandraBrowseRefusal.sorting } let condition = try whereCondition(filters: filters, matchAll: matchAll) return CassandraBrowseStatement( - window: .init(offset: max(offset, 0), limit: max(limit, 0), values: condition.values, refusal: nil), + window: .init( + offset: max(offset, 0), limit: max(limit, 0), values: condition.values, refusal: nil, + keyspace: keyspace?.isEmpty == false ? keyspace : nil, table: table + ), cql: appending(condition, to: source) ) } catch { diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraConnection.swift b/apple/Plugins/CassandraDriverPlugin/CassandraConnection.swift index bbc86f6312..effc22404e 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraConnection.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraConnection.swift @@ -347,13 +347,15 @@ actor CassandraConnectionActor { return extractResult(from: result, startTime: startTime) } - /// Reads the rows a table browse shows, walking the paging state past the ones before its offset. A page - /// is counted by the rows it carried rather than by its size, because ScyllaDB answers a filtered page short. + // Reads the rows a table browse shows, walking the paging state past the ones before its offset. A page is + // counted by the rows it carried rather than by its size, because ScyllaDB answers a filtered page short. + // `fallback` runs instead when the server refuses to prepare `browse`. func executeBrowse( _ browse: CassandraBrowseStatement, + fallback: CassandraBrowseStatement? = nil, consistency: CassandraConsistency?, cancellation: CassandraCancellation - ) throws -> CassandraRawResult { + ) throws -> (result: CassandraRawResult, ranFallback: Bool) { if let refusal = browse.window.refusal { throw CassandraBrowseRefusal(pluginErrorMessage: refusal) } @@ -363,7 +365,18 @@ actor CassandraConnectionActor { let startTime = Date() try cancellation.check() - let prepared = try prepare(browse.cql, on: session, cancellation: cancellation) + var browse = browse + var ranFallback = false + let prepared: OpaquePointer + do { + prepared = try prepare(browse.cql, on: session, cancellation: cancellation) + } catch let error where !(error is CancellationError) { + guard let fallback else { throw error } + Self.logger.info("Browse refused, running it without TTL selectors: \(error.localizedDescription)") + browse = fallback + ranFallback = true + prepared = try prepare(browse.cql, on: session, cancellation: cancellation) + } defer { cass_prepared_free(prepared) } let pageSize = CassandraResumePoints.pageSize(forLimit: browse.window.limit) @@ -385,13 +398,14 @@ actor CassandraConnectionActor { read = try self.walk(walk, from: nil, prepared: prepared, session: session, cancellation: cancellation) } - return CassandraRawResult( + let result = CassandraRawResult( columns: read.header?.columns ?? [], columnTypeNames: read.header?.typeNames ?? [], rows: read.rows, rowsAffected: read.rows.count, executionTime: Date().timeIntervalSince(startTime) ) + return (result, ranFallback) } private func walk( @@ -669,16 +683,8 @@ actor CassandraConnectionActor { case CASS_VALUE_TYPE_ASCII, CASS_VALUE_TYPE_TEXT, CASS_VALUE_TYPE_VARCHAR: var output: UnsafePointer? var outputLength: Int = 0 - let rc = cass_value_get_string(value, &output, &outputLength) - if rc == CASS_OK, let output { - return String( - bytesNoCopy: UnsafeMutableRawPointer(mutating: output), - length: outputLength, - encoding: .utf8, - freeWhenDone: false - ) - } - return nil + guard cass_value_get_string(value, &output, &outputLength) == CASS_OK, let output else { return nil } + return wireText(output, length: outputLength) case CASS_VALUE_TYPE_INT: var intVal: Int32 = 0 @@ -817,21 +823,19 @@ actor CassandraConnectionActor { return CassandraVarint.decimalString(unscaled: Data(bytes: unscaled, count: length), scale: scale) default: - // Fallback: try reading as string var output: UnsafePointer? var outputLength: Int = 0 if cass_value_get_string(value, &output, &outputLength) == CASS_OK, let output { - return String( - bytesNoCopy: UnsafeMutableRawPointer(mutating: output), - length: outputLength, - encoding: .utf8, - freeWhenDone: false - ) + return wireText(output, length: outputLength) } return "" } } + private static func wireText(_ pointer: UnsafePointer, length: Int) -> String? { + CassandraWireText.text(UnsafeRawBufferPointer(start: pointer, count: length)) + } + private static func extractCollectionString( _ value: OpaquePointer, open: String, @@ -871,7 +875,7 @@ actor CassandraConnectionActor { var namePointer: UnsafePointer? var nameLength: Int = 0 let name = cass_iterator_get_user_type_field_name(iterator, &namePointer, &nameLength) == CASS_OK - ? namePointer.flatMap { String(bytes: UnsafeRawBufferPointer(start: $0, count: nameLength), encoding: .utf8) } + ? namePointer.flatMap { wireText($0, length: nameLength) } : nil let fieldValue = cass_iterator_get_user_type_field_value(iterator).flatMap(nestedText) ?? "null" fields.append("\(name ?? "?"): \(fieldValue)") diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift index 64b527cbf6..52165b1366 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraPlugin.swift @@ -135,6 +135,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen private let config: DriverConnectionConfig let connectionActor = CassandraConnectionActor() let activeBrowse = CassandraActiveBrowse() + let cellLifetimes = CassandraCellLifetimeStore() private let stateLock = NSLock() nonisolated(unsafe) private var _currentKeyspace: String? nonisolated(unsafe) private var _sessionConsistency: CassandraConsistency? @@ -287,6 +288,7 @@ internal final class CassandraPluginDriver: PluginDatabaseDriver, @unchecked Sen func disconnect() { activeBrowse.cancel() + cellLifetimes.removeAll() Task.detached(priority: .utility) { [connectionActor] in await connectionActor.close() } diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+Browse.swift b/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+Browse.swift index 7e072991f0..c7de4f443b 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+Browse.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+Browse.swift @@ -4,8 +4,12 @@ // import Foundation +import os +import TableProCassandraCore import TableProPluginKit +private let logger = Logger(subsystem: "com.TablePro.CassandraDriver", category: "Browse") + extension CassandraPluginDriver { func buildBrowseQuery( table: String, @@ -125,7 +129,19 @@ extension CassandraPluginDriver { ) async throws -> PluginQueryResult { let cancellation = activeBrowse.begin(consumerLeft: consumerLeft) defer { activeBrowse.end(cancellation) } - let raw = try await connectionActor.executeBrowse(browse, consistency: consistency, cancellation: cancellation) + let capture = try await lifetimeCapture(for: browse, cancellation: cancellation) + // Taken before the request, so the time a kept TTL has run counts the whole request. + let readAt = ContinuousClock.now + let run = try await connectionActor.executeBrowse( + capture.map { browse.sending($0.plan.cql) } ?? browse, + fallback: capture == nil ? nil : browse, + consistency: consistency, + cancellation: cancellation + ) + var raw = run.result + if let capture, !run.ranFallback { + raw = keepLifetimes(of: raw, capture: capture, readAt: readAt) + } let isTruncated = rowCap.map { raw.rows.count > $0 } ?? false let rows = isTruncated ? Array(raw.rows.prefix(rowCap ?? raw.rows.count)) : raw.rows return PluginQueryResult( @@ -166,6 +182,134 @@ extension CassandraPluginDriver { } } + private struct LifetimeCapture: Sendable { + let plan: CassandraLifetimeRead.Plan + let table: CassandraTableIdentity + let layout: CassandraTableLayout + } + + // A browse from the first row reads the table's columns again, so a column added since is shown. A read larger + // than the store holds, such as a copy or an export, runs as written. + private func lifetimeCapture( + for browse: CassandraBrowseStatement, + cancellation: CassandraCancellation + ) async throws -> LifetimeCapture? { + let window = browse.window + guard window.refusal == nil, window.limit <= CassandraCellLifetimeStore.maximumRows, + let tableName = window.table, + let keyspace = window.keyspace.flatMap({ $0.isEmpty ? nil : $0 }) ?? currentSchema + else { return nil } + let table = CassandraTableIdentity(keyspace: keyspace, table: tableName) + guard let layout = try await tableLayout(table, refresh: window.offset == 0, cancellation: cancellation), + let plan = CassandraLifetimeRead.plan( + cql: browse.cql, + target: CassandraBrowseRenderer.qualifiedTable(keyspace: window.keyspace, table: tableName), + layout: layout + ) + else { return nil } + return LifetimeCapture(plan: plan, table: table, layout: layout) + } + + private func tableLayout( + _ table: CassandraTableIdentity, + refresh: Bool, + cancellation: CassandraCancellation + ) async throws -> CassandraTableLayout? { + if !refresh, let known = cellLifetimes.layout(of: table) { return known.layout } + let columns: CassandraRawResult + let options: CassandraRawResult + do { + columns = try await connectionActor.executeQuery( + CassandraTableLayout.columnsQuery(keyspace: table.keyspace, table: table.table), + consistency: nil, + cancellation: cancellation + ) + options = try await connectionActor.executeQuery( + CassandraTableLayout.defaultTimeToLiveQuery(keyspace: table.keyspace, table: table.table), + consistency: nil, + cancellation: cancellation + ) + } catch let error as CancellationError { + throw error + } catch { + logger.info("Browse runs without TTL selectors, the table's columns were not read: \(error.localizedDescription)") + return nil + } + let layout = Self.layout(columns: columns.rows, options: options.rows) + cellLifetimes.setLayout(layout, of: table) + return layout + } + + private static func layout(columns: [[PluginCellValue]], options: [[PluginCellValue]]) -> CassandraTableLayout? { + var parsed: [CassandraTableLayout.Column] = [] + for row in columns { + guard let column = CassandraTableLayout.column( + name: row[safe: 0]?.asText, + type: row[safe: 1]?.asText, + kind: row[safe: 2]?.asText, + position: row[safe: 3]?.asText + ) else { return nil } + parsed.append(column) + } + let defaultTimeToLive = options.first?[safe: 0]?.asText.flatMap { Int($0) } + return CassandraTableLayout(columns: parsed, defaultTimeToLive: defaultTimeToLive) + } + + // The host sees the columns it asked for. Every row read replaces what was kept for its key, or drops it. + private func keepLifetimes( + of raw: CassandraRawResult, + capture: LifetimeCapture, + readAt: ContinuousClock.Instant + ) -> CassandraRawResult { + let plan = capture.plan + let shownCount = plan.shownColumns.count + guard raw.columns.count == shownCount + plan.lifetimeColumns.count else { return raw } + var read: [(row: CassandraRowIdentity, lifetimes: CassandraRowLifetimes?)] = [] + read.reserveCapacity(raw.rows.count) + let rows = raw.rows.map { row -> [PluginCellValue] in + if let key = Self.key(of: row, plan: plan) { + read.append(( + CassandraRowIdentity(table: capture.table, key: key), + Self.lifetimes(of: row, capture: capture, readAt: readAt) + )) + } + return Array(row.prefix(shownCount)) + } + cellLifetimes.keep(read) + return CassandraRawResult( + columns: Array(raw.columns.prefix(shownCount)), + columnTypeNames: Array(raw.columnTypeNames.prefix(shownCount)), + rows: rows, + rowsAffected: raw.rowsAffected, + executionTime: raw.executionTime + ) + } + + private static func key(of row: [PluginCellValue], plan: CassandraLifetimeRead.Plan) -> [String: PluginCellValue]? { + var key: [String: PluginCellValue] = [:] + for (column, index) in plan.keyIndices { + guard index < row.count, !row[index].isNull else { return nil } + key[column] = row[index] + } + return key + } + + private static func lifetimes( + of row: [PluginCellValue], + capture: LifetimeCapture, + readAt: ContinuousClock.Instant + ) -> CassandraRowLifetimes? { + let plan = capture.plan + let lifetimes = plan.lifetimeValueIndices.enumerated().map { selector, valueIndex -> CassandraCellLifetime? in + let timeToLiveIndex = plan.shownColumns.count + selector + guard valueIndex < row.count, timeToLiveIndex < row.count else { return nil } + return CassandraCellLifetime(valueIsNull: row[valueIndex].isNull, timeToLive: row[timeToLiveIndex].asText) + } + return CassandraRowLifetimes( + columns: plan.lifetimeColumns, lifetimes: lifetimes, readAt: readAt, layout: capture.layout + ) + } + private static func matchesAll(_ logicMode: String) -> Bool { logicMode.lowercased() != "or" } @@ -203,3 +347,64 @@ final class CassandraActiveBrowse: @unchecked Sendable { running?.cancel() } } + +struct CassandraTableIdentity: Hashable, Sendable { + let keyspace: String + let table: String +} + +struct CassandraRowIdentity: Hashable, Sendable { + let table: CassandraTableIdentity + let key: [String: PluginCellValue] +} + +// What the browses on one connection read about cell lifetimes, so a later edit or restore writes each cell with +// the TTL it had left. Rows read longest ago drop out first. +final class CassandraCellLifetimeStore: @unchecked Sendable { + static let maximumRows = 20_000 + static let maximumTables = 64 + + // A table system_schema lists no layout for is remembered too, so its browse pages ask once. + struct KnownLayout: Sendable { + let layout: CassandraTableLayout? + } + + private let lock = NSLock() + private var rows = CassandraRecentEntries( + capacity: CassandraCellLifetimeStore.maximumRows + ) + private var layouts = CassandraRecentEntries( + capacity: CassandraCellLifetimeStore.maximumTables + ) + + func layout(of table: CassandraTableIdentity) -> KnownLayout? { + lock.withLock { layouts[table] } + } + + func setLayout(_ layout: CassandraTableLayout?, of table: CassandraTableIdentity) { + lock.withLock { layouts.set(KnownLayout(layout: layout), for: table) } + } + + func keep(_ read: [(row: CassandraRowIdentity, lifetimes: CassandraRowLifetimes?)]) { + lock.withLock { + for entry in read { + if let lifetimes = entry.lifetimes { + rows.set(lifetimes, for: entry.row) + } else { + rows.removeValue(for: entry.row) + } + } + } + } + + func lifetimes(of row: CassandraRowIdentity) -> CassandraRowLifetimes? { + lock.withLock { rows[row] } + } + + func removeAll() { + lock.withLock { + rows.removeAll() + layouts.removeAll() + } + } +} diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+RowWrites.swift b/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+RowWrites.swift index 75d0a49af6..4d509259a8 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+RowWrites.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraPluginDriver+RowWrites.swift @@ -4,6 +4,7 @@ // import Foundation +import TableProCassandraCore import TableProPluginKit extension CassandraPluginDriver { @@ -23,7 +24,8 @@ extension CassandraPluginDriver { columns: columns, primaryKeyColumns: primaryKeyColumns, changes: changes, - insertedRowData: insertedRowData + insertedRowData: insertedRowData, + lifetimes: keptLifetimes(keyspace: schema, table: table) ) } @@ -35,6 +37,24 @@ extension CassandraPluginDriver { rows: [[PluginCellValue]], absentCells: [Int: Set] ) -> [(statement: String, parameters: [PluginCellValue])]? { - CassandraRowWriter.restoreInserts(keyspace: schema, table: table, columns: columns, rows: rows) + CassandraRowWriter.restoreInserts( + keyspace: schema, + table: table, + columns: columns, + primaryKeyColumns: primaryKeyColumns, + rows: rows, + lifetimes: keptLifetimes(keyspace: schema, table: table) + ) + } + + // An unqualified write lands in the session's keyspace, the one an unqualified browse read. + private func keptLifetimes(keyspace schema: String?, table: String) -> CassandraRowWriter.Lifetimes { + let keyspace = schema.flatMap { $0.isEmpty ? nil : $0 } ?? currentSchema + return { [cellLifetimes] key in + guard let keyspace else { return nil } + return cellLifetimes.lifetimes( + of: CassandraRowIdentity(table: CassandraTableIdentity(keyspace: keyspace, table: table), key: key) + ) + } } } diff --git a/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift b/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift index 0f9731a695..be924c7b54 100644 --- a/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift +++ b/apple/Plugins/CassandraDriverPlugin/CassandraRowWriter.swift @@ -13,16 +13,22 @@ import TableProPluginKit enum CassandraRowWriter { static let unappliedMarker = CassandraConditionalWrite.appliedColumn + // What a browse kept for the row with these primary key values, by column name. + typealias Lifetimes = ([String: PluginCellValue]) -> CassandraRowLifetimes? + static func rowWrites( keyspace: String?, table: String, columns: [String], primaryKeyColumns: [String], changes: [PluginRowChange], - insertedRowData: [Int: [PluginCellValue]] + insertedRowData: [Int: [PluginCellValue]], + lifetimes: @escaping Lifetimes = { _ in nil } ) throws -> [PluginRowWrite] { let target = CassandraBrowseRenderer.qualifiedTable(keyspace: keyspace, table: table) - let writer = Writer(target: target, columns: columns, primaryKeyColumns: primaryKeyColumns) + let writer = Writer( + target: target, columns: columns, primaryKeyColumns: primaryKeyColumns, lifetimes: lifetimes + ) return try changes.compactMap { change in switch change.type { case .insert: @@ -35,23 +41,73 @@ enum CassandraRowWriter { } } - /// Writes deleted rows back as they were, key included, to undo a delete. A plain `INSERT` rather than - /// `IF NOT EXISTS`: restoring a row means making it what it was, whatever took its place since. + // Writes deleted rows back as they were, key included and each cell with the TTL it had left, to undo a delete. + // A plain `INSERT` rather than `IF NOT EXISTS`: restoring a row means making it what it was, whatever took its + // place since. static func restoreInserts( keyspace: String?, table: String, columns: [String], - rows: [[PluginCellValue]] + primaryKeyColumns: [String], + rows: [[PluginCellValue]], + lifetimes: Lifetimes = { _ in nil } ) -> [(statement: String, parameters: [PluginCellValue])] { let target = CassandraBrowseRenderer.qualifiedTable(keyspace: keyspace, table: table) + let now = ContinuousClock.now return rows.map { row in let width = min(row.count, columns.count) - let names = columns.prefix(width).map(CassandraBrowseRenderer.quote).joined(separator: ", ") - let markers = Array(repeating: "?", count: width).joined(separator: ", ") - return ("INSERT INTO \(target) (\(names)) VALUES (\(markers))", Array(row.prefix(width))) + let names = Array(columns.prefix(width)) + let values = Array(row.prefix(width)) + let kept = rowKey(primaryKeyColumns, columns: names, row: values).flatMap(lifetimes) + let keys = kept.flatMap { kept -> CassandraLifetimeWrite.RowKeys? in + guard let whole = condition(naming: primaryKeyColumns, columns: names, row: values), + let partition = condition(naming: kept.layout.partitionKey, columns: names, row: values) + else { return nil } + return CassandraLifetimeWrite.RowKeys(row: whole, partition: partition) + } + let cells = zip(names, values).map { name, value in + CassandraLifetimeWrite.RestoredCell( + column: name, + value: value, + isKey: primaryKeyColumns.contains(name), + isNull: value.isNull, + isStatic: kept?.layout.staticColumns.contains(name) ?? false, + timeToLive: kept?.timeToLive(of: name, at: now) + ) + } + let write = CassandraLifetimeWrite.restore(target: target, cells: cells, keys: keys) + return (write.cql, write.parameters) } } + static func rowKey( + _ keyColumns: [String], + columns: [String], + row: [PluginCellValue] + ) -> [String: PluginCellValue]? { + guard !keyColumns.isEmpty else { return nil } + var key: [String: PluginCellValue] = [:] + for keyColumn in keyColumns { + guard let index = columns.firstIndex(of: keyColumn), index < row.count, !row[index].isNull else { + return nil + } + key[keyColumn] = row[index] + } + return key + } + + static func condition( + naming keyColumns: [String], + columns: [String], + row: [PluginCellValue] + ) -> CassandraLifetimeWrite.KeyCondition? { + guard let key = rowKey(keyColumns, columns: columns, row: row) else { return nil } + return CassandraLifetimeWrite.KeyCondition( + cql: keyColumns.map { "\(CassandraBrowseRenderer.quote($0)) = ?" }.joined(separator: " AND "), + values: keyColumns.compactMap { key[$0] } + ) + } + static func unappliedWriteRefusal(statement: String, columns: [String], rows: [[PluginCellValue]]) -> String? { let refusal = CassandraConditionalWrite.refusal( statement: statement, firstColumn: columns.first, firstValue: rows.first?.first?.asText @@ -70,6 +126,7 @@ enum CassandraRowWriter { let target: String let columns: [String] let primaryKeyColumns: [String] + let lifetimes: Lifetimes func insert(_ change: PluginRowChange, values: [PluginCellValue]?) throws -> PluginRowWrite { guard !primaryKeyColumns.isEmpty else { @@ -112,8 +169,11 @@ enum CassandraRowWriter { func update(_ change: PluginRowChange) throws -> PluginRowWrite? { guard !change.cellChanges.isEmpty else { return nil } let expressionColumns = change.expressionColumns ?? [] - var assignments: [String] = [] - var parameters: [PluginCellValue] = [] + let kept = change.originalRow + .flatMap { CassandraRowWriter.rowKey(primaryKeyColumns, columns: columns, row: $0) } + .flatMap(lifetimes) + let now = ContinuousClock.now + var assignments: [CassandraLifetimeWrite.Assignment] = [] for cell in change.cellChanges { if primaryKeyColumns.contains(cell.columnName) { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( @@ -128,19 +188,31 @@ enum CassandraRowWriter { localized: "Cassandra columns have no default value." )) } - let column = CassandraBrowseRenderer.quote(cell.columnName) - if let call = expression(cell.newValue, isMarked: expressionColumns.contains(cell.columnIndex)) { - assignments.append("\(column) = \(call)") - } else { - assignments.append("\(column) = ?") - parameters.append(cell.newValue) + if kept?.hasExpired(cell.columnName, at: now) == true { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( + format: String( + localized: "The value in %@ expired after the grid read it. Refresh, then edit the row again." + ), + cell.columnName + )) } + let column = CassandraBrowseRenderer.quote(cell.columnName) + let call = expression(cell.newValue, isMarked: expressionColumns.contains(cell.columnIndex)) + assignments.append(CassandraLifetimeWrite.Assignment( + cql: "\(column) = \(call ?? "?")", + parameters: call == nil ? [cell.newValue] : [], + timeToLive: kept?.timeToLive(of: cell.columnName, at: now), + isStatic: kept?.layout.staticColumns.contains(cell.columnName) ?? false + )) } let key = try keyCondition(for: change) - let statement = CassandraConditionalWrite.updateStatement( - target: target, assignments: assignments, keyCondition: key.cql + let partition = change.originalRow.flatMap { original in + kept.flatMap { CassandraRowWriter.condition(naming: $0.layout.partitionKey, columns: columns, row: original) } + } + let write = CassandraLifetimeWrite.update( + target: target, assignments: assignments, row: key, partition: partition ) - return PluginRowWrite(statement: statement, parameters: parameters + key.values, rowIndices: [change.rowIndex]) + return PluginRowWrite(statement: write.cql, parameters: write.parameters, rowIndices: [change.rowIndex]) } func delete(_ change: PluginRowChange) throws -> PluginRowWrite { @@ -150,24 +222,15 @@ enum CassandraRowWriter { ) } - private func keyCondition(for change: PluginRowChange) throws -> (cql: String, values: [PluginCellValue]) { - guard !primaryKeyColumns.isEmpty, let original = change.originalRow else { + private func keyCondition(for change: PluginRowChange) throws -> CassandraLifetimeWrite.KeyCondition { + guard let original = change.originalRow, + let key = CassandraRowWriter.condition(naming: primaryKeyColumns, columns: columns, row: original) + else { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( localized: "The row's primary key is not loaded, so the row cannot be found." )) } - var conditions: [String] = [] - var values: [PluginCellValue] = [] - for keyColumn in primaryKeyColumns { - guard let index = columns.firstIndex(of: keyColumn), index < original.count, !original[index].isNull else { - throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: String( - localized: "The row's primary key is not loaded, so the row cannot be found." - )) - } - conditions.append("\(CassandraBrowseRenderer.quote(keyColumn)) = ?") - values.append(original[index]) - } - return (conditions.joined(separator: " AND "), values) + return key } private func rowFromCellChanges(_ change: PluginRowChange) -> [PluginCellValue] { diff --git a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver+Execution.swift b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver+Execution.swift index 63a0b38d19..437740eb29 100644 --- a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver+Execution.swift +++ b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver+Execution.swift @@ -193,6 +193,7 @@ extension ElasticsearchPluginDriver { """) let hits = try await fetchHits(index: parsed.index, parsed: parsed, fields: fields, conn: conn) + rememberArrayText(in: hits, index: parsed.index, mappingColumns: mappingColumns) return renderHits(hits, mappingColumns: mappingColumns, fields: fields, length: length, startTime: startTime) } @@ -396,7 +397,7 @@ extension ElasticsearchPluginDriver { return fields[column]?.type ?? "" } } - return PluginQueryResult( + var result = PluginQueryResult( columns: columns, columnTypeNames: typeNames, rows: rows, @@ -404,6 +405,8 @@ extension ElasticsearchPluginDriver { executionTime: Date().timeIntervalSince(startTime), columnMeta: ElasticsearchTypeClassification.columnMeta(columns: columns, typeNames: typeNames) ) + result.rowLocators = hits.map { ElasticsearchDocumentLocator(hit: $0)?.locator } + return result } private func renderObjects( diff --git a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift index bb48629513..bf836cfae5 100644 --- a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift +++ b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchPluginDriver.swift @@ -7,6 +7,7 @@ import Foundation import os +import TableProDocumentPath import TableProPluginKit internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked Sendable { @@ -14,6 +15,7 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked private let lock = NSLock() private var _connection: ElasticsearchConnection? private var _mappingCache: [String: [ElasticsearchColumn]] = [:] + private var _arrayTextColumns: [String: Set] = [:] static let logger = Logger(subsystem: "com.TablePro", category: "ElasticsearchPluginDriver") static let maxResultWindow = 10_000 @@ -58,6 +60,7 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked _connection?.disconnect() _connection = nil _mappingCache.removeAll() + _arrayTextColumns.removeAll() } } @@ -247,6 +250,24 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set + ) throws -> [PluginRowWrite]? { + try generateRowWrites( + table: table, schema: schema, columns: columns, primaryKeyColumns: primaryKeyColumns, changes: changes, + insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, + insertedRowIndices: insertedRowIndices, context: PluginRowWriteContext() + ) + } + + func generateRowWrites( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set, + context: PluginRowWriteContext ) throws -> [PluginRowWrite]? { if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes.filter { $0.type != .insert }) { throw refusal @@ -258,7 +279,9 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked index: table, columns: columns, columnTypeNames: typeNames, - columnPathKeys: columnPathKeys(index: table) + columnPathKeys: columnPathKeys(index: table), + rowLocators: context.rowLocators, + arrayTextColumns: lock.withLock { _arrayTextColumns[table] } ?? [] ) return try generator.generateRowWrites( from: changes, @@ -308,7 +331,28 @@ internal final class ElasticsearchPluginDriver: PluginDatabaseDriver, @unchecked /// nothing. The whole cache goes rather than one index: a console request can name a list, a /// wildcard or an alias, and refilling it costs one `GET //_mapping` per index reopened. func invalidateMappingCache() { - lock.withLock { _mappingCache.removeAll() } + lock.withLock { + _mappingCache.removeAll() + _arrayTextColumns.removeAll() + } + } + + // The grid shows a string and an array that read the same alike, so a new row's text in such a field stays text. + func rememberArrayText(in hits: [[String: Any]], index: String, mappingColumns: [ElasticsearchColumn]) { + let stringColumns = mappingColumns.filter { ElasticsearchStatementGenerator.stringTypes.contains($0.type) } + guard !stringColumns.isEmpty else { return } + var found: Set = [] + for hit in hits { + guard let source = hit["_source"] as? [String: Any] else { continue } + for column in stringColumns where !found.contains(column.name) { + if let text = DocumentPath.value(in: source, atPath: column.name) as? String, + ElasticsearchStatementGenerator.parsedArray(text) != nil { + found.insert(column.name) + } + } + } + guard !found.isEmpty else { return } + lock.withLock { _arrayTextColumns[index, default: []].formUnion(found) } } private func columnPathKeys(index: String) -> [String: [String]] { diff --git a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift index 0bfc05bcd3..548e7ed1e6 100644 --- a/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift +++ b/apple/Plugins/ElasticsearchDriverPlugin/ElasticsearchStatementGenerator.swift @@ -17,12 +17,54 @@ struct ElasticsearchWriteRequest: Equatable { // The fields a partial update sets, each as its mapping keys. The stored document decides how // each one is spelled, so the request is aligned to it when it runs. let leafPaths: [[String]] + // Leaves of `leafPaths` sent as text that reads as a JSON array. The grid shows the string `["a"]` and the array + // `["a"]` alike, so the stored document decides which one the edit is. + let arrayCandidates: [[String]] - init(method: String, path: String, body: String?, leafPaths: [[String]] = []) { + init(method: String, path: String, body: String?, leafPaths: [[String]] = [], arrayCandidates: [[String]] = []) { self.method = method self.path = path self.body = body self.leafPaths = leafPaths + self.arrayCandidates = arrayCandidates + } +} + +// Where a hit lives: its concrete index (a search can go through an alias or a wildcard) and, for a document indexed +// with custom routing, the routing that finds its shard again. With custom routing an `_id` is unique only per shard. +struct ElasticsearchDocumentLocator: Equatable { + let index: String + let id: String + let routing: String? + + private static let indexKey = "_index" + private static let idKey = "_id" + private static let routingKey = "_routing" + + init(index: String, id: String, routing: String?) { + self.index = index + self.id = id + self.routing = routing + } + + init?(hit: [String: Any]) { + guard let index = hit[Self.indexKey] as? String, let id = hit[Self.idKey] as? String else { return nil } + self.init(index: index, id: id, routing: hit[Self.routingKey] as? String) + } + + init?(locator: String) { + guard let object = try? JSONSerialization.jsonObject(with: Data(locator.utf8)) as? [String: Any] else { + return nil + } + self.init(hit: object) + } + + // Sorted keys, so the same hit always gives the same text. + var locator: String? { + var object: [String: Any] = [Self.indexKey: index, Self.idKey: id] + if let routing { object[Self.routingKey] = routing } + guard let data = try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys]) else { return nil } + return String(data: data, encoding: .utf8) } } @@ -32,10 +74,16 @@ struct ElasticsearchStatementGenerator { // Plain `PUT _doc/` replaces a document that already has the id. private static let createOnlyQuery = "&op_type=create" private static let updateEndpoint = "_update" + private static let documentEndpoint = "_doc" + private static let routingParameter = "routing=" let index: String let columns: [String] let columnTypeNames: [String] + // By row index, the `ElasticsearchDocumentLocator` text each hit was read with. + let rowLocators: [Int: String] + // String fields some hit held a string reading as a JSON array in. + let arrayTextColumns: Set /// A `nested` column carries the whole array of objects, and its dotted leaves are views of /// those same bytes. Writing both makes Elasticsearch expand the dotted key into the object the @@ -45,11 +93,20 @@ struct ElasticsearchStatementGenerator { private let nestedParentByLeaf: [String: String] private let columnPathKeys: [String: [String]] - init(index: String, columns: [String], columnTypeNames: [String], columnPathKeys: [String: [String]] = [:]) { + init( + index: String, + columns: [String], + columnTypeNames: [String], + columnPathKeys: [String: [String]] = [:], + rowLocators: [Int: String] = [:], + arrayTextColumns: Set = [] + ) { self.index = index self.columns = columns self.columnTypeNames = columnTypeNames self.columnPathKeys = columnPathKeys + self.rowLocators = rowLocators + self.arrayTextColumns = arrayTextColumns self.nestedParentByLeaf = Self.nestedParents(columns: columns, typeNames: columnTypeNames) } @@ -202,12 +259,11 @@ struct ElasticsearchStatementGenerator { private func updateRequest(for change: PluginRowChange) throws -> ElasticsearchWriteRequest? { guard !change.cellChanges.isEmpty else { return nil } - guard let id = documentId(from: change) else { - throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.missingIdReason) - } + let target = try document(for: change) var doc: [String: Any] = [:] var leafPaths: [[String]] = [] + var arrayCandidates: [[String]] = [] for cellChange in change.cellChanges { let column = cellChange.columnName if metaColumns.contains(column) { @@ -222,8 +278,15 @@ struct ElasticsearchStatementGenerator { Self.isShortened(cellChange.newValue) || Self.isShortened(cellChange.oldValue) { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.shortenedValueReason(column)) } - let value: Any = cellChange.newValue.asText.map { jsonValue($0, for: column) } ?? NSNull() let keys = pathKeys(for: column) + let text = cellChange.newValue.asText + let value: Any + if let text, holdsTextOrArray(column), Self.parsedArray(text) != nil { + value = text + arrayCandidates.append(keys) + } else { + value = text.map { jsonValue($0, for: column) } ?? NSNull() + } guard DocumentPathPlacement.set(value, atKeys: keys, in: &doc) else { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.notJSONReason) } @@ -235,9 +298,10 @@ struct ElasticsearchStatementGenerator { } return .init( method: "POST", - path: "/\(encodedIndex)/\(Self.updateEndpoint)/\(encodePathComponent(id))\(Self.refreshQuery)", + path: documentPath(Self.updateEndpoint, for: target), body: body, - leafPaths: leafPaths + leafPaths: leafPaths, + arrayCandidates: arrayCandidates ) } @@ -247,11 +311,16 @@ struct ElasticsearchStatementGenerator { request.path.hasSuffix(createOnlyQuery) } + // A routed document is found only on its routing's shard, so the read carries the update's routing. static func storedDocumentPath(for request: ElasticsearchWriteRequest) -> String? { guard !request.leafPaths.isEmpty else { return nil } - let parts = (request.path.split(separator: "?", maxSplits: 1).first ?? "").split(separator: "/") + let pathAndQuery = request.path.split(separator: "?", maxSplits: 1) + let parts = (pathAndQuery.first ?? "").split(separator: "/") guard parts.count == 3, parts[1] == updateEndpoint else { return nil } - return "/\(parts[0])/_doc/\(parts[2])" + let routing = pathAndQuery.dropFirst().first? + .split(separator: "&") + .first(where: { $0.hasPrefix(routingParameter) }) + return "/\(parts[0])/\(documentEndpoint)/\(parts[2])" + (routing.map { "?\($0)" } ?? "") } static func aligned( @@ -267,7 +336,11 @@ struct ElasticsearchStatementGenerator { var placed: [String: Any] = [:] for keys in request.leafPaths { - guard let value = DocumentPathPlacement.value(in: doc, atKeys: keys) else { return request } + guard var value = DocumentPathPlacement.value(in: doc, atKeys: keys) else { return request } + if request.arrayCandidates.contains(keys), let text = value as? String, let array = parsedArray(text), + storedHoldsArray(atKeys: keys, in: stored) { + value = array + } try DocumentPathPlacement.place(value, atKeys: keys, in: &placed, following: stored) } guard let alignedBody = serialize(["doc": placed]) else { return request } @@ -279,13 +352,33 @@ struct ElasticsearchStatementGenerator { return ElasticsearchWriteRequest(method: request.method, path: path, body: alignedBody) } + // Every spelling the stored document has for the field holds an array. + private static func storedHoldsArray(atKeys keys: [String], in stored: [String: Any]) -> Bool { + let atoms = keys.flatMap { $0.split(separator: ".", omittingEmptySubsequences: false).map(String.init) } + let found = storedValues(of: atoms[...], in: stored) + return !found.isEmpty && found.allSatisfy { $0 is [Any] } + } + + // The same walk as `DocumentPathPlacement.place`: a level can spell the next part of the path as one dotted key. + private static func storedValues(of atoms: ArraySlice, in level: [String: Any]) -> [Any] { + var found: [Any] = [] + for end in atoms.indices.map({ $0 + 1 }) { + let key = atoms[atoms.startIndex.. ElasticsearchWriteRequest { - guard let id = documentId(from: change) else { - throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.missingIdReason) - } - return .init(method: "DELETE", path: docPath(id: id), body: nil) + let target = try document(for: change) + return .init(method: "DELETE", path: documentPath(Self.documentEndpoint, for: target), body: nil) } // MARK: - Refusals @@ -317,6 +410,20 @@ struct ElasticsearchStatementGenerator { // MARK: - Helpers + // A row read from a table tab carries its hit's locator. Without one, the tab's index and the `_id` cell. + private func document(for change: PluginRowChange) throws -> ElasticsearchDocumentLocator { + if let locator = rowLocators[change.rowIndex] { + guard let document = ElasticsearchDocumentLocator(locator: locator) else { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.missingIdReason) + } + return document + } + guard let id = documentId(from: change) else { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: Self.missingIdReason) + } + return ElasticsearchDocumentLocator(index: index, id: id, routing: nil) + } + private func documentId(from change: PluginRowChange) -> String? { guard let originalRow = change.originalRow, let idIndex = columns.firstIndex(of: ElasticsearchMappingFlattener.idColumn), @@ -332,7 +439,13 @@ struct ElasticsearchStatementGenerator { } private func docPath(id: String) -> String { - "/\(encodedIndex)/_doc/\(encodePathComponent(id))\(Self.refreshQuery)" + documentPath(Self.documentEndpoint, for: ElasticsearchDocumentLocator(index: index, id: id, routing: nil)) + } + + private func documentPath(_ endpoint: String, for document: ElasticsearchDocumentLocator) -> String { + let path = "/\(encodePathComponent(document.index))/\(endpoint)/\(encodePathComponent(document.id))\(Self.refreshQuery)" + guard let routing = document.routing else { return path } + return path + "&" + Self.routingParameter + Self.encodeQueryValue(routing) } private var encodedIndex: String { @@ -341,17 +454,49 @@ struct ElasticsearchStatementGenerator { private static let pathComponentAllowed: CharacterSet = .urlPathAllowed.subtracting(CharacterSet(charactersIn: "/")) + // RFC 3986 unreserved: `&`, `=`, `+`, `#` and `%` in a routing value would end or change the parameter. + private static let queryValueAllowed = CharacterSet( + charactersIn: "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~" + ) private static let structuredTypes: Set = ["object", "nested", "flattened", "join"] + static let stringTypes: Set = [ + "text", "keyword", "wildcard", "constant_keyword", "match_only_text", "search_as_you_type", "annotated_text" + ] private func encodePathComponent(_ value: String) -> String { value.addingPercentEncoding(withAllowedCharacters: Self.pathComponentAllowed) ?? value } - private func jsonValue(_ text: String, for column: String) -> Any { - let typeName = columns.firstIndex(of: column).flatMap { index in + private static func encodeQueryValue(_ value: String) -> String { + value.addingPercentEncoding(withAllowedCharacters: queryValueAllowed) ?? value + } + + private func mappedType(of column: String) -> String { + columns.firstIndex(of: column).flatMap { index in index < columnTypeNames.count ? columnTypeNames[index] : nil } ?? "" + } + + // A string field, or one whose type is unknown, can hold a string that reads as a JSON array. + private func holdsTextOrArray(_ column: String) -> Bool { + let typeName = mappedType(of: column) + return typeName.isEmpty || Self.stringTypes.contains(typeName) + } + + // JSON allows white space before the bracket; anything else is not an array and is never copied and parsed. + static func parsedArray(_ text: String) -> [Any]? { + guard text.unicodeScalars.first(where: { !" \t\n\r".unicodeScalars.contains($0) }) == "[" else { return nil } + return try? JSONSerialization.jsonObject(with: Data(text.utf8)) as? [Any] + } + + private func jsonValue(_ text: String, for column: String) -> Any { + let typeName = mappedType(of: column) + if Self.stringTypes.contains(typeName) { + // A new row has no stored value to compare, so an array is read as one unless the field held such text. + if !arrayTextColumns.contains(column), let array = Self.parsedArray(text) { return array } + return text + } if ElasticsearchQueryBuilder.numericTypes.contains(typeName) { if let intVal = Int(text) { return intVal } if let doubleVal = Double(text) { return doubleVal } @@ -385,7 +530,11 @@ struct ElasticsearchStatementGenerator { guard !request.leafPaths.isEmpty, let leafData = try? JSONSerialization.data(withJSONObject: request.leafPaths) else { return statement } - return "\(statement):\(leafData.base64EncodedString())" + let withLeaves = "\(statement):\(leafData.base64EncodedString())" + guard !request.arrayCandidates.isEmpty, + let candidateData = try? JSONSerialization.data(withJSONObject: request.arrayCandidates) + else { return withLeaves } + return "\(withLeaves):\(candidateData.base64EncodedString())" } static func decode(_ statement: String) -> ElasticsearchWriteRequest? { @@ -396,17 +545,19 @@ struct ElasticsearchStatementGenerator { let path = decodeBase64(parts[1]) else { return nil } let body = decodeBase64(parts[2]) - let leafPaths = parts.count > 3 - ? Data(base64Encoded: parts[3]).flatMap { try? JSONSerialization.jsonObject(with: $0) as? [[String]] } - : nil return ElasticsearchWriteRequest( method: method, path: path, body: (body?.isEmpty ?? true) ? nil : body, - leafPaths: leafPaths ?? [] + leafPaths: parts.count > 3 ? decodePaths(parts[3]) : [], + arrayCandidates: parts.count > 4 ? decodePaths(parts[4]) : [] ) } + private static func decodePaths(_ part: String) -> [[String]] { + Data(base64Encoded: part).flatMap { try? JSONSerialization.jsonObject(with: $0) as? [[String]] } ?? [] + } + static func isTaggedStatement(_ statement: String) -> Bool { statement.hasPrefix(writeTag) } diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaBrowseEngine.swift b/apple/Plugins/KafkaDriverPlugin/KafkaBrowseEngine.swift index 2273558179..00276ccaa8 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaBrowseEngine.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaBrowseEngine.swift @@ -9,6 +9,8 @@ struct KafkaScanWindow: Sendable { let tail: [Int32: Int64] /// True when the page is the newest rows of the window rather than the oldest. let readsBackward: Bool + // The earliest offset of each partition a tail scan may walk back to. Empty for a forward scan. + var floor: [Int32: Int64] = [:] } struct KafkaBrowsePage: Sendable { @@ -37,6 +39,10 @@ enum KafkaBrowseEngine { /// slicing is the honest way to answer that, but it has to be bounded or a deep page would /// pull an unbounded amount of the log into memory. private static let maximumOverFetch = 50_000 + // Past the page's own range a walk reads at most this many bytes per partition: a tail of aborted transactions, + // or a range that runs from an empty stretch into a dense one, would otherwise be read without bound. The page is + // then cut short and says so. + private static let tailWalkByteBudget = 256 * 1_024 * 1_024 static func consume(_ query: KafkaConsumeQuery, cluster: KafkaCluster) async throws -> KafkaBrowsePage { let metadata = try await cluster.metadata(topics: [query.topic]) @@ -58,10 +64,46 @@ enum KafkaBrowseEngine { // Each partition contributes at most `wanted` records, because the merge cannot know // in advance how the messages are distributed: one partition may hold the whole page. var collected: [KafkaRecord] = [] - var truncated = false + var truncated = query.skip + query.limit > maximumOverFetch + var walkedStarts = window.start for partition in selected { try Task.checkCancellation() guard let start = window.start[partition] else { continue } + // A tail scan must not read past the end it was anchored to, or page two would show + // messages produced after page one and the window would not be a window at all. + if window.readsBackward, let end = window.tail[partition] { + var walk = KafkaTailWalk( + wanted: wanted, + end: end, + firstStart: start, + floor: window.floor[partition] ?? start + ) + // The page's own range is read as before; the ranges past it share the budget. + var walkBytesLeft: Int? + while let range = walk.nextRange { + try Task.checkCancellation() + let result = try await KafkaFetchRequest.fetch( + topic: query.topic, + partition: partition, + startOffset: range.start, + maximumRecords: walk.needed, + endOffset: range.end, + keepingNewest: walk.needed, + byteBudget: walkBytesLeft, + cluster: cluster + ) + if result.stoppedAtByteBudget { + truncated = true + break + } + walkBytesLeft = walkBytesLeft.map { $0 - result.bytesRead } ?? tailWalkByteBudget + if result.truncated { truncated = true } + walk.absorb(result.records) + } + collected.append(contentsOf: walk.records) + walkedStarts[partition] = walk.start + continue + } let result = try await KafkaFetchRequest.fetch( topic: query.topic, partition: partition, @@ -69,13 +111,7 @@ enum KafkaBrowseEngine { maximumRecords: wanted, cluster: cluster ) - var records = result.records - // A tail scan must not read past the end it was anchored to, or page two would show - // messages produced after page one and the window would not be a window at all. - if let end = window.tail[partition] { - records = records.filter { $0.offset < end } - } - collected.append(contentsOf: records) + collected.append(contentsOf: result.records) if result.truncated { truncated = true } } @@ -87,7 +123,10 @@ enum KafkaBrowseEngine { readsBackward: window.readsBackward ) if ordered.count > query.skip + query.limit { truncated = true } - return KafkaBrowsePage(records: page, window: window, truncated: truncated) + let walked = KafkaScanWindow( + start: walkedStarts, tail: window.tail, readsBackward: window.readsBackward, floor: window.floor + ) + return KafkaBrowsePage(records: page, window: walked, truncated: truncated) } /// The partitions to read, or an error naming the ones the topic does not have. @@ -132,14 +171,16 @@ enum KafkaBrowseEngine { cluster: cluster ) var starts: [Int32: Int64] = [:] + var floors: [Int32: Int64] = [:] for (partition, end) in kept { // No fallback offset. A partition whose earliest offset did not come back has // an unknown floor, and anchoring it at zero would send the page to the start of // a log whose first surviving message may be far past it. guard let floor = earliest[partition] else { continue } starts[partition] = KafkaOffsetMath.stepBack(from: end, by: step, floor: floor) + floors[partition] = floor } - return KafkaScanWindow(start: starts, tail: kept, readsBackward: true) + return KafkaScanWindow(start: starts, tail: kept, readsBackward: true, floor: floors) case .oldest: let starts = try await KafkaOffsetsRequest.listOffsets( @@ -200,11 +241,13 @@ enum KafkaBrowseEngine { // says and what the partition count would otherwise hide. var starts: [Int32: Int64] = [:] var ends: [Int32: Int64] = [:] + var floors: [Int32: Int64] = [:] for bound in bounds { starts[bound.partition] = KafkaOffsetMath.stepBack(from: bound.latest, by: step, floor: bound.earliest) ends[bound.partition] = bound.latest + floors[bound.partition] = bound.earliest } - return KafkaScanWindow(start: starts, tail: ends, readsBackward: true) + return KafkaScanWindow(start: starts, tail: ends, readsBackward: true, floor: floors) } } } diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaFetchRequest.swift b/apple/Plugins/KafkaDriverPlugin/KafkaFetchRequest.swift index 9365bfa506..0558392347 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaFetchRequest.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaFetchRequest.swift @@ -9,6 +9,13 @@ struct KafkaFetchResult: Sendable { /// return more. Distinguishing this from "the partition ended" is what keeps a page from /// silently stopping early. let truncated: Bool + // Where the next read starts, past control, aborted and compacted batches too. For one reply, nil when no batch + // came back whole; for a whole fetch, nil only when nothing can follow Int64.max. + let nextOffset: Int64? + // The replies passed `byteBudget` before the read reached its end, so it returns no records. + var stoppedAtByteBudget = false + // Record batch bytes the replies carried, aborted and control batches included. + var bytesRead = 0 } enum KafkaFetchRequest { @@ -20,39 +27,62 @@ enum KafkaFetchRequest { private static let defaultPartitionBudget: Int32 = 1 * 1_024 * 1_024 private static let maximumPartitionBudget: Int32 = 64 * 1_024 * 1_024 - // Kafka guarantees progress only for the first message in range (KIP-74), so the byte budget grows on an empty reply. + // Kafka guarantees progress only for the first message in range (KIP-74), so the byte budget grows on a reply + // that held no whole batch. `endOffset` stops the read there, so a tail window never reads past its anchor. + // With `keepingNewest` the read runs to `endOffset` and holds only that many of the newest records it passed. + // It gives up once the replies carried more than `byteBudget` bytes. static func fetch( topic: String, partition: Int32, startOffset: Int64, maximumRecords: Int, + endOffset: Int64? = nil, + keepingNewest: Int? = nil, + byteBudget: Int? = nil, cluster: KafkaCluster ) async throws -> KafkaFetchResult { var budget = defaultPartitionBudget var collected: [KafkaRecord] = [] var highWatermark: Int64 = startOffset - var offset = startOffset + var offset: Int64? = startOffset var sawTruncation = false var decompression: DecompressionBudget? + var bytesRead = 0 - while collected.count < maximumRecords { + while keepingNewest != nil || collected.count < maximumRecords, let from = offset { + if let endOffset, from >= endOffset { break } try Task.checkCancellation() let allowance = try KafkaCompression.fetchBudget(requesting: budget, after: decompression) let (page, spent) = try await fetchOnce( topic: topic, partition: partition, - startOffset: offset, + startOffset: from, budget: budget, decompression: allowance, cluster: cluster ) decompression = spent highWatermark = page.highWatermark + bytesRead += page.bytesRead + if let byteBudget, bytesRead > byteBudget { + return KafkaFetchResult( + records: [], highWatermark: page.highWatermark, truncated: true, nextOffset: nil, + stoppedAtByteBudget: true, bytesRead: bytesRead + ) + } if page.records.isEmpty { - // Nothing came back but the log says there is more: the next message does not + // Commit and abort markers, aborted runs and batches compaction emptied hold offsets but no + // records, so a reply of only those is progress, not a message too big for the budget. + if let next = page.nextOffset, next > from { + offset = next + budget = defaultPartitionBudget + if next >= page.highWatermark { break } + continue + } + // No whole batch came back but the log says there is more: the next message does not // fit the budget. Grow it and try once more before giving up on the partition. - let hasMore = offset < page.highWatermark + let hasMore = from < page.highWatermark if hasMore, budget < maximumPartitionBudget { budget = min(budget * 4, maximumPartitionBudget) continue @@ -61,18 +91,26 @@ enum KafkaFetchRequest { break } - collected.append(contentsOf: page.records) - guard let last = page.records.last, let next = KafkaOffsetMath.offset(after: last.offset) else { break } - offset = next + let kept = endOffset.map { end in page.records.filter { $0.offset < end } } ?? page.records + collected.append(contentsOf: kept) + if let keepingNewest, collected.count > keepingNewest { + collected.removeFirst(collected.count - keepingNewest) + } + let afterRecords = page.records.last.flatMap { KafkaOffsetMath.offset(after: $0.offset) } + offset = afterRecords.map { max($0, page.nextOffset ?? $0) } budget = defaultPartitionBudget - if offset >= page.highWatermark { break } + if let next = offset, next >= page.highWatermark { break } } - if collected.count > maximumRecords { + if keepingNewest == nil, collected.count > maximumRecords { collected = Array(collected.prefix(maximumRecords)) sawTruncation = true + offset = collected.last.flatMap { KafkaOffsetMath.offset(after: $0.offset) } } - return KafkaFetchResult(records: collected, highWatermark: highWatermark, truncated: sawTruncation) + return KafkaFetchResult( + records: collected, highWatermark: highWatermark, truncated: sawTruncation, nextOffset: offset, + bytesRead: bytesRead + ) } // Returns the decompression budget as the reply left it, so the next page continues from it. @@ -170,7 +208,7 @@ enum KafkaFetchRequest { if flexible { try partitionReader.taggedFields() } guard let blob, !blob.isEmpty else { - return KafkaFetchResult(records: [], highWatermark: highWatermark, truncated: false) + return KafkaFetchResult(records: [], highWatermark: highWatermark, truncated: false, nextOffset: nil) } let decoded = try KafkaRecordBatchDecoder.decode( blob: blob, @@ -185,7 +223,9 @@ enum KafkaFetchRequest { return KafkaFetchResult( records: trimmed, highWatermark: highWatermark, - truncated: decoded.truncated + truncated: decoded.truncated, + nextOffset: decoded.nextOffset, + bytesRead: blob.count ) } if flexible { try reader.taggedFields() } @@ -193,7 +233,7 @@ enum KafkaFetchRequest { } // One topic and one partition were requested, so there is one answer. let result = topics.flatMap { $0 }.first - ?? KafkaFetchResult(records: [], highWatermark: startOffset, truncated: false) + ?? KafkaFetchResult(records: [], highWatermark: startOffset, truncated: false, nextOffset: nil) return (result, spent) } } diff --git a/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver+LiveMessages.swift b/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver+LiveMessages.swift index 9f811a0625..4cf3de5658 100644 --- a/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver+LiveMessages.swift +++ b/apple/Plugins/KafkaDriverPlugin/KafkaPluginDriver+LiveMessages.swift @@ -99,11 +99,11 @@ final class KafkaLiveSubscription: PluginLiveSubscription, @unchecked Sendable { for record in page.records { continuation.yield(KafkaFollowCursor.message(from: record, topic: cursor.topic)) } - if let last = page.records.last { - guard let next = KafkaOffsetMath.offset(after: last.offset) else { return } - offset = next - } else if ContinuousClock.now - started < Self.idleWait / 2 { - // A trailing transaction marker answers at once and reads as no record, + guard let next = page.nextOffset else { return } + let moved = next > offset + offset = next + if !moved, ContinuousClock.now - started < Self.idleWait / 2 { + // A reply that answers at once with nothing to move past would spin, // where an idle partition makes the broker wait out the long poll. try await Task.sleep(for: Self.idleWait) } diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift index 813c3d6258..155c700630 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedError.swift @@ -12,6 +12,7 @@ enum MemcachedError: Error, PluginDriverError { case server(MemcachedServerError) case command(MemcachedCommandError) case filter(MemcachedFilterRefusal) + case compareAndSwap(MemcachedCASRefusal) case refused(String) case unexpectedReply(String) @@ -35,6 +36,10 @@ enum MemcachedError: Error, PluginDriverError { return Self.describe(error) case .filter(let refusal): return Self.describe(refusal) + case .compareAndSwap(.changed): + return String(localized: "The item changed since its CAS value was read, so nothing was saved. Refresh to see its current value.") + case .compareAndSwap(.notFound): + return String(localized: "The item does not exist, so nothing was saved. Refresh to see the current keys.") case .refused(let message): return message case .unexpectedReply(let line): diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift index 93d1cda577..058fdf7902 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedPluginDriver.swift @@ -23,8 +23,9 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private var keySource: MemcachedKeySource = .metadump private var queryTimeout = 0 private var shownExpiries: [String: MemcachedKnownExpiry] = [:] + private var shownItems: [String: MemcachedLoadedItem] = [:] private var serverClockOffset: Int64 = 0 - private static let shownExpiryLimit = 100_000 + private static let shownKeyLimit = 100_000 init(config: DriverConnectionConfig) { tunneledServerName = config.tunneledServerName @@ -159,7 +160,11 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { if case .metadump = command { return try await metadumpResult(command, started: started) } - switch try await connection.send(command) { + let reply = try await connection.send(command) + if case .status(let line) = reply, let refusal = MemcachedCASRefusal(replyTo: command, status: line) { + throw MemcachedError.compareAndSwap(refusal) + } + switch reply { case .values(let values): let withCAS: Bool switch command { @@ -222,7 +227,10 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private func browse(_ spec: MemcachedBrowseSpec, started: Date) async throws -> PluginQueryResult { let connection = try liveConnection() let page = try await browsePage(spec, on: connection) - let rows = try await rows(of: page.entries, serverTime: page.serverTime, on: connection) + let keys = page.entries.map(\.key) + let values = try await values(of: keys, on: connection) + rememberLoadedItems(of: keys, values: values) + let rows = rows(of: page.entries, values: values, serverTime: page.serverTime) return result( columns: MemcachedColumn.allCases.map(\.rawValue), types: MemcachedColumn.allCases.map(\.typeName), @@ -238,40 +246,57 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { serverClockOffset = MemcachedServerClock.offset( serverTime: serverTime, localTime: Int64(Date().timeIntervalSince1970) ) - if shownExpiries.count + page.count > Self.shownExpiryLimit { shownExpiries.removeAll() } + if shownExpiries.count + page.count > Self.shownKeyLimit { shownExpiries.removeAll() } for entry in page { shownExpiries[entry.key] = MemcachedKnownExpiry(expiresAt: entry.expiresAt) } } return BrowsePage(entries: page, serverTime: serverTime, isCut: listing.isCut) } - // A key removed since the listing has no value and no row. - private func rows( - of entries: some Collection, - serverTime: Int64, - on connection: MemcachedConnection - ) async throws -> [PluginRow] { + // `gets`, so each value comes with the unique a `cas` needs to write it back. + private func values(of keys: [String], on connection: MemcachedConnection) async throws -> [String: MemcachedValue] { var values: [String: MemcachedValue] = [:] - let keys = entries.map(\.key) var start = keys.startIndex while start < keys.endIndex { try Task.checkCancellation() let end = min(start + Self.valuesPerRequest, keys.endIndex) - if case .values(let read) = try await connection.send(.get(keys: Array(keys[start ..< end]), withCAS: false)) { + if case .values(let read) = try await connection.send(.get(keys: Array(keys[start ..< end]), withCAS: true)) { for value in read { values[value.key] = value } } start = end } - return entries.compactMap { entry in + return values + } + + // A key removed since the listing has no value and no row. + private func rows( + of entries: some Collection, + values: [String: MemcachedValue], + serverTime: Int64 + ) -> [PluginRow] { + entries.compactMap { entry in guard let value = values[entry.key] else { return nil } return [ .text(entry.key), Self.cell(value.data), PluginCellValue.fromOptional(entry.secondsLeft(at: serverTime).map(String.init)), - .text(String(value.flags)), + Self.flagsCell(value.flags), ] } } + private func rememberLoadedItems(of keys: [String], values: [String: MemcachedValue]) { + lock.withLock { + if shownItems.count + keys.count > Self.shownKeyLimit { shownItems.removeAll() } + for key in keys { + shownItems[key] = values[key].flatMap { value in + value.unique.map { + MemcachedLoadedItem(unique: $0, flags: Self.flagsCell(value.flags), value: Self.cell(value.data)) + } + } + } + } + } + private func listKeys() async throws -> (entries: [MemcachedKeyEntry], isCut: Bool) { if lock.withLock({ keySource }) == .metadump { do { @@ -346,7 +371,9 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { while start < page.entries.endIndex { try Task.checkCancellation() let end = min(start + Self.rowsPerStreamBatch, page.entries.endIndex) - let rows = try await self.rows(of: page.entries[start ..< end], serverTime: page.serverTime, on: connection) + let batch = page.entries[start ..< end] + let values = try await self.values(of: batch.map(\.key), on: connection) + let rows = self.rows(of: batch, values: values, serverTime: page.serverTime) if !rows.isEmpty { continuation.yield(.rows(rows)) } start = end } @@ -505,12 +532,12 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { insertedRowIndices: Set ) throws -> [PluginRowWrite]? { if let refusal = PluginRowWriteRefusal.columnDefaultsUnsupported(in: changes) { throw refusal } - let offset = lock.withLock { serverClockOffset } - let expiries = lock.withLock { shownExpiries } + let (offset, expiries, items) = lock.withLock { (serverClockOffset, shownExpiries, shownItems) } let generator = MemcachedStatementGenerator( columns: columns, now: { MemcachedServerClock.serverTime(localTime: Int64(Date().timeIntervalSince1970), offset: offset) }, - knownExpiry: { expiries[$0] } + knownExpiry: { expiries[$0] }, + loadedItem: { items[$0] } ) return try generator.generateRowWrites( from: changes, @@ -522,8 +549,14 @@ final class MemcachedPluginDriver: PluginDatabaseDriver, @unchecked Sendable { // MARK: - Helpers + // String(data:encoding:) drops a leading byte order mark, and a save that carries the value forward would too. private static func cell(_ data: Data) -> PluginCellValue { - String(data: data, encoding: .utf8).map(PluginCellValue.text) ?? .bytes(data) + let text = String(decoding: data, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + return text.utf8.elementsEqual(data) ? .text(text) : .bytes(data) + } + + private static func flagsCell(_ flags: UInt32) -> PluginCellValue { + .text(String(flags)) } private func result( diff --git a/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift b/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift index a0e40fe779..78815b317e 100644 --- a/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift +++ b/apple/Plugins/MemcachedDriverPlugin/MemcachedStatementGenerator.swift @@ -1,3 +1,4 @@ +import CryptoKit import Foundation import TableProMemcachedCore import TableProPluginKit @@ -7,13 +8,47 @@ struct MemcachedKnownExpiry: Sendable { let expiresAt: Int64? } -/// Turns grid edits into console commands: `set` for a new row or a new value or flags, `touch` for a TTL alone, -/// `delete` for a removed row. The key names the item, so it is never rewritten in place. +// The unique `gets` returned for a key, with a digest of the Flags and Value the grid showed. A later read, from +// another tab, holds a newer unique, which must not let this row's older Value or Flags be written back. +struct MemcachedLoadedItem: Sendable { + private let casUnique: UInt64 + private let digest: Data + + init(unique: UInt64, flags: PluginCellValue, value: PluginCellValue) { + casUnique = unique + digest = Self.digest(flags: flags, value: value) + } + + func unique(ifShowing flags: PluginCellValue?, _ value: PluginCellValue?) -> UInt64? { + guard let flags, let value, Self.digest(flags: flags, value: value) == digest else { return nil } + return casUnique + } + + private static func digest(flags: PluginCellValue, value: PluginCellValue) -> Data { + var hash = SHA256() + for cell in [flags, value] { + let tagged: (tag: UInt8, bytes: Data) + switch cell { + case .null: tagged = (0, Data()) + case .text(let text): tagged = (1, Data(text.utf8)) + case .bytes(let data): tagged = (2, data) + } + hash.update(data: [tagged.tag]) + withUnsafeBytes(of: UInt64(tagged.bytes.count).littleEndian) { hash.update(bufferPointer: $0) } + hash.update(data: tagged.bytes) + } + return Data(hash.finalize()) + } +} + +// Grid edits as console commands: `set` for a new row or one that writes both Value and Flags, `cas` for one that keeps +// the loaded Value or Flags, `touch` for a TTL alone, `delete` for a removed row. The key is never rewritten in place. struct MemcachedStatementGenerator { let columns: [String] /// The server's clock, so an absolute expiry is compared with the time the server keeps. var now: () -> Int64 = { Int64(Date().timeIntervalSince1970) } var knownExpiry: (String) -> MemcachedKnownExpiry? = { _ in nil } + var loadedItem: (String) -> MemcachedLoadedItem? = { _ in nil } func generateRowWrites( from changes: [PluginRowChange], @@ -82,27 +117,38 @@ struct MemcachedStatementGenerator { } edited[column] = cell.newValue } - if let ttl = edited[.ttl], edited[.value] == nil, edited[.flags] == nil { - let seconds = try number(ttl, of: .ttl, rowIndex: change.rowIndex) + guard let write = MemcachedItemEdit.write( + editsValue: edited[.value] != nil, editsFlags: edited[.flags] != nil, editsTTL: edited[.ttl] != nil, + unique: loadedItem(key)?.unique(ifShowing: original(.flags, of: change), original(.value, of: change)) + ) else { return nil } + switch write { + case .touch: + let seconds = try number(edited[.ttl], of: .ttl, rowIndex: change.rowIndex) return MemcachedStatements.touch(key: key, exptime: try exptime(forTTL: seconds, rowIndex: change.rowIndex)) + case .needsRefresh: + throw PluginRowWriteRefusal( + rowIndex: change.rowIndex, + reason: String(localized: "Refresh before changing this item, so the save can check that it has not changed since it was loaded.") + ) + case .set, .compareAndSwap: + break } - guard edited[.value] != nil || edited[.flags] != nil else { return nil } let exptime: Int64 if let ttl = edited[.ttl] { exptime = try self.exptime(forTTL: try number(ttl, of: .ttl, rowIndex: change.rowIndex), rowIndex: change.rowIndex) } else { exptime = try currentExptime(of: key, change: change) } - return setStatement( - key: key, - flags: try number(edited[.flags] ?? original(.flags, of: change), of: .flags, rowIndex: change.rowIndex) ?? 0, - exptime: exptime, - value: edited[.value] ?? original(.value, of: change) ?? .null - ) + let flags = try number(edited[.flags] ?? original(.flags, of: change), of: .flags, rowIndex: change.rowIndex) ?? 0 + let value = edited[.value] ?? original(.value, of: change) ?? .null + guard case .compareAndSwap(let unique) = write else { + return setStatement(key: key, flags: flags, exptime: exptime, value: value) + } + return casStatement(key: key, flags: flags, exptime: exptime, unique: unique, value: value) } - /// A `set` replaces the expiry too, so one that leaves TTL alone sends the time the item already ends at, - /// rather than the TTL the grid read, which has been counting down since. + // A `set` or `cas` replaces the expiry too, so one that leaves TTL alone sends the time the item already ends at, + // rather than the TTL the grid read, which has been counting down since. private func currentExptime(of key: String, change: PluginRowChange) throws -> Int64 { guard let known = knownExpiry(key) else { return try exptime(forTTL: try number(original(.ttl, of: change), of: .ttl, rowIndex: change.rowIndex), rowIndex: change.rowIndex) @@ -125,6 +171,14 @@ struct MemcachedStatementGenerator { return MemcachedStatements.set(key: key, flags: flags, exptime: exptime, value: value.asText ?? "") } + private func casStatement(key: String, flags: Int64, exptime: Int64, unique: UInt64, value: PluginCellValue) -> String { + let flags = UInt32(flags) + if case .bytes(let data) = value { + return MemcachedStatements.cas(key: key, flags: flags, exptime: exptime, unique: unique, bytes: data) + } + return MemcachedStatements.cas(key: key, flags: flags, exptime: exptime, unique: unique, value: value.asText ?? "") + } + private func exptime(forTTL ttl: Int64?, rowIndex: Int) throws -> Int64 { guard let exptime = MemcachedStatements.exptime(forTTL: ttl, now: now()) else { throw PluginRowWriteRefusal(rowIndex: rowIndex, reason: String(localized: "This TTL is too large for Memcached.")) diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBNumberKinds.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBNumberKinds.swift new file mode 100644 index 0000000000..1e61cc450d --- /dev/null +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBNumberKinds.swift @@ -0,0 +1,123 @@ +import Foundation +import TableProNumberFormatting +import TableProPluginKit + +// The numeric type each top-level number of a read document held, by the document's row locator. One field can hold +// int32, int64, double and decimal128 side by side, and the column's majority kind would retype a cell on save. +nonisolated struct MongoDBNumberKinds: Sendable { + private var kindsByLocator: [String: [String: BsonValueKind]] = [:] + private(set) var count = 0 + + static let empty = MongoDBNumberKinds() + + static let cellLimit = 100_000 + + private static let numberKinds: Set = [.int32, .int64, .double, .decimal128] + + // A document holding no number is kept as an empty entry, so merging it clears what an earlier read recorded. + static func recording(_ read: MongoReadDocuments, representation: MongoDBUuidRepresentation) -> MongoDBNumberKinds { + var recorded = MongoDBNumberKinds() + for (document, text) in zip(read.dictionaries, read.texts) { + guard let locator = MongoDocumentIdentity.locator(inDocument: text) else { continue } + var kinds: [String: BsonValueKind] = [:] + for (field, value) in document where field != MongoDBCollectionDDL.idField { + let kind = BsonDocumentFlattener.valueKind(for: value, representation: representation) + if numberKinds.contains(kind) { kinds[field] = kind } + } + recorded.kindsByLocator[locator] = kinds + recorded.count += kinds.count + } + return recorded + } + + var isEmpty: Bool { kindsByLocator.isEmpty } + + func kinds(ofRow locator: String) -> [String: BsonValueKind]? { + kindsByLocator[locator] + } + + // A document read again replaces what was recorded for it, since its fields can have changed type or gone. + func merging(_ page: MongoDBNumberKinds) -> MongoDBNumberKinds { + var merged = self + for (locator, kinds) in page.kindsByLocator { + merged.count -= merged.kindsByLocator[locator]?.count ?? 0 + merged.kindsByLocator[locator] = kinds.isEmpty ? nil : kinds + merged.count += kinds.count + } + return merged + } +} + +// What a driver's reads recorded, by the same collection key as its other caches. Past `cellLimit` a collection starts +// over from the latest read, and a row no longer recorded is typed by its column. +nonisolated final class MongoDBNumberKindStore: @unchecked Sendable { + private let lock = NSLock() + private var byCollection: [String: MongoDBNumberKinds] = [:] + + func remember(_ read: MongoReadDocuments, representation: MongoDBUuidRepresentation, collection key: String) { + let page = MongoDBNumberKinds.recording(read, representation: representation) + guard !page.isEmpty else { return } + lock.withLock { + let merged = (byCollection[key] ?? .empty).merging(page) + byCollection[key] = merged.count > MongoDBNumberKinds.cellLimit ? MongoDBNumberKinds.empty.merging(page) : merged + } + } + + func kinds(ofRows rowLocators: [Int: String], collection key: String) -> [Int: [String: BsonValueKind]] { + guard let kinds = lock.withLock({ byCollection[key] }) else { return [:] } + return rowLocators.compactMapValues { kinds.kinds(ofRow: $0) } + } + + func forget(collectionsWhere isStale: (String) -> Bool) { + lock.withLock { byCollection = byCollection.filter { !isStale($0.key) } } + } +} + +nonisolated enum MongoDBDoubleText { + // `0.1` and `4` are doubles exactly as typed; `9007199254740993` is not, a double rounds it to ...992. + static func isExact(_ text: String) -> Bool { + guard let parsed = Double(text), parsed.isFinite, + let typed = DecimalDigits(text), let held = DecimalDigits(parsed.description) else { return false } + return typed == held + } + + // A JSON number as sign, significant digits and power of ten, so `4`, `4.0` and `0.4e1` compare equal. + private struct DecimalDigits: Equatable { + let isNegative: Bool + let significand: String + let exponent: Int + + init?(_ text: String) { + guard NumberText.isJSONNumberLiteral(text) else { return nil } + var rest = Substring(text) + let isNegative = rest.first == "-" + if isNegative { rest = rest.dropFirst() } + var exponent = 0 + if let marker = rest.firstIndex(where: { $0 == "e" || $0 == "E" }) { + guard let power = Int(rest[rest.index(after: marker)...]), power.magnitude < 1 << 32 else { return nil } + exponent = power + rest = rest[..] = [:] private var binarySubtypesByCollection: [String: MongoDBBinarySubtypes] = [:] private var fieldKindsByCollection: [String: MongoDBFieldKinds] = [:] + private let numberKinds = MongoDBNumberKindStore() private static let logger = Logger(subsystem: "com.TablePro", category: "MongoDBPluginDriver") @@ -835,8 +836,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se insertedRowIndices: Set, context: PluginRowWriteContext ) throws -> [PluginRowWrite]? { - var generator = writeGenerator(for: table, columns: columns, database: context.database) - generator.rowLocators = context.rowLocators + let generator = writeGenerator(for: table, columns: columns, database: context.database, rowLocators: context.rowLocators) return try generator.generateRowWrites( from: changes, insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, insertedRowIndices: insertedRowIndices @@ -866,8 +866,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se absentCells: [Int: Set], context: PluginRowWriteContext ) -> [(statement: String, parameters: [PluginCellValue])]? { - var generator = writeGenerator(for: table, columns: columns, database: context.database) - generator.rowLocators = context.rowLocators + let generator = writeGenerator(for: table, columns: columns, database: context.database, rowLocators: context.rowLocators) return generator.generateRestore(rows: rows, absentCells: absentCells) } @@ -884,7 +883,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se return query + #".collation({"locale":"simple"})"# } - private func writeGenerator(for table: String, columns: [String], database: String? = nil) -> MongoDBStatementGenerator { + private func writeGenerator(for table: String, columns: [String], database: String?, rowLocators: [Int: String]) -> MongoDBStatementGenerator { let key = columnKindKey(table, database: database) return columnKindLock.withLock { let declared = declaredSchemasByCollection[key] ?? .empty @@ -894,6 +893,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se columnKinds: columnKindsByCollection[key] ?? [:], declaredKinds: declared.valueKinds, identityKinds: identityKindsByCollection[key], + rowLocators: rowLocators, readNumberKinds: numberKinds.kinds(ofRows: rowLocators, collection: key), fieldKinds: fieldKindsByCollection[key] ?? .empty, binarySubtypes: binarySubtypesByCollection[key] ?? .empty, declaredBinaryFields: Self.binaryFields(declaredBy: declared), @@ -1072,6 +1072,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se rememberIdentityKinds(of: documents, collection: collection) rememberBinarySubtypes(of: documents, collection: collection) rememberFieldKinds(of: documents, collection: collection) + if !collection.isEmpty { numberKinds.remember(read, representation: uuidRepresentation, collection: columnKindKey(collection)) } } let unseen = MongoDBCollectionShape.declaredColumnsMissing(from: sampled, schema: declared) let columns = sampled + unseen @@ -1255,6 +1256,7 @@ nonisolated final class MongoDBPluginDriver: PluginDatabaseDriver, @unchecked Se declaredSchemasByCollection = declaredSchemasByCollection.filter { !isStale($0.key) } identityKindsByCollection = identityKindsByCollection.filter { !isStale($0.key) } } + numberKinds.forget(collectionsWhere: isStale) } } diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift index 2729efa23b..b4082e663f 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBQueryBuilder.swift @@ -409,21 +409,26 @@ nonisolated struct MongoDBQueryBuilder { return (parts[0], parts[1]) } + // Equal values come back in any order, so each skip/limit page could repeat or miss documents. `_id` is unique and + // makes the order total (MongoDB `cursor.sort()`, Sort Consistency). private func buildSortDocument( sortColumns: [(columnIndex: Int, ascending: Bool)], columns: [String] ) -> String? { - guard !sortColumns.isEmpty else { return nil } - - let parts = sortColumns.compactMap { sortCol -> String? in + var parts: [String] = [] + var sortsById = false + for sortCol in sortColumns { guard sortCol.columnIndex >= 0, sortCol.columnIndex < columns.count, - Self.isPlainFieldName(columns[sortCol.columnIndex]) else { return nil } - let columnName = Self.escapeJsonString(columns[sortCol.columnIndex]) - let direction = sortCol.ascending ? 1 : -1 - return "\"\(columnName)\": \(direction)" + Self.isPlainFieldName(columns[sortCol.columnIndex]) else { continue } + let column = columns[sortCol.columnIndex] + sortsById = sortsById || column == MongoDBCollectionDDL.idField + parts.append("\"\(Self.escapeJsonString(column))\": \(sortCol.ascending ? 1 : -1)") } guard !parts.isEmpty else { return nil } + if !sortsById { + parts.append("\"\(MongoDBCollectionDDL.idField)\": 1") + } return "{\(parts.joined(separator: ", "))}" } diff --git a/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift b/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift index 574e17e26f..f366ee5e50 100644 --- a/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift +++ b/apple/Plugins/MongoDBDriverPlugin/MongoDBStatementGenerator.swift @@ -26,6 +26,8 @@ nonisolated struct MongoDBStatementGenerator { // By `PluginRowChange.rowIndex`. Display text cannot say which value an `_id` came from: a string and an ObjectId // with the same hex read the same. var rowLocators: [Int: String] = [:] + // By row index like `rowLocators`: the numeric type each top-level number of the row was read as. + var readNumberKinds: [Int: [String: BsonValueKind]] = [:] /// Every kind each field held in the documents read, which says whether a cell showing `{` or /// `[` holds a document or an array or a string that reads the same. var fieldKinds: MongoDBFieldKinds = .empty @@ -140,19 +142,19 @@ nonisolated struct MongoDBStatementGenerator { return "\(collectionAccessor).insertOne({\(entries.joined(separator: ", "))})" } - /// A value of a whole document the grid writes: a new row, a duplicate or a paste, or a - /// deleted document put back. An `_id` the row holds is kept, typed the way the filters type it; - /// the grid leaves a new row's `_id` as DEFAULT, so the server generates it. - /// - /// The shell inserts through libmongoc, which refuses a document holding an empty key at any - /// depth, and a JavaScript object reads `__proto__` as its prototype and drops it. An update - /// carries both, so only a new document refuses them, before anything in the save is sent. - private func documentValueJson(_ value: PluginCellValue, field: String, provenance: Provenance) throws -> String { + // libmongoc refuses an inserted document holding an empty key at any depth, and JavaScript reads `__proto__` as the + // prototype and drops it. An update carries both, so only a new or restored document refuses them, before any write. + private func documentValueJson( + _ value: PluginCellValue, + field: String, + provenance: Provenance, + readKind: BsonValueKind? = nil + ) throws -> String { guard !field.isEmpty else { throw MongoDBWriteRefusal.emptyFieldNameInNewDocument } guard field != "__proto__" else { throw MongoDBWriteRefusal.prototypeFieldInNewDocument } let json = field == MongoDBCollectionDDL.idField ? try idValueJson(value) - : try valueJson(value, field: field, provenance: provenance) + : try valueJson(value, field: field, provenance: provenance, readKind: readKind) if opensContainer(json), case .text(let text) = value, parsedContainer(text)?.holdsEmptyKey == true { throw MongoDBWriteRefusal.emptyKeyInNewDocument(field: field) } @@ -161,12 +163,8 @@ nonisolated struct MongoDBStatementGenerator { // MARK: - Restore - /// Puts a deleted document back with the `_id` it had. - /// - /// A new row leaves `_id` to the server. Undoing a delete is the opposite requirement: a new - /// `_id` is a different document, and anything that referenced the old one still points at - /// nothing. A value that cannot be written refuses the restore rather than dropping the field. - /// A field the document did not have stays missing, and one that held null holds it again. + // An undone delete keeps its `_id`, since a new one is another document that nothing referencing the old one finds. + // A value that cannot be written refuses the restore rather than dropping the field. func generateRestore( rows: [[PluginCellValue]], absentCells: [Int: Set] = [:] @@ -178,12 +176,14 @@ nonisolated struct MongoDBStatementGenerator { guard idIndex < row.count else { throw MongoDBWriteRefusal.missingIdentity } guard let locator = rowLocators[rowIndex] else { throw MongoDBWriteRefusal.unlocatedRow } let absent = absentCells[rowIndex] ?? [] + let readKinds = readNumberKinds[rowIndex] ?? [:] let idField = MongoDBCollectionDDL.idField var entries = ["\(quotedKey(idField)): \(try Self.identityJson(locator: locator))"] for (index, value) in row.enumerated() where index != idIndex && index < columns.count { guard !absent.contains(index) else { continue } let field = columns[index] - entries.append("\(quotedKey(field)): \(try documentValueJson(value, field: field, provenance: .restored))") + let json = try documentValueJson(value, field: field, provenance: .restored, readKind: readKinds[field]) + entries.append("\(quotedKey(field)): \(json)") } let statement = "\(collectionAccessor).insertOne({\(entries.joined(separator: ", "))})" return (statement: statement, parameters: []) @@ -212,15 +212,16 @@ nonisolated struct MongoDBStatementGenerator { } } - /// A field the change names in `absentColumns` is removed with `$unset`; NULL is stored as null. private func updateStatement(for change: PluginRowChange) throws -> String? { guard !change.cellChanges.isEmpty else { return nil } let identity = try identityFilter(of: change) let removed = change.absentColumns ?? [] + let readKinds = readNumberKinds[change.rowIndex] ?? [:] let cellWrites = try change.cellChanges.map { try cellWrite( field: $0.columnName, from: $0.oldValue, to: $0.newValue, - removesField: removed.contains($0.columnIndex), isDefault: change.isColumnDefault($0.columnIndex) + removesField: removed.contains($0.columnIndex), isDefault: change.isColumnDefault($0.columnIndex), + readKind: readKinds[$0.columnName] ) } let update = try updateDocument(for: cellWrites) @@ -232,7 +233,8 @@ nonisolated struct MongoDBStatementGenerator { from oldValue: PluginCellValue, to newValue: PluginCellValue, removesField: Bool, - isDefault: Bool + isDefault: Bool, + readKind: BsonValueKind? ) throws -> CellWrite { guard field != MongoDBCollectionDDL.idField else { throw MongoDBWriteRefusal.identityChanged } guard !removesField else { return .remove(field: field) } @@ -246,7 +248,8 @@ nonisolated struct MongoDBStatementGenerator { if let nested = try nestedEdit(of: field, from: oldValue, to: text) { return nested } - return .whole(field: field, json: try valueJson(newValue, field: field, provenance: .edit(replacing: oldValue))) + let json = try valueJson(newValue, field: field, provenance: .edit(replacing: oldValue), readKind: readKind) + return .whole(field: field, json: json) } } @@ -375,8 +378,12 @@ nonisolated struct MongoDBStatementGenerator { return type == "array" || type == "regex" } - /// A cell's value as the JSON the statement carries. - private func valueJson(_ value: PluginCellValue, field: String, provenance: Provenance) throws -> String { + private func valueJson( + _ value: PluginCellValue, + field: String, + provenance: Provenance, + readKind: BsonValueKind? = nil + ) throws -> String { switch value { case .null: return "null" @@ -389,7 +396,7 @@ nonisolated struct MongoDBStatementGenerator { guard !JSONTruncation.isIncompleteStructure(text) else { throw MongoDBWriteRefusal.truncatedValue(field: field) } - return try jsonValue(for: text, field: field, provenance: provenance) + return try jsonValue(for: text, field: field, provenance: provenance, readKind: readKind) } } @@ -579,18 +586,23 @@ nonisolated struct MongoDBStatementGenerator { return nsValue.length == 24 && value.allSatisfy { $0.isHexDigit } } - /// A cell's text as the value it stands for, in the field's type when that type is known. - /// - /// The statement is JavaScript the shell evaluates, so text is only ever pasted in when it is - /// strict JSON. A stored string that merely starts with `[` and ends with `]` would otherwise run - /// as code the moment its row is duplicated or its delete is undone. - private func jsonValue(for value: String, field: String, provenance: Provenance) throws -> String { + // The statement is JavaScript the shell evaluates, so text goes in only as strict JSON: a stored string reading + // `[...]` would otherwise run as code once its row is duplicated or its delete undone. + private func jsonValue( + for value: String, + field: String, + provenance: Provenance, + readKind: BsonValueKind? + ) throws -> String { if declaredKinds[field] == .string || spellsAnotherKindInAStringField(value, field: field) { return "\"\(escapeJsonString(value))\"" } if value == "true" || value == "false" || value == "null" { return value } + if let typed = readNumberJson(value, field: field, readKind: readKind, provenance: provenance) { + return typed + } if let typed = typedJson(value, kind: kind(of: field)) { return typed } @@ -606,6 +618,34 @@ nonisolated struct MongoDBStatementGenerator { return "\"\(escapeJsonString(value))\"" } + // A number keeps the type its cell was read as when the text is a value of that type, since a field can mix numeric + // types and the column's majority would retype it. A validator's declared kind is what the server enforces. + private func readNumberJson( + _ value: String, + field: String, + readKind: BsonValueKind?, + provenance: Provenance + ) -> String? { + guard declaredKinds[field] == nil, let readKind, readsAsNumber(provenance) else { return nil } + switch readKind { + case .int32, .int64, .decimal128: + return typedJson(value, kind: readKind) + case .double: + return MongoDBDoubleText.isExact(value) ? typedJson(value, kind: .double) : nil + default: + return nil + } + } + + private func readsAsNumber(_ provenance: Provenance) -> Bool { + switch provenance { + case .edit(replacing: .text), .restored: + return true + case .edit, .typedIntoNewDocument, .copiedIntoNewDocument: + return false + } + } + // A cell shows the string "90210" and the number 90210 as the same text. In a field that held // strings and never a number, boolean or null, text of that shape is a string like the rest. private func spellsAnotherKindInAStringField(_ value: String, field: String) -> Bool { diff --git a/apple/Plugins/RedisDriverPlugin/RedisCommandChannel.swift b/apple/Plugins/RedisDriverPlugin/RedisCommandChannel.swift index db79b92cfa..dae6c7f971 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisCommandChannel.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisCommandChannel.swift @@ -228,7 +228,7 @@ enum RedisScanReply { let cursor: String switch parts[0] { case .string(let value), .status(let value): cursor = value - case .data(let value): cursor = String(data: value, encoding: .utf8) ?? "0" + case .data(let value): cursor = RedisWireText.decode(value) ?? "0" case .integer(let value): cursor = String(value) default: cursor = "0" } @@ -236,7 +236,7 @@ enum RedisScanReply { let keys = keyReplies.compactMap { item -> String? in switch item { case .string(let key), .status(let key): return key - case .data(let value): return String(data: value, encoding: .utf8) + case .data(let value): return RedisWireText.decode(value) default: return nil } } diff --git a/apple/Plugins/RedisDriverPlugin/RedisKeyContentsRead.swift b/apple/Plugins/RedisDriverPlugin/RedisKeyContentsRead.swift index 4890870559..fe99890600 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisKeyContentsRead.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisKeyContentsRead.swift @@ -30,6 +30,7 @@ struct RedisKeyContents { let kind: RedisKeyKind let length: Int? let preview: RedisReply? + var stringByteLimit: Int? var lengthCell: PluginCellValue { .fromOptional(length.map(String.init)) @@ -48,11 +49,12 @@ extension RedisCommandChannel { } } - /// A key whose type is unknown gets no probe at all, because the length and preview commands - /// are chosen by type. + // A key whose type is unknown gets no probe at all, because the length and preview commands + // are chosen by type. func readContents( of keys: [String], - describedAs descriptions: [RedisKeyDescription] + describedAs descriptions: [RedisKeyDescription], + stringByteLimit: Int? ) async throws -> [RedisKeyContents?] { let kinds = zip(keys, descriptions).map { (key: $0, kind: $1.kind) } var commands: [[String]] = [] @@ -60,7 +62,7 @@ extension RedisCommandChannel { for entry in kinds { guard let kind = entry.kind else { continue } commands.append(RedisKeySummary.lengthCommand(for: kind, key: entry.key)) - commands.append(RedisKeySummary.previewCommand(for: kind, key: entry.key)) + commands.append(RedisKeySummary.previewCommand(for: kind, key: entry.key, stringByteLimit: stringByteLimit)) } let answers = try await runMetadataReads(commands) @@ -72,7 +74,12 @@ extension RedisCommandChannel { contents.append(nil) continue } - contents.append(RedisKeyContents(kind: kind, length: answers[next]?.intValue, preview: answers[next + 1])) + contents.append(RedisKeyContents( + kind: kind, + length: answers[next]?.intValue, + preview: answers[next + 1], + stringByteLimit: stringByteLimit + )) next += 2 } return contents diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginConnection.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginConnection.swift index ac7b9cbaa8..41964cadbb 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginConnection.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginConnection.swift @@ -942,7 +942,7 @@ enum HiredisReply { switch reply.pointee.type { case REDIS_REPLY_STRING, REDIS_REPLY_BIGNUM, REDIS_REPLY_VERB: guard let data = bytes(of: reply) else { return .null } - return String(data: data, encoding: .utf8).map(RedisReply.string) ?? .data(data) + return RedisWireText.decode(data).map(RedisReply.string) ?? .data(data) case REDIS_REPLY_INTEGER, REDIS_REPLY_BOOL: return .integer(reply.pointee.integer) @@ -966,10 +966,10 @@ enum HiredisReply { return .null case REDIS_REPLY_STATUS: - return .status(bytes(of: reply).flatMap { String(data: $0, encoding: .utf8) } ?? "") + return .status(bytes(of: reply).flatMap(RedisWireText.decode) ?? "") case REDIS_REPLY_ERROR: - return .error(bytes(of: reply).flatMap { String(data: $0, encoding: .utf8) } ?? "Unknown error") + return .error(bytes(of: reply).flatMap(RedisWireText.decode) ?? "Unknown error") case REDIS_REPLY_DOUBLE: return .string(bytes(of: reply).flatMap { String(data: $0, encoding: .utf8) } ?? String(reply.pointee.dval)) diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift index 6ac131744d..e06de0f3a8 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Operations.swift @@ -166,7 +166,11 @@ extension RedisPluginDriver { let capped = Array(keys.prefix(PluginRowLimits.emergencyMax)) let keysTruncated = keys.count > PluginRowLimits.emergencyMax return try await buildKeyBrowseResult( - keys: capped, connection: conn, startTime: startTime, isTruncated: keysTruncated + keys: capped, + connection: conn, + startTime: startTime, + isTruncated: keysTruncated, + batchSize: Self.keySummaryBatchSize ) case .scan(let cursor, let pattern, let count, let type): diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+ResultBuilding.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+ResultBuilding.swift index e705b0bb9c..167701e122 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+ResultBuilding.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+ResultBuilding.swift @@ -34,18 +34,24 @@ extension RedisPluginDriver { ) } + static let keySummaryBatchSize = 200 + + // Nil sends every key in one pipeline: fine for a grid page the user sized, not for KEYS or SCAN. func buildKeyBrowseResult( keys: [String], connection conn: any RedisCommandChannel, startTime: Date, isTruncated: Bool = false, - statusMessage: String? = nil + statusMessage: String? = nil, + batchSize: Int? = nil ) async throws -> PluginQueryResult { guard !keys.isEmpty else { return buildEmptyKeyResult(startTime: startTime, statusMessage: statusMessage) } - let rows = try await buildKeySummaryRows(keys: keys, connection: conn) + let rows = try await buildKeySummaryRows( + keys: keys, connection: conn, stringByteLimit: RedisStringPreview.byteLimit, batchSize: batchSize + ) return PluginQueryResult( columns: Self.keyBrowseColumns, columnTypeNames: Self.keyBrowseColumnTypeNames, @@ -57,14 +63,35 @@ extension RedisPluginDriver { ) } + // A nil byte limit reads every string whole, which only an export wants. func buildKeySummaryRows( keys: [String], - connection conn: any RedisCommandChannel + connection conn: any RedisCommandChannel, + stringByteLimit: Int?, + batchSize: Int? = nil + ) async throws -> [PluginRow] { + guard let batchSize, batchSize > 0, keys.count > batchSize else { + return try await summarizeKeys(keys, connection: conn, stringByteLimit: stringByteLimit) + } + var rows: [PluginRow] = [] + rows.reserveCapacity(keys.count) + for start in stride(from: 0, to: keys.count, by: batchSize) { + try Task.checkCancellation() + let batch = Array(keys[start ..< min(start + batchSize, keys.count)]) + rows += try await summarizeKeys(batch, connection: conn, stringByteLimit: stringByteLimit) + } + return rows + } + + private func summarizeKeys( + _ keys: [String], + connection conn: any RedisCommandChannel, + stringByteLimit: Int? ) async throws -> [PluginRow] { guard !keys.isEmpty else { return [] } let descriptions = try await conn.describeKeys(keys) - let contents = try await conn.readContents(of: keys, describedAs: descriptions) + let contents = try await conn.readContents(of: keys, describedAs: descriptions, stringByteLimit: stringByteLimit) return zip(keys, zip(descriptions, contents)).map { key, summary in let (description, content) = summary return [ @@ -78,7 +105,24 @@ extension RedisPluginDriver { } private func valueCell(for content: RedisKeyContents) -> PluginCellValue? { - content.preview.map { previewCell($0, kind: content.kind) } + guard let preview = content.preview else { return nil } + guard content.kind == .string, let byteLimit = content.stringByteLimit else { + return previewCell(preview, kind: content.kind) + } + return shownStringCell(preview, length: content.length, byteLimit: byteLimit) + } + + private func shownStringCell(_ reply: RedisReply, length: Int?, byteLimit: Int) -> PluginCellValue { + let bytes: Data + switch reply { + case .string(let text): bytes = Data(text.utf8) + case .data(let data): bytes = data + default: return stringCell(from: reply) + } + switch RedisStringPreview.shown(bytes, length: length, byteLimit: byteLimit) { + case .text(let text): return .text(text) + case .bytes(let data): return .bytes(data) + } } func previewCell(_ reply: RedisReply, kind: RedisKeyKind) -> PluginCellValue { diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Scan.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Scan.swift index 519e6b4dd1..7a740f7949 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Scan.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver+Scan.swift @@ -81,12 +81,14 @@ extension RedisPluginDriver { startTime: Date ) async throws -> PluginQueryResult { let outcome = RedisScanPageOutcome(page: page, rowLimit: PluginRowLimits.emergencyMax) + // SCAN COUNT is the user's to set, so a page can hold any number of keys. return try await buildKeyBrowseResult( keys: outcome.keys, connection: conn, startTime: startTime, isTruncated: outcome.isTruncated, - statusMessage: outcome.statusMessage + statusMessage: outcome.statusMessage, + batchSize: Self.keySummaryBatchSize ) } } diff --git a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift index 75ff9f44ec..18c3a3ff75 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisPluginDriver.swift @@ -558,6 +558,8 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation: continuation ) } + case .keys(let pattern): + try await streamKeyRows(connection: conn, pattern: pattern, continuation: continuation) default: let startTime = Date() let result = try await executeOperation(operation, connection: conn, startTime: startTime) @@ -589,7 +591,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { ))) var cursor = RedisClusterCursor.start - let batchSize = 200 + let batchSize = Self.keySummaryBatchSize repeat { try Task.checkCancellation() @@ -604,7 +606,7 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { try Task.checkCancellation() let batchEnd = min(batchStart + batchSize, page.keys.count) let batchKeys = Array(page.keys[batchStart ..< batchEnd]) - let rowBatch = try await buildKeySummaryRows(keys: batchKeys, connection: conn) + let rowBatch = try await buildKeySummaryRows(keys: batchKeys, connection: conn, stringByteLimit: nil) if !rowBatch.isEmpty { continuation.yield(.rows(rowBatch)) } @@ -615,6 +617,29 @@ final class RedisPluginDriver: PluginDatabaseDriver, @unchecked Sendable { continuation.finish() } + // KEYS names every match at once; their values are read a batch at a time, each string whole. + private func streamKeyRows( + connection conn: any RedisCommandChannel, + pattern: String, + continuation: AsyncThrowingStream.Continuation + ) async throws { + let keys = (try await conn.run(["KEYS", pattern]).arrayValue ?? []).map(\.displayText) + continuation.yield(.header(PluginStreamHeader( + columns: Self.keyBrowseColumns, + columnTypeNames: Self.keyBrowseColumnTypeNames, + estimatedRowCount: keys.count + ))) + for start in stride(from: 0, to: keys.count, by: Self.keySummaryBatchSize) { + try Task.checkCancellation() + let batch = Array(keys[start ..< min(start + Self.keySummaryBatchSize, keys.count)]) + let rows = try await buildKeySummaryRows(keys: batch, connection: conn, stringByteLimit: nil) + if !rows.isEmpty { + continuation.yield(.rows(rows)) + } + } + continuation.finish() + } + // MARK: - Query Building func buildBrowseQuery( diff --git a/apple/Plugins/RedisDriverPlugin/RedisReply.swift b/apple/Plugins/RedisDriverPlugin/RedisReply.swift index a4284b2381..0aaeb72278 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisReply.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisReply.swift @@ -9,6 +9,7 @@ import Foundation import TableProPluginKit +import TableProRedisCore // MARK: - Reply Type @@ -24,7 +25,7 @@ enum RedisReply { var stringValue: String? { switch self { case .string(let s), .status(let s): return s - case .data(let d): return String(data: d, encoding: .utf8) + case .data(let d): return RedisWireText.decode(d) default: return nil } } @@ -55,7 +56,7 @@ enum RedisReply { case .string(let text), .status(let text): return text case .error(let message): return "(error) \(message)" case .integer(let value): return String(value) - case .data(let bytes): return String(data: bytes, encoding: .utf8) ?? bytes.base64EncodedString() + case .data(let bytes): return RedisWireText.decode(bytes) ?? bytes.base64EncodedString() case .array(let items): return "[\(items.map(\.displayText).joined(separator: ", "))]" case .null: return "(nil)" } diff --git a/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift b/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift index 1bb4883226..58c1bbd7b1 100644 --- a/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift +++ b/apple/Plugins/RedisDriverPlugin/RedisStatementGenerator.swift @@ -54,6 +54,10 @@ struct RedisStatementGenerator { columns.firstIndex(of: "TTL") } + private var lengthColumnIndex: Int? { + columns.firstIndex(of: "Length") + } + private static let insertableTypes: Set = ["string", "hash", "list", "set", "zset"] // MARK: - Public API @@ -114,8 +118,10 @@ struct RedisStatementGenerator { ) throws -> [String] { var key: String? var value: String? + var valueCell: PluginCellValue? var type: String? var ttlText: String? + var lengthText: String? if let values = insertedRowData[change.rowIndex] { if let ki = keyColumnIndex, ki < values.count { @@ -126,17 +132,24 @@ struct RedisStatementGenerator { } if let vi = valueColumnIndex, vi < values.count { value = Self.encodedArgument(values[vi]) + valueCell = values[vi] } if let ttli = ttlColumnIndex, ttli < values.count { ttlText = values[ttli].asText } + if let li = lengthColumnIndex, li < values.count { + lengthText = values[li].asText + } } else { for cellChange in change.cellChanges { switch cellChange.columnName { case "Key": key = cellChange.newValue.asText case "Type": type = cellChange.newValue.asText - case "Value": value = Self.encodedArgument(cellChange.newValue) + case "Value": + value = Self.encodedArgument(cellChange.newValue) + valueCell = cellChange.newValue case "TTL": ttlText = cellChange.newValue.asText + case "Length": lengthText = cellChange.newValue.asText default: break } } @@ -150,6 +163,15 @@ struct RedisStatementGenerator { guard typeName.isEmpty || Self.insertableTypes.contains(typeName) else { throw Self.notAddable(typeName, rowIndex: change.rowIndex) } + if let length = lengthText.flatMap({ Int($0) }), let valueCell, + Self.holdsPart(of: length, value: valueCell, typeName: typeName) { + throw PluginRowWriteRefusal( + rowIndex: change.rowIndex, + reason: String( + localized: "This row holds only the part of a value the grid shows, so adding it would store that part. Add the key with a command in the query editor." + ) + ) + } var ttl: Int? if let ttlText { @@ -187,6 +209,23 @@ struct RedisStatementGenerator { private static let collectionTypes: Set = ["hash", "list", "set", "zset"] + // A duplicated row carries the Length the read reported beside the preview it showed. + private static func holdsPart(of length: Int, value: PluginCellValue, typeName: String) -> Bool { + switch (typeName, value) { + case ("string", .text(let text)), ("", .text(let text)): + return length > RedisStringPreview.byteLimit && text.hasSuffix(RedisStringPreview.marker) + && text.utf8.count - RedisStringPreview.marker.utf8.count <= RedisStringPreview.byteLimit + case ("string", .bytes(let bytes)), ("", .bytes(let bytes)): + return length > RedisStringPreview.byteLimit && bytes.count <= RedisStringPreview.byteLimit + case (_, .text(let text)) where collectionTypes.contains(typeName): + guard let parsed = try? JSONSerialization.jsonObject(with: Data(text.utf8)) else { return false } + let count = (parsed as? [String: Any])?.count ?? (parsed as? [Any])?.count + return count.map { $0 < length } ?? false + default: + return false + } + } + private static func notAddable(_ typeName: String, rowIndex: Int) -> PluginRowWriteRefusal { PluginRowWriteRefusal( rowIndex: rowIndex, @@ -289,8 +328,8 @@ struct RedisStatementGenerator { return valueCommands + ttlCommands + (rename.map { [$0] } ?? []) } - /// Only a string's value is the whole of what the grid shows. A collection shows a preview, - /// and a `SET` over it would replace the structure with that text. + // A collection shows a preview and a long string only its start, and a `SET` over either + // would replace the value with that text. private func valueCommands( setting newValue: PluginCellValue, of change: PluginRowChange, @@ -318,12 +357,35 @@ struct RedisStatementGenerator { ) ) } + guard !isShownInPart(change) else { + throw PluginRowWriteRefusal( + rowIndex: change.rowIndex, + reason: String( + localized: "The grid shows only the start of this value, so it cannot be edited here. Change it with a command in the query editor." + ) + ) + } let set = "SET \(RedisArgumentCodec.quote(key)) \(encodedValue)" guard !keepsTTLOnSet else { return ["\(set) KEEPTTL"] } guard restoringTTL, let seconds = originalTTL(of: change), seconds > 0 else { return [set] } return [set, "EXPIRE \(RedisArgumentCodec.quote(key)) \(seconds)"] } + // Only the key browse has a Length column, and it reads each string with GETRANGE. + private func isShownInPart(_ change: PluginRowChange) -> Bool { + guard let lengthIndex = lengthColumnIndex, let originalRow = change.originalRow, lengthIndex < originalRow.count else { + return false + } + let length = originalRow[lengthIndex].asText.flatMap { Int($0) } + let shownByteCount: Int + switch valueColumnIndex.flatMap({ $0 < originalRow.count ? originalRow[$0] : nil }) { + case .text(let text)?: shownByteCount = text.utf8.count + case .bytes(let bytes)?: shownByteCount = bytes.count + case .null?, nil: shownByteCount = 0 + } + return RedisStringPreview.isShortened(length: length, shownByteCount: shownByteCount) + } + private func originalTTL(of change: PluginRowChange) -> Int? { guard let ttlIndex = ttlColumnIndex, let originalRow = change.originalRow, ttlIndex < originalRow.count else { return nil diff --git a/apple/Plugins/RedisDriverTests/HiredisReplyTests.swift b/apple/Plugins/RedisDriverTests/HiredisReplyTests.swift index a1e5a4aa40..5d4cf11db7 100644 --- a/apple/Plugins/RedisDriverTests/HiredisReplyTests.swift +++ b/apple/Plugins/RedisDriverTests/HiredisReplyTests.swift @@ -61,6 +61,16 @@ struct HiredisReplyTests { #expect(try Self.convert(wire).displayText == expected) } + @Test("A bulk string that starts with a byte order mark reads with it") + func bulkStringKeepsByteOrderMark() throws { + let reply = try Self.convert("$9\r\n\u{FEFF}user:1\r\n") + guard case .string(let text) = reply else { + Issue.record("expected text, got \(reply.displayText)") + return + } + #expect(Data(text.utf8) == Data([0xEF, 0xBB, 0xBF] + Array("user:1".utf8))) + } + private struct Outcome: Sendable { let status: Int32 let error: Int32 diff --git a/apple/Plugins/RedisDriverTests/RedisKeyBrowseValueTests.swift b/apple/Plugins/RedisDriverTests/RedisKeyBrowseValueTests.swift new file mode 100644 index 0000000000..a3e3c50776 --- /dev/null +++ b/apple/Plugins/RedisDriverTests/RedisKeyBrowseValueTests.swift @@ -0,0 +1,117 @@ +// +// RedisKeyBrowseValueTests.swift +// RedisDriverTests +// + +import Foundation +import TableProPluginKit +import TableProRedisCore +import Testing + +struct RedisKeyBrowseValueTests { + private func driver() -> RedisPluginDriver { + RedisPluginDriver(config: DriverConnectionConfig( + host: "127.0.0.1", port: 6_379, username: "", password: "", database: "0" + )) + } + + private static func scanReply(_ keys: [String]) -> RedisReply { + .array([.string("0"), .array(keys.map(RedisReply.string))]) + } + + @Test("The grid reads a long string's start and marks it shortened, with its whole length") + func gridShowsTheStartOfALongString() async throws { + let start = String(repeating: "a", count: RedisStringPreview.byteLimit) + let channel = StubRedisChannel([ + Self.scanReply(["page:1"]), + .status("string"), .integer(-1), + .integer(900_000), .string(start), + ]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("KEYBROWSE LIMIT 100 OFFSET 0"), connection: channel, startTime: Date() + ) + #expect(channel.sentCommands.last == ["GETRANGE", "page:1", "0", String(RedisStringPreview.byteLimit - 1)]) + let row = try #require(result.rows.first) + #expect(row[3] == .text("900000")) + #expect(row[4] == .text(start + RedisStringPreview.marker)) + } + + @Test("A short string in the grid shows whole and unmarked") + func gridShowsAShortStringWhole() async throws { + let channel = StubRedisChannel([ + Self.scanReply(["k"]), + .status("string"), .integer(-1), + .integer(5), .string("hello"), + ]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("KEYBROWSE LIMIT 100 OFFSET 0"), connection: channel, startTime: Date() + ) + #expect(result.rows.first?[4] == .text("hello")) + } + + @Test("A long binary string in the grid stays bytes") + func gridKeepsBinaryAsBytes() async throws { + let blob = Data([0x1F, 0x8B, 0x08, 0x00, 0xFF, 0xFE]) + let channel = StubRedisChannel([ + Self.scanReply(["bin"]), + .status("string"), .integer(-1), + .integer(900_000), .data(blob), + ]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("KEYBROWSE LIMIT 100 OFFSET 0"), connection: channel, startTime: Date() + ) + #expect(result.rows.first?[4] == .bytes(blob)) + } + + @Test("The export summary reads every string whole") + func exportSummaryReadsWholeValues() async throws { + let whole = String(repeating: "b", count: RedisStringPreview.byteLimit + 10) + let channel = StubRedisChannel([ + .status("string"), .integer(-1), + .integer(Int64(whole.utf8.count)), .string(whole), + ]) + let rows = try await driver().buildKeySummaryRows(keys: ["doc"], connection: channel, stringByteLimit: nil) + #expect(channel.sentCommands.last == ["GET", "doc"]) + #expect(rows.first?[4] == .text(whole)) + } + + @Test("KEYS summarizes its keys in batches, never one pipeline for every key") + func keysSummarizedInBatches() async throws { + let keys = (0 ..< 250).map { "k\($0)" } + let channel = StubRedisChannel([.array(keys.map(RedisReply.string))]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("KEYS *"), connection: channel, startTime: Date() + ) + #expect(result.rows.count == 250) + #expect(result.rows.map { $0[0] } == keys.map(PluginCellValue.text)) + #expect(channel.pipelineSizes == [RedisPluginDriver.keySummaryBatchSize * 2, 100]) + } + + @Test("A SCAN page summarizes its keys in batches") + func scanPageSummarizedInBatches() async throws { + let keys = (0 ..< 250).map { "k\($0)" } + let channel = StubRedisChannel([Self.scanReply(keys)]) + let result = try await driver().executeOperation( + try RedisCommandParser.parse("SCAN 0 COUNT 300"), connection: channel, startTime: Date() + ) + #expect(result.rows.count == 250) + #expect(channel.pipelineSizes == [RedisPluginDriver.keySummaryBatchSize * 2, 100]) + } + + @Test("A key that starts with a byte order mark is deleted by its own name") + func byteOrderMarkKeyDeletesItself() throws { + let markedKey = "\u{FEFF}user:1" + let generator = RedisStatementGenerator(namespaceName: "", columns: ["Key", "Type", "TTL", "Length", "Value"]) + let change = PluginRowChange( + rowIndex: 0, type: .delete, cellChanges: [], originalRow: [.text(markedKey), "string", "-1", "1", "v"] + ) + let writes = try generator.generateRowWrites( + from: [change], insertedRowData: [:], deletedRowIndices: [0], insertedRowIndices: [] + ) + guard case .del(let deleted)? = try? RedisCommandParser.parse(writes[0].statement) else { + Issue.record("expected a DEL") + return + } + #expect(deleted.map { Data($0.utf8) } == [Data([0xEF, 0xBB, 0xBF] + Array("user:1".utf8))]) + } +} diff --git a/apple/Plugins/RedisDriverTests/RedisKeyMetadataReadTests.swift b/apple/Plugins/RedisDriverTests/RedisKeyMetadataReadTests.swift index f0e9226b82..9289f25820 100644 --- a/apple/Plugins/RedisDriverTests/RedisKeyMetadataReadTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisKeyMetadataReadTests.swift @@ -165,7 +165,8 @@ struct RedisKeyContentsReadTests { let channel = StubRedisChannel([]) let contents = try await channel.readContents( of: ["other:1"], - describedAs: [RedisKeyDescription(typeName: nil, ttlSeconds: nil)] + describedAs: [RedisKeyDescription(typeName: nil, ttlSeconds: nil)], + stringByteLimit: RedisStringPreview.byteLimit ) #expect(contents.count == 1) #expect(contents[0] == nil) @@ -180,7 +181,8 @@ struct RedisKeyContentsReadTests { let channel = StubRedisChannel([.integer(1), keyDenied]) let contents = try await channel.readContents( of: ["app:h"], - describedAs: [RedisKeyDescription(typeName: "hash", ttlSeconds: -1)] + describedAs: [RedisKeyDescription(typeName: "hash", ttlSeconds: -1)], + stringByteLimit: RedisStringPreview.byteLimit ) let hash = try #require(contents[0]) #expect(hash.kind == .hash) @@ -190,12 +192,13 @@ struct RedisKeyContentsReadTests { #expect(channel.sentCommands == [["HLEN", "app:h"], ["HSCAN", "app:h", "0", "COUNT", "100"]]) } - @Test("A refused GET leaves the string length the server gave") + @Test("A refused string read leaves the string length the server gave") func refusedGetKeepsLength() async throws { let channel = StubRedisChannel([.integer(1), keyDenied]) let contents = try await channel.readContents( of: ["app:1"], - describedAs: [RedisKeyDescription(typeName: "string", ttlSeconds: nil)] + describedAs: [RedisKeyDescription(typeName: "string", ttlSeconds: nil)], + stringByteLimit: RedisStringPreview.byteLimit ) let string = try #require(contents[0]) #expect(string.length == 1) @@ -211,7 +214,8 @@ struct RedisKeyContentsReadTests { RedisKeyDescription(typeName: "string", ttlSeconds: -1), RedisKeyDescription(typeName: nil, ttlSeconds: nil), RedisKeyDescription(typeName: "list", ttlSeconds: -1), - ] + ], + stringByteLimit: nil ) #expect(contents.count == 3) #expect(contents[0]?.length == 3) @@ -223,6 +227,26 @@ struct RedisKeyContentsReadTests { #expect(channel.sentCommands.map(\.first) == ["STRLEN", "GET", "LLEN", "LRANGE"]) } + @Test("With a byte limit a string is read with GETRANGE over that range, without one with GET") + func stringReadFollowsTheLimit() async throws { + let limited = StubRedisChannel([.integer(3), .string("abc")]) + let contents = try await limited.readContents( + of: ["a"], + describedAs: [RedisKeyDescription(typeName: "string", ttlSeconds: -1)], + stringByteLimit: 65_536 + ) + #expect(limited.sentCommands == [["STRLEN", "a"], ["GETRANGE", "a", "0", "65535"]]) + #expect(contents[0]?.stringByteLimit == 65_536) + + let whole = StubRedisChannel([.integer(3), .string("abc")]) + _ = try await whole.readContents( + of: ["a"], + describedAs: [RedisKeyDescription(typeName: "string", ttlSeconds: -1)], + stringByteLimit: nil + ) + #expect(whole.sentCommands == [["STRLEN", "a"], ["GET", "a"]]) + } + @Test("A key whose type changed under the probe fails the page rather than showing wrong contents") func wrongTypeThrows() async throws { let channel = StubRedisChannel([ @@ -232,7 +256,8 @@ struct RedisKeyContentsReadTests { await #expect(throws: RedisPluginError.self) { try await channel.readContents( of: ["app:h"], - describedAs: [RedisKeyDescription(typeName: "hash", ttlSeconds: -1)] + describedAs: [RedisKeyDescription(typeName: "hash", ttlSeconds: -1)], + stringByteLimit: RedisStringPreview.byteLimit ) } } diff --git a/apple/Plugins/RedisDriverTests/RedisReplyTests.swift b/apple/Plugins/RedisDriverTests/RedisReplyTests.swift index 8269d0744c..8ecb837754 100644 --- a/apple/Plugins/RedisDriverTests/RedisReplyTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisReplyTests.swift @@ -58,6 +58,35 @@ struct RedisReplyStringValueTests { let reply = RedisReply.array([.string("a")]) #expect(reply.stringValue == nil) } + + @Test("data that starts with a byte order mark keeps it in stringValue and displayText") + func dataKeepsByteOrderMark() { + let reply = RedisReply.data(Data([0xEF, 0xBB, 0xBF] + Array("user:1".utf8))) + #expect(reply.stringValue?.unicodeScalars.first == "\u{FEFF}") + #expect(reply.displayText.unicodeScalars.first == "\u{FEFF}") + } +} + +// MARK: - SCAN reply + +struct RedisScanReplyTests { + private static let markedKey = Data([0xEF, 0xBB, 0xBF] + Array("user:1".utf8)) + + @Test("A key that starts with a byte order mark is listed with it") + func keyKeepsByteOrderMark() throws { + let page = RedisScanReply.parse(.array([.string("0"), .array([.data(Self.markedKey), .string("user:1")])])) + #expect(page.keys.count == 2) + let first = try #require(page.keys.first) + #expect(Data(first.utf8) == Self.markedKey) + #expect(page.keys.last == "user:1") + } + + @Test("A key that is not UTF-8 is still left out") + func binaryKeyLeftOut() { + let page = RedisScanReply.parse(.array([.string("7"), .array([.data(Data([0xFF, 0xFE])), .string("a")])])) + #expect(page.cursor == "7") + #expect(page.keys == ["a"]) + } } // MARK: - intValue diff --git a/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift b/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift index 8710b7d525..1739b3dccb 100644 --- a/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift +++ b/apple/Plugins/RedisDriverTests/RedisStatementGeneratorTests.swift @@ -1219,4 +1219,105 @@ struct RedisStatementGeneratorRefusalTests { #expect(written.map(\.statement) == ["EXPIRE mykey 60", "DEL {u}a {u}b", "DEL x"]) #expect(written.map(\.rowIndices) == [[0], [1, 3], [2]]) } + + private static let shownInPart = PluginRowWriteRefusal( + rowIndex: 0, + reason: "The grid shows only the start of this value, so it cannot be edited here. " + + "Change it with a command in the query editor." + ) + + @Test("A Value edit of a string longer than the grid reads is refused") + func valueShownInPartRefused() { + let change = edit( + [(columnIndex: 4, columnName: "Value", oldValue: "start...", newValue: "start")], + original: ["page:1", "string", "-1", .text(String(RedisStringPreview.byteLimit + 1)), "start..."] + ) + #expect(throws: Self.shownInPart) { try writes(for: [change]) } + } + + @Test("A string exactly as long as the grid reads is written") + func valueAtTheLimitWritten() throws { + let change = edit( + [(columnIndex: 4, columnName: "Value", oldValue: "old", newValue: "new")], + original: ["page:1", "string", "-1", .text(String(RedisStringPreview.byteLimit)), "old"] + ) + #expect(try writes(for: [change]).map(\.statement) == ["SET page:1 new KEEPTTL"]) + } + + @Test("A TTL edit and a rename of a string shown in part still write") + func otherEditsOfAShortenedRowWrite() throws { + let change = edit( + [ + (columnIndex: 2, columnName: "TTL", oldValue: "-1", newValue: "60"), + (columnIndex: 0, columnName: "Key", oldValue: "page:1", newValue: "page:2"), + ], + original: ["page:1", "string", "-1", "900000", "start..."] + ) + #expect(try writes(for: [change]).map(\.statement) == ["EXPIRE page:1 60", "MUSTAPPLY RENAMENX page:1 page:2"]) + } + + @Test("Without a length, a Value cell that filled the read range is refused") + func unknownLengthFilledRangeRefused() throws { + let filled = String(repeating: "a", count: RedisStringPreview.byteLimit - 3) + RedisStringPreview.marker + let refused = edit( + [(columnIndex: 4, columnName: "Value", oldValue: .text(filled), newValue: "b")], + original: ["page:1", "string", "-1", .null, .text(filled)] + ) + #expect(throws: Self.shownInPart) { try writes(for: [refused]) } + + let short = edit( + [(columnIndex: 4, columnName: "Value", oldValue: "old", newValue: "new")], + original: ["page:1", "string", "-1", .null, "old"] + ) + #expect(try writes(for: [short]).map(\.statement) == ["SET page:1 new KEEPTTL"]) + } +} + +// Duplicate copies the Value cell the grid showed, which for a long string or a large collection is only its start. +struct RedisCopiedPreviewInsertTests { + private static let browseColumns = ["Key", "Type", "TTL", "Length", "Value"] + + private static let partRefusal = PluginRowWriteRefusal( + rowIndex: 0, + reason: "This row holds only the part of a value the grid shows, so adding it would store that part. " + + "Add the key with a command in the query editor." + ) + + private func statements(type: String, length: PluginCellValue, value: PluginCellValue) throws -> [String] { + var gen = RedisStatementGenerator(namespaceName: "", columns: Self.browseColumns) + gen.stagingToken = { "t" } + let change = PluginRowChange(rowIndex: 0, type: .insert, cellChanges: [], originalRow: nil) + return try gen.generateRowWrites( + from: [change], + insertedRowData: [0: ["copy", .text(type), nil, length, value]], + deletedRowIndices: [], + insertedRowIndices: [0] + ).map(\.statement) + } + + @Test("A copied string the grid showed in part is refused") + func copiedShortenedStringRefused() { + let shown = String(repeating: "a", count: RedisStringPreview.byteLimit) + RedisStringPreview.marker + let length = PluginCellValue.text(String(RedisStringPreview.byteLimit + 10)) + #expect(throws: Self.partRefusal) { try statements(type: "string", length: length, value: .text(shown)) } + #expect(throws: Self.partRefusal) { + try statements(type: "string", length: length, value: .bytes(Data(repeating: 0xFF, count: RedisStringPreview.byteLimit))) + } + } + + @Test("A copied string shown whole is added") + func copiedWholeStringAdded() throws { + #expect(try statements(type: "string", length: "5", value: "hello") == ["MUSTAPPLY SET copy hello NX"]) + } + + @Test("A copied collection holding fewer elements than its length is refused") + func copiedCollectionPreviewRefused() { + #expect(throws: Self.partRefusal) { try statements(type: "list", length: "250", value: #"["a","b"]"#) } + #expect(throws: Self.partRefusal) { try statements(type: "hash", length: "3", value: #"{"f":"v"}"#) } + } + + @Test("A copied collection holding every element is added") + func copiedWholeCollectionAdded() throws { + #expect(try statements(type: "set", length: "2", value: #"["a","b"]"#).count == 3) + } } diff --git a/apple/Plugins/RedisDriverTests/StubRedisChannel.swift b/apple/Plugins/RedisDriverTests/StubRedisChannel.swift index c8d8c7a578..edf21d90e9 100644 --- a/apple/Plugins/RedisDriverTests/StubRedisChannel.swift +++ b/apple/Plugins/RedisDriverTests/StubRedisChannel.swift @@ -9,6 +9,7 @@ import Foundation import TableProPluginKit +import TableProRedisCore /// Driven by one task at a time, so the outcomes are handed out in order with no synchronisation. /// It also stands in for one node of a cluster, which hands it the cluster's database. @@ -16,6 +17,7 @@ final class StubRedisChannel: RedisClusterNodeConnection, @unchecked Sendable { private var outcomes: [Result] private(set) var sentCommands: [[String]] = [] private(set) var sentScopes: [RedisCommandScope] = [] + private(set) var pipelineSizes: [Int] = [] private(set) var footprint = RedisSessionFootprint() let supportsDatabaseSelection: Bool private(set) var sessionDatabase: RedisSessionDatabase @@ -61,6 +63,7 @@ final class StubRedisChannel: RedisClusterNodeConnection, @unchecked Sendable { func forgetSentCommands() { sentCommands = [] sentScopes = [] + pipelineSizes = [] } /// Mirrors the hiredis connection: a SELECT queued in an open block moves nothing yet and @@ -105,6 +108,7 @@ final class StubRedisChannel: RedisClusterNodeConnection, @unchecked Sendable { /// connection does, because every command is on the wire before the first reply is read. func executePipeline(_ commands: [[Data]], scope: RedisCommandScope) async throws -> [RedisReply] { let pipeline = commands.map(decoded) + pipelineSizes.append(pipeline.count) try admit(scope, command: pipeline.first ?? []) try moveToCommandDatabase(command: pipeline.first?.first) let replies = try pipeline.map { try send($0, scope: scope) } @@ -136,7 +140,7 @@ final class StubRedisChannel: RedisClusterNodeConnection, @unchecked Sendable { } private func decoded(_ args: [Data]) -> [String] { - args.map { String(data: $0, encoding: .utf8) ?? "" } + args.map { RedisWireText.decode($0) ?? "" } } private func admit(_ scope: RedisCommandScope, command: [String]) throws { diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealCBOR.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealCBOR.swift index 46d05263f4..9db4a8b98c 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealCBOR.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealCBOR.swift @@ -121,10 +121,7 @@ public enum SurrealCBOR { return .bytes(Data(try cursor.next(count))) case 3: let count = try checkedCount(try readArgument(&cursor, info: info)) - guard let text = String(bytes: try cursor.next(count), encoding: .utf8) else { - throw SurrealCBORError.invalidUTF8 - } - return .string(text) + return .string(try exactText(try cursor.next(count))) case 4: let count = try checkedCount(try readArgument(&cursor, info: info)) var items: [SurrealValue] = [] @@ -154,6 +151,13 @@ public enum SurrealCBOR { } } + // String(bytes:encoding:) drops a leading byte order mark, which names a different record or field. + private static func exactText(_ bytes: [UInt8]) throws -> String { + let text = String(decoding: bytes, as: UTF8.self) // swiftlint:disable:this optional_data_string_conversion + guard text.utf8.elementsEqual(bytes) else { throw SurrealCBORError.invalidUTF8 } + return text + } + private static func fallbackKey(_ value: SurrealValue) -> String { switch value { case let .int(number): diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift index fb2c57d7a3..4aafdff170 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealDBPluginDriver.swift @@ -345,13 +345,15 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { let flattened = SurrealRowFlattener.flatten(last.value, length: length) let affected = rowsAffected(last.value) - return PluginQueryResult( + var result = PluginQueryResult( columns: flattened.columns, columnTypeNames: flattened.columnTypeNames, rows: flattened.rows, rowsAffected: affected, executionTime: elapsed ) + result.rowLocators = SurrealRowFlattener.rowLocators(last.value) + return result } private static func rowsAffected(_ value: SurrealValue) -> Int { @@ -457,6 +459,24 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, insertedRowIndices: Set + ) throws -> [PluginRowWrite]? { + try generateRowWrites( + table: table, schema: schema, columns: columns, primaryKeyColumns: primaryKeyColumns, changes: changes, + insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, + insertedRowIndices: insertedRowIndices, context: PluginRowWriteContext() + ) + } + + func generateRowWrites( + table: String, + schema: String?, + columns: [String], + primaryKeyColumns: [String], + changes: [PluginRowChange], + insertedRowData: [Int: [PluginCellValue]], + deletedRowIndices: Set, + insertedRowIndices: Set, + context: PluginRowWriteContext ) throws -> [PluginRowWrite]? { try SurrealStatementGenerator.rowWrites( table: table, @@ -469,7 +489,8 @@ final class SurrealDBPluginDriver: PluginDatabaseDriver, @unchecked Sendable { changes: changes, insertedRowData: insertedRowData, deletedRowIndices: deletedRowIndices, - insertedRowIndices: insertedRowIndices + insertedRowIndices: insertedRowIndices, + rowLocators: context.rowLocators ) } diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealQL.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealQL.swift index bf54b3a6a0..97bb1803c3 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealQL.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealQL.swift @@ -46,6 +46,21 @@ public enum SurrealQL { } } + // `recordIdPart` backticks a uuid, an array and an object id like a string that needs quoting, so a backticked + // string is that string only when it cannot read as one of them. + public static func isRecoverableFromText(_ id: SurrealValue) -> Bool { + switch id { + case .int: + return true + case let .string(text): + if isSimpleIdentifier(text) { return true } + guard UUID(uuidString: text) == nil, let first = text.unicodeScalars.first else { return false } + return first != "[" && first != "{" + default: + return false + } + } + public static func recordLiteral(_ record: SurrealRecordID) -> String { quoteIdentifier(record.table) + ":" + recordIdPart(record.id) } diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealRowFlattener.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealRowFlattener.swift index a5d692e2ee..b681cf4449 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealRowFlattener.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealRowFlattener.swift @@ -49,6 +49,18 @@ public enum SurrealRowFlattener { ) } + // One per row `flatten` returns: the record's id for a row that is a record, nil for any other row. Nil when the + // rows are not records, which `flatten` shows in a single column with no id to edit by. + public static func rowLocators(_ value: SurrealValue) -> [String?]? { + let records = normalize(value) + guard !records.isEmpty, records.allSatisfy({ $0.objectPairs != nil }) else { return nil } + let locators = records.map { record -> String? in + guard case let .recordId(recordId)? = record[SurrealInfoParser.recordIdColumn] else { return nil } + return SurrealRowLocator.locator(for: recordId) + } + return locators.contains(where: { $0 != nil }) ? locators : nil + } + public static func cell(_ value: SurrealValue, length: SurrealTextLength) -> PluginCellValue { switch value { case .null, .none: @@ -114,3 +126,17 @@ public enum SurrealRowFlattener { } } } + +// The id cell shows a uuid, an array, an object and a string id that needs quoting all as backticked text, so a row +// is found again by its id as CBOR: the same record always encodes to the same bytes. +public enum SurrealRowLocator { + public static func locator(for record: SurrealRecordID) -> String { + SurrealCBOR.encode(.recordId(record)).base64EncodedString() + } + + public static func record(from locator: String) -> SurrealRecordID? { + guard let data = Data(base64Encoded: locator), + case let .recordId(record)? = try? SurrealCBOR.decode(data) else { return nil } + return record + } +} diff --git a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift index f8fb1604eb..627c0bff72 100644 --- a/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift +++ b/apple/Plugins/SurrealDBDriverPlugin/SurrealStatementGenerator.swift @@ -19,7 +19,8 @@ public enum SurrealStatementGenerator { changes: [PluginRowChange], insertedRowData: [Int: [PluginCellValue]], deletedRowIndices: Set, - insertedRowIndices: Set + insertedRowIndices: Set, + rowLocators: [Int: String] = [:] ) throws(PluginRowWriteRefusal) -> [PluginRowWrite] { var writes: [PluginRowWrite] = [] var insertedWritten: Set = [] @@ -41,12 +42,15 @@ public enum SurrealStatementGenerator { )) continue } + let locator = rowLocators[index] if change.type == .update, - let write = try update(table: table, scope: scope, columns: columns, typing: typing, change: change) { + let write = try update( + table: table, scope: scope, columns: columns, typing: typing, change: change, locator: locator + ) { writes.append(write) } if change.type == .delete || deletedRowIndices.contains(index), - let write = delete(table: table, scope: scope, columns: columns, change: change) { + let write = try delete(table: table, scope: scope, columns: columns, change: change, locator: locator) { writes.append(write) } } @@ -90,15 +94,18 @@ public enum SurrealStatementGenerator { scope: SurrealScope, columns: [String], typing: FieldTyping, - change: PluginRowChange + change: PluginRowChange, + locator: String? ) throws(PluginRowWriteRefusal) -> PluginRowWrite? { if let reserved = change.cellChanges.first(where: { SurrealInfoParser.isReservedColumn($0.columnName) }) { throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: reservedColumnReason(reserved.columnName)) } - let recordKey = Self.recordText(columns: columns, originalRow: change.originalRow) - guard let record = recordKey.flatMap({ SurrealQL.parseRecordId($0, fallbackTable: table) }) else { return nil } let editable = change.cellChanges.filter { !change.isColumnDefault($0.columnIndex) } - guard !editable.isEmpty else { return nil } + guard !editable.isEmpty, + let record = try targetRecord(table: table, columns: columns, change: change, locator: locator) + else { return nil } + // The record kinds are keyed by the literal the read showed. + let recordKey = record.literal var parameters: [PluginCellValue] = [SurrealCellCoder.parameter(.recordId(record))] var assignments: [String] = [] @@ -169,9 +176,12 @@ public enum SurrealStatementGenerator { table: String, scope: SurrealScope, columns: [String], - change: PluginRowChange - ) -> PluginRowWrite? { - guard let record = recordId(table: table, columns: columns, originalRow: change.originalRow) else { return nil } + change: PluginRowChange, + locator: String? + ) throws(PluginRowWriteRefusal) -> PluginRowWrite? { + guard let record = try targetRecord(table: table, columns: columns, change: change, locator: locator) else { + return nil + } return PluginRowWrite( statement: SurrealQueryBuilder.compose(scope: scope, statement: "DELETE $p0;"), parameters: [SurrealCellCoder.parameter(.recordId(record))], @@ -206,14 +216,35 @@ public enum SurrealStatementGenerator { return String(format: format, column) } - private static func recordId( + private static var unreadableRecordIdReason: String { + String( + localized: "This record's id reads like a UUID, array or object id, so its text does not say which record it is. Open the table to edit it, or use a query." + ) + } + + private static var unknownRecordReason: String { + String(localized: "This record cannot be found again from what was read. Refresh and try again.") + } + + // The locator is the id the row was read with. A row without one, from a query tab, is named by its id cell. + private static func targetRecord( table: String, columns: [String], - originalRow: [PluginCellValue]? - ) -> SurrealRecordID? { - recordText(columns: columns, originalRow: originalRow).flatMap { - SurrealQL.parseRecordId($0, fallbackTable: table) + change: PluginRowChange, + locator: String? + ) throws(PluginRowWriteRefusal) -> SurrealRecordID? { + if let locator { + guard let record = SurrealRowLocator.record(from: locator) else { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: unknownRecordReason) + } + return record + } + guard let text = recordText(columns: columns, originalRow: change.originalRow), + let record = SurrealQL.parseRecordId(text, fallbackTable: table) else { return nil } + guard SurrealQL.isRecoverableFromText(record.id) else { + throw PluginRowWriteRefusal(rowIndex: change.rowIndex, reason: unreadableRecordIdReason) } + return record } private static func recordText(columns: [String], originalRow: [PluginCellValue]?) -> String? { diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 59ea7fab79..1caafe4100 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -146205,6 +146205,10 @@ } } }, + "Refresh before changing this item, so the save can check that it has not changed since it was loaded." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "Refresh concurrently" : { "localizations" : { "fr" : { @@ -189409,6 +189413,10 @@ }, "The global_instrumentation consumer is disabled, so the server records no index reads." : { + }, + "The grid shows only the start of this value, so it cannot be edited here. Change it with a command in the query editor." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "the group coordinator is not available" : { "extractionState" : "manual", @@ -189612,6 +189620,14 @@ } } }, + "The item changed since its CAS value was read, so nothing was saved. Refresh to see its current value." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "The item does not exist, so nothing was saved. Refresh to see the current keys." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "The JSON array must contain only objects." : { }, @@ -196171,6 +196187,10 @@ }, "The value ends early, at byte %lld." : { + }, + "The value in %@ expired after the grid read it. Refresh, then edit the row again." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "The value in %@ is shortened for display, so saving it would store only the part shown. Change this field with a query." : { "extractionState" : "manual", @@ -201703,6 +201723,14 @@ } } }, + "This record cannot be found again from what was read. Refresh and try again." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, + "This record's id reads like a UUID, array or object id, so its text does not say which record it is. Open the table to edit it, or use a query." : { + "extractionState" : "manual", + "generatesSymbol" : false + }, "This replaces %@ on this Mac." : { }, @@ -201759,6 +201787,10 @@ }, "This routine takes no arguments." : { + }, + "This row cannot be found again from what was read. Reload the table." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "This row does not name a stored document." : { "extractionState" : "manual", @@ -201797,6 +201829,10 @@ }, "This row has no value for this field." : { + }, + "This row holds only the part of a value the grid shows, so adding it would store that part. Add the key with a command in the query editor." : { + "extractionState" : "manual", + "generatesSymbol" : false }, "This row's _id is binary data whose subtype is not known, so TablePro cannot tell which document it names." : { "extractionState" : "manual", diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBOnIOSTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBOnIOSTests.swift index 3d73e531ea..98201d824a 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBOnIOSTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/MongoDBOnIOSTests.swift @@ -73,7 +73,7 @@ struct MongoDBOnIOSTests { offset: 100 ) - #expect(query == #"db.users.find({"$and": [{"name": "ann"}, {"$or": [{"name": {"$regex": "x", "$options": "i"}}, {"email": {"$regex": "x", "$options": "i"}}]}]}).sort({"name": 1}).skip(100).limit(50)"#) + #expect(query == #"db.users.find({"$and": [{"name": "ann"}, {"$or": [{"name": {"$regex": "x", "$options": "i"}}, {"email": {"$regex": "x", "$options": "i"}}]}]}).sort({"name": 1, "_id": 1}).skip(100).limit(50)"#) } @Test("From MongoDB 4.2, search matches a number, a date or an ObjectId by its text as well") diff --git a/apple/changelog.d/4220.fixed.1.md b/apple/changelog.d/4220.fixed.1.md new file mode 100644 index 0000000000..582a62eb1e --- /dev/null +++ b/apple/changelog.d/4220.fixed.1.md @@ -0,0 +1 @@ +Cassandra and ScyllaDB edits and undone deletes no longer remove the time to live of the cells they write. diff --git a/apple/changelog.d/4220.fixed.2.md b/apple/changelog.d/4220.fixed.2.md new file mode 100644 index 0000000000..0a4102b63c --- /dev/null +++ b/apple/changelog.d/4220.fixed.2.md @@ -0,0 +1 @@ +Kafka pages from the newest message no longer skip messages in topics with transactions or compaction. diff --git a/apple/changelog.d/4220.fixed.3.md b/apple/changelog.d/4220.fixed.3.md new file mode 100644 index 0000000000..80c1da3c1c --- /dev/null +++ b/apple/changelog.d/4220.fixed.3.md @@ -0,0 +1 @@ +MongoDB pages sorted by a column with repeated values no longer repeat or skip documents. diff --git a/apple/changelog.d/4220.fixed.4.md b/apple/changelog.d/4220.fixed.4.md new file mode 100644 index 0000000000..ac0200526b --- /dev/null +++ b/apple/changelog.d/4220.fixed.4.md @@ -0,0 +1 @@ +Redis, Cassandra and Bigtable keys that start with a byte order mark are no longer edited or deleted as another key. diff --git a/apple/changelog.d/4220.fixed.5.md b/apple/changelog.d/4220.fixed.5.md new file mode 100644 index 0000000000..74dd454109 --- /dev/null +++ b/apple/changelog.d/4220.fixed.5.md @@ -0,0 +1 @@ +Browsing a Redis database no longer reads every string value whole. diff --git a/apple/changelog.d/4220.fixed.6.md b/apple/changelog.d/4220.fixed.6.md new file mode 100644 index 0000000000..0e7ad833ef --- /dev/null +++ b/apple/changelog.d/4220.fixed.6.md @@ -0,0 +1 @@ +Elasticsearch edits and deletes of documents indexed with custom routing no longer fail or change another document. diff --git a/apple/changelog.d/4220.fixed.7.md b/apple/changelog.d/4220.fixed.7.md new file mode 100644 index 0000000000..373d89711e --- /dev/null +++ b/apple/changelog.d/4220.fixed.7.md @@ -0,0 +1 @@ +Editing an Elasticsearch text or keyword value that reads as a JSON array no longer stores an array. diff --git a/apple/changelog.d/4220.fixed.8.md b/apple/changelog.d/4220.fixed.8.md new file mode 100644 index 0000000000..457a1f2032 --- /dev/null +++ b/apple/changelog.d/4220.fixed.8.md @@ -0,0 +1 @@ +Editing a MongoDB number no longer changes its type to the one most of the column holds. diff --git a/apple/changelog.d/4220.fixed.9.md b/apple/changelog.d/4220.fixed.9.md new file mode 100644 index 0000000000..2bdcfdc29c --- /dev/null +++ b/apple/changelog.d/4220.fixed.9.md @@ -0,0 +1 @@ +Changing a Memcached item's flags no longer writes back the value the grid loaded. diff --git a/apple/changelog.d/4220.fixed.md b/apple/changelog.d/4220.fixed.md new file mode 100644 index 0000000000..bb2ef3c042 --- /dev/null +++ b/apple/changelog.d/4220.fixed.md @@ -0,0 +1 @@ +SurrealDB edits and deletes of a record whose id is a UUID, array or object now change that record.