diff --git a/apple/Packages/TableProCore/Sources/TableProModels/Cell.swift b/apple/Packages/TableProCore/Sources/TableProModels/Cell.swift index 2a9a28bd92..9f35ef576b 100644 --- a/apple/Packages/TableProCore/Sources/TableProModels/Cell.swift +++ b/apple/Packages/TableProCore/Sources/TableProModels/Cell.swift @@ -90,26 +90,68 @@ public extension Row { } } } + + // A one-line list never needs more than a preview, even of a value the cell holds whole. + func displayValues(maxBytes: Int = Cell.previewByteLimit) -> [String?] { + cells.map { cell -> String? in + switch cell { + case .null: + return nil + case .text(let value): + return Cell.preview(of: value, maxBytes: maxBytes) + case .truncatedText, .binary: + return cell.displayString + } + } + } } public extension Cell { + static let previewByteLimit = 4_096 + + // For text whose kind only the column's declared type tells, such as a row read back as strings. static func from(legacyValue value: String?, columnTypeName: String?, options: StreamOptions, ref: CellRef? = nil) -> Cell { guard let value else { return .null } - let bytes = value.utf8.count - let upper = (columnTypeName ?? "").uppercased() - let isBinary = upper.contains("BLOB") || upper.contains("BYTEA") || upper.contains("BINARY") || upper.contains("VARBINARY") || upper.contains("IMAGE") + if let columnTypeName, holdsBytes(typeName: columnTypeName) { + return .binary(byteCount: value.utf8.count, ref: ref) + } + return from(text: value, options: options, ref: ref) + } - if isBinary { - return .binary(byteCount: bytes, ref: ref) + // For a value the driver knows is text, whatever its column's type is called. + static func from(text value: String, options: StreamOptions, ref: CellRef? = nil) -> Cell { + let bytes = value.utf8.count + guard bytes > options.textTruncationBytes else { return .text(value) } + // A value with a ref can be read again by its key, so it never spends the budget. + if ref == nil, let budget = options.textBudget, budget.admit(bytes) { + return .text(value) } + return .truncatedText(prefix: preview(of: value, maxBytes: options.textTruncationBytes), totalBytes: bytes, ref: ref) + } - if bytes > options.textTruncationBytes { - let prefixSlice = value.prefix(options.textTruncationBytes) - return .truncatedText(prefix: String(prefixSlice), totalBytes: bytes, ref: ref) + // At most `maxBytes` of UTF-8, ending on a Character boundary. + static func preview(of value: String, maxBytes: Int) -> String { + guard value.utf8.count > maxBytes else { return value } + var end = value.startIndex + var used = 0 + while end < value.endIndex { + let next = value.index(after: end) + used += value.utf8.distance(from: end, to: next) + guard used <= maxBytes else { break } + end = next } + return String(value[.. Bool { + let base = typeName.prefix { $0 != "(" }.trimmingCharacters(in: .whitespaces).uppercased() + return byteTypeNames.contains(base) } + + private static let byteTypeNames: Set = [ + "BLOB", "TINYBLOB", "MEDIUMBLOB", "LONGBLOB", "BYTEA", "BINARY", "VARBINARY", "IMAGE" + ] } private let byteCountFormatter = OSAllocatedUnfairLock(uncheckedState: { diff --git a/apple/Packages/TableProCore/Sources/TableProModels/StreamingResult.swift b/apple/Packages/TableProCore/Sources/TableProModels/StreamingResult.swift index fd619d01e1..46043693fe 100644 --- a/apple/Packages/TableProCore/Sources/TableProModels/StreamingResult.swift +++ b/apple/Packages/TableProCore/Sources/TableProModels/StreamingResult.swift @@ -1,4 +1,5 @@ import Foundation +import os public enum StreamElement: Sendable { case columns([ColumnInfo]) @@ -20,22 +21,55 @@ public struct StreamOptions: Sendable { public let inlineBinary: Bool public let maxRows: Int public let lazyContext: LazyContext? + // Long text with no ref stays whole while this admits it; nil cuts every long value. + public var textBudget: TextBudget? public init( - textTruncationBytes: Int = 4_096, + textTruncationBytes: Int = Cell.previewByteLimit, inlineBinary: Bool = false, maxRows: Int = 100_000, - lazyContext: LazyContext? = nil + lazyContext: LazyContext? = nil, + textBudget: TextBudget? = nil ) { self.textTruncationBytes = textTruncationBytes self.inlineBinary = inlineBinary self.maxRows = maxRows self.lazyContext = lazyContext + self.textBudget = textBudget } public static let `default` = StreamOptions() } +// Shared by every cell of one request, and read from whatever thread the driver decodes on. +public final class TextBudget: Sendable { + private struct State: Sendable { + var remaining: Int + var refused = false + } + + private let state: OSAllocatedUnfairLock + + public init(bytes: Int) { + state = OSAllocatedUnfairLock(initialState: State(remaining: max(0, bytes))) + } + + public func admit(_ bytes: Int) -> Bool { + state.withLock { state in + guard bytes <= state.remaining else { + state.refused = true + return false + } + state.remaining -= bytes + return true + } + } + + public var hasRefused: Bool { + state.withLock { $0.refused } + } +} + public struct LazyContext: Sendable { public let table: String public let primaryKeyColumns: [String] diff --git a/apple/Packages/TableProCore/Tests/TableProModelsTests/CellTests.swift b/apple/Packages/TableProCore/Tests/TableProModelsTests/CellTests.swift new file mode 100644 index 0000000000..6b6714d5f0 --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProModelsTests/CellTests.swift @@ -0,0 +1,133 @@ +import Foundation +import Testing + +@testable import TableProModels + +struct CellTests { + private static let long = String(repeating: "a", count: 10_000) + + @Test("A budget keeps long values whole until it is spent, then the next keeps only a preview") + func budgetKeepsWholeUntilSpent() { + let budget = TextBudget(bytes: 15_000) + let options = StreamOptions(textBudget: budget) + + guard case .text(let kept) = Cell.from(text: Self.long, options: options) else { + Issue.record("the first long value should be kept whole") + return + } + #expect(kept == Self.long) + #expect(!budget.hasRefused) + + guard case .truncatedText(let prefix, let totalBytes, let ref) = Cell.from(text: Self.long, options: options) else { + Issue.record("the second long value should be cut") + return + } + #expect(prefix.utf8.count == Cell.previewByteLimit) + #expect(totalBytes == 10_000) + #expect(ref == nil) + #expect(budget.hasRefused) + } + + @Test("A value within the cut spends nothing") + func shortValuesSpendNothing() { + let budget = TextBudget(bytes: 10_000) + let options = StreamOptions(textBudget: budget) + for _ in 0 ..< 10 { + _ = Cell.from(text: String(repeating: "b", count: Cell.previewByteLimit), options: options) + } + + guard case .text = Cell.from(text: Self.long, options: options) else { + Issue.record("the long value should still fit the budget") + return + } + #expect(!budget.hasRefused) + } + + @Test("Without a budget every long value is cut") + func noBudgetCuts() { + guard case .truncatedText(_, let totalBytes, .none) = Cell.from(text: Self.long, options: .default) else { + Issue.record("the long value should be cut") + return + } + #expect(totalBytes == 10_000) + } + + @Test("A value with a ref stays cut and loadable, and spends none of the budget") + func refValuesSpendNothing() { + let budget = TextBudget(bytes: 10_000) + let options = StreamOptions(textBudget: budget) + let ref = CellRef(table: "notes", column: "body", primaryKey: [PrimaryKeyComponent(column: "id", value: "1")]) + + guard case .truncatedText(_, _, let kept?) = Cell.from(text: Self.long, options: options, ref: ref) else { + Issue.record("a value with a ref should stay cut") + return + } + #expect(kept == ref) + guard case .text = Cell.from(text: Self.long, options: options) else { + Issue.record("the budget should be untouched") + return + } + } + + @Test( + "The preview holds at most the cut in bytes and ends on a whole character", + arguments: ["界", "👍🏽", "e\u{301}", "x"] + ) + func previewIsBoundedInBytes(_ unit: String) { + let value = String(repeating: unit, count: 5_000) + + let preview = Cell.preview(of: value, maxBytes: 4_096) + + #expect(preview.utf8.count <= 4_096) + #expect(preview.utf8.count > 4_096 - unit.utf8.count) + #expect(preview == String(repeating: unit, count: preview.count)) + } + + @Test("A cut value's preview is bounded in bytes, not characters") + func cutPreviewIsBoundedInBytes() { + let value = String(repeating: "界", count: 5_000) + + guard case .truncatedText(let prefix, let totalBytes, _) = Cell.from(text: value, options: .default) else { + Issue.record("the value should be cut") + return + } + #expect(prefix.utf8.count <= Cell.previewByteLimit) + #expect(totalBytes == 15_000) + } + + @Test( + "Oracle's BINARY_DOUBLE, BINARY_FLOAT and BINARY_INTEGER hold numbers, not bytes", + arguments: ["binary_double", "BINARY_FLOAT", "binary_integer"] + ) + func oracleBinaryNumbersStayText(_ typeName: String) { + guard case .text(let value) = Cell.from(legacyValue: "1.5", columnTypeName: typeName, options: .default) else { + Issue.record("\(typeName) should read as text") + return + } + #expect(value == "1.5") + } + + @Test( + "A byte type, with or without a length, reads as binary", + arguments: ["BLOB", "bytea", "VARBINARY(16)", "binary(16)", "LONGBLOB", "image", "BLOB(3)"] + ) + func byteTypesAreBinary(_ typeName: String) { + guard case .binary(let byteCount, _) = Cell.from(legacyValue: "0A0B", columnTypeName: typeName, options: .default) else { + Issue.record("\(typeName) should read as binary") + return + } + #expect(byteCount == 4) + } + + @Test("Display values cut a long value the cell holds whole") + func displayValuesCutKeptText() { + let row = Row(cells: [.text(Self.long), .null, .truncatedText(prefix: "a", totalBytes: 10_000, ref: nil)]) + + let shown = row.displayValues() + + #expect(shown[0]?.utf8.count == Cell.previewByteLimit) + #expect(shown[1] == nil) + #expect(shown[2] == row.cells[2].displayString) + #expect(row.legacyValues[0] == Self.long) + } +} diff --git a/apple/TableProMobile/TableProMobile/AppState.swift b/apple/TableProMobile/TableProMobile/AppState.swift index 859465654b..6b93ba3fc2 100644 --- a/apple/TableProMobile/TableProMobile/AppState.swift +++ b/apple/TableProMobile/TableProMobile/AppState.swift @@ -40,6 +40,7 @@ final class AppState { let backgroundRelease: BackgroundReleaseCoordinator let queryActivities = QueryActivityController() let catalogChanges = CatalogChangeBoard() + let sessionModels = ConnectionSessionModels() let syncCoordinator: IOSSyncCoordinator @ObservationIgnored private var automaticPresentationOwner: UUID? @@ -186,6 +187,7 @@ final class AppState { private func sessionEnded(_ connectionId: UUID, driver: any DatabaseDriver) { sessionTransactions.sessionEnded(connectionId, driver: driver) + sessionModels.existing(connectionId)?.sessionEnded(driver) bumpSessionRevision(connectionId) } @@ -225,6 +227,7 @@ final class AppState { let removedIds = Set(connections.map(\.id)).subtracting(merged.map(\.id)) guard persist(connections: merged, onlyIfSaved: true) else { return false } localState.purge(removedIds) + sessionModels.purge(removedIds) savedQueries.removeRecords(ofConnections: removedIds, tombstoning: false) publishLibrary() return true @@ -441,6 +444,7 @@ final class AppState { guard !removed.isEmpty else { return } persist(connections: connections.filter { !ids.contains($0.id) }) localState.purge(Set(removed.map(\.id))) + sessionModels.purge(Set(removed.map(\.id))) let syncedIds = Set(removed.filter(\.participatesInSync).map(\.id)) savedQueries.removeRecords(ofConnections: syncedIds, tombstoning: true) savedQueries.removeRecords(ofConnections: Set(removed.map(\.id)).subtracting(syncedIds), tombstoning: false) diff --git a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinator.swift b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinator.swift index cf501acb3f..f5be407722 100644 --- a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinator.swift +++ b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinator.swift @@ -4,39 +4,46 @@ import os import SwiftUI import TableProConnectionLibrary import TableProDatabase -import TableProLogRedaction import TableProModels import TableProSQLGrammar +// One window's view of a connection. The session, its lists and its lifecycle are the connection's +// `ConnectionSessionModel`, shared by every window; this keeps what is the window's own and asks the model for the rest. @MainActor @Observable final class ConnectionCoordinator { - private(set) var connection: DatabaseConnection - - private(set) var session: ConnectionSession? - private(set) var phase: ConnectionPhase = .connecting - private(set) var tables: [TableInfo] = [] { - didSet { selectedTable = TableSelectionResolver.keeping(selectedTable, in: tables) } - } - private(set) var databases: [String] = [] - private(set) var schemas: [String] = [] - private(set) var activeDatabase: String = "" - private(set) var activeSchema: String = "public" + private let model: ConnectionSessionModel + private let appState: AppState - private(set) var isSwitching = false { - didSet { resumeFollowIfPending() } - } + // False until the window connects, and again once it walks away: it then shows no session, whatever the model + // still holds for other windows. + private(set) var isAttached = false + // What the window showed when the session moved under its unsaved edit; it stays there until the edit ends. + private var pin: ConnectionSnapshot? + + var connection: DatabaseConnection { model.record } + var session: ConnectionSession? { isAttached ? shownState.session : nil } + var phase: ConnectionPhase { isAttached ? shownState.phase : .notConnected } + var isReconnecting: Bool { isAttached && shownState.isReconnecting } + var tables: [TableInfo] { shownCatalog.tables } + var databases: [String] { shownCatalog.databases } + var schemas: [String] { shownCatalog.schemas } + var activeDatabase: String { shownCatalog.placement.database } + var activeSchema: String { shownCatalog.placement.schema } + + private var shownState: ConnectionSessionState { pin?.state ?? model.state } + private var shownCatalog: ConnectionCatalog { pin?.catalog ?? model.catalog } + + private(set) var isSwitching = false private(set) var switchTarget: String? - private(set) var isReconnecting = false { - didSet { resumeFollowIfPending() } - } - // Another window moved the session while this one was busy; it follows once it is free. - @ObservationIgnored private var followPending = false - // A follow that changes the database rebuilds the table screen, which would drop an unsaved row edit. + // A move of the session would rebuild the table screen, which would drop an unsaved row edit. var isHeldByEditor = false { didSet { guard isHeldByEditor != oldValue else { return } appState.setEditorHold(isHeldByEditor, by: self, on: connection.id) - resumeFollowIfPending() + guard !isHeldByEditor, pin != nil else { return } + pin = nil + selectedTable = TableSelectionResolver.keeping(selectedTable, in: tables) + resumedFollow = Task { await followLiveSession() } } } @@ -44,19 +51,8 @@ final class ConnectionCoordinator { appState.holdsEditor(connection.id, besides: self) } - // One follow at a time, so an older one's late tables never land over a newer session's. - @ObservationIgnored private var isFollowing = false { - didSet { resumeFollowIfPending() } - } @ObservationIgnored private(set) var resumedFollow: Task? - @ObservationIgnored private var sessionCheck: Task? - @ObservationIgnored private var statementFailedDuringCheck = false - @ObservationIgnored private var metadataRefused = false - @ObservationIgnored private var seenCatalogGeneration: Int - @ObservationIgnored private var isReloadingCatalog = false - @ObservationIgnored private var catalogReloadRequested = false - // A redial that failed over an open transaction; the user hears about it once a later dial lands. - @ObservationIgnored private var unsettledLoss: SessionLoss? + @ObservationIgnored private var joiners = AttemptJoiners() private(set) var failureAlertTitle: String? var failureAlertMessage: String? var showFailureAlert = false @@ -74,10 +70,7 @@ final class ConnectionCoordinator { private(set) var keepsDamagedHistoryCopy = false private var historyStorage: QueryHistoryStorage { appState.queryHistory } - private let appState: AppState - var connectionManager: ConnectionManager { appState.connectionManager } - private static let logger = Logger(subsystem: "com.TablePro", category: "ConnectionCoordinator") enum ConnectionPhase: Sendable { case notConnected @@ -124,28 +117,14 @@ final class ConnectionCoordinator { } } - var displayName: String { - connection.name.isEmpty ? connection.host : connection.name - } - - var supportsDatabaseSwitching: Bool { - connection.type == .mysql || connection.type == .mariadb || - connection.type == .tidb || connection.type == .oceanbase || - connection.type == .postgresql || connection.type == .redshift || - connection.type == .mssql || connection.type == .clickhouse || - connection.type == .mongodb || connection.type == .cloudflareD1 - } - - var supportsSchemas: Bool { - connection.type == .postgresql || connection.type == .redshift || - connection.type == .mssql || connection.type == .duckdb || - connection.type == .oracle - } + var displayName: String { model.displayName } + var supportsDatabaseSwitching: Bool { model.supportsDatabaseSwitching } + var supportsSchemas: Bool { model.supportsSchemas } init(connection: DatabaseConnection, appState: AppState) { - self.connection = connection self.appState = appState - seenCatalogGeneration = appState.catalogChanges.generation(for: connection.id) + model = appState.sessionModels.model(for: connection, appState: appState) + model.register(self) } // MARK: - Persisted State @@ -156,87 +135,59 @@ final class ConnectionCoordinator { let tab = ConnectedTab(rawValue: savedTab) { selectedTab = tab } - activeDatabase = defaults.string(forKey: ConnectionDefaultsKey.lastDB.name(for: connection.id)) ?? "" - activeSchema = defaults.string(forKey: ConnectionDefaultsKey.lastSchema.name(for: connection.id)) ?? "public" + model.restorePersistedPlacement() } // MARK: - Connection Lifecycle - // The attempt allowed to write `session` and `phase`. Cancelling mints a new one. - private var attemptToken = UUID() - private var connectTask: Task? { - didSet { resumeFollowIfPending() } - } - - private var joiners = AttemptJoiners() - let prompts = ConnectionPromptQueue() - // Returning early without touching `phase` is what left the connecting screen up for good. + // The last caller to go away walks the window away, so a screen that closed mid-open leaves nothing waiting. func connect() async { prompts.expectScreen() - if let inFlight = connectTask { - await join(inFlight) - return - } - - let token = UUID() - attemptToken = token - phase = .connecting - - let task = Task { [weak self] in - guard let self else { return } - await self.runAttempt(token: token) - } - connectTask = task - await join(task) - if connectTask == task { connectTask = nil } - } - - // The last caller to go away abandons the attempt, so the next screen never joins it and waits forever. - private func join(_ task: Task) async { + isAttached = true let joiner = joiners.join() await withTaskCancellationHandler { - await task.value + await model.connect(from: self) _ = joiners.leave(joiner) } onCancel: { Task { @MainActor [weak self] in - guard let self else { return } - guard joiners.leave(joiner), connectTask == task else { return } + guard let self, joiners.leave(joiner), case .connecting = phase else { return } cancelConnect() } } + await model.runHeldBackWork(from: self) } func adopt(_ record: DatabaseConnection) { - guard record.id == connection.id, record != connection else { return } - connection = record + model.adopt(record) } func retire() { - attemptToken = UUID() - connectTask?.cancel() - connectTask = nil + isAttached = false + pin = nil prompts.cancelAll() if appState.sessionRedialer.leave(connection.id, consumer: self) { appState.connectionManager.invalidateAttempt(for: connection.id) } appState.setEditorHold(false, by: self, on: connection.id) - session = nil + model.leave(self) + } + + // The record changed how it dials, so no window keeps the session. + func dropSession() { + model.reset() } // Never waits on the driver, which ignores `Task.cancel()`. The questions go first: a reconnect or a - // database switch asks without owning `connectTask`, and a suspended one holds its tunnel open. - // A lost session's redial is the session check, not `connectTask`; retiring mints a new attempt token, so a dial - // that returns after Cancel is never adopted. + // database switch asks without being the open, and a suspended one holds its tunnel open. func cancelConnect() { prompts.cancelAll() - guard connectTask != nil || (isLost && isReconnecting) else { return } + guard isAttached, model.state.isOpening || (isLost && isReconnecting) else { return } retire() - phase = .notConnected } - // A lost session redials where this window was, so the screen of a row edit in progress is not rebuilt. + // A lost session redials where it was, so the screen of a row edit in progress is not rebuilt. func reconnect() async { if isLost { await restoreSession(after: .whileIdle) @@ -245,111 +196,6 @@ final class ConnectionCoordinator { } } - private func runAttempt(token: UUID) async { - // Another window's dial is shared once its schema is back, never torn down mid-dial. - await appState.sessionRedialer.settled(connection.id, consumer: self, prompter: prompts) - guard attemptToken == token else { return } - if let existing = appState.connectionManager.session(for: connection.id) { - do { - let existingTables = try await existing.driver.fetchTables(schema: nil) - guard attemptToken == token else { return } - session = existing - tables = existingTables - await loadDatabases() - await loadSchemas() - await refreshTransaction() - guard attemptToken == token else { return } - phase = .connected - appState.libraryPreferences.recordConnected(connection.id) - reportDroppedTransaction(unsettledLoss) - return - } catch { - guard attemptToken == token else { return } - if await keepsRefusingSession(existing, token: token, refusal: error) { return } - guard attemptToken == token else { return } - session = nil - await appState.sessionRedialer.settled(connection.id, consumer: self, prompter: prompts) - guard attemptToken == token else { return } - if let live = appState.connectionManager.session(for: connection.id), live.driver !== existing.driver { - await runAttempt(token: token) - return - } - await appState.connectionManager.disconnect(connection.id) - } - } - - guard attemptToken == token else { return } - await connectFresh(token: token) - } - - // `allowSignIn` is false on the retry that follows a sign-in, so a connection that keeps - // failing cannot put the prompt up again and again. That retry dials with the new credentials - // rather than join a dial another window started without them. - private func connectFresh(token: UUID, allowSignIn: Bool = true) async { - Telemetry.shared.connectionAttempted(databaseType: connection.type.rawValue) - - do { - let outcome = try await appState.sessionRedialer.redial( - connection, - schema: nil, - prompter: prompts, - joining: allowSignIn, - consumer: self, - stillWanted: { [weak self] in self?.attemptToken == token } - ) - let newSession = outcome.session - let newTables = try await newSession.driver.fetchTables(schema: nil) - guard attemptToken == token else { return } - if !outcome.dialedHere { - adoptPlacement(of: newSession, includingDatabase: true) - } - session = newSession - tables = newTables - if outcome.dialedHere { - appState.bumpSessionRevision(connection.id) - } - await loadDatabases() - await loadSchemas() - await refreshTransaction() - guard attemptToken == token else { return } - phase = .connected - appState.libraryPreferences.recordConnected(connection.id) - Telemetry.shared.connectionSucceeded(databaseType: connection.type.rawValue) - navigateToPendingTable() - reportDroppedTransaction(unsettledLoss) - } catch { - guard attemptToken == token else { return } - var failure = error - // A sign-in that expired is recoverable, so offer it once and retry rather than - // leaving the user on an error screen whose only button repeats the same failure. - if allowSignIn, - let method = SignInOffer.method(for: error, connection: connection, secureStore: appState.secureStore) { - switch await SignInOffer.offer(method, prompts: prompts) { - case .signedIn: - guard attemptToken == token else { return } - await connectFresh(token: token, allowSignIn: false) - return - case .failed(let signInError): - failure = signInError - case .declined: - break - } - } - guard attemptToken == token else { return } - phase = .failed(classify(failure, operation: "connect")) - } - } - - private func classify(_ error: any Error, operation: String) -> AppError { - ErrorClassifier.classify(error, context: ErrorContext( - operation: operation, - databaseType: connection.type, - host: connection.host, - sshEnabled: connection.sshEnabled, - signsInWithIAM: connection.iamSignIn != nil - )) - } - func reconnectIfNeeded() async { await restoreSession(after: .whileIdle) } @@ -370,265 +216,35 @@ final class ConnectionCoordinator { // True when the redial dropped a held transaction and the outcome is already settled for the user. @discardableResult private func restoreSession(after loss: SessionLoss) async -> Bool { - // One check at a time, and a statement that failed while it ran makes the loss ambiguous. - if let sessionCheck { - if loss == .duringWork { statementFailedDuringCheck = true } - return await sessionCheck.value - } - statementFailedDuringCheck = false - let check = Task { - let settled = await checkSession(after: loss) - sessionCheck = nil - return settled - } - sessionCheck = check - return await check.value - } - - private func checkSession(after loss: SessionLoss) async -> Bool { - let held = session - // A failed redial dropped the session, so there is nothing to ping and this trigger dials again. - guard held != nil || isLost, !isSwitching, !isReconnecting, connectTask == nil else { return false } - let token = attemptToken - var workWasRunning = appState.sessionTransactions.isRunningStatements(connection.id) - - switch await newerSession(than: held, token: token) { - case .taken: - return held == nil && reportDroppedTransaction(unsettledLoss) - case .deferred: - return false - case .none: - break - } - if let held { - if appState.connectionManager.session(for: connection.id) != nil, await Self.answersPing(held.driver) { - return false - } - guard attemptToken == token else { return false } - // Another window can finish a whole redial while this ping is out. - if await newerSession(than: held, token: token) != .none { return false } - } - - workWasRunning = workWasRunning || appState.sessionTransactions.isRunningStatements(connection.id) - let ended = appState.sessionTransactions.endedHold(for: connection.id, on: held?.driver) - workWasRunning = workWasRunning || ended?.duringWork == true - let heldTransaction = transactionHold != nil || ended != nil - let released = ended.flatMap { releasedLoss($0, on: held, asked: loss) } - isReconnecting = true - defer { isReconnecting = false } - do { - let outcome = try await appState.sessionRedialer.redial( - dialTarget, - schema: supportsSchemas ? activeSchema : nil, - prompter: prompts, - consumer: self, - stillWanted: { [weak self] in self?.attemptToken == token } - ) - guard attemptToken == token else { return false } - await takeSession(outcome.session, token: token, dialedHere: outcome.dialedHere) - guard attemptToken == token else { return false } - let dropped = heldTransaction ? released ?? settledLoss(loss, workWasRunning: workWasRunning) : unsettledLoss - return reportDroppedTransaction(dropped) - } catch { - guard attemptToken == token else { return false } - if heldTransaction { - unsettledLoss = released ?? settledLoss(loss, workWasRunning: workWasRunning) - } - phase = .lost(classify(error, operation: "reconnect")) - self.session = nil - return false - } - } - - private func settledLoss(_ loss: SessionLoss, workWasRunning: Bool) -> SessionLoss { - workWasRunning || statementFailedDuringCheck ? .duringWork : loss - } - - // The release closed the file once its running statement ended, so a later failure on the closed file decides nothing. - private func releasedLoss( - _ ended: SessionTransactionMonitor.EndedHold, - on held: ConnectionSession?, - asked loss: SessionLoss - ) -> SessionLoss? { - guard held?.driver.holdsSuspensionBlockingResource == true else { return nil } - if ended.duringWork { return .inBackgroundAfterStatement } - return loss == .duringRollBack ? .duringRollBack : .inBackground - } - - // True when the transaction a redial ended is settled for the user. - @discardableResult - private func reportDroppedTransaction(_ loss: SessionLoss?) -> Bool { - unsettledLoss = nil - guard let loss else { return false } - appState.sessionTransactions.noteRolledBack(connection.id) - if let message = Self.lostTransactionMessage(loss, connectionName: displayName) { - presentFailure(title: nil, message: message) - } - return true - } - - private enum NewerSession { - case none - case taken - // Another window holds it on another database or schema, and this window's unsaved edit belongs where it is. - case deferred - } - - private func newerSession(than held: ConnectionSession?, token: UUID) async -> NewerSession { - // A redial another window started is not ready to share until its schema is back. - await appState.sessionRedialer.settled(connection.id, consumer: self, prompter: prompts) - guard attemptToken == token, - let live = appState.connectionManager.session(for: connection.id), - live.driver !== held?.driver else { - return .none - } - guard !editPins(against: live) else { - followPending = true - return .deferred - } - return await takeSession(live, token: token, dialedHere: false) ? .taken : .deferred - } - - // An unsaved edit belongs to the placement it was made in, so a session elsewhere waits until the edit ends; - // taking it would rebuild the editor's screen, or point its save at a same-named table in another database. - private func editPins(against live: ConnectionSession) -> Bool { - guard isHeldByEditor else { return false } - let moved = movedPlacement(of: live, includingDatabase: true) - return moved.database != nil || moved.schema != nil - } - - private var dialTarget: DatabaseConnection { - guard supportsDatabaseSwitching, !activeDatabase.isEmpty else { return connection } - var target = connection - target.database = activeDatabase - return target - } - - @discardableResult - private func takeSession( - _ newSession: ConnectionSession, - token: UUID, - dialedHere: Bool, - yieldingToEdits: Bool = false - ) async -> Bool { - let placementAsked = movedPlacement(of: newSession, includingDatabase: !dialedHere) - let fetched: [TableInfo]? - do { - fetched = try await newSession.driver.fetchTables(schema: nil) - } catch { - Self.logger.warning("Failed to load tables on the new session: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - fetched = nil - } - guard attemptToken == token else { return false } - if !dialedHere, editPins(against: newSession) { - followPending = true - return false - } - // A row edit started while the tables loaded lives on a screen this would rebuild, and another switch that - // landed meanwhile made these tables the wrong database's. - if yieldingToEdits, - !canFollowLiveSession || movedPlacement(of: newSession, includingDatabase: !dialedHere) != placementAsked { - followPending = true - return false - } - // The dial restored this window's schema, or another window's; a failed restore leaves the default. - let movedPlacement = adoptPlacement(of: newSession, includingDatabase: !dialedHere) - if let fetched { - tables = fetched - } else { - metadataRefused = true - if movedPlacement { tables = [] } - } - session = newSession - phase = .connected - if dialedHere { - appState.bumpSessionRevision(connection.id) + guard isAttached else { return false } + let settled = await model.restoreSession(after: loss, from: self) + await model.runHeldBackWork(from: self) + return settled + } + + // An unsaved edit belongs to the placement it was made in, so while it is open the window keeps what it showed + // when the session moved elsewhere: taking it would rebuild the editor's screen, or point its save at a same-named + // table in another database. A move this window asked for itself reaches it. + func sessionModelChanged(from old: ConnectionSnapshot) { + guard pin == nil else { return } + if isHeldByEditor, !isSwitching, model.moves(from: old) { + // An open or a switch in flight is not where the edit was made; the loss or session before it is. + pin = ConnectionSnapshot(state: old.state.atRest, catalog: old.catalog) + return } - await refreshTransaction() - return true - } - - // A session another window dialed or moved sits on that window's database and schema, so this one - // follows it before any write can name a table from the old list. - @discardableResult - private func adoptPlacement(of taken: ConnectionSession, includingDatabase: Bool) -> Bool { - let moved = movedPlacement(of: taken, includingDatabase: includingDatabase) - if let database = moved.database { activeDatabase = database } - if let schema = moved.schema { activeSchema = schema } - return moved.database != nil || moved.schema != nil + guard old.catalog.tables != model.catalog.tables else { return } + selectedTable = TableSelectionResolver.keeping(selectedTable, in: model.catalog.tables) } - private func movedPlacement(of taken: ConnectionSession, includingDatabase: Bool) -> (database: String?, schema: String?) { - var database: String? - if includingDatabase { - let candidate = sessionDatabase(of: taken) - if supportsDatabaseSwitching, !candidate.isEmpty, candidate != activeDatabase { database = candidate } - } - var schema: String? - if supportsSchemas, let current = taken.driver.currentSchema, current != activeSchema { schema = current } - return (database, schema) + // It shows the model's session and no edit holds it, so it can hear what the model does for every window. + var showsLiveSession: Bool { + isAttached && pin == nil && !isHeldByEditor } - // Another window replaced the shared session, or switched its database or schema. + // The manager's session moved, or another party replaced it. func followLiveSession() async { - guard canFollowLiveSession, !isFollowing else { - followPending = true - return - } - // A window whose redial failed takes the session another window dialed. - if isLost { - guard appState.connectionManager.session(for: connection.id) != nil else { return } - await restoreSession(after: .whileIdle) - return - } - isFollowing = true - defer { isFollowing = false } - let token = attemptToken - await appState.sessionRedialer.settled(connection.id, consumer: self, prompter: prompts) - guard attemptToken == token else { return } - guard canFollowLiveSession else { - followPending = true - return - } - guard let held = session, let live = appState.connectionManager.session(for: connection.id) else { return } - if live.driver !== held.driver { - guard await takeSession(live, token: token, dialedHere: false, yieldingToEdits: true), - attemptToken == token else { return } - await loadSchemas() - return - } - let moved = movedPlacement(of: live, includingDatabase: true) - guard moved.database != nil || moved.schema != nil else { return } - let fetched: [TableInfo]? - do { - fetched = try await live.driver.fetchTables(schema: supportsSchemas ? moved.schema ?? activeSchema : nil) - } catch { - Self.logger.warning("Failed to load tables where another window moved the session: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - fetched = nil - } - guard attemptToken == token, session?.driver === live.driver else { return } - // A row edit started while the tables loaded lives on a screen the new placement would rebuild, and another - // switch that landed meanwhile made these tables the wrong database's. - guard canFollowLiveSession, movedPlacement(of: live, includingDatabase: true) == moved else { - followPending = true - return - } - // Placement and tables change in one step, so a row of the old list never pairs with the new placement. - if let database = moved.database { activeDatabase = database } - if let schema = moved.schema { activeSchema = schema } - session = live - if let fetched { - tables = fetched - } else { - metadataRefused = true - tables = [] - } - // SQL Server lists another database's schemas. - if moved.database != nil { - await loadSchemas() - guard attemptToken == token else { return } - } - await refreshTransaction() + guard isAttached, pin == nil else { return } + await model.follow(from: self) } // Where an unqualified table name resolves for this window. Another window's switch can move it under an open @@ -651,41 +267,6 @@ final class ConnectionCoordinator { String(localized: "Another window has unsaved changes on this connection. Save or discard them before switching database.") } - private var canFollowLiveSession: Bool { - (session != nil || isLost) && !isSwitching && !isReconnecting && connectTask == nil && !isHeldByEditor - } - - private func resumeFollowIfPending() { - guard followPending, canFollowLiveSession, !isFollowing else { return } - followPending = false - resumedFollow = Task { await followLiveSession() } - } - - // A transaction the server aborted refuses catalog reads, yet the session still holds the user's work. - private func keepsRefusingSession(_ existing: ConnectionSession, token: UUID, refusal: any Error) async -> Bool { - guard await Self.answersPing(existing.driver), attemptToken == token else { return false } - await refreshTransaction() - guard attemptToken == token, - appState.sessionTransactions.hold(for: connection.id, on: existing.driver) != nil else { return false } - Self.logger.info("Kept a session that refused its catalog: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") - let database = sessionDatabase(of: existing) - if supportsDatabaseSwitching, !database.isEmpty { - activeDatabase = database - } - metadataRefused = true - session = existing - phase = .connected - appState.libraryPreferences.recordConnected(connection.id) - return true - } - - private func sessionDatabase(of taken: ConnectionSession) -> String { - guard let live = appState.connectionManager.session(for: connection.id), live.driver === taken.driver else { - return taken.activeDatabase - } - return live.activeDatabase - } - static func lostTransactionMessage(_ loss: SessionLoss, connectionName: String) -> String? { switch loss { case .whileIdle: @@ -721,37 +302,18 @@ final class ConnectionCoordinator { } } - private static func answersPing(_ driver: any DatabaseDriver) async -> Bool { - do { - _ = try await driver.ping() - return true - } catch { - return false - } - } - // MARK: - Transactions var transactionHold: SessionTransactionMonitor.Hold? { appState.sessionTransactions.hold(for: connection.id, on: session?.driver) } - // PostgreSQL, Redshift and SQL Server keep DDL inside the transaction that ran it, where only the session sees it; - // a session that cannot say whether it holds one counts as holding one. var catalogSeesSessionDDL: Bool { - guard Self.transactionalDDLTypes.contains(connection.type) else { return true } - return appState.sessionTransactions.isKnownIdle(connection.id, on: session?.driver) + model.catalogSeesSessionDDL(on: session?.driver) } - private static let transactionalDDLTypes: Set = [.postgresql, .redshift, .mssql] - - // Completion reads beside the session only while both see the same tables: same DDL, same database, no table - // only a procedure on the session made. var catalogMatchesSession: Bool { - guard catalogSeesSessionDDL else { return false } - return !appState.catalogChanges.sessionLeftCatalog( - for: connection.id, driver: session.map { ObjectIdentifier($0.driver) } - ) + model.catalogMatchesSession(on: session?.driver) } var transactionToken: SessionTransactionMonitor.HoldToken? { @@ -778,7 +340,7 @@ final class ConnectionCoordinator { if heldBefore, !lossSettled, rolledBack(value) { appState.sessionTransactions.noteRolledBack(connection.id) } - await reloadMetadataIfReleased() + await model.reloadMetadataIfReleased(from: self) return value } catch { await restoreSession(after: loss) @@ -810,7 +372,7 @@ final class ConnectionCoordinator { guard await !restoreSession(after: .duringWork) else { return } throw error } - await reloadMetadataIfReleased() + await model.reloadMetadataIfReleased(from: self) // A catalog connection sees DDL the transaction ran only once it commits. noteCatalogChanged([.tables, .schemas, .databases]) } @@ -822,47 +384,19 @@ final class ConnectionCoordinator { guard await !restoreSession(after: .duringRollBack) else { return } throw error } - await reloadMetadataIfReleased() + await model.reloadMetadataIfReleased(from: self) // The lists may show a schema or a database the rolled-back transaction created. noteCatalogChanged([.schemas, .databases]) } func refreshTransaction() async { - await appState.sessionTransactions.refresh(connection.id) + await model.refreshTransaction() } - // A Roll Back from any window can undo DDL; a failed read keeps the list it has. + // A Roll Back from any window can undo DDL. func reloadAfterRollBack() async { - guard !metadataRefused else { - await reloadMetadataIfReleased() - return - } - guard let session else { return } - do { - let reloaded = try await session.driver.fetchTables(schema: supportsSchemas ? activeSchema : nil) - guard self.session?.driver === session.driver else { return } - tables = reloaded - } catch { - Self.logger.warning("Failed to reload tables after a Roll Back: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - } - // With autocommit off, a MySQL read opens a transaction. - await refreshTransaction() - } - - private func reloadMetadataIfReleased() async { - guard metadataRefused, let session, - appState.sessionTransactions.hold(for: connection.id, on: session.driver) == nil else { return } - do { - let reloaded = try await session.driver.fetchTables(schema: nil) - guard self.session?.driver === session.driver else { return } - tables = reloaded - metadataRefused = false - } catch { - Self.logger.warning("Tables are still refused: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - return - } - await loadDatabases() - await loadSchemas() + guard isAttached else { return } + await model.reloadAfterRollBack(from: self) } // MARK: - Database / Schema Switching @@ -886,160 +420,55 @@ final class ConnectionCoordinator { } func switchDatabase(to name: String) async { - guard session != nil, name != activeDatabase, !isSwitching else { return } + await switchDatabase(to: name, inside: nil) + } + + // `open` is the open this switch puts back where the connection was; the switch runs as a step of it. + func switchDatabase(to name: String, inside open: ConnectionAttempt?) async { + guard isAttached, session != nil || model.isSwitchingDatabase, name != activeDatabase, !isSwitching else { return } guard !anotherWindowHoldsEditor else { presentFailure(title: Self.switchDatabaseFailedTitle, message: Self.editorHeldElsewhereMessage) return } let failure = await whileSwitching(to: name) { - let failure: Failure? - if connection.type == .postgresql || connection.type == .redshift { - await reconnectWithDatabase(name) - failure = nil - } else { - failure = await useDatabase(name) - } - await refreshTransaction() - appState.bumpSessionRevision(connection.id) - return failure + await model.switchDatabase(to: name, from: self, inside: open) } + await model.loadSchemasAfterSwitch(to: name, from: self) guard let failure else { return } await report(failure, operation: "switchDatabase") } - private func useDatabase(_ name: String) async -> Failure? { - do { - try await appState.connectionManager.switchDatabase(connection.id, to: name) - } catch { - return Failure(error: error, title: Self.switchDatabaseFailedTitle, redialRepeats: false) - } - if let freshSession = appState.connectionManager.session(for: connection.id) { - self.session = freshSession - } - UserDefaults.standard.set(name, forKey: ConnectionDefaultsKey.lastDB.name(for: connection.id)) - do { - let fetched = try await session?.driver.fetchTables(schema: nil) ?? [] - // Placement and tables change in one step, so a row of the old list never pairs with the new database. - activeDatabase = name - tables = fetched - return nil - } catch { - // The session is on the new database either way, so its list empties until a reload can read it. - activeDatabase = name - metadataRefused = true - tables = [] - return Failure(error: error, title: Self.refreshTablesFailedTitle, redialRepeats: true) - } - } - - // Lets another window's redial land before the teardown that would cancel it, and dials where other windows join. - private func reconnectWithDatabase(_ database: String) async { - let token = attemptToken - await appState.sessionRedialer.settled(connection.id) - // A window closed during that wait must not tear down the session the other windows now use. - guard attemptToken == token else { return } - let previous = dialTarget - await appState.connectionManager.disconnect(connection.id) - self.session = nil - - var newConnection = connection - newConnection.database = database - - do { - let newSession = try await appState.sessionRedialer.redial( - newConnection, - schema: nil, - prompter: prompts, - joining: false, - consumer: self, - stillWanted: { [weak self] in self?.attemptToken == token } - ).session - guard attemptToken == token else { return } - let fetched = try await newSession.driver.fetchTables(schema: nil) - guard attemptToken == token else { return } - self.session = newSession - self.tables = fetched - activeDatabase = database - UserDefaults.standard.set(database, forKey: ConnectionDefaultsKey.lastDB.name(for: connection.id)) - await loadSchemas() - } catch { - guard attemptToken == token else { return } - Self.logger.error("Failed to switch to database \(database, privacy: .private(mask: .hash)): \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - // Back to the database the picker still shows, not the saved one, so the picker stays true. - do { - let fallback = try await appState.sessionRedialer.redial( - previous, - schema: supportsSchemas ? activeSchema : nil, - prompter: prompts, - joining: false, - consumer: self, - stillWanted: { [weak self] in self?.attemptToken == token } - ) - guard attemptToken == token else { return } - await takeSession(fallback.session, token: token, dialedHere: fallback.dialedHere) - guard attemptToken == token else { return } - presentFailure( - title: Self.switchDatabaseFailedTitle, - message: classify(error, operation: "switchDatabase").message - ) - } catch { - guard attemptToken == token else { return } - phase = .lost(classify(error, operation: "switchDatabase")) - self.session = nil - } - } - } - func switchSchema(to name: String) async { guard let session, name != activeSchema, !isSwitching else { return } let failure = await whileSwitching(to: name) { - do { - try await session.driver.switchSchema(to: name) - } catch { - return Failure(error: error, title: Self.switchSchemaFailedTitle, redialRepeats: false) - } - UserDefaults.standard.set(name, forKey: ConnectionDefaultsKey.lastSchema.name(for: connection.id)) - appState.bumpSessionRevision(connection.id) - do { - let fetched = try await session.driver.fetchTables(schema: name) - activeSchema = name - tables = fetched - return nil - } catch { - activeSchema = name - metadataRefused = true - tables = [] - return Failure(error: error, title: Self.refreshTablesFailedTitle, redialRepeats: true) - } + await model.switchSchema(to: name, on: session) } guard let failure else { return } await report(failure, operation: "switchSchema") } - private func whileSwitching(to target: String, _ work: () async -> Failure?) async -> Failure? { + private func whileSwitching( + to target: String, + _ work: () async -> ConnectionFailure? + ) async -> ConnectionFailure? { switchTarget = target isSwitching = true - defer { - isSwitching = false - switchTarget = nil - } - return await work() - } - - private struct Failure { - let error: any Error - let title: String - // A failed read that a redial repeats needs no alert once the redial lands. - let redialRepeats: Bool + model.beginSwitch() + let failure = await work() + model.endSwitch() + isSwitching = false + switchTarget = nil + await model.runHeldBackWork(from: self) + return failure } // A failure on a dead session redials first. The alert then says why, unless the session is lost, which its own // screen says. - private func report(_ failure: Failure, operation: String) async { + func report(_ failure: ConnectionFailure, operation: String) async { let failedOn = session?.driver if await restoreSession(after: .whileIdle) { return } guard !isLost, !failure.redialRepeats || session?.driver === failedOn else { return } - presentFailure(title: failure.title, message: classify(failure.error, operation: operation).message) + presentFailure(title: failure.title, message: model.classify(failure.error, operation: operation).message) } func presentFailure(title: String?, message: String) { @@ -1055,13 +484,9 @@ final class ConnectionCoordinator { // MARK: - Tables func refreshTables() async { - guard let session else { return } - do { - let schema = supportsSchemas ? activeSchema : nil - self.tables = try await session.driver.fetchTables(schema: schema) - } catch { - Self.logger.warning("Failed to refresh tables: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - await report(Failure(error: error, title: Self.refreshTablesFailedTitle, redialRepeats: true), operation: "refreshTables") + guard session != nil else { return } + if let failure = await model.refreshTables() { + await report(failure, operation: "refreshTables") } // With autocommit off, a MySQL read opens a transaction. await refreshTransaction() @@ -1071,109 +496,12 @@ final class ConnectionCoordinator { // `reloadedHere`: the caller already read the session, the one place a transaction's own DDL shows. func noteCatalogChanged(_ kinds: SQLCatalogObjectKinds, reloadedHere: Bool = false) { - let board = appState.catalogChanges - let caughtUp = board.generation(for: connection.id) == seenCatalogGeneration - board.post(kinds, for: connection.id) - if reloadedHere, caughtUp { - seenCatalogGeneration = board.generation(for: connection.id) - } + model.noteCatalogChanged(kinds, reloadedHere: reloadedHere) } - // A statement a window ran may have created or dropped what the lists show; failed reads keep the lists. - // One reload at a time: a later post waits for it, so an older answer never lands over a newer one. func reloadChangedCatalog() async { - guard !isReloadingCatalog else { - catalogReloadRequested = true - return - } - isReloadingCatalog = true - defer { isReloadingCatalog = false } - repeat { - catalogReloadRequested = false - await reloadChangedCatalogOnce() - } while catalogReloadRequested - } - - private func reloadChangedCatalogOnce() async { - let board = appState.catalogChanges - let generation = board.generation(for: connection.id) - guard generation != seenCatalogGeneration, let session, !isSwitching else { return } - var wanted = board.kinds(for: connection.id, since: seenCatalogGeneration) - if !supportsSchemas { wanted.remove(.schemas) } - if !supportsDatabaseSwitching { wanted.remove(.databases) } - let kinds = wanted - let database = supportsDatabaseSwitching ? activeDatabase : "" - let schema = supportsSchemas ? activeSchema : nil - let place = CatalogPlace(driver: ObjectIdentifier(session.driver), database: activeDatabase, schema: activeSchema) - let monitor = appState.sessionTransactions - - var snapshot: CatalogSnapshot? - // On the session a read would replace ROW_COUNT() and the warnings, so it is only where the DDL is visible, or - // where a USE or a role moved the session off what the catalog connection sees, as Refresh reads it there too. - if catalogMatchesSession { - do { - snapshot = try await appState.connectionManager.readCatalog(for: connection.id, database: database) { - try await Self.readCatalog(kinds, schema: schema, from: $0) - } - } catch { - Self.logger.warning("Reloading the catalog after a change failed: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - seenCatalogGeneration = generation - return - } - } - if snapshot == nil { - // A server that has a catalog connection but could not open one keeps its lists: a read on the session would - // replace what the user's last statement left there. Only its own uncommitted DDL sends the read there. - if catalogMatchesSession, appState.connectionManager.opensCatalogConnections(for: connection.id) { - seenCatalogGeneration = generation - return - } - // A stream on the session would end under this read, so it waits for the session to go idle. - guard !monitor.isBusy(connection.id) else { return } - board.beginSessionRead(for: connection.id) - snapshot = try? await monitor.track(connection.id, reading: true) { - try await Self.readCatalog(kinds, schema: schema, from: session.driver) - } - board.endSessionRead(for: connection.id) - } - seenCatalogGeneration = generation - guard let snapshot, place == CatalogPlace( - driver: self.session.map { ObjectIdentifier($0.driver) }, database: activeDatabase, schema: activeSchema - ) else { return } - if let reloaded = snapshot.tables { tables = reloaded } - if let reloaded = snapshot.schemas { schemas = reloaded } - if let reloaded = snapshot.databases { databases = reloaded } - } - - private struct CatalogPlace: Equatable { - let driver: ObjectIdentifier? - let database: String - let schema: String - } - - private struct CatalogSnapshot: Sendable { - var tables: [TableInfo]? - var schemas: [String]? - var databases: [String]? - } - - // The table list is the read that matters: its failure fails the reload, so a dead connection is dropped. - nonisolated private static func readCatalog( - _ kinds: SQLCatalogObjectKinds, - schema: String?, - from driver: any DatabaseDriver - ) async throws -> CatalogSnapshot { - var snapshot = CatalogSnapshot() - if kinds.contains(.tables) { - snapshot.tables = try await driver.fetchTables(schema: schema) - } - if kinds.contains(.schemas) { - snapshot.schemas = try? await driver.fetchSchemas() - } - if kinds.contains(.databases) { - snapshot.databases = try? await driver.fetchDatabases() - } - return snapshot + guard isAttached else { return } + await model.reloadChangedCatalog() } // MARK: - Query History @@ -1205,47 +533,4 @@ final class ConnectionCoordinator { selectedTab = .tables selectedTable = table } - - // MARK: - Private Helpers - - private func loadDatabases() async { - guard let session, supportsDatabaseSwitching else { return } - do { - databases = try await session.driver.fetchDatabases() - if !activeDatabase.isEmpty, databases.contains(activeDatabase) { - let sessionDB = appState.connectionManager.session(for: connection.id)?.activeDatabase ?? connection.database - if activeDatabase != sessionDB { - let target = activeDatabase - activeDatabase = sessionDB - await switchDatabase(to: target) - } - } else { - activeDatabase = appState.connectionManager.session(for: connection.id)?.activeDatabase ?? connection.database - // A database the connection names by its id is listed by its name, which the driver reports. - if !databases.contains(activeDatabase), let reported = session.driver.currentDatabase, - databases.contains(reported) { - await switchDatabase(to: reported) - } - } - } catch { - Self.logger.warning("Failed to load databases: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - } - } - - private func loadSchemas() async { - guard let session, supportsSchemas else { return } - do { - schemas = try await session.driver.fetchSchemas() - let currentSchema = session.driver.currentSchema ?? "public" - if schemas.contains(activeSchema), activeSchema != currentSchema { - let target = activeSchema - activeSchema = currentSchema - await switchSchema(to: target) - } else if !schemas.contains(activeSchema) { - activeSchema = currentSchema - } - } catch { - Self.logger.warning("Failed to load schemas: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") - } - } } diff --git a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinatorStore.swift b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinatorStore.swift index 7960b26091..1368652e4e 100644 --- a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinatorStore.swift +++ b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionCoordinatorStore.swift @@ -72,9 +72,11 @@ final class ConnectionCoordinatorStore { } private func rebuild(_ id: UUID, droppingSession: Bool) { - coordinators.removeValue(forKey: id)?.retire() + let retired = coordinators.removeValue(forKey: id) + retired?.retire() generations[id, default: 0] += 1 guard droppingSession else { return } + retired?.dropSession() disconnect(id) } @@ -83,6 +85,7 @@ final class ConnectionCoordinatorStore { if let retired = coordinators.removeValue(forKey: id) { removedRecords[id] = retired.connection retired.retire() + retired.dropSession() } disconnect(id) } diff --git a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionModel.swift b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionModel.swift new file mode 100644 index 0000000000..f8435c26d4 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionModel.swift @@ -0,0 +1,1202 @@ +import Foundation +import Observation +import os +import TableProConnectionLibrary +import TableProDatabase +import TableProLogRedaction +import TableProModels +import TableProSQLGrammar + +// What a window shows of its connection: the model's, or what it showed when the session moved under an unsaved edit. +struct ConnectionSnapshot { + let state: ConnectionSessionState + let catalog: ConnectionCatalog +} + +struct ConnectionFailure { + let error: any Error + let title: String + // A failed read that a redial repeats needs no alert once the redial lands. + let redialRepeats: Bool +} + +// One per connection, kept while a window holds it. +final class ConnectionSessionModels { + private struct WeakModel { + weak var model: ConnectionSessionModel? + } + + private var models: [UUID: WeakModel] = [:] + + func model(for connection: DatabaseConnection, appState: AppState) -> ConnectionSessionModel { + if let existing = models[connection.id]?.model { + existing.adopt(connection) + return existing + } + let created = ConnectionSessionModel(record: connection, appState: appState) + models[connection.id] = WeakModel(model: created) + return created + } + + func existing(_ connectionId: UUID) -> ConnectionSessionModel? { + models[connectionId]?.model + } + + func purge(_ connectionIds: Set) { + for connectionId in connectionIds { + models.removeValue(forKey: connectionId) + } + } +} + +// The session every window of one connection shows: one lifecycle, one table list, one check, one catalog reload. +// A window asks through its `ConnectionCoordinator`, which keeps what is the window's own. +@MainActor @Observable +final class ConnectionSessionModel { + private(set) var record: DatabaseConnection + + // Written only by `enter(_:)`, which refuses a step of an attempt the state no longer holds. + private(set) var state: ConnectionSessionState = .notConnected { + didSet { scenesSaw(ConnectionSnapshot(state: oldValue, catalog: catalog)) } + } + private(set) var catalog = ConnectionCatalog() { + didSet { scenesSaw(ConnectionSnapshot(state: state, catalog: oldValue)) } + } + + // A transaction lost with no window told yet; the next transition that lands a session reports it. + @ObservationIgnored private var pendingLoss: SessionLoss? + @ObservationIgnored private var seenCatalogGeneration: Int + @ObservationIgnored private var isReloadingCatalog = false + @ObservationIgnored private var catalogReloadRequested = false + @ObservationIgnored private var isReloadingAfterRollBack = false + @ObservationIgnored private var rollBackReloadRequested = false + @ObservationIgnored private var catalogReloadHeldBack = false + @ObservationIgnored private var isFollowing = false + @ObservationIgnored private var followRequested = false + @ObservationIgnored private weak var followRequester: ConnectionCoordinator? + // Each request that moves the placement bumps it, so only the newest one lands. + @ObservationIgnored private var placementEpoch = 0 + @ObservationIgnored private var switchesInFlight = 0 + @ObservationIgnored private var placementRestored = false + @ObservationIgnored private var scenes: [WeakScene] = [] + + private let appState: AppState + private static let logger = Logger(subsystem: "com.TablePro", category: "ConnectionSessionModel") + + typealias SessionLoss = ConnectionCoordinator.SessionLoss + private typealias Placement = ConnectionCatalog.Placement + + private struct WeakScene { + weak var scene: ConnectionCoordinator? + } + + init(record: DatabaseConnection, appState: AppState) { + self.record = record + self.appState = appState + seenCatalogGeneration = appState.catalogChanges.generation(for: record.id) + } + + var connectionId: UUID { record.id } + + var session: ConnectionSession? { state.session } + + var activeDatabase: String { catalog.placement.database } + var activeSchema: String { catalog.placement.schema } + + private var isLost: Bool { + if case .lost = state.phase { return true } + return false + } + + var displayName: String { + record.name.isEmpty ? record.host : record.name + } + + var supportsDatabaseSwitching: Bool { + record.type == .mysql || record.type == .mariadb || + record.type == .tidb || record.type == .oceanbase || + record.type == .postgresql || record.type == .redshift || + record.type == .mssql || record.type == .clickhouse || + record.type == .mongodb || record.type == .cloudflareD1 + } + + var supportsSchemas: Bool { + record.type == .postgresql || record.type == .redshift || + record.type == .mssql || record.type == .duckdb || + record.type == .oracle + } + + private var redialsToSwitchDatabase: Bool { + record.type == .postgresql || record.type == .redshift + } + + func adopt(_ record: DatabaseConnection) { + guard record.id == self.record.id, record != self.record else { return } + self.record = record + } + + // The first window restores where the connection was; a later one finds the model there already. + func restorePersistedPlacement() { + guard !placementRestored else { return } + placementRestored = true + let defaults = UserDefaults.standard + catalog.placement = Placement( + database: defaults.string(forKey: ConnectionDefaultsKey.lastDB.name(for: connectionId)) ?? "", + schema: defaults.string(forKey: ConnectionDefaultsKey.lastSchema.name(for: connectionId)) ?? "public" + ) + } + + func classify(_ error: any Error, operation: String) -> AppError { + ErrorClassifier.classify(error, context: ErrorContext( + operation: operation, + databaseType: record.type, + host: record.host, + sshEnabled: record.sshEnabled, + signsInWithIAM: record.iamSignIn != nil + )) + } + + // MARK: - Windows + + func register(_ scene: ConnectionCoordinator) { + scenes.removeAll { $0.scene == nil } + scenes.append(WeakScene(scene: scene)) + } + + private func scenesSaw(_ old: ConnectionSnapshot) { + for scene in scenes.compactMap(\.scene) { + scene.sessionModelChanged(from: old) + } + } + + private func consumers(of attempt: ConnectionAttempt) -> [ConnectionCoordinator] { + let live = scenes.compactMap(\.scene) + return attempt.consumers.compactMap { id in live.first { ObjectIdentifier($0) == id } } + } + + // The window that asked, or the next one still waiting: it answers the attempt's questions and hears its outcome. + private func lead(of attempt: ConnectionAttempt) -> ConnectionCoordinator? { + consumers(of: attempt).first + } + + // True when the session the model now holds sits somewhere else than `old` showed, so an unsaved edit made there + // would land on another table. + func moves(from old: ConnectionSnapshot) -> Bool { + if catalog.placement != old.catalog.placement { return true } + guard let live = state.session, live.driver !== old.state.session?.driver else { return false } + let moved = movedPlacement(of: live, from: old.catalog.placement, includingDatabase: true) + return moved.database != nil || moved.schema != nil + } + + // MARK: - Lifecycle + + @discardableResult + private func enter(_ event: ConnectionSessionState.Event) -> Bool { + guard let next = state.applying(event) else { return false } + state = next + return true + } + + private func isCurrent(_ attempt: ConnectionAttempt) -> Bool { + state.attempt === attempt + } + + func connect(from scene: ConnectionCoordinator) async { + switch state { + case .connecting(let open, _): + await wait(on: open, from: scene) + case .checking(let running, _), .reconnecting(let running, _): + await wait(on: running, from: scene) + if case .connected = state { await attach() } + case .connected(let held): + if appState.connectionManager.session(for: connectionId)?.driver === held.driver { + await attach() + } else { + // The session ended under every window, as the background release does to a file. + await restoreSession(after: .whileIdle, from: scene) + } + case .notConnected, .failed, .lost: + await open(from: scene) + } + } + + // A window that joins answers the dial's questions too, and keeps the dial when the window that asked goes. + private func wait(on attempt: ConnectionAttempt, from scene: ConnectionCoordinator) async { + attempt.join(scene) + await appState.sessionRedialer.settled(connectionId, consumer: scene, prompter: scene.prompts) + _ = await attempt.task?.value + } + + private func open(from scene: ConnectionCoordinator) async { + let attempt = ConnectionAttempt(.open, origin: state.openOrigin) + attempt.join(scene) + let wanted = catalog.placement + placementRestored = true + enter(.open(attempt)) + attempt.task = Task { [weak self] in + await self?.runAttempt(attempt, restoring: wanted) + return false + } + _ = await attempt.task?.value + } + + // A window that opens on a connected session reads the lists itself, which covers every change posted before. + private func attach() async { + guard let session else { return } + let generation = appState.catalogChanges.generation(for: connectionId) + var load = await loadTables(in: supportsSchemas ? activeSchema : nil, on: session) + if supportsDatabaseSwitching { load.databases = try? await session.driver.fetchDatabases() } + if supportsSchemas { load.schemas = try? await session.driver.fetchSchemas() } + if load.failure == nil, commit(load, onFailure: .keep), generation > seenCatalogGeneration { + seenCatalogGeneration = generation + } + await refreshTransaction() + appState.libraryPreferences.recordConnected(connectionId) + } + + // The last window to leave abandons what is in flight, so the next one never joins it and waits forever. + func leave(_ scene: ConnectionCoordinator) { + guard let attempt = state.attempt, attempt.leave(scene) else { return } + enter(.abandon(attempt)) + if attempt.purpose == .open { attempt.task?.cancel() } + leaveDial() + } + + // The record changed how it dials, so the next window opens it again. + func reset() { + let attempt = state.attempt + enter(.cancel) + if attempt?.purpose == .open { attempt?.task?.cancel() } + leaveDial() + } + + private func leaveDial() { + if appState.sessionRedialer.leave(connectionId, consumer: self) { + appState.connectionManager.invalidateAttempt(for: connectionId) + } + } + + private func runAttempt(_ attempt: ConnectionAttempt, restoring wanted: Placement) async { + // Another dial is shared once its schema is back, never torn down mid-dial. + await appState.sessionRedialer.settled(connectionId, consumer: self, prompter: lead(of: attempt)?.prompts) + guard isCurrent(attempt) else { return } + if let existing = appState.connectionManager.session(for: connectionId) { + let load = await loadTables(in: nil, on: existing) + guard isCurrent(attempt) else { return } + if load.failure == nil { + let placement = adoptedPlacement(of: existing, includingDatabase: true) + enter(.adopt(existing, by: attempt)) + commit(load, at: placement, onFailure: .keep) + await finishOpen(attempt, restoring: wanted) + return + } + if await keepsRefusingSession(existing, refused: load, by: attempt) { return } + guard isCurrent(attempt) else { return } + await appState.sessionRedialer.settled(connectionId, consumer: self, prompter: lead(of: attempt)?.prompts) + guard isCurrent(attempt) else { return } + if let live = appState.connectionManager.session(for: connectionId), live.driver !== existing.driver { + await runAttempt(attempt, restoring: wanted) + return + } + await appState.connectionManager.disconnect(connectionId) + } + + guard isCurrent(attempt) else { return } + await connectFresh(attempt, restoring: wanted) + } + + // `allowSignIn` is false on the retry that follows a sign-in, so a connection that keeps + // failing cannot put the prompt up again and again. That retry dials with the new credentials + // rather than join a dial started without them. + private func connectFresh(_ attempt: ConnectionAttempt, restoring wanted: Placement, allowSignIn: Bool = true) async { + Telemetry.shared.connectionAttempted(databaseType: record.type.rawValue) + + do { + let outcome = try await appState.sessionRedialer.redial( + record, + schema: nil, + prompter: lead(of: attempt)?.prompts, + joining: allowSignIn, + consumer: self, + stillWanted: { [weak self] in self?.isCurrent(attempt) == true } + ) + let newSession = outcome.session + let load = await loadTables(in: nil, on: newSession) + if let refusal = load.failure { throw refusal } + guard isCurrent(attempt) else { return } + let placement = adoptedPlacement(of: newSession, includingDatabase: true) + enter(.adopt(newSession, by: attempt)) + commit(load, at: placement, onFailure: .keep) + if outcome.dialedHere { + appState.bumpSessionRevision(connectionId) + } + guard await finishOpen(attempt, restoring: wanted) else { return } + Telemetry.shared.connectionSucceeded(databaseType: record.type.rawValue) + for scene in consumers(of: attempt) { + scene.navigateToPendingTable() + } + } catch { + guard isCurrent(attempt) else { return } + var failure = error + // A sign-in that expired is recoverable, so offer it once and retry rather than + // leaving the user on an error screen whose only button repeats the same failure. + if allowSignIn, let prompts = lead(of: attempt)?.prompts, + let method = SignInOffer.method(for: error, connection: record, secureStore: appState.secureStore) { + switch await SignInOffer.offer(method, prompts: prompts) { + case .signedIn: + guard isCurrent(attempt) else { return } + await connectFresh(attempt, restoring: wanted, allowSignIn: false) + return + case .failed(let signInError): + failure = signInError + case .declined: + break + } + } + guard isCurrent(attempt) else { return } + enter(.fail(classify(failure, operation: "connect"), attempt)) + } + } + + // The open landed where the session is; this puts it back where the connection was, through the asking window. + @discardableResult + private func finishOpen(_ attempt: ConnectionAttempt, restoring wanted: Placement) async -> Bool { + await loadDatabases(restoring: wanted.database, from: lead(of: attempt), inside: attempt) + await loadSchemas(restoring: wanted.schema, from: lead(of: attempt)) + await refreshTransaction() + // A database restore that failed both ways left the open lost, and it stays lost. + guard enter(.opened(attempt)) else { return false } + appState.libraryPreferences.recordConnected(connectionId) + reportDroppedTransaction(pendingLoss, to: lead(of: attempt)) + return true + } + + // A transaction the server aborted refuses catalog reads, yet the session still holds the user's work. + private func keepsRefusingSession( + _ existing: ConnectionSession, + refused load: CatalogLoad, + by attempt: ConnectionAttempt + ) async -> Bool { + guard let refusal = load.failure, await Self.answersPing(existing.driver), isCurrent(attempt) else { return false } + await refreshTransaction() + guard isCurrent(attempt), + appState.sessionTransactions.hold(for: connectionId, on: existing.driver) != nil else { return false } + Self.logger.info("Kept a session that refused its catalog: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") + var placement = catalog.placement + let database = sessionDatabase(of: existing) + if supportsDatabaseSwitching, !database.isEmpty { + placement.database = database + } + enter(.adopt(existing, by: attempt)) + commit(load, at: placement, onFailure: .refuse) + enter(.opened(attempt)) + appState.libraryPreferences.recordConnected(connectionId) + return true + } + + // Another party ended the session the model holds, as the background release does to a file. The next + // transition that lands a session reports what it held, whichever window asks. + func sessionEnded(_ driver: any DatabaseDriver) { + guard case .connected(let held) = state, held.driver === driver, + let ended = appState.sessionTransactions.endedHold(for: connectionId, on: driver) else { return } + if driver.holdsSuspensionBlockingResource { + pendingLoss = ended.duringWork ? .inBackgroundAfterStatement : .inBackground + } else { + pendingLoss = ended.duringWork ? .duringWork : .whileIdle + } + } + + // MARK: - Session Check + + // True when the redial dropped a held transaction and the outcome is already settled for the user. + @discardableResult + func restoreSession(after loss: SessionLoss, from scene: ConnectionCoordinator) async -> Bool { + // One check per connection, and a statement that failed while it ran makes the loss ambiguous. + if let running = state.check, let task = running.task { + if loss == .duringWork { running.ambiguous = true } + running.join(scene) + return await task.value + } + guard !scene.isSwitching else { return false } + let check = ConnectionAttempt(.redial) + check.join(scene) + guard enter(.check(check)) else { return false } + let task = Task { await checkSession(check, after: loss) } + check.task = task + return await task.value + } + + private func checkSession(_ check: ConnectionAttempt, after loss: SessionLoss) async -> Bool { + let held = session + var workWasRunning = appState.sessionTransactions.isRunningStatements(connectionId) + + switch await newerSession(than: held, during: check) { + case .taken: + return reportDroppedTransaction(pendingLoss, to: lead(of: check)) + case .refused: + enter(.settle(check)) + return false + case .none: + break + } + if let held { + if appState.connectionManager.session(for: connectionId) != nil, await Self.answersPing(held.driver) { + enter(.settle(check)) + return false + } + guard isCurrent(check) else { return false } + // Another dial can land a whole session while this ping is out. + switch await newerSession(than: held, during: check) { + case .taken: + return reportDroppedTransaction(pendingLoss, to: lead(of: check)) + case .refused: + enter(.settle(check)) + return false + case .none: + break + } + } + + workWasRunning = workWasRunning || appState.sessionTransactions.isRunningStatements(connectionId) + let ended = appState.sessionTransactions.endedHold(for: connectionId, on: held?.driver) + workWasRunning = workWasRunning || ended?.duringWork == true + let heldTransaction = appState.sessionTransactions.hold(for: connectionId, on: held?.driver) != nil + || ended != nil + let released = ended.flatMap { releasedLoss($0, on: held, asked: loss) } + guard enter(.dial(check)) else { return false } + do { + let outcome = try await appState.sessionRedialer.redial( + dialTarget, + schema: supportsSchemas ? activeSchema : nil, + prompter: lead(of: check)?.prompts, + consumer: self, + stillWanted: { [weak self] in self?.isCurrent(check) == true } + ) + guard isCurrent(check), + await takeSession(outcome.session, dialedHere: outcome.dialedHere, by: check) else { return false } + let dropped = heldTransaction + ? released ?? Self.settledLoss(loss, workWasRunning: workWasRunning, ambiguous: check.ambiguous) + : pendingLoss + return reportDroppedTransaction(dropped, to: lead(of: check)) + } catch { + guard isCurrent(check) else { return false } + if heldTransaction { + pendingLoss = released + ?? Self.settledLoss(loss, workWasRunning: workWasRunning, ambiguous: check.ambiguous) + } + enter(.lose(classify(error, operation: "reconnect"), check)) + return false + } + } + + private static func settledLoss(_ loss: SessionLoss, workWasRunning: Bool, ambiguous: Bool) -> SessionLoss { + workWasRunning || ambiguous ? .duringWork : loss + } + + // The release closed the file once its running statement ended, so a later failure on the closed file decides nothing. + private func releasedLoss( + _ ended: SessionTransactionMonitor.EndedHold, + on held: ConnectionSession?, + asked loss: SessionLoss + ) -> SessionLoss? { + guard held?.driver.holdsSuspensionBlockingResource == true else { return nil } + if ended.duringWork { return .inBackgroundAfterStatement } + return loss == .duringRollBack ? .duringRollBack : .inBackground + } + + // True when the transaction a redial ended is settled for the user. + @discardableResult + private func reportDroppedTransaction(_ loss: SessionLoss?, to scene: ConnectionCoordinator?) -> Bool { + pendingLoss = nil + guard let loss else { return false } + appState.sessionTransactions.noteRolledBack(connectionId) + if let message = ConnectionCoordinator.lostTransactionMessage(loss, connectionName: displayName) { + scene?.presentFailure(title: nil, message: message) + } + return true + } + + private enum NewerSession { + case none + case taken + case refused + } + + private func newerSession(than held: ConnectionSession?, during check: ConnectionAttempt) async -> NewerSession { + // A dial in flight is not ready to share until its schema is back. + await appState.sessionRedialer.settled(connectionId, consumer: self, prompter: lead(of: check)?.prompts) + guard isCurrent(check), + let live = appState.connectionManager.session(for: connectionId), + live.driver !== held?.driver else { + return .none + } + return await takeSession(live, dialedHere: false, by: check) ? .taken : .refused + } + + private var dialTarget: DatabaseConnection { + guard supportsDatabaseSwitching, !activeDatabase.isEmpty else { return record } + var target = record + target.database = activeDatabase + return target + } + + // `lander` is the attempt that dialed or found the session; a follow takes it with none. + @discardableResult + private func takeSession( + _ newSession: ConnectionSession, + dialedHere: Bool, + by lander: ConnectionAttempt?, + yieldingToBusy: Bool = false + ) async -> Bool { + let placementAsked = movedPlacement(of: newSession, includingDatabase: !dialedHere) + let load = await loadTables(in: nil, on: newSession) + if let refusal = load.failure { + Self.logger.warning("Failed to load tables on the new session: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") + } + if let lander, !isCurrent(lander) { return false } + // Another switch that landed while the tables loaded made them the wrong database's. + if yieldingToBusy, + !canFollow || movedPlacement(of: newSession, includingDatabase: !dialedHere) != placementAsked { + return false + } + // The dial restored the connection's schema; a failed restore leaves the default. + let placement = adoptedPlacement(of: newSession, includingDatabase: !dialedHere) + let moved = placement != catalog.placement + guard enter(.adopt(newSession, by: lander)) else { return false } + commit(load, at: placement, onFailure: moved ? .empty : .refuse) + if dialedHere { + appState.bumpSessionRevision(connectionId) + } + await refreshTransaction() + return true + } + + // A session dialed or moved elsewhere sits on that database and schema, so the lists follow it before any write + // can name a table from the old list. + private func adoptedPlacement(of taken: ConnectionSession, includingDatabase: Bool) -> Placement { + let moved = movedPlacement(of: taken, includingDatabase: includingDatabase) + return Placement(database: moved.database ?? activeDatabase, schema: moved.schema ?? activeSchema) + } + + private func movedPlacement( + of taken: ConnectionSession, + from placement: Placement? = nil, + includingDatabase: Bool + ) -> (database: String?, schema: String?) { + let base = placement ?? catalog.placement + var database: String? + if includingDatabase { + let candidate = sessionDatabase(of: taken) + if supportsDatabaseSwitching, !candidate.isEmpty, candidate != base.database { database = candidate } + } + var schema: String? + if supportsSchemas, let current = taken.driver.currentSchema, current != base.schema { schema = current } + return (database, schema) + } + + private func sessionDatabase(of taken: ConnectionSession) -> String { + guard let live = appState.connectionManager.session(for: connectionId), live.driver === taken.driver else { + return taken.activeDatabase + } + return live.activeDatabase + } + + private static func answersPing(_ driver: any DatabaseDriver) async -> Bool { + do { + _ = try await driver.ping() + return true + } catch { + return false + } + } + + // MARK: - Following + + private var canFollow: Bool { + (session != nil || isLost) && switchesInFlight == 0 && !state.isReconnecting && !state.isOpening + } + + // The manager holds a session this model did not land, or moved its own: the lists follow it there. A follow + // asked while the model is busy runs once it is free, so a later move is never missed. + func follow(from scene: ConnectionCoordinator) async { + followRequester = scene + guard canFollow, !isFollowing else { + followRequested = true + return + } + isFollowing = true + defer { isFollowing = false } + repeat { + followRequested = false + await followOnce(from: followLead(or: scene)) + } while followRequested && canFollow + } + + // The window that asked hears the follow's outcome only while it still shows the live session; a retired one cannot + // answer its questions, and a pinned one's alert would describe a session it does not show. + private func followLead(or scene: ConnectionCoordinator) -> ConnectionCoordinator { + guard let followRequester, followRequester.showsLiveSession else { return scene } + return followRequester + } + + // What a busy model held back: a follow, and a catalog reload that would have read a placement in motion. + func runHeldBackWork(from scene: ConnectionCoordinator) async { + if followRequested, canFollow, !isFollowing { + await follow(from: followLead(or: scene)) + } + if catalogReloadHeldBack, switchesInFlight == 0 { + catalogReloadHeldBack = false + await reloadChangedCatalog() + } + } + + private func followOnce(from scene: ConnectionCoordinator) async { + if isLost { + guard appState.connectionManager.session(for: connectionId) != nil else { return } + await restoreSession(after: .whileIdle, from: scene) + return + } + await appState.sessionRedialer.settled(connectionId, consumer: scene, prompter: scene.prompts) + guard canFollow else { + followRequested = true + return + } + guard let held = session, let live = appState.connectionManager.session(for: connectionId) else { return } + if live.driver !== held.driver { + guard await takeSession(live, dialedHere: false, by: nil, yieldingToBusy: true) else { + followRequested = true + return + } + reportDroppedTransaction(pendingLoss, to: scene) + await loadSchemas(restoring: activeSchema, from: scene) + return + } + let moved = movedPlacement(of: live, includingDatabase: true) + guard moved.database != nil || moved.schema != nil else { return } + let request = beginPlacementRequest() + let target = Placement(database: moved.database ?? activeDatabase, schema: moved.schema ?? activeSchema) + let load = await loadTables(in: supportsSchemas ? target.schema : nil, on: live) + if let refusal = load.failure { + Self.logger.warning("Failed to load tables where the session moved: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") + } + // Another move that landed while the tables loaded made them the wrong placement's. + guard canFollow, session?.driver === live.driver, + movedPlacement(of: live, includingDatabase: true) == moved else { + followRequested = true + return + } + enter(.adopt(live, by: nil)) + commit(load, at: target, onFailure: .empty, for: request) + // SQL Server lists another database's schemas. + if moved.database != nil { + await loadSchemas(restoring: activeSchema, from: scene) + } + await refreshTransaction() + } + + // MARK: - Database / Schema Switching + + func beginSwitch() { + switchesInFlight += 1 + } + + func endSwitch() { + switchesInFlight -= 1 + } + + private func beginPlacementRequest() -> Int { + placementEpoch += 1 + return placementEpoch + } + + func switchDatabase( + to name: String, + from scene: ConnectionCoordinator, + inside open: ConnectionAttempt? + ) async -> ConnectionFailure? { + let failure: ConnectionFailure? + if redialsToSwitchDatabase { + await reconnectWithDatabase(name, from: scene, inside: open) + failure = nil + } else { + failure = await useDatabase(name) + } + await refreshTransaction() + appState.bumpSessionRevision(connectionId) + return failure + } + + // SQL Server lists another database's schemas; a redial already read them on its new session. It runs once the + // switch ends, so restoring a schema there is not refused as a switch in flight. + func loadSchemasAfterSwitch(to name: String, from scene: ConnectionCoordinator) async { + guard !redialsToSwitchDatabase, activeDatabase == name else { return } + await loadSchemas(restoring: activeSchema, from: scene) + } + + private func useDatabase(_ name: String) async -> ConnectionFailure? { + let request = beginPlacementRequest() + do { + try await appState.connectionManager.switchDatabase(connectionId, to: name) + } catch { + return ConnectionFailure(error: error, title: ConnectionCoordinator.switchDatabaseFailedTitle, redialRepeats: false) + } + if let freshSession = appState.connectionManager.session(for: connectionId) { + enter(.adopt(freshSession, by: nil)) + } + UserDefaults.standard.set(name, forKey: ConnectionDefaultsKey.lastDB.name(for: connectionId)) + guard let session else { return nil } + let load = await loadTables(in: nil, on: session) + // The session is on the new database either way, so a refused list empties until a reload can read it. + commit(load, at: Placement(database: name, schema: activeSchema), onFailure: .empty, for: request) + guard let refusal = load.failure else { return nil } + return ConnectionFailure(error: refusal, title: ConnectionCoordinator.refreshTablesFailedTitle, redialRepeats: true) + } + + // Inside its open the switch is the open's step; otherwise it is an attempt of its own, which no older step outlives. + private func beginDatabaseSwitch(from scene: ConnectionCoordinator, inside open: ConnectionAttempt?) -> ConnectionAttempt? { + if let open { + return enter(.teardown(open)) ? open : nil + } + let attempt = ConnectionAttempt(.switchDatabase) + attempt.join(scene) + return enter(.switchDatabase(attempt)) ? attempt : nil + } + + // Every window reads no session while a PostgreSQL switch runs; a switch asked then runs after it. + var isSwitchingDatabase: Bool { + guard case .reconnecting(let attempt, _) = state else { return false } + return attempt.purpose == .switchDatabase + } + + // A dial, a session check or another window's switch in flight lands first: the teardown would cancel the dial, + // and the check's lost-transaction notice or the other switch would never land. The newest switch then wins. + // `open` is the open this switch is a step of: it is the one attempt the switch never waits for. + private func reconnectWithDatabase( + _ database: String, + from scene: ConnectionCoordinator, + inside open: ConnectionAttempt? + ) async { + while true { + await appState.sessionRedialer.settled(connectionId) + guard let running = state.attempt, running !== open, let task = running.task else { break } + _ = await task.value + if state.attempt === running { break } + } + // A window closed during that wait must not tear down the session the other windows now use. + guard scene.isAttached, activeDatabase != database, + let switching = beginDatabaseSwitch(from: scene, inside: open) else { return } + guard switching.purpose == .switchDatabase else { + await runDatabaseSwitch(to: database, as: switching, from: scene) + return + } + let task = Task { + await runDatabaseSwitch(to: database, as: switching, from: scene) + return false + } + switching.task = task + _ = await task.value + } + + private func runDatabaseSwitch( + to database: String, + as switching: ConnectionAttempt, + from scene: ConnectionCoordinator + ) async { + let request = beginPlacementRequest() + let previous = dialTarget + await appState.connectionManager.disconnect(connectionId) + + var newConnection = record + newConnection.database = database + + do { + let newSession = try await appState.sessionRedialer.redial( + newConnection, + schema: nil, + prompter: scene.prompts, + joining: false, + consumer: self, + stillWanted: { [weak self] in self?.isCurrent(switching) == true } + ).session + guard isCurrent(switching) else { return } + let load = await loadTables(in: nil, on: newSession) + if let refusal = load.failure { throw refusal } + guard isCurrent(switching) else { return } + enter(.adopt(newSession, by: switching)) + commit(load, at: Placement(database: database, schema: activeSchema), onFailure: .keep, for: request) + UserDefaults.standard.set(database, forKey: ConnectionDefaultsKey.lastDB.name(for: connectionId)) + await loadSchemas(restoring: activeSchema, from: scene) + } catch { + guard isCurrent(switching) else { return } + Self.logger.error("Failed to switch to database \(database, privacy: .private(mask: .hash)): \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + // Back to the database the picker still shows, not the saved one, so the picker stays true. + do { + let fallback = try await appState.sessionRedialer.redial( + previous, + schema: supportsSchemas ? activeSchema : nil, + prompter: scene.prompts, + joining: false, + consumer: self, + stillWanted: { [weak self] in self?.isCurrent(switching) == true } + ) + guard isCurrent(switching), + await takeSession(fallback.session, dialedHere: fallback.dialedHere, by: switching) else { return } + scene.presentFailure( + title: ConnectionCoordinator.switchDatabaseFailedTitle, + message: classify(error, operation: "switchDatabase").message + ) + } catch { + guard isCurrent(switching) else { return } + enter(.lose(classify(error, operation: "switchDatabase"), switching)) + } + } + } + + func switchSchema(to name: String, on session: ConnectionSession) async -> ConnectionFailure? { + let request = beginPlacementRequest() + do { + try await session.driver.switchSchema(to: name) + } catch { + return ConnectionFailure(error: error, title: ConnectionCoordinator.switchSchemaFailedTitle, redialRepeats: false) + } + UserDefaults.standard.set(name, forKey: ConnectionDefaultsKey.lastSchema.name(for: connectionId)) + appState.bumpSessionRevision(connectionId) + let load = await loadTables(in: name, on: session) + // The session is on the new schema either way, so a refused list empties until a reload can read it. + commit(load, at: Placement(database: activeDatabase, schema: name), onFailure: .empty, for: request) + guard let refusal = load.failure else { return nil } + return ConnectionFailure(error: refusal, title: ConnectionCoordinator.refreshTablesFailedTitle, redialRepeats: true) + } + + // MARK: - Tables + + func refreshTables() async -> ConnectionFailure? { + guard let session else { return nil } + let load = await loadTables(in: supportsSchemas ? activeSchema : nil, on: session) + commit(load, onFailure: .keep) + guard let refusal = load.failure else { return nil } + Self.logger.warning("Failed to refresh tables: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") + return ConnectionFailure(error: refusal, title: ConnectionCoordinator.refreshTablesFailedTitle, redialRepeats: true) + } + + func refreshTransaction() async { + await appState.sessionTransactions.refresh(connectionId) + } + + // A Roll Back from any window can undo DDL; every window asks, and one reload serves them all. + func reloadAfterRollBack(from scene: ConnectionCoordinator) async { + guard !isReloadingAfterRollBack else { + rollBackReloadRequested = true + return + } + isReloadingAfterRollBack = true + defer { isReloadingAfterRollBack = false } + repeat { + rollBackReloadRequested = false + await reloadAfterRollBackOnce(from: scene) + } while rollBackReloadRequested + } + + // A failed read keeps the list it has. + private func reloadAfterRollBackOnce(from scene: ConnectionCoordinator) async { + guard !catalog.tablesRefused else { + await reloadMetadataIfReleased(from: scene) + return + } + guard let session else { return } + let load = await loadTables(in: supportsSchemas ? activeSchema : nil, on: session) + commit(load, onFailure: .keep) + if let refusal = load.failure { + Self.logger.warning("Failed to reload tables after a Roll Back: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") + } + // With autocommit off, a MySQL read opens a transaction. + await refreshTransaction() + } + + func reloadMetadataIfReleased(from scene: ConnectionCoordinator) async { + guard catalog.tablesRefused, let session, + appState.sessionTransactions.hold(for: connectionId, on: session.driver) == nil else { return } + let load = await loadTables(in: nil, on: session) + if let refusal = load.failure { + Self.logger.warning("Tables are still refused: \(LogRedaction.publicDescription(of: refusal), privacy: .public) \(refusal.localizedDescription, privacy: .private)") + return + } + guard commit(load, onFailure: .keep, clearingRefusal: true) else { return } + await loadDatabases(restoring: activeDatabase, from: scene) + await loadSchemas(restoring: activeSchema, from: scene) + } + + private enum OnFailure { + case keep + // The list stays, and reloads in full once nothing holds the session. + case refuse + case empty + } + + private struct CatalogLoad { + let driver: ObjectIdentifier + let from: Placement + let epoch: Int + var tables: Result<[TableInfo], any Error>? + var schemas: [String]? + var databases: [String]? + + var failure: (any Error)? { + guard case .failure(let error)? = tables else { return nil } + return error + } + } + + private func loadTables(in schema: String?, on session: ConnectionSession) async -> CatalogLoad { + let from = catalog.placement + let epoch = placementEpoch + let driver = ObjectIdentifier(session.driver) + do { + let tables = try await session.driver.fetchTables(schema: schema) + return CatalogLoad(driver: driver, from: from, epoch: epoch, tables: .success(tables)) + } catch { + return CatalogLoad(driver: driver, from: from, epoch: epoch, tables: .failure(error)) + } + } + + // The one place a read lands. Placement and lists change in one step, and only on the session the read used: a + // placement request only while it is the newest, any other read only while nothing moved since it started. + @discardableResult + private func commit( + _ load: CatalogLoad, + at placement: Placement? = nil, + onFailure: OnFailure, + clearingRefusal: Bool = false, + for request: Int? = nil + ) -> Bool { + guard let session, ObjectIdentifier(session.driver) == load.driver else { return false } + if let request { + guard request == placementEpoch else { return false } + } else { + guard load.epoch == placementEpoch, catalog.placement == load.from else { return false } + } + var next = catalog + next.placement = placement ?? catalog.placement + switch load.tables { + case .success(let tables)?: + next.tables = tables + if clearingRefusal { next.tablesRefused = false } + case .failure?: + switch onFailure { + case .keep: + break + case .refuse: + next.tablesRefused = true + case .empty: + next.tables = [] + next.tablesRefused = true + } + case nil: + break + } + if let schemas = load.schemas { next.schemas = schemas } + if let databases = load.databases { next.databases = databases } + if next != catalog { catalog = next } + return true + } + + // MARK: - Catalog Changes + + // `reloadedHere`: the caller already read the session, the one place a transaction's own DDL shows. + func noteCatalogChanged(_ kinds: SQLCatalogObjectKinds, reloadedHere: Bool) { + let board = appState.catalogChanges + let caughtUp = board.generation(for: connectionId) == seenCatalogGeneration + board.post(kinds, for: connectionId) + if reloadedHere, caughtUp { + seenCatalogGeneration = board.generation(for: connectionId) + } + } + + // A statement a window ran may have created or dropped what the lists show; failed reads keep the lists. + // One reload per connection: a later post waits for it, so an older answer never lands over a newer one. + func reloadChangedCatalog() async { + guard !isReloadingCatalog else { + catalogReloadRequested = true + return + } + isReloadingCatalog = true + defer { isReloadingCatalog = false } + repeat { + catalogReloadRequested = false + await reloadChangedCatalogOnce() + } while catalogReloadRequested + } + + private func reloadChangedCatalogOnce() async { + let board = appState.catalogChanges + let generation = board.generation(for: connectionId) + guard generation != seenCatalogGeneration else { return } + // A placement in motion would make this read the wrong one's, and a PostgreSQL switch leaves no session to + // read; the switch's end runs it again. + guard switchesInFlight == 0, !isSwitchingDatabase else { + catalogReloadHeldBack = true + return + } + guard let session else { return } + var wanted = board.kinds(for: connectionId, since: seenCatalogGeneration) + if !supportsSchemas { wanted.remove(.schemas) } + if !supportsDatabaseSwitching { wanted.remove(.databases) } + let kinds = wanted + let database = supportsDatabaseSwitching ? activeDatabase : "" + let schema = supportsSchemas ? activeSchema : nil + let readFrom = catalog.placement + let epoch = placementEpoch + let monitor = appState.sessionTransactions + let matchesSession = catalogMatchesSession(on: session.driver) + + var snapshot: CatalogSnapshot? + // On the session a read would replace ROW_COUNT() and the warnings, so it is only where the DDL is visible, or + // where a USE or a role moved the session off what the catalog connection sees, as Refresh reads it there too. + if matchesSession { + do { + snapshot = try await appState.connectionManager.readCatalog(for: connectionId, database: database) { + try await Self.readCatalog(kinds, schema: schema, from: $0) + } + } catch { + Self.logger.warning("Reloading the catalog after a change failed: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + seenCatalogGeneration = generation + return + } + } + if snapshot == nil { + // A server that has a catalog connection but could not open one keeps its lists: a read on the session would + // replace what the user's last statement left there. Only its own uncommitted DDL sends the read there. + if matchesSession, appState.connectionManager.opensCatalogConnections(for: connectionId) { + seenCatalogGeneration = generation + return + } + // A stream on the session would end under this read, so it waits for the session to go idle. + guard !monitor.isBusy(connectionId) else { return } + board.beginSessionRead(for: connectionId) + snapshot = try? await monitor.track(connectionId, reading: true) { + try await Self.readCatalog(kinds, schema: schema, from: session.driver) + } + board.endSessionRead(for: connectionId) + } + seenCatalogGeneration = generation + guard let snapshot else { return } + var load = CatalogLoad(driver: ObjectIdentifier(session.driver), from: readFrom, epoch: epoch) + if let tables = snapshot.tables { load.tables = .success(tables) } + load.schemas = snapshot.schemas + load.databases = snapshot.databases + commit(load, onFailure: .keep) + } + + private struct CatalogSnapshot: Sendable { + var tables: [TableInfo]? + var schemas: [String]? + var databases: [String]? + } + + // The table list is the read that matters: its failure fails the reload, so a dead connection is dropped. + nonisolated private static func readCatalog( + _ kinds: SQLCatalogObjectKinds, + schema: String?, + from driver: any DatabaseDriver + ) async throws -> CatalogSnapshot { + var snapshot = CatalogSnapshot() + if kinds.contains(.tables) { + snapshot.tables = try await driver.fetchTables(schema: schema) + } + if kinds.contains(.schemas) { + snapshot.schemas = try? await driver.fetchSchemas() + } + if kinds.contains(.databases) { + snapshot.databases = try? await driver.fetchDatabases() + } + return snapshot + } + + private static let transactionalDDLTypes: Set = [.postgresql, .redshift, .mssql] + + // PostgreSQL, Redshift and SQL Server keep DDL inside the transaction that ran it, where only the session sees it; + // a session that cannot say whether it holds one counts as holding one. + func catalogSeesSessionDDL(on driver: (any DatabaseDriver)?) -> Bool { + guard Self.transactionalDDLTypes.contains(record.type) else { return true } + return appState.sessionTransactions.isKnownIdle(connectionId, on: driver) + } + + // Completion reads beside the session only while both see the same tables: same DDL, same database, no table + // only a procedure on the session made. + func catalogMatchesSession(on driver: (any DatabaseDriver)?) -> Bool { + guard catalogSeesSessionDDL(on: driver) else { return false } + return !appState.catalogChanges.sessionLeftCatalog( + for: connectionId, driver: driver.map { ObjectIdentifier($0) } + ) + } + + // MARK: - Lists + + // `wanted` is where the connection was; the session may have opened elsewhere, and the asking window switches back, + // as a step of `open` when an open asks. + private func loadDatabases( + restoring wanted: String, + from scene: ConnectionCoordinator?, + inside open: ConnectionAttempt? = nil + ) async { + guard let session, supportsDatabaseSwitching else { return } + do { + let listed = try await session.driver.fetchDatabases() + catalog.databases = listed + let sessionDB = appState.connectionManager.session(for: connectionId)?.activeDatabase ?? record.database + if !wanted.isEmpty, listed.contains(wanted) { + catalog.placement.database = sessionDB + if wanted != sessionDB { + await scene?.switchDatabase(to: wanted, inside: open) + } + } else { + catalog.placement.database = sessionDB + // A database the connection names by its id is listed by its name, which the driver reports. + if !listed.contains(activeDatabase), let reported = session.driver.currentDatabase, + listed.contains(reported) { + await scene?.switchDatabase(to: reported, inside: open) + } + } + } catch { + Self.logger.warning("Failed to load databases: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + } + } + + private func loadSchemas(restoring wanted: String, from scene: ConnectionCoordinator?) async { + guard let session, supportsSchemas else { return } + do { + let listed = try await session.driver.fetchSchemas() + catalog.schemas = listed + let currentSchema = session.driver.currentSchema ?? "public" + if listed.contains(wanted), wanted != currentSchema { + catalog.placement.schema = currentSchema + // A switch in flight lands its own schema, which the restore never moves back. + if switchesInFlight == 0 { + await restoreSchema(wanted, on: session, reportingTo: scene) + } + } else if !listed.contains(activeSchema) { + catalog.placement.schema = currentSchema + } + } catch { + Self.logger.warning("Failed to load schemas: \(LogRedaction.publicDescription(of: error), privacy: .public) \(error.localizedDescription, privacy: .private)") + } + } + + // On the model's own session, as a switch every window waits for: a window that asked may be pinned on another + // session. Its failure is that window's to hear only while it shows the live session. + private func restoreSchema(_ name: String, on session: ConnectionSession, reportingTo scene: ConnectionCoordinator?) async { + beginSwitch() + let failure = await switchSchema(to: name, on: session) + endSwitch() + if let failure, let scene, scene.showsLiveSession { + await scene.report(failure, operation: "switchSchema") + } + // What the restore held back goes now, as it does after a window's own switch. + if let lead = scene ?? followRequester { + await runHeldBackWork(from: lead) + } else if catalogReloadHeldBack, switchesInFlight == 0 { + catalogReloadHeldBack = false + await reloadChangedCatalog() + } + } +} diff --git a/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionState.swift b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionState.swift new file mode 100644 index 0000000000..f614df3246 --- /dev/null +++ b/apple/TableProMobile/TableProMobile/Coordinators/ConnectionSessionState.swift @@ -0,0 +1,260 @@ +import Foundation +import TableProDatabase +import TableProModels + +// One connection's session lifecycle. Each async step carries the attempt it started under and lands only while the +// state still holds that attempt, so a dial or a load that returns after Cancel, or after a newer attempt, changes +// nothing. +enum ConnectionSessionState { + case notConnected + // `provisional` is the session an open reattached or dialed, held while the lists load. + case connecting(ConnectionAttempt, provisional: ConnectionSession?) + case connected(ConnectionSession) + // A session check pinging the session, before it knows whether to redial. + case checking(ConnectionAttempt, over: Resting) + // A redial, or a PostgreSQL database switch, which leaves no session until the new one lands. + case reconnecting(ConnectionAttempt, over: Resting) + case failed(AppError) + case lost(AppError) + + enum Resting { + case connected(ConnectionSession) + case lost(AppError) + + var phase: ConnectionCoordinator.ConnectionPhase { + switch self { + case .connected: return .connected + case .lost(let error): return .lost(error) + } + } + + var settled: ConnectionSessionState { + switch self { + case .connected(let session): return .connected(session) + case .lost(let error): return .lost(error) + } + } + } + + enum Event { + case open(ConnectionAttempt) + case check(ConnectionAttempt) + case dial(ConnectionAttempt) + case switchDatabase(ConnectionAttempt) + // A database switch inside an open drops the session the open had. + case teardown(ConnectionAttempt) + // With no attempt: a session another window moved or replaced, or a USE moved, taken in place. + case adopt(ConnectionSession, by: ConnectionAttempt?) + case settle(ConnectionAttempt) + case opened(ConnectionAttempt) + case fail(AppError, ConnectionAttempt) + case lose(AppError, ConnectionAttempt) + // The last window waiting left: an open goes back to the loss or failure it started from, so a window that + // still shows it keeps its Reconnect, and a check or a switch goes back to the session it had. + case abandon(ConnectionAttempt) + // The record changed how it dials, so nothing of the old session is kept. + case cancel + } + + // Nil when the state refuses the event: a step of an attempt it no longer holds, or one it cannot take. + func applying(_ event: Event) -> ConnectionSessionState? { + switch event { + case .open(let attempt): + if isOpening { return nil } + return .connecting(attempt, provisional: nil) + case .check(let attempt): + // A failed redial left nothing to ping, so a check over a loss dials again. + guard let resting else { return nil } + return .checking(attempt, over: resting) + case .dial(let attempt): + guard case .checking(let current, let over) = self, current === attempt else { return nil } + return .reconnecting(attempt, over: over) + case .switchDatabase(let attempt): + // A check in flight finishes first, or its lost-transaction notice would never be given. + guard let resting else { return nil } + return .reconnecting(attempt, over: resting) + case .teardown(let attempt): + guard case .connecting(let current, _) = self, current === attempt else { return nil } + return .connecting(attempt, provisional: nil) + case .adopt(let session, let lander): + guard let lander else { return replacingSession(with: session) } + guard self.attempt === lander else { return nil } + if isOpening { return .connecting(lander, provisional: session) } + return .connected(session) + case .settle(let attempt): + guard self.attempt === attempt else { return nil } + switch self { + case .checking(_, let over), .reconnecting(_, let over): return over.settled + case .notConnected, .connecting, .connected, .failed, .lost: return nil + } + case .opened(let attempt): + guard case .connecting(let current, let provisional?) = self, current === attempt else { return nil } + return .connected(provisional) + case .fail(let error, let attempt): + guard case .connecting(let current, _) = self, current === attempt else { return nil } + return .failed(error) + case .lose(let error, let attempt): + guard self.attempt === attempt else { return nil } + if case .checking = self { return nil } + return .lost(error) + case .abandon(let attempt): + guard self.attempt === attempt else { return nil } + return atRest + case .cancel: + return .notConnected + } + } + + // What a window shows once nothing is in flight: the loss or failure an open started from, or the session a + // check or a switch started over. + var atRest: ConnectionSessionState { + switch self { + case .connecting(let attempt, _): + switch attempt.origin { + case .failed(let error)?: return .failed(error) + case .lost(let error)?: return .lost(error) + case nil: return .notConnected + } + case .checking(_, let over), .reconnecting(_, let over): + return over.settled + case .notConnected, .connected, .failed, .lost: + return self + } + } + + // What an open started here goes back to when every window waiting on it leaves. + var openOrigin: ConnectionAttempt.Origin? { + switch self { + case .failed(let error): return .failed(error) + case .lost(let error): return .lost(error) + case .notConnected, .connecting, .connected, .checking, .reconnecting: return nil + } + } + + var attempt: ConnectionAttempt? { + switch self { + case .connecting(let attempt, _), .checking(let attempt, _), .reconnecting(let attempt, _): return attempt + case .notConnected, .connected, .failed, .lost: return nil + } + } + + // The running session check, which a later trigger joins. + var check: ConnectionAttempt? { + guard let attempt, attempt.purpose == .redial else { return nil } + return attempt + } + + var session: ConnectionSession? { + switch self { + case .connecting(_, let provisional): + return provisional + case .connected(let session), .checking(_, .connected(let session)): + return session + case .reconnecting(let attempt, .connected(let session)): + return attempt.purpose == .switchDatabase ? nil : session + case .notConnected, .checking(_, .lost), .reconnecting(_, .lost), .failed, .lost: + return nil + } + } + + var phase: ConnectionCoordinator.ConnectionPhase { + switch self { + case .notConnected: return .notConnected + case .connecting: return .connecting + case .connected: return .connected + case .checking(_, let over), .reconnecting(_, let over): return over.phase + case .failed(let error): return .failed(error) + case .lost(let error): return .lost(error) + } + } + + // A database switch names its target instead. + var isReconnecting: Bool { + guard case .reconnecting(let attempt, _) = self else { return false } + return attempt.purpose == .redial + } + + var isOpening: Bool { + if case .connecting = self { return true } + return false + } + + private var resting: Resting? { + switch self { + case .connected(let session): return .connected(session) + case .lost(let error): return .lost(error) + case .notConnected, .connecting, .checking, .reconnecting, .failed: return nil + } + } + + private func replacingSession(with session: ConnectionSession) -> ConnectionSessionState? { + switch self { + case .connecting(let attempt, _): + return .connecting(attempt, provisional: session) + case .connected: + return .connected(session) + case .checking(let attempt, .connected): + return .checking(attempt, over: .connected(session)) + case .reconnecting(let attempt, .connected) where attempt.purpose == .redial: + return .reconnecting(attempt, over: .connected(session)) + case .notConnected, .checking(_, .lost), .reconnecting, .failed, .lost: + return nil + } + } +} + +// One open, session check or PostgreSQL database switch. Its identity is what a late step is checked against. +final class ConnectionAttempt { + nonisolated enum Purpose { + case open + case redial + case switchDatabase + } + + enum Origin { + case failed(AppError) + case lost(AppError) + } + + let purpose: Purpose + // For an open: the loss or failure it started from. + let origin: Origin? + // The windows that asked for it or wait on it, first asker first; the last one to leave abandons it. + private(set) var consumers: [ObjectIdentifier] = [] + // A statement failed while this redial ran, so it may have ended the transaction itself. + var ambiguous = false + var task: Task? + + init(_ purpose: Purpose, origin: Origin? = nil) { + self.purpose = purpose + self.origin = origin + } + + func join(_ consumer: AnyObject) { + let id = ObjectIdentifier(consumer) + guard !consumers.contains(id) else { return } + consumers.append(id) + } + + // True when that consumer was the last one. + func leave(_ consumer: AnyObject) -> Bool { + guard let index = consumers.firstIndex(of: ObjectIdentifier(consumer)) else { return false } + consumers.remove(at: index) + return consumers.isEmpty + } +} + +// What a window's lists show and where they were read. +nonisolated struct ConnectionCatalog: Equatable { + nonisolated struct Placement: Hashable { + var database: String + var schema: String + } + + var placement = Placement(database: "", schema: "public") + var databases: [String] = [] + var schemas: [String] = [] + var tables: [TableInfo] = [] + // A table read was refused; the lists reload in full once nothing holds the session. + var tablesRefused = false +} diff --git a/apple/TableProMobile/TableProMobile/Coordinators/SessionRedialer.swift b/apple/TableProMobile/TableProMobile/Coordinators/SessionRedialer.swift index e7fcb97b34..6c26522c89 100644 --- a/apple/TableProMobile/TableProMobile/Coordinators/SessionRedialer.swift +++ b/apple/TableProMobile/TableProMobile/Coordinators/SessionRedialer.swift @@ -5,8 +5,8 @@ import TableProLogRedaction import TableProModels import TableProSSHTransport -// Each scene has its own coordinator on the one shared session, so two windows coming back at once -// would each dial, and the later attempt cancels the earlier one's. +// One dial per connection at a time: a dial that would start a second one joins the first, or waits for it when it +// needs its own target, since the later attempt cancels the earlier one's. @MainActor final class SessionRedialer { struct Outcome { diff --git a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Decode.swift b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Decode.swift index c9f143bcce..0cd74fc0b7 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Decode.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Decode.swift @@ -3,6 +3,25 @@ import Foundation import TableProModels extension DuckDBActor { + // A streamed BLOB is its bytes, as on every other engine. `decodeCell` keeps the base64 text `execute` returns. + static func streamedCell( + vector: duckdb_vector, + row: idx_t, + column: DuckDBStreamColumn, + options: StreamOptions, + ref: CellRef? + ) -> Cell { + guard !column.castToText, column.type == DUCKDB_TYPE_BLOB else { + return decodeCell(vector: vector, row: row, column: column, options: options, ref: ref) + } + if let validity = duckdb_vector_get_validity(vector), !duckdb_validity_row_is_valid(validity, row) { + return .null + } + guard let data = duckdb_vector_get_data(vector) else { return .null } + let value = data.assumingMemoryBound(to: duckdb_string_t.self)[Int(row)] + return .binary(byteCount: Int(duckdb_string_t_length(value)), ref: ref) + } + static func decodeCell( vector: duckdb_vector, row: idx_t, @@ -83,11 +102,7 @@ extension DuckDBActor { let length = Int(duckdb_string_t_length(value)) guard length > 0, let pointer = duckdb_string_t_data(&value) else { return .text("") } let text = String(bytes: UnsafeRawBufferPointer(start: pointer, count: length), encoding: .utf8) ?? "" - if length > options.textTruncationBytes { - let prefix = String(text.prefix(options.textTruncationBytes)) - return .truncatedText(prefix: prefix, totalBytes: length, ref: ref) - } - return .text(text) + return Cell.from(text: text, options: options, ref: ref) } private static func decodeBlob(_ data: UnsafeMutableRawPointer, row: idx_t) -> Cell { diff --git a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Streaming.swift b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Streaming.swift index 79056a3882..c5e7f64c31 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Streaming.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/DuckDBDriver+Streaming.swift @@ -50,7 +50,12 @@ nonisolated extension DuckDBDriver { break } let cells = legacyRow.enumerated().map { index, value -> Cell in - Cell.from(legacyValue: value, columnTypeName: index < columns.count ? columns[index].typeName : nil, options: options) + let typeName = index < columns.count ? columns[index].typeName : nil + // `execute` carries a BLOB as base64, so the cell counts the bytes it decodes to. + if let value, typeName == DuckDBActor.typeName(for: DUCKDB_TYPE_BLOB) { + return .binary(byteCount: Self.base64ByteCount(value), ref: nil) + } + return Cell.from(legacyValue: value, columnTypeName: typeName, options: options) } continuation.yield(.row(Row(cells: cells))) emitted += 1 @@ -60,6 +65,12 @@ nonisolated extension DuckDBDriver { } } + private static func base64ByteCount(_ text: String) -> Int { + let utf8 = text.utf8 + let padding = utf8.reversed().prefix(2).prefix { $0 == UInt8(ascii: "=") }.count + return utf8.count / 4 * 3 - padding + } + func executeStreaming(query: String, options: StreamOptions) -> AsyncThrowingStream { let actor = self.actor let interruptHandle = actor.interruptHandle @@ -213,7 +224,7 @@ extension DuckDBActor { cells.append(.null) continue } - cells.append(Self.decodeCell( + cells.append(Self.streamedCell( vector: vector, row: row, column: columns[index], diff --git a/apple/TableProMobile/TableProMobile/Drivers/MSSQLDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/MSSQLDriver.swift index 11186e7582..98bdec9fd4 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/MSSQLDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/MSSQLDriver.swift @@ -199,7 +199,6 @@ nonisolated final class MSSQLDriver: DatabaseDriver, @unchecked Sendable { let cells = zip(headerColumns, row).map { columnInfo, rawCell in cell( from: rawCell, - columnTypeName: columnInfo.typeName, options: options, ref: rowKey?.ref(column: columnInfo.name, primaryKey: primaryKey) ) @@ -234,12 +233,12 @@ nonisolated final class MSSQLDriver: DatabaseDriver, @unchecked Sendable { } // A binary value reads back through `execute` as text, not its bytes, so it offers no full value to load. - private func cell(from raw: MSSQLRawCell, columnTypeName: String, options: StreamOptions, ref: CellRef?) -> Cell { + private func cell(from raw: MSSQLRawCell, options: StreamOptions, ref: CellRef?) -> Cell { switch raw { case .null: return .null case .string(let value): - return Cell.from(legacyValue: value, columnTypeName: columnTypeName, options: options, ref: ref) + return Cell.from(text: value, options: options, ref: ref) case .bytes(let data): return .binary(byteCount: data.count, ref: nil) } diff --git a/apple/TableProMobile/TableProMobile/Drivers/MySQLDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/MySQLDriver.swift index 3170d07c7d..cf0a58a580 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/MySQLDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/MySQLDriver.swift @@ -1094,12 +1094,7 @@ private actor MySQLActor { case .bytes(let data): return .binary(byteCount: data.count, ref: ref) case .text(let text): - return Cell.from( - legacyValue: text, - columnTypeName: column.typeName, - options: options, - ref: ref - ) + return Cell.from(text: text, options: options, ref: ref) @unknown default: return .null } diff --git a/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift index 184e1fdade..8e7d52ce96 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/OracleDriver.swift @@ -170,7 +170,6 @@ nonisolated final class OracleDriver: DatabaseDriver, @unchecked Sendable { let cells = zip(headerColumns, row).map { columnInfo, rawCell in cell( from: rawCell, - columnTypeName: columnInfo.typeName, options: options, ref: rowKey?.ref(column: columnInfo.name, primaryKey: primaryKey) ) @@ -198,12 +197,12 @@ nonisolated final class OracleDriver: DatabaseDriver, @unchecked Sendable { } // A binary value reads back through `execute` as text, not its bytes, so it offers no full value to load. - private func cell(from raw: OracleRawCell, columnTypeName: String, options: StreamOptions, ref: CellRef?) -> Cell { + private func cell(from raw: OracleRawCell, options: StreamOptions, ref: CellRef?) -> Cell { switch raw { case .null: return .null case .string(let value): - return Cell.from(legacyValue: value, columnTypeName: columnTypeName, options: options, ref: ref) + return Cell.from(text: value, options: options, ref: ref) case .bytes(let data): return .binary(byteCount: data.count, ref: nil) } diff --git a/apple/TableProMobile/TableProMobile/Drivers/PluginBackedDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/PluginBackedDriver.swift index 0130bd43be..61c8b7b681 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/PluginBackedDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/PluginBackedDriver.swift @@ -225,8 +225,7 @@ nonisolated class PluginBackedDriver: DatabaseDriver, @unchecked Sendable { case .bytes(let data): return .binary(byteCount: data.count, ref: ref) case .text(let text): - let typeName = index < columns.count ? columns[index].typeName : nil - return Cell.from(legacyValue: text, columnTypeName: typeName, options: options, ref: ref) + return Cell.from(text: text, options: options, ref: ref) } } continuation.yield(.row(Row(cells: cells, locator: locator))) diff --git a/apple/TableProMobile/TableProMobile/Drivers/SQLiteDriver.swift b/apple/TableProMobile/TableProMobile/Drivers/SQLiteDriver.swift index 980ad0e87e..c411752fdd 100644 --- a/apple/TableProMobile/TableProMobile/Drivers/SQLiteDriver.swift +++ b/apple/TableProMobile/TableProMobile/Drivers/SQLiteDriver.swift @@ -542,12 +542,8 @@ private actor SQLiteActor { default: if let str = Self.text(of: stmt, column: columnIndex) { let ref = makeCellRef(column: columns[i].name, columns: columns, statement: stmt, options: options) - cells.append(Cell.from( - legacyValue: str, - columnTypeName: columns[i].typeName, - options: options, - ref: ref - )) + // The storage class, not the declared type: a BLOB column can hold text. + cells.append(Cell.from(text: str, options: options, ref: ref)) } else { cells.append(.null) } diff --git a/apple/TableProMobile/TableProMobile/Helpers/ClipboardExporter.swift b/apple/TableProMobile/TableProMobile/Helpers/ClipboardExporter.swift index 46638ebd8c..dfc7c70383 100644 --- a/apple/TableProMobile/TableProMobile/Helpers/ClipboardExporter.swift +++ b/apple/TableProMobile/TableProMobile/Helpers/ClipboardExporter.swift @@ -32,8 +32,20 @@ nonisolated enum ClipboardExporter { } nonisolated struct TooLarge: LocalizedError, Equatable { + enum Source: Equatable { + case page + case queryResult + } + + var source: Source = .page + var errorDescription: String? { - String(localized: "The full values on this page need more memory than this device has free. Choose a smaller page size and try again.") + switch source { + case .page: + String(localized: "The full values on this page need more memory than this device has free. Choose a smaller page size and try again.") + case .queryResult: + String(localized: "The full values in this result need more memory than this device has free. Run a query that returns fewer rows and try again.") + } } } diff --git a/apple/TableProMobile/TableProMobile/Localizable.xcstrings b/apple/TableProMobile/TableProMobile/Localizable.xcstrings index 8032af820e..2588102bc1 100644 --- a/apple/TableProMobile/TableProMobile/Localizable.xcstrings +++ b/apple/TableProMobile/TableProMobile/Localizable.xcstrings @@ -21153,6 +21153,9 @@ } } } + }, + "Shown in part. Run a query that returns fewer rows or columns to see it in full." : { + }, "SID" : { "localizations" : { @@ -21460,6 +21463,9 @@ }, "Some documents may already have been changed when this failed. MongoDB does not undo them, so check the data before you run it again." : { + }, + "Some long values are shown in part to stay within memory." : { + }, "Sort" : { "localizations" : { @@ -24942,6 +24948,9 @@ "The FreeTDS configuration could not be written: %@" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "The full values in this result need more memory than this device has free. Run a query that returns fewer rows and try again." : { + }, "The full values on this page need more memory than this device has free. Choose a smaller page size and try again." : { @@ -29459,6 +29468,9 @@ }, "View as JSON" : { + }, + "View Full Value" : { + }, "Views" : { "localizations" : { diff --git a/apple/TableProMobile/TableProMobile/Models/StreamingResultBuffer.swift b/apple/TableProMobile/TableProMobile/Models/StreamingResultBuffer.swift index 8a262c15cc..805a260972 100644 --- a/apple/TableProMobile/TableProMobile/Models/StreamingResultBuffer.swift +++ b/apple/TableProMobile/TableProMobile/Models/StreamingResultBuffer.swift @@ -51,7 +51,8 @@ final class StreamingResultBuffer { flushTask?.cancel() flushTask = nil guard !pendingRows.isEmpty else { return } - let legacyBatch = pendingRows.map(\.legacyValues) + // Display text only: a long value kept whole stays whole in `window` for copy, export and the row screen. + let legacyBatch = pendingRows.map { $0.displayValues() } window.append(contentsOf: pendingRows) legacyRows.append(contentsOf: legacyBatch) if legacyRows.count > window.count { diff --git a/apple/TableProMobile/TableProMobile/Platform/MemoryPressureMonitor.swift b/apple/TableProMobile/TableProMobile/Platform/MemoryPressureMonitor.swift index 8b58f12408..6ffa538a61 100644 --- a/apple/TableProMobile/TableProMobile/Platform/MemoryPressureMonitor.swift +++ b/apple/TableProMobile/TableProMobile/Platform/MemoryPressureMonitor.swift @@ -56,4 +56,18 @@ final class MemoryPressureMonitor { guard available > 0 else { return true } return available > requiredBytes } + + nonisolated func textBudgetBytes() -> Int { + Self.textBudgetBytes(availableBytes: availableMemoryBytes()) + } + + // A sixth of the headroom past the margin: the kept text, the four copies `exportFits` wants room for, and a + // share for the rest of the app. The call answers 0 outside an app, where only the ceiling bounds it. + nonisolated static func textBudgetBytes(availableBytes: Int) -> Int { + guard availableBytes > 0 else { return textBudgetCeilingBytes } + return min(textBudgetCeilingBytes, max(0, availableBytes - textBudgetMarginBytes) / 6) + } + + nonisolated private static let textBudgetCeilingBytes = 64 * 1_024 * 1_024 + nonisolated private static let textBudgetMarginBytes = 64 * 1_024 * 1_024 } diff --git a/apple/TableProMobile/TableProMobile/ViewModels/DataBrowserViewModel.swift b/apple/TableProMobile/TableProMobile/ViewModels/DataBrowserViewModel.swift index 924322f039..ddf69bbe3e 100644 --- a/apple/TableProMobile/TableProMobile/ViewModels/DataBrowserViewModel.swift +++ b/apple/TableProMobile/TableProMobile/ViewModels/DataBrowserViewModel.swift @@ -18,6 +18,7 @@ final class DataBrowserViewModel { private static let logger = Logger(subsystem: "com.TablePro", category: "DataBrowserViewModel") private let buffer: StreamingResultBuffer + private let textBudgetBytes: @MainActor () -> Int private(set) var totalRows: Int? private(set) var phase: Phase = .idle private(set) var executionTime: TimeInterval = 0 @@ -63,8 +64,12 @@ final class DataBrowserViewModel { case found } - init(windowCapacity: Int = 1_000) { + init( + windowCapacity: Int = 1_000, + textBudgetBytes: @escaping @MainActor () -> Int = { MemoryPressureMonitor.shared.textBudgetBytes() } + ) { self.buffer = StreamingResultBuffer(capacity: windowCapacity) + self.textBudgetBytes = textBudgetBytes self.pagination = PaginationState(pageSize: AppPreferences.defaultPageSize, currentPage: 0) } @@ -259,7 +264,9 @@ final class DataBrowserViewModel { pagination.totalRows = page.totalCount let result = page.result - await loadPage(options: pageOptions(lazyContext: nil), startedAt: start, generation: generation) { options in + // A key's values have no key of their own to be read again by, so they stay whole within a budget. + let keyOptions = pageOptions(lazyContext: nil, textBudget: TextBudget(bytes: textBudgetBytes())) + await loadPage(options: keyOptions, startedAt: start, generation: generation) { options in QueryResultStreaming.stream(options: options) { result } } guard isCurrent(generation) else { return } @@ -284,12 +291,13 @@ final class DataBrowserViewModel { pagination.totalRows = pagination.currentOffset + legacyRows.count } - private func pageOptions(lazyContext: LazyContext?) -> StreamOptions { + private func pageOptions(lazyContext: LazyContext?, textBudget: TextBudget? = nil) -> StreamOptions { StreamOptions( textTruncationBytes: 4_096, inlineBinary: false, maxRows: pagination.pageSize, - lazyContext: lazyContext + lazyContext: lazyContext, + textBudget: textBudget ) } @@ -349,11 +357,7 @@ final class DataBrowserViewModel { } private func searchableColumns() -> [ColumnInfo] { - columns.filter { col in - let upper = col.typeName.uppercased() - return !upper.contains("BLOB") && !upper.contains("BYTEA") && !upper.contains("BINARY") - && !upper.contains("VARBINARY") && !upper.contains("IMAGE") - } + columns.filter { !Cell.holdsBytes(typeName: $0.typeName) } } private func fetchTotalRows(session: ConnectionSession, table: TableInfo, generation: Int) async { diff --git a/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift b/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift index 0b8a94ef79..1e062df830 100644 --- a/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift +++ b/apple/TableProMobile/TableProMobile/ViewModels/QueryEditorViewModel.swift @@ -47,12 +47,14 @@ final class QueryEditorViewModel { private static let budgetCheckInterval = 500 private let buffer: StreamingResultBuffer + private let textBudgetBytes: @MainActor () -> Int private(set) var phase: Phase = .idle private(set) var executionTime: TimeInterval = 0 private(set) var requestCount = 0 private(set) var currentRequest = 0 private(set) var rowsAffectedTotal = 0 private(set) var interruption: ScriptInterruption? + private(set) var cutLongValues = false // Each request as it went to the server, split again after a statement changed how the session reads literals. @ObservationIgnored private(set) var sentStatements: [String] = [] @ObservationIgnored private(set) var ranTransactionRollBack = false @@ -61,9 +63,14 @@ final class QueryEditorViewModel { @ObservationIgnored private var runID: UInt64 = 0 @ObservationIgnored private var fetchTask: Task? @ObservationIgnored private var startedAt: Date? + @ObservationIgnored private var requestBudget: TextBudget? - init(windowCapacity: Int = QueryEditorViewModel.maxBufferedRows) { + init( + windowCapacity: Int = QueryEditorViewModel.maxBufferedRows, + textBudgetBytes: @escaping @MainActor () -> Int = { MemoryPressureMonitor.shared.textBudgetBytes() } + ) { self.buffer = StreamingResultBuffer(capacity: windowCapacity) + self.textBudgetBytes = textBudgetBytes } var columns: [ColumnInfo] { buffer.columns } @@ -101,6 +108,21 @@ final class QueryEditorViewModel { } } + var longValuesNotice: String? { + cutLongValues ? String(localized: "Some long values are shown in part to stay within memory.") : nil + } + + // The export text and the pasteboard's copy of it are held beside the values already on screen. + func resultsFitExport(availableBytes: Int) -> Bool { + let textBytes = buffer.window.rows.reduce(0) { total, row in + row.cells.reduce(total) { sum, cell in + guard case .text(let value) = cell else { return sum } + return sum + value.utf8.count + } + } + return DataBrowserViewModel.exportFits(pendingBytes: textBytes, availableBytes: availableBytes) + } + var progressMessage: String { guard requestCount > 1 else { return String(localized: "Executing...") } switch unitKind { @@ -232,6 +254,8 @@ final class QueryEditorViewModel { executionTime = 0 rowsAffectedTotal = 0 interruption = nil + requestBudget = nil + cutLongValues = false ranTransactionRollBack = false unitKind = script.unitKind requestCount = script.requestCount @@ -259,6 +283,8 @@ final class QueryEditorViewModel { executionTime = 0 rowsAffectedTotal = 0 interruption = nil + requestBudget = nil + cutLongValues = false ranTransactionRollBack = false requestCount = 0 currentRequest = 0 @@ -351,6 +377,7 @@ final class QueryEditorViewModel { guard run == runID else { return } buffer.flush() finalizeTiming() + settleLongValues() resolvePhase() } @@ -370,8 +397,13 @@ final class QueryEditorViewModel { /// False when the request was cut short: by Stop, by memory pressure, or by another query that took over the /// connection, whose driver then ends this stream as cancelled. private func stream(_ sql: String, driver: DatabaseDriver, options: StreamOptions) async throws -> Bool { + // Only this request's rows stay on screen, so it gets a budget of its own, sized from the headroom now. + let budget = TextBudget(bytes: textBudgetBytes()) + requestBudget = budget + var requestOptions = options + requestOptions.textBudget = budget var sinceBudgetCheck = 0 - for try await element in driver.executeStreaming(query: sql, options: options) { + for try await element in driver.executeStreaming(query: sql, options: requestOptions) { if Task.isCancelled { return false } buffer.apply(element) guard case .row = element else { continue } @@ -393,6 +425,7 @@ final class QueryEditorViewModel { guard run == runID else { return } buffer.flush() finalizeTiming() + settleLongValues() let reason = buffer.truncation ?? .cancelled if requestCount > 1 { let cause: ScriptInterruption.Cause @@ -424,6 +457,7 @@ final class QueryEditorViewModel { guard run == runID else { return } buffer.flush() finalizeTiming() + settleLongValues() if requestCount > 1 { interruption = ScriptInterruption( cause: .failed(sql: sql), @@ -462,6 +496,10 @@ final class QueryEditorViewModel { } } + private func settleLongValues() { + cutLongValues = requestBudget?.hasRefused ?? false + } + private func finalizeTiming() { if let startedAt { executionTime = Date().timeIntervalSince(startedAt) diff --git a/apple/TableProMobile/TableProMobile/ViewModels/RowDetailViewModel.swift b/apple/TableProMobile/TableProMobile/ViewModels/RowDetailViewModel.swift index 224de52ca0..bf6976f3df 100644 --- a/apple/TableProMobile/TableProMobile/ViewModels/RowDetailViewModel.swift +++ b/apple/TableProMobile/TableProMobile/ViewModels/RowDetailViewModel.swift @@ -156,6 +156,27 @@ final class RowDetailViewModel { } } + // Past this the row shows an excerpt, and the whole value opens on a screen of its own. + static let inlineByteLimit = Cell.previewByteLimit + + static func excerpt(of value: String) -> String { + Cell.preview(of: value, maxBytes: inlineByteLimit) + } + + func longValue(forRow rowIndex: Int, at index: Int) -> String? { + guard let value = completeValue(forRow: rowIndex, at: index), value.utf8.count > Self.inlineByteLimit else { + return nil + } + return value + } + + // Cut with no key to read it again by: a query result past its memory budget, or a table without a primary key. + func isShownInPart(forRow rowIndex: Int, at index: Int) -> Bool { + let rowCells = cells(at: rowIndex) + guard rowCells.indices.contains(index), case .truncatedText(_, _, .none) = rowCells[index] else { return false } + return true + } + // MariaDB's catalog names a JSON column longtext while its result column says JSON, so either type decides. func isJSON(at index: Int) -> Bool { guard index >= 0, index < columns.count else { return false } diff --git a/apple/TableProMobile/TableProMobile/Views/ConnectedView.swift b/apple/TableProMobile/TableProMobile/Views/ConnectedView.swift index 6b87d7f96f..71f3fc0270 100644 --- a/apple/TableProMobile/TableProMobile/Views/ConnectedView.swift +++ b/apple/TableProMobile/TableProMobile/Views/ConnectedView.swift @@ -88,7 +88,11 @@ struct ConnectedView: View { case .connected: resolved.loadHistory() hapticSuccess.toggle() - case .failed, .lost: + case .lost: + // A lost open comes back to the tabs through Reconnect, which never runs this task. + resolved.loadHistory() + hapticError.toggle() + case .failed: hapticError.toggle() case .notConnected, .connecting: break @@ -123,24 +127,32 @@ struct ConnectedView: View { case .notConnected: statusScreen { notConnectedView } case .failed(let error): - statusScreen { unavailableView(error, title: nil, systemImage: nil) } + statusScreen { unavailableView(error, title: nil, systemImage: nil) { retries += 1 } } case .lost(let error): let presentation = ConnectionStatusPresentation(coordinator.displayStatus) statusScreen { - unavailableView(error, title: presentation.title, systemImage: presentation.symbol?.systemName) + // The toolbar's Reconnect: a redial where the window was, not a fresh open of the saved database. + unavailableView(error, title: presentation.title, systemImage: presentation.symbol?.systemName) { + await coordinator.reconnect() + } } case .tabs: connectedContent(coordinator) } } - private func unavailableView(_ error: AppError, title: String?, systemImage: String?) -> some View { + private func unavailableView( + _ error: AppError, + title: String?, + systemImage: String?, + retry: @escaping () async -> Void + ) -> some View { ErrorView( error: error, title: title, systemImage: systemImage, retryTitle: String(localized: "Reconnect"), - onRetry: error.canRetry ? { retries += 1 } : nil, + onRetry: error.canRetry ? retry : nil, onEditConnection: connection.isSample ? nil : { presenter.presentConnectionEditor(for: connection.id) } ) } diff --git a/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift b/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift index b2b420f9ac..10c61e5e8e 100644 --- a/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift +++ b/apple/TableProMobile/TableProMobile/Views/QueryEditorView.swift @@ -44,7 +44,7 @@ struct QueryEditorView: View { @State private var showClearConfirmation = false @State private var showShareSheet = false @State private var shareText = "" - @State private var resultsExportError: ClipboardExporter.IncompleteValues? + @State private var resultsExportError: (any Error)? @State private var hapticSuccess = false @State private var hapticError = false @State private var containerSize: CGSize = .zero @@ -521,6 +521,10 @@ struct QueryEditorView: View { noticeBanner(message, systemImage: "exclamationmark.triangle.fill", tint: .orange) Divider() } + if includesTruncation, let notice = viewModel.longValuesNotice { + noticeBanner(notice, systemImage: "info.circle.fill", tint: .secondary) + Divider() + } if let summary = viewModel.scriptSummary { noticeBanner(summary, systemImage: "checkmark.circle.fill", tint: .green) Divider() @@ -562,7 +566,7 @@ struct QueryEditorView: View { keyColumn: RowKey.columnIndex(columns: columns, primaryKeys: []) ) { index, line in NavigationLink { - RowDetailView(columns: columns, rows: viewModel.window.rows, initialIndex: index) + resultRowDetail(columns: columns, initialIndex: index) } label: { line } @@ -584,11 +588,7 @@ struct QueryEditorView: View { let rowIndex = item.id let row = item.values NavigationLink { - RowDetailView( - columns: viewModel.columns, - rows: viewModel.window.rows, - initialIndex: rowIndex - ) + resultRowDetail(columns: viewModel.columns, initialIndex: rowIndex) } label: { resultRowCard(columns: viewModel.columns, row: row) } @@ -600,6 +600,17 @@ struct QueryEditorView: View { .listStyle(.plain) } + // No table, so nothing to edit; the engine's type and session decide how Copy Row quotes and escapes. + private func resultRowDetail(columns: [ColumnInfo], initialIndex: Int) -> some View { + RowDetailView( + columns: columns, + rows: viewModel.window.rows, + initialIndex: initialIndex, + session: session, + databaseType: databaseType + ) + } + private func resultRowCard(columns: [ColumnInfo], row: [String?]) -> some View { let budget = DuoLayoutResolver.previewFieldCount(for: duoWidthClass) let preview = Array(zip(columns, row).prefix(budget)) @@ -705,12 +716,8 @@ struct QueryEditorView: View { Section("Share Results") { ForEach(ExportFormat.offered(for: databaseType)) { format in Button { - do throws(ClipboardExporter.IncompleteValues) { - shareText = try ClipboardExporter.exportRows( - columns: viewModel.columns, rows: viewModel.window.rows.map(\.cells), - format: format, - databaseType: databaseType, driver: coordinator.session?.driver - ) + do { + shareText = try exportedResults(as: format) showShareSheet = true } catch { resultsExportError = error @@ -723,12 +730,8 @@ struct QueryEditorView: View { Section("Copy Results") { ForEach(ExportFormat.offered(for: databaseType)) { format in Button { - do throws(ClipboardExporter.IncompleteValues) { - let text = try ClipboardExporter.exportRows( - columns: viewModel.columns, rows: viewModel.window.rows.map(\.cells), - format: format, - databaseType: databaseType, driver: coordinator.session?.driver - ) + do { + let text = try exportedResults(as: format) ClipboardExporter.copyToClipboard(text) } catch { resultsExportError = error @@ -766,6 +769,17 @@ struct QueryEditorView: View { } } + private func exportedResults(as format: ExportFormat) throws -> String { + guard viewModel.resultsFitExport(availableBytes: MemoryPressureMonitor.shared.availableMemoryBytes()) else { + throw ClipboardExporter.TooLarge(source: .queryResult) + } + return try ClipboardExporter.exportRows( + columns: viewModel.columns, rows: viewModel.window.rows.map(\.cells), + format: format, + databaseType: databaseType, driver: coordinator.session?.driver + ) + } + // MARK: - Execution private func runAtCursor(dismissingKeyboard: Bool) { diff --git a/apple/TableProMobile/TableProMobile/Views/RowDetailView.swift b/apple/TableProMobile/TableProMobile/Views/RowDetailView.swift index e0572c3948..e9d4fa1614 100644 --- a/apple/TableProMobile/TableProMobile/Views/RowDetailView.swift +++ b/apple/TableProMobile/TableProMobile/Views/RowDetailView.swift @@ -15,6 +15,7 @@ struct RowDetailView: View { @State private var showSaveConfirmation = false @State private var jsonEditorField: IndexedRow? @State private var jsonViewerRoute: JSONValueViewer.Route? + @State private var fullValueRoute: FullValueRoute? private let coordinator: ConnectionCoordinator? init( @@ -89,6 +90,12 @@ struct RowDetailView: View { .navigationDestination(item: $jsonViewerRoute) { route in JSONValueViewer(route: route) } + .navigationDestination(item: $fullValueRoute) { route in + FullValueScreen( + title: viewModel.columns.indices.contains(route.column) ? viewModel.columns[route.column].name : "", + text: viewModel.completeValue(forRow: route.row, at: route.column) ?? "" + ) + } .toolbar { rowDetailToolbar } // An unsaved edit keeps this screen up over a lost session, so the state and Reconnect show here. .connectionStatusToolbar(for: connectionCoordinator) @@ -238,7 +245,11 @@ struct RowDetailView: View { } } else { let jsonText = viewModel.jsonText(forRow: rowIndex, at: index) - fieldContent(value: value, jsonPreview: viewModel.isJSON(at: index) ? jsonText : nil) + let longValue = viewModel.longValue(forRow: rowIndex, at: index) + fieldContent( + value: longValue.map { RowDetailViewModel.excerpt(of: $0) } ?? value, + jsonPreview: viewModel.isJSON(at: index) ? jsonText : nil + ) .contextMenu { if let value = viewModel.completeValue(forRow: rowIndex, at: index) { Button { @@ -259,6 +270,17 @@ struct RowDetailView: View { } label: { Label("View as JSON", systemImage: "curlybraces") } + } else if longValue != nil { + Button { + fullValueRoute = FullValueRoute(row: rowIndex, column: index) + } label: { + Label("View Full Value", systemImage: "doc.plaintext") + } + } + if viewModel.isShownInPart(forRow: rowIndex, at: index) { + Text("Shown in part. Run a query that returns fewer rows or columns to see it in full.") + .font(.footnote) + .foregroundStyle(.secondary) } if index < cells.count, cells[index].isLoadable, !viewModel.hasOverride(forRow: rowIndex, cellIndex: index) { @@ -424,4 +446,29 @@ struct RowDetailView: View { hapticError.toggle() } } + + // Names the cell rather than carrying its text, so the route stays cheap to compare. + private struct FullValueRoute: Hashable { + let row: Int + let column: Int + } +} + +private struct FullValueScreen: View { + let title: String + let text: String + + var body: some View { + JSONTextView(readOnlyText: text) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .navigationTitle(title) + .navigationBarTitleDisplayMode(.inline) + .toolbar { + ToolbarItem(placement: .primaryAction) { + Button("Copy", systemImage: "doc.on.doc") { + ClipboardExporter.copyToClipboard(text) + } + } + } + } } diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorBackgroundTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorBackgroundTests.swift index 3011a5fb98..7e6d2ecb4f 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorBackgroundTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorBackgroundTests.swift @@ -112,6 +112,29 @@ struct ConnectionCoordinatorBackgroundTests { ) } + // iPad: the transaction ran in one window, and the user comes back to the app in another. + @Test("A transaction lost in the background is reported once, in the window that reopens the file") + func reopenInAnotherWindowReportsOnce() async throws { + let left = try await connectedCoordinator(on: fileDriver(.explicitTransaction)) + #expect(left.transactionHold == .open) + + await releaseForTheBackground() + let reopened = fileDriver(.idle) + factory.queue(reopened) + let right = ConnectionCoordinator(connection: connection, appState: appState) + await right.connect() + await left.followLiveSession() + + #expect(right.session?.driver === reopened) + #expect(left.session?.driver === reopened) + #expect( + right.failureAlertMessage + == ConnectionCoordinator.lostTransactionMessage(.inBackground, connectionName: "Ledger") + ) + #expect(!left.showFailureAlert) + #expect(monitor.rollBackCount(for: connection.id) == 1) + } + @Test("A reopen that fails keeps the loss and reports it when a later reopen lands") func failedReopenReportsLater() async throws { let coordinator = try await connectedCoordinator(on: fileDriver(.explicitTransaction)) diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorReconnectTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorReconnectTests.swift index c1d210522b..cd77db9040 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorReconnectTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorReconnectTests.swift @@ -3,6 +3,7 @@ import TableProDatabase @testable import TableProMobile import TableProModels import TableProQuery +import TableProSQLGrammar import Testing @MainActor @@ -339,21 +340,22 @@ struct ConnectionCoordinatorReconnectTests { #expect(following.tablePlacement != chosen) } + // The connection reads the new schema's tables once, for both windows. @Test("A window following a schema switch keeps its old placement until the new schema's tables arrive") func followCommitsPlacementWithItsTables() async throws { let first = driver(tables: ["facts"]) first.currentSchema = "public" let (switching, following) = try await warehouseCoordinators(on: first) - await switching.switchSchema(to: "staging") let fetch = TestGate() first.scriptedTables = [TableInfo(name: "staging_facts")] first.beforeFetchTables = { await fetch.enter() } - let follow = Task { await following.followLiveSession() } + let switched = Task { await switching.switchSchema(to: "staging") } await fetch.waitUntilEntered() let duringFetch = (following.tablePlacement.schema, tableNames(following)) await fetch.open() - await follow.value + await switched.value + await following.followLiveSession() #expect(duringFetch.0 == "public") #expect(duringFetch.1 == ["facts"]) @@ -366,18 +368,18 @@ struct ConnectionCoordinatorReconnectTests { let first = driver(tables: ["facts"]) first.currentSchema = "public" let (switching, following) = try await warehouseCoordinators(on: first) - await switching.switchSchema(to: "staging") let fetch = TestGate() first.scriptedTables = [TableInfo(name: "staging_facts")] first.beforeFetchTables = { await fetch.enter() } - let follow = Task { await following.followLiveSession() } + let switched = Task { await switching.switchSchema(to: "staging") } await fetch.waitUntilEntered() following.isHeldByEditor = true first.beforeFetchTables = nil await fetch.open() - await follow.value + await switched.value + #expect(switching.tablePlacement.schema == "staging") #expect(following.tablePlacement.schema == "public") #expect(tableNames(following) == ["facts"]) @@ -387,31 +389,234 @@ struct ConnectionCoordinatorReconnectTests { #expect(tableNames(following) == ["staging_facts"]) } - @Test("A follow that another switch overtakes while its tables load lands on the newer placement") - func followOvertakenBySecondSwitchFollowsTheNewest() async throws { + @Test("A switch that another window's switch overtakes while its tables load never lands") + func overtakenSwitchNeverLands() async throws { let first = driver(tables: ["facts"]) first.currentSchema = "public" let (switching, following) = try await warehouseCoordinators(on: first) - await switching.switchSchema(to: "staging") let fetch = TestGate() first.scriptedTables = [TableInfo(name: "staging_facts")] first.beforeFetchTables = { await fetch.enter() } - let follow = Task { await following.followLiveSession() } + let overtaken = Task { await switching.switchSchema(to: "staging") } await fetch.waitUntilEntered() first.beforeFetchTables = nil - first.currentSchema = "archive" first.scriptedTables = [TableInfo(name: "archive_facts")] + await following.switchSchema(to: "archive") await fetch.open() - await follow.value - let afterOvertaken = following.tablePlacement.schema - await following.resumedFollow?.value + await overtaken.value - #expect(afterOvertaken == "public") + #expect(switching.tablePlacement.schema == "archive") #expect(following.tablePlacement.schema == "archive") #expect(tableNames(following) == ["archive_facts"]) } + @Test("A database switch asked during a check over an open transaction waits for it, so the loss is still reported") + func switchDuringACheckKeepsTheLossNotice() async throws { + let first = driver(tables: ["facts"], state: .explicitTransaction) + first.currentSchema = "public" + let (coordinator, _) = try await warehouseCoordinators(on: first) + #expect(coordinator.transactionHold == .open) + + first.pingError = MockDatabaseDriver.MockError.scripted + let redialed = driver(tables: ["facts"]) + redialed.currentSchema = "public" + let loading = TestGate() + redialed.beforeFetchTables = { await loading.enter() } + factory.queue(redialed) + let staging = driver(tables: ["staging_facts"]) + staging.currentSchema = "public" + factory.queue(staging) + + let check = Task { await coordinator.reconnectIfNeeded() } + await loading.waitUntilEntered() + let switched = Task { await coordinator.switchDatabase(to: "staging") } + await ObservedCondition.wait { coordinator.isSwitching } + redialed.beforeFetchTables = nil + await loading.open() + await check.value + await switched.value + + #expect( + coordinator.failureAlertMessage + == ConnectionCoordinator.lostTransactionMessage(.whileIdle, connectionName: "Warehouse") + ) + #expect(monitor.rollBackCount(for: coordinator.connection.id) == 1) + #expect(coordinator.activeDatabase == "staging") + #expect(coordinator.session?.driver === staging) + } + + @Test("A PostgreSQL switch asked while another window's runs lands after it, so the newest switch wins") + func overlappingPostgreSQLSwitchesKeepTheNewest() async throws { + let first = driver(tables: ["facts"]) + first.currentSchema = "public" + let (busy, other) = try await warehouseCoordinators(on: first) + let staging = driver(tables: ["staging_facts"]) + staging.currentSchema = "public" + let dialing = TestGate() + staging.beforeConnect = { await dialing.enter() } + factory.queue(staging) + let archive = driver(tables: ["archive_facts"]) + archive.currentSchema = "public" + factory.queue(archive) + + let busySwitch = Task { await busy.switchDatabase(to: "staging") } + await dialing.waitUntilEntered() + let otherSwitch = Task { await other.switchDatabase(to: "archive") } + await ObservedCondition.wait { other.isSwitching } + await dialing.open() + await busySwitch.value + await otherSwitch.value + + #expect(factory.dialed.suffix(2).map { $0.database } == ["staging", "archive"]) + #expect(busy.session?.driver === archive) + #expect(other.activeDatabase == "archive") + #expect(tableNames(busy) == ["archive_facts"]) + } + + @Test("A catalog reload held back while another window switches schema runs once the switch ends") + func catalogReloadHeldBackBySwitchRunsAfterIt() async throws { + let first = driver(tables: ["facts"]) + first.currentSchema = "public" + first.scriptedSchemas = ["public", "sales"] + let (posting, switching) = try await warehouseCoordinators(on: first) + let fetch = TestGate() + first.beforeFetchTables = { await fetch.enter() } + let switched = Task { await switching.switchSchema(to: "sales") } + await fetch.waitUntilEntered() + + first.scriptedSchemas = ["public", "sales", "reporting"] + posting.noteCatalogChanged(.schemas) + await posting.reloadChangedCatalog() + let whileSwitching = posting.schemas + await fetch.open() + await switched.value + + #expect(whileSwitching == ["public", "sales"]) + #expect(posting.schemas == ["public", "sales", "reporting"]) + } + + @Test("A follow asked while another window switches runs once the switch ends") + func followAskedDuringASwitchRunsAfterIt() async throws { + let first = driver(tables: ["facts"]) + first.currentSchema = "public" + let (switching, following) = try await warehouseCoordinators(on: first) + let fetch = TestGate() + first.beforeFetchTables = { await fetch.enter() } + let switched = Task { await switching.switchSchema(to: "staging") } + await fetch.waitUntilEntered() + + let replacement = driver(tables: ["archive_facts"]) + replacement.currentSchema = "public" + factory.queue(replacement) + var archive = following.connection + archive.database = "archive" + _ = try await appState.connectionManager.connect(archive) + await following.followLiveSession() + let whileSwitching = following.session?.driver + first.beforeFetchTables = nil + await fetch.open() + await switched.value + + #expect(whileSwitching === first) + #expect(following.session?.driver === replacement) + #expect(following.activeDatabase == "archive") + #expect(tableNames(following) == ["archive_facts"]) + } + + @Test("A follow held back for a window that has since pinned an edit reports to the window that freed the model") + func heldBackFollowSkipsAPinnedRequester() async throws { + let first = driver(tables: ["facts"], state: .explicitTransaction) + first.currentSchema = "public" + let (switching, editing) = try await warehouseCoordinators(on: first) + let fetch = TestGate() + first.beforeFetchTables = { await fetch.enter() } + let switched = Task { await switching.switchSchema(to: "staging") } + await fetch.waitUntilEntered() + + let replacement = driver(tables: ["archive_facts"]) + replacement.currentSchema = "public" + factory.queue(replacement) + var archive = editing.connection + archive.database = "archive" + _ = try await appState.connectionManager.connect(archive) + await ObservedCondition.wait { monitor.hold(for: archive.id) == nil } + await editing.followLiveSession() + editing.isHeldByEditor = true + first.beforeFetchTables = nil + await fetch.open() + await switched.value + + #expect(switching.session?.driver === replacement) + #expect( + switching.failureAlertMessage + == ConnectionCoordinator.lostTransactionMessage(.whileIdle, connectionName: "Warehouse") + ) + #expect(!editing.showFailureAlert) + #expect(editing.session?.driver === first) + } + + @Test("A database change posted during a PostgreSQL switch reloads the list once the switch lands") + func catalogPostDuringAPostgreSQLSwitchReloadsAfterIt() async throws { + let first = driver(tables: ["facts"]) + first.currentSchema = "public" + first.scriptedDatabases = ["analytics", "staging"] + let (posting, switching) = try await warehouseCoordinators(on: first) + let staging = driver(tables: ["staging_facts"]) + staging.currentSchema = "public" + staging.scriptedDatabases = ["analytics", "staging", "reporting"] + let dialing = TestGate() + staging.beforeConnect = { await dialing.enter() } + factory.queue(staging) + let switched = Task { await switching.switchDatabase(to: "staging") } + await dialing.waitUntilEntered() + + posting.noteCatalogChanged(.databases) + await posting.reloadChangedCatalog() + let whileSwitching = posting.databases + await dialing.open() + await switched.value + + #expect(whileSwitching == ["analytics", "staging"]) + #expect(posting.databases == ["analytics", "staging", "reporting"]) + } + + @Test("A table created while a SQL Server schema restore reads its list shows in both windows once the restore ends") + func catalogPostDuringASchemaRestoreReloadsAfterIt() async throws { + let first = driver(tables: ["orders"]) + first.currentSchema = "dbo" + first.scriptedSchemas = ["dbo", "sales"] + let ledger = DatabaseConnection(name: "Ledger", type: .mssql, host: "sql.example.com", port: 1_433, database: "app") + factory.queue(first) + _ = try await appState.connectionManager.connect(ledger) + let switching = ConnectionCoordinator(connection: ledger, appState: appState) + await switching.connect() + let creating = ConnectionCoordinator(connection: ledger, appState: appState) + await creating.connect() + await switching.switchSchema(to: "sales") + // The session lands on the new database's default schema, so the USE puts it back on sales. + first.currentSchema = "dbo" + let restoring = TestGate() + first.beforeFetchTables = { [weak first] in + if first?.currentSchema == "sales" { await restoring.enter() } + } + factory.opensCatalogs = true + factory.queue(driver(tables: ["orders", "invoices"])) + + let switched = Task { await switching.switchDatabase(to: "history") } + await restoring.waitUntilEntered() + creating.noteCatalogChanged(.tables) + await creating.reloadChangedCatalog() + let whileRestoring = tableNames(creating) + await restoring.open() + await switched.value + + #expect(whileRestoring == ["orders"]) + #expect(switching.activeSchema == "sales") + #expect(tableNames(switching) == ["orders", "invoices"]) + #expect(tableNames(creating) == ["orders", "invoices"]) + } + @Test("A window closed while its database switch waits on another window's dial leaves that dial's session alone") func closedWindowSwitchDoesNotTearDownTheSharedSession() async throws { let first = driver(tables: ["facts"]) @@ -463,6 +668,25 @@ struct ConnectionCoordinatorReconnectTests { #expect(following.schemas == ["dbo", "archive"]) } + // The list the switching window had belongs to the database it left. + @Test("A window that switches a SQL Server database lists the new database's schemas") + func switchReloadsSchemasForTheNewDatabase() async { + let first = driver(tables: ["orders"]) + first.currentSchema = "dbo" + first.scriptedSchemas = ["dbo", "sales"] + let ledger = DatabaseConnection(name: "Ledger", type: .mssql, host: "sql.example.com", port: 1_433, database: "app") + factory.queue(first) + let switching = ConnectionCoordinator(connection: ledger, appState: appState) + await switching.connect() + #expect(switching.schemas == ["dbo", "sales"]) + + first.scriptedSchemas = ["dbo", "archive"] + await switching.switchDatabase(to: "history") + + #expect(switching.activeDatabase == "history") + #expect(switching.schemas == ["dbo", "archive"]) + } + @Test("A PostgreSQL database switch in one window moves the other window onto the new session") func otherWindowFollowsAPostgreSQLSwitch() async throws { let first = driver(tables: ["facts"]) diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStatusTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStatusTests.swift index b89b21270a..f37c63eedb 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStatusTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorStatusTests.swift @@ -181,6 +181,45 @@ struct ConnectionCoordinatorStatusTests { #expect(isConnected(coordinator)) } + // The lost screen's Reconnect calls this too, so neither reopens the saved database and switches back. + @Test("Reconnect on a lost PostgreSQL window redials the database and schema it was on, in one dial") + func reconnectRedialsWhereTheWindowWas() async { + let warehouse = DatabaseConnection( + name: "Warehouse", + type: .postgresql, + host: "pg.example.com", + port: 5_432, + database: "analytics" + ) + let first = driver(tables: ["facts"]) + first.currentSchema = "public" + factory.queue(first) + let coordinator = ConnectionCoordinator(connection: warehouse, appState: appState) + await coordinator.connect() + let staging = driver(tables: ["staging_facts"]) + staging.currentSchema = "public" + factory.queue(staging) + await coordinator.switchDatabase(to: "staging") + await coordinator.switchSchema(to: "sales") + staging.pingError = MockDatabaseDriver.MockError.scripted + factory.queue(refusingDriver()) + await coordinator.reconnectIfNeeded() + #expect(coordinator.displayStatus == .lost(cause: ScriptedFailure.unreachable.localizedDescription)) + let dialsBefore = factory.dialed.count + + let recovered = driver(tables: ["sales_facts"]) + recovered.currentSchema = "public" + factory.queue(recovered) + await coordinator.reconnect() + + #expect(factory.dialed.count == dialsBefore + 1) + #expect(factory.dialed.last?.database == "staging") + #expect(recovered.currentSchema == "sales") + #expect(coordinator.session?.driver === recovered) + #expect(coordinator.activeDatabase == "staging") + #expect(coordinator.activeSchema == "sales") + } + @Test("Tracked work that fails on a lost session dials again") func failedWorkOnALostSessionRedials() async throws { let coordinator = try await lostCoordinator(on: driver(tables: ["orders"])) @@ -208,6 +247,25 @@ struct ConnectionCoordinatorStatusTests { #expect(appState.sessionTransactions.rollBackCount(for: connection.id) == 1) } + @Test("Cancel in a window opening a lost connection leaves another window on its lost screen") + func cancelledOpenKeepsAnotherWindowLost() async throws { + let lost = try await lostCoordinator(on: driver(tables: ["orders"])) + let opening = ConnectionCoordinator(connection: connection, appState: appState) + let dialed = driver(tables: ["orders"]) + let connecting = TestGate() + dialed.beforeConnect = { await connecting.enter() } + factory.queue(dialed) + let open = Task { await opening.connect() } + await connecting.waitUntilEntered() + + opening.cancelConnect() + await connecting.open() + await open.value + + #expect(opening.displayStatus == .notConnected) + #expect(lost.displayStatus == .lost(cause: ScriptedFailure.unreachable.localizedDescription)) + } + @Test("A window whose redial failed takes the session another window dialed") func lostWindowFollowsAnotherWindowsDial() async throws { let lost = try await lostCoordinator(on: driver(tables: ["orders"])) @@ -416,9 +474,11 @@ struct ConnectionCoordinatorStatusTests { let saved = await viewModel.saveChanges() #expect(coordinator.session == nil) + #expect(coordinator.displayStatus == .lost(cause: ScriptedFailure.unreachable.localizedDescription)) #expect(coordinator.activeDatabase == "app_staging") #expect(coordinator.tables.map { $0.name } == ["staging_orders"]) #expect(!saved) + #expect(viewModel.operationError?.title == ConnectionStatusPresentation(.lost(cause: nil)).title) #expect(!ranUpdate(second)) coordinator.isHeldByEditor = false @@ -430,23 +490,33 @@ struct ConnectionCoordinatorStatusTests { #expect(isConnected(coordinator)) } + // The other window's open reads the saved database's tables once, for both windows. @Test("An unsaved edit that starts while a lost window loads another database's tables keeps the window where it was") func editArrivingDuringTheFetchPinsTheWindow() async throws { let coordinator = try await lostOnStaging() - let second = await reconnectAnotherWindow() + let second = driver(tables: ["orders"]) let loading = TestGate() second.beforeFetchTables = { await loading.enter() } + factory.queue(second) + let other = ConnectionCoordinator(connection: connection, appState: appState) - let check = Task { await coordinator.reconnectIfNeeded() } + let opening = Task { await other.connect() } await loading.waitUntilEntered() coordinator.isHeldByEditor = true second.beforeFetchTables = nil await loading.open() - await check.value + await opening.value + #expect(appState.connectionManager.session(for: connection.id)?.activeDatabase == "app_prod") #expect(coordinator.session == nil) + #expect(coordinator.displayStatus == .lost(cause: ScriptedFailure.unreachable.localizedDescription)) #expect(coordinator.activeDatabase == "app_staging") #expect(coordinator.tables.map { $0.name } == ["staging_orders"]) + guard case .failure(let refusal) = coordinator.writableSession else { + Issue.record("Expected the pinned window to refuse a write") + return + } + #expect(refusal.title == ConnectionStatusPresentation(.lost(cause: nil)).title) coordinator.isHeldByEditor = false await coordinator.resumedFollow?.value diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorSwitchTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorSwitchTests.swift index 66a22a8243..b449b2e087 100644 --- a/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorSwitchTests.swift +++ b/apple/TableProMobile/TableProMobileTests/ConnectionCoordinatorSwitchTests.swift @@ -155,6 +155,34 @@ struct ConnectionCoordinatorSwitchTests { #expect(coordinator.displayStatus == .lost(cause: ScriptedFailure.unreachable.localizedDescription)) } + @Test("An open whose saved database cannot be restored, and whose way back fails too, is lost and redials on the next trigger") + func failedRestoreDuringAnOpenIsLost() async { + let lastDB = ConnectionDefaultsKey.lastDB.name(for: warehouse.id) + UserDefaults.standard.set("staging", forKey: lastDB) + defer { UserDefaults.standard.removeObject(forKey: lastDB) } + let first = driver(tables: ["facts"]) + first.scriptedDatabases = ["analytics", "staging"] + factory.queue(first) + factory.queue(refusingDriver()) + factory.queue(refusingDriver()) + let coordinator = ConnectionCoordinator(connection: warehouse, appState: appState) + coordinator.restorePersistedState() + + await coordinator.connect() + + #expect(factory.dialed.suffix(2).map { $0.database } == ["staging", "analytics"]) + #expect(coordinator.session == nil) + #expect(coordinator.displayStatus == .lost(cause: ScriptedFailure.unreachable.localizedDescription)) + + let recovered = driver(tables: ["facts"]) + factory.queue(recovered) + await coordinator.reconnectIfNeeded() + + #expect(factory.dialed.last?.database == "analytics") + #expect(coordinator.session?.driver === recovered) + #expect(coordinator.displayStatus == .connected) + } + @Test("A refresh that fails on a dead session redials and shows the new list without an alert") func refreshOnADeadSessionRedials() async throws { let first = driver(tables: ["orders"]) diff --git a/apple/TableProMobile/TableProMobileTests/ConnectionSessionStateTests.swift b/apple/TableProMobile/TableProMobileTests/ConnectionSessionStateTests.swift new file mode 100644 index 0000000000..a1539cdef1 --- /dev/null +++ b/apple/TableProMobile/TableProMobileTests/ConnectionSessionStateTests.swift @@ -0,0 +1,203 @@ +import Foundation +import TableProDatabase +@testable import TableProMobile +import Testing + +@MainActor +struct ConnectionSessionStateTests { + private let failure = AppError( + category: .network, + title: "Connection Failed", + message: "The server closed the connection.", + recovery: nil, + underlying: nil + ) + + private func session(on driver: MockDatabaseDriver) -> ConnectionSession { + ConnectionSession(connectionId: UUID(), driver: driver, activeDatabase: "app") + } + + private func isConnected(_ state: ConnectionSessionState?, on driver: MockDatabaseDriver) -> Bool { + guard case .connected(let landed)? = state else { return false } + return landed.driver === driver + } + + private func isLost(_ state: ConnectionSessionState?) -> Bool { + guard case .lost? = state else { return false } + return true + } + + private func isNotConnected(_ state: ConnectionSessionState?) -> Bool { + guard case .notConnected? = state else { return false } + return true + } + + private func showsConnected(_ state: ConnectionSessionState?) -> Bool { + guard case .connected? = state?.phase else { return false } + return true + } + + private func showsLost(_ state: ConnectionSessionState?) -> Bool { + guard case .lost? = state?.phase else { return false } + return true + } + + @Test("An open lands only with a session, so a connected state always has one") + func openLandsOnlyWithASession() { + let open = ConnectionAttempt(.open) + let driver = MockDatabaseDriver() + let started = ConnectionSessionState.notConnected.applying(.open(open)) + + #expect(started?.isOpening == true) + #expect(started?.session == nil) + #expect(started?.applying(.opened(open)) == nil) + + let loading = started?.applying(.adopt(session(on: driver), by: open)) + #expect(loading?.isOpening == true) + #expect(loading?.session?.driver === driver) + #expect(isConnected(loading?.applying(.opened(open)), on: driver)) + } + + @Test("A database restore that fails both ways inside an open ends it lost, and the open can no longer land") + func failedRestoreEndsTheOpenLost() { + let open = ConnectionAttempt(.open) + let loading = ConnectionSessionState.connecting(open, provisional: session(on: MockDatabaseDriver())) + + let tornDown = loading.applying(.teardown(open)) + #expect(tornDown?.isOpening == true) + #expect(tornDown?.session == nil) + + let lost = tornDown?.applying(.lose(failure, open)) + #expect(isLost(lost)) + #expect(lost?.applying(.opened(open)) == nil) + #expect(lost?.applying(.adopt(session(on: MockDatabaseDriver()), by: open)) == nil) + } + + @Test("A step of an attempt the state no longer holds is refused") + func staleStepsAreRefused() { + let driver = MockDatabaseDriver() + let first = ConnectionAttempt(.open) + let abandoned = ConnectionSessionState.connecting(first, provisional: nil).applying(.abandon(first)) + + #expect(isNotConnected(abandoned)) + #expect(abandoned?.applying(.adopt(session(on: driver), by: first)) == nil) + #expect(abandoned?.applying(.fail(failure, first)) == nil) + #expect(ConnectionSessionState.connecting(first, provisional: nil).applying(.open(ConnectionAttempt(.open))) == nil) + + let check = ConnectionAttempt(.redial) + let checking = ConnectionSessionState.connected(session(on: driver)).applying(.check(check)) + let reopened = checking?.applying(.open(ConnectionAttempt(.open))) + #expect(reopened?.isOpening == true) + #expect(reopened?.applying(.dial(check)) == nil) + #expect(reopened?.applying(.lose(failure, check)) == nil) + #expect(reopened?.applying(.adopt(session(on: driver), by: check)) == nil) + } + + @Test("A check pings in place, and only its redial reads as reconnecting") + func checkReconnectsOnlyOnceItDials() { + let held = MockDatabaseDriver() + let check = ConnectionAttempt(.redial) + let checking = ConnectionSessionState.connected(session(on: held)).applying(.check(check)) + + #expect(checking?.check === check) + #expect(checking?.isReconnecting == false) + #expect(checking?.session?.driver === held) + #expect(showsConnected(checking)) + + let dialing = checking?.applying(.dial(check)) + #expect(dialing?.isReconnecting == true) + #expect(dialing?.session?.driver === held) + #expect(isLost(dialing?.applying(.lose(failure, check)))) + + let redialed = MockDatabaseDriver() + let retry = ConnectionAttempt(.redial) + let lostChecking = ConnectionSessionState.lost(failure).applying(.check(retry)) + #expect(showsLost(lostChecking)) + #expect(lostChecking?.session == nil) + let lostDialing = lostChecking?.applying(.dial(retry)) + #expect(lostDialing?.isReconnecting == true) + #expect(isConnected(lostDialing?.applying(.adopt(session(on: redialed), by: retry)), on: redialed)) + } + + @Test("A check starts only over a connected or a lost session") + func checkNeedsASessionOrALoss() { + let driver = MockDatabaseDriver() + let switching = ConnectionAttempt(.switchDatabase) + let states: [ConnectionSessionState] = [ + .notConnected, + .failed(failure), + .connecting(ConnectionAttempt(.open), provisional: session(on: driver)), + .reconnecting(switching, over: .connected(session(on: driver))) + ] + for state in states { + #expect(state.applying(.check(ConnectionAttempt(.redial))) == nil) + } + } + + @Test("A session another window moved lands under a running check without ending it") + func followLandsUnderACheck() { + let check = ConnectionAttempt(.redial) + let moved = MockDatabaseDriver() + let checking = ConnectionSessionState.checking(check, over: .connected(session(on: MockDatabaseDriver()))) + + let followed = checking.applying(.adopt(session(on: moved), by: nil)) + #expect(followed?.check === check) + #expect(followed?.session?.driver === moved) + #expect(isConnected(followed?.applying(.settle(check)), on: moved)) + } + + @Test("A PostgreSQL database switch shows no session until the new one lands, and is not a reconnect") + func databaseSwitchHasNoSessionUntilItLands() { + let switching = ConnectionAttempt(.switchDatabase) + let staging = MockDatabaseDriver() + let torn = ConnectionSessionState.connected(session(on: MockDatabaseDriver())).applying(.switchDatabase(switching)) + + #expect(torn?.session == nil) + #expect(torn?.isReconnecting == false) + #expect(showsConnected(torn)) + #expect(torn?.applying(.adopt(session(on: staging), by: nil)) == nil) + #expect(isConnected(torn?.applying(.adopt(session(on: staging), by: switching)), on: staging)) + #expect(isLost(torn?.applying(.lose(failure, switching)))) + } + + @Test("A database switch waits for a running check, and may start over a loss") + func databaseSwitchWaitsForACheck() { + let held = session(on: MockDatabaseDriver()) + let check = ConnectionAttempt(.redial) + let checking = ConnectionSessionState.checking(check, over: .connected(held)) + let redialing = ConnectionSessionState.reconnecting(check, over: .connected(held)) + + #expect(checking.applying(.switchDatabase(ConnectionAttempt(.switchDatabase))) == nil) + #expect(redialing.applying(.switchDatabase(ConnectionAttempt(.switchDatabase))) == nil) + + let overLoss = ConnectionSessionState.lost(failure).applying(.switchDatabase(ConnectionAttempt(.switchDatabase))) + #expect(overLoss?.session == nil) + #expect(showsLost(overLoss)) + #expect(overLoss?.isReconnecting == false) + } + + @Test("An open every window left goes back to the loss it started from, and a reset ends neutral") + func abandonedOpenKeepsItsLoss() { + let fromLoss = ConnectionAttempt(.open, origin: .lost(failure)) + let reopening = ConnectionSessionState.connecting(fromLoss, provisional: session(on: MockDatabaseDriver())) + let fresh = ConnectionAttempt(.open) + + #expect(isLost(reopening.applying(.abandon(fromLoss)))) + #expect(isNotConnected(ConnectionSessionState.connecting(fresh, provisional: nil).applying(.abandon(fresh)))) + #expect(reopening.applying(.abandon(fresh)) == nil) + #expect(isNotConnected(reopening.applying(.cancel))) + } + + @Test("At rest, an open shows the loss it started from and a switch the session it started over") + func restingStateSkipsWhatIsInFlight() { + let held = MockDatabaseDriver() + let reopening = ConnectionSessionState.connecting(ConnectionAttempt(.open, origin: .lost(failure)), provisional: nil) + let switching = ConnectionSessionState.reconnecting( + ConnectionAttempt(.switchDatabase), over: .connected(session(on: held)) + ) + + #expect(isLost(reopening.atRest)) + #expect(isConnected(switching.atRest, on: held)) + #expect(isNotConnected(ConnectionSessionState.connecting(ConnectionAttempt(.open), provisional: nil).atRest)) + } +} diff --git a/apple/TableProMobile/TableProMobileTests/DataBrowserViewModelTests.swift b/apple/TableProMobile/TableProMobileTests/DataBrowserViewModelTests.swift index 9de74f2a30..8c3f42995f 100644 --- a/apple/TableProMobile/TableProMobileTests/DataBrowserViewModelTests.swift +++ b/apple/TableProMobile/TableProMobileTests/DataBrowserViewModelTests.swift @@ -1021,6 +1021,37 @@ struct DataBrowserViewModelTests { #expect(vm.isLoading == false) #expect(driver.executedQueries.isEmpty) } + + @Test("a long value in a key's contents stays whole within the budget, and the grid shows a preview of it") + func longKeyValueStaysWhole() async { + let long = String(repeating: "v", count: 10_000) + let driver = MockKeyContentsDriver() + driver.scriptedPages = [ + .success(KeyContentsPage( + result: QueryResult( + columns: [ColumnInfo(name: "value", typeName: "string", ordinalPosition: 0)], + rows: [[long]], + rowsAffected: 0, + executionTime: 0 + ), + totalCount: 1 + )) + ] + let vm = DataBrowserViewModel(textBudgetBytes: { 1_000_000 }) + let session = ConnectionSession(connectionId: UUID(), driver: driver, activeDatabase: "db0", tables: []) + vm.attach(session: session, table: TableInfo(name: "profile"), databaseType: .redis, host: "localhost") + + await vm.load(isInitial: true) + + guard case .text(let kept)? = vm.cells(at: 0).first else { + Issue.record("the value should be kept whole, got \(vm.cells(at: 0))") + return + } + #expect(kept == long) + #expect(DataBrowserViewModel.canComplete(vm.cells(at: 0))) + let shown = vm.legacyRows.first?.first ?? nil + #expect(shown?.utf8.count == Cell.previewByteLimit) + } } private actor ExecuteGate { diff --git a/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBDriverTests.swift b/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBDriverTests.swift index 4b674276ca..92f843a957 100644 --- a/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBDriverTests.swift +++ b/apple/TableProMobile/TableProMobileTests/Drivers/DuckDBDriverTests.swift @@ -95,7 +95,15 @@ final class DuckDBDriverTests: XCTestCase { XCTAssertEqual(streamed.count, 1) XCTAssertEqual(materialized.rows.count, 1) - XCTAssertEqual(streamed[0], materialized.rows[0], "streamed output must match the materialized path exactly") + // A streamed BLOB is a binary cell, as on every other engine; `execute` still carries its base64. + let payload = 10 + XCTAssertEqual(streamed[0][payload], Cell.binary(byteCount: 3, ref: nil).displayString) + XCTAssertEqual(materialized.rows[0][payload], Data("abc".utf8).base64EncodedString()) + var streamedRest = streamed[0] + var materializedRest = materialized.rows[0] + streamedRest.remove(at: payload) + materializedRest.remove(at: payload) + XCTAssertEqual(streamedRest, materializedRest, "streamed output must match the materialized path for every other type") } func testStreamingMultiRowOrder() async throws { @@ -323,6 +331,30 @@ final class DuckDBDriverTests: XCTestCase { } } +struct DuckDBBlobStreamingTests { + @Test("A streamed BLOB is a binary cell of its byte count, with the ref every other column gets") + func blobStreamsAsBinary() async throws { + let driver = DuckDBDriver(source: .inMemory) + try await driver.connect() + _ = try await driver.execute(query: "CREATE TABLE files (id INTEGER PRIMARY KEY, payload BLOB)") + _ = try await driver.execute(query: "INSERT INTO files VALUES (1, 'abc'::BLOB)") + let options = StreamOptions(maxRows: 10, lazyContext: LazyContext(table: "files", primaryKeyColumns: ["id"])) + + var cells: [Cell] = [] + for try await element in driver.executeStreaming(query: "SELECT id, payload FROM files", options: options) { + if case .row(let row) = element { cells = row.cells } + } + try await driver.disconnect() + + guard cells.count == 2, case .binary(let byteCount, let ref) = cells[1] else { + Issue.record("the BLOB should stream as a binary cell, got \(cells)") + return + } + #expect(byteCount == 3) + #expect(ref?.primaryKey == [PrimaryKeyComponent(column: "id", value: "1")]) + } +} + struct DuckDBDriverTransactionStateTests { private func connectedDriver() async throws -> DuckDBDriver { let driver = DuckDBDriver(source: .inMemory) diff --git a/apple/TableProMobile/TableProMobileTests/QueryEditorViewModelTests.swift b/apple/TableProMobile/TableProMobileTests/QueryEditorViewModelTests.swift index 4a135755a6..ab17c1a25a 100644 --- a/apple/TableProMobile/TableProMobileTests/QueryEditorViewModelTests.swift +++ b/apple/TableProMobile/TableProMobileTests/QueryEditorViewModelTests.swift @@ -401,6 +401,94 @@ struct QueryEditorViewModelTests { ]) #expect(vm.requestCount == 3) } + + private static let longValue = String(repeating: "x", count: 10_000) + + private func longValueResult(rows: Int) -> QueryResult { + QueryResult( + columns: [ColumnInfo(name: "body", typeName: "TEXT", ordinalPosition: 0)], + rows: Array(repeating: [Self.longValue], count: rows), + rowsAffected: 0, + executionTime: 0 + ) + } + + @Test("a long value comes back whole, and one past the budget keeps a preview and says so") + func longValuesStayWholeWithinTheBudget() async { + let driver = MockDatabaseDriver() + driver.scriptedExecuteResults = [.success(longValueResult(rows: 2))] + + let vm = QueryEditorViewModel(windowCapacity: 100, textBudgetBytes: { 15_000 }) + await vm.run(driver: driver, script: .statements("SELECT body FROM notes")) + + let cells = vm.window.rows.compactMap { $0.cells.first } + guard cells.count == 2, case .text(let kept) = cells[0], case .truncatedText(_, let totalBytes, let ref) = cells[1] else { + Issue.record("expected the first value whole and the second cut, got \(cells)") + return + } + #expect(kept == Self.longValue) + #expect(totalBytes == 10_000) + #expect(ref == nil) + #expect(vm.longValuesNotice == String(localized: "Some long values are shown in part to stay within memory.")) + let shown = vm.legacyRows.first?.first ?? nil + #expect(shown?.utf8.count == Cell.previewByteLimit) + } + + @Test("each request of a script starts with a budget of its own") + func eachRequestGetsAFreshBudget() async { + let driver = MockDatabaseDriver() + driver.scriptedExecuteResults = [.success(longValueResult(rows: 1)), .success(longValueResult(rows: 1))] + + let vm = QueryEditorViewModel(windowCapacity: 100, textBudgetBytes: { 15_000 }) + await vm.run(driver: driver, script: .statements("SELECT body FROM a", "SELECT body FROM b")) + + #expect(driver.executedQueries == ["SELECT body FROM a", "SELECT body FROM b"]) + guard case .text(let kept)? = vm.window.rows.first?.cells.first else { + Issue.record("the second request's value should be whole, got \(vm.window.rows)") + return + } + #expect(kept == Self.longValue) + #expect(vm.longValuesNotice == nil) + } + + @Test("a new run clears the notice of the one before") + func newRunClearsTheNotice() async { + let driver = MockDatabaseDriver() + driver.scriptedExecuteResults = [.success(longValueResult(rows: 2)), .success(longValueResult(rows: 1))] + let vm = QueryEditorViewModel(windowCapacity: 100, textBudgetBytes: { 15_000 }) + + await vm.run(driver: driver, script: .statements("SELECT body FROM notes")) + #expect(vm.longValuesNotice != nil) + await vm.run(driver: driver, script: .statements("SELECT body FROM notes LIMIT 1")) + + #expect(vm.longValuesNotice == nil) + } + + @Test("Copy and Share Results ask for the memory the data browser's export asks for") + func resultsExportChecksMemory() async { + let driver = MockDatabaseDriver() + driver.scriptedExecuteResults = [.success(longValueResult(rows: 1))] + let vm = QueryEditorViewModel(windowCapacity: 100, textBudgetBytes: { 15_000 }) + await vm.run(driver: driver, script: .statements("SELECT body FROM notes")) + let margin = 64 * 1_024 * 1_024 + + #expect(vm.resultsFitExport(availableBytes: 0)) + #expect(vm.resultsFitExport(availableBytes: margin + 40_001)) + #expect(!vm.resultsFitExport(availableBytes: margin + 40_000)) + } + + @Test("the budget is a quarter of the headroom past 64 MB, at most 64 MB, and 64 MB when the headroom is unknown") + func textBudgetFollowsTheHeadroom() { + let megabyte = 1_024 * 1_024 + #expect(MemoryPressureMonitor.textBudgetBytes(availableBytes: 0) == 64 * megabyte) + #expect(MemoryPressureMonitor.textBudgetBytes(availableBytes: 32 * megabyte) == 0) + #expect(MemoryPressureMonitor.textBudgetBytes(availableBytes: 100 * megabyte) == 6 * megabyte) + // A result that spent its whole budget still exports, with the kept text off the headroom. + let available = 200 * megabyte + let spent = MemoryPressureMonitor.textBudgetBytes(availableBytes: available) + #expect(DataBrowserViewModel.exportFits(pendingBytes: spent, availableBytes: available - spent)) + #expect(MemoryPressureMonitor.textBudgetBytes(availableBytes: 4_096 * megabyte) == 64 * megabyte) + } } private extension QueryScript { diff --git a/apple/TableProMobile/TableProMobileTests/RowDetailViewModelTests.swift b/apple/TableProMobile/TableProMobileTests/RowDetailViewModelTests.swift index 2f0824d27a..3e8167407d 100644 --- a/apple/TableProMobile/TableProMobileTests/RowDetailViewModelTests.swift +++ b/apple/TableProMobile/TableProMobileTests/RowDetailViewModelTests.swift @@ -1125,4 +1125,59 @@ struct RowDetailViewModelTests { #expect(vm.exportCurrentRow(as: .csv)?.contains("abc and the rest") == true) #expect(vm.operationError == nil) } + + @Test("A query result's row copies as an INSERT in its engine's quoting, not SQLite's") + func queryResultRowExportsInTheEnginesQuoting() throws { + let vm = RowDetailViewModel( + columns: makeColumns(), rows: makeRows(), initialIndex: 0, + session: makeSession(driver: MockDatabaseDriver()), databaseType: .mysql + ) + + let insert = try #require(vm.exportCurrentRow(as: .sqlInsert)) + + #expect(insert.hasPrefix("INSERT INTO `table` (`id`, `name`) VALUES")) + #expect(!vm.canEdit) + } + + @Test("A long value shows an excerpt, while Copy Value and the full view get all of it") + func longValueShowsAnExcerpt() { + let long = String(repeating: "y", count: 10_000) + let vm = RowDetailViewModel( + columns: makeColumns(), rows: [Row(cells: [.text("1"), .text(long)])], initialIndex: 0 + ) + + #expect(vm.longValue(forRow: 0, at: 1) == long) + #expect(vm.completeValue(forRow: 0, at: 1) == long) + #expect(vm.longValue(forRow: 0, at: 0) == nil) + #expect(RowDetailViewModel.excerpt(of: long).utf8.count == RowDetailViewModel.inlineByteLimit) + #expect(!vm.isShownInPart(forRow: 0, at: 1)) + } + + @Test("A table value loaded in full is long enough to need the full view too") + func loadedValueShowsAnExcerpt() async { + let long = String(repeating: "z", count: 10_000) + let vm = makeViewModel( + lastCell: .truncatedText(prefix: "zzz", totalBytes: 10_000, ref: bodyRef(id: "1")), + loadFullValue: { _ in long } + ) + #expect(vm.longValue(forRow: 0, at: 2) == nil) + + await vm.loadFullValue(ref: bodyRef(id: "1"), forRow: 0, cellIndex: 2) + + #expect(vm.longValue(forRow: 0, at: 2) == long) + } + + @Test("Only a value cut with no key to load it by is shown in part") + func cutValueWithoutRefIsShownInPart() { + let vm = RowDetailViewModel( + columns: makeColumns(), + rows: [Row(cells: [.text("1"), .truncatedText(prefix: "a", totalBytes: 5_000, ref: nil)])], + initialIndex: 0 + ) + let loadable = makeViewModel(lastCell: .truncatedText(prefix: "a", totalBytes: 5_000, ref: bodyRef(id: "1"))) + + #expect(vm.isShownInPart(forRow: 0, at: 1)) + #expect(vm.longValue(forRow: 0, at: 1) == nil) + #expect(!loadable.isShownInPart(forRow: 0, at: 2)) + } } diff --git a/apple/TableProMobile/TableProMobileTests/StreamingResultBufferTests.swift b/apple/TableProMobile/TableProMobileTests/StreamingResultBufferTests.swift index 141c959b6c..4e46275d7d 100644 --- a/apple/TableProMobile/TableProMobileTests/StreamingResultBufferTests.swift +++ b/apple/TableProMobile/TableProMobileTests/StreamingResultBufferTests.swift @@ -36,6 +36,23 @@ struct StreamingResultBufferTests { #expect(buffer.window.count == 4) } + @Test("the grid gets a preview of a long value the row keeps whole") + func gridTextOfKeptValueIsCapped() throws { + let buffer = StreamingResultBuffer(capacity: 10) + let long = String(repeating: "界", count: 5_000) + buffer.apply(.row(Row(cells: [.text(long)]))) + buffer.flush() + + let shown = try #require(buffer.legacyRows.first?.first ?? nil) + #expect(shown.utf8.count <= Cell.previewByteLimit) + #expect(long.hasPrefix(shown)) + guard case .text(let kept)? = buffer.window.rows.first?.cells.first else { + Issue.record("the row should keep the whole value") + return + } + #expect(kept == long) + } + @Test("reset clears all state") func resetClears() { let buffer = StreamingResultBuffer(capacity: 100) diff --git a/apple/changelog.d/4201.changed.md b/apple/changelog.d/4201.changed.md new file mode 100644 index 0000000000..8367d4080e --- /dev/null +++ b/apple/changelog.d/4201.changed.md @@ -0,0 +1 @@ +On iPad, every window of a connection now shows Reconnecting or Lost at the same time. diff --git a/apple/changelog.d/4201.fixed.1.md b/apple/changelog.d/4201.fixed.1.md new file mode 100644 index 0000000000..bfdd221262 --- /dev/null +++ b/apple/changelog.d/4201.fixed.1.md @@ -0,0 +1 @@ +A DuckDB BLOB on iOS showed as base64 text, and saving an edit stored that text as the blob's bytes. diff --git a/apple/changelog.d/4201.fixed.2.md b/apple/changelog.d/4201.fixed.2.md new file mode 100644 index 0000000000..64e63ef093 --- /dev/null +++ b/apple/changelog.d/4201.fixed.2.md @@ -0,0 +1 @@ +Oracle BINARY_DOUBLE, BINARY_FLOAT and BINARY_INTEGER values on iOS showed as BLOBs and could not be copied or searched. diff --git a/apple/changelog.d/4201.fixed.3.md b/apple/changelog.d/4201.fixed.3.md new file mode 100644 index 0000000000..e01d17ded8 --- /dev/null +++ b/apple/changelog.d/4201.fixed.3.md @@ -0,0 +1 @@ +A Redis value over 4 KB on iOS could never be read in full. diff --git a/apple/changelog.d/4201.fixed.4.md b/apple/changelog.d/4201.fixed.4.md new file mode 100644 index 0000000000..19cd0c3680 --- /dev/null +++ b/apple/changelog.d/4201.fixed.4.md @@ -0,0 +1 @@ +Copy Row as SQL INSERT from an iOS query result quoted names the SQLite way on every engine. diff --git a/apple/changelog.d/4201.fixed.5.md b/apple/changelog.d/4201.fixed.5.md new file mode 100644 index 0000000000..e161edca9b --- /dev/null +++ b/apple/changelog.d/4201.fixed.5.md @@ -0,0 +1 @@ +An iOS connection could show Connected with no session after a failed switch back to its last database. diff --git a/apple/changelog.d/4201.fixed.6.md b/apple/changelog.d/4201.fixed.6.md new file mode 100644 index 0000000000..2ee86b2fa6 --- /dev/null +++ b/apple/changelog.d/4201.fixed.6.md @@ -0,0 +1 @@ +After a database switch on SQL Server on iOS, the schema list still showed the old database's schemas. diff --git a/apple/changelog.d/4201.fixed.7.md b/apple/changelog.d/4201.fixed.7.md new file mode 100644 index 0000000000..302f601988 --- /dev/null +++ b/apple/changelog.d/4201.fixed.7.md @@ -0,0 +1 @@ +On iPad, Reconnect on the lost-connection screen moved the connection's other windows to the saved database and back. diff --git a/apple/changelog.d/4201.fixed.8.md b/apple/changelog.d/4201.fixed.8.md new file mode 100644 index 0000000000..6458673e7f --- /dev/null +++ b/apple/changelog.d/4201.fixed.8.md @@ -0,0 +1 @@ +On iPad, a transaction rolled back in the background went unreported when another window reopened the connection first. diff --git a/apple/changelog.d/4201.fixed.md b/apple/changelog.d/4201.fixed.md new file mode 100644 index 0000000000..f2edc063f6 --- /dev/null +++ b/apple/changelog.d/4201.fixed.md @@ -0,0 +1 @@ +Text over 4 KB in an iOS query result could not be read in full or copied. diff --git a/core/ARCHITECTURE.md b/core/ARCHITECTURE.md index ef8f437f68..8a4fc27513 100644 --- a/core/ARCHITECTURE.md +++ b/core/ARCHITECTURE.md @@ -116,6 +116,7 @@ Each boundary error has a stable `code()`, such as `driver.timeout` or `ssh.host - The library owns one tokio runtime. Each async export runs on it and is aborted when the foreign side drops the call; a `CancelToken` stops it sooner, since UniFFI cannot cancel. - A result set takes rows as the driver reads them. Each column keeps a kind byte, a raw `i64` and the exact text `tablepro_core::export::value_text` writes, or a binary value's bytes, in chunks of 65,536 rows. A window only cuts and copies. - A `CoreError` carries the `code()` of each error from the outermost in; the innermost decides. +- A secret crosses as the `SecretString` custom type, a `String` to the app, so Rust holds it in `secrecy` from the lift and wipes it on drop. Still plain: the serialized buffer UniFFI frees for a record, enum, option or callback answer that carries one; for a secret going to the app (`HostSecrets::store` and `store_item`, `ParsedConnectionUrl.password`), the `String` lowered and its buffer; every string the app holds, which the generated C# records also print in full. The blob Credential Manager hands back is wiped before `CredFree`. How the bindings are generated, and where they live: `windows/AGENTS.md`, Bindings, for C#, and `apple/AGENTS.md`, Repository map, for Swift. @@ -125,7 +126,7 @@ How an app reaches a database: the contract in `tablepro-core`, a transport that ### The contract -`DatabaseDriver` describes an engine before any connection exists, through `DriverCapabilities`, and opens a `Connection`. The `Connection` serves the workspace from a pool; every call takes `CallOptions` (a deadline and a cancel token) and runs on a task the connection owns (`OwnedCalls`), so a stop reaches the server instead of leaving the statement running. Saves report what they left in the database through a `WriteLedger`. Each editor tab gets an `EditorSession` of its own that streams results into a `ResultSink`. [docs/adding-drivers.md](../linux/docs/adding-drivers.md) is the full contract, [ADR 0016](../linux/docs/decisions/0016-driver-contract-v2.md) the reasons for it, and the engine pages say how each driver meets it: [PostgreSQL](../linux/docs/drivers/postgresql.md), [MySQL and MariaDB](../linux/docs/drivers/mysql.md), [SQLite](../linux/docs/drivers/sqlite.md), [DuckDB](../linux/docs/drivers/duckdb.md), [SQL Server](../linux/docs/drivers/sql-server.md), [ClickHouse](../linux/docs/drivers/clickhouse.md), [Cassandra and ScyllaDB](../linux/docs/drivers/cassandra.md), [Redis](../linux/docs/drivers/redis.md), [MongoDB](../linux/docs/drivers/mongodb.md). An engine without SQL says so through `DriverCapabilities::data_model`: the workspace lists a key-value engine's keys and opens them in key tabs ([ADR 0021](../linux/docs/decisions/0021-engines-without-sql.md)), and opens a document engine's collections in table tabs whose statements are the engine's shell calls ([ADR 0026](../linux/docs/decisions/0026-documents-in-the-table-tab.md)). +`DatabaseDriver` describes an engine before any connection exists, through `DriverCapabilities`, and opens a `Connection`. The `Connection` serves the workspace from a pool; every call takes `CallOptions` (a deadline and a cancel token) and runs on a task the connection owns (`OwnedCalls`), so a stop reaches the server instead of leaving the statement running. Saves report what they left in the database through a `WriteLedger`. Each editor tab gets an `EditorSession` of its own that streams results into a `ResultSink`, and the `Session` that opens it puts a `ByteLimitSink` in front of that sink, so a run stops keeping rows once they pass `RunOptions::byte_limit` bytes across its results, the row that crosses it kept whole ([ADR 0020](docs/decisions/0020-a-run-keeps-a-byte-limit-of-rows.md)). [docs/adding-drivers.md](../linux/docs/adding-drivers.md) is the full contract, [ADR 0016](../linux/docs/decisions/0016-driver-contract-v2.md) the reasons for it, and the engine pages say how each driver meets it: [PostgreSQL](../linux/docs/drivers/postgresql.md), [MySQL and MariaDB](../linux/docs/drivers/mysql.md), [SQLite](../linux/docs/drivers/sqlite.md), [DuckDB](../linux/docs/drivers/duckdb.md), [SQL Server](../linux/docs/drivers/sql-server.md), [ClickHouse](../linux/docs/drivers/clickhouse.md), [Cassandra and ScyllaDB](../linux/docs/drivers/cassandra.md), [Redis](../linux/docs/drivers/redis.md), [MongoDB](../linux/docs/drivers/mongodb.md). An engine without SQL says so through `DriverCapabilities::data_model`: the workspace lists a key-value engine's keys and opens them in key tabs ([ADR 0021](../linux/docs/decisions/0021-engines-without-sql.md)), and opens a document engine's collections in table tabs whose statements are the engine's shell calls ([ADR 0026](../linux/docs/decisions/0026-documents-in-the-table-tab.md)). ### SQL scripts diff --git a/core/crates/core/src/browse/row_reference.rs b/core/crates/core/src/browse/row_reference.rs index cd11ac1306..b27c0ad3d3 100644 --- a/core/crates/core/src/browse/row_reference.rs +++ b/core/crates/core/src/browse/row_reference.rs @@ -166,6 +166,7 @@ fn stored_form(value: &Value) -> Option> { Some(match value { Value::Null => return None, Value::Bool(flag) => StoredForm::Integer(i128::from(*flag)), + Value::Int32(integer) => StoredForm::Integer(i128::from(*integer)), Value::Int(integer) => StoredForm::Integer(i128::from(*integer)), Value::UInt(integer) => StoredForm::Integer(i128::from(*integer)), Value::WideInt(integer) => StoredForm::Integer(integer.to_i128()?), diff --git a/core/crates/core/src/byte_limit.rs b/core/crates/core/src/byte_limit.rs new file mode 100644 index 0000000000..18a6031a81 --- /dev/null +++ b/core/crates/core/src/byte_limit.rs @@ -0,0 +1,143 @@ +use std::num::NonZeroU64; + +use crate::value::{OtherValue, UndecodedValue, Value}; + +// Zero is a run that can keep nothing, so it cannot be built. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct ByteLimit(NonZeroU64); + +impl ByteLimit { + // SI bytes, as the Linux banner prints them: "256.0 MB". + pub const EDITOR_DEFAULT: Self = Self(NonZeroU64::new(256_000_000).expect("256 MB is not zero")); + + pub fn new(bytes: u64) -> Option { + NonZeroU64::new(bytes).map(Self) + } + + pub const fn get(self) -> u64 { + self.0.get() + } +} + +// A wide integer is at most 256 bits. +const WIDE_INTEGER_BYTES: usize = 32; + +impl Value { + // Lengths, not capacities, so the same rows measure the same everywhere. + pub fn retained_bytes(&self) -> usize { + size_of::() + self.heap_bytes() + } + + fn heap_bytes(&self) -> usize { + match self { + Value::Null + | Value::Bool(_) + | Value::Int32(_) + | Value::Int(_) + | Value::UInt(_) + | Value::Float32(_) + | Value::Float64(_) + | Value::Uuid(_) + | Value::Date(_) + | Value::Time(_) + | Value::TimeTz(_) + | Value::Timestamp(_) + | Value::TimestampTz(_) + | Value::Interval(_) => 0, + Value::WideInt(_) => WIDE_INTEGER_BYTES, + Value::Decimal(decimal) => decimal.parts().map_or(0, |(_, digits, _)| digits.len()), + Value::Text(text) => text.len(), + Value::Bytes(bytes) => bytes.len(), + Value::Json(json) => json.as_str().len(), + Value::Bits(bits) => bits.as_bytes().len(), + Value::Array(items) => items.iter().map(Value::retained_bytes).sum(), + Value::Other(other) => size_of::() + other.type_name.len() + other.text.len(), + Value::Undecodable(undecoded) => size_of::() + undecoded.type_name.len(), + } + } +} + +pub(crate) fn row_bytes(row: &[Value]) -> u64 { + row.iter() + .map(|value| u64::try_from(value.retained_bytes()).unwrap_or(u64::MAX)) + .fold(0, u64::saturating_add) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::value::{BitString, JsonText, SqlDecimal, UndecodableReason, WideInteger}; + + const SLOT: usize = size_of::(); + + #[test] + fn a_limit_of_no_bytes_cannot_be_built() { + assert_eq!(ByteLimit::new(0), None); + assert_eq!(ByteLimit::new(1).map(ByteLimit::get), Some(1)); + } + + #[test] + fn the_editor_default_is_256_mb() { + assert_eq!(ByteLimit::EDITOR_DEFAULT.get(), 256_000_000); + } + + #[test] + fn a_scalar_is_its_slot_alone() { + for value in [ + Value::Null, + Value::Bool(true), + Value::Int32(7), + Value::Int(7), + Value::UInt(7), + Value::Float32(0.5), + Value::Float64(0.5), + Value::Uuid(uuid::Uuid::nil()), + ] { + assert_eq!(value.retained_bytes(), SLOT, "{value:?}"); + } + } + + #[test] + fn a_value_with_a_heap_part_adds_its_length() { + let decimal: SqlDecimal = "-123.45".parse().expect("a decimal"); + let json = JsonText::parse("{\"a\":1}".to_owned()).expect("json"); + let bits = BitString::from_u64(0b1011, 12); + + assert_eq!(Value::Text("héllo".to_owned()).retained_bytes(), SLOT + 6); + assert_eq!(Value::Bytes(vec![0; 1_000]).retained_bytes(), SLOT + 1_000); + assert_eq!(Value::Json(json).retained_bytes(), SLOT + 7); + assert_eq!(Value::Decimal(decimal).retained_bytes(), SLOT + 5); + assert_eq!(Value::Decimal(SqlDecimal::nan()).retained_bytes(), SLOT); + assert_eq!(Value::Bits(bits).retained_bytes(), SLOT + 2); + assert_eq!( + Value::WideInt(WideInteger::from_u128(u128::MAX)).retained_bytes(), + SLOT + WIDE_INTEGER_BYTES + ); + } + + #[test] + fn a_boxed_or_nested_value_counts_what_it_holds() { + let other = Value::Other(Box::new(OtherValue { + type_name: "inet".to_owned(), + text: "10.0.0.1".to_owned(), + })); + let undecodable = Value::Undecodable(Box::new(UndecodedValue { + type_name: "geometry".to_owned(), + reason: UndecodableReason::UnsupportedType, + })); + let array = Value::Array(vec![Value::Int(1), Value::Text("ab".to_owned())]); + + assert_eq!(other.retained_bytes(), SLOT + size_of::() + 4 + 8); + assert_eq!(undecodable.retained_bytes(), SLOT + size_of::() + 8); + assert_eq!(array.retained_bytes(), SLOT + SLOT + SLOT + 2); + assert_eq!(Value::Array(Vec::new()).retained_bytes(), SLOT); + } + + #[test] + fn a_row_is_the_sum_of_its_values() { + let row = [Value::Int(1), Value::Text("abc".to_owned())]; + + assert_eq!(row_bytes(&row), (SLOT * 2 + 3) as u64); + assert_eq!(row_bytes(&[]), 0); + } +} diff --git a/core/crates/core/src/byte_limited_sink.rs b/core/crates/core/src/byte_limited_sink.rs new file mode 100644 index 0000000000..a172b3a784 --- /dev/null +++ b/core/crates/core/src/byte_limited_sink.rs @@ -0,0 +1,370 @@ +use async_trait::async_trait; + +use crate::byte_limit::{ByteLimit, row_bytes}; +use crate::result_event::{ResultEvent, Truncation}; +use crate::result_sink::{ResultSink, SinkFlow}; +use crate::value::Value; + +// The row that crosses the limit is kept whole, so a run shows at least one row and every kept value is exact. +pub struct ByteLimitSink { + inner: Box, + limit: u64, + kept: u64, + set_rows: u64, + set_cut: bool, +} + +impl ByteLimitSink { + pub fn new(inner: Box, limit: ByteLimit) -> Self { + Self { + inner, + limit: limit.get(), + kept: 0, + set_rows: 0, + set_cut: false, + } + } + + // Never ConsumerGone past the cut: that cancels the statement, which aborts the user's transaction on some engines. + async fn rows(&mut self, mut rows: Vec>) -> SinkFlow { + if self.set_cut { + return SinkFlow::Continue; + } + let mut keep = 0; + for row in &rows { + if self.kept >= self.limit { + break; + } + self.kept = self.kept.saturating_add(row_bytes(row)); + keep += 1; + } + self.set_rows += keep as u64; + if keep == rows.len() { + return self.inner.accept(ResultEvent::Rows(rows)).await; + } + rows.truncate(keep); + if !rows.is_empty() && self.inner.accept(ResultEvent::Rows(rows)).await == SinkFlow::ConsumerGone { + return SinkFlow::ConsumerGone; + } + self.set_cut = true; + self.inner + .accept(ResultEvent::ByteLimitReached { + retained: self.set_rows, + }) + .await + } + + fn start_set(&mut self) { + self.set_rows = 0; + self.set_cut = false; + } +} + +#[async_trait] +impl ResultSink for ByteLimitSink { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + let event = match event { + ResultEvent::Rows(rows) => return self.rows(rows).await, + ResultEvent::ResultSetStarted { columns } => { + self.start_set(); + ResultEvent::ResultSetStarted { columns } + } + ResultEvent::RowLimitReached { .. } | ResultEvent::ByteLimitReached { .. } if self.set_cut => { + return SinkFlow::Continue; + } + // A driver that holds a set before its first event stops at the limit itself. + ResultEvent::ByteLimitReached { retained } => { + self.set_cut = true; + self.kept = self.kept.max(self.limit); + ResultEvent::ByteLimitReached { retained } + } + ResultEvent::ResultSetFinished { + total_rows, truncation, .. + } if self.set_cut => { + // A statement the server stopped did less than one read to its end, which the app has to say. + let truncation = match truncation { + Truncation::ServerStopped => Truncation::ServerStopped, + Truncation::Complete | Truncation::ReadPastLimit | Truncation::ByteLimit => Truncation::ByteLimit, + }; + let finished = ResultEvent::ResultSetFinished { + retained: self.set_rows, + total_rows, + truncation, + }; + self.start_set(); + finished + } + ResultEvent::ResultSetFinished { + retained, + total_rows, + truncation, + } => { + self.start_set(); + ResultEvent::ResultSetFinished { + retained, + total_rows, + truncation, + } + } + other => other, + }; + self.inner.accept(event).await + } +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use super::*; + use crate::column::{ColumnType, ResultColumn}; + + const SLOT: usize = size_of::(); + + #[derive(Clone, Default)] + struct Seen { + events: Arc>>, + gone: bool, + } + + #[async_trait] + impl ResultSink for Seen { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + self.events.lock().expect("the events").push(event); + match self.gone { + true => SinkFlow::ConsumerGone, + false => SinkFlow::Continue, + } + } + } + + impl Seen { + fn events(&self) -> Vec { + self.events.lock().expect("the events").clone() + } + } + + fn sink(seen: &Seen, bytes: u64) -> ByteLimitSink { + ByteLimitSink::new(Box::new(seen.clone()), ByteLimit::new(bytes).expect("a limit")) + } + + fn started() -> ResultEvent { + let columns: Arc<[ResultColumn]> = vec![ResultColumn::new("label", ColumnType::untyped_text(""))].into(); + ResultEvent::ResultSetStarted { columns } + } + + // One text value whose row measures exactly `bytes`. + fn row(bytes: usize, mark: char) -> Vec { + vec![Value::Text(mark.to_string().repeat(bytes - SLOT))] + } + + fn rows(count: usize, bytes: usize) -> Vec> { + (0..count).map(|_| row(bytes, 'x')).collect() + } + + fn finished(retained: u64, total_rows: Option, truncation: Truncation) -> ResultEvent { + ResultEvent::ResultSetFinished { + retained, + total_rows, + truncation, + } + } + + async fn play(sink: &mut ByteLimitSink, events: Vec) -> Vec { + let mut flows = Vec::new(); + for event in events { + flows.push(sink.accept(event).await); + } + flows + } + + #[tokio::test] + async fn a_set_under_the_limit_passes_as_it_came() { + let seen = Seen::default(); + let events = vec![ + started(), + ResultEvent::Rows(rows(3, 100)), + finished(3, Some(3), Truncation::Complete), + ]; + + play(&mut sink(&seen, 1_000), events.clone()).await; + + assert_eq!(seen.events(), events); + } + + #[tokio::test] + async fn a_set_read_to_its_end_is_cut_with_its_exact_total() { + let seen = Seen::default(); + + let flows = play( + &mut sink(&seen, 250), + vec![ + started(), + ResultEvent::Rows(rows(10, 100)), + finished(10, Some(10), Truncation::Complete), + ], + ) + .await; + + assert_eq!( + seen.events(), + [ + started(), + ResultEvent::Rows(rows(3, 100)), + ResultEvent::ByteLimitReached { retained: 3 }, + finished(3, Some(10), Truncation::ByteLimit), + ] + ); + assert!(flows.iter().all(|flow| *flow == SinkFlow::Continue), "{flows:?}"); + } + + #[tokio::test] + async fn the_drivers_row_limit_is_swallowed_for_a_set_already_cut() { + let seen = Seen::default(); + + play( + &mut sink(&seen, 250), + vec![ + started(), + ResultEvent::Rows(rows(2, 100)), + ResultEvent::Rows(rows(8, 100)), + ResultEvent::RowLimitReached { retained: 10 }, + finished(10, None, Truncation::ServerStopped), + ], + ) + .await; + + assert_eq!( + seen.events(), + [ + started(), + ResultEvent::Rows(rows(2, 100)), + ResultEvent::Rows(rows(1, 100)), + ResultEvent::ByteLimitReached { retained: 3 }, + finished(3, None, Truncation::ServerStopped), + ] + ); + } + + #[tokio::test] + async fn a_counted_total_past_the_row_limit_passes_through() { + let seen = Seen::default(); + + play( + &mut sink(&seen, 150), + vec![ + started(), + ResultEvent::Rows(rows(10, 100)), + ResultEvent::RowLimitReached { retained: 10 }, + finished(10, Some(50), Truncation::ReadPastLimit), + ], + ) + .await; + + assert_eq!( + seen.events().last(), + Some(&finished(2, Some(50), Truncation::ByteLimit)) + ); + } + + #[tokio::test] + async fn the_row_that_crosses_the_limit_is_kept_whole() { + let seen = Seen::default(); + + play( + &mut sink(&seen, 10), + vec![ + started(), + ResultEvent::Rows(vec![row(5_000, 'a'), row(5_000, 'b')]), + finished(2, Some(2), Truncation::Complete), + ], + ) + .await; + + assert_eq!(seen.events()[1], ResultEvent::Rows(vec![row(5_000, 'a')])); + assert_eq!(seen.events()[2], ResultEvent::ByteLimitReached { retained: 1 }); + } + + #[tokio::test] + async fn a_set_after_the_limit_keeps_nothing_and_an_empty_one_stays_complete() { + let seen = Seen::default(); + + play( + &mut sink(&seen, 200), + vec![ + started(), + ResultEvent::Rows(rows(2, 100)), + finished(2, Some(2), Truncation::Complete), + started(), + ResultEvent::Rows(rows(4, 100)), + finished(4, Some(4), Truncation::Complete), + started(), + finished(0, Some(0), Truncation::Complete), + ResultEvent::CommandCompleted { rows_affected: Some(1) }, + ], + ) + .await; + + assert_eq!( + seen.events(), + [ + started(), + ResultEvent::Rows(rows(2, 100)), + finished(2, Some(2), Truncation::Complete), + started(), + ResultEvent::ByteLimitReached { retained: 0 }, + finished(0, Some(4), Truncation::ByteLimit), + started(), + finished(0, Some(0), Truncation::Complete), + ResultEvent::CommandCompleted { rows_affected: Some(1) }, + ] + ); + } + + #[tokio::test] + async fn a_byte_limit_the_driver_sent_passes_once_and_fills_the_run() { + let seen = Seen::default(); + + play( + &mut sink(&seen, 1_000), + vec![ + started(), + ResultEvent::Rows(rows(2, 100)), + ResultEvent::ByteLimitReached { retained: 2 }, + ResultEvent::ByteLimitReached { retained: 2 }, + finished(2, None, Truncation::ByteLimit), + started(), + ResultEvent::Rows(rows(1, 100)), + finished(1, Some(1), Truncation::Complete), + ], + ) + .await; + + assert_eq!( + seen.events(), + [ + started(), + ResultEvent::Rows(rows(2, 100)), + ResultEvent::ByteLimitReached { retained: 2 }, + finished(2, None, Truncation::ByteLimit), + started(), + ResultEvent::ByteLimitReached { retained: 0 }, + finished(0, Some(1), Truncation::ByteLimit), + ] + ); + } + + #[tokio::test] + async fn the_inner_sinks_consumer_gone_passes() { + let seen = Seen { + gone: true, + ..Seen::default() + }; + let mut sink = sink(&seen, 150); + + let flows = play(&mut sink, vec![started(), ResultEvent::Rows(rows(4, 100))]).await; + + assert_eq!(flows, [SinkFlow::ConsumerGone, SinkFlow::ConsumerGone]); + assert_eq!(seen.events().len(), 2, "nothing after the inner sink left"); + } +} diff --git a/core/crates/core/src/connection_url/mod.rs b/core/crates/core/src/connection_url/mod.rs index 3816258177..9ab9af2c46 100644 --- a/core/crates/core/src/connection_url/mod.rs +++ b/core/crates/core/src/connection_url/mod.rs @@ -233,7 +233,7 @@ struct Draft { host: String, port: Option, username: String, - password: Option, + password: Option, database: String, tls: TlsHints, options: NetworkOptions, @@ -241,7 +241,7 @@ struct Draft { instance: bool, direct: Option, query_user: Option, - query_password: Option, + query_password: Option, unread: Vec, path_database: bool, } @@ -250,7 +250,7 @@ fn network_url(engine: Engine, rest: &str, properties: bool) -> Result match userinfo.split_once(':') { - Some((user, secret)) => (decode(user)?, Some(decode(secret)?)), + Some((user, secret)) => (decode(user)?, Some(SecretString::from(decode(secret)?))), None => (decode(userinfo)?, None), }, None => (String::new(), None), @@ -294,7 +294,7 @@ fn network_url(engine: Engine, rest: &str, properties: bool) -> Result draft.instance = !value.trim().is_empty(), (Engine::Mysql { .. }, "socket" | "unix_socket") => draft.socket = Some(value), (_, "user" | "username") => draft.query_user = Some(value), - (_, "password") => draft.query_password = Some(value), + (_, "password") => draft.query_password = Some(SecretString::from(value)), _ => draft.unread(key), } Ok(()) diff --git a/core/crates/core/src/dialect/test_dialect.rs b/core/crates/core/src/dialect/test_dialect.rs index 5b35a44ed3..dc59821b1e 100644 --- a/core/crates/core/src/dialect/test_dialect.rs +++ b/core/crates/core/src/dialect/test_dialect.rs @@ -224,6 +224,7 @@ impl SqlDialect for TestDialect { match value { Value::Null => Ok("NULL".to_owned()), Value::Bool(flag) => Ok(flag.to_string()), + Value::Int32(number) => Ok(number.to_string()), Value::Int(number) => Ok(number.to_string()), Value::UInt(number) => Ok(number.to_string()), Value::Text(text) => Ok(format!("'{}'", text.replace('\'', "''"))), diff --git a/core/crates/core/src/dml/build_update.rs b/core/crates/core/src/dml/build_update.rs index 93d1152ab0..a5f517629a 100644 --- a/core/crates/core/src/dml/build_update.rs +++ b/core/crates/core/src/dml/build_update.rs @@ -21,9 +21,10 @@ pub fn build_update( identity: &RowIdentity, key: &[Value], assignments: &[(usize, CellInput)], + read: &[(usize, Value)], ) -> Result { if crate::document::speaks_shell(dialect) { - return crate::document::update(table, columns, identity, key, assignments); + return crate::document::update(table, columns, identity, key, assignments, read); } if assignments.is_empty() { return Err(BuildSqlError::NothingToUpdate); diff --git a/core/crates/core/src/dml/tests.rs b/core/crates/core/src/dml/tests.rs index d43e781e29..a3bb180c64 100644 --- a/core/crates/core/src/dml/tests.rs +++ b/core/crates/core/src/dml/tests.rs @@ -77,6 +77,7 @@ fn build_update_binds_every_value_and_names_the_row() { &by_id(), &[Value::Int(7)], &[(1, CellInput::Value(Value::Text("new".to_owned())))], + &[], ) .expect("the update"); @@ -107,6 +108,7 @@ fn an_update_of_partition_scoped_columns_alone_names_only_the_partition() { &identity, &key, &[(2, CellInput::Value(Value::Int(3)))], + &[], ) .expect("the update"); let mixed = build_update( @@ -119,6 +121,7 @@ fn an_update_of_partition_scoped_columns_alone_names_only_the_partition() { (1, CellInput::Value(Value::Text("grace".to_owned()))), (2, CellInput::Value(Value::Int(3))), ], + &[], ) .expect("the update"); @@ -148,6 +151,7 @@ fn build_update_null_key_renders_is_null_without_marker() { &identity, &[Value::Int(7), Value::Null], &[(1, CellInput::Value(Value::Text("new".to_owned())))], + &[], ) .expect("the update"); @@ -170,6 +174,7 @@ fn build_update_undecodable_key_is_error() { &by_id(), &[undecodable()], &[(1, CellInput::Value(Value::Text("new".to_owned())))], + &[], ) .expect_err("an unreadable key"); @@ -180,8 +185,8 @@ fn build_update_undecodable_key_is_error() { fn build_update_with_no_assignments_is_error() { let dialect = TestDialect::exact(); - let error = - build_update(&dialect, &table(), &columns(), &by_id(), &[Value::Int(1)], &[]).expect_err("nothing to change"); + let error = build_update(&dialect, &table(), &columns(), &by_id(), &[Value::Int(1)], &[], &[]) + .expect_err("nothing to change"); assert_eq!(error, BuildSqlError::NothingToUpdate); } @@ -355,6 +360,7 @@ fn build_update_engine_row_id_uses_part_columns_and_targets() { &identity, &key, &[(1, CellInput::Value(Value::Text("new".to_owned())))], + &[], ) .expect("the update"); @@ -387,6 +393,7 @@ fn positional_markers_bind_assignments_before_the_key() { &identity, &[Value::Int(7), Value::Int(3)], &[(1, CellInput::Value(Value::Text("new".to_owned())))], + &[], ) .expect("the update"); @@ -421,6 +428,7 @@ fn a_value_of_the_wrong_type_is_refused_before_it_reaches_the_server() { &by_id(), &[Value::Int(7)], &[(0, CellInput::Value(Value::Text("not a number".to_owned())))], + &[], ) .expect_err("text into an integer column"); @@ -454,6 +462,7 @@ fn an_expression_is_written_into_the_update_and_never_bound() { &by_id(), &[Value::Int(7)], &[(1, now()), (2, CellInput::Value(Value::Int(3)))], + &[], ) .expect("the update"); @@ -496,6 +505,7 @@ fn a_default_in_an_update_is_written_only_where_the_engine_parses_it() { &by_id(), &[Value::Int(7)], &assignments, + &[], ) .expect("the update"); assert_eq!( @@ -510,6 +520,7 @@ fn a_default_in_an_update_is_written_only_where_the_engine_parses_it() { &by_id(), &[Value::Int(7)], &assignments, + &[], ); assert_eq!( refused, diff --git a/core/crates/core/src/document/document_write.rs b/core/crates/core/src/document/document_write.rs index db02eeb9c0..e385f9db4d 100644 --- a/core/crates/core/src/document/document_write.rs +++ b/core/crates/core/src/document/document_write.rs @@ -81,6 +81,7 @@ pub(crate) fn update( identity: &RowIdentity, key: &[Value], assignments: &[(usize, CellInput)], + read: &[(usize, Value)], ) -> Result { if assignments.is_empty() { return Err(BuildSqlError::NothingToUpdate); @@ -107,7 +108,10 @@ pub(crate) fn update( Value::Null if !names.contains(&column.name) => { unset.push((column.name.clone(), ShellValue::string(""))); } - _ => set.push((column.name.clone(), params.bind(value.clone(), Some(column))?)), + _ => { + let read = read.iter().find(|(at, _)| at == index).map(|(_, value)| value); + set.push((column.name.clone(), params.bind_edit(value.clone(), column, read)?)); + } } } let filter = key_filter(columns, identity, key, &mut params)?; diff --git a/core/crates/core/src/document/json_cells.rs b/core/crates/core/src/document/json_cells.rs index f78e934cb8..ef6a848ed3 100644 --- a/core/crates/core/src/document/json_cells.rs +++ b/core/crates/core/src/document/json_cells.rs @@ -32,7 +32,8 @@ fn as_json(value: Value) -> Value { Value::Null | Value::Json(_) | Value::Undecodable(_) => return value, Value::Text(text) => quoted(text), Value::Bool(flag) => flag.to_string(), - Value::Int(number) => number.to_string(), + Value::Int32(number) => number.to_string(), + Value::Int(number) => wrapped("$numberLong", "ed(&number.to_string())), Value::UInt(number) => number.to_string(), Value::WideInt(number) => wrapped("$numberDecimal", "ed(&number.to_string())), Value::Float64(number) => double(*number), @@ -143,6 +144,7 @@ mod tests { })); let rows = vec![ vec![Value::Text("a\"b".to_owned()), Value::Text("plain".to_owned())], + vec![Value::Int32(5), Value::Null], vec![Value::Int(5), Value::Null], vec![Value::Float64(2.0), Value::Null], vec![id, Value::Null], @@ -172,6 +174,7 @@ mod tests { [ r#""a\"b""#, "5", + r#"{"$numberLong": "5"}"#, "2.0", r#"{"$oid": "5f1d7a0c9b1e8b2a3c4d5e6f"}"#, r#"{"$binary": {"base64": "aGkh", "subType": "00"}}"#, diff --git a/core/crates/core/src/document/shell_literal.rs b/core/crates/core/src/document/shell_literal.rs index 6414de11da..a536d41fe3 100644 --- a/core/crates/core/src/document/shell_literal.rs +++ b/core/crates/core/src/document/shell_literal.rs @@ -13,9 +13,10 @@ pub fn shell_literal(value: &Value, column: &ColumnType) -> Result "null".to_owned(), Value::Bool(flag) => flag.to_string(), + Value::Int32(number) => number.to_string(), Value::Int(number) => match (column.name.as_str(), i32::try_from(*number)) { - ("long", _) | (_, Err(_)) => format!("NumberLong({})", quoted(&number.to_string())), - (_, Ok(small)) => small.to_string(), + ("int", Ok(small)) => small.to_string(), + _ => format!("NumberLong({})", quoted(&number.to_string())), }, Value::UInt(number) => match i64::try_from(*number) { Ok(number) => format!("NumberLong({})", quoted(&number.to_string())), @@ -115,6 +116,9 @@ mod tests { (Value::Int(5), "int", "5"), (Value::Int(5), "long", r#"NumberLong("5")"#), (Value::Int(5_000_000_000), "int", r#"NumberLong("5000000000")"#), + (Value::Int32(5), "long", "5"), + (Value::Int(5), "double", r#"NumberLong("5")"#), + (Value::Int(5), "mixed", r#"NumberLong("5")"#), (Value::Float64(2.0), "double", "2.0"), (Value::Text("a\"b".to_owned()), "string", r#""a\"b""#), ( diff --git a/core/crates/core/src/document/shell_params.rs b/core/crates/core/src/document/shell_params.rs index 3c962bb395..2065f82639 100644 --- a/core/crates/core/src/document/shell_params.rs +++ b/core/crates/core/src/document/shell_params.rs @@ -1,8 +1,11 @@ +use std::cmp::Ordering; + use crate::column::{ColumnInfo, SqlTypeExpr}; use crate::dialect::BindError; use crate::sql_syntax::mongo::{ShellCall, ShellValue}; use crate::statement::{BoundParam, ParamType, Statement, StatementBuilder}; use crate::value::Value; +use crate::value::order::decimal_of; /// The values a generated shell statement binds, each written into the /// statement as `$n`. @@ -19,7 +22,17 @@ impl ShellParams { /// Bind `value` as the next parameter, typed by `column` where there /// is one, and return the marker that stands for it. pub(crate) fn bind(&mut self, value: Value, column: Option<&ColumnInfo>) -> Result { - let param = bind_param(value, column)?; + let param = bind_param(value, column, None)?; + Ok(ShellValue::Param(self.builder.push_bound(param))) + } + + pub(crate) fn bind_edit( + &mut self, + value: Value, + column: &ColumnInfo, + read: Option<&Value>, + ) -> Result { + let param = bind_param(value, Some(column), read)?; Ok(ShellValue::Param(self.builder.push_bound(param))) } @@ -57,12 +70,16 @@ fn renumbered(value: &ShellValue, by: usize) -> ShellValue { /// The parameter a value binds as. A field's own BSON type, which names /// its column's type, decides between an int and a long and says which /// text is an ObjectId; a value with no column is typed by itself. -pub fn bind_param(value: Value, column: Option<&ColumnInfo>) -> Result { +pub fn bind_param(value: Value, column: Option<&ColumnInfo>, read: Option<&Value>) -> Result { + // A column's type is the widest number its sample held, so an edited number keeps the type it was read as. + if let Some((value, param_type)) = read.and_then(|read| in_read_type(&value, read)) { + return BoundParam::new(value, param_type); + } let type_name = column.map(|column| column.column_type.name.as_str()); let param_type = match (&value, type_name) { (Value::Null, Some(name)) => param_type_named(name), (Value::Null, None) | (Value::Text(_), _) => ParamType::Text, - (Value::Int(_), Some("int")) => ParamType::Int32, + (Value::Int32(_), _) | (Value::Int(_), Some("int")) => ParamType::Int32, (Value::Int(_), _) => ParamType::Int64, (Value::UInt(_), _) => ParamType::UInt64, (Value::Float32(_) | Value::Float64(_), _) => ParamType::Float64, @@ -93,6 +110,61 @@ pub fn bind_param(value: Value, column: Option<&ColumnInfo>) -> Result Option<(Value, ParamType)> { + if !is_number(value) { + return None; + } + match read { + Value::Int32(_) => exact_integer(value) + .and_then(|number| i32::try_from(number).ok()) + .map(|number| (Value::Int32(number), ParamType::Int32)), + Value::Int(_) => exact_integer(value).map(|number| (Value::Int(number), ParamType::Int64)), + Value::Float64(_) => exact_double(value).map(|number| (Value::Float64(number), ParamType::Float64)), + Value::Decimal(_) => decimal_of(value).map(|number| (Value::Decimal(number.into_owned()), ParamType::Decimal)), + _ => None, + } +} + +fn is_number(value: &Value) -> bool { + matches!( + value, + Value::Int32(_) + | Value::Int(_) + | Value::UInt(_) + | Value::WideInt(_) + | Value::Float32(_) + | Value::Float64(_) + | Value::Decimal(_) + ) +} + +// A float, or a decimal typed with a fraction such as `4.0`, is not an integer even when it is whole. +fn exact_integer(value: &Value) -> Option { + match value { + Value::Int32(number) => Some(i64::from(*number)), + Value::Int(number) => Some(*number), + Value::UInt(number) => i64::try_from(*number).ok(), + Value::WideInt(number) => number.to_i128().and_then(|number| i64::try_from(number).ok()), + Value::Decimal(decimal) => match decimal.to_scaled_i128()? { + (mantissa, 0) => i64::try_from(mantissa).ok(), + _ => None, + }, + _ => None, + } +} + +// A double whose shortest form is the same number, so `2.5` is one and `9007199254740993` is not. +fn exact_double(value: &Value) -> Option { + if let Value::Float64(number) = value { + return Some(*number); + } + let decimal = decimal_of(value)?; + let double = decimal.to_string().parse::().ok()?; + let read_back = Value::Float64(double); + let shortest = decimal_of(&read_back)?; + (shortest.cmp_numeric(&decimal) == Ordering::Equal).then_some(double) +} + /// What a null of a field of this BSON type binds as. fn param_type_named(name: &str) -> ParamType { match name { diff --git a/core/crates/core/src/document/tests.rs b/core/crates/core/src/document/tests.rs index c5f438c3ab..4d90028887 100644 --- a/core/crates/core/src/document/tests.rs +++ b/core/crates/core/src/document/tests.rs @@ -2,7 +2,7 @@ use crate::browse::{ ExportFields, ExportRequest, PageRequest, RowKeyLayout, SortKey, build_count, build_export, build_page, }; use crate::column::{ - ColumnDefault, ColumnInfo, ColumnKind, ColumnType, IntegerKind, IntegerWidth, ReadForm, TextLength, + ColumnDefault, ColumnInfo, ColumnKind, ColumnType, FloatWidth, IntegerKind, IntegerWidth, ReadForm, TextLength, }; use crate::dialect::TestDialect; use crate::dml::{BuildSqlError, build_delete, build_insert, build_multi_row_insert, build_update}; @@ -321,7 +321,7 @@ fn a_filter_that_is_not_one_document_or_names_a_dotted_field_is_refused() { #[test] fn build_filter_writes_the_document_after_parameters_already_bound() { let mut builder = StatementBuilder::new(); - builder.push_bound(crate::document::bind_param(Value::Int(1), None).expect("a param")); + builder.push_bound(crate::document::bind_param(Value::Int(1), None, None).expect("a param")); assert_eq!(build_filter(&mut builder, &shell(), &columns(), &filter()), Ok(true)); @@ -458,6 +458,7 @@ fn a_change_names_its_document_by_id_and_sets_only_the_fields_changed() { (1, CellInput::Value(Value::Text("ann".to_owned()))), (2, CellInput::Value(Value::Int(3))), ], + &[], ) .expect("an update"); assert_eq!( @@ -513,7 +514,16 @@ fn a_change_names_its_document_by_id_and_sets_only_the_fields_changed() { #[test] fn a_change_to_null_removes_the_field_and_a_key_keeps_set() { let change = |assignments: &[(usize, CellInput)]| { - build_update(&shell(), &users(), &columns(), &by_id(), &[object_id()], assignments).expect("an update") + build_update( + &shell(), + &users(), + &columns(), + &by_id(), + &[object_id()], + assignments, + &[], + ) + .expect("an update") }; let both = change(&[ @@ -619,6 +629,7 @@ fn a_dotted_field_cannot_be_updated_by_name() { &by_id(), &[object_id()], &[(3, CellInput::Value(Value::Text("x".to_owned())))], + &[], ); assert_eq!( @@ -636,6 +647,7 @@ fn a_document_field_is_never_set_to_a_value_the_server_computes() { &by_id(), &[object_id()], &[(1, CellInput::Default)], + &[], ); assert_eq!( @@ -645,3 +657,108 @@ fn a_document_field_is_never_set_to_a_value_the_server_computes() { }) ); } + +fn numbers() -> Vec { + vec![ + column("_id", "objectId", ColumnKind::Other), + column( + "qty", + "long", + ColumnKind::Integer(IntegerKind { + width: IntegerWidth::W64, + signed: true, + }), + ), + column("price", "double", ColumnKind::Float(FloatWidth::F64)), + column( + "amount", + "decimal", + ColumnKind::Decimal { + precision: Some(34), + scale: None, + }, + ), + ] +} + +fn edited_params(edits: &[(usize, Value, Value)]) -> Vec<(Value, ParamType)> { + let assignments: Vec<(usize, CellInput)> = edits + .iter() + .map(|(column, _, value)| (*column, CellInput::Value(value.clone()))) + .collect(); + let read: Vec<(usize, Value)> = edits.iter().map(|(column, read, _)| (*column, read.clone())).collect(); + let update = build_update( + &shell(), + &users(), + &numbers(), + &by_id(), + &[object_id()], + &assignments, + &read, + ) + .expect("an update"); + checked(&update); + update.params()[..edits.len()] + .iter() + .map(|param| (param.value().clone(), param.param_type().clone())) + .collect() +} + +fn decimal(text: &str) -> Value { + Value::Decimal(text.parse().expect("a decimal")) +} + +#[test] +fn an_edited_number_binds_as_the_type_it_was_read_as() { + assert_eq!( + edited_params(&[ + (1, Value::Int32(3), Value::Int(7)), + (2, Value::Int32(3), Value::Int32(4)), + (3, Value::Float64(1.5), decimal("2.5")), + ]), + [ + (Value::Int32(7), ParamType::Int32), + (Value::Int32(4), ParamType::Int32), + (Value::Float64(2.5), ParamType::Float64), + ] + ); + assert_eq!( + edited_params(&[(1, Value::Int(3), Value::Int(7)), (3, Value::Int(3), decimal("8"))]), + [(Value::Int(7), ParamType::Int64), (Value::Int(8), ParamType::Int64)] + ); +} + +#[test] +fn an_edit_its_read_type_cannot_hold_takes_the_columns_type() { + assert_eq!( + edited_params(&[ + (2, Value::Int32(3), Value::Float64(4.5)), + (1, Value::Int32(3), Value::Int(3_000_000_000)), + (3, Value::Float64(1.5), decimal("0.10000000000000000001")), + ]), + [ + (Value::Float64(4.5), ParamType::Float64), + (Value::Int(3_000_000_000), ParamType::Int64), + (decimal("0.10000000000000000001"), ParamType::Decimal), + ] + ); + assert_eq!( + edited_params(&[(3, Value::Int32(3), decimal("4.0"))]), + [(decimal("4.0"), ParamType::Decimal)], + "a fraction typed into an int field became an int" + ); +} + +#[test] +fn a_cell_read_as_null_or_not_a_number_takes_the_columns_type() { + assert_eq!( + edited_params(&[ + (1, Value::Null, Value::Int(7)), + (2, Value::Text("x".to_owned()), Value::Float64(4.0)), + ]), + [ + (Value::Int(7), ParamType::Int64), + (Value::Float64(4.0), ParamType::Float64), + ] + ); +} diff --git a/core/crates/core/src/document/value_order.rs b/core/crates/core/src/document/value_order.rs index 63f8aa9c23..229cde208a 100644 --- a/core/crates/core/src/document/value_order.rs +++ b/core/crates/core/src/document/value_order.rs @@ -40,7 +40,8 @@ fn bracket_named(alias: &str) -> Option { fn bracket_of(value: &Value) -> Option { let alias = match value { Value::Null => "null", - Value::Int(_) + Value::Int32(_) + | Value::Int(_) | Value::UInt(_) | Value::Float32(_) | Value::Float64(_) @@ -123,6 +124,10 @@ mod tests { Some(vec!["bool", "date", "timestamp", "regex", "maxKey"]) ); assert_eq!(aliases_beyond(&Value::Int(5), false), Some(vec!["minKey", "null"])); + assert_eq!( + aliases_beyond(&Value::Int32(5), true), + aliases_beyond(&Value::Int(5), true) + ); assert_eq!( aliases_beyond(&Value::Text("a".to_owned()), true).map(|aliases| aliases.len()), Some(9) diff --git a/core/crates/core/src/dump/dump_writer.rs b/core/crates/core/src/dump/dump_writer.rs index c0227d0ee0..0d30aa5a7a 100644 --- a/core/crates/core/src/dump/dump_writer.rs +++ b/core/crates/core/src/dump/dump_writer.rs @@ -513,6 +513,7 @@ impl ResultSink for RowsSink { } } ResultEvent::RowLimitReached { .. } + | ResultEvent::ByteLimitReached { .. } | ResultEvent::CommandCompleted { .. } | ResultEvent::Notice(_) | ResultEvent::NoticeLimitReached { .. } => SinkFlow::Continue, diff --git a/core/crates/core/src/edit/parse_edit_text.rs b/core/crates/core/src/edit/parse_edit_text.rs index 30910262a6..305480e4d3 100644 --- a/core/crates/core/src/edit/parse_edit_text.rs +++ b/core/crates/core/src/edit/parse_edit_text.rs @@ -1,17 +1,21 @@ use crate::column::ColumnInfo; use crate::value::Value; +use super::parse_literal_text::parse_as_read; use super::{CellInput, EditContext, EditParseError, EmptyInputMeaning, empty_input_meaning, parse_literal_text}; -/// Turn what the user typed into a cell into the input to save. -/// -/// Empty input follows `empty_input_meaning`. Whitespace alone is empty -/// for a column that cannot hold the empty string, and a value for one -/// that can. -pub fn parse_edit_text(input: &str, column: &ColumnInfo, context: EditContext) -> Result { +pub fn parse_edit_text( + input: &str, + column: &ColumnInfo, + context: EditContext, + read: Option<&Value>, +) -> Result { let accepts_empty = column.column_type.kind.accepts_empty_string(); let empty = input.is_empty() || (!accepts_empty && input.trim().is_empty()); if !empty { + if let Some(value) = read.and_then(|read| parse_as_read(input, read, &column.column_type)) { + return Ok(CellInput::Value(value)); + } return parse_literal_text(input, &column.column_type).map(CellInput::Value); } match empty_input_meaning(column, context) { @@ -25,7 +29,8 @@ pub fn parse_edit_text(input: &str, column: &ColumnInfo, context: EditContext) - #[cfg(test)] mod tests { use crate::column::{ - ColumnDefault, ColumnKind, ColumnType, IntegerKind, IntegerWidth, ReadForm, SqlExpression, TextLength, + ColumnDefault, ColumnKind, ColumnType, FloatWidth, IntegerKind, IntegerWidth, ReadForm, SqlExpression, + TextLength, }; use super::*; @@ -71,12 +76,12 @@ mod tests { for context in [EditContext::InsertDraft, EditContext::UpdateRow] { assert_eq!( - parse_edit_text("", &text, context), + parse_edit_text("", &text, context, None), Ok(CellInput::Value(Value::Text(String::new()))) ); } assert_eq!( - parse_edit_text(" ", &text, EditContext::UpdateRow), + parse_edit_text(" ", &text, EditContext::UpdateRow, None), Ok(CellInput::Value(Value::Text(" ".to_owned()))), "spaces in a text column were thrown away" ); @@ -100,12 +105,12 @@ mod tests { ); assert_eq!( - parse_edit_text("", &set, EditContext::UpdateRow), + parse_edit_text("", &set, EditContext::UpdateRow, None), Ok(CellInput::Value(Value::Text(String::new()))), "clearing a set wrote NULL instead of the empty set" ); assert_eq!( - parse_edit_text("", &enumeration, EditContext::UpdateRow), + parse_edit_text("", &enumeration, EditContext::UpdateRow, None), Ok(CellInput::Value(Value::Null)) ); } @@ -118,19 +123,19 @@ mod tests { serial.is_auto_increment = true; assert_eq!( - parse_edit_text("", &defaulted, EditContext::InsertDraft), + parse_edit_text("", &defaulted, EditContext::InsertDraft, None), Ok(CellInput::Default) ); assert_eq!( - parse_edit_text(" ", &serial, EditContext::InsertDraft), + parse_edit_text(" ", &serial, EditContext::InsertDraft, None), Ok(CellInput::Default) ); assert_eq!( - parse_edit_text("", &integer(true), EditContext::InsertDraft), + parse_edit_text("", &integer(true), EditContext::InsertDraft, None), Ok(CellInput::Value(Value::Null)) ); assert_eq!( - parse_edit_text("", &integer(false), EditContext::InsertDraft), + parse_edit_text("", &integer(false), EditContext::InsertDraft, None), Err(EditParseError::Required) ); } @@ -141,12 +146,12 @@ mod tests { defaulted.default = ColumnDefault::Expression(SqlExpression::from_catalog_text("0")); assert_eq!( - parse_edit_text("", &defaulted, EditContext::UpdateRow), + parse_edit_text("", &defaulted, EditContext::UpdateRow, None), Err(EditParseError::Required), "an UPDATE writing NULL into a NOT NULL column" ); assert_eq!( - parse_edit_text("", &integer(true), EditContext::UpdateRow), + parse_edit_text("", &integer(true), EditContext::UpdateRow, None), Ok(CellInput::Value(Value::Null)) ); } @@ -154,12 +159,77 @@ mod tests { #[test] fn typed_text_parses_as_the_columns_type() { assert_eq!( - parse_edit_text("42", &integer(false), EditContext::UpdateRow), + parse_edit_text("42", &integer(false), EditContext::UpdateRow, None), Ok(CellInput::Value(Value::Int(42))) ); assert_eq!( - parse_edit_text("4x", &integer(false), EditContext::UpdateRow), + parse_edit_text("4x", &integer(false), EditContext::UpdateRow, None), Err(EditParseError::InvalidInteger) ); } + + fn edited(input: &str, column: &ColumnInfo, read: Value) -> Result { + parse_edit_text(input, column, EditContext::UpdateRow, Some(&read)) + } + + fn decimal(text: &str) -> Value { + Value::Decimal(text.parse().expect("a decimal")) + } + + #[test] + fn a_number_a_sampled_column_does_not_type_parses_as_the_number_it_is() { + let double = column(ColumnKind::Float(FloatWidth::F64), true); + let decimal_column = column( + ColumnKind::Decimal { + precision: Some(34), + scale: None, + }, + true, + ); + + assert_eq!( + edited("4", &double, Value::Int32(3)), + Ok(CellInput::Value(Value::Int32(4))) + ); + assert_eq!( + edited("4.5", &double, Value::Int32(3)), + Ok(CellInput::Value(Value::Float64(4.5))) + ); + assert_eq!( + edited("9007199254740995", &double, Value::Int(9_007_199_254_740_993)), + Ok(CellInput::Value(Value::Int(9_007_199_254_740_995))), + "a long in a double field lost its digits" + ); + assert_eq!( + edited("0.1", &decimal_column, Value::Float64(2.5)), + Ok(CellInput::Value(Value::Float64(0.1))) + ); + assert_eq!( + edited("9007199254740993", &decimal_column, Value::Float64(2.5)), + Ok(CellInput::Value(decimal("9007199254740993"))), + "a number no double holds was rounded to one" + ); + } + + #[test] + fn a_long_past_the_sample_of_an_int_field_is_not_refused() { + assert_eq!( + edited("5000000001", &integer(true), Value::Int(5_000_000_000)), + Ok(CellInput::Value(Value::Int(5_000_000_001))) + ); + assert_eq!( + edited("5000000001", &integer(true), Value::Int(1)), + Err(EditParseError::IntegerOutOfRange { + kind: IntegerKind { + width: IntegerWidth::W32, + signed: true, + }, + }), + "a value the column holds lifted the column's range" + ); + assert_eq!( + edited("5000000001", &integer(true), Value::Null), + parse_edit_text("5000000001", &integer(true), EditContext::UpdateRow, None) + ); + } } diff --git a/core/crates/core/src/edit/parse_literal_text.rs b/core/crates/core/src/edit/parse_literal_text.rs index 9bae8c469f..2fcabf5fc0 100644 --- a/core/crates/core/src/edit/parse_literal_text.rs +++ b/core/crates/core/src/edit/parse_literal_text.rs @@ -21,6 +21,50 @@ pub fn parse_literal_text(input: &str, column: &ColumnType) -> Result Option { + let census = matches!( + column.kind, + ColumnKind::Integer(_) + | ColumnKind::Float(_) + | ColumnKind::Decimal { .. } + | ColumnKind::Money { .. } + | ColumnKind::Json { .. } + ); + if column.dynamic_storage || !census || kind_holds(&column.kind, read) { + return None; + } + let trimmed = input.trim(); + match read { + Value::Int32(_) => trimmed.parse::().ok().map(Value::Int32), + Value::Int(_) => trimmed.parse::().ok().map(Value::Int), + Value::Float64(_) => exact_double(trimmed).map(Value::Float64), + Value::Decimal(_) => trimmed.parse::().ok().map(Value::Decimal), + _ => None, + } +} + +fn kind_holds(kind: &ColumnKind, read: &Value) -> bool { + match (kind, read) { + (ColumnKind::Integer(kind), Value::Int(number)) => WideInteger::from_i128(i128::from(*number)).fits(*kind), + (ColumnKind::Float(_), Value::Float64(_)) => true, + (ColumnKind::Decimal { .. } | ColumnKind::Money { .. }, Value::Decimal(_)) => true, + _ => false, + } +} + +// `0.1` is a double by its shortest form, and `9007199254740993` is not one. +fn exact_double(input: &str) -> Option { + if let Some(value) = non_finite(input) { + return Some(value); + } + let decimal = input.parse::().ok()?; + let double = input.parse::().ok()?; + let read_back = Value::Float64(double); + let shortest = crate::value::order::decimal_of(&read_back)?; + (shortest.cmp_numeric(&decimal) == std::cmp::Ordering::Equal).then_some(double) +} + /// What the parsed value is for, which decides whether the column's /// declared precision applies. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/core/crates/core/src/export/encoder/cells_encoder.rs b/core/crates/core/src/export/encoder/cells_encoder.rs index d7d8272f6c..ea8b75e45b 100644 --- a/core/crates/core/src/export/encoder/cells_encoder.rs +++ b/core/crates/core/src/export/encoder/cells_encoder.rs @@ -77,7 +77,7 @@ impl RowEncoder for CellsEncoder { /// The storage a value's text does not carry. fn stored_as(value: &Value) -> Option<&'static str> { match value { - Value::Int(_) | Value::UInt(_) | Value::WideInt(_) => Some("int"), + Value::Int32(_) | Value::Int(_) | Value::UInt(_) | Value::WideInt(_) => Some("int"), // A decimal's text is exact, which a binary float is not. Value::Float32(_) | Value::Float64(_) => Some("real"), Value::Bytes(_) => Some("bytes"), diff --git a/core/crates/core/src/export/json.rs b/core/crates/core/src/export/json.rs index 61e89b7faa..46d6b0888e 100644 --- a/core/crates/core/src/export/json.rs +++ b/core/crates/core/src/export/json.rs @@ -52,6 +52,7 @@ impl Serialize for JsonCell<'_> { match self.value { Value::Null | Value::Undecodable(_) => serializer.serialize_none(), Value::Bool(flag) => serializer.serialize_bool(*flag), + Value::Int32(number) => serializer.serialize_i32(*number), Value::Int(number) => serializer.serialize_i64(*number), Value::UInt(number) => serializer.serialize_u64(*number), Value::WideInt(number) => number_token(serializer, &number.to_string()), diff --git a/core/crates/core/src/export/value_text.rs b/core/crates/core/src/export/value_text.rs index 9960e0cc33..1bc7fc7710 100644 --- a/core/crates/core/src/export/value_text.rs +++ b/core/crates/core/src/export/value_text.rs @@ -131,6 +131,7 @@ pub fn value_to_text(v: &Value) -> Option { match v { Value::Null => None, Value::Bool(b) => Some(if *b { "true".to_string() } else { "false".to_string() }), + Value::Int32(i) => Some(i.to_string()), Value::Int(i) => Some(i.to_string()), Value::UInt(i) => Some(i.to_string()), Value::WideInt(i) => Some(i.to_string()), @@ -299,6 +300,7 @@ mod tests { assert_eq!(value_to_text(&Value::Bool(true)), Some("true".to_string())); assert_eq!(value_to_text(&Value::Bool(false)), Some("false".to_string())); assert_eq!(value_to_text(&Value::Int(42)), Some("42".to_string())); + assert_eq!(value_to_text(&Value::Int32(-42)), Some("-42".to_string())); assert_eq!(value_to_text(&Value::UInt(u64::MAX)), Some(u64::MAX.to_string())); assert_eq!(value_to_text(&Value::Float64(1.5)), Some("1.5".to_string())); assert_eq!(value_to_text(&Value::Text("hi".into())), Some("hi".to_string())); diff --git a/core/crates/core/src/export/xlsx.rs b/core/crates/core/src/export/xlsx.rs index 5ee065d76d..761db439af 100644 --- a/core/crates/core/src/export/xlsx.rs +++ b/core/crates/core/src/export/xlsx.rs @@ -211,6 +211,7 @@ fn sheet_number(columns: &[ResultColumn], column: u16, value: &Value) -> Option< match value { Value::Float32(number) => Some(real_number(*number)), Value::Float64(number) => Some(*number), + Value::Int32(number) => Some(f64::from(*number)), Value::Int(_) | Value::UInt(_) | Value::Decimal(_) | Value::WideInt(_) => exact_number(columns, column, value), _ => None, } @@ -383,6 +384,10 @@ mod tests { let columns = cols(&["id"]); assert_eq!(sheet_number(&columns, 0, &Value::Int(42)), Some(42.0)); + assert_eq!( + sheet_number(&columns, 0, &Value::Int32(i32::MIN)), + Some(f64::from(i32::MIN)) + ); assert_eq!( sheet_number(&columns, 0, &Value::Int(-999_999_999_999_999)), Some(-999_999_999_999_999.0) diff --git a/core/crates/core/src/import/pasted_table.rs b/core/crates/core/src/import/pasted_table.rs index f75b0a5d42..444284df0e 100644 --- a/core/crates/core/src/import/pasted_table.rs +++ b/core/crates/core/src/import/pasted_table.rs @@ -422,7 +422,7 @@ fn pasted_cell(field: Option<&PastedField>, column: &ColumnInfo) -> Result, diff --git a/core/crates/core/src/row_batch.rs b/core/crates/core/src/row_batch.rs new file mode 100644 index 0000000000..b13f5d9adb --- /dev/null +++ b/core/crates/core/src/row_batch.rs @@ -0,0 +1,75 @@ +use crate::byte_limit::row_bytes; +use crate::value::Value; + +// Bounded in bytes too, so a batch of wide rows is not held several times over while a sink copies it. +#[derive(Debug, Default)] +pub struct RowBatch { + rows: Vec>, + bytes: u64, +} + +impl RowBatch { + pub const MAX_ROWS: usize = 256; + pub const MAX_BYTES: u64 = 4 * 1024 * 1024; + + pub fn new() -> Self { + Self::default() + } + + pub fn push(&mut self, row: Vec) -> bool { + self.bytes = self.bytes.saturating_add(row_bytes(&row)); + self.rows.push(row); + self.is_full() + } + + pub fn is_full(&self) -> bool { + self.rows.len() >= Self::MAX_ROWS || self.bytes >= Self::MAX_BYTES + } + + pub fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + pub fn len(&self) -> usize { + self.rows.len() + } + + pub fn take(&mut self) -> Vec> { + self.bytes = 0; + std::mem::take(&mut self.rows) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_batch_of_small_rows_is_full_at_256() { + let mut batch = RowBatch::new(); + + let full: Vec = (0..RowBatch::MAX_ROWS) + .map(|n| batch.push(vec![Value::Int(n as i64)])) + .collect(); + + assert!(full[..RowBatch::MAX_ROWS - 1].iter().all(|full| !full)); + assert_eq!(full.last(), Some(&true)); + assert_eq!(batch.take().len(), RowBatch::MAX_ROWS); + assert!(batch.is_empty()); + } + + #[test] + fn a_batch_of_wide_rows_is_full_at_4_mib() { + let mut batch = RowBatch::new(); + let wide = || vec![Value::Text("x".repeat(1024 * 1024))]; + + assert!(!batch.push(wide())); + assert!(!batch.push(wide())); + assert!(!batch.push(wide())); + assert!(batch.push(wide())); + assert_eq!(batch.len(), 4); + + batch.take(); + assert!(!batch.push(vec![Value::Null]), "taking the rows starts the count again"); + } +} diff --git a/core/crates/core/src/run_options.rs b/core/crates/core/src/run_options.rs index fa7ae8b1ec..443c06b827 100644 --- a/core/crates/core/src/run_options.rs +++ b/core/crates/core/src/run_options.rs @@ -2,12 +2,14 @@ use std::time::Duration; use tokio_util::sync::CancellationToken; +use crate::byte_limit::ByteLimit; use crate::row_limit::RowLimit; /// What one run of editor SQL is allowed to do. #[derive(Debug, Clone)] pub struct RunOptions { pub row_limit: RowLimit, + pub byte_limit: ByteLimit, pub deadline: Option, pub cancel: CancellationToken, } @@ -16,6 +18,7 @@ impl RunOptions { pub fn new(cancel: CancellationToken) -> Self { Self { row_limit: RowLimit::EDITOR_DEFAULT, + byte_limit: ByteLimit::EDITOR_DEFAULT, deadline: None, cancel, } @@ -25,6 +28,10 @@ impl RunOptions { Self { row_limit, ..self } } + pub fn with_byte_limit(self, byte_limit: ByteLimit) -> Self { + Self { byte_limit, ..self } + } + pub fn with_deadline(self, deadline: Option) -> Self { Self { deadline, ..self } } diff --git a/core/crates/core/src/save/change_set.rs b/core/crates/core/src/save/change_set.rs index d66b1a5fa2..cc7e91c6f4 100644 --- a/core/crates/core/src/save/change_set.rs +++ b/core/crates/core/src/save/change_set.rs @@ -34,6 +34,8 @@ pub struct RowUpdate { pub key: Vec, /// Column position and its new input. Only the cells that moved. pub assignments: Vec<(usize, CellInput)>, + // A document engine writes an edited number back in the type the cell was read as; SQL engines ignore it. + pub read: Vec<(usize, Value)>, } impl ChangeSet { @@ -70,6 +72,7 @@ mod tests { RowChange::Update(RowUpdate { key: vec![Value::Int(1)], assignments: vec![(1, CellInput::Value(Value::Text("x".to_owned())))], + read: vec![(1, Value::Text("w".to_owned()))], }), RowChange::Delete { key: vec![Value::Int(2)], diff --git a/core/crates/core/src/save/plan_save.rs b/core/crates/core/src/save/plan_save.rs index 432e933da3..d3e54ee0cc 100644 --- a/core/crates/core/src/save/plan_save.rs +++ b/core/crates/core/src/save/plan_save.rs @@ -50,7 +50,15 @@ pub fn plan_save( )? } RowChange::Update(update) => { - let statement = build_update(dialect, table, columns, identity, &update.key, &update.assignments)?; + let statement = build_update( + dialect, + table, + columns, + identity, + &update.key, + &update.assignments, + &update.read, + )?; match sets_partition_only(columns, &update.assignments) { // Its count says how many partitions it reached, not // whether the row is there. @@ -234,6 +242,7 @@ mod tests { RowChange::Update(RowUpdate { key: vec![key], assignments: vec![(1, CellInput::Value(Value::Text(name.to_owned())))], + read: Vec::new(), }) } diff --git a/core/crates/core/src/statement/param_type.rs b/core/crates/core/src/statement/param_type.rs index b37a0694c7..36041e9abb 100644 --- a/core/crates/core/src/statement/param_type.rs +++ b/core/crates/core/src/statement/param_type.rs @@ -51,13 +51,13 @@ impl ParamType { matches!( (self, value), (Self::Bool, Value::Bool(_)) - | (Self::Int16 | Self::Int32 | Self::Int64, Value::Int(_)) + | (Self::Int16 | Self::Int32 | Self::Int64, Value::Int(_) | Value::Int32(_)) | (Self::Int64 | Self::UInt64, Value::UInt(_)) | (Self::Float32, Value::Float32(_)) | (Self::Float64, Value::Float64(_) | Value::Float32(_)) | ( Self::Decimal, - Value::Decimal(_) | Value::Int(_) | Value::UInt(_) | Value::WideInt(_) + Value::Decimal(_) | Value::Int32(_) | Value::Int(_) | Value::UInt(_) | Value::WideInt(_) ) | (Self::Text | Self::FixedText | Self::ServerText(_), Value::Text(_)) | (Self::ServerText(_), Value::Other(_)) @@ -185,6 +185,12 @@ mod tests { assert!(ParamType::Float64.accepts(&Value::Float32(1.5))); assert!(ParamType::Decimal.accepts(&Value::Int(3))); assert!(ParamType::Int64.accepts(&Value::UInt(3))); + assert!(ParamType::Int64.accepts(&Value::Int32(3))); + assert!(ParamType::Decimal.accepts(&Value::Int32(3))); + assert!( + !ParamType::Float64.accepts(&Value::Int32(3)), + "an int32 bound as a double" + ); assert!( !ParamType::Float32.accepts(&Value::Float64(1.5)), "narrowing was accepted" diff --git a/core/crates/core/src/value/identity.rs b/core/crates/core/src/value/identity.rs index e1776cabff..ec485e8fa5 100644 --- a/core/crates/core/src/value/identity.rs +++ b/core/crates/core/src/value/identity.rs @@ -35,6 +35,9 @@ impl Hash for ValueIdentity { pub fn same_value(left: &Value, right: &Value) -> bool { match (left, right) { + (Value::Int32(narrow), Value::Int(wide)) | (Value::Int(wide), Value::Int32(narrow)) => { + i64::from(*narrow) == *wide + } (Value::Float32(left), Value::Float32(right)) => left.to_bits() == right.to_bits(), (Value::Float64(left), Value::Float64(right)) => left.to_bits() == right.to_bits(), (Value::Array(left), Value::Array(right)) => { @@ -45,10 +48,14 @@ pub fn same_value(left: &Value, right: &Value) -> bool { } fn hash_value(value: &Value, state: &mut H) { + if let Value::Int32(narrow) = value { + return hash_value(&Value::Int(i64::from(*narrow)), state); + } std::mem::discriminant(value).hash(state); match value { Value::Null => {} Value::Bool(value) => value.hash(state), + Value::Int32(value) => value.hash(state), Value::Int(value) => value.hash(state), Value::UInt(value) => value.hash(state), Value::WideInt(value) => value.hash(state), @@ -117,4 +124,18 @@ mod tests { .collect(); assert_eq!(identities.len(), 5); } + + #[test] + fn an_int32_is_the_long_of_the_same_number() { + assert!(same_value(&Value::Int32(7), &Value::Int(7))); + assert!(same_value(&Value::Int(7), &Value::Int32(7))); + assert!(!same_value(&Value::Int32(7), &Value::Int(8))); + assert!(!same_value(&Value::Int32(7), &Value::Float64(7.0))); + + let identities: HashSet = [Value::Int32(7), Value::Int(7), Value::Int(8)] + .into_iter() + .map(ValueIdentity::new) + .collect(); + assert_eq!(identities.len(), 2); + } } diff --git a/core/crates/core/src/value/mod.rs b/core/crates/core/src/value/mod.rs index 821f02c539..892f8976ff 100644 --- a/core/crates/core/src/value/mod.rs +++ b/core/crates/core/src/value/mod.rs @@ -29,6 +29,8 @@ pub use wide_integer::{WideInteger, WideIntegerParseError}; pub enum Value { Null, Bool(bool), + // MongoDB's int32, read apart from a long so an edit writes it back as one; other engines read Int. + Int32(i32), Int(i64), UInt(u64), WideInt(WideInteger), @@ -60,7 +62,8 @@ impl Value { /// type holding only such values lines up as numbers do. pub fn is_number(&self) -> bool { match self { - Self::Int(_) + Self::Int32(_) + | Self::Int(_) | Self::UInt(_) | Self::WideInt(_) | Self::Float32(_) @@ -76,6 +79,7 @@ impl Value { match self { Self::Null => "Null", Self::Bool(_) => "Bool", + Self::Int32(_) => "Int32", Self::Int(_) => "Int", Self::UInt(_) => "UInt", Self::WideInt(_) => "WideInt", @@ -101,6 +105,7 @@ impl Value { pub fn as_exact_u64(&self) -> Option { match self { + Self::Int32(value) => u64::try_from(*value).ok(), Self::Int(value) => u64::try_from(*value).ok(), Self::UInt(value) => Some(*value), Self::Decimal(decimal) => match decimal.to_scaled_i128() { @@ -125,6 +130,7 @@ mod tests { fn as_exact_u64_accepts_only_exact_non_negative_integers() { assert_eq!(Value::Int(7).as_exact_u64(), Some(7)); assert_eq!(Value::Int(-1).as_exact_u64(), None); + assert_eq!(Value::Int32(7).as_exact_u64(), Some(7)); assert_eq!(Value::UInt(u64::MAX).as_exact_u64(), Some(u64::MAX)); assert_eq!(Value::Decimal("42".parse().unwrap()).as_exact_u64(), Some(42)); assert_eq!(Value::Decimal("42.0".parse().unwrap()).as_exact_u64(), None); diff --git a/core/crates/core/src/value/order.rs b/core/crates/core/src/value/order.rs index dc50cdc7df..7ea4c358e4 100644 --- a/core/crates/core/src/value/order.rs +++ b/core/crates/core/src/value/order.rs @@ -155,7 +155,8 @@ fn rank(value: &Value) -> u8 { match value { Value::Null => 0, Value::Bool(_) => 1, - Value::Int(_) + Value::Int32(_) + | Value::Int(_) | Value::UInt(_) | Value::WideInt(_) | Value::Float32(_) @@ -217,7 +218,13 @@ fn interval_span(interval: &SqlInterval) -> i128 { fn is_number(value: &Value) -> bool { matches!( value, - Value::Int(_) | Value::UInt(_) | Value::WideInt(_) | Value::Float32(_) | Value::Float64(_) | Value::Decimal(_) + Value::Int32(_) + | Value::Int(_) + | Value::UInt(_) + | Value::WideInt(_) + | Value::Float32(_) + | Value::Float64(_) + | Value::Decimal(_) ) } @@ -257,6 +264,7 @@ fn float_of(value: &Value) -> Option { fn integer_of(value: &Value) -> Option { match value { + Value::Int32(int) => Some(i128::from(*int)), Value::Int(int) => Some(i128::from(*int)), Value::UInt(int) => Some(i128::from(*int)), Value::WideInt(wide) => wide.to_i128(), @@ -399,6 +407,15 @@ mod tests { assert_eq!(order(&Value::Float64(-0.0), &Value::Float64(0.0)), Ordering::Less); } + #[test] + fn an_int32_orders_as_the_number_it_holds() { + assert_eq!(order(&Value::Int32(2), &Value::Int(2)), Ordering::Equal); + assert_eq!(order(&Value::Int32(3), &Value::Int(2)), Ordering::Greater); + assert_eq!(order(&Value::Int32(2), &Value::Float64(2.5)), Ordering::Less); + assert_eq!(order(&Value::Int32(2), &decimal("2.0")), Ordering::Equal); + assert_eq!(order(&Value::Bool(true), &Value::Int32(0)), Ordering::Less); + } + #[test] fn nan_sorts_after_numbers() { let nan = Value::Float64(f64::NAN); diff --git a/core/crates/drivers/cassandra/src/bind.rs b/core/crates/drivers/cassandra/src/bind.rs index ee0bb6ff4a..92f0059b25 100644 --- a/core/crates/drivers/cassandra/src/bind.rs +++ b/core/crates/drivers/cassandra/src/bind.rs @@ -104,6 +104,7 @@ fn native_value(value: &Value, native: &NativeType) -> Option { fn integer(value: &Value) -> Option { match value { + Value::Int32(number) => Some(i64::from(*number)), Value::Int(number) => Some(*number), Value::UInt(number) => i64::try_from(*number).ok(), _ => None, @@ -114,6 +115,7 @@ fn integer(value: &Value) -> Option { /// the app holds it in. NaN and the infinities have none. fn unscaled(value: &Value) -> Option<(BigInt, u32)> { match value { + Value::Int32(number) => Some((BigInt::from(*number), 0)), Value::Int(number) => Some((BigInt::from(*number), 0)), Value::UInt(number) => Some((BigInt::from(*number), 0)), Value::WideInt(number) => Some((number.to_string().parse().ok()?, 0)), diff --git a/core/crates/drivers/cassandra/src/dialect.rs b/core/crates/drivers/cassandra/src/dialect.rs index a82cdcdfe8..fcffec716e 100644 --- a/core/crates/drivers/cassandra/src/dialect.rs +++ b/core/crates/drivers/cassandra/src/dialect.rs @@ -433,6 +433,7 @@ fn scalar_literal(value: &Value) -> Result { Ok(match value { Value::Null => "null".to_owned(), Value::Bool(flag) => flag.to_string(), + Value::Int32(number) => number.to_string(), Value::Int(number) => number.to_string(), Value::UInt(number) => number.to_string(), Value::WideInt(number) => number.to_string(), diff --git a/core/crates/drivers/cassandra/tests/integration/writes.rs b/core/crates/drivers/cassandra/tests/integration/writes.rs index 4235b0d2d2..f4ca7494cb 100644 --- a/core/crates/drivers/cassandra/tests/integration/writes.rs +++ b/core/crates/drivers/cassandra/tests/integration/writes.rs @@ -31,6 +31,7 @@ fn rename(team: &str, id: i64, name: &str) -> RowChange { RowChange::Update(RowUpdate { key: key(team, id), assignments: vec![(2, CellInput::Value(Value::Text(name.to_owned())))], + read: Vec::new(), }) } @@ -165,6 +166,7 @@ async fn a_static_column_is_saved_for_its_partition() { vec![RowChange::Update(RowUpdate { key: key("a", 1), assignments: vec![(motto, CellInput::Value(Value::Text("new".to_owned())))], + read: Vec::new(), })], ) .await diff --git a/core/crates/drivers/clickhouse/src/dialect.rs b/core/crates/drivers/clickhouse/src/dialect.rs index a326be4865..f2472c946f 100644 --- a/core/crates/drivers/clickhouse/src/dialect.rs +++ b/core/crates/drivers/clickhouse/src/dialect.rs @@ -315,6 +315,7 @@ fn literal(value: &Value) -> Result { Ok(match value { Value::Null => "NULL".to_owned(), Value::Bool(flag) => flag.to_string(), + Value::Int32(int) => int.to_string(), Value::Int(int) => int.to_string(), Value::UInt(int) => int.to_string(), Value::WideInt(int) => int.to_string(), diff --git a/core/crates/drivers/clickhouse/src/editor_run.rs b/core/crates/drivers/clickhouse/src/editor_run.rs index 1cbe2a5a63..61a6caba3f 100644 --- a/core/crates/drivers/clickhouse/src/editor_run.rs +++ b/core/crates/drivers/clickhouse/src/editor_run.rs @@ -3,8 +3,8 @@ use std::time::Duration; use tablepro_core::column::ResultColumn; use tablepro_core::{ - DriverError, ErrorCategory, LossPhase, ResultEvent, ResultSink, RowLimit, RunEnd, RunOptions, RunSummary, SinkFlow, - TimeoutPhase, TransactionState, Truncation, UserSql, Value, + DriverError, ErrorCategory, LossPhase, ResultEvent, ResultSink, RowBatch, RowLimit, RunEnd, RunOptions, RunSummary, + SinkFlow, TimeoutPhase, TransactionState, Truncation, UserSql, Value, }; use tokio::time::Instant; use tokio_util::sync::CancellationToken; @@ -18,10 +18,6 @@ use crate::lingering_query::LingeringQuery; use crate::result_origin::{TracedTable, source, traced_table, with_origins}; use crate::result_reader::{ReadEvent, ResultReader}; -/// Rows per `ResultEvent::Rows`. The sink is awaited per batch, so a slow -/// grid slows the read instead of growing a queue. -const BATCH_ROWS: usize = 256; - /// Why a run stopped before its statement did. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum Interrupt { @@ -355,7 +351,7 @@ impl Stop { struct Emitter { sink: Box, limit: RowLimit, - batch: Vec>, + batch: RowBatch, retained: u64, consumer_gone: bool, } @@ -365,7 +361,7 @@ impl Emitter { Self { sink, limit, - batch: Vec::new(), + batch: RowBatch::new(), retained: 0, consumer_gone: false, } @@ -381,9 +377,9 @@ impl Emitter { if self.retained >= self.limit.get() { return true; } - self.batch.push(row); + let full = self.batch.push(row); self.retained += 1; - if self.batch.len() >= BATCH_ROWS { + if full { self.flush().await; } false @@ -424,7 +420,7 @@ impl Emitter { if self.batch.is_empty() { return; } - let rows = std::mem::take(&mut self.batch); + let rows = self.batch.take(); self.send(ResultEvent::Rows(rows)).await; } diff --git a/core/crates/drivers/clickhouse/src/param_value.rs b/core/crates/drivers/clickhouse/src/param_value.rs index b09352453a..e640829e2b 100644 --- a/core/crates/drivers/clickhouse/src/param_value.rs +++ b/core/crates/drivers/clickhouse/src/param_value.rs @@ -30,6 +30,7 @@ impl Serialize for ParamValue<'_> { match self.0 { Value::Null => serializer.serialize_none(), Value::Bool(flag) => serializer.serialize_bool(*flag), + Value::Int32(int) => serializer.serialize_i32(*int), Value::Int(int) => serializer.serialize_i64(*int), Value::UInt(int) => serializer.serialize_u64(*int), Value::WideInt(int) => serializer.serialize_str(&int.to_string()), diff --git a/core/crates/drivers/clickhouse/tests/integration/keys_and_decimals.rs b/core/crates/drivers/clickhouse/tests/integration/keys_and_decimals.rs index d389698fd8..c92de531f5 100644 --- a/core/crates/drivers/clickhouse/tests/integration/keys_and_decimals.rs +++ b/core/crates/drivers/clickhouse/tests/integration/keys_and_decimals.rs @@ -13,6 +13,7 @@ async fn edit(connection: &dyn Connection, name: &str, key: Vec, column: changes: vec![RowChange::Update(RowUpdate { key, assignments: vec![(column, CellInput::Value(value))], + read: Vec::new(), })], }; let batch = planned(connection, &table(name), &changes).await; diff --git a/core/crates/drivers/clickhouse/tests/integration/writes.rs b/core/crates/drivers/clickhouse/tests/integration/writes.rs index 58c024fe7b..e4d7f9ef28 100644 --- a/core/crates/drivers/clickhouse/tests/integration/writes.rs +++ b/core/crates/drivers/clickhouse/tests/integration/writes.rs @@ -77,6 +77,7 @@ async fn duplicate_sorting_key_edit_sends_no_mutation_row_guard_failed() { changes: vec![RowChange::Update(RowUpdate { key: vec![Value::Int(1)], assignments: vec![(1, CellInput::Value(Value::Text("x".to_owned())))], + read: Vec::new(), })], }; let batch = planned(connection.as_ref(), &table(&name), &changes).await; @@ -128,6 +129,7 @@ async fn an_insert_then_an_update_of_the_same_key_stops_the_update() { RowChange::Update(RowUpdate { key: vec![Value::Int(5)], assignments: vec![(1, CellInput::Value(Value::Text("edited".to_owned())))], + read: Vec::new(), }), ], }; @@ -216,6 +218,7 @@ async fn write_statements_never_carry_max_execution_time() { changes: vec![RowChange::Update(RowUpdate { key: vec![Value::Int(1)], assignments: vec![(1, CellInput::Value(Value::Text("b".to_owned())))], + read: Vec::new(), })], }; let batch = planned(connection.as_ref(), &table(&name), &changes).await; diff --git a/core/crates/drivers/duckdb/src/dialect.rs b/core/crates/drivers/duckdb/src/dialect.rs index 74ee7a6858..e9dee7b9f5 100644 --- a/core/crates/drivers/duckdb/src/dialect.rs +++ b/core/crates/drivers/duckdb/src/dialect.rs @@ -134,6 +134,7 @@ impl SqlDialect for DuckDbDialect { match value { Value::Null => Ok("NULL".to_owned()), Value::Bool(flag) => Ok(if *flag { "TRUE" } else { "FALSE" }.to_owned()), + Value::Int32(integer) => Ok(integer.to_string()), Value::Int(integer) => Ok(integer.to_string()), Value::UInt(integer) => Ok(integer.to_string()), Value::WideInt(integer) => Ok(integer.to_string()), @@ -305,7 +306,7 @@ fn value_param(value: Value, column: &ColumnType) -> Result<(Value, ParamType), let param_type = match &value { Value::Null | Value::Text(_) => ParamType::Text, Value::Bool(_) => ParamType::Bool, - Value::Int(_) => ParamType::Int64, + Value::Int32(_) | Value::Int(_) => ParamType::Int64, Value::UInt(_) => ParamType::UInt64, Value::WideInt(_) | Value::Decimal(_) => ParamType::Decimal, Value::Float32(_) => ParamType::Float32, diff --git a/core/crates/drivers/duckdb/src/encode.rs b/core/crates/drivers/duckdb/src/encode.rs index a601547051..3220da4122 100644 --- a/core/crates/drivers/duckdb/src/encode.rs +++ b/core/crates/drivers/duckdb/src/encode.rs @@ -22,6 +22,7 @@ pub(crate) fn bind(value: &Value) -> Result { Ok(match value { Value::Null => Bound::Null, Value::Bool(flag) => Bound::Boolean(*flag), + Value::Int32(integer) => Bound::BigInt(i64::from(*integer)), Value::Int(integer) => Bound::BigInt(*integer), Value::UInt(integer) => Bound::UBigInt(*integer), Value::WideInt(integer) => match (integer.to_i128(), integer.to_u128()) { diff --git a/core/crates/drivers/duckdb/src/result_stream.rs b/core/crates/drivers/duckdb/src/result_stream.rs index 5a0205f0c6..5fc49ed813 100644 --- a/core/crates/drivers/duckdb/src/result_stream.rs +++ b/core/crates/drivers/duckdb/src/result_stream.rs @@ -15,7 +15,7 @@ use tablepro_core::sql_syntax::SqlGrammar; use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptDiagnostic, ScriptPlan, ScriptToken, ScriptTokenKind}; use tablepro_core::statement::{BoundParam, Statement}; use tablepro_core::value::Value; -use tablepro_core::{DriverError, QueryResult, ResultEvent, RowLimit, Truncation, UnsupportedFeature}; +use tablepro_core::{DriverError, QueryResult, ResultEvent, RowBatch, RowLimit, Truncation, UnsupportedFeature}; use tokio::sync::mpsc; use crate::decode::value_at; @@ -25,11 +25,6 @@ use crate::errors::map_error; use crate::instance::{Link, attach_unless_open}; use crate::result_origin::{TracedTable, traced_table}; -/// How many rows travel in one `Rows` event: enough that a large read -/// is not one await per row, few enough that the first rows show at -/// once. -const ROWS_PER_EVENT: usize = 256; - /// What a run of statements did. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub(crate) struct StreamOutcome { @@ -74,15 +69,15 @@ impl Shape { } fn rows(&self, chunk: &StructArray, from: usize, to: usize) -> Vec> { - (from..to) - .map(|row| { - chunk - .columns() - .iter() - .zip(&self.types) - .map(|(column, duck)| value_at(column.as_ref(), duck, row)) - .collect() - }) + (from..to).map(|row| self.row(chunk, row)).collect() + } + + fn row(&self, chunk: &StructArray, row: usize) -> Vec { + chunk + .columns() + .iter() + .zip(&self.types) + .map(|(column, duck)| value_at(column.as_ref(), duck, row)) .collect() } } @@ -344,26 +339,30 @@ fn stream_rows( let keep = limit.map_or(u64::MAX, RowLimit::get); let mut retained = 0u64; let mut truncation = Truncation::Complete; + let mut batch = RowBatch::new(); 'chunks: while let Some(chunk) = next_chunk(running)? { - let mut from = 0; - while from < chunk.len() { + for row in 0..chunk.len() { if retained >= keep { truncation = Truncation::ReadPastLimit; break 'chunks; } - let room = usize::try_from(keep - retained).unwrap_or(usize::MAX); - let to = chunk.len().min(from + ROWS_PER_EVENT).min(from.saturating_add(room)); - let rows = shape.rows(&chunk, from, to); - retained += rows.len() as u64; - outcome.rows += rows.len() as u64; - if !events.send(ResultEvent::Rows(rows)) { + retained += 1; + outcome.rows += 1; + if batch.push(shape.row(&chunk, row)) && !events.send(ResultEvent::Rows(batch.take())) { outcome.consumer_gone = true; return Ok(()); } - from = to; + } + // A chunk's rows show without waiting for the next chunk. + if !batch.is_empty() && !events.send(ResultEvent::Rows(batch.take())) { + outcome.consumer_gone = true; + return Ok(()); } } let mut closing = Vec::new(); + if !batch.is_empty() { + closing.push(ResultEvent::Rows(batch.take())); + } if truncation == Truncation::ReadPastLimit { closing.push(ResultEvent::RowLimitReached { retained }); } diff --git a/core/crates/drivers/mongodb/src/cells.rs b/core/crates/drivers/mongodb/src/cells.rs index f38b3cb6f3..6e85e6625d 100644 --- a/core/crates/drivers/mongodb/src/cells.rs +++ b/core/crates/drivers/mongodb/src/cells.rs @@ -11,7 +11,7 @@ use crate::extended_json::extended_json; pub(crate) fn cell(value: &Bson) -> Value { match value { Bson::Double(number) => Value::Float64(*number), - Bson::Int32(number) => Value::Int(i64::from(*number)), + Bson::Int32(number) => Value::Int32(*number), Bson::Int64(number) => Value::Int(*number), Bson::Decimal128(number) => match number.to_string().parse::() { Ok(decimal) if decimal.is_finite() => Value::Decimal(decimal), @@ -55,7 +55,8 @@ mod tests { #[test] fn plain_values_show_plainly_and_the_rest_as_extended_json() { let id = ObjectId::parse_str("5f1d7a0c9b1e8b2a3c4d5e6f").expect("an id"); - assert_eq!(cell(&Bson::Int32(3)), Value::Int(3)); + assert_eq!(cell(&Bson::Int32(3)), Value::Int32(3)); + assert_eq!(cell(&Bson::Int64(3)), Value::Int(3)); assert_eq!(cell(&Bson::String("a".to_owned())), Value::Text("a".to_owned())); assert_eq!( cell(&Bson::ObjectId(id)), diff --git a/core/crates/drivers/mongodb/src/cursor_rows.rs b/core/crates/drivers/mongodb/src/cursor_rows.rs index 801044669a..eba4173ac4 100644 --- a/core/crates/drivers/mongodb/src/cursor_rows.rs +++ b/core/crates/drivers/mongodb/src/cursor_rows.rs @@ -1,12 +1,19 @@ use bson::Document; use mongodb::{ClientSession, SessionCursor}; -use tablepro_core::{DriverError, LossPhase, ResultEvent, ResultSink, RowLimit, SinkFlow, Truncation}; +use tablepro_core::{ByteLimit, DriverError, LossPhase, ResultEvent, ResultSink, RowLimit, SinkFlow, Truncation}; use crate::call::ReadStop; use crate::endpoint::MongoEndpoint; use crate::rows::{BATCH_ROWS, Layout}; use crate::shell_command::Columns; +// Found columns hold every kept document before the first event, so no sink can stop them at the byte limit. +#[derive(Debug, Clone, Copy)] +pub(crate) struct EditorLimits { + pub(crate) rows: RowLimit, + pub(crate) bytes: ByteLimit, +} + /// Stream a cursor into `sink` as the result set `set_index`. With a /// `limit`, as an editor reads, at most that many documents are kept; with /// none, as an export reads, every one, under columns fixed before the @@ -15,13 +22,13 @@ pub(crate) async fn stream( read: Read<'_>, columns: Columns, sink: &mut dyn ResultSink, - limit: Option, + limit: Option, set_index: usize, ) -> Result<(SinkFlow, u64), DriverError> { let discovered = matches!(columns, Columns::Discovered); match (limit, discovered) { (Some(limit), true) => discovered_kept(read, columns, sink, limit).await, - (Some(limit), false) => named_kept(read, columns, sink, limit, set_index).await, + (Some(limit), false) => named_kept(read, columns, sink, limit.rows, set_index).await, (None, _) => every(read, columns, sink).await, } } @@ -62,6 +69,11 @@ impl<'a> Read<'a> { } } + // The BSON size of the document `next` returned last. + fn current_bytes(&self) -> u64 { + self.cursor.current().as_bytes().len() as u64 + } + /// Up to a batch of the `room` documents left to keep, and how the set /// ends when it does: a full `room` reads one more to say whether the /// result was cut. @@ -93,15 +105,22 @@ async fn discovered_kept( mut read: Read<'_>, columns: Columns, sink: &mut dyn ResultSink, - limit: RowLimit, + limit: EditorLimits, ) -> Result<(SinkFlow, u64), DriverError> { let mut documents = Vec::new(); - let mut more = false; + let mut end = Truncation::Complete; + // The document that crosses the byte limit is kept whole, as the session's sink keeps a row. + let mut held: u64 = 0; while let Some(document) = read.next().await? { - if documents.len() as u64 == limit.get() { - more = true; + if documents.len() as u64 == limit.rows.get() { + end = Truncation::ReadPastLimit; break; } + if held >= limit.bytes.get() { + end = Truncation::ByteLimit; + break; + } + held = held.saturating_add(read.current_bytes()); documents.push(document); } drop(read); @@ -122,10 +141,6 @@ async fn discovered_kept( return Ok((SinkFlow::ConsumerGone, retained)); } } - let end = match more { - true => Truncation::ReadPastLimit, - false => Truncation::Complete, - }; Ok((close(sink, retained, end).await, retained)) } @@ -182,12 +197,18 @@ async fn named_kept( Ok((close(sink, retained, end).await, retained)) } -/// End a kept set: `RowLimitReached` first when the result was cut. +/// End a kept set: `RowLimitReached`, or `ByteLimitReached`, first when the result was cut. async fn close(sink: &mut dyn ResultSink, retained: u64, end: Truncation) -> SinkFlow { match end { Truncation::Complete => finish(sink, retained, Some(retained), Truncation::Complete).await, cut => { - if sink.accept(ResultEvent::RowLimitReached { retained }).await == SinkFlow::ConsumerGone { + let reached = match cut { + Truncation::ByteLimit => ResultEvent::ByteLimitReached { retained }, + Truncation::Complete | Truncation::ServerStopped | Truncation::ReadPastLimit => { + ResultEvent::RowLimitReached { retained } + } + }; + if sink.accept(reached).await == SinkFlow::ConsumerGone { return SinkFlow::ConsumerGone; } finish(sink, retained, None, cut).await diff --git a/core/crates/drivers/mongodb/src/dialect.rs b/core/crates/drivers/mongodb/src/dialect.rs index 88fa529e5e..029a52f0b0 100644 --- a/core/crates/drivers/mongodb/src/dialect.rs +++ b/core/crates/drivers/mongodb/src/dialect.rs @@ -46,7 +46,7 @@ impl SqlDialect for MongoDialect { BindTarget::Column(column) => Some(column), BindTarget::Pattern | BindTarget::EngineRowId(_) => None, }; - let param = tablepro_core::document::bind_param(value, column)?; + let param = tablepro_core::document::bind_param(value, column, None)?; Ok(Placeholder::bound(format!("${}", ordinal + 1), param)) } @@ -134,7 +134,7 @@ mod tests { #[test] fn an_exported_insert_leaves_a_null_field_out() { let columns = ["_id", "name", "email"].map(|name| ResultColumn::new(name, ColumnType::unknown())); - let rows = [vec![Value::Int(1), Value::Text("ann".to_owned()), Value::Null]]; + let rows = [vec![Value::Int32(1), Value::Text("ann".to_owned()), Value::Null]]; let sql = render_sql_insert( &MongoDialect, @@ -192,9 +192,13 @@ mod tests { let dialect = MongoDialect; assert_eq!( - dialect.literal(&Value::Int(1), &ColumnType::unknown()).as_deref(), + dialect.literal(&Value::Int32(1), &ColumnType::unknown()).as_deref(), Ok("1") ); + assert_eq!( + dialect.literal(&Value::Int(1), &ColumnType::unknown()).as_deref(), + Ok(r#"NumberLong("1")"#) + ); assert!(dialect.parse_type("int").is_err()); assert!(dialect.parse_expression("1 + 1").is_err()); assert_eq!(dialect.grammar(), SqlGrammar::Mongo); diff --git a/core/crates/drivers/mongodb/src/editor_run.rs b/core/crates/drivers/mongodb/src/editor_run.rs index bc84b37e2a..edc1eb96f5 100644 --- a/core/crates/drivers/mongodb/src/editor_run.rs +++ b/core/crates/drivers/mongodb/src/editor_run.rs @@ -15,6 +15,7 @@ use tokio::time::Instant; use tokio_util::sync::CancellationToken; use crate::call::{SessionKill, release, remaining_millis, session}; +use crate::cursor_rows::EditorLimits; use crate::endpoint::MongoEndpoint; use crate::errors::{stopped_by_kill, syntax_error}; use crate::execute::{Source, start}; @@ -155,7 +156,13 @@ impl EditorRun { *self.database.lock().unwrap_or_else(PoisonError::into_inner) = next.clone(); (Source::Used(next), None) } - source => (source, Some(options.row_limit)), + source => ( + source, + Some(EditorLimits { + rows: options.row_limit, + bytes: options.byte_limit, + }), + ), }; emit( &self.endpoint, diff --git a/core/crates/drivers/mongodb/src/rows.rs b/core/crates/drivers/mongodb/src/rows.rs index 008022f7a9..05973776a0 100644 --- a/core/crates/drivers/mongodb/src/rows.rs +++ b/core/crates/drivers/mongodb/src/rows.rs @@ -9,7 +9,7 @@ use tablepro_core::{DriverError, QueryResult, ResultEvent, ResultSink, RowLimit, use crate::call::ReadStop; use crate::cells::cell; -use crate::cursor_rows::Read; +use crate::cursor_rows::{EditorLimits, Read}; use crate::endpoint::MongoEndpoint; use crate::execute::Source; use crate::field_types::{column_type, shared_type, type_alias}; @@ -162,10 +162,12 @@ pub(crate) async fn emit( source: Source, session: &mut ClientSession, sink: &mut dyn ResultSink, - limit: Option, + limit: Option, sets: &mut usize, stop: &ReadStop, ) -> Result<(SinkFlow, u64), DriverError> { + // A reply's documents are already held, and one reply is at most 16 MiB, so only a cursor stops at the byte limit. + let row_limit = limit.map(|limit| limit.rows); let set_index = *sets; if !matches!( source, @@ -197,7 +199,7 @@ pub(crate) async fn emit( .. } => Err(error), Source::Write { affected, summary, .. } => { - let (flow, rows) = emit_documents(sink, vec![summary], &Columns::Discovered, limit).await; + let (flow, rows) = emit_documents(sink, vec![summary], &Columns::Discovered, row_limit).await; if flow == SinkFlow::ConsumerGone { return Ok((flow, rows)); } @@ -210,9 +212,9 @@ pub(crate) async fn emit( } Source::Count(count) => { let counted = bson::doc! { "count": Bson::Int64(i64::try_from(count).unwrap_or(i64::MAX)) }; - Ok(emit_documents(sink, vec![counted], &Columns::Discovered, limit).await) + Ok(emit_documents(sink, vec![counted], &Columns::Discovered, row_limit).await) } - Source::Documents { documents, columns } => Ok(emit_documents(sink, documents, &columns, limit).await), + Source::Documents { documents, columns } => Ok(emit_documents(sink, documents, &columns, row_limit).await), Source::Cursor { cursor, columns } => { let read = Read::new(endpoint, *cursor, session, stop); crate::cursor_rows::stream(read, columns, sink, limit, set_index).await @@ -291,7 +293,7 @@ mod tests { assert_eq!( layout.row(&doc! { "b": false, "_id": 4_i32 }), - [Value::Int(4), Value::Null, Value::Bool(false)] + [Value::Int32(4), Value::Null, Value::Bool(false)] ); let names: Vec = layout.columns().iter().map(|column| column.name.clone()).collect(); assert_eq!(names, ["_id", "a", "b"]); @@ -321,8 +323,8 @@ mod tests { layout.row(&doc! { "a": 1_i32, "__field_2": 3_i32, "n": 4_i64 }), [ Value::Null, - Value::Int(1), - Value::Int(3), + Value::Int32(1), + Value::Int32(3), Value::Json(tablepro_core::value::JsonText::parse(r#"{"$numberLong": "4"}"#.to_owned()).expect("JSON")) ] ); diff --git a/core/crates/drivers/mongodb/src/to_bson.rs b/core/crates/drivers/mongodb/src/to_bson.rs index aa490ffd85..f3795f59e2 100644 --- a/core/crates/drivers/mongodb/src/to_bson.rs +++ b/core/crates/drivers/mongodb/src/to_bson.rs @@ -62,6 +62,7 @@ pub(crate) fn param_bson(param: &BoundParam) -> Converted { Ok(match param.value() { Value::Null => Bson::Null, Value::Bool(flag) => Bson::Boolean(*flag), + Value::Int32(number) => Bson::Int32(*number), Value::Int(number) => match (param.param_type(), i32::try_from(*number)) { (ParamType::Int32, Ok(small)) => Bson::Int32(small), _ => Bson::Int64(*number), @@ -471,25 +472,44 @@ mod tests { #[test] fn a_bound_value_keeps_its_columns_type() { - let int = bind_param(Value::Int(5), None).expect("a long"); + let int = bind_param(Value::Int(5), None, None).expect("a long"); assert_eq!(param_bson(&int), Ok(Bson::Int64(5))); + let int32 = bind_param(Value::Int32(5), None, None).expect("an int"); + assert_eq!(param_bson(&int32), Ok(Bson::Int32(5))); let id = Value::Other(Box::new(OtherValue { type_name: "objectId".to_owned(), text: "5f1d7a0c9b1e8b2a3c4d5e6f".to_owned(), })); assert!(matches!( - param_bson(&bind_param(id, None).expect("an id")), + param_bson(&bind_param(id, None, None).expect("an id")), Ok(Bson::ObjectId(_)) )); let bad = Value::Other(Box::new(OtherValue { type_name: "objectId".to_owned(), text: "nope".to_owned(), })); - assert!(param_bson(&bind_param(bad, None).expect("text")).is_err()); + assert!(param_bson(&bind_param(bad, None, None).expect("text")).is_err()); let json = Value::Json(JsonText::parse(r#"{"a": {"$numberLong": "2"}}"#.to_owned()).expect("JSON")); assert_eq!( - param_bson(&bind_param(json, None).expect("JSON")), + param_bson(&bind_param(json, None, None).expect("JSON")), Ok(Bson::Document(bson::doc! { "a": 2_i64 })) ); } + + #[test] + fn an_edited_number_keeps_the_type_it_was_read_as() { + let long = crate::field_types::field_column("qty", &["int", "long"].into_iter().collect()); + let edited = + |value: Value, read: Option<&Value>| param_bson(&bind_param(value, Some(&long), read).expect("a number")); + + assert_eq!(edited(Value::Int(7), Some(&Value::Int32(3))), Ok(Bson::Int32(7))); + assert_eq!(edited(Value::Int(7), Some(&Value::Int(3))), Ok(Bson::Int64(7))); + assert_eq!( + edited(Value::Int(3_000_000_000), Some(&Value::Int32(3))), + Ok(Bson::Int64(3_000_000_000)), + "a number past an int32 kept the type it no longer fits" + ); + assert_eq!(edited(Value::Int(7), None), Ok(Bson::Int64(7))); + assert_eq!(edited(Value::Int(7), Some(&Value::Null)), Ok(Bson::Int64(7))); + } } diff --git a/core/crates/drivers/mongodb/tests/integration/conformance_engine.rs b/core/crates/drivers/mongodb/tests/integration/conformance_engine.rs index 92a2c42630..5573ed6828 100644 --- a/core/crates/drivers/mongodb/tests/integration/conformance_engine.rs +++ b/core/crates/drivers/mongodb/tests/integration/conformance_engine.rs @@ -127,6 +127,7 @@ impl DocumentEngine for MongoEngine { } match rows(&events).first().and_then(|row| row.first()) { None => Ok(0), + Some(Value::Int32(count)) => u64::try_from(*count).map_err(ConformanceFailure::setup), Some(Value::Int(count)) => u64::try_from(*count).map_err(ConformanceFailure::setup), Some(other) => Err(ConformanceFailure::setup(format!("$currentOp counted {other:?}"))), } diff --git a/core/crates/drivers/mongodb/tests/integration/statements.rs b/core/crates/drivers/mongodb/tests/integration/statements.rs index c7bf0246de..16ffa01922 100644 --- a/core/crates/drivers/mongodb/tests/integration/statements.rs +++ b/core/crates/drivers/mongodb/tests/integration/statements.rs @@ -5,9 +5,10 @@ use std::str::FromStr; use secrecy::SecretString; +use tablepro_core::browse::SortKey; use tablepro_core::browse::{PageRead, PageRequest, read_page}; use tablepro_core::column::ColumnKind; -use tablepro_core::edit::CellInput; +use tablepro_core::edit::{CellInput, EditContext, parse_edit_text}; use tablepro_core::filter::FilterSet; use tablepro_core::meta::{ObjectKind, ObjectScope, PagePosition, RowIdentity, TableRef}; use tablepro_core::save::{ChangeSet, RowChange, RowUpdate, plan_save}; @@ -100,8 +101,8 @@ async fn a_find_reads_the_fields_it_projects_in_the_order_it_sorts() { assert_eq!( rows(&events), [ - vec![Value::Int(3), Value::Text("three".to_owned())], - vec![Value::Int(2), Value::Text("two".to_owned())], + vec![Value::Int32(3), Value::Text("three".to_owned())], + vec![Value::Int32(2), Value::Text("two".to_owned())], ] ); @@ -274,7 +275,7 @@ async fn values_reach_the_grid_as_their_own_types() { ); let row = rows(&events).pop().expect("the document"); let at = chrono::DateTime::parse_from_rfc3339("2024-05-06T07:08:09.010Z").expect("a time"); - assert_eq!(row[0], Value::Int(1)); + assert_eq!(row[0], Value::Int32(1)); assert_eq!( row[1], Value::Other(Box::new(OtherValue { @@ -340,11 +341,12 @@ async fn a_nested_field_edited_as_json_keeps_its_types() { &identity, &ChangeSet { changes: vec![RowChange::Update(RowUpdate { - key: vec![Value::Int(1)], + key: vec![Value::Int32(1)], assignments: vec![( nested, CellInput::Value(Value::Json(JsonText::parse(edited.to_owned()).expect("JSON"))), )], + read: Vec::new(), })], }, ) @@ -373,6 +375,131 @@ async fn a_nested_field_edited_as_json_keeps_its_types() { assert_eq!(rows(&events), [vec![Value::Int(1)]], "the edit kept the BSON types"); } +// Each edit carries the value its cell was read as, as a table tab's save does. +fn number_edits( + rows: &[Vec], + columns: &[ColumnInfo], + edits: [(i32, [(usize, &str); 2]); N], +) -> Vec { + edits + .into_iter() + .map(|(id, cells)| { + let row = rows + .iter() + .find(|row| row.first() == Some(&Value::Int32(id))) + .expect("the document"); + let mut assignments = Vec::new(); + let mut read = Vec::new(); + for (column, text) in cells { + let edited = parse_edit_text(text, &columns[column], EditContext::UpdateRow, Some(&row[column])) + .expect("a number"); + assignments.push((column, edited)); + read.push((column, row[column].clone())); + } + RowChange::Update(RowUpdate { + key: vec![row[0].clone()], + assignments, + read, + }) + }) + .collect() +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn an_edited_number_keeps_its_own_bson_type() { + let fixture = start().await; + let connection = connect(&fixture).await; + let session = editor(connection.as_ref()).await; + let collection = unique("widths"); + // The census reads the first 1,000 documents, so the int32 values after them sit in a long and a double column. + let sampled: Vec = (1..=1_000) + .map(|id| format!("{{_id: {id}, qty: NumberLong({id}), price: {id}.5}}")) + .collect(); + let past = "{_id: 1001, qty: NumberInt(3), price: 7}, {_id: 1002, qty: NumberInt(5), price: 9}"; + let (inserted, _) = run( + session.as_ref(), + &format!( + "db.getCollection('{collection}').insertMany([{}, {past}])", + sampled.join(", ") + ), + ) + .await; + assert_eq!(inserted.end, RunEnd::Completed); + let table = TableRef::new(Some(DATABASE.to_owned()), collection.clone()); + let columns = connection + .fetch_columns(&table, within(CALL)) + .await + .expect("the fields"); + let types: Vec<&str> = columns.iter().map(|column| column.column_type.name.as_str()).collect(); + assert_eq!(types, ["int", "long", "double"], "the census saw one type per field"); + let identity = connection + .row_identity(&table, within(CALL)) + .await + .expect("an identity"); + let request = PageRequest { + table: &table, + columns: &columns, + filter: &FilterSet::default(), + sort: &[SortKey { + column: 0, + ascending: false, + }], + identity: &identity, + position: &PagePosition::First, + limit: 3, + }; + let read = read_page::>(connection.as_ref(), &request, &within(CALL)) + .await + .expect("a page"); + let changes = number_edits( + &read.result.rows, + &columns, + [ + (1002, [(1, "3000000000"), (2, "9.5")]), + (1001, [(1, "4"), (2, "8")]), + (1000, [(1, "6"), (2, "2.5")]), + ], + ); + let plan = plan_save( + &*connection.dialect(), + &table, + &columns, + &identity, + &ChangeSet { changes }, + ) + .expect("a save plan"); + let report = connection + .write_batch(&plan.batch, within(CALL)) + .await + .expect("the save"); + assert_eq!(report.rows_affected, [Some(1), Some(1), Some(1)]); + + for (conditions, why) in [ + ( + "{_id: 1001}, {qty: 4}, {qty: {$type: 'int'}}, {price: 8}, {price: {$type: 'int'}}", + "an int32 past the census took its column's type", + ), + ( + "{_id: 1000}, {qty: 6}, {qty: {$type: 'long'}}, {price: 2.5}, {price: {$type: 'double'}}", + "a number of its column's own type changed type", + ), + ( + "{_id: 1002}, {qty: NumberLong('3000000000')}, {qty: {$type: 'long'}}, {price: 9.5}, \ + {price: {$type: 'double'}}", + "an int32 edited past its range did not take the column's type", + ), + ] { + let (typed, events) = run( + session.as_ref(), + &format!("db.getCollection('{collection}').countDocuments({{$and: [{conditions}]}})"), + ) + .await; + assert_eq!(typed.end, RunEnd::Completed); + assert_eq!(rows(&events), [vec![Value::Int(1)]], "{why}"); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[ignore = "requires docker"] async fn a_result_set_event_names_the_fields_a_later_document_brings() { @@ -450,6 +577,7 @@ async fn a_field_of_two_number_types_is_json_and_an_edit_keeps_its_type() { qty, CellInput::Value(Value::Json(JsonText::parse("4".to_owned()).expect("JSON"))), )], + read: Vec::new(), })], }, ) diff --git a/core/crates/drivers/mongodb/tests/integration/support.rs b/core/crates/drivers/mongodb/tests/integration/support.rs index ce31d71e6b..8b433e6ade 100644 --- a/core/crates/drivers/mongodb/tests/integration/support.rs +++ b/core/crates/drivers/mongodb/tests/integration/support.rs @@ -183,6 +183,7 @@ pub async fn save_field( changes: vec![RowChange::Update(RowUpdate { key: vec![key], assignments: vec![(column, CellInput::Value(value))], + read: Vec::new(), })], }; let plan = plan_save(&*connection.dialect(), table, &columns, &identity, &changes).expect("a save plan"); diff --git a/core/crates/drivers/mssql/src/dialect.rs b/core/crates/drivers/mssql/src/dialect.rs index 83a545d121..578b12cac3 100644 --- a/core/crates/drivers/mssql/src/dialect.rs +++ b/core/crates/drivers/mssql/src/dialect.rs @@ -132,6 +132,7 @@ impl SqlDialect for MssqlDialect { Value::Null => Ok("NULL".to_owned()), // BIT takes 0 or 1; there is no boolean literal. Value::Bool(flag) => Ok(if *flag { "1" } else { "0" }.to_owned()), + Value::Int32(integer) => Ok(integer.to_string()), Value::Int(integer) => Ok(integer.to_string()), Value::UInt(integer) => Ok(integer.to_string()), Value::WideInt(integer) => Ok(integer.to_string()), diff --git a/core/crates/drivers/mssql/src/editor_run.rs b/core/crates/drivers/mssql/src/editor_run.rs index 7e4a9c0f86..bb92febcb7 100644 --- a/core/crates/drivers/mssql/src/editor_run.rs +++ b/core/crates/drivers/mssql/src/editor_run.rs @@ -4,8 +4,8 @@ use tablepro_core::column::ResultColumn; use tablepro_core::sql_syntax::SqlGrammar; use tablepro_core::sql_syntax::script::{BatchErrorPolicy, LexicalSettings, ScriptPlan}; use tablepro_core::{ - DriverError, ResultEvent, ResultSink, RowLimit, RunEnd, SinkFlow, StopReason, StopRequest, TransactionState, - Truncation, Value, + DriverError, ResultEvent, ResultSink, RowBatch, RowLimit, RunEnd, SinkFlow, StopReason, StopRequest, + TransactionState, Truncation, Value, }; use tiberius::QueryItem; @@ -18,10 +18,6 @@ use crate::physical::MssqlClient; use crate::result_origin::describe; use crate::transport_io::ReadTally; -/// Rows per `ResultEvent::Rows`. The sink is awaited per batch, so a slow -/// grid slows the read instead of growing a queue. -const BATCH_ROWS: usize = 256; - /// How a run ended, and whether its connection can take the next one. #[derive(Debug)] pub(crate) struct ScriptOutcome { @@ -44,7 +40,7 @@ pub(crate) struct EditorRun<'a> { struct OpenSet { retained: u64, total: u64, - batch: Vec>, + batch: RowBatch, limit_reported: bool, } @@ -190,20 +186,18 @@ async fn keep( ) -> SinkFlow { set.total += 1; if set.retained < limit { - set.batch.push(values()); + let full = set.batch.push(values()); set.retained += 1; - if set.batch.len() < BATCH_ROWS { + if !full { return SinkFlow::Continue; } - return sink.accept(ResultEvent::Rows(std::mem::take(&mut set.batch))).await; + return sink.accept(ResultEvent::Rows(set.batch.take())).await; } if set.limit_reported { return SinkFlow::Continue; } set.limit_reported = true; - if !set.batch.is_empty() - && sink.accept(ResultEvent::Rows(std::mem::take(&mut set.batch))).await == SinkFlow::ConsumerGone - { + if !set.batch.is_empty() && sink.accept(ResultEvent::Rows(set.batch.take())).await == SinkFlow::ConsumerGone { return SinkFlow::ConsumerGone; } sink.accept(ResultEvent::RowLimitReached { retained: set.retained }) @@ -211,9 +205,7 @@ async fn keep( } async fn finish(sink: &mut dyn ResultSink, mut set: OpenSet) -> SinkFlow { - if !set.batch.is_empty() - && sink.accept(ResultEvent::Rows(std::mem::take(&mut set.batch))).await == SinkFlow::ConsumerGone - { + if !set.batch.is_empty() && sink.accept(ResultEvent::Rows(set.batch.take())).await == SinkFlow::ConsumerGone { return SinkFlow::ConsumerGone; } sink.accept(ResultEvent::ResultSetFinished { diff --git a/core/crates/drivers/mssql/src/encode.rs b/core/crates/drivers/mssql/src/encode.rs index 12948ecc04..2f54f64182 100644 --- a/core/crates/drivers/mssql/src/encode.rs +++ b/core/crates/drivers/mssql/src/encode.rs @@ -158,6 +158,7 @@ fn text_data(text: String) -> ColumnData<'static> { /// say whether it fits. fn whole(param_type: &ParamType, value: &Value) -> Result { match value { + Value::Int32(integer) => Ok(i128::from(*integer)), Value::Int(integer) => Ok(i128::from(*integer)), Value::UInt(integer) => Ok(i128::from(*integer)), other => Err(refused(param_type, other)), @@ -166,6 +167,7 @@ fn whole(param_type: &ParamType, value: &Value) -> Result { fn numeric(param_type: &ParamType, value: &Value) -> Result { let (mantissa, scale) = match value { + Value::Int32(integer) => (i128::from(*integer), 0), Value::Int(integer) => (i128::from(*integer), 0), Value::UInt(integer) => (i128::from(*integer), 0), Value::WideInt(integer) => ( diff --git a/core/crates/drivers/mssql/tests/integration/write_batch.rs b/core/crates/drivers/mssql/tests/integration/write_batch.rs index 1a0393d35e..6905bb6e57 100644 --- a/core/crates/drivers/mssql/tests/integration/write_batch.rs +++ b/core/crates/drivers/mssql/tests/integration/write_batch.rs @@ -23,6 +23,7 @@ async fn update_label(connection: &dyn tablepro_core::Connection, table: &TableR changes: vec![RowChange::Update(RowUpdate { key: vec![key], assignments: vec![(label, CellInput::Value(Value::Text("changed".to_owned())))], + read: Vec::new(), })], }; plan_save(connection.dialect().as_ref(), table, &columns, &identity, &changes) @@ -54,6 +55,7 @@ async fn edited_xml_cell_saves() { changes: vec![RowChange::Update(RowUpdate { key: vec![Value::Int(1)], assignments: vec![(1, CellInput::Value(edited))], + read: Vec::new(), })], }; let save = plan_save(connection.dialect().as_ref(), &table, &columns, &identity, &changes) @@ -94,6 +96,7 @@ async fn datetime_key_with_a_third_of_a_millisecond_saves() { changes: vec![RowChange::Update(RowUpdate { key: vec![key], assignments: vec![(1, CellInput::Value(Value::Text("changed".to_owned())))], + read: Vec::new(), })], }; let save = plan_save(connection.dialect().as_ref(), &table, &columns, &identity, &changes) diff --git a/core/crates/drivers/mysql/src/dialect.rs b/core/crates/drivers/mysql/src/dialect.rs index 4ef98a7b6a..cbdc36d0b9 100644 --- a/core/crates/drivers/mysql/src/dialect.rs +++ b/core/crates/drivers/mysql/src/dialect.rs @@ -362,6 +362,9 @@ fn adapt(value: Value, param_type: ParamType) -> Result<(Value, ParamType), Bind (Value::Int(number), ParamType::UInt64) if number >= 0 => { (Value::UInt(number.unsigned_abs()), ParamType::UInt64) } + (Value::Int32(number), ParamType::UInt64) if number >= 0 => { + (Value::UInt(u64::from(number.unsigned_abs())), ParamType::UInt64) + } (Value::Other(other), _) => (Value::Text(other.text), ParamType::Text), (value, param_type) => (value, param_type), }) @@ -374,6 +377,7 @@ fn literal(value: &Value) -> Result { true => "TRUE".to_owned(), false => "FALSE".to_owned(), }, + Value::Int32(integer) => integer.to_string(), Value::Int(integer) => integer.to_string(), Value::UInt(integer) => integer.to_string(), Value::WideInt(integer) => integer.to_string(), diff --git a/core/crates/drivers/mysql/src/editor_run.rs b/core/crates/drivers/mysql/src/editor_run.rs index 1f166cedc5..03f712a575 100644 --- a/core/crates/drivers/mysql/src/editor_run.rs +++ b/core/crates/drivers/mysql/src/editor_run.rs @@ -7,7 +7,7 @@ use sqlx::{AssertSqlSafe, Either, Executor as _, Row as _, SqlSafeStr as _, Stat use tablepro_core::column::ResultColumn; use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptTokenKind}; use tablepro_core::sql_syntax::{SourceTable, SqlGrammar, StatementAccess, source_table, statement_access}; -use tablepro_core::{DriverError, ErrorCategory, ResultEvent, ResultSink, RowLimit, SinkFlow, Truncation, Value}; +use tablepro_core::{DriverError, ErrorCategory, ResultEvent, ResultSink, RowBatch, RowLimit, SinkFlow, Truncation}; use tokio_util::sync::CancellationToken; use crate::catalog::result_column; @@ -20,8 +20,6 @@ use crate::mysql_wire_format::MySqlWireFormat; use crate::script_split; use crate::session_statements::SelectLimit; -const ROWS_PER_EVENT: usize = 256; - const SELECT_OPENERS: [&str; 4] = ["SELECT", "WITH", "TABLE", "VALUES"]; // Ordered, so a text the server runs whole takes its strictest statement's. @@ -217,7 +215,7 @@ async fn described( } struct OpenSet { - batch: Vec>, + batch: RowBatch, retained: u64, seen: u64, } @@ -268,7 +266,7 @@ impl Emitter<'_> { self.release().await; self.open_set(row.columns()).await; self.open = Some(OpenSet { - batch: Vec::new(), + batch: RowBatch::new(), retained: 0, seen: 0, }); @@ -281,11 +279,12 @@ impl Emitter<'_> { if set.retained >= limit { return; } - set.batch + let full = set + .batch .push(decode_row(row, MySqlWireFormat::Text, MySqlTimestampMode::SessionLocal)); set.retained += 1; - if set.batch.len() >= ROWS_PER_EVENT { - let rows = std::mem::take(&mut set.batch); + if full { + let rows = set.batch.take(); self.send(ResultEvent::Rows(rows)).await; } } @@ -294,7 +293,7 @@ impl Emitter<'_> { if self.gone { return; } - let Some(set) = self.open.take() else { + let Some(mut set) = self.open.take() else { if may_be_an_empty_set { self.held_back = Some(rows_affected); return; @@ -307,7 +306,7 @@ impl Emitter<'_> { return; }; if !set.batch.is_empty() { - self.send(ResultEvent::Rows(set.batch)).await; + self.send(ResultEvent::Rows(set.batch.take())).await; } let (truncation, total_rows) = truncation(set.seen, self.limit); if truncation != Truncation::Complete { diff --git a/core/crates/drivers/mysql/src/encode.rs b/core/crates/drivers/mysql/src/encode.rs index de7219ba1a..be6d3e234a 100644 --- a/core/crates/drivers/mysql/src/encode.rs +++ b/core/crates/drivers/mysql/src/encode.rs @@ -27,6 +27,7 @@ fn bind_one<'q>(query: MySqlQuery<'q>, param: &BoundParam) -> Result query.bind(None::), (_, Value::Bool(flag)) => query.bind(*flag), + (_, Value::Int32(integer)) => query.bind(*integer), (_, Value::Int(integer)) => query.bind(*integer), (_, Value::UInt(integer)) => query.bind(*integer), (ParamType::Float32, Value::Float32(real)) => query.bind(*real), diff --git a/core/crates/drivers/mysql/tests/integration/decode.rs b/core/crates/drivers/mysql/tests/integration/decode.rs index 0da732d6b3..710d7f7378 100644 --- a/core/crates/drivers/mysql/tests/integration/decode.rs +++ b/core/crates/drivers/mysql/tests/integration/decode.rs @@ -94,6 +94,7 @@ async fn edit_by_max_id(connection: &Arc, id_type: &str, id: Val changes: vec![RowChange::Update(RowUpdate { key: vec![read[0][0].clone()], assignments: vec![(label, CellInput::Value(Value::Text("after".to_owned())))], + read: Vec::new(), })], }; let dialect = connection.dialect(); diff --git a/core/crates/drivers/postgres/src/codec/pg_param.rs b/core/crates/drivers/postgres/src/codec/pg_param.rs index 2aa4a326d8..cd1097b1ec 100644 --- a/core/crates/drivers/postgres/src/codec/pg_param.rs +++ b/core/crates/drivers/postgres/src/codec/pg_param.rs @@ -106,6 +106,9 @@ fn encode(param: &BoundParam, out: &mut BytesMut) -> Result (ParamType::Int16, Value::Int(int)) => wire::int2_to_sql(i16::try_from(*int)?, out), (ParamType::Int32, Value::Int(int)) => wire::int4_to_sql(i32::try_from(*int)?, out), (ParamType::Int64, Value::Int(int)) => wire::int8_to_sql(*int, out), + (ParamType::Int16, Value::Int32(int)) => wire::int2_to_sql(i16::try_from(*int)?, out), + (ParamType::Int32, Value::Int32(int)) => wire::int4_to_sql(*int, out), + (ParamType::Int64, Value::Int32(int)) => wire::int8_to_sql(i64::from(*int), out), (ParamType::Int64, Value::UInt(int)) => wire::int8_to_sql(i64::try_from(*int)?, out), (ParamType::Float32, Value::Float32(float)) => wire::float4_to_sql(*float, out), (ParamType::Float64, Value::Float64(float)) => wire::float8_to_sql(*float, out), @@ -162,6 +165,7 @@ fn micros(time: &SqlTime) -> Result { fn decimal_text(value: &Value) -> Option { match value { Value::Decimal(decimal) => Some(decimal.to_string()), + Value::Int32(int) => Some(int.to_string()), Value::Int(int) => Some(int.to_string()), Value::UInt(int) => Some(int.to_string()), Value::WideInt(int) => Some(int.to_string()), diff --git a/core/crates/drivers/postgres/src/dialect.rs b/core/crates/drivers/postgres/src/dialect.rs index bf3882bc7b..657af44c40 100644 --- a/core/crates/drivers/postgres/src/dialect.rs +++ b/core/crates/drivers/postgres/src/dialect.rs @@ -258,7 +258,7 @@ fn row_id_binding(part: EngineRowIdPart, value: &Value) -> Result { + (EngineRowIdPart::PostgresTableOid, Value::Int32(_) | Value::Int(_) | Value::UInt(_)) => { Binding::cast(ParamType::Int64, named("oid")) } (EngineRowIdPart::PostgresTableOid, _) => Binding::server_text(named("oid")), @@ -348,6 +348,7 @@ fn literal(value: &Value) -> Result { Ok(match value { Value::Null => "NULL".to_owned(), Value::Bool(flag) => flag.to_string(), + Value::Int32(int) => int.to_string(), Value::Int(int) => int.to_string(), Value::UInt(int) => int.to_string(), Value::WideInt(int) => int.to_string(), diff --git a/core/crates/drivers/postgres/src/editor/run.rs b/core/crates/drivers/postgres/src/editor/run.rs index 53cbb7da48..65b9ca4513 100644 --- a/core/crates/drivers/postgres/src/editor/run.rs +++ b/core/crates/drivers/postgres/src/editor/run.rs @@ -4,7 +4,7 @@ use std::sync::atomic::Ordering; use futures_util::StreamExt; use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptToken}; use tablepro_core::sql_syntax::{SourceTable, SqlGrammar}; -use tablepro_core::{DriverError, ResultEvent, ResultSink, RowLimit, RunEnd, SinkFlow, Truncation, Value}; +use tablepro_core::{DriverError, ResultEvent, ResultSink, RowBatch, RowLimit, RunEnd, SinkFlow, Truncation, Value}; use tokio_postgres::SimpleQueryMessage; use tracing::debug; @@ -21,10 +21,6 @@ use crate::cancel::acknowledges_stop; use crate::catalog::text_column; use crate::errors::{map_error, read_only_refusal, sqlstate}; -/// Rows per `ResultEvent::Rows`. The sink is awaited per batch, so a slow -/// grid slows the read instead of growing a queue. -const BATCH_ROWS: usize = 256; - /// What a message is for. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Role { @@ -59,7 +55,7 @@ struct ResultSet { open: bool, retained: u64, total: u64, - batch: Vec>, + batch: RowBatch, limit_reported: bool, } @@ -397,9 +393,9 @@ impl<'a> Run<'a> { None => Value::Null, }) .collect(); - set.batch.push(values); + let full = set.batch.push(values); set.retained += 1; - if set.batch.len() >= BATCH_ROWS { + if full { self.flush(set).await; } } else if !set.limit_reported { @@ -436,7 +432,7 @@ impl<'a> Run<'a> { if set.batch.is_empty() { return; } - let rows = std::mem::take(&mut set.batch); + let rows = set.batch.take(); self.send(ResultEvent::Rows(rows)).await; } diff --git a/core/crates/drivers/postgres/tests/integration/decode.rs b/core/crates/drivers/postgres/tests/integration/decode.rs index ef1b028752..a5f2678c47 100644 --- a/core/crates/drivers/postgres/tests/integration/decode.rs +++ b/core/crates/drivers/postgres/tests/integration/decode.rs @@ -229,6 +229,7 @@ async fn set_null_saves_typed_null() { changes: vec![RowChange::Update(RowUpdate { key: vec![Value::Int(1)], assignments: (1..=4).map(|column| (column, CellInput::Value(Value::Null))).collect(), + read: Vec::new(), })], }; @@ -260,6 +261,7 @@ async fn char10_key_update_affects_one_row() { changes: vec![RowChange::Update(RowUpdate { key: vec![Value::Text("ab".to_owned())], assignments: vec![(1, CellInput::Value(Value::Text("changed".to_owned())))], + read: Vec::new(), })], }; diff --git a/core/crates/drivers/postgres/tests/integration/metadata.rs b/core/crates/drivers/postgres/tests/integration/metadata.rs index 4769651aa2..9fef08b532 100644 --- a/core/crates/drivers/postgres/tests/integration/metadata.rs +++ b/core/crates/drivers/postgres/tests/integration/metadata.rs @@ -490,6 +490,7 @@ async fn stale_ctid_edit_after_vacuum_reuse_row_guard_failed() { changes: vec![RowChange::Update(RowUpdate { key: before[0].clone(), assignments: vec![(0, CellInput::Value(Value::Text("changed".to_owned())))], + read: Vec::new(), })], }; let plan = plan_save(dialect.as_ref(), &target, &columns, &identity, &changes).expect("a plan"); @@ -1370,6 +1371,7 @@ async fn a_save_casts_to_the_built_in_type_a_lookalike_shares_a_name_with() { changes: vec![RowChange::Update(RowUpdate { key: vec![Value::Int(1)], assignments: vec![(1, CellInput::Value(Value::Text("10.0.0.2".to_owned())))], + read: Vec::new(), })], }; let dialect = connection.dialect(); diff --git a/core/crates/drivers/sqlite/src/dialect.rs b/core/crates/drivers/sqlite/src/dialect.rs index 60f6865897..804e403a44 100644 --- a/core/crates/drivers/sqlite/src/dialect.rs +++ b/core/crates/drivers/sqlite/src/dialect.rs @@ -89,6 +89,7 @@ impl SqlDialect for SqliteDialect { match value { Value::Null => Ok("NULL".to_owned()), Value::Bool(flag) => Ok(if *flag { "1" } else { "0" }.to_owned()), + Value::Int32(integer) => Ok(integer.to_string()), Value::Int(integer) => Ok(integer.to_string()), Value::UInt(integer) => Ok(integer.to_string()), Value::WideInt(integer) => Ok(integer.to_string()), @@ -244,7 +245,7 @@ fn value_param(value: Value) -> Result<(Value, ParamType), BindError> { let param_type = match &value { Value::Null | Value::Text(_) => ParamType::Text, Value::Bool(_) => ParamType::Bool, - Value::Int(_) | Value::UInt(_) => ParamType::Int64, + Value::Int32(_) | Value::Int(_) | Value::UInt(_) => ParamType::Int64, Value::WideInt(_) | Value::Decimal(_) => ParamType::Decimal, Value::Float32(_) | Value::Float64(_) => ParamType::Float64, Value::Bytes(_) => ParamType::Bytes, diff --git a/core/crates/drivers/sqlite/src/encode.rs b/core/crates/drivers/sqlite/src/encode.rs index cf4fae41e1..88578074af 100644 --- a/core/crates/drivers/sqlite/src/encode.rs +++ b/core/crates/drivers/sqlite/src/encode.rs @@ -31,6 +31,7 @@ fn bind_one<'q>( Ok(match param.value() { Value::Null => query.bind(None::), Value::Bool(flag) => query.bind(i64::from(*flag)), + Value::Int32(integer) => query.bind(i64::from(*integer)), Value::Int(integer) => query.bind(*integer), // SQLite has no unsigned integer. Past i64 the digits are bound // as text, which keeps them exact where a real would round. diff --git a/core/crates/drivers/sqlite/src/result_stream.rs b/core/crates/drivers/sqlite/src/result_stream.rs index b4b7ca0575..ad6bdd122a 100644 --- a/core/crates/drivers/sqlite/src/result_stream.rs +++ b/core/crates/drivers/sqlite/src/result_stream.rs @@ -17,7 +17,7 @@ use tablepro_core::sql_syntax::script::{ }; use tablepro_core::statement::{BoundParam, Statement}; use tablepro_core::value::Value; -use tablepro_core::{DriverError, QueryResult, ResultEvent, ResultSink, RowLimit, SinkFlow, Truncation}; +use tablepro_core::{DriverError, QueryResult, ResultEvent, ResultSink, RowBatch, RowLimit, SinkFlow, Truncation}; use crate::catalog::{result_column_type, storage_class_type}; use crate::column_origins::column_origins; @@ -25,11 +25,6 @@ use crate::decode::decode_row; use crate::encode::bind_all; use crate::errors::map_error; -/// How many rows travel in one `Rows` event: enough that a large read -/// is not one await per row, few enough that the first rows show at -/// once. -const ROWS_PER_EVENT: usize = 256; - /// What a run of statements did. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub(crate) struct StreamOutcome { @@ -245,7 +240,7 @@ async fn run_statement( } let mut stream = query.fetch(&mut *connection); - let mut batch: Vec> = Vec::with_capacity(ROWS_PER_EVENT); + let mut batch = RowBatch::new(); let mut retained = 0u64; let mut truncation = Truncation::Complete; while let Some(row) = stream.try_next().await.map_err(map_error)? { @@ -265,9 +260,9 @@ async fn run_statement( return Ok(()); } } - batch.push(values); + let full = batch.push(values); retained += 1; - if batch.len() == ROWS_PER_EVENT && gone(sink.accept(ResultEvent::Rows(std::mem::take(&mut batch))).await) { + if full && gone(sink.accept(ResultEvent::Rows(batch.take())).await) { outcome.rows += retained; outcome.consumer_gone = true; return Ok(()); @@ -280,7 +275,7 @@ async fn run_statement( let mut closing = Vec::new(); if !batch.is_empty() { - closing.push(ResultEvent::Rows(batch)); + closing.push(ResultEvent::Rows(batch.take())); } if truncation == Truncation::ReadPastLimit { closing.push(ResultEvent::RowLimitReached { retained }); diff --git a/core/crates/drivers/sqlite/tests/integration/decode.rs b/core/crates/drivers/sqlite/tests/integration/decode.rs index 967b1e1d4f..178a547ce1 100644 --- a/core/crates/drivers/sqlite/tests/integration/decode.rs +++ b/core/crates/drivers/sqlite/tests/integration/decode.rs @@ -113,6 +113,7 @@ async fn strict_table_round_trip() { (3, CellInput::Value(Value::Text("ü'".to_owned()))), (4, CellInput::Value(Value::Bytes(vec![1, 2, 3]))), ], + read: Vec::new(), })], }; let dialect = connection.dialect(); diff --git a/core/crates/ffi/src/connection_url.rs b/core/crates/ffi/src/connection_url.rs index 993fb36439..a70c9b58f8 100644 --- a/core/crates/ffi/src/connection_url.rs +++ b/core/crates/ffi/src/connection_url.rs @@ -1,19 +1,19 @@ use std::path::Path; -use secrecy::ExposeSecret; +use secrecy::{ExposeSecret, SecretString}; use tablepro_core::connection_url::{self, ConnectionUrl}; use tablepro_core::is_local_socket_host; use crate::error::CoreError; use crate::library::{Redacted, TlsModeTag}; -#[derive(Clone, PartialEq, Eq, uniffi::Record)] +#[derive(Clone, uniffi::Record)] pub struct ParsedConnectionUrl { pub driver_id: String, pub host: String, pub port: Option, pub username: String, - pub password: Option, + pub password: Option, // The database name, or for a file engine the file's path. pub database: String, pub tls_mode: Option, @@ -42,8 +42,9 @@ impl std::fmt::Debug for ParsedConnectionUrl { } #[uniffi::export] -pub fn parse_connection_url(url: String) -> Result { - let parsed = connection_url::parse(&url).map_err(|error| CoreError::failed(error.code(), error.to_string()))?; +pub fn parse_connection_url(url: SecretString) -> Result { + let text = url.expose_secret(); + let parsed = connection_url::parse(text).map_err(|error| CoreError::failed(error.code(), error.to_string()))?; if let Some(setting) = unkept_setting(&parsed) { return Err(CoreError::failed( "url.settings_unsupported", @@ -62,7 +63,7 @@ pub fn parse_connection_url(url: String) -> Result ParsedConnectionUrl { - parse_connection_url(url.to_owned()).unwrap() + parse_connection_url(SecretString::from(url)).unwrap() } fn refusal(url: &str) -> String { - parse_connection_url(url.to_owned()).unwrap_err().code().to_owned() + parse_connection_url(SecretString::from(url)) + .unwrap_err() + .code() + .to_owned() } #[test] @@ -115,7 +119,7 @@ mod tests { assert_eq!(parsed.host, "db.example.com"); assert_eq!(parsed.port, Some(6543)); assert_eq!(parsed.username, "app@corp"); - assert_eq!(parsed.password.as_deref(), Some("p@ss")); + assert_eq!(parsed.password.as_ref().map(ExposeSecret::expose_secret), Some("p@ss")); assert_eq!(parsed.database, "shop db"); assert_eq!(parsed.tls_mode, Some(TlsModeTag::VerifyFull)); } diff --git a/core/crates/ffi/src/core_handle.rs b/core/crates/ffi/src/core_handle.rs index 33f763036e..2fac15fbc0 100644 --- a/core/crates/ffi/src/core_handle.rs +++ b/core/crates/ffi/src/core_handle.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; use futures::future::BoxFuture; -use secrecy::{ExposeSecret, SecretString}; +use secrecy::SecretString; use tablepro_core::credentials::{CredentialInteraction, CredentialPrompter, SecretItems, SecretKind, SecretVault}; use tablepro_core::{ ConnectEnvironment, Credentials, DatabaseDriver, DriverError, DriverRegistry, FileEndpoint, LivenessPolicy, @@ -421,7 +421,7 @@ fn fill(request: &mut ConnectRequest, typed: &[(SecretKind, SecretString)]) { return; }; for (kind, secret) in typed { - let secret = SecretString::from(secret.expose_secret().to_owned()); + let secret = secret.clone(); match (kind, &mut *credentials, ssh.as_mut().map(|ssh| &mut ssh.auth)) { (SecretKind::DatabasePassword, Credentials::Password { password, .. }, _) => *password = secret, (SecretKind::SshPassword, _, Some(SshAuth::Password { password })) => *password = secret, @@ -492,6 +492,7 @@ mod tests { use std::sync::Mutex; use async_trait::async_trait; + use secrecy::ExposeSecret; use tablepro_core::credentials::{ CredentialPrompt, PromptPurpose, PromptReason, PromptReply, SecretError, SecretItem, SecretLookup, }; @@ -575,7 +576,7 @@ mod tests { stored.options.dns_seed_list = true; core.store.as_ref().unwrap().upsert_blocking(stored.clone()).unwrap(); let typed = || ConnectionSecrets { - database_password: Some("typed".to_owned()), + database_password: Some(SecretString::from("typed")), ..ConnectionSecrets::default() }; @@ -635,9 +636,9 @@ mod tests { fn secrets(value: &str) -> ConnectionSecrets { ConnectionSecrets { - database_password: Some(value.to_owned()), - ssh_password: Some(value.to_owned()), - ssh_passphrase: Some(value.to_owned()), + database_password: Some(SecretString::from(value)), + ssh_password: Some(SecretString::from(value)), + ssh_passphrase: Some(SecretString::from(value)), } } diff --git a/core/crates/ffi/src/credential_manager.rs b/core/crates/ffi/src/credential_manager.rs index af455a73c9..309cd0280b 100644 --- a/core/crates/ffi/src/credential_manager.rs +++ b/core/crates/ffi/src/credential_manager.rs @@ -1,4 +1,5 @@ use async_trait::async_trait; +use secrecy::zeroize::{Zeroize, Zeroizing}; use secrecy::{ExposeSecret, SecretString}; use tablepro_core::AiProvider; use tablepro_core::credentials::{SecretError, SecretItem, SecretItems, SecretKind, SecretLookup, SecretVault}; @@ -93,7 +94,7 @@ fn failure(action: &str) -> SecretError { } // The blob holds UTF-16, as cmdkey and the Credential Manager control panel write it. -fn decode(blob: &[u8]) -> Result { +fn decode(blob: &[u8]) -> Result { if blob.len() % 2 != 0 { return Err(SecretError::InvalidEncoding); } @@ -101,10 +102,18 @@ fn decode(blob: &[u8]) -> Result { .chunks_exact(2) .map(|pair| u16::from_le_bytes([pair[0], pair[1]])) .collect(); - String::from_utf16(&units).map_err(|_| SecretError::InvalidEncoding) + let units = Zeroizing::new(units); + // Sized exactly, so neither a push nor SecretString's shrink to fit leaves a copy in freed memory. + let mut size = 0; + for decoded in char::decode_utf16(units.iter().copied()) { + size += decoded.map_err(|_| SecretError::InvalidEncoding)?.len_utf8(); + } + let mut text = String::with_capacity(size); + text.extend(char::decode_utf16(units.iter().copied()).flatten()); + Ok(SecretString::from(text)) } -fn read(target: &str) -> Result, SecretError> { +fn read(target: &str) -> Result, SecretError> { let name = wide(target); let mut found: *mut CREDENTIALW = std::ptr::null_mut(); // SAFETY: `name` is NUL-terminated and outlives the call; `found` receives a buffer CredFree releases. @@ -116,16 +125,20 @@ fn read(target: &str) -> Result, SecretError> { } return Err(failure("read")); } - // SAFETY: CredReadW succeeded, so `found` is live until CredFree and a non-null blob holds its size. + // SAFETY: CredReadW succeeded, so `found` is live until CredFree, the process owns it, and a non-null blob + // holds its size. let decoded = unsafe { let credential = &*found; let size = credential.CredentialBlobSize as usize; - let blob = if credential.CredentialBlob.is_null() || size == 0 { - &[][..] + if credential.CredentialBlob.is_null() || size == 0 { + decode(&[]) } else { - std::slice::from_raw_parts(credential.CredentialBlob, size) - }; - decode(blob) + let blob = std::slice::from_raw_parts_mut(credential.CredentialBlob, size); + let decoded = decode(blob); + // CredFree releases the block as it is, so the decrypted text would stay in freed memory. + blob.zeroize(); + decoded + } }; // SAFETY: `found` came from CredReadW and is freed once. unsafe { CredFree(found.cast_const().cast()) }; @@ -136,7 +149,9 @@ fn write(target: &str, secret: &str, label: &str) -> Result<(), SecretError> { let name = wide(target); let mut comment = wide(label); let mut user = wide("TablePro"); - let mut blob: Vec = secret.encode_utf16().flat_map(u16::to_le_bytes).collect(); + // Sized up front: a collect that grew would leave parts of the secret in the buffers it gave up. + let mut blob: Zeroizing> = Zeroizing::new(Vec::with_capacity(secret.encode_utf16().count() * 2)); + blob.extend(secret.encode_utf16().flat_map(u16::to_le_bytes)); let size = u32::try_from(blob.len()).map_err(|_| SecretError::Backend { detail: "the secret is too long for Credential Manager".to_owned(), service_message: None, @@ -156,7 +171,6 @@ fn write(target: &str, secret: &str, label: &str) -> Result<(), SecretError> { credential.UserName = user.as_mut_ptr(); // SAFETY: every pointer in `credential` points into a buffer that outlives the call. let ok = unsafe { CredWriteW(&raw const credential, 0) }; - blob.fill(0); if ok == 0 { return Err(failure("write")); } @@ -178,23 +192,22 @@ fn delete(target: &str) -> Result<(), SecretError> { impl SecretVault for CredentialManager { async fn store(&self, id: Uuid, kind: SecretKind, secret: &SecretString, label: &str) -> Result<(), SecretError> { let target = self.connection_target(id, kind); - let (secret, label) = (secret.expose_secret().to_owned(), label.to_owned()); - self.blocking(move || write(&target, &secret, &label)).await + let (secret, label) = (secret.clone(), label.to_owned()); + self.blocking(move || write(&target, secret.expose_secret(), &label)) + .await } async fn load(&self, id: Uuid, kind: SecretKind) -> Result { let target = self.connection_target(id, kind); let found = self.blocking(move || read(&target)).await?; - Ok(found.map_or(SecretLookup::NotStored, |secret| { - SecretLookup::Found(SecretString::from(secret)) - })) + Ok(found.map_or(SecretLookup::NotStored, SecretLookup::Found)) } async fn relabel(&self, id: Uuid, kind: SecretKind, label: &str) -> Result<(), SecretError> { let target = self.connection_target(id, kind); let label = label.to_owned(); self.blocking(move || match read(&target)? { - Some(secret) => write(&target, &secret, &label), + Some(secret) => write(&target, secret.expose_secret(), &label), None => Ok(()), }) .await @@ -219,15 +232,15 @@ impl SecretVault for CredentialManager { impl SecretItems for CredentialManager { async fn load_item(&self, item: SecretItem) -> Result, SecretError> { let target = self.item_target(item); - let found = self.blocking(move || read(&target)).await?; - Ok(found.map(SecretString::from)) + self.blocking(move || read(&target)).await } async fn store_item(&self, item: SecretItem, secret: &SecretString) -> Result<(), SecretError> { let target = self.item_target(item); - let secret = secret.expose_secret().to_owned(); + let secret = secret.clone(); let label = english_item_label(item); - self.blocking(move || write(&target, &secret, &label)).await + self.blocking(move || write(&target, secret.expose_secret(), &label)) + .await } async fn delete_item(&self, item: SecretItem) -> Result<(), SecretError> { diff --git a/core/crates/ffi/src/editor.rs b/core/crates/ffi/src/editor.rs index 2cdf839eee..1efd9a6bcb 100644 --- a/core/crates/ffi/src/editor.rs +++ b/core/crates/ffi/src/editor.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::time::Duration; -use tablepro_core::{EditorSession, EditorSessionState, RowLimit, RunOptions, UserSql}; +use tablepro_core::{ByteLimit, EditorSession, EditorSessionState, RowLimit, RunOptions, UserSql}; use tablepro_session::runtime::{Tasks, catch_panic}; use crate::error::CoreError; @@ -16,12 +16,14 @@ pub struct QueryEditor { tasks: Tasks, } -/// What one run may do. `None` takes the editor's defaults: 10,000 rows -/// and no deadline beyond the connection's. +/// What one run may do. `None` takes the editor's defaults: 10,000 rows, +/// 256,000,000 bytes of rows across the run, and no deadline beyond the +/// connection's. #[derive(Debug, Clone, Default, PartialEq, Eq, uniffi::Record)] pub struct RunSettings { pub row_limit: Option, pub timeout_ms: Option, + pub byte_limit: Option, } impl QueryEditor { @@ -41,8 +43,11 @@ impl QueryEditor { if let Some(limit) = settings.row_limit.and_then(RowLimit::new) { options = options.with_row_limit(limit); } + if let Some(limit) = settings.byte_limit.and_then(ByteLimit::new) { + options = options.with_byte_limit(limit); + } let editor = self.editor.clone(); - let sink = RunSink::new(state.clone()); + let sink = RunSink::new(state.clone(), options.byte_limit); let run = state.clone(); drop(self.tasks.spawn_task(async move { let ran = catch_panic(editor.run(UserSql::new(sql), options, Box::new(sink))).await; diff --git a/core/crates/ffi/src/host.rs b/core/crates/ffi/src/host.rs index d8bdceb7ac..6c319ff5c4 100644 --- a/core/crates/ffi/src/host.rs +++ b/core/crates/ffi/src/host.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use async_trait::async_trait; -use secrecy::{ExposeSecret, SecretString}; +use secrecy::SecretString; use tablepro_core::AiProvider; use tablepro_core::credentials::{SecretError, SecretItem, SecretItems, SecretKind, SecretLookup, SecretVault}; use tablepro_session::runtime::Tasks; @@ -21,13 +21,13 @@ pub struct HostConfig { /// core makes it on a blocking thread, never on a runtime worker. #[uniffi::export(callback_interface)] pub trait HostSecrets: Send + Sync { - fn load(&self, connection_id: String, kind: SecretKindTag) -> Result, HostSecretError>; + fn load(&self, connection_id: String, kind: SecretKindTag) -> Result, HostSecretError>; fn store( &self, connection_id: String, kind: SecretKindTag, - secret: String, + secret: SecretString, label: String, ) -> Result<(), HostSecretError>; @@ -37,10 +37,10 @@ pub trait HostSecrets: Send + Sync { fn delete_connection(&self, connection_id: String) -> Result<(), HostSecretError>; - fn load_item(&self, item: SecretItemTag) -> Result, HostSecretError>; + fn load_item(&self, item: SecretItemTag) -> Result, HostSecretError>; /// The app names the item in its own words. - fn store_item(&self, item: SecretItemTag, secret: String) -> Result<(), HostSecretError>; + fn store_item(&self, item: SecretItemTag, secret: SecretString) -> Result<(), HostSecretError>; fn delete_item(&self, item: SecretItemTag) -> Result<(), HostSecretError>; } @@ -148,23 +148,17 @@ impl std::fmt::Debug for HostSecretStore { } } -fn exposed(secret: &SecretString) -> String { - secret.expose_secret().to_owned() -} - #[async_trait] impl SecretVault for HostSecretStore { async fn store(&self, id: Uuid, kind: SecretKind, secret: &SecretString, label: &str) -> Result<(), SecretError> { - let (secret, label) = (exposed(secret), label.to_owned()); + let (secret, label) = (secret.clone(), label.to_owned()); self.call(move |host| host.store(id.to_string(), kind.into(), secret, label)) .await } async fn load(&self, id: Uuid, kind: SecretKind) -> Result { let found = self.call(move |host| host.load(id.to_string(), kind.into())).await?; - Ok(found.map_or(SecretLookup::NotStored, |secret| { - SecretLookup::Found(SecretString::from(secret)) - })) + Ok(found.map_or(SecretLookup::NotStored, SecretLookup::Found)) } async fn relabel(&self, id: Uuid, kind: SecretKind, label: &str) -> Result<(), SecretError> { @@ -185,12 +179,11 @@ impl SecretVault for HostSecretStore { #[async_trait] impl SecretItems for HostSecretStore { async fn load_item(&self, item: SecretItem) -> Result, SecretError> { - let found = self.call(move |host| host.load_item(item.into())).await?; - Ok(found.map(SecretString::from)) + self.call(move |host| host.load_item(item.into())).await } async fn store_item(&self, item: SecretItem, secret: &SecretString) -> Result<(), SecretError> { - let secret = exposed(secret); + let secret = secret.clone(); self.call(move |host| host.store_item(item.into(), secret)).await } @@ -204,18 +197,20 @@ mod tests { use std::collections::HashMap; use std::sync::Mutex; + use secrecy::ExposeSecret; + use super::*; use crate::runtime; #[derive(Default)] struct MemorySecrets { - stored: Mutex>, - items: Mutex>, + stored: Mutex>, + items: Mutex>, locked: bool, } impl HostSecrets for MemorySecrets { - fn load(&self, connection_id: String, kind: SecretKindTag) -> Result, HostSecretError> { + fn load(&self, connection_id: String, kind: SecretKindTag) -> Result, HostSecretError> { if self.locked { return Err(HostSecretError::Locked); } @@ -227,7 +222,7 @@ mod tests { &self, connection_id: String, kind: SecretKindTag, - secret: String, + secret: SecretString, label: String, ) -> Result<(), HostSecretError> { self.stored @@ -254,11 +249,11 @@ mod tests { Ok(()) } - fn load_item(&self, item: SecretItemTag) -> Result, HostSecretError> { + fn load_item(&self, item: SecretItemTag) -> Result, HostSecretError> { Ok(self.items.lock().unwrap().get(&item).cloned()) } - fn store_item(&self, item: SecretItemTag, secret: String) -> Result<(), HostSecretError> { + fn store_item(&self, item: SecretItemTag, secret: SecretString) -> Result<(), HostSecretError> { self.items.lock().unwrap().insert(item, secret); Ok(()) } @@ -287,7 +282,10 @@ mod tests { store.delete_connection(id).await.unwrap(); let after_delete = store.load(id, SecretKind::DatabasePassword).await.unwrap(); - assert_eq!(found.found().map(|secret| exposed(&secret)), Some("hunter2".to_owned())); + assert_eq!( + found.found().map(|secret| secret.expose_secret().to_owned()), + Some("hunter2".to_owned()) + ); assert!(!after_delete.is_stored()); } @@ -300,7 +298,10 @@ mod tests { store.delete_connection(Uuid::new_v4()).await.unwrap(); let found = store.load_item(item).await.unwrap(); - assert_eq!(found.map(|secret| exposed(&secret)), Some("sk-test".to_owned())); + assert_eq!( + found.map(|secret| secret.expose_secret().to_owned()), + Some("sk-test".to_owned()) + ); } #[tokio::test] diff --git a/core/crates/ffi/src/lib.rs b/core/crates/ffi/src/lib.rs index 018bf0c4b0..55034ff845 100644 --- a/core/crates/ffi/src/lib.rs +++ b/core/crates/ffi/src/lib.rs @@ -48,4 +48,13 @@ pub use safe_mode_gate::RunGate; pub use session::{ColumnSort, DatabaseSession, FailureCauseTag, SessionHealthTag, TableQuery}; pub use statement_at::StatementSpan; +use secrecy::{ExposeSecret, SecretString}; + uniffi::setup_scaffolding!(); + +// The app sees a String; here a lifted secret is wiped on drop. core/ARCHITECTURE.md, Bindings, lists what is not. +uniffi::custom_type!(SecretString, String, { + remote, + lower: |secret| secret.expose_secret().to_owned(), + try_lift: |text| Ok(SecretString::from(text)), +}); diff --git a/core/crates/ffi/src/library.rs b/core/crates/ffi/src/library.rs index c38c1e5d6e..7d6054b31c 100644 --- a/core/crates/ffi/src/library.rs +++ b/core/crates/ffi/src/library.rs @@ -118,15 +118,14 @@ pub struct ConnectionRecord { } // Each secret as the form holds it: None keeps what is stored, an empty string is stored as one. -#[derive(Clone, Default, PartialEq, Eq, uniffi::Record)] +#[derive(Clone, Default, uniffi::Record)] pub struct ConnectionSecrets { - pub database_password: Option, - pub ssh_password: Option, - pub ssh_passphrase: Option, + pub database_password: Option, + pub ssh_password: Option, + pub ssh_passphrase: Option, } impl ConnectionSecrets { - // Moves each string into a SecretString, which wipes it on drop, instead of leaving a plain copy behind. pub(crate) fn into_typed(self) -> Vec<(SecretKind, SecretString)> { [ (SecretKind::DatabasePassword, self.database_password), @@ -134,7 +133,7 @@ impl ConnectionSecrets { (SecretKind::SshPassphrase, self.ssh_passphrase), ] .into_iter() - .filter_map(|(kind, secret)| secret.map(|secret| (kind, SecretString::from(secret)))) + .filter_map(|(kind, secret)| secret.map(|secret| (kind, secret))) .collect() } } @@ -838,9 +837,9 @@ mod tests { #[test] fn debug_output_says_which_secrets_are_held_but_not_what_they_are() { let secrets = ConnectionSecrets { - database_password: Some("hunter2".to_owned()), + database_password: Some(SecretString::from("hunter2")), ssh_password: None, - ssh_passphrase: Some(String::new()), + ssh_passphrase: Some(SecretString::from("")), }; assert_eq!( diff --git a/core/crates/ffi/src/prompter.rs b/core/crates/ffi/src/prompter.rs index 3983551973..d20b94935f 100644 --- a/core/crates/ffi/src/prompter.rs +++ b/core/crates/ffi/src/prompter.rs @@ -62,10 +62,10 @@ pub struct PromptFieldInfo { pub secret: bool, } -#[derive(Clone, PartialEq, Eq, uniffi::Enum)] +#[derive(Clone, uniffi::Enum)] pub enum PromptAnswer { // One value per field; a host key or a confirmation is trusted with none. - Submitted { values: Vec, remember: bool }, + Submitted { values: Vec, remember: bool }, Cancelled, } @@ -177,10 +177,7 @@ impl From for PromptReasonTag { impl From for PromptReply { fn from(answer: PromptAnswer) -> Self { match answer { - PromptAnswer::Submitted { values, remember } => Self::Submitted { - values: values.into_iter().map(SecretString::from).collect(), - remember, - }, + PromptAnswer::Submitted { values, remember } => Self::Submitted { values, remember }, PromptAnswer::Cancelled => Self::Cancelled, } } @@ -242,7 +239,7 @@ mod tests { let prompter = bridge(Answering { asked: asked.clone(), answer: PromptAnswer::Submitted { - values: vec!["correct horse".to_owned()], + values: vec![SecretString::from("correct horse")], remember: true, }, }); @@ -277,7 +274,7 @@ mod tests { #[test] fn debug_output_of_an_answer_holds_no_typed_value() { let answer = PromptAnswer::Submitted { - values: vec!["correct horse".to_owned()], + values: vec![SecretString::from("correct horse")], remember: true, }; diff --git a/core/crates/ffi/src/results/column_cells.rs b/core/crates/ffi/src/results/column_cells.rs index c9c111e48a..2e0ad6c039 100644 --- a/core/crates/ffi/src/results/column_cells.rs +++ b/core/crates/ffi/src/results/column_cells.rs @@ -15,6 +15,8 @@ use super::ValueKind; // Outside every `ValueKind`. const NULL_CELL: u8 = u8::MAX; +// Also outside: an int32 crosses as an `Integer`, and the store keeps its width for an edit to write back. +const INT32_CELL: u8 = u8::MAX - 1; // A value is formatted once, on arrival, into the text the core's export writes; a binary value keeps its bytes. #[derive(Debug, Clone, Default)] @@ -31,6 +33,7 @@ pub(super) struct Cell<'a> { pub kind: Option, pub raw: i64, pub bytes: &'a [u8], + pub int32: bool, } impl ColumnCells { @@ -45,6 +48,11 @@ impl ColumnCells { self.bytes.extend_from_slice(if *flag { b"true" } else { b"false" }); i64::from(*flag) } + Value::Int32(number) => { + let _ = write!(self.bytes, "{number}"); + self.close(INT32_CELL, i64::from(*number)); + return; + } Value::Int(number) => { let _ = write!(self.bytes, "{number}"); *number @@ -78,9 +86,14 @@ impl ColumnCells { }; let kind = self.kinds[index]; Cell { - kind: (kind != NULL_CELL).then(|| ValueKind::from_byte(kind)), + kind: match kind { + NULL_CELL => None, + INT32_CELL => Some(ValueKind::Integer), + kind => Some(ValueKind::from_byte(kind)), + }, raw: self.raw[index], bytes: &self.bytes[start..self.ends[index]], + int32: kind == INT32_CELL, } } @@ -130,6 +143,7 @@ pub(super) fn cell_value(cell: Cell<'_>) -> Value { let raw = cell.raw; let parsed = match kind { ValueKind::Bool => Some(Value::Bool(raw != 0)), + ValueKind::Integer if cell.int32 => i32::try_from(raw).ok().map(Value::Int32), ValueKind::Integer => Some(Value::Int(raw)), ValueKind::UnsignedInteger => Some(Value::UInt(u64::from_ne_bytes(raw.to_ne_bytes()))), ValueKind::Float => Some(Value::Float64(f64::from_bits(u64::from_ne_bytes(raw.to_ne_bytes())))), @@ -355,6 +369,7 @@ mod tests { }; let groups: Vec> = vec![ vec![Value::Int(-5), Value::Int(3)], + vec![Value::Int32(-5), Value::Int(-4), Value::Int32(3)], vec![Value::UInt(7), Value::UInt(u64::MAX)], vec![Value::Float64(-0.5), Value::Float64(f64::NAN)], vec![ @@ -430,6 +445,23 @@ mod tests { assert_eq!(cell_value(cells(&[Value::Null]).cell(0)), Value::Null); } + #[test] + fn an_int32_crosses_as_an_integer_and_reads_back_as_an_int32() { + let mut target = cells(&[Value::Int(1)]); + let source = cells(&[Value::Int32(-7), Value::Int(-7), Value::Int32(i32::MAX)]); + target.extend_from(&source, 0..3); + + let int32 = target.cell(1); + assert_eq!( + (int32.kind, int32.raw, text(int32)), + (Some(ValueKind::Integer), -7, "-7") + ); + assert_eq!(cell_value(int32), Value::Int32(-7)); + assert_eq!(cell_value(target.cell(2)), Value::Int(-7)); + assert_eq!(cell_value(target.cell(3)), Value::Int32(i32::MAX)); + assert_eq!(target.cell(2).kind, Some(ValueKind::Integer)); + } + #[test] fn a_range_copied_into_another_column_reads_the_same() { let source = cells(&[ diff --git a/core/crates/ffi/src/results/store.rs b/core/crates/ffi/src/results/store.rs index 087389c75f..740977e727 100644 --- a/core/crates/ffi/src/results/store.rs +++ b/core/crates/ffi/src/results/store.rs @@ -118,8 +118,13 @@ impl ResultStore { state.rows } + // A set the last chunk holds would otherwise keep the slack its arrays grew by. pub(crate) fn finish(&self) { - self.state.write().unwrap_or_else(PoisonError::into_inner).finished = true; + let mut state = self.state.write().unwrap_or_else(PoisonError::into_inner); + state.finished = true; + if let Some(chunk) = state.chunks.last_mut() { + chunk.columns.iter_mut().for_each(ColumnCells::shrink); + } } // `Some(None)` for null, and every byte of a binary value as hex. @@ -152,7 +157,7 @@ impl ResultStore { let mut values = Vec::with_capacity(usize::try_from(state.rows).unwrap_or(0)); for chunk in &state.chunks { let cells = chunk.columns.get(column)?; - values.extend((0..chunk.rows).map(|row| cell_value(cells.cell(row)))); + values.extend((0..chunk.rows).map(|row| cell_value(sort_head(cells.cell(row))))); } drop(state); Some( @@ -183,6 +188,38 @@ impl StoreState { } } +// The core orders text by its first 4,096 characters (`value::order`), which fit in four bytes each. +const SORT_HEAD_BYTES: usize = 4 * 4_096; + +// A cell that reads back as text keeps only what its order reads, so a sort does not copy a whole long column. +fn sort_head(cell: Cell<'_>) -> Cell<'_> { + match cell.kind { + Some( + ValueKind::Text + | ValueKind::Json + | ValueKind::Xml + | ValueKind::Array + | ValueKind::Network + | ValueKind::Spatial + | ValueKind::Other, + ) => Cell { + bytes: text_head(cell.bytes), + ..cell + }, + _ => cell, + } +} + +fn text_head(bytes: &[u8]) -> &[u8] { + let Some(head) = bytes.get(..SORT_HEAD_BYTES) else { + return bytes; + }; + match std::str::from_utf8(head) { + Err(error) if error.error_len().is_none() => head.get(..error.valid_up_to()).unwrap_or(head), + _ => head, + } +} + fn kinds_of(columns: &[ResultColumn]) -> Vec { columns .iter() @@ -259,6 +296,41 @@ mod tests { assert_eq!(store.row_order(2, true), None); } + #[test] + fn long_texts_sort_by_the_head_the_core_orders_by() { + let store = store(1); + let wide = "😀".repeat(4_095); + let tail = "z".repeat(100_000); + store.append(&[ + vec![Value::Text(format!("{wide}b{tail}"))], + vec![Value::Text(format!("{wide}a{tail}"))], + vec![Value::Text("short".to_owned())], + ]); + + assert_eq!(store.row_order(0, true), Some(vec![2, 1, 0])); + assert_eq!( + text_head(format!("{wide}b{tail}").as_bytes()).len(), + SORT_HEAD_BYTES, + "the head ends on a whole character" + ); + assert_eq!(text_head("é".repeat(9_000).as_bytes()).len(), SORT_HEAD_BYTES); + assert_eq!( + text_head(&[0xe4_u8, 0xb8, 0xad].repeat(6_000)).len(), + SORT_HEAD_BYTES - 1 + ); + } + + #[test] + fn a_finished_set_reads_back_every_row() { + let store = store(2); + store.append(&rows(0..3)); + + store.finish(); + + assert!(store.is_finished()); + assert_eq!(store.full_text(2, 1), Some(Some("row 2".to_owned()))); + } + #[test] fn columns_resolved_before_any_row_replace_the_set_s_columns() { let store = store(1); diff --git a/core/crates/ffi/src/results/value_kind.rs b/core/crates/ffi/src/results/value_kind.rs index 413b2f198e..ce290f1b5a 100644 --- a/core/crates/ffi/src/results/value_kind.rs +++ b/core/crates/ffi/src/results/value_kind.rs @@ -90,7 +90,7 @@ impl ValueKind { pub fn of_value(value: &Value) -> Self { match value { Value::Bool(_) => Self::Bool, - Value::Int(_) => Self::Integer, + Value::Int32(_) | Value::Int(_) => Self::Integer, Value::UInt(_) => Self::UnsignedInteger, Value::WideInt(_) => Self::WideInteger, Value::Float32(_) | Value::Float64(_) => Self::Float, diff --git a/core/crates/ffi/src/row_edits.rs b/core/crates/ffi/src/row_edits.rs index fafb30372e..300a2c2f9a 100644 --- a/core/crates/ffi/src/row_edits.rs +++ b/core/crates/ffi/src/row_edits.rs @@ -218,9 +218,15 @@ impl TableEditor { .map(|(_, column)| column.name.clone()) } - fn input(&self, column: &ColumnInfo, edit: &CellEdit, context: EditContext) -> Result { + fn input( + &self, + column: &ColumnInfo, + edit: &CellEdit, + context: EditContext, + read: Option<&Value>, + ) -> Result { let input = match edit { - CellEdit::Text { text } => parse_edit_text(text, column, context).map_err(parse_error)?, + CellEdit::Text { text } => parse_edit_text(text, column, context, read).map_err(parse_error)?, CellEdit::Null if column.nullable => CellInput::Value(Value::Null), CellEdit::Null => return Err(parse_error(EditParseError::Required)), CellEdit::Default => CellInput::Default, @@ -284,6 +290,7 @@ impl TableEditor { column: &ColumnInfo, edit: &CellEdit, new_row: bool, + read: Option<&Value>, ) -> Result { let open = match new_row { true => editable_in_new_row(column), @@ -304,7 +311,7 @@ impl TableEditor { true => EditContext::InsertDraft, false => EditContext::UpdateRow, }; - self.input(column, edit, context) + self.input(column, edit, context, read) } fn update( @@ -316,13 +323,14 @@ impl TableEditor { cells: &[CellChange], ) -> Result, CoreError> { let mut assignments: Vec<(usize, CellInput)> = Vec::with_capacity(cells.len()); + let mut originals: Vec<(usize, Value)> = Vec::with_capacity(cells.len()); for cell in cells { let column = self.column(cell.column)?; let index = cell.column as usize; - let input = self.check_cell(Some(dialect), column, &cell.edit, false)?; let original = rows .stored_value(row, index) .and_then(|value| value_for_column(&value, &read[index].column_type, &column.column_type)); + let input = self.check_cell(Some(dialect), column, &cell.edit, false, original.as_ref())?; let unchanged = match (&input, &original) { (CellInput::Value(edited), Some(original)) => is_unchanged(original, edited, &column.column_type), _ => false, @@ -332,13 +340,20 @@ impl TableEditor { if !unchanged { assignments.push((index, input)); } + if let Some(original) = original { + originals.retain(|(position, _)| *position != index); + originals.push((index, original)); + } } if assignments.is_empty() { return Ok(None); } + // A document engine saves an edited number in the type it was read as, which the store keeps. + originals.retain(|(position, _)| assignments.iter().any(|(assigned, _)| assigned == position)); Ok(Some(RowChange::Update(RowUpdate { key: self.key(rows, read, row)?, assignments, + read: originals, }))) } @@ -362,8 +377,8 @@ impl TableEditor { } // A column with no text to type, such as a blob, still takes NULL or the server's value in a new row. let input = match edit { - CellEdit::Text { .. } => self.check_cell(None, column, edit, true)?, - CellEdit::Null | CellEdit::Default => self.input(column, edit, EditContext::InsertDraft)?, + CellEdit::Text { .. } => self.check_cell(None, column, edit, true, None)?, + CellEdit::Null | CellEdit::Default => self.input(column, edit, EditContext::InsertDraft, None)?, }; inputs.push(input); } @@ -441,7 +456,8 @@ impl TableEditor { pub fn check(&self, column: u32, edit: CellEdit, new_row: bool) -> Result<(), CoreError> { let info = self.column(column)?; let dialect = self.session.current_dialect(); - self.check_cell(dialect.as_deref(), info, &edit, new_row).map(|_| ()) + self.check_cell(dialect.as_deref(), info, &edit, new_row, None) + .map(|_| ()) } // The statements the edits save as, at the session's Safe Mode level now. `rows` is the result set the rows were diff --git a/core/crates/ffi/src/run/run_event.rs b/core/crates/ffi/src/run/run_event.rs index 0f392dd79a..e95bef496e 100644 --- a/core/crates/ffi/src/run/run_event.rs +++ b/core/crates/ffi/src/run/run_event.rs @@ -26,6 +26,12 @@ pub enum RunEvent { set_index: u32, retained: u64, }, + // The run kept `limit_bytes` of rows; the set's rows after its first `retained` were read and not kept. + ByteLimitReached { + set_index: u32, + retained: u64, + limit_bytes: u64, + }, ResultSetFinished { set_index: u32, row_count: u64, @@ -83,6 +89,7 @@ pub enum TruncationTag { ServerStopped, /// There were rows past the limit, which were not kept. ReadPastLimit, + ByteLimit, } /// How a run ended. A statement the server refused is `Failed`, not an @@ -133,6 +140,7 @@ impl From for TruncationTag { Truncation::Complete => Self::Complete, Truncation::ServerStopped => Self::ServerStopped, Truncation::ReadPastLimit => Self::ReadPastLimit, + Truncation::ByteLimit => Self::ByteLimit, } } } diff --git a/core/crates/ffi/src/run/run_sink.rs b/core/crates/ffi/src/run/run_sink.rs index 1234e1d6c7..1fb41da928 100644 --- a/core/crates/ffi/src/run/run_sink.rs +++ b/core/crates/ffi/src/run/run_sink.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use async_trait::async_trait; -use tablepro_core::{ResultEvent, ResultSink, SinkFlow}; +use tablepro_core::{ByteLimit, ResultEvent, ResultSink, SinkFlow}; use super::{RunEvent, RunState}; use crate::results::{ResultColumnInfo, ResultStore}; @@ -10,11 +10,16 @@ use crate::results::{ResultColumnInfo, ResultStore}; pub(crate) struct RunSink { state: Arc, current: Option<(u32, Arc)>, + limit_bytes: u64, } impl RunSink { - pub(crate) fn new(state: Arc) -> Self { - Self { state, current: None } + pub(crate) fn new(state: Arc, byte_limit: ByteLimit) -> Self { + Self { + state, + current: None, + limit_bytes: byte_limit.get(), + } } } @@ -56,6 +61,11 @@ impl ResultSink for RunSink { set_index: self.current_index(), retained, }, + ResultEvent::ByteLimitReached { retained } => RunEvent::ByteLimitReached { + set_index: self.current_index(), + retained, + limit_bytes: self.limit_bytes, + }, ResultEvent::ResultSetFinished { total_rows, truncation, .. } => { @@ -90,10 +100,11 @@ impl RunSink { #[cfg(test)] mod tests { - use tablepro_core::ServerNotice; + use tablepro_core::column::{ColumnType, ResultColumn}; + use tablepro_core::{ServerNotice, Truncation, Value}; use super::*; - use crate::run::RunListener; + use crate::run::{RunListener, TruncationTag}; struct Quiet; @@ -101,10 +112,48 @@ mod tests { fn events_pending(&self) {} } + #[tokio::test] + async fn a_set_cut_for_bytes_reaches_the_app_with_the_runs_limit() { + let state = Arc::new(RunState::new(Box::new(Quiet))); + let limit = ByteLimit::new(4_096).unwrap(); + let mut sink = RunSink::new(state.clone(), limit); + let columns: Arc<[ResultColumn]> = vec![ResultColumn::new("blob", ColumnType::untyped_text(""))].into(); + + sink.accept(ResultEvent::ResultSetStarted { columns }).await; + sink.accept(ResultEvent::Rows(vec![vec![Value::Text("a".repeat(5_000))]])) + .await; + sink.accept(ResultEvent::ByteLimitReached { retained: 1 }).await; + sink.accept(ResultEvent::ResultSetFinished { + retained: 1, + total_rows: Some(3), + truncation: Truncation::ByteLimit, + }) + .await; + + let events = state.drain(); + assert_eq!( + events[2], + RunEvent::ByteLimitReached { + set_index: 0, + retained: 1, + limit_bytes: 4_096, + } + ); + assert_eq!( + events[3], + RunEvent::ResultSetFinished { + set_index: 0, + row_count: 1, + total_rows: Some(3), + truncation: TruncationTag::ByteLimit, + } + ); + } + #[tokio::test] async fn the_notice_limit_reaches_the_app_after_the_notices_kept() { let state = Arc::new(RunState::new(Box::new(Quiet))); - let mut sink = RunSink::new(state.clone()); + let mut sink = RunSink::new(state.clone(), ByteLimit::EDITOR_DEFAULT); sink.accept(ResultEvent::Notice(ServerNotice { severity: "NOTICE".to_owned(), diff --git a/core/crates/ffi/tests/byte_limit.rs b/core/crates/ffi/tests/byte_limit.rs new file mode 100644 index 0000000000..cb7cd069a4 --- /dev/null +++ b/core/crates/ffi/tests/byte_limit.rs @@ -0,0 +1,129 @@ +#![cfg(feature = "engines")] +#![expect( + clippy::unwrap_used, + reason = "the setup helpers fail a test the way the test functions themselves do" +)] + +use std::sync::Arc; + +use secrecy::SecretString; +use tablepro_ffi::{ + Core, FileOpenRequest, HostConfig, HostSecretError, HostSecrets, RunEvent, RunListener, RunOutcome, RunSettings, + SecretItemTag, SecretKindTag, TruncationTag, +}; + +struct NoSecrets; + +impl HostSecrets for NoSecrets { + fn load(&self, _: String, _: SecretKindTag) -> Result, HostSecretError> { + Ok(None) + } + + fn store(&self, _: String, _: SecretKindTag, _: SecretString, _: String) -> Result<(), HostSecretError> { + Ok(()) + } + + fn relabel(&self, _: String, _: SecretKindTag, _: String) -> Result<(), HostSecretError> { + Ok(()) + } + + fn delete(&self, _: String, _: SecretKindTag) -> Result<(), HostSecretError> { + Ok(()) + } + + fn delete_connection(&self, _: String) -> Result<(), HostSecretError> { + Ok(()) + } + + fn load_item(&self, _: SecretItemTag) -> Result, HostSecretError> { + Ok(None) + } + + fn store_item(&self, _: SecretItemTag, _: SecretString) -> Result<(), HostSecretError> { + Ok(()) + } + + fn delete_item(&self, _: SecretItemTag) -> Result<(), HostSecretError> { + Ok(()) + } +} + +struct Quiet; + +impl RunListener for Quiet { + fn events_pending(&self) {} +} + +fn core() -> Arc { + Core::new( + HostConfig { + client_version: "test".to_owned(), + storage_dir: None, + }, + Box::new(NoSecrets), + ) + .unwrap() +} + +const BLOBS: &str = "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c WHERE x < 100) \ + SELECT x, zeroblob(65536) FROM c"; + +#[tokio::test] +async fn a_run_keeps_whole_rows_up_to_its_byte_limit() { + let dir = tempfile::tempdir().unwrap(); + let request = FileOpenRequest { + path: dir.path().join("bytes.sqlite").display().to_string(), + driver_id: None, + read_only: false, + create: true, + }; + let core = core(); + let session = core.open_file(request, None).await.unwrap(); + let editor = session.open_editor(None).await.unwrap(); + let settings = RunSettings { + byte_limit: Some(1 << 20), + ..RunSettings::default() + }; + + let query = editor.run(BLOBS.to_owned(), settings, Box::new(Quiet)); + let outcome = query.outcome().await.unwrap(); + let events = query.drain_events(); + + assert!(matches!(outcome, RunOutcome::Completed { .. }), "{outcome:?}"); + let reached: Vec<&RunEvent> = events + .iter() + .filter(|event| matches!(event, RunEvent::ByteLimitReached { .. })) + .collect(); + // Each row is two values and 65,536 bytes, so the 16th crosses 1 MiB and is kept whole. + assert_eq!( + reached, + [&RunEvent::ByteLimitReached { + set_index: 0, + retained: 16, + limit_bytes: 1 << 20, + }] + ); + let finished = events + .iter() + .position(|event| matches!(event, RunEvent::ResultSetFinished { .. })) + .unwrap(); + assert_eq!( + events[finished], + RunEvent::ResultSetFinished { + set_index: 0, + row_count: 16, + total_rows: Some(100), + truncation: TruncationTag::ByteLimit, + } + ); + let cut = events.iter().position(|event| event == reached[0]).unwrap(); + assert!(cut < finished, "{events:?}"); + let set = query.result_set(0).unwrap(); + assert_eq!(set.row_count(), 16); + let last = set.cell_text(15, 1).unwrap().unwrap(); + assert_eq!(last.len(), 2 + 2 * 65_536, "the row that crossed the limit was cut"); + + let next = editor.run("SELECT 1".to_owned(), RunSettings::default(), Box::new(Quiet)); + let after = next.outcome().await.unwrap(); + assert!(matches!(after, RunOutcome::Completed { .. }), "{after:?}"); +} diff --git a/core/crates/ffi/tests/cloud_library.rs b/core/crates/ffi/tests/cloud_library.rs index 6b1c5aba12..83a58ea935 100644 --- a/core/crates/ffi/tests/cloud_library.rs +++ b/core/crates/ffi/tests/cloud_library.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::sync::{Arc, Mutex}; +use secrecy::{ExposeSecret, SecretString}; use tablepro_ffi::{ CloudConnectionFields, CloudGroupFields, CloudLibraryPull, CloudLibraryRecords, ConnectionLibrary, ConnectionRecord, ConnectionSecrets, Core, HostConfig, HostSecretError, HostSecrets, SafeModeTag, SecretItemTag, @@ -13,13 +14,13 @@ use tablepro_ffi::{ }; use tempfile::TempDir; -type Stored = Arc>>; +type Stored = Arc>>; #[derive(Default, Clone)] struct MemorySecrets(Stored); impl HostSecrets for MemorySecrets { - fn load(&self, connection_id: String, kind: SecretKindTag) -> Result, HostSecretError> { + fn load(&self, connection_id: String, kind: SecretKindTag) -> Result, HostSecretError> { Ok(self.0.lock().unwrap().get(&(connection_id, kind)).cloned()) } @@ -27,7 +28,7 @@ impl HostSecrets for MemorySecrets { &self, connection_id: String, kind: SecretKindTag, - secret: String, + secret: SecretString, _: String, ) -> Result<(), HostSecretError> { self.0.lock().unwrap().insert((connection_id, kind), secret); @@ -48,11 +49,11 @@ impl HostSecrets for MemorySecrets { Ok(()) } - fn load_item(&self, _: SecretItemTag) -> Result, HostSecretError> { + fn load_item(&self, _: SecretItemTag) -> Result, HostSecretError> { Ok(None) } - fn store_item(&self, _: SecretItemTag, _: String) -> Result<(), HostSecretError> { + fn store_item(&self, _: SecretItemTag, _: SecretString) -> Result<(), HostSecretError> { Ok(()) } @@ -201,7 +202,7 @@ async fn cloud_deletions_remove_secrets_only_after_storage_succeeds() { .store( id.clone(), SecretKindTag::DatabasePassword, - "password".to_owned(), + SecretString::from("password"), "test".to_owned(), ) .unwrap(); @@ -218,12 +219,13 @@ async fn cloud_deletions_remove_secrets_only_after_storage_succeeds() { secrets .load(id.clone(), SecretKindTag::DatabasePassword) .unwrap() - .as_deref(), + .as_ref() + .map(ExposeSecret::expose_secret), Some("password") ); std::fs::remove_dir(&file).unwrap(); std::fs::write(&file, contents).unwrap(); library.apply_cloud_pull(pull).await.unwrap(); assert!(library.snapshot().await.unwrap().connections.is_empty()); - assert_eq!(secrets.load(id, SecretKindTag::DatabasePassword).unwrap(), None); + assert!(secrets.load(id, SecretKindTag::DatabasePassword).unwrap().is_none()); } diff --git a/core/crates/ffi/tests/library.rs b/core/crates/ffi/tests/library.rs index 521fe87921..02bc1dee6f 100644 --- a/core/crates/ffi/tests/library.rs +++ b/core/crates/ffi/tests/library.rs @@ -12,6 +12,7 @@ use std::sync::{Arc, Mutex}; use russh::keys::HashAlg; use russh::keys::ssh_key::PrivateKey; use russh::keys::ssh_key::private::Ed25519Keypair; +use secrecy::{ExposeSecret, SecretString}; use tablepro_ffi::{ ConnectionRecord, ConnectionSecrets, Core, CoreError, FileOpenRequest, HostConfig, HostPrompter, HostSecretError, HostSecrets, ObjectScopeArg, PromptAnswer, PromptPurposeTag, PromptRequest, RunOutcome, RunSettings, SafeModeTag, @@ -19,13 +20,13 @@ use tablepro_ffi::{ }; use tokio::net::TcpListener; -type Stored = Arc>>; +type Stored = Arc>>; #[derive(Default, Clone)] struct MemorySecrets(Stored); impl HostSecrets for MemorySecrets { - fn load(&self, connection_id: String, kind: SecretKindTag) -> Result, HostSecretError> { + fn load(&self, connection_id: String, kind: SecretKindTag) -> Result, HostSecretError> { Ok(self.0.lock().unwrap().get(&(connection_id, kind)).cloned()) } @@ -33,7 +34,7 @@ impl HostSecrets for MemorySecrets { &self, connection_id: String, kind: SecretKindTag, - secret: String, + secret: SecretString, _: String, ) -> Result<(), HostSecretError> { self.0.lock().unwrap().insert((connection_id, kind), secret); @@ -54,11 +55,11 @@ impl HostSecrets for MemorySecrets { Ok(()) } - fn load_item(&self, _: SecretItemTag) -> Result, HostSecretError> { + fn load_item(&self, _: SecretItemTag) -> Result, HostSecretError> { Ok(None) } - fn store_item(&self, _: SecretItemTag, _: String) -> Result<(), HostSecretError> { + fn store_item(&self, _: SecretItemTag, _: SecretString) -> Result<(), HostSecretError> { Ok(()) } @@ -88,7 +89,12 @@ impl RefusingSecrets { } fn value(&self, id: &str, kind: SecretKindTag) -> Option { - self.memory.0.lock().unwrap().get(&(id.to_owned(), kind)).cloned() + self.memory + .0 + .lock() + .unwrap() + .get(&(id.to_owned(), kind)) + .map(|secret| secret.expose_secret().to_owned()) } fn deleting(&self) -> Result<(), HostSecretError> { @@ -100,7 +106,7 @@ impl RefusingSecrets { } impl HostSecrets for RefusingSecrets { - fn load(&self, connection_id: String, kind: SecretKindTag) -> Result, HostSecretError> { + fn load(&self, connection_id: String, kind: SecretKindTag) -> Result, HostSecretError> { self.memory.load(connection_id, kind) } @@ -108,7 +114,7 @@ impl HostSecrets for RefusingSecrets { &self, connection_id: String, kind: SecretKindTag, - secret: String, + secret: SecretString, label: String, ) -> Result<(), HostSecretError> { if self.refused_kinds.lock().unwrap().contains(&kind) { @@ -134,11 +140,11 @@ impl HostSecrets for RefusingSecrets { self.memory.delete_connection(connection_id) } - fn load_item(&self, item: SecretItemTag) -> Result, HostSecretError> { + fn load_item(&self, item: SecretItemTag) -> Result, HostSecretError> { self.memory.load_item(item) } - fn store_item(&self, item: SecretItemTag, secret: String) -> Result<(), HostSecretError> { + fn store_item(&self, item: SecretItemTag, secret: SecretString) -> Result<(), HostSecretError> { self.memory.store_item(item, secret) } @@ -272,7 +278,7 @@ async fn a_saved_password_goes_to_the_secret_store_and_leaves_with_the_connectio .save( record("shop", "mysql", "shop"), ConnectionSecrets { - database_password: Some("hunter2".to_owned()), + database_password: Some(SecretString::from("hunter2")), ..ConnectionSecrets::default() }, ) @@ -286,18 +292,39 @@ async fn a_saved_password_goes_to_the_secret_store_and_leaves_with_the_connectio .cloned(); library.delete(saved.id.clone()).await.unwrap(); - assert_eq!(kept.as_deref(), Some("hunter2")); + assert_eq!(kept.as_ref().map(ExposeSecret::expose_secret), Some("hunter2")); assert!(secrets.0.lock().unwrap().is_empty()); assert!(library.snapshot().await.unwrap().connections.is_empty()); } fn with_password(password: &str) -> ConnectionSecrets { ConnectionSecrets { - database_password: Some(password.to_owned()), + database_password: Some(SecretString::from(password)), ..ConnectionSecrets::default() } } +#[tokio::test] +async fn a_save_refused_before_any_write_stores_none_of_its_secrets() { + let dir = tempfile::tempdir().unwrap(); + let secrets = MemorySecrets::default(); + let core = core(dir.path(), secrets.clone()); + let library = core.library().unwrap(); + let mut bad_id = record("shop", "mysql", "shop"); + bad_id.id = "not-an-id".to_owned(); + + let unnamed = library + .save(record(" ", "mysql", "shop"), with_password("hunter2")) + .await + .unwrap_err(); + let invalid = library.save(bad_id, with_password("hunter2")).await.unwrap_err(); + + assert_eq!(unnamed.code(), "library.name_required"); + assert_eq!(invalid.code(), "library.invalid_id"); + assert!(secrets.0.lock().unwrap().is_empty()); + assert!(library.snapshot().await.unwrap().connections.is_empty()); +} + #[tokio::test] async fn a_delete_the_secret_store_refuses_keeps_the_connection_and_its_password() { let dir = tempfile::tempdir().unwrap(); @@ -356,7 +383,7 @@ async fn a_refused_ssh_password_takes_back_the_new_connection_and_its_database_p let core = core(dir.path(), secrets.clone()); let library = core.library().unwrap(); let both = ConnectionSecrets { - ssh_password: Some("tunnel".to_owned()), + ssh_password: Some(SecretString::from("tunnel")), ..with_password("hunter2") }; @@ -438,7 +465,7 @@ async fn an_edit_refused_at_its_second_secret_takes_back_the_first() { .save( through_ssh(22), ConnectionSecrets { - ssh_password: Some("old-tunnel".to_owned()), + ssh_password: Some(SecretString::from("old-tunnel")), ..ConnectionSecrets::default() }, ) @@ -452,7 +479,7 @@ async fn an_edit_refused_at_its_second_secret_takes_back_the_first() { .save( moved, ConnectionSecrets { - ssh_password: Some("new-tunnel".to_owned()), + ssh_password: Some(SecretString::from("new-tunnel")), ..with_password("new-db") }, ) @@ -608,7 +635,7 @@ async fn a_password_typed_at_connect_is_used_when_none_is_saved() { .connect_saved( saved.id, ConnectionSecrets { - database_password: Some("typed".to_owned()), + database_password: Some(SecretString::from("typed")), ..ConnectionSecrets::default() }, None, @@ -680,8 +707,8 @@ async fn an_ssh_login_asks_the_app_about_a_host_key_it_does_not_know() { let fingerprint = key.public_key().fingerprint(HashAlg::Sha256).to_string(); let prompter = Declining::default(); let secrets = ConnectionSecrets { - database_password: Some(String::new()), - ssh_password: Some("typed".to_owned()), + database_password: Some(SecretString::from("")), + ssh_password: Some(SecretString::from("typed")), ssh_passphrase: None, }; @@ -729,7 +756,7 @@ async fn an_empty_database_password_survives_a_separately_stored_ssh_password() .save( through_ssh(port), ConnectionSecrets { - ssh_password: Some("saved ssh password".to_owned()), + ssh_password: Some(SecretString::from("saved ssh password")), ..ConnectionSecrets::default() }, ) @@ -739,7 +766,7 @@ async fn an_empty_database_password_survives_a_separately_stored_ssh_password() .connect_saved( saved.id, ConnectionSecrets { - database_password: Some(String::new()), + database_password: Some(SecretString::from("")), ..ConnectionSecrets::default() }, None, @@ -758,7 +785,7 @@ async fn an_empty_database_password_survives_a_separately_stored_ssh_password() .test_connection( with_key, ConnectionSecrets { - database_password: Some(String::new()), + database_password: Some(SecretString::from("")), ..ConnectionSecrets::default() }, None, diff --git a/core/crates/ffi/tests/row_edits.rs b/core/crates/ffi/tests/row_edits.rs index b51a85ea45..f089cc0d1d 100644 --- a/core/crates/ffi/tests/row_edits.rs +++ b/core/crates/ffi/tests/row_edits.rs @@ -7,6 +7,7 @@ use std::path::Path; use std::sync::Arc; +use secrecy::SecretString; use tablepro_ffi::{ CellChange, CellEdit, ConnectionRecord, ConnectionSecrets, Core, DatabaseSession, EditLock, EditStart, FileOpenRequest, HostConfig, HostSecretError, HostSecrets, ResultSet, RowEdit, RowSaveOutcome, RunListener, @@ -16,11 +17,11 @@ use tablepro_ffi::{ struct NoSecrets; impl HostSecrets for NoSecrets { - fn load(&self, _: String, _: SecretKindTag) -> Result, HostSecretError> { + fn load(&self, _: String, _: SecretKindTag) -> Result, HostSecretError> { Ok(None) } - fn store(&self, _: String, _: SecretKindTag, _: String, _: String) -> Result<(), HostSecretError> { + fn store(&self, _: String, _: SecretKindTag, _: SecretString, _: String) -> Result<(), HostSecretError> { Ok(()) } @@ -36,11 +37,11 @@ impl HostSecrets for NoSecrets { Ok(()) } - fn load_item(&self, _: SecretItemTag) -> Result, HostSecretError> { + fn load_item(&self, _: SecretItemTag) -> Result, HostSecretError> { Ok(None) } - fn store_item(&self, _: SecretItemTag, _: String) -> Result<(), HostSecretError> { + fn store_item(&self, _: SecretItemTag, _: SecretString) -> Result<(), HostSecretError> { Ok(()) } diff --git a/core/crates/ffi/tests/safe_mode_gate.rs b/core/crates/ffi/tests/safe_mode_gate.rs index 1aac542fb9..74ab7da6b2 100644 --- a/core/crates/ffi/tests/safe_mode_gate.rs +++ b/core/crates/ffi/tests/safe_mode_gate.rs @@ -7,6 +7,7 @@ use std::path::Path; use std::sync::Arc; +use secrecy::SecretString; use tablepro_ffi::{ ConnectionRecord, ConnectionSecrets, Core, DatabaseSession, FileOpenRequest, HostConfig, HostSecretError, HostSecrets, RunGate, RunListener, RunOutcome, RunSettings, SafeModeTag, SecretItemTag, SecretKindTag, TlsModeTag, @@ -16,11 +17,11 @@ use tablepro_ffi::{ struct NoSecrets; impl HostSecrets for NoSecrets { - fn load(&self, _: String, _: SecretKindTag) -> Result, HostSecretError> { + fn load(&self, _: String, _: SecretKindTag) -> Result, HostSecretError> { Ok(None) } - fn store(&self, _: String, _: SecretKindTag, _: String, _: String) -> Result<(), HostSecretError> { + fn store(&self, _: String, _: SecretKindTag, _: SecretString, _: String) -> Result<(), HostSecretError> { Ok(()) } @@ -36,11 +37,11 @@ impl HostSecrets for NoSecrets { Ok(()) } - fn load_item(&self, _: SecretItemTag) -> Result, HostSecretError> { + fn load_item(&self, _: SecretItemTag) -> Result, HostSecretError> { Ok(None) } - fn store_item(&self, _: SecretItemTag, _: String) -> Result<(), HostSecretError> { + fn store_item(&self, _: SecretItemTag, _: SecretString) -> Result<(), HostSecretError> { Ok(()) } diff --git a/core/crates/ffi/tests/sqlite_path.rs b/core/crates/ffi/tests/sqlite_path.rs index 13c1780d8e..9ab13ef427 100644 --- a/core/crates/ffi/tests/sqlite_path.rs +++ b/core/crates/ffi/tests/sqlite_path.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::{Duration, Instant}; +use secrecy::SecretString; use tablepro_ffi::window_abi::{TpCursor, TpWindow, tablepro_cursor_fetch, tablepro_cursor_free}; use tablepro_ffi::{ ColumnSort, Core, CoreError, DatabaseSession, EndpointInfo, FileOpenRequest, HostConfig, HostSecretError, @@ -20,11 +21,11 @@ use tablepro_ffi::{ struct NoSecrets; impl HostSecrets for NoSecrets { - fn load(&self, _: String, _: SecretKindTag) -> Result, HostSecretError> { + fn load(&self, _: String, _: SecretKindTag) -> Result, HostSecretError> { Ok(None) } - fn store(&self, _: String, _: SecretKindTag, _: String, _: String) -> Result<(), HostSecretError> { + fn store(&self, _: String, _: SecretKindTag, _: SecretString, _: String) -> Result<(), HostSecretError> { Ok(()) } @@ -40,11 +41,11 @@ impl HostSecrets for NoSecrets { Ok(()) } - fn load_item(&self, _: SecretItemTag) -> Result, HostSecretError> { + fn load_item(&self, _: SecretItemTag) -> Result, HostSecretError> { Ok(None) } - fn store_item(&self, _: SecretItemTag, _: String) -> Result<(), HostSecretError> { + fn store_item(&self, _: SecretItemTag, _: SecretString) -> Result<(), HostSecretError> { Ok(()) } @@ -316,6 +317,7 @@ async fn a_million_rows_stream_in_and_any_window_reads_back() { RunSettings { row_limit: Some(1_000_000), timeout_ms: None, + byte_limit: None, }, Box::new(wakes.clone()), ); diff --git a/core/crates/mcp/src/collected_results.rs b/core/crates/mcp/src/collected_results.rs index cb348ecaee..2330754f06 100644 --- a/core/crates/mcp/src/collected_results.rs +++ b/core/crates/mcp/src/collected_results.rs @@ -137,7 +137,7 @@ impl ResultCollector { } ResultEvent::Rows(_) if state.sets_left_out => {} ResultEvent::Rows(rows) => return self.keep_rows(&mut state, rows), - ResultEvent::RowLimitReached { .. } => { + ResultEvent::RowLimitReached { .. } | ResultEvent::ByteLimitReached { .. } => { if !state.sets_left_out && let Some(set) = state.sets.last_mut() { diff --git a/core/crates/session/Cargo.toml b/core/crates/session/Cargo.toml index 191354b057..4a4fc8480c 100644 --- a/core/crates/session/Cargo.toml +++ b/core/crates/session/Cargo.toml @@ -18,6 +18,7 @@ tablepro-net.workspace = true tablepro-ssh.workspace = true tablepro-storage.workspace = true tablepro-test-support = { workspace = true, optional = true } +async-trait.workspace = true futures.workspace = true secrecy.workspace = true thiserror.workspace = true @@ -32,7 +33,6 @@ tablepro-driver-postgres.workspace = true tablepro-driver-redis.workspace = true tablepro-test-fixtures.workspace = true tablepro-test-support.workspace = true -async-trait.workspace = true tempfile.workspace = true tokio = { workspace = true, features = ["test-util"] } diff --git a/core/crates/session/src/session/byte_limited_editor.rs b/core/crates/session/src/session/byte_limited_editor.rs new file mode 100644 index 0000000000..a12f8d765e --- /dev/null +++ b/core/crates/session/src/session/byte_limited_editor.rs @@ -0,0 +1,55 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use tablepro_core::{ + ByteLimitSink, CallOptions, DriverError, EditorSession, EditorSessionId, EditorSessionState, ResultSink, + RunOptions, RunSummary, TransactionState, UserSql, +}; + +// Every app opens its editor sessions here, so none can run one without the run's byte limit. +pub(super) struct ByteLimitedEditor { + inner: Arc, +} + +impl ByteLimitedEditor { + pub(super) fn wrap(inner: Arc) -> Arc { + Arc::new(Self { inner }) + } +} + +#[async_trait] +impl EditorSession for ByteLimitedEditor { + fn id(&self) -> EditorSessionId { + self.inner.id() + } + + fn state(&self) -> EditorSessionState { + self.inner.state() + } + + async fn run( + &self, + sql: UserSql, + options: RunOptions, + sink: Box, + ) -> Result { + let limited = Box::new(ByteLimitSink::new(sink, options.byte_limit)); + self.inner.run(sql, options, limited).await + } + + async fn transaction_state(&self, options: CallOptions) -> Result { + self.inner.transaction_state(options).await + } + + fn database(&self) -> Option { + self.inner.database() + } + + async fn use_database(&self, database: &str, options: CallOptions) -> Result<(), DriverError> { + self.inner.use_database(database, options).await + } + + async fn close(&self) { + self.inner.close().await; + } +} diff --git a/core/crates/session/src/session/mod.rs b/core/crates/session/src/session/mod.rs index ab26702314..987a55af97 100644 --- a/core/crates/session/src/session/mod.rs +++ b/core/crates/session/src/session/mod.rs @@ -1,6 +1,7 @@ //! One window's session: its connection link, the operation chokepoint, //! the health monitor, the schema catalog, and an ordered shutdown. +mod byte_limited_editor; mod editor_session_shutdown; mod object_list_snapshot; mod schema_catalog; diff --git a/core/crates/session/src/session/session.rs b/core/crates/session/src/session/session.rs index 3f49f9a79e..6735034a61 100644 --- a/core/crates/session/src/session/session.rs +++ b/core/crates/session/src/session/session.rs @@ -16,6 +16,7 @@ use tokio_util::sync::{CancellationToken, DropGuard}; use tokio_util::task::AbortOnDropHandle; use uuid::Uuid; +use super::byte_limited_editor::ByteLimitedEditor; use super::{ObjectListSnapshot, SchemaCatalog, SessionDeps}; use crate::connect::{ConnectError, ConnectRequest, ConnectionMetadata, PendingSession, RequestTarget}; use crate::health::{ @@ -385,6 +386,7 @@ impl Session { lease .open_editor_session(options, call.cancel().clone()) .await + .map(ByteLimitedEditor::wrap) .map_err(OperationError::from) } }) @@ -431,9 +433,14 @@ impl std::fmt::Debug for Session { mod tests { use std::sync::Weak; - use tablepro_core::{DriverError, LivenessPolicy}; + use tablepro_core::column::{ColumnType, ResultColumn}; + use tablepro_core::value::Value; + use tablepro_core::{ + ByteLimit, DriverError, LivenessPolicy, ResultEvent, ResultSink, RunEnd, RunOptions, SinkFlow, Truncation, + UserSql, + }; use tablepro_test_support::{ - CallCounters, CloseBehaviour, ConnectionScript, FakeConnection, FakeEditorSession, FakeReply, Replies, + CallCounters, CloseBehaviour, ConnectionScript, FakeConnection, FakeEditorSession, FakeReply, FakeRun, Replies, }; use super::*; @@ -635,4 +642,63 @@ mod tests { assert_eq!(opened.id(), editor.id()); assert_eq!(CallCounters::get(&connection.counters().open_editor_session), 1); } + + #[derive(Clone, Default)] + struct Seen(Arc>>); + + #[async_trait::async_trait] + impl ResultSink for Seen { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + self.0.lock().unwrap().push(event); + SinkFlow::Continue + } + } + + #[tokio::test(start_paused = true)] + async fn an_opened_editor_session_keeps_the_runs_byte_limit() { + let wide = |mark: &str| vec![Value::Text(mark.repeat(1_000))]; + let columns: Arc<[ResultColumn]> = vec![ResultColumn::new("blob", ColumnType::untyped_text(""))].into(); + let mut runs = Replies::default(); + runs.always(FakeRun::completed(vec![ + ResultEvent::ResultSetStarted { columns }, + ResultEvent::Rows(vec![wide("a"), wide("b"), wide("c")]), + ResultEvent::ResultSetFinished { + retained: 3, + total_rows: Some(3), + truncation: Truncation::Complete, + }, + ])); + let editor: Arc = Arc::new(FakeEditorSession::new(runs)); + let mut script = ConnectionScript::default(); + script.open_editor_session.once(FakeReply::ok(editor)); + let (session, _) = start(script); + let opened = session + .open_editor_session(EditorSessionOptions::default(), &CancellationToken::new()) + .await + .unwrap(); + let seen = Seen::default(); + let limit = ByteLimit::new(1_500).unwrap(); + + let summary = opened + .run( + UserSql::new("SELECT blob"), + RunOptions::new(CancellationToken::new()).with_byte_limit(limit), + Box::new(seen.clone()), + ) + .await + .unwrap(); + + assert_eq!(summary.end, RunEnd::Completed); + let events = seen.0.lock().unwrap().clone(); + assert_eq!(events[1], ResultEvent::Rows(vec![wide("a"), wide("b")])); + assert_eq!(events[2], ResultEvent::ByteLimitReached { retained: 2 }); + assert_eq!( + events[3], + ResultEvent::ResultSetFinished { + retained: 2, + total_rows: Some(3), + truncation: Truncation::ByteLimit, + } + ); + } } diff --git a/core/crates/session/tests/session_mongodb.rs b/core/crates/session/tests/session_mongodb.rs index 375a7baf63..0a6d1ee443 100644 --- a/core/crates/session/tests/session_mongodb.rs +++ b/core/crates/session/tests/session_mongodb.rs @@ -154,6 +154,7 @@ impl Grid { changes: vec![RowChange::Update(RowUpdate { key: vec![Value::Int(id)], assignments: vec![(self.column(field), CellInput::Value(Value::Text(text.to_owned())))], + read: Vec::new(), })], }; let grid = self.clone(); diff --git a/core/crates/test-support/src/cell_expressions.rs b/core/crates/test-support/src/cell_expressions.rs index dc9f4b3474..ea14fbc8cf 100644 --- a/core/crates/test-support/src/cell_expressions.rs +++ b/core/crates/test-support/src/cell_expressions.rs @@ -96,6 +96,7 @@ fn update(index: usize, input: CellInput) -> RowChange { RowChange::Update(RowUpdate { key: vec![Value::Int(1)], assignments: vec![(index, input)], + read: Vec::new(), }) } diff --git a/core/crates/test-support/src/conformance/cases/byte_limit.rs b/core/crates/test-support/src/conformance/cases/byte_limit.rs new file mode 100644 index 0000000000..bcef66450f --- /dev/null +++ b/core/crates/test-support/src/conformance/cases/byte_limit.rs @@ -0,0 +1,211 @@ +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use tablepro_core::meta::TableRef; +use tablepro_core::value::Value; +use tablepro_core::{ + ByteLimit, ByteLimitSink, EditorSessionOptions, ResultEvent, ResultSink, RunEnd, RunOptions, SinkFlow, Truncation, + UserSql, WriteKind, +}; +use tokio_util::sync::CancellationToken; + +use crate::conformance::support::{ + apply, connect, create_scratch_table, dial, expect, insert_sql, run_completed, run_options, unique, writable, +}; +use crate::conformance::{ConformanceEngine, ConformanceFailure, DocumentEngine, test_liveness}; +use crate::lock; + +const CASE: &str = "byte_limit_keeps_whole_rows_and_later_statements"; +const DOCUMENT_CASE: &str = "a_document_read_stops_at_the_byte_limit"; + +const ROWS: usize = 40; +// Under the narrowest scratch label column, SQL Server's nvarchar(200). +const LABEL_CHARS: usize = 160; +const KEPT_ROWS: u64 = 16; +const DOCUMENTS: u32 = 1_234; +const DOCUMENT_LIMIT_BYTES: u64 = 1_024; + +fn label(id: i64) -> String { + format!("{id:04}{}", "x".repeat(LABEL_CHARS - 4)) +} + +fn text_of(value: &Value) -> Option<&str> { + match value { + Value::Text(text) => Some(text), + Value::Bytes(bytes) => std::str::from_utf8(bytes).ok(), + Value::Other(other) => Some(&other.text), + _ => None, + } +} + +fn row_bytes(row: &[Value]) -> u64 { + row.iter().map(|value| value.retained_bytes() as u64).sum() +} + +#[derive(Clone, Default)] +struct Events(Arc>>); + +#[async_trait] +impl ResultSink for Events { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + lock(&self.0).push(event); + SinkFlow::Continue + } +} + +impl Events { + fn rows(&self) -> Vec> { + lock(&self.0) + .iter() + .filter_map(|event| match event { + ResultEvent::Rows(rows) => Some(rows.clone()), + _ => None, + }) + .flatten() + .collect() + } + + fn column(&self, name: &str) -> Option { + lock(&self.0).iter().find_map(|event| match event { + ResultEvent::ResultSetStarted { columns } => columns.iter().position(|column| column.name == name), + _ => None, + }) + } + + // One `ByteLimitReached` for the rows kept, and a finish that says the set was cut for size. + fn check_cut(&self, case: &'static str, kept: usize) -> Result<(), ConformanceFailure> { + let events = lock(&self.0).clone(); + let reached: Vec = events + .iter() + .filter_map(|event| match event { + ResultEvent::ByteLimitReached { retained } => Some(*retained), + _ => None, + }) + .collect(); + expect(case, reached == [kept as u64], || { + format!("{kept} rows were kept, but the byte limit events were {reached:?}") + })?; + let truncations: Vec = events + .iter() + .filter_map(|event| match event { + ResultEvent::ResultSetFinished { truncation, .. } => Some(*truncation), + _ => None, + }) + .collect(); + expect(case, truncations == [Truncation::ByteLimit], || { + format!("the read finished with {truncations:?}") + }) + } +} + +// Through the sink the session puts around every editor run, as an app's run goes. +pub async fn byte_limit_keeps_whole_rows_and_later_statements( + engine: E, +) -> Result<(), ConformanceFailure> { + let connection = connect(CASE, &engine, writable()).await?; + let table = create_scratch_table(&engine, connection.as_ref(), "byte_limit").await?; + let dialect = connection.dialect(); + let inserts = (1..=ROWS as i64) + .map(|id| insert_sql(dialect.as_ref(), &table, id, &label(id))) + .collect(); + apply(connection.as_ref(), &table, WriteKind::RowChanges, inserts).await?; + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await?; + let row = (2 * size_of::() + LABEL_CHARS) as u64; + let limit = ByteLimit::new(row * KEPT_ROWS).ok_or_else(|| ConformanceFailure::setup("a byte limit of zero"))?; + let select = format!( + "SELECT {}, {} FROM {}", + dialect.quote_identifier("id"), + dialect.quote_identifier("label"), + dialect.qualify(&table) + ); + + let events = Events::default(); + let sink = Box::new(ByteLimitSink::new(Box::new(events.clone()), limit)); + let options = RunOptions::new(CancellationToken::new()).with_byte_limit(limit); + let summary = session.run(UserSql::new(select), options, sink).await?; + + expect(CASE, summary.end == RunEnd::Completed, || { + format!("the limited read ended {:?}", summary.end) + })?; + let rows = events.rows(); + let kept = rows.len(); + expect(CASE, kept > 0 && kept < ROWS, || { + format!("the read kept {kept} of {ROWS} rows") + })?; + let before_last: u64 = rows.iter().take(kept - 1).map(|row| row_bytes(row)).sum(); + let with_last = rows.iter().map(|row| row_bytes(row)).sum::(); + expect(CASE, before_last < limit.get() && with_last >= limit.get(), || { + format!( + "{kept} rows were kept: {before_last} bytes before the last and {with_last} with it, under {}", + limit.get() + ) + })?; + let whole = rows.iter().all(|row| { + row.get(1) + .and_then(text_of) + .is_some_and(|text| text.len() == LABEL_CHARS && text.ends_with('x')) + }); + expect(CASE, whole, || "a kept label was not whole".to_owned())?; + events.check_cut(CASE, kept)?; + let after = run_completed(CASE, session.as_ref(), &engine.row_source_sql(1), run_options()).await?; + let counted = after.rows().len(); + expect(CASE, counted == 1, || { + format!("the statement after the cut returned {counted} rows") + })?; + + session.close().await; + connection.close().await?; + Ok(()) +} + +// No sink here: a document engine holds a cursor's documents before its first row, so the driver itself stops. +pub async fn a_document_read_stops_at_the_byte_limit(engine: E) -> Result<(), ConformanceFailure> { + let options = engine.connect_options(writable(), test_liveness()); + let connection = dial(DOCUMENT_CASE, E::driver(), options).await??; + let session = connection + .open_editor_session(EditorSessionOptions::default(), CancellationToken::new()) + .await?; + let table = TableRef::new(Some(engine.database().to_owned()), unique("documents")); + let fill = engine.fill_statements(&table.name, DOCUMENTS); + run_completed(DOCUMENT_CASE, session.as_ref(), &fill, run_options()).await?; + let limit = + ByteLimit::new(DOCUMENT_LIMIT_BYTES).ok_or_else(|| ConformanceFailure::setup("a byte limit of zero"))?; + + let events = Events::default(); + let find = format!("db.getCollection(\"{}\").find()", table.name); + let options = RunOptions::new(CancellationToken::new()).with_byte_limit(limit); + let summary = session + .run(UserSql::new(find), options, Box::new(events.clone())) + .await?; + + expect(DOCUMENT_CASE, summary.end == RunEnd::Completed, || { + format!("the limited find ended {:?}", summary.end) + })?; + let label = events + .column("label") + .ok_or_else(|| ConformanceFailure::setup("the find has no label column"))?; + let rows = events.rows(); + let kept = rows.len(); + expect(DOCUMENT_CASE, kept > 0 && kept < DOCUMENTS as usize, || { + format!("the find kept {kept} of {DOCUMENTS} documents under {DOCUMENT_LIMIT_BYTES} bytes") + })?; + let whole = rows.iter().all(|row| { + row.get(label) + .and_then(text_of) + .is_some_and(|text| text.starts_with("row ")) + }); + expect(DOCUMENT_CASE, whole, || "a kept label was not whole".to_owned())?; + events.check_cut(DOCUMENT_CASE, kept)?; + let one = format!("db.getCollection(\"{}\").find({{_id: 1}})", table.name); + let after = run_completed(DOCUMENT_CASE, session.as_ref(), &one, run_options()).await?; + let found = after.rows().len(); + expect(DOCUMENT_CASE, found == 1, || { + format!("the find after the cut returned {found} rows") + })?; + + session.close().await; + connection.close().await?; + Ok(()) +} diff --git a/core/crates/test-support/src/conformance/cases/document.rs b/core/crates/test-support/src/conformance/cases/document.rs index 9fbe9757b4..ced6731d52 100644 --- a/core/crates/test-support/src/conformance/cases/document.rs +++ b/core/crates/test-support/src/conformance/cases/document.rs @@ -108,7 +108,7 @@ fn ids(read: &PageRead) -> Vec { .rows .iter() .filter_map(|row| match row.first() { - Some(Value::Int(id)) => Some(*id), + Some(Value::Int32(id)) => Some(i64::from(*id)), _ => None, }) .collect() @@ -294,7 +294,7 @@ async fn label_of( case: &'static str, connection: &dyn Connection, table: &TableRef, - id: i64, + id: i32, ) -> Result, ConformanceFailure> { let read = page( case, @@ -309,7 +309,7 @@ async fn label_of( .result .rows .iter() - .find(|row| row.first() == Some(&Value::Int(id))) + .find(|row| row.first() == Some(&Value::Int32(id))) .and_then(|row| row.get(1).cloned())) } @@ -324,8 +324,9 @@ pub async fn a_field_edit_saves_and_reads_back(engine: E) -> connection.as_ref(), &table, vec![RowChange::Update(RowUpdate { - key: vec![Value::Int(7)], + key: vec![Value::Int32(7)], assignments: vec![(1, CellInput::Value(Value::Text("renamed".to_owned())))], + read: Vec::new(), })], ) .await??; @@ -354,7 +355,7 @@ pub async fn a_save_on_a_document_that_is_gone_changes_nothing(engine: E) -> Result ], }, RowChange::Delete { - key: vec![Value::Int(1)], + key: vec![Value::Int32(1)], }, ], ) @@ -477,6 +479,7 @@ pub async fn a_save_that_fails_at_its_second_step_keeps_the_first Re fn id_of(value: &Value) -> Option { match value { + Value::Int32(id) => u64::try_from(*id).ok(), Value::Int(id) => u64::try_from(*id).ok(), Value::UInt(id) => Some(*id), _ => None, diff --git a/core/crates/test-support/src/conformance/cases/mod.rs b/core/crates/test-support/src/conformance/cases/mod.rs index 23ba24b6b4..be0c55d74c 100644 --- a/core/crates/test-support/src/conformance/cases/mod.rs +++ b/core/crates/test-support/src/conformance/cases/mod.rs @@ -1,6 +1,7 @@ //! One file per area of the contract. Every case gets an engine started //! for it alone and returns a `ConformanceFailure` rather than panicking. +pub mod byte_limit; pub mod cancellation; pub mod close; pub mod commit_resolution; diff --git a/core/crates/test-support/src/conformance/cases/row_guards.rs b/core/crates/test-support/src/conformance/cases/row_guards.rs index 944e80e927..9560d9e3e4 100644 --- a/core/crates/test-support/src/conformance/cases/row_guards.rs +++ b/core/crates/test-support/src/conformance/cases/row_guards.rs @@ -87,6 +87,7 @@ fn relabel( changes: vec![RowChange::Update(RowUpdate { key: vec![Value::Int(id)], assignments: vec![(label, CellInput::Value(Value::Text("changed".to_owned())))], + read: Vec::new(), })], }; let plan = plan_save(connection.dialect().as_ref(), table, columns, identity, &changes) diff --git a/core/crates/test-support/src/conformance/cases/value_corpus.rs b/core/crates/test-support/src/conformance/cases/value_corpus.rs index 79f5d00e03..5b614f31fb 100644 --- a/core/crates/test-support/src/conformance/cases/value_corpus.rs +++ b/core/crates/test-support/src/conformance/cases/value_corpus.rs @@ -242,6 +242,7 @@ fn by_id(columns: &[ResultColumn], rows: Vec>) -> Result Some(i64::from(*key)), Some(Value::Int(key)) => Some(*key), Some(Value::UInt(key)) => i64::try_from(*key).ok(), Some(Value::Text(key)) => key.parse().ok(), diff --git a/core/crates/test-support/src/conformance/connection_suite_macro.rs b/core/crates/test-support/src/conformance/connection_suite_macro.rs index 31fe9b2611..6a1ef09b1b 100644 --- a/core/crates/test-support/src/conformance/connection_suite_macro.rs +++ b/core/crates/test-support/src/conformance/connection_suite_macro.rs @@ -227,6 +227,12 @@ macro_rules! connection_conformance_suite { stream_read_streams_every_row_in_bounded_batches, $crate::conformance::cases::streaming::stream_read_streams_every_row_in_bounded_batches::<$engine> } + $crate::__conformance_case! { + $docker, + $engine as ConformanceEngine, + byte_limit_keeps_whole_rows_and_later_statements, + $crate::conformance::cases::byte_limit::byte_limit_keeps_whole_rows_and_later_statements::<$engine> + } $crate::__conformance_endpoint_cases! { $endpoint, $stop, $docker, $engine } $crate::__conformance_pool_case! { $pool, $endpoint, $stop, $docker, $engine } $crate::__conformance_editor_case! { $editor, $docker, $engine } diff --git a/core/crates/test-support/src/conformance/document_suite_macro.rs b/core/crates/test-support/src/conformance/document_suite_macro.rs index 21c0917766..c0f97395f0 100644 --- a/core/crates/test-support/src/conformance/document_suite_macro.rs +++ b/core/crates/test-support/src/conformance/document_suite_macro.rs @@ -28,6 +28,12 @@ macro_rules! document_conformance_suite { document_close_is_idempotent, a_lost_transport_reports_connection_lost_to_a_read, } + $crate::__conformance_case! { + true, + $engine as DocumentEngine, + a_document_read_stops_at_the_byte_limit, + $crate::conformance::cases::byte_limit::a_document_read_stops_at_the_byte_limit::<$engine> + } #[::tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn document_suite_matches_driver_capabilities() diff --git a/core/crates/test-support/src/conformance/support.rs b/core/crates/test-support/src/conformance/support.rs index 43e640b8a1..59d2d12248 100644 --- a/core/crates/test-support/src/conformance/support.rs +++ b/core/crates/test-support/src/conformance/support.rs @@ -240,6 +240,7 @@ pub(crate) async fn run_completed( /// The single integer a count query returned. pub(crate) fn single_count(rows: &[Vec]) -> Option { match rows.first()?.first()? { + Value::Int32(count) => u64::try_from(*count).ok(), Value::Int(count) => u64::try_from(*count).ok(), Value::UInt(count) => Some(*count), Value::Text(count) => count.trim().parse().ok(), diff --git a/core/crates/test-support/src/fake_dialect.rs b/core/crates/test-support/src/fake_dialect.rs index e6c8a78327..b8fa63bbfe 100644 --- a/core/crates/test-support/src/fake_dialect.rs +++ b/core/crates/test-support/src/fake_dialect.rs @@ -41,6 +41,7 @@ impl SqlDialect for FakeDialect { match value { Value::Null => Ok("NULL".to_owned()), Value::Bool(flag) => Ok(flag.to_string()), + Value::Int32(number) => Ok(number.to_string()), Value::Int(number) => Ok(number.to_string()), Value::UInt(number) => Ok(number.to_string()), Value::Text(text) => Ok(format!("'{}'", text.replace('\'', "''"))), @@ -116,6 +117,7 @@ const CREDENTIAL_RULES: CredentialRules = CredentialRules { fn param_type_of(value: &Value) -> ParamType { match value { Value::Bool(_) => ParamType::Bool, + Value::Int32(_) => ParamType::Int32, Value::Int(_) => ParamType::Int64, Value::UInt(_) => ParamType::UInt64, Value::Float32(_) => ParamType::Float32, diff --git a/core/docs/decisions/0003-result-windows-for-native-grids.md b/core/docs/decisions/0003-result-windows-for-native-grids.md index 71204e8e1f..5c6a89b653 100644 --- a/core/docs/decisions/0003-result-windows-for-native-grids.md +++ b/core/docs/decisions/0003-result-windows-for-native-grids.md @@ -1,6 +1,6 @@ # 0003: The core owns the result set, and a native grid pulls windows of text -- **Status**: Accepted; amended by 0007 +- **Status**: Accepted; amended by 0007; the row limit as the bound on memory superseded by 0020 - **Date**: 2026-11-02 ## Context diff --git a/core/docs/decisions/0020-a-run-keeps-a-byte-limit-of-rows.md b/core/docs/decisions/0020-a-run-keeps-a-byte-limit-of-rows.md new file mode 100644 index 0000000000..3580228ca5 --- /dev/null +++ b/core/docs/decisions/0020-a-run-keeps-a-byte-limit-of-rows.md @@ -0,0 +1,35 @@ +# 0020: A run keeps a byte limit of rows + +- **Status**: Accepted +- **Date**: 2026-11-11 + +## Context + +[ADR 0003](0003-result-windows-for-native-grids.md) has the row limit bound what an editor run holds. A row has no size bound, so 10,000 rows of 2 MB values are 20 GB, and the FFI's store holds a batch about three times while it appends it. Every driver sent 256 rows per `Rows` event, whatever their size. MongoDB held every document up to the row limit before its first event, so no sink could stop it. + +## Decision + +`RunOptions::byte_limit` bounds the bytes of rows one editor run keeps, 256,000,000 by default, measured by `Value::retained_bytes`. `ByteLimitSink` in `tablepro-core` enforces it, and `Session::open_editor_session` wraps every editor session it opens, so each run goes through that sink. + +## Rationale + +- **Per run, not per value.** A kept row stays exact: a query result has no row identity to read the rest of a cut value by. The row that crosses the limit is kept whole, so a run shows at least one row. +- **Not a stop.** The sink drops rows past the limit and answers `Continue`, never `ConsumerGone`. A stop cancels the statement, and on PostgreSQL a cancelled statement aborts the user's transaction. Rows past the cut are still read up to the row limit, as rows past the row limit are. +- **Once, in the session.** One sink sees every engine's events, and the Linux app, the FFI and MCP only set `RunOptions`, so a later app cannot leave it out. A driver that holds a set before its first event stops on its own: MongoDB stops taking documents once their BSON bytes reach the limit, closes the cursor and sends `ByteLimitReached` itself. +- **The measure.** `retained_bytes` is the value's 32 bytes plus the length of its text, bytes or digits. It counts lengths, not capacities, so the same rows measure the same on every platform. The 10,000-row default reaches 256 MB only past about 25.6 KB a row, and a million rows of three short columns, about 107 MB, stays under it. +- **Its own words.** `Truncation::ByteLimit`, and `ResultEvent::ByteLimitReached { retained }` once per set at the cut. `ServerStopped` says the server stopped the statement, and `ReadPastLimit` names the row limit. + +## Consequences + +- A set the sink cut finishes `ByteLimit` with the driver's total: exact where the driver read to the end, `None` where it stopped at the row limit. A set the server also stopped finishes `ServerStopped`, since the user has to know the statement did not run to its end; `ByteLimitReached` has already said the size cut it. A set after the limit is used up keeps no rows, and an empty one stays `Complete`. +- A batch goes at 256 rows or 4 MiB of `retained_bytes` (`RowBatch`), so a batch of wide rows is not held several times over while a sink copies it. +- The FFI's `RunSettings.byte_limit` sets the limit per run, and `RunEvent::ByteLimitReached` carries it. The Linux editor gives each statement the bytes its run has left, as it does rows. +- Table pages are exempt. Linux reads a page through `read_page`, not a sink, and Windows pages pass the largest limit, because both pagers read a short page as the last. A page of wide rows is bounded by its row count alone until a page reads a prefix and the length of a long value, with the whole value read by row identity. +- Cassandra's page and Redis's reply are still decoded whole before the sink sees them; the sink bounds what is kept, not that. +- What crosses the wire does not change. + +## Alternatives considered + +- **A cap per value.** Each long value cut to a prefix. A query result cannot read the rest by key, so copy, the detail pane and an edit would act on the prefix. +- **The limit in each driver.** Nine copies of one rule and its words, and one more to remember for each engine added. +- **Stop the statement at the cut.** The read ends sooner, but an open PostgreSQL transaction is aborted, and each engine needs its own stop, such as a chunked FETCH for PostgreSQL's cursor. diff --git a/core/docs/decisions/0021-an-int32-keeps-its-width.md b/core/docs/decisions/0021-an-int32-keeps-its-width.md new file mode 100644 index 0000000000..8a74cd4bd6 --- /dev/null +++ b/core/docs/decisions/0021-an-int32-keeps-its-width.md @@ -0,0 +1,34 @@ +# 0021: An int32 keeps its width, and an edited number keeps its type + +- **Status**: Accepted +- **Date**: 2026-11-11 + +## Context + +[Linux ADR 0014](../../../linux/docs/decisions/0014-lossless-values-and-dialects.md) gives `Value` one signed integer, `Int(i64)`, and leaves an integer's width to its column. That holds where the column declares a type. A MongoDB column is a census instead: the field's type over the first 1,000 documents ([drivers/mongodb.md](../../../linux/docs/drivers/mongodb.md#catalogue)). A field the census finds with numbers of two BSON types is a JSON column, which keeps each value's type. A document past the census can still hold an `int` in a field the census typed `long` or `double`. The driver read an int32 and a long as the same `Int`, and an edit was parsed by the column's kind and bound by its type name, so that edit rewrote the value's BSON type: an `int` became a `long` or a `double`. A long past 2^53 lost digits in a `double` field, and a long past the census in an `int` field was refused. Extended JSON and Copy as mongosh wrote a long that fits 32 bits as an int. + +## Decision + +`Value::Int32(i32)` holds an int32 that an engine types apart from its column, and an edited number is written back in the type it was read as when the new value is exactly one of that type. + +## Rationale + +- **The width lives in the value.** ADR 0014 puts what the server sent in the value, with a variant per width where one matters (`Float32`, `Float64`). The read value already reaches the edit, the tracker and the FFI store, so only the int32 width was missing. Only the MongoDB driver produces `Int32`; every other engine keeps `Int` and the width on the column. +- **The same number either way.** `same_value`, `ValueIdentity`, `value::order` and the document brackets treat `Int32(n)` and `Int(n)` as one number, so an edit back to the read value is no edit and an `_id` key matches as MongoDB's index does. +- **The save carries the read value.** `RowUpdate::read` holds each assigned cell's read value, which the Linux tracker already kept as `CellEdit.original`. `document::bind_param` binds a number in the read's type when the new value is exactly a value of it: an integer in range, or a double whose shortest form is the decimal typed. Otherwise, and for a null, missing or new cell, the column's type applies. SQL engines ignore it. +- **The edit parses as the cell.** `parse_edit_text` takes the read value and parses as its number type first, but only where the column's kind does not hold the read, which a declared column never shows. So a long keeps its digits in a `double` field and is not refused in an `int` one. +- **Text writes the width.** Extended JSON writes `Int32` bare and `Int` as `$numberLong`; a mongosh literal writes `Int` as `NumberLong` unless the column is `int` and the value fits. +- **The FFI does not change.** `ValueKind::of_value` maps `Int32` to `Integer`; the result store keeps the width in a byte outside `ValueKind`, as it keeps a null, so a window still sends `Integer`. + +## Consequences + +- Every exhaustive match on `Value` has an `Int32` arm; each SQL driver binds and spells it as the integer it is. +- An edited int32 stays an int32, a long a long, a double a double and a Decimal128 a Decimal128, unless the new value is not one of those. +- A long that fits 32 bits in an `int` field, past the census, is still refused when edited past 32 bits: the column's range applies, since nothing tells it from an int. +- Windows sends `Integer` for an int32 cell. A save from Windows needs the read value with its width, which the core's result store holds. + +## Alternatives considered + +- **Only the census decides.** It makes a field of mixed numbers JSON, but it cannot see a value past its first 1,000 documents. +- **Read the document at save time.** A race between the read and the write, and the statement no longer shows what is sent, which the write confirmation compares. +- **A per-row type vector beside the rows.** Every layer that carries a cell would carry it too: the page, the grid, the tracker and the FFI store. diff --git a/core/docs/decisions/README.md b/core/docs/decisions/README.md index 776cfa3b6f..8984d95167 100644 --- a/core/docs/decisions/README.md +++ b/core/docs/decisions/README.md @@ -10,7 +10,7 @@ The format, the statuses and how a later ADR changes an earlier one follow [the |---|---|---|---| | [0001](0001-shared-core.md) | One shared Rust core, a native UI per platform | Accepted; SSH on Windows amended by 0008 | The UI-free crates move to `core/crates` and build on Linux, Windows and macOS; each app keeps a native UI and passes its platform services in. | | [0002](0002-uniffi-bindings.md) | One UniFFI API for Swift and C#, with a C ABI beside it | Accepted | UniFFI proc macros in the core; Swift through UniFFI's generator and C# through uniffi-bindgen-cs v0.11.1+v0.31.0, each with a thin layer for cancel, progress, streams and off-UI-thread calls; `extern "C"` and csbindgen for hot paths and as the fallback. | -| [0003](0003-result-windows-for-native-grids.md) | The core owns the result set, and a native grid pulls windows of text | Accepted; amended by 0007 | A columnar result set in the core; one packed UTF-8 window per visible range with exact, locale-free text; one coalesced rows-arrived callback; Arrow only for bulk export. | +| [0003](0003-result-windows-for-native-grids.md) | The core owns the result set, and a native grid pulls windows of text | Accepted; amended by 0007; the row limit as the bound on memory superseded by 0020 | A columnar result set in the core; one packed UTF-8 window per visible range with exact, locale-free text; one coalesced rows-arrived callback; Arrow only for bulk export. | | [0004](0004-driver-plugin-abi.md) | Downloadable drivers over a small C ABI of our own | Accepted | A static descriptor, minor-version negotiation, protobuf messages, Arrow C Data rows, a runtime per driver and `catch_unwind` at every export; signed per platform, never unloaded, linked statically on iOS. | | [0005](0005-sql-server-transport.md) | SQL Server on tiberius everywhere, with Kerberos added and ODBC as a Windows option | Accepted | tiberius on every platform with SSPI Negotiate added in a fork, Entra ID tokens from the core, and an optional Windows connection type over an installed ODBC Driver 18 for Always Encrypted; msodbcsql18 is never bundled. | | [0006](0006-connection-library.md) | The connection library holds a group tree, manual order and favourites | Accepted; how the Linux app shows groups superseded by linux 0032, the version 2 upgrade by linux 0033 | Groups form a tree by id, connections and groups carry an explicit order, favourites are one ordered list; version 2 group names become top-level groups. | @@ -27,3 +27,5 @@ The format, the statuses and how a later ADR changes an earlier one follow [the | [0017](0017-favourites.md) | Favourites keep folders and keywords on saved queries, and the tables and databases a user marks | Accepted | One migration in the history's database adds per-connection folders that nest, a keyword and an order on saved queries, and favourite tables and databases with the four macOS environments; stable error codes, no UUID ids, and a database an older build wrote is copied first. | | [0018](0018-command-line.md) | One command line for the apps on the core | Accepted | `tablepro-cli` parses with clap, writes the macOS command's formats and exit status, and reaches the app's MCP server with the token the app's host reads; `classify` leaves files, links and GUI options to the app. | | [0019](0019-row-edits-through-the-ffi.md) | A table tab's row edits cross the FFI as typed text, and the core plans and runs the save | Accepted | `TableEditor` over the catalog's primary key reads keys and stored values from the result set, parses typed text with `parse_edit_text` and plans with `plan_save`; `RowSave` shows its statements with their values, gates at `SafeModeLevel::gate(Write)`, plans again on the writing lease and reports saved, rolled back or unknown. | +| [0020](0020-a-run-keeps-a-byte-limit-of-rows.md) | A run keeps a byte limit of rows | Accepted | `RunOptions::byte_limit`, 256 MB by default, bounds the rows an editor run keeps, measured by `Value::retained_bytes`; the session wraps every editor session in `ByteLimitSink`, which keeps whole rows, never stops the statement and finishes a cut set as `Truncation::ByteLimit`; MongoDB stops reading itself; table pages are exempt. | +| [0021](0021-an-int32-keeps-its-width.md) | An int32 keeps its width, and an edited number keeps its type | Accepted | `Value::Int32` holds MongoDB's int32, equal to the `Int` of the same number; a save carries each edited cell's read value, and a number binds in the read's BSON type when it is exactly one, else the column's. | diff --git a/linux/changelog.d/4201.changed.md b/linux/changelog.d/4201.changed.md new file mode 100644 index 0000000000..ca3a8be051 --- /dev/null +++ b/linux/changelog.d/4201.changed.md @@ -0,0 +1 @@ +A query run stops keeping rows once they pass 256 MB, and the banner over a result says when that cut it short. diff --git a/linux/changelog.d/4201.fixed.1.md b/linux/changelog.d/4201.fixed.1.md new file mode 100644 index 0000000000..d51f6be302 --- /dev/null +++ b/linux/changelog.d/4201.fixed.1.md @@ -0,0 +1 @@ +The banner over a PostgreSQL result cut at the row limit said the statement ran no further than the rows shown. diff --git a/linux/changelog.d/4201.fixed.md b/linux/changelog.d/4201.fixed.md new file mode 100644 index 0000000000..218f17bee7 --- /dev/null +++ b/linux/changelog.d/4201.fixed.md @@ -0,0 +1 @@ +Editing a number in a MongoDB collection could change its stored type, such as saving an int as a long. diff --git a/linux/crates/app/src/services/change_origins.rs b/linux/crates/app/src/services/change_origins.rs index 8ef43b7680..31eff402c1 100644 --- a/linux/crates/app/src/services/change_origins.rs +++ b/linux/crates/app/src/services/change_origins.rs @@ -36,6 +36,7 @@ mod tests { RowChange::Update(RowUpdate { key: vec![Value::Int(1)], assignments: Vec::new(), + read: Vec::new(), }), RowChange::Delete { key: vec![Value::Int(2)], diff --git a/linux/crates/app/src/services/change_tracker.rs b/linux/crates/app/src/services/change_tracker.rs index 7e12f7a1ab..28b3d49fde 100644 --- a/linux/crates/app/src/services/change_tracker.rs +++ b/linux/crates/app/src/services/change_tracker.rs @@ -11,6 +11,7 @@ use tablepro_core::dml::BuildSqlError; use tablepro_core::edit::CellInput; use tablepro_core::meta::RowIdentity; use tablepro_core::save::{ChangeSet, RowChange, RowUpdate}; +use tablepro_core::value::same_value; use tablepro_core::{ColumnInfo, Value}; use super::cell_overlay::{CellMarks, CellOverlay}; @@ -80,7 +81,10 @@ pub struct CellEdit { impl CellEdit { /// Whether the edit leaves the cell as it was before it. fn is_no_change(&self) -> bool { - matches!((&self.pending, &self.original), (CellInput::Value(pending), Some(original)) if pending == original) + matches!( + (&self.pending, &self.original), + (CellInput::Value(pending), Some(original)) if same_value(pending, original) + ) } } @@ -746,6 +750,10 @@ impl TabChangeTracker { require_key(columns)?; cells.sort_by_key(|(col, _)| *col); let touched = cells.iter().map(|(_, edit)| edit.touched).max().unwrap_or_default(); + let read = cells + .iter() + .filter_map(|(col, edit)| edit.original.clone().map(|value| (*col, value))) + .collect(); rows.push(( touched, RowChange::Update(RowUpdate { @@ -754,6 +762,7 @@ impl TabChangeTracker { .into_iter() .map(|(col, edit)| (col, edit.pending.clone())) .collect(), + read, }), StatementSource::Update { row_key: row_key.clone(), @@ -1702,6 +1711,7 @@ mod tests { vec![RowChange::Update(RowUpdate { key: vec![Value::Int(1), Value::Int(2)], assignments: vec![(2, CellInput::Value(Value::Text("new".into())))], + read: vec![(2, Value::Text("old".into()))], })] ); assert_eq!( @@ -1712,6 +1722,54 @@ mod tests { ); } + #[test] + fn a_change_set_carries_each_edited_cells_read_value() { + let mut t = TabChangeTracker::default(); + let columns = vec![pk_col("id"), data_col("qty"), data_col("price")]; + let key = RowKey::from_pk_values(&[Value::Int(1)]).unwrap(); + for (col, original, value) in [ + (2, Value::Float64(2.5), Value::Float64(3.5)), + (1, Value::Int32(3), Value::Int(4)), + ] { + t.apply(TrackerEdit::Cell { + row_key: key.clone(), + col, + original, + value: CellInput::Value(value), + }); + } + + let tracked = t.change_set(&columns).unwrap(); + + assert_eq!( + tracked.changes.changes, + vec![RowChange::Update(RowUpdate { + key: vec![Value::Int(1)], + assignments: vec![ + (1, CellInput::Value(Value::Int(4))), + (2, CellInput::Value(Value::Float64(3.5))), + ], + read: vec![(1, Value::Int32(3)), (2, Value::Float64(2.5))], + })] + ); + + t.apply(TrackerEdit::Cell { + row_key: key, + col: 1, + original: Value::Int32(3), + value: CellInput::Value(Value::Int(3)), + }); + assert_eq!( + t.change_set(&columns).unwrap().changes.changes, + vec![RowChange::Update(RowUpdate { + key: vec![Value::Int(1)], + assignments: vec![(2, CellInput::Value(Value::Float64(3.5)))], + read: vec![(2, Value::Float64(2.5))], + })], + "an int32 edited back to its number stayed an edit" + ); + } + #[test] #[expect( clippy::disallowed_methods, diff --git a/linux/crates/app/src/services/grid/display_text.rs b/linux/crates/app/src/services/grid/display_text.rs index 1e6b3fb3ec..5f1a90cd4c 100644 --- a/linux/crates/app/src/services/grid/display_text.rs +++ b/linux/crates/app/src/services/grid/display_text.rs @@ -130,6 +130,7 @@ fn write_value(value: &Value, column: &ColumnType, date_format: DateFormat, scra Value::UInt(flag @ (0 | 1)) if column.kind == ColumnKind::Bool(BoolStorage::Integer) => { scratch.write_str(if *flag == 1 { "true" } else { "false" }) } + Value::Int32(number) => write!(scratch, "{number}"), Value::Int(number) => write!(scratch, "{number}"), Value::UInt(number) => write!(scratch, "{number}"), Value::Float32(number) if number.is_finite() => write!(scratch, "{number}"), @@ -337,6 +338,7 @@ mod tests { assert_eq!(shown(&Value::Bool(true)), "true"); assert_eq!(shown(&Value::Int(42)), "42"); assert_eq!(shown(&Value::UInt(7)), "7"); + assert_eq!(shown(&Value::Int32(-7)), "-7"); assert_eq!(shown(&Value::Float64(1.5)), "1.5"); assert_eq!(shown(&Value::Float32(0.1)), "0.1"); assert_eq!(shown(&Value::Text("hello".into())), "hello"); diff --git a/linux/crates/app/src/services/grid/edit_parse.rs b/linux/crates/app/src/services/grid/edit_parse.rs index 5d3defc081..e2e951099f 100644 --- a/linux/crates/app/src/services/grid/edit_parse.rs +++ b/linux/crates/app/src/services/grid/edit_parse.rs @@ -64,7 +64,7 @@ pub fn commit_edit( if matches!(&opened_with, EditText::Editable(text) if text == input) { return CommitResult::Unchanged; } - match parse_cell_text(input, catalog, &start.row, dialect) { + match parse_cell_text(input, catalog, &start.row, &start.read, dialect) { Ok(parsed) => commit_input(column, start, parsed), Err(error) => { tracing::debug!(%error, column = %catalog.name, "refused an edit"); @@ -79,11 +79,12 @@ pub fn parse_cell_text( text: &str, column: &ColumnInfo, row: &GridRowId, + read: &Value, dialect: Option<&dyn SqlDialect>, ) -> Result { match dialect.and_then(|dialect| typed_cell_expression(dialect, &column.column_type, text)) { Some(expression) => Ok(CellInput::Expression(expression)), - None => parse_edit_text(text, column, edit_context(row)), + None => parse_edit_text(text, column, edit_context(row), Some(read)), } } @@ -129,13 +130,13 @@ pub fn empty_placeholder(column: &GridColumn, row: &GridRowId) -> Option #[cfg(test)] mod tests { - use tablepro_core::column::{ColumnDefault, ColumnKind}; + use tablepro_core::column::{ColumnDefault, ColumnKind, FloatWidth}; use tablepro_core::edit::{CellInput, EditParseError}; use tablepro_core::{ColumnInfo, Value}; use super::{CommitResult, EditStart, commit_edit, empty_placeholder}; use crate::services::change_tracker::RowKey; - use crate::services::grid::test_columns::{grid, info, persisted, result_only}; + use crate::services::grid::test_columns::{grid, info, info_of_kind, persisted, result_only}; use crate::services::grid::{GridColumn, GridRowId}; use crate::services::tracker_edit::TrackerEdit; @@ -249,6 +250,28 @@ mod tests { assert_eq!(commit_edit(&text, &start, "old", None), CommitResult::Unchanged); } + #[test] + fn a_number_a_sampled_column_does_not_type_keeps_the_type_it_was_read_as() { + let mut catalog = info_of_kind("c", ColumnKind::Float(FloatWidth::F64)); + catalog.column_type.dynamic_storage = false; + let mut double = grid(catalog); + double.index = 1; + + assert_eq!( + value_of(commit_edit(&double, &saved(Value::Int32(3)), "4", None)), + Some(CellInput::Value(Value::Int32(4))) + ); + assert_eq!( + value_of(commit_edit(&double, &saved(Value::Int32(3)), "4.5", None)), + Some(CellInput::Value(Value::Float64(4.5))) + ); + assert_eq!( + value_of(commit_edit(&double, &saved(Value::Float64(2.5)), "4", None)), + Some(CellInput::Value(Value::Float64(4.0))), + "a double the column types parsed as the read's type" + ); + } + #[test] fn a_value_the_column_cannot_hold_is_refused_with_its_reason() { let integer = column("integer", false, None); diff --git a/linux/crates/app/src/services/grid/selection_summary.rs b/linux/crates/app/src/services/grid/selection_summary.rs index 2f8e8f7941..d6cb7edad0 100644 --- a/linux/crates/app/src/services/grid/selection_summary.rs +++ b/linux/crates/app/src/services/grid/selection_summary.rs @@ -156,6 +156,7 @@ impl Tally { /// what its cell shows, so `0.1 + 0.2` is `0.3`. fn number_of(value: &Value) -> Option { let decimal = match value { + Value::Int32(n) => return Some(Exact::integer(BigInt::from(*n))), Value::Int(n) => return Some(Exact::integer(BigInt::from(*n))), Value::UInt(n) => return Some(Exact::integer(BigInt::from(*n))), Value::Text(text) => Cow::Owned(text.parse::().ok()?), @@ -309,6 +310,7 @@ mod tests { "-1" ); assert_eq!(numbers(&[Value::Int(-1), Value::Int(-2)], NUMERIC).average, "-1.5"); + assert_eq!(numbers(&[Value::Int32(-1), Value::Int(-2)], NUMERIC).average, "-1.5"); assert_eq!( numbers(&[Value::Int(1), Value::Int(1), Value::Int(0)], NUMERIC).average, "0.6667" diff --git a/linux/crates/app/src/ui/data_grid/edit/paste.rs b/linux/crates/app/src/ui/data_grid/edit/paste.rs index c9016072fe..34ae366d39 100644 --- a/linux/crates/app/src/ui/data_grid/edit/paste.rs +++ b/linux/crates/app/src/ui/data_grid/edit/paste.rs @@ -144,10 +144,14 @@ impl EditController { return; }; let dialect = grid.sql_dialect(); + let read = row + .cells() + .and_then(|cells| cells.get(column).cloned()) + .unwrap_or(Value::Null); let input = match text { None if cell_presets(&grid_column, &row.id(), dialect.as_deref()).null => Ok(CellInput::Value(Value::Null)), None => Err(EditParseError::Required), - Some(text) => parse_cell_text(text, catalog, &row.id(), dialect.as_deref()), + Some(text) => parse_cell_text(text, catalog, &row.id(), &read, dialect.as_deref()), }; match input { Ok(input) => { diff --git a/linux/crates/app/src/ui/data_grid/edit/row_edits.rs b/linux/crates/app/src/ui/data_grid/edit/row_edits.rs index 62bebac19d..9f880d3078 100644 --- a/linux/crates/app/src/ui/data_grid/edit/row_edits.rs +++ b/linux/crates/app/src/ui/data_grid/edit/row_edits.rs @@ -114,7 +114,7 @@ impl EditController { let presets = cell_presets(&grid_column, &row.id(), grid.sql_dialect().as_deref()); let input = match preset { CellPreset::Empty => { - parse_edit_text("", catalog, edit_context(&row.id())).map_err(|error| edit_parse_message(&error)) + parse_edit_text("", catalog, edit_context(&row.id()), None).map_err(|error| edit_parse_message(&error)) } CellPreset::Null if presets.null => Ok(CellInput::Value(Value::Null)), CellPreset::Null => Err(edit_parse_message(&EditParseError::Required)), @@ -282,8 +282,8 @@ pub(super) fn toggled(value: &Value, default: bool, storage: BoolStorage) -> Val let truth = match (default, value) { (true, _) => None, (false, Value::Bool(flag)) => Some(*flag), - (false, Value::Int(0) | Value::UInt(0)) => Some(false), - (false, Value::Int(1) | Value::UInt(1)) => Some(true), + (false, Value::Int(0) | Value::Int32(0) | Value::UInt(0)) => Some(false), + (false, Value::Int(1) | Value::Int32(1) | Value::UInt(1)) => Some(true), (false, _) => None, }; let next = truth.is_none_or(|truth| !truth); diff --git a/linux/crates/app/src/ui/editor/mod.rs b/linux/crates/app/src/ui/editor/mod.rs index 40e27ec2d1..4c3d37e30c 100644 --- a/linux/crates/app/src/ui/editor/mod.rs +++ b/linux/crates/app/src/ui/editor/mod.rs @@ -45,7 +45,7 @@ use libadwaita::prelude::*; use sourceview5::prelude::*; use tablepro_core::dialect::SqlDialect; use tablepro_core::sql_syntax::SqlGrammar; -use tablepro_core::{RowLimit, UserSql}; +use tablepro_core::{ByteLimit, RowLimit, UserSql}; use tablepro_session::runtime::{ProcessHold, Tasks}; use tokio_util::sync::CancellationToken; @@ -512,6 +512,7 @@ impl SqlEditor { .resolve(deps.settings.query_timeout()), row_limit: RowLimit::EDITOR_DEFAULT, run_row_limit: RUN_ROW_LIMIT, + byte_limit: ByteLimit::EDITOR_DEFAULT, }; let metadata = session.metadata().clone(); // Read again at the end from the run's own session, never the diff --git a/linux/crates/app/src/ui/editor/run_log.rs b/linux/crates/app/src/ui/editor/run_log.rs index 53344ddba3..801d89f7d2 100644 --- a/linux/crates/app/src/ui/editor/run_log.rs +++ b/linux/crates/app/src/ui/editor/run_log.rs @@ -1,7 +1,7 @@ use tablepro_core::sql_syntax::script::BatchErrorPolicy; use tablepro_core::{ - DriverError, EditorSessionState, ErrorCategory, ResultEvent, RunEnd, RunSummary, TransactionState, Truncation, - UserSql, + ByteLimit, DriverError, EditorSessionState, ErrorCategory, ResultEvent, RunEnd, RunSummary, TransactionState, + Truncation, UserSql, }; use super::script_runner::sql_preview; @@ -103,13 +103,19 @@ pub struct RowRoom { /// The statement was sent with `left` as its row limit, below its /// own, so a result the driver cut short was cut by the run's limit. pub limits_statement: bool, + // The run's byte limit, which a result it cuts short names, and the bytes of rows the run can still keep. + pub run_bytes: u64, + pub bytes_left: u64, } impl RowRoom { + // A result the session cut for size without a room from the run names the editor's own limit. pub const UNLIMITED: Self = Self { run_limit: u64::MAX, left: u64::MAX, limits_statement: false, + run_bytes: ByteLimit::EDITOR_DEFAULT.get(), + bytes_left: u64::MAX, }; } @@ -161,13 +167,19 @@ impl BatchCollector { tracing::warn!(rows = rows.len(), "rows arrived before their result set"); return; } - let kept = usize::try_from(self.room.left).unwrap_or(usize::MAX).min(rows.len()); + // The session keeps the row that crosses its limit, so a statement with no bytes left still brings one. + let full = self.room.bytes_left == 0; + let room = if full { 0 } else { self.room.left }; + let kept = usize::try_from(room).unwrap_or(usize::MAX).min(rows.len()); self.room.left -= kept as u64; - let cut_by = (kept < rows.len()).then_some(self.room.run_limit); + let cut = kept < rows.len(); + let cut_by = (cut && !full).then_some(self.room.run_limit); + let cut_for_size = (cut && full).then_some(self.room.run_bytes); if let Some(result) = self.last_set() { result.rows.extend(rows.into_iter().take(kept).map(Into::into)); result.summary.retained = result.rows.len() as u64; result.summary.run_limit = result.summary.run_limit.or(cut_by); + result.summary.size_limit = result.summary.size_limit.or(cut_for_size); } } // Until the set ends and says how far the read went, all @@ -180,6 +192,19 @@ impl BatchCollector { total_rows: None, truncation: Truncation::ServerStopped, run_limit: result.summary.run_limit.or(cut_by), + size_limit: result.summary.size_limit, + }; + } + } + ResultEvent::ByteLimitReached { .. } => { + let size = self.room.run_bytes; + if let Some(result) = self.last_set() { + result.summary = ResultSummary { + retained: result.rows.len() as u64, + total_rows: None, + truncation: Truncation::ByteLimit, + run_limit: result.summary.run_limit, + size_limit: result.summary.size_limit.or(Some(size)), }; } } @@ -188,12 +213,14 @@ impl BatchCollector { } => { let cut_by = (self.room.limits_statement && truncation == Truncation::ReadPastLimit) .then_some(self.room.run_limit); + let cut_for_size = (truncation == Truncation::ByteLimit).then_some(self.room.run_bytes); if let Some(result) = self.last_set() { result.summary = ResultSummary { retained: result.rows.len() as u64, total_rows, truncation, run_limit: result.summary.run_limit.or(cut_by), + size_limit: result.summary.size_limit.or(cut_for_size), }; } } @@ -603,6 +630,7 @@ mod tests { total_rows: Some(5), truncation: Truncation::ServerStopped, run_limit: None, + size_limit: None, } ); assert_eq!(result.statement.as_str(), "SELECT a FROM t"); @@ -657,6 +685,7 @@ mod tests { run_limit: 100, left: 1, limits_statement: true, + ..RowRoom::UNLIMITED }; let mut collector = BatchCollector::new(UserSql::new("SELECT a FROM t")).within(room); for event in [ @@ -682,6 +711,85 @@ mod tests { assert_eq!(result.summary.run_limit, Some(100)); } + fn first_summary(entries: &[EntryKind]) -> ResultSummary { + match entries.first() { + Some(EntryKind::ResultSet(result)) => result.summary, + other => panic!("expected rows: {other:?}"), + } + } + + #[test] + fn a_set_cut_for_size_says_so_before_and_after_it_ends() { + let cut = || { + vec![ + ResultEvent::ResultSetStarted { + columns: columns(&["a"]), + }, + ResultEvent::Rows(vec![vec![Value::Int(1)], vec![Value::Int(2)]]), + ResultEvent::ByteLimitReached { retained: 2 }, + ] + }; + let mut finished = cut(); + finished.push(ResultEvent::ResultSetFinished { + retained: 2, + total_rows: Some(9), + truncation: Truncation::ByteLimit, + }); + + let (stopped, _) = collect(cut(), RunEnd::Cancelled); + let (ended, _) = collect(finished, RunEnd::Completed); + + let size_limit = Some(ByteLimit::EDITOR_DEFAULT.get()); + assert_eq!( + first_summary(&stopped), + ResultSummary { + retained: 2, + total_rows: None, + truncation: Truncation::ByteLimit, + run_limit: None, + size_limit, + } + ); + assert_eq!( + first_summary(&ended), + ResultSummary { + retained: 2, + total_rows: Some(9), + truncation: Truncation::ByteLimit, + run_limit: None, + size_limit, + } + ); + } + + #[test] + fn a_batch_with_no_bytes_left_keeps_no_rows_and_says_why() { + let room = RowRoom { + run_bytes: 1_000, + bytes_left: 0, + ..RowRoom::UNLIMITED + }; + let mut collector = BatchCollector::new(UserSql::new("SELECT a FROM t")).within(room); + for event in [ + ResultEvent::ResultSetStarted { + columns: columns(&["a"]), + }, + ResultEvent::Rows(vec![vec![Value::Int(1)]]), + ResultEvent::ByteLimitReached { retained: 1 }, + ResultEvent::ResultSetFinished { + retained: 1, + total_rows: Some(3), + truncation: Truncation::ByteLimit, + }, + ] { + collector.record(event); + } + let (entries, _, _) = collector.finish(summary(RunEnd::Completed), EditorSessionState::Ready, false, 0); + + let kept = first_summary(&entries); + assert_eq!((kept.retained, kept.size_limit, kept.run_limit), (0, Some(1_000), None)); + } + #[test] fn a_set_cut_by_the_limit_and_then_stopped_stays_truncated() { let (entries, _) = collect( @@ -714,12 +822,14 @@ mod tests { total_rows: None, truncation: Truncation::ServerStopped, run_limit: None, + size_limit: None, }; let counted = ResultSummary { retained: 100, total_rows: Some(250), truncation: Truncation::ReadPastLimit, run_limit: None, + size_limit: None, }; let history = |entries: Vec| { let mut log = RunLog::new(vec![UserSql::new("SELECT a FROM t")], Vec::new()); diff --git a/linux/crates/app/src/ui/editor/script_runner.rs b/linux/crates/app/src/ui/editor/script_runner.rs index 4d0d1356cd..ba2cb7ed08 100644 --- a/linux/crates/app/src/ui/editor/script_runner.rs +++ b/linux/crates/app/src/ui/editor/script_runner.rs @@ -7,8 +7,8 @@ use tablepro_core::sql_syntax::SqlGrammar; use tablepro_core::sql_syntax::script::text_offsets::CharCounter; use tablepro_core::sql_syntax::script::{BatchErrorPolicy, LexicalSettings, ScriptDiagnostic, ScriptPlan}; use tablepro_core::{ - DriverError, EditorSession, EditorSessionState, ReadOnlyRefusal, ResultEvent, ResultSink, RowLimit, RunOptions, - SinkFlow, TransactionState, UserSql, + ByteLimit, DriverError, EditorSession, EditorSessionState, ReadOnlyRefusal, ResultEvent, ResultSink, RowLimit, + RunOptions, SinkFlow, TransactionState, UserSql, }; use tablepro_session::{MonitorSignal, Session}; use tokio::sync::mpsc; @@ -44,20 +44,50 @@ pub struct ScriptRun { pub statement_timeout: Option, pub row_limit: RowLimit, pub run_row_limit: u64, + pub byte_limit: ByteLimit, } -/// What a statement may keep once the run has kept `kept` rows, and the -/// row limit it goes out with. With no room left it still runs, for -/// what it changes, and reads one row, which is dropped. -fn room_after(run: &ScriptRun, kept: u64) -> (RowRoom, RowLimit) { - let left = run.run_row_limit.saturating_sub(kept); +#[derive(Debug, Clone, Copy, Default)] +struct Kept { + rows: u64, + bytes: u64, +} + +impl Kept { + fn add(&mut self, entries: &[EntryKind]) { + for entry in entries { + if let EntryKind::ResultSet(result) = entry { + self.rows += result.row_count() as u64; + self.bytes = result + .rows + .iter() + .flat_map(|row| row.iter()) + .map(|value| value.retained_bytes() as u64) + .fold(self.bytes, u64::saturating_add); + } + } + } +} + +/// What a statement may keep once the run has kept `kept`, and the row +/// and byte limits it goes out with. With no room left it still runs, +/// for what it changes, and reads one row, which is dropped. +fn room_after(run: &ScriptRun, kept: Kept) -> (RowRoom, RowLimit, ByteLimit) { + let left = run.run_row_limit.saturating_sub(kept.rows); let own = run.row_limit.get(); + let bytes_left = run.byte_limit.get().saturating_sub(kept.bytes); let room = RowRoom { run_limit: run.run_row_limit, left, limits_statement: left < own, + run_bytes: run.byte_limit.get(), + bytes_left, }; - (room, RowLimit::new(left.clamp(1, own)).unwrap_or(run.row_limit)) + ( + room, + RowLimit::new(left.clamp(1, own)).unwrap_or(run.row_limit), + ByteLimit::new(bytes_left.max(1)).unwrap_or(run.byte_limit), + ) } /// The batches of the scoped text and what the plan warns about. @@ -139,7 +169,7 @@ pub async fn run_batches( transaction, rollback_told: false, }; - let mut kept: u64 = 0; + let mut kept = Kept::default(); 'batches: for (index, batch) in run.batches.iter().enumerate() { if batch.refused { if events.send(RunEvent::Batch(refusal(index))).await.is_err() { @@ -155,11 +185,12 @@ pub async fn run_batches( break 'batches; } let sql = UserSql::new(batch.text.as_str()); - let (room, row_limit) = room_after(run, kept); + let (room, row_limit, byte_limit) = room_after(run, kept); let collector = Arc::new(Mutex::new(BatchCollector::new(sql.clone()).within(room))); let options = RunOptions::new(cancel.child_token()) .with_deadline(run.statement_timeout) - .with_row_limit(row_limit); + .with_row_limit(row_limit) + .with_byte_limit(byte_limit); let sink = Box::new(CollectingSink { collector: collector.clone(), }); @@ -178,13 +209,7 @@ pub async fn run_batches( } after.rollback_told |= lost && in_transaction && entries.iter().any(|entry| matches!(entry, EntryKind::Failed { .. })); - kept += entries - .iter() - .map(|entry| match entry { - EntryKind::ResultSet(result) => result.row_count() as u64, - _ => 0, - }) - .sum::(); + kept.add(&entries); let report = BatchReport { batch: index, entries, @@ -331,7 +356,8 @@ fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { #[cfg(test)] mod tests { - use tablepro_core::{DriverError, LossPhase, RunEnd, TransactionState}; + use tablepro_core::column::{ColumnType, ResultColumn}; + use tablepro_core::{DriverError, LossPhase, RunEnd, TransactionState, Truncation, Value}; use tablepro_test_support::{FakeEditorSession, FakeRun, Replies}; use super::super::editor_session_slot::tests::session_opening; @@ -355,6 +381,7 @@ mod tests { statement_timeout: None, row_limit: RowLimit::EDITOR_DEFAULT, run_row_limit: RUN_ROW_LIMIT, + byte_limit: ByteLimit::EDITOR_DEFAULT, }, statements, warnings, @@ -591,6 +618,16 @@ mod tests { /// every event into a results list. async fn run_in_slot(session: &Arc, slot: &Arc, text: &str) -> RunLog { let (run, statements, warnings) = script(text, SqlGrammar::PostgreSql); + run_script_in_slot(session, slot, run, statements, warnings).await + } + + async fn run_script_in_slot( + session: &Arc, + slot: &Arc, + run: ScriptRun, + statements: Vec, + warnings: Vec, + ) -> RunLog { let (events, mut received) = mpsc::channel(8); let running = run_in_tab( session.clone(), @@ -776,8 +813,8 @@ mod tests { fn a_statement_reads_only_the_rows_its_run_has_room_for() { let (mut run, _, _) = script("SELECT 1", SqlGrammar::PostgreSql); run.run_row_limit = 100_000; - let limit = |kept| { - let (room, limit) = room_after(&run, kept); + let limit = |rows| { + let (room, limit, _) = room_after(&run, Kept { rows, bytes: 0 }); (room.left, room.limits_statement, limit.get()) }; @@ -787,6 +824,61 @@ mod tests { assert_eq!(limit(250_000), (0, true, 1)); } + #[test] + fn a_statement_goes_out_with_the_bytes_its_run_has_left() { + let (mut run, _, _) = script("SELECT 1", SqlGrammar::PostgreSql); + run.byte_limit = ByteLimit::new(1_000).unwrap(); + let bytes = |bytes| { + let (room, _, limit) = room_after(&run, Kept { rows: 0, bytes }); + (room.run_bytes, room.bytes_left, limit.get()) + }; + + assert_eq!(bytes(0), (1_000, 1_000, 1_000)); + assert_eq!(bytes(600), (1_000, 400, 400)); + assert_eq!(bytes(1_000), (1_000, 0, 1)); + assert_eq!(bytes(5_000), (1_000, 0, 1)); + } + + fn wide_rows(rows: usize) -> FakeRun { + let columns: Arc<[ResultColumn]> = vec![ResultColumn::new("blob", ColumnType::untyped_text("text"))].into(); + FakeRun::completed(vec![ + ResultEvent::ResultSetStarted { columns }, + ResultEvent::Rows((0..rows).map(|_| vec![Value::Text("x".repeat(1_000))]).collect()), + ResultEvent::ResultSetFinished { + retained: rows as u64, + total_rows: Some(rows as u64), + truncation: Truncation::Complete, + }, + ]) + } + + // Through the window's session, whose editor sessions cut each statement at the bytes its run has left. + #[tokio::test] + async fn a_run_keeps_bytes_up_to_its_limit_across_all_results() { + let mut runs = Replies::default(); + runs.always(wide_rows(60)); + let editor = Arc::new(FakeEditorSession::new(runs)); + let (session, _) = session_opening(vec![editor.clone()]); + let slot = Arc::new(EditorSessionSlot::new()); + let (mut run, statements, warnings) = script("SELECT 1; SELECT 2; SELECT 3", SqlGrammar::PostgreSql); + let row = Value::Text("x".repeat(1_000)).retained_bytes() as u64; + run.byte_limit = ByteLimit::new(row * 100).unwrap(); + let limit = run.byte_limit.get(); + + let log = run_script_in_slot(&session, &slot, run, statements, warnings).await; + + assert_eq!(texts(&editor).len(), 3, "a statement past the limit was not sent"); + let kept: Vec<(usize, Option)> = log + .entries() + .iter() + .filter_map(|entry| match &entry.kind { + EntryKind::ResultSet(result) => Some((result.row_count(), result.summary.size_limit)), + _ => None, + }) + .collect(); + assert_eq!(kept, [(60, None), (40, Some(limit)), (0, Some(limit))]); + } + #[test] fn batches_know_where_they_start_in_the_buffer() { let scoped = ScopedText { diff --git a/linux/crates/app/src/ui/editor/tests.rs b/linux/crates/app/src/ui/editor/tests.rs index 72b5ef2683..1061d9ff82 100644 --- a/linux/crates/app/src/ui/editor/tests.rs +++ b/linux/crates/app/src/ui/editor/tests.rs @@ -885,7 +885,7 @@ fn truncated_result_reveals_banner_and_complete_result_hides_it() { assert!(capped.is_revealed(), "the capped result has no banner"); assert_eq!( capped.title(), - "Stopped after the first 3 rows: the statement did not run past them" + "Showing the first 3 rows: the server stopped the statement at the row limit, before its end" ); assert_eq!(capped.button_label().as_deref(), Some("Export All…")); assert!(!complete.is_revealed(), "a complete result shows a banner"); diff --git a/linux/crates/app/src/ui/result_pane/result_summary.rs b/linux/crates/app/src/ui/result_pane/result_summary.rs index e2cb90afc4..0b3a5a8ee9 100644 --- a/linux/crates/app/src/ui/result_pane/result_summary.rs +++ b/linux/crates/app/src/ui/result_pane/result_summary.rs @@ -15,6 +15,8 @@ pub struct ResultSummary { /// The run's limit on the rows of all its results, when that limit /// cut this one short. pub run_limit: Option, + // The run's limit on the bytes of all its results, when that limit cut this one short. + pub size_limit: Option, } impl ResultSummary { @@ -25,20 +27,21 @@ impl ResultSummary { total_rows: None, truncation: Truncation::Complete, run_limit: None, + size_limit: None, } } pub(crate) fn is_truncated(&self) -> bool { - self.truncation != Truncation::Complete || self.run_limit.is_some() + self.truncation != Truncation::Complete || self.run_limit.is_some() || self.size_limit.is_some() } /// What the banner over a cut-short result says. `None` for a /// complete one, which shows no banner. pub(crate) fn limit_title(&self) -> Option { - self.limit_title_with(format_thousands) + self.limit_title_with(format_thousands, |bytes| gtk4::glib::format_size(bytes).to_string()) } - fn limit_title_with(&self, format: impl Fn(u64) -> String) -> Option { + fn limit_title_with(&self, format: impl Fn(u64) -> String, format_size: impl Fn(u64) -> String) -> Option { let shown = format(self.retained); if let Some(limit) = self.run_limit { let limit = format(limit); @@ -55,31 +58,64 @@ impl ResultSummary { ), }); } + if let Some(bytes) = self.size_limit { + return Some(self.size_title(&shown, &format_size(bytes), format)); + } match (self.truncation, self.total_rows) { (Truncation::Complete, _) => None, - (Truncation::ReadPastLimit, Some(total)) => Some(ngettext_f( + (Truncation::ReadPastLimit | Truncation::ByteLimit, Some(total)) => Some(ngettext_f( "Showing {shown} of {total} row", "Showing {shown} of {total} rows", total, &[("shown", &shown), ("total", &format(total))], )), - (Truncation::ReadPastLimit, None) => Some(ngettext_f( + (Truncation::ReadPastLimit | Truncation::ByteLimit, None) => Some(ngettext_f( "Showing the first {shown} row", "Showing the first {shown} rows", self.retained, &[("shown", &shown)], )), - // A SELECT that calls a function per row did that for these - // rows only, which "showing" alone would hide. + // A SELECT that calls a function per row may have done that for + // only part of the rows, which "showing" alone would hide. (Truncation::ServerStopped, _) => Some(ngettext_f( - "Stopped after the first {shown} row: the statement did not run past it", - "Stopped after the first {shown} rows: the statement did not run past them", + "Showing the first {shown} row: the server stopped the statement at the row limit, before its end", + "Showing the first {shown} rows: the server stopped the statement at the row limit, before its end", self.retained, &[("shown", &shown)], )), } } + // The row that crosses the size is kept whole, so the size is where keeping stops, not a cap. + fn size_title(&self, shown: &str, size: &str, format: impl Fn(u64) -> String) -> String { + let stopped = self.truncation == Truncation::ServerStopped; + match (self.retained, self.total_rows) { + (0, _) if stopped => gettext_f( + "No rows kept; a run stops keeping rows past {size}, and the server stopped the statement at the row limit, before its end", + &[("size", size)], + ), + (0, _) => gettext_f("No rows kept; a run stops keeping rows past {size}", &[("size", size)]), + (retained, _) if stopped => ngettext_f( + "Showing the first {shown} row; a run stops keeping rows past {size}, and the server stopped the statement at the row limit, before its end", + "Showing the first {shown} rows; a run stops keeping rows past {size}, and the server stopped the statement at the row limit, before its end", + retained, + &[("shown", shown), ("size", size)], + ), + (_, Some(total)) => ngettext_f( + "Showing {shown} of {total} row; a run stops keeping rows past {size}", + "Showing {shown} of {total} rows; a run stops keeping rows past {size}", + total, + &[("shown", shown), ("total", &format(total)), ("size", size)], + ), + (retained, None) => ngettext_f( + "Showing the first {shown} row; a run stops keeping rows past {size}", + "Showing the first {shown} rows; a run stops keeping rows past {size}", + retained, + &[("shown", shown), ("size", size)], + ), + } + } + /// The result's line in the results list: its place in the run and /// how many rows it has, with a `+` when there were more than it /// kept and nobody counted them. @@ -119,12 +155,18 @@ mod tests { Thousands::for_posix_locale("en_US.UTF-8").format(n) } + // A fixed spelling, since glib writes the decimal point of the test's locale. + fn megabytes(bytes: u64) -> String { + format!("{} MB", bytes / 1_000_000) + } + fn summary(retained: u64, total_rows: Option, truncation: Truncation) -> ResultSummary { ResultSummary { retained, total_rows, truncation, run_limit: None, + size_limit: None, } } @@ -134,11 +176,11 @@ mod tests { let read_past = summary(1_000, None, Truncation::ReadPastLimit); assert_eq!( - stopped.limit_title_with(english).as_deref(), - Some("Stopped after the first 1,000 rows: the statement did not run past them") + stopped.limit_title_with(english, megabytes).as_deref(), + Some("Showing the first 1,000 rows: the server stopped the statement at the row limit, before its end") ); assert_eq!( - read_past.limit_title_with(english).as_deref(), + read_past.limit_title_with(english, megabytes).as_deref(), Some("Showing the first 1,000 rows") ); } @@ -148,7 +190,7 @@ mod tests { let read_past = summary(10_000, Some(1_234_567), Truncation::ReadPastLimit); assert_eq!( - read_past.limit_title_with(english).as_deref(), + read_past.limit_title_with(english, megabytes).as_deref(), Some("Showing 10,000 of 1,234,567 rows") ); } @@ -161,24 +203,69 @@ mod tests { }; assert_eq!( - cut(2_500).limit_title_with(english).as_deref(), + cut(2_500).limit_title_with(english, megabytes).as_deref(), Some("Showing the first 2,500 rows; a run keeps at most 100,000 rows across its results") ); assert_eq!( - cut(0).limit_title_with(english).as_deref(), + cut(0).limit_title_with(english, megabytes).as_deref(), Some("No rows kept; a run keeps at most 100,000 rows across its results") ); assert_eq!(cut(0).tab_label_with(4, english), "Result 5 (0+)"); assert_eq!(cut(0).history_row_count(), None); } + #[test] + fn limit_title_names_the_size_limit_when_it_cut_the_result() { + let cut = |retained, total_rows| ResultSummary { + size_limit: Some(256_000_000), + ..summary(retained, total_rows, Truncation::ByteLimit) + }; + + assert_eq!( + cut(128, None).limit_title_with(english, megabytes).as_deref(), + Some("Showing the first 128 rows; a run stops keeping rows past 256 MB") + ); + assert_eq!( + cut(1_280, Some(10_000)).limit_title_with(english, megabytes).as_deref(), + Some("Showing 1,280 of 10,000 rows; a run stops keeping rows past 256 MB") + ); + assert_eq!( + cut(0, Some(4)).limit_title_with(english, megabytes).as_deref(), + Some("No rows kept; a run stops keeping rows past 256 MB") + ); + let stopped = ResultSummary { + truncation: Truncation::ServerStopped, + ..cut(128, None) + }; + assert_eq!( + stopped.limit_title_with(english, megabytes).as_deref(), + Some( + "Showing the first 128 rows; a run stops keeping rows past 256 MB, and the server stopped the statement at the row limit, before its end" + ) + ); + let stopped_empty = ResultSummary { + truncation: Truncation::ServerStopped, + ..cut(0, None) + }; + assert_eq!( + stopped_empty.limit_title_with(english, megabytes).as_deref(), + Some( + "No rows kept; a run stops keeping rows past 256 MB, and the server stopped the statement at the row limit, before its end" + ) + ); + assert_eq!(cut(128, None).tab_label_with(0, english), "Result 1 (128+)"); + assert_eq!(cut(1_280, Some(10_000)).tab_label_with(1, english), "Result 2 (10,000)"); + assert_eq!(cut(0, Some(4)).tab_label_with(2, english), "Result 3 (4)"); + assert_eq!(cut(128, None).history_row_count(), None); + } + #[test] fn limit_title_none_when_complete() { assert_eq!( - summary(5, Some(5), Truncation::Complete).limit_title_with(english), + summary(5, Some(5), Truncation::Complete).limit_title_with(english, megabytes), None ); - assert_eq!(ResultSummary::complete(0).limit_title_with(english), None); + assert_eq!(ResultSummary::complete(0).limit_title_with(english, megabytes), None); } #[test] diff --git a/linux/crates/app/src/ui/write_confirmation.rs b/linux/crates/app/src/ui/write_confirmation.rs index b96bc5d436..aac3428db1 100644 --- a/linux/crates/app/src/ui/write_confirmation.rs +++ b/linux/crates/app/src/ui/write_confirmation.rs @@ -200,6 +200,7 @@ fn shown_value(value: &Value) -> String { let text = text.replace(['\n', '\r'], " "); match value { Value::Bool(_) + | Value::Int32(_) | Value::Int(_) | Value::UInt(_) | Value::WideInt(_) @@ -281,6 +282,14 @@ mod tests { buffer.text(&buffer.start_iter(), &buffer.end_iter(), false).to_string() } + #[test] + fn a_bound_number_shows_bare_and_text_quoted() { + assert_eq!(shown_value(&Value::Int32(5)), "5"); + assert_eq!(shown_value(&Value::Int(5)), "5"); + assert_eq!(shown_value(&Value::Text("it's".to_owned())), "'it''s'"); + assert_eq!(shown_value(&Value::Null), "NULL"); + } + #[gtk4::test] fn the_whole_statement_shows_and_cancel_is_the_default() { adw::init().expect("libadwaita initialises under the test backend"); diff --git a/linux/docs/adding-drivers.md b/linux/docs/adding-drivers.md index 39ce3216b7..7a59bf408a 100644 --- a/linux/docs/adding-drivers.md +++ b/linux/docs/adding-drivers.md @@ -293,7 +293,7 @@ An editor tab holds one session for as long as it is open, because the state a u - It is never pooled and never reopened behind the user's back. `state()` is `Ready`, `Running` or `Lost`. A lost connection is `EditorSessionState::Lost`, and the tab opens a new session on its next run. - One run at a time: a second `run` fails with `EditorSessionBusy`. - `transaction_state(options)` answers what the session holds now, outside a run. -- `run(sql, options, sink)` takes `UserSql` as typed and `RunOptions`: `row_limit`, `deadline` and `cancel`. A caller stops a run through `RunOptions::cancel`. +- `run(sql, options, sink)` takes `UserSql` as typed and `RunOptions`: `row_limit`, `byte_limit`, `deadline` and `cancel`. A caller stops a run through `RunOptions::cancel`. - A dropped caller is a cancel. The run goes on a task of its own, or an actor's, and `run` arms `options.cancel.clone().drop_guard()` until the outcome arrives, so the stop still goes out and the session settles to `Ready` or `Lost`. `MysqlEditorSession::run` is the shape to copy. - The app's editor splits a script with the engine's grammar and sends one statement per run, or one `GO` batch on SQL Server. A driver still runs whatever text it is given, a whole script included. - A stopped run sends nothing more, whether or not the stop reached the server: the statement running, or the batch the server runs as one (a `GO` batch on SQL Server, text MySQL has to send whole), is its last. `TimedOut { server_cancelled: true }` means the server said it stopped the statement, never only that the stop went out. @@ -307,9 +307,10 @@ A run streams `ResultEvent`s into a `ResultSink`: - `ResultSetStarted { columns }` opens a result set. - `ColumnsResolved { set_index, columns }` replaces the columns of a set whose types the driver could only learn from its rows (SQLite). -- `Rows` carries a batch of rows, 256 in the shipped drivers. +- `Rows` carries a batch of rows. `RowBatch` sends one at 256 rows or 4 MiB of `Value::retained_bytes`, whichever comes first. - `RowLimitReached { retained }` comes once per set, before `ResultSetFinished`. -- `ResultSetFinished { retained, total_rows, truncation }` closes the set. `Truncation` is `Complete`, `ServerStopped` or `ReadPastLimit`, and `total_rows` is `None` when the driver did not read the rest. +- `ByteLimitReached { retained }` comes once per set the run's byte limit cut, before `ResultSetFinished`. The session's `ByteLimitSink` sends it; a driver sends it only when it holds a set before its first event (The row-limit rule, below). +- `ResultSetFinished { retained, total_rows, truncation }` closes the set. `Truncation` is `Complete`, `ServerStopped`, `ReadPastLimit` or `ByteLimit`, and `total_rows` is `None` when the driver did not read the rest. - `CommandCompleted { rows_affected }` reports a statement that returned no rows; `None` is "not reported", not zero. - `Notice` passes on a server notice or warning. - `NoticeLimitReached { retained }` comes once per run when the driver kept fewer notices than the server sent, after the `retained` ones it passed on. @@ -326,6 +327,8 @@ A run streams `ResultEvent`s into a `ResultSink`: - The driver never rewrites the user's SQL to add a `LIMIT`. - `stream_read` has no limit. +`RunOptions::byte_limit` bounds the bytes of rows a whole run keeps, `ByteLimit::EDITOR_DEFAULT` (256,000,000) unless the caller sets one (core ADR 0020, `core/docs/decisions/0020-a-run-keeps-a-byte-limit-of-rows.md`). The driver does not enforce it: `Session::open_editor_session` wraps every editor session so its runs go through `ByteLimitSink`. The sink keeps the row that crosses the limit whole, sends `ByteLimitReached`, drops the run's later rows and finishes a cut set as `Truncation::ByteLimit` with the driver's `total_rows`, or keeps the driver's `ServerStopped`, which says more. It answers `Continue` for the rows it drops, never `ConsumerGone`, so the limit ends neither the statement nor the user's transaction, and rows past the cut are still read up to the row limit. A driver that holds a set before its first event, as MongoDB holds a cursor's documents, stops holding at the limit itself, ends the set with `ByteLimitReached` and `Truncation::ByteLimit`, and the sink passes them on. The conformance case `byte_limit_keeps_whole_rows_and_later_statements` checks the sink on every engine with an editor, and `a_document_read_stops_at_the_byte_limit` the driver's own stop. + ## 7. Dialects `Connection::dialect()` returns the engine's `SqlDialect`, built from `ServerInfo` at connect. Every builder in `tablepro-core` (pages, filters, row edits, exports) goes through it: quoting, placeholders, literals, the page window, keyset comparison, the engine's row address and type parsing. A dialect never reads session state. Where ordering by a table's key sorts the whole table, as on DuckDB, whose indexes do not order a scan, `key_order_sorts_the_table` says so, and an export with no sort of the user's keeps the scan's order instead of holding every row. `qualify` names a table in the connection's own statements and `portable_name` in an export and a foreign key's REFERENCES; they differ on DuckDB, which names its file's catalogue only in the first. `SqlDialect::ddl()` returns the engine's `DdlDialect`, or `None` where the app does not edit the engine's structure. Never match on a driver id in core; add a dialect method. diff --git a/linux/docs/decisions/0014-lossless-values-and-dialects.md b/linux/docs/decisions/0014-lossless-values-and-dialects.md index 3f994a36d4..4e4cf7adc7 100644 --- a/linux/docs/decisions/0014-lossless-values-and-dialects.md +++ b/linux/docs/decisions/0014-lossless-values-and-dialects.md @@ -1,6 +1,6 @@ # 0014: Lossless values and engine dialects -- **Status**: Accepted +- **Status**: Accepted; the integer values amended by core 0021 - **Date**: 2026-10-15 ## Context diff --git a/linux/docs/decisions/0016-driver-contract-v2.md b/linux/docs/decisions/0016-driver-contract-v2.md index 071dbd2f56..171818e995 100644 --- a/linux/docs/decisions/0016-driver-contract-v2.md +++ b/linux/docs/decisions/0016-driver-contract-v2.md @@ -1,6 +1,6 @@ # 0016: Driver contract v2 -- **Status**: Accepted; the no-fork rule for tiberius superseded by core 0005 (`core/docs/decisions/0005-sql-server-transport.md`), a database change outside a transaction amended by core 0013 (`core/docs/decisions/0013-catalog-operations.md`) +- **Status**: Accepted; the no-fork rule for tiberius superseded by core 0005 (`core/docs/decisions/0005-sql-server-transport.md`), a database change outside a transaction amended by core 0013 (`core/docs/decisions/0013-catalog-operations.md`), a run's byte limit beside its row limit amended by core 0020 (`core/docs/decisions/0020-a-run-keeps-a-byte-limit-of-rows.md`) - **Date**: 2026-10-13 ## Context diff --git a/linux/docs/decisions/README.md b/linux/docs/decisions/README.md index 815eb3bfc0..a2add40761 100644 --- a/linux/docs/decisions/README.md +++ b/linux/docs/decisions/README.md @@ -61,9 +61,9 @@ A fact the repository keeps elsewhere, such as a workflow trigger, a branch, a v | [0010](0010-ddl-plans-and-table-definitions.md) | DDL plans against table definitions | Accepted; truncating and schema and database changes amended by core 0013 | Each engine plans structure changes through its `DdlDialect` against the table's definition; the UI offers only what the plan accepts. | | [0012](0012-window-per-session.md) | One window per connection session, built from gtk-rs subclasses | Accepted; what a window holds superseded by 0028 | One `TableProApplication` per profile through D-Bus uniqueness; a window holds at most one session and returns to the picker on disconnect; the application, the window and its workspace are subclasses, not Relm4 components. | | [0013](0013-openssh-client-transport.md) | SSH through the system OpenSSH client | Accepted; the forced options amended by 0019, the TCP port amended by 0023, the russh consequence amended by core 0008 | ControlMaster per connection, Unix-socket forwards, prompts through an askpass helper. | -| [0014](0014-lossless-values-and-dialects.md) | Lossless values and engine dialects | Accepted | A value reaches the user and goes back without loss; every statement is spelled by the dialect of the connection that runs it. | +| [0014](0014-lossless-values-and-dialects.md) | Lossless values and engine dialects | Accepted; the integer values amended by core 0021 | A value reaches the user and goes back without loss; every statement is spelled by the dialect of the connection that runs it. | | [0015](0015-gnome-50-platform-floor.md) | GNOME 50 platform floor | Accepted; the runtime row amended by 0018 | GTK 4.22, libadwaita 1.9, GtkSourceView 5.18, GLib 2.88; AdwShortcutsDialog replaces the shortcuts window. | -| [0016](0016-driver-contract-v2.md) | Driver contract v2 | Accepted; the no-fork rule for tiberius superseded by core 0005, a database change outside a transaction amended by core 0013 | Owned calls with engine stops, phase-aware write outcomes, server-side row limits; PostgreSQL on tokio-postgres; no forks. | +| [0016](0016-driver-contract-v2.md) | Driver contract v2 | Accepted; the no-fork rule for tiberius superseded by core 0005, a database change outside a transaction amended by core 0013, a run's byte limit beside its row limit amended by core 0020 | Owned calls with engine stops, phase-aware write outcomes, server-side row limits; PostgreSQL on tokio-postgres; no forks. | | [0017](0017-distribution-channels.md) | Distribution channels | Superseded by 0018 | Flathub, a .deb for Ubuntu 26.04 on GitHub Releases built with rustup, and an .rpm for Fedora 44 on COPR, all from `meson install` and one vendor tarball; no AppImage. | | [0018](0018-ppa-copr-and-flathub-later.md) | A PPA and COPR first, Flathub later | Accepted; the source tarball amended by core 0001; release channels and the Snap alternative superseded by 0041; Flathub permission review amended by 0041; the source of `debian/copyright` superseded by 0040 | A Launchpad PPA built with `rustc-1.97` and a COPR project built on each release are the primary channels; Flathub follows as a submission a maintainer writes; Rust 1.97 is the MSRV. | | [0019](0019-ssh-key-file-identities-only.md) | A chosen SSH key file limits the keys ssh offers | Superseded by 0020 | A key file comes with `IdentitiesOnly=yes`, so the agent's other keys cannot use up `MaxAuthTries` first. | diff --git a/linux/docs/drivers/mongodb.md b/linux/docs/drivers/mongodb.md index e6b72b5356..3ce8c1bcbc 100644 --- a/linux/docs/drivers/mongodb.md +++ b/linux/docs/drivers/mongodb.md @@ -46,7 +46,7 @@ Databases are the sidebar's schemas, collections its tables and views its views. A collection's columns are its top-level fields, found by the server from the first 1,000 documents in their natural order: the fields, the BSON types each holds, and where each first appears. No document crosses the wire for it. `_id` comes first, then the fields in the order the documents place them. -A field's column type is its values' type: one numeric type is a number, a string is text, a date a timestamp with a zone, an `objectId` its own type, and UUIDs (binary subtype 4) a UUID. `$type` names every binary subtype `binData`, so the census tells a UUID by where it sorts: binary data orders by length, then subtype, then bytes. A field that holds documents, arrays, other binary data, numbers of more than one type (`int` and `long`, or `int` and `double`) or more than one kind of value is a JSON column, written as relaxed Extended JSON from the BSON itself, with each `long` in its canonical `{"$numberLong": "..."}` form, so the type survives an edit, a copy and an export. A field missing from a document reads as NULL. +A field's column type is its values' type: one numeric type is a number, a string is text, a date a timestamp with a zone, an `objectId` its own type, and UUIDs (binary subtype 4) a UUID. `$type` names every binary subtype `binData`, so the census tells a UUID by where it sorts: binary data orders by length, then subtype, then bytes. A field that holds documents, arrays, other binary data, numbers of more than one type (`int` and `long`, or `int` and `double`) or more than one kind of value is a JSON column, written as relaxed Extended JSON from the BSON itself, with each `long` in its canonical `{"$numberLong": "..."}` form, so the type survives an edit, a copy and an export. A field missing from a document reads as NULL. Each cell still reads as its own BSON type, an `int` apart from a `long`, so a number past the census in a field it typed `long` or `double` saves an edit as its own type when the new value is one; a null, missing or new cell takes the column's type. A connection that has not read a collection's fields reads them before its first page or export of it, so a page after a reconnect shows the same JSON columns. @@ -84,7 +84,7 @@ The editor runs mongosh statements. A statement ends at `;` or at the end of a l Anything else, a JavaScript function, a variable, `forEach`, `bulkWrite`, is refused before it is sent with a `SyntaxError` at its place, or `Unsupported`. As mongosh checks, `updateOne` and `updateMany` take update operators or a pipeline and `replaceOne` a document with none, so a forgotten `$set` never replaces a document, and `dropIndex()` has to name its index. A write's `writeConcern`, `ordered`, `comment` and `let` options go with its command. A statement's documents show one per row, a column per field: the fields a projection names, in its order, or every field the documents hold, in the order they first show it. A write shows what mongosh prints for it (`insertedId`, `matchedCount`, `modifiedCount`). -The run's row limit keeps 10,000 documents, and the cursor reads one more to say the result was cut; closing it frees the rest on the server. When a projection names the columns, the documents go to the grid a batch at a time, each batch read once the grid took the last, and the columns' types are settled when the last one is read. Otherwise the documents kept are all read before the first row is shown, so the columns hold every field they have. A tab's `use` changes only its own database. +The run's row limit keeps 10,000 documents, and the cursor reads one more to say the result was cut; closing it frees the rest on the server. When a projection names the columns, the documents go to the grid a batch at a time, each batch read once the grid took the last, and the columns' types are settled when the last one is read. Otherwise the documents kept are all read before the first row is shown, so the columns hold every field they have, and the session's byte limit cannot cut them: once the documents held reach the run's byte limit in BSON bytes, the driver closes the cursor and ends the set with `ByteLimitReached` and `Truncation::ByteLimit` (`cursor_rows.rs`), the document that crosses it kept whole. A command's inline reply is one document of at most 16 MiB, already held, so only the session's sink bounds it. A tab's `use` changes only its own database. A table export reads every field the documents hold, not only the tab's columns, so a field none of the first 1,000 documents holds is still written. Before the first row, the server tallies the fields the export's documents hold and the types each holds, the same census the columns come from but over every matching document, and no document crosses the wire for it. A field of nested or mixed values is written as Extended JSON. An export that matches no document names the collection's columns. An export of an editor result whose columns are not named by a projection finds them the same way for an `aggregate`, and reads a `find`'s documents twice: once for the fields they hold, then to write them under those columns. A document that gains a field between the two reads fails the export rather than losing the field. The MCP server's `browse_table` and `export_data` read the tab's columns, since they stop at a row limit and a deadline. diff --git a/linux/docs/drivers/postgresql.md b/linux/docs/drivers/postgresql.md index 572854bd85..a6480d028b 100644 --- a/linux/docs/drivers/postgresql.md +++ b/linux/docs/drivers/postgresql.md @@ -127,7 +127,8 @@ CLOSE tablepro_cursor COMMIT -- or RELEASE SAVEPOINT tablepro_cursor_savepoint ``` -- One row past the limit tells a full result from a cut one. A cut result has `total_rows: None` and `Truncation::ServerStopped`: the statement never ran past the rows fetched, so a function it calls per row, such as `SELECT archive_order(id) FROM orders`, ran for those rows only, and the driver's `COMMIT` keeps what they did. The result says so rather than read the rest: `MOVE FORWARD ALL` would run the whole statement for every plain `SELECT` too, which is what the limit is there to avoid. +- One row past the limit tells a full result from a cut one. A cut result has `total_rows: None` and `Truncation::ServerStopped`: the server ran the statement only as far as the fetch needed, so a function it calls per row, such as `SELECT archive_order(id) FROM orders`, ran for the rows fetched, the one past the limit included, and the driver's `COMMIT` keeps what they did. A plan that reads every row before it returns one, such as a sort on the function's result, ran the function for every row. The result says so rather than read the rest: `MOVE FORWARD ALL` would run the whole statement for every plain `SELECT` too, which is what the limit is there to avoid. +- This is how PostgreSQL runs a statement under a row limit. A cursor is a portal ([DECLARE](https://www.postgresql.org/docs/current/sql-declare.html), Notes). An Execute with a row count runs a portal only that far and suspends it, and the Sync after it commits the implicit transaction when nothing failed ([protocol](https://www.postgresql.org/docs/current/protocol-flow.html), Extended Query). libpq offers no other stop: a cancel request ends the statement with an error and rolls its implicit transaction back, rows already shown included, and otherwise every row is read ([libpq](https://www.postgresql.org/docs/current/libpq-async.html)). The Linux result summary and the Windows status line both say the server stopped the statement, a set the byte limit also cut included. - Each step goes as its own message, so a trailing `--` comment in the statement cannot swallow the `FETCH`. - A `DECLARE` the server refuses because the statement cannot be a cursor (a data-modifying `WITH`: 0A000; `SELECT INTO`: 42601) is rolled back to the savepoint or out of the driver's transaction, and the statement runs again as a plain statement. Inside the user's transaction any other failure aborts their transaction, as it would without the limit. - Error positions are corrected for the `DECLARE` prefix, so they count from the start of the user's statement. diff --git a/linux/po/vi.po b/linux/po/vi.po index 61196d3623..1862060aa3 100644 --- a/linux/po/vi.po +++ b/linux/po/vi.po @@ -7222,11 +7222,15 @@ msgstr[0] "Đang hiện {shown} dòng đầu tiên" #: crates/app/src/ui/result_pane/result_summary.rs #, rust-format -msgid "Stopped after the first {shown} row: the statement did not run past it" +msgid "" +"Showing the first {shown} row: the server stopped the statement at the row " +"limit, before its end" msgid_plural "" -"Stopped after the first {shown} rows: the statement did not run past them" +"Showing the first {shown} rows: the server stopped the statement at the row " +"limit, before its end" msgstr[0] "" -"Đã dừng sau {shown} dòng đầu tiên: câu lệnh không chạy tiếp quá các dòng đó" +"Đang hiện {shown} dòng đầu tiên: máy chủ đã dừng câu lệnh ở giới hạn số dòng, " +"trước khi chạy hết" #: crates/app/src/ui/result_pane/result_summary.rs #, rust-format diff --git a/windows/src/TablePro.App.Model/ConnectionLife.cs b/windows/src/TablePro.App.Model/ConnectionLife.cs index 5ffcf44f29..461f7950e4 100644 --- a/windows/src/TablePro.App.Model/ConnectionLife.cs +++ b/windows/src/TablePro.App.Model/ConnectionLife.cs @@ -10,7 +10,15 @@ public enum ConnectionState Failed, } -public sealed record SecretAnswer(string Secret, bool Save); +public sealed record SecretAnswer(string Secret, bool Save) +{ + // A record's ToString prints every property, and this one holds a password. + private bool PrintMembers(System.Text.StringBuilder builder) + { + builder.Append("Secret = [REDACTED], Save = ").Append(Save); + return true; + } +} // The core calls a connect makes. public interface IConnector diff --git a/windows/src/TablePro.App.Model/RunResults.cs b/windows/src/TablePro.App.Model/RunResults.cs index 34e5046a47..bb84f3377f 100644 --- a/windows/src/TablePro.App.Model/RunResults.cs +++ b/windows/src/TablePro.App.Model/RunResults.cs @@ -15,6 +15,12 @@ public sealed class RunResult(int setIndex, IReadOnlyList columns) public long? TotalRows { get; internal set; } public bool Truncated { get; internal set; } + + // The server stopped the statement, so work it does per row may have run for only part of the rows. + public bool StoppedOnServer { get; internal set; } + + // Set from ByteLimitReached alone, since a finish carries no limit. + public long? SizeLimit { get; internal set; } } // Each result set of a run in the order it started, labelled "Result {n} ({rows})" as the Linux app lists them. @@ -42,10 +48,15 @@ public RunResult Add(int setIndex, IReadOnlyList columns) case QueryEvent.RowLimitReached limit when Find(limit.SetIndex) is { } cut: cut.Truncated = true; return cut; + case QueryEvent.ByteLimitReached bytes when Find(bytes.SetIndex) is { } full: + full.Truncated = true; + full.SizeLimit = bytes.LimitBytes; + return full; case QueryEvent.ResultSetFinished finished when Find(finished.SetIndex) is { } ended: ended.Rows = finished.RowCount; ended.TotalRows = finished.TotalRows; ended.Truncated |= finished.Truncation != Truncation.Complete; + ended.StoppedOnServer = finished.Truncation == Truncation.ServerStopped; return ended; default: return null; diff --git a/windows/src/TablePro.App.Model/TableTab.cs b/windows/src/TablePro.App.Model/TableTab.cs index 6b7c482102..ec00fb6812 100644 --- a/windows/src/TablePro.App.Model/TableTab.cs +++ b/windows/src/TablePro.App.Model/TableTab.cs @@ -111,6 +111,9 @@ public interface ITableTabView // A sort is an ORDER BY in the page's statement, so it orders the whole table, not the rows already read. public sealed class TableTab : IWorkspaceTab { + // A page is bounded by its size, and the pager reads a short page as the last, so no byte limit cuts one. + private const long PageByteLimit = long.MaxValue; + // The refusals the core makes before it sends anything, so a save after one cannot write twice. private static readonly HashSet NothingSent = new(StringComparer.Ordinal) { @@ -662,7 +665,7 @@ private async Task LoadPageAsync(long offset, CancellationToken cancellationToke _run?.Dispose(); _page = page with { CanSeek = query.CanSeek }; // The statement holds the page; a run limit one past it keeps a full page from reading as cut short. - _run = _editor.Run(query.Sql, new QueryOptions(page.Size + 1), cancellationToken); + _run = _editor.Run(query.Sql, new QueryOptions(page.Size + 1, ByteLimit: PageByteLimit), cancellationToken); await _results.ShowRunAsync(_run, _sort, query.VisibleColumns, cancellationToken).ConfigureAwait(true); if (_work.IsClosed) { diff --git a/windows/src/TablePro.App/Strings/en-US/Resources.resw b/windows/src/TablePro.App/Strings/en-US/Resources.resw index 2bce812887..af7fe3acca 100644 --- a/windows/src/TablePro.App/Strings/en-US/Resources.resw +++ b/windows/src/TablePro.App/Strings/en-US/Resources.resw @@ -750,6 +750,15 @@ Rows: {0:N0}, the row limit · {1:N0} ms + + Rows: {0:N0}, the server stopped the statement at the row limit, before its end · {1:N0} ms + + + Rows: {0:N0}, a run stops keeping rows past {2:N0} MB · {1:N0} ms + + + Rows: {0:N0}, a run stops keeping rows past {2:N0} MB, and the server stopped the statement at the row limit, before its end · {1:N0} ms + No statement at the cursor diff --git a/windows/src/TablePro.App/Views/ResultsPane.xaml.cs b/windows/src/TablePro.App/Views/ResultsPane.xaml.cs index 2a16024cda..c3ba52e33e 100644 --- a/windows/src/TablePro.App/Views/ResultsPane.xaml.cs +++ b/windows/src/TablePro.App/Views/ResultsPane.xaml.cs @@ -167,12 +167,15 @@ void Adopt() } break; - case QueryEvent.RowsAppended or QueryEvent.RowLimitReached or QueryEvent.ResultSetFinished: + case QueryEvent.RowsAppended + or QueryEvent.RowLimitReached + or QueryEvent.ByteLimitReached + or QueryEvent.ResultSetFinished: if (results.Apply(queryEvent) is { } changed && incoming.Find(entry => ReferenceEquals(entry.Result, changed)) is { } grown) { grown.Item.Content = results.Label(changed); - if (queryEvent is not QueryEvent.RowLimitReached) + if (queryEvent is QueryEvent.RowsAppended or QueryEvent.ResultSetFinished) { RowsArrived(grown, ended: queryEvent is QueryEvent.ResultSetFinished); } @@ -410,7 +413,18 @@ private void Finish(QueryOutcome outcome, bool hasRows, long? affected) private string RowsText(RunResult result, long milliseconds) => RowsStatus?.Invoke(result.Rows, milliseconds, result.Truncated) - ?? Strings.Format(result.Truncated ? "StatusRowsLimited" : "StatusRows", result.Rows, milliseconds); + ?? Strings.Format( + result switch + { + { SizeLimit: not null, StoppedOnServer: true } => "StatusRowsSizeLimitedStopped", + { SizeLimit: not null } => "StatusRowsSizeLimited", + { StoppedOnServer: true } => "StatusRowsStopped", + { Truncated: true } => "StatusRowsLimited", + _ => "StatusRows", + }, + result.Rows, + milliseconds, + result.SizeLimit / 1_000_000.0); // Notices that arrive together are laid out once, after the batch: a TextBox lays out all its text on each change. private void QueueMessages() diff --git a/windows/src/TablePro.Interop/Generated/tablepro_ffi.cs b/windows/src/TablePro.Interop/Generated/tablepro_ffi.cs index 89842843ec..2f95dec5d7 100644 --- a/windows/src/TablePro.Interop/Generated/tablepro_ffi.cs +++ b/windows/src/TablePro.Interop/Generated/tablepro_ffi.cs @@ -21,6 +21,8 @@ namespace TablePro.Interop.Native; using BoxedErrorInfo = ErrorInfo; using FfiConverterTypeBoxedErrorInfo = FfiConverterTypeErrorInfo; +using FfiConverterTypeSecretString = FfiConverterString; +using SecretString = String; @@ -3644,8 +3646,8 @@ static void uniffiCheckContractApiVersion() { static void uniffiCheckApiChecksums() { { var checksum = _UniFFILib.uniffi_tablepro_ffi_checksum_func_parse_connection_url(); - if (checksum != 61414) { - throw new UniffiContractChecksumException($"TablePro.Interop.Native: uniffi bindings expected function `uniffi_tablepro_ffi_checksum_func_parse_connection_url` checksum `61414`, library returned `{checksum}`"); + if (checksum != 48047) { + throw new UniffiContractChecksumException($"TablePro.Interop.Native: uniffi bindings expected function `uniffi_tablepro_ffi_checksum_func_parse_connection_url` checksum `48047`, library returned `{checksum}`"); } } { @@ -4094,14 +4096,14 @@ static void uniffiCheckApiChecksums() { } { var checksum = _UniFFILib.uniffi_tablepro_ffi_checksum_method_hostsecrets_load(); - if (checksum != 5070) { - throw new UniffiContractChecksumException($"TablePro.Interop.Native: uniffi bindings expected function `uniffi_tablepro_ffi_checksum_method_hostsecrets_load` checksum `5070`, library returned `{checksum}`"); + if (checksum != 39136) { + throw new UniffiContractChecksumException($"TablePro.Interop.Native: uniffi bindings expected function `uniffi_tablepro_ffi_checksum_method_hostsecrets_load` checksum `39136`, library returned `{checksum}`"); } } { var checksum = _UniFFILib.uniffi_tablepro_ffi_checksum_method_hostsecrets_store(); - if (checksum != 47632) { - throw new UniffiContractChecksumException($"TablePro.Interop.Native: uniffi bindings expected function `uniffi_tablepro_ffi_checksum_method_hostsecrets_store` checksum `47632`, library returned `{checksum}`"); + if (checksum != 40708) { + throw new UniffiContractChecksumException($"TablePro.Interop.Native: uniffi bindings expected function `uniffi_tablepro_ffi_checksum_method_hostsecrets_store` checksum `40708`, library returned `{checksum}`"); } } { @@ -4124,14 +4126,14 @@ static void uniffiCheckApiChecksums() { } { var checksum = _UniFFILib.uniffi_tablepro_ffi_checksum_method_hostsecrets_load_item(); - if (checksum != 18393) { - throw new UniffiContractChecksumException($"TablePro.Interop.Native: uniffi bindings expected function `uniffi_tablepro_ffi_checksum_method_hostsecrets_load_item` checksum `18393`, library returned `{checksum}`"); + if (checksum != 56817) { + throw new UniffiContractChecksumException($"TablePro.Interop.Native: uniffi bindings expected function `uniffi_tablepro_ffi_checksum_method_hostsecrets_load_item` checksum `56817`, library returned `{checksum}`"); } } { var checksum = _UniFFILib.uniffi_tablepro_ffi_checksum_method_hostsecrets_store_item(); - if (checksum != 44779) { - throw new UniffiContractChecksumException($"TablePro.Interop.Native: uniffi bindings expected function `uniffi_tablepro_ffi_checksum_method_hostsecrets_store_item` checksum `44779`, library returned `{checksum}`"); + if (checksum != 48405) { + throw new UniffiContractChecksumException($"TablePro.Interop.Native: uniffi bindings expected function `uniffi_tablepro_ffi_checksum_method_hostsecrets_store_item` checksum `48405`, library returned `{checksum}`"); } } { @@ -7045,9 +7047,9 @@ public override void Write(ConnectionRecord value, BigEndianStream stream) { internal record ConnectionSecrets ( - string? DatabasePassword, - string? SshPassword, - string? SshPassphrase + SecretString? DatabasePassword, + SecretString? SshPassword, + SecretString? SshPassphrase ) { } @@ -7056,23 +7058,23 @@ class FfiConverterTypeConnectionSecrets: FfiConverterRustBuffer -/// What one run may do. `None` takes the editor's defaults: 10,000 rows -/// and no deadline beyond the connection's. +/// What one run may do. `None` takes the editor's defaults: 10,000 rows, +/// 256,000,000 bytes of rows across the run, and no deadline beyond the +/// connection's. /// internal record RunSettings ( ulong? RowLimit, - ulong? TimeoutMs + ulong? TimeoutMs, + ulong? ByteLimit ) { } @@ -7832,19 +7836,22 @@ class FfiConverterTypeRunSettings: FfiConverterRustBuffer { public override RunSettings Read(BigEndianStream stream) { return new RunSettings( RowLimit: FfiConverterOptionalUInt64.INSTANCE.Read(stream), - TimeoutMs: FfiConverterOptionalUInt64.INSTANCE.Read(stream) + TimeoutMs: FfiConverterOptionalUInt64.INSTANCE.Read(stream), + ByteLimit: FfiConverterOptionalUInt64.INSTANCE.Read(stream) ); } public override int AllocationSize(RunSettings value) { return 0 + FfiConverterOptionalUInt64.INSTANCE.AllocationSize(value.RowLimit) - + FfiConverterOptionalUInt64.INSTANCE.AllocationSize(value.TimeoutMs); + + FfiConverterOptionalUInt64.INSTANCE.AllocationSize(value.TimeoutMs) + + FfiConverterOptionalUInt64.INSTANCE.AllocationSize(value.ByteLimit); } public override void Write(RunSettings value, BigEndianStream stream) { FfiConverterOptionalUInt64.INSTANCE.Write(value.RowLimit, stream); FfiConverterOptionalUInt64.INSTANCE.Write(value.TimeoutMs, stream); + FfiConverterOptionalUInt64.INSTANCE.Write(value.ByteLimit, stream); } } @@ -8927,7 +8934,7 @@ public override void Write(ObjectScopeArg value, BigEndianStream stream) { internal record PromptAnswer { public record Submitted ( - string[] Values, + SecretString[] Values, bool Remember ) : PromptAnswer {} @@ -8946,7 +8953,7 @@ public override PromptAnswer Read(BigEndianStream stream) { switch (value) { case 1: return new PromptAnswer.Submitted( - FfiConverterSequenceString.INSTANCE.Read(stream), + FfiConverterSequenceTypeSecretString.INSTANCE.Read(stream), FfiConverterBoolean.INSTANCE.Read(stream) ); case 2: @@ -8961,7 +8968,7 @@ public override int AllocationSize(PromptAnswer value) { switch (value) { case PromptAnswer.Submitted variant_value: return 4 - + FfiConverterSequenceString.INSTANCE.AllocationSize(variant_value.Values) + + FfiConverterSequenceTypeSecretString.INSTANCE.AllocationSize(variant_value.Values) + FfiConverterBoolean.INSTANCE.AllocationSize(variant_value.Remember); case PromptAnswer.Cancelled variant_value: return 4; @@ -8974,7 +8981,7 @@ public override void Write(PromptAnswer value, BigEndianStream stream) { switch (value) { case PromptAnswer.Submitted variant_value: stream.WriteInt(1); - FfiConverterSequenceString.INSTANCE.Write(variant_value.Values, stream); + FfiConverterSequenceTypeSecretString.INSTANCE.Write(variant_value.Values, stream); FfiConverterBoolean.INSTANCE.Write(variant_value.Remember, stream); break; case PromptAnswer.Cancelled variant_value: @@ -9431,6 +9438,12 @@ public record RowLimitReached ( ulong Retained ) : RunEvent {} + public record ByteLimitReached ( + uint SetIndex, + ulong Retained, + ulong LimitBytes + ) : RunEvent {} + public record ResultSetFinished ( uint SetIndex, ulong RowCount, @@ -9499,27 +9512,33 @@ public override RunEvent Read(BigEndianStream stream) { FfiConverterUInt64.INSTANCE.Read(stream) ); case 5: + return new RunEvent.ByteLimitReached( + FfiConverterUInt32.INSTANCE.Read(stream), + FfiConverterUInt64.INSTANCE.Read(stream), + FfiConverterUInt64.INSTANCE.Read(stream) + ); + case 6: return new RunEvent.ResultSetFinished( FfiConverterUInt32.INSTANCE.Read(stream), FfiConverterUInt64.INSTANCE.Read(stream), FfiConverterOptionalUInt64.INSTANCE.Read(stream), FfiConverterTypeTruncationTag.INSTANCE.Read(stream) ); - case 6: + case 7: return new RunEvent.CommandCompleted( FfiConverterOptionalUInt64.INSTANCE.Read(stream) ); - case 7: + case 8: return new RunEvent.Notice( FfiConverterString.INSTANCE.Read(stream), FfiConverterString.INSTANCE.Read(stream), FfiConverterOptionalString.INSTANCE.Read(stream) ); - case 8: + case 9: return new RunEvent.NoticeLimitReached( FfiConverterUInt64.INSTANCE.Read(stream) ); - case 9: + case 10: return new RunEvent.Finished( FfiConverterTypeRunOutcome.INSTANCE.Read(stream), FfiConverterTypeTransactionStateTag.INSTANCE.Read(stream) @@ -9547,6 +9566,11 @@ public override int AllocationSize(RunEvent value) { return 4 + FfiConverterUInt32.INSTANCE.AllocationSize(variant_value.SetIndex) + FfiConverterUInt64.INSTANCE.AllocationSize(variant_value.Retained); + case RunEvent.ByteLimitReached variant_value: + return 4 + + FfiConverterUInt32.INSTANCE.AllocationSize(variant_value.SetIndex) + + FfiConverterUInt64.INSTANCE.AllocationSize(variant_value.Retained) + + FfiConverterUInt64.INSTANCE.AllocationSize(variant_value.LimitBytes); case RunEvent.ResultSetFinished variant_value: return 4 + FfiConverterUInt32.INSTANCE.AllocationSize(variant_value.SetIndex) @@ -9595,29 +9619,35 @@ public override void Write(RunEvent value, BigEndianStream stream) { FfiConverterUInt32.INSTANCE.Write(variant_value.SetIndex, stream); FfiConverterUInt64.INSTANCE.Write(variant_value.Retained, stream); break; - case RunEvent.ResultSetFinished variant_value: + case RunEvent.ByteLimitReached variant_value: stream.WriteInt(5); FfiConverterUInt32.INSTANCE.Write(variant_value.SetIndex, stream); + FfiConverterUInt64.INSTANCE.Write(variant_value.Retained, stream); + FfiConverterUInt64.INSTANCE.Write(variant_value.LimitBytes, stream); + break; + case RunEvent.ResultSetFinished variant_value: + stream.WriteInt(6); + FfiConverterUInt32.INSTANCE.Write(variant_value.SetIndex, stream); FfiConverterUInt64.INSTANCE.Write(variant_value.RowCount, stream); FfiConverterOptionalUInt64.INSTANCE.Write(variant_value.TotalRows, stream); FfiConverterTypeTruncationTag.INSTANCE.Write(variant_value.Truncation, stream); break; case RunEvent.CommandCompleted variant_value: - stream.WriteInt(6); + stream.WriteInt(7); FfiConverterOptionalUInt64.INSTANCE.Write(variant_value.RowsAffected, stream); break; case RunEvent.Notice variant_value: - stream.WriteInt(7); + stream.WriteInt(8); FfiConverterString.INSTANCE.Write(variant_value.Severity, stream); FfiConverterString.INSTANCE.Write(variant_value.Message, stream); FfiConverterOptionalString.INSTANCE.Write(variant_value.Code, stream); break; case RunEvent.NoticeLimitReached variant_value: - stream.WriteInt(8); + stream.WriteInt(9); FfiConverterUInt64.INSTANCE.Write(variant_value.Retained, stream); break; case RunEvent.Finished variant_value: - stream.WriteInt(9); + stream.WriteInt(10); FfiConverterTypeRunOutcome.INSTANCE.Write(variant_value.Outcome, stream); FfiConverterTypeTransactionStateTag.INSTANCE.Write(variant_value.Transaction, stream); break; @@ -10285,7 +10315,8 @@ internal enum TruncationTag: int { /// /// There were rows past the limit, which were not kept. /// - ReadPastLimit + ReadPastLimit, + ByteLimit } class FfiConverterTypeTruncationTag: FfiConverterRustBuffer { @@ -10297,6 +10328,7 @@ public override TruncationTag Read(BigEndianStream stream) { case 1: return TruncationTag.Complete; case 2: return TruncationTag.ServerStopped; case 3: return TruncationTag.ReadPastLimit; + case 4: return TruncationTag.ByteLimit; default: throw new InternalException(String.Format("invalid enum value '{0}' in FfiConverterTypeTruncationTag.Read()", value)); } } @@ -10310,6 +10342,7 @@ public override void Write(TruncationTag value, BigEndianStream stream) { case TruncationTag.Complete: stream.WriteInt(1); break; case TruncationTag.ServerStopped: stream.WriteInt(2); break; case TruncationTag.ReadPastLimit: stream.WriteInt(3); break; + case TruncationTag.ByteLimit: stream.WriteInt(4); break; default: throw new InternalException(String.Format("invalid enum value '{0}' in FfiConverterTypeTruncationTag.Write()", value)); } } @@ -10630,9 +10663,9 @@ public override void Write(HostPrompter value, BigEndianStream stream) { /// internal interface HostSecrets { /// - string? Load(string @connectionId, SecretKindTag @kind); + SecretString? Load(string @connectionId, SecretKindTag @kind); /// - void Store(string @connectionId, SecretKindTag @kind, string @secret, string @label); + void Store(string @connectionId, SecretKindTag @kind, SecretString @secret, string @label); /// void Relabel(string @connectionId, SecretKindTag @kind, string @label); /// @@ -10640,12 +10673,12 @@ internal interface HostSecrets { /// void DeleteConnection(string @connectionId); /// - string? LoadItem(SecretItemTag @item); + SecretString? LoadItem(SecretItemTag @item); /// /// The app names the item in its own words. /// /// - void StoreItem(SecretItemTag @item, string @secret); + void StoreItem(SecretItemTag @item, SecretString @secret); /// void DeleteItem(SecretItemTag @item); } @@ -10662,7 +10695,7 @@ static void Load(ulong @uniffiHandle,RustBuffer @connectionId,RustBuffer @kind,I FfiConverterString.INSTANCE.Lift(@connectionId), FfiConverterTypeSecretKindTag.INSTANCE.Lift(@kind)); unsafe { - *(RustBuffer*)uniffiOutReturn = FfiConverterOptionalString.INSTANCE.Lower(result); + *(RustBuffer*)uniffiOutReturn = FfiConverterOptionalTypeSecretString.INSTANCE.Lower(result); } _uniffi_out_err.code = UniffiCallbackResponseStatus.SUCCESS; @@ -10693,7 +10726,7 @@ static void Store(ulong @uniffiHandle,RustBuffer @connectionId,RustBuffer @kind, uniffiObject.Store( FfiConverterString.INSTANCE.Lift(@connectionId), FfiConverterTypeSecretKindTag.INSTANCE.Lift(@kind), - FfiConverterString.INSTANCE.Lift(@secret), + FfiConverterTypeSecretString.INSTANCE.Lift(@secret), FfiConverterString.INSTANCE.Lift(@label)); _uniffi_out_err.code = UniffiCallbackResponseStatus.SUCCESS; @@ -10812,7 +10845,7 @@ static void LoadItem(ulong @uniffiHandle,RustBuffer @item,IntPtr /*RustBuffer*/ uniffiObject.LoadItem( FfiConverterTypeSecretItemTag.INSTANCE.Lift(@item)); unsafe { - *(RustBuffer*)uniffiOutReturn = FfiConverterOptionalString.INSTANCE.Lower(result); + *(RustBuffer*)uniffiOutReturn = FfiConverterOptionalTypeSecretString.INSTANCE.Lower(result); } _uniffi_out_err.code = UniffiCallbackResponseStatus.SUCCESS; @@ -10842,7 +10875,7 @@ static void StoreItem(ulong @uniffiHandle,RustBuffer @item,RustBuffer @secret,In } uniffiObject.StoreItem( FfiConverterTypeSecretItemTag.INSTANCE.Lift(@item), - FfiConverterString.INSTANCE.Lift(@secret)); + FfiConverterTypeSecretString.INSTANCE.Lift(@secret)); _uniffi_out_err.code = UniffiCallbackResponseStatus.SUCCESS; } @@ -11667,6 +11700,37 @@ public override void Write(BoxedErrorInfo? value, BigEndianStream stream) { +class FfiConverterOptionalTypeSecretString: FfiConverterRustBuffer { + public static FfiConverterOptionalTypeSecretString INSTANCE = new FfiConverterOptionalTypeSecretString(); + + public override SecretString? Read(BigEndianStream stream) { + if (stream.ReadByte() == 0) { + return null; + } + return FfiConverterTypeSecretString.INSTANCE.Read(stream); + } + + public override int AllocationSize(SecretString? value) { + if (value == null) { + return 1; + } else { + return 1 + FfiConverterTypeSecretString.INSTANCE.AllocationSize((SecretString)value); + } + } + + public override void Write(SecretString? value, BigEndianStream stream) { + if (value == null) { + stream.WriteByte(0); + } else { + stream.WriteByte(1); + FfiConverterTypeSecretString.INSTANCE.Write((SecretString)value, stream); + } + } +} + + + + class FfiConverterSequenceUInt16: FfiConverterRustBuffer { public static FfiConverterSequenceUInt16 INSTANCE = new FfiConverterSequenceUInt16(); @@ -12586,6 +12650,62 @@ public override void Write(TlsModeTag[] value, BigEndianStream stream) { + +class FfiConverterSequenceTypeSecretString: FfiConverterRustBuffer { + public static FfiConverterSequenceTypeSecretString INSTANCE = new FfiConverterSequenceTypeSecretString(); + + public override SecretString[] Read(BigEndianStream stream) { + var length = stream.ReadInt(); + if (length == 0) { + return []; + } + + var result = new SecretString[length]; + var readFn = FfiConverterTypeSecretString.INSTANCE.Read; + for (int i = 0; i < length; i++) { + result[i] = readFn(stream); + } + return result; + } + + public override int AllocationSize(SecretString[] value) { + var sizeForLength = 4; + + // details/1-empty-list-as-default-method-parameter.md + if (value == null) { + return sizeForLength; + } + + var allocationSizeFn = FfiConverterTypeSecretString.INSTANCE.AllocationSize; + var sizeForItems = value.Sum(item => allocationSizeFn(item)); + return sizeForLength + sizeForItems; + } + + public override void Write(SecretString[] value, BigEndianStream stream) { + // details/1-empty-list-as-default-method-parameter.md + if (value == null) { + stream.WriteInt(0); + return; + } + + stream.WriteInt(value.Length); + var writerFn = FfiConverterTypeSecretString.INSTANCE.Write; + value.ForEach(item => writerFn(item, stream)); + } +} + + + +/** + * Typealias from the type name used in the UDL file to the builtin type. This + * is needed because the UDL type name is used in function/method signatures. + * It's also what we have an external type that references a custom type. + */ + + + + + /** * Typealias from the type name used in the UDL file to the builtin type. This * is needed because the UDL type name is used in function/method signatures. @@ -12725,10 +12845,10 @@ CallStatusErrorHandler errorHandler #pragma warning restore 8625 internal static class CoreMethods { /// - public static ParsedConnectionUrl ParseConnectionUrl(string @url) { + public static ParsedConnectionUrl ParseConnectionUrl(SecretString @url) { return FfiConverterTypeParsedConnectionUrl.INSTANCE.Lift( _UniffiHelpers.RustCallWithError(FfiConverterTypeCoreError.INSTANCE, (ref UniffiRustCallStatus _status) => - _UniFFILib.uniffi_tablepro_ffi_fn_func_parse_connection_url(FfiConverterString.INSTANCE.Lower(@url), ref _status) + _UniFFILib.uniffi_tablepro_ffi_fn_func_parse_connection_url(FfiConverterTypeSecretString.INSTANCE.Lower(@url), ref _status) )); } diff --git a/windows/src/TablePro.Interop/Mapping.cs b/windows/src/TablePro.Interop/Mapping.cs index 2559e81df2..67e6b5cf0f 100644 --- a/windows/src/TablePro.Interop/Mapping.cs +++ b/windows/src/TablePro.Interop/Mapping.cs @@ -107,7 +107,8 @@ public static ResultColumn ResultColumn(Native.ResultColumnInfo column) => public static Native.RunSettings Settings(QueryOptions options) => new( options.RowLimit is { } limit ? checked((ulong)limit) : null, - options.Timeout is { } timeout ? checked((ulong)timeout.TotalMilliseconds) : null); + options.Timeout is { } timeout ? checked((ulong)timeout.TotalMilliseconds) : null, + options.ByteLimit is { } bytes ? checked((ulong)bytes) : null); public static QueryEvent Event(Native.RunEvent nativeEvent) => nativeEvent switch { @@ -118,15 +119,22 @@ public static ResultColumn ResultColumn(Native.ResultColumnInfo column) => Native.RunEvent.RowsAppended rows => new QueryEvent.RowsAppended(Index(rows.SetIndex), Count(rows.RowCount)), Native.RunEvent.RowLimitReached limit => new QueryEvent.RowLimitReached(Index(limit.SetIndex), Count(limit.Retained)), + Native.RunEvent.ByteLimitReached bytes => new QueryEvent.ByteLimitReached( + Index(bytes.SetIndex), + Count(bytes.Retained), + Count(bytes.LimitBytes)), Native.RunEvent.ResultSetFinished finished => new QueryEvent.ResultSetFinished( Index(finished.SetIndex), Count(finished.RowCount), finished.TotalRows is { } total ? Count(total) : null, finished.Truncation switch { + Native.TruncationTag.Complete => Truncation.Complete, Native.TruncationTag.ServerStopped => Truncation.ServerStopped, Native.TruncationTag.ReadPastLimit => Truncation.ReadPastLimit, - _ => Truncation.Complete, + Native.TruncationTag.ByteLimit => Truncation.ByteLimit, + _ => throw new InvalidOperationException( + $"The core sent a truncation this build does not know: {finished.Truncation}"), }), Native.RunEvent.CommandCompleted command => new QueryEvent.CommandCompleted(command.RowsAffected is { } affected ? Count(affected) : null), diff --git a/windows/src/TablePro.Interop/QueryEvent.cs b/windows/src/TablePro.Interop/QueryEvent.cs index 63d498e8ca..16005676f0 100644 --- a/windows/src/TablePro.Interop/QueryEvent.cs +++ b/windows/src/TablePro.Interop/QueryEvent.cs @@ -1,7 +1,7 @@ namespace TablePro.Interop; -// RowLimit: the most rows each set keeps, 10,000 by default. Timeout: beyond the connection's own limit. -public sealed record QueryOptions(long? RowLimit = null, TimeSpan? Timeout = null); +// Null takes the core's: 10,000 rows per set, 256,000,000 bytes of rows per run, no timeout past the connection's. +public sealed record QueryOptions(long? RowLimit = null, TimeSpan? Timeout = null, long? ByteLimit = null); // TypeName: the type as the server spells it, for display. public sealed record ResultColumn(string Name, string TypeName, ValueKind ValueKind); @@ -15,6 +15,9 @@ public enum Truncation // There were rows past the limit, which were not kept. ReadPastLimit, + + // Rows past the cut were not kept; the engine may still have read them up to the row limit. + ByteLimit, } // In order, Finished last. Rows that arrive between two reads come as one RowsAppended with the latest count. @@ -33,6 +36,9 @@ public sealed record RowsAppended(int SetIndex, long RowCount) : QueryEvent; public sealed record RowLimitReached(int SetIndex, long Retained) : QueryEvent; + // LimitBytes: what the whole run keeps. The set's rows after its first Retained were read and not kept. + public sealed record ByteLimitReached(int SetIndex, long Retained, long LimitBytes) : QueryEvent; + // TotalRows: every row the statement produced, where the engine counted past the limit. public sealed record ResultSetFinished(int SetIndex, long RowCount, long? TotalRows, Truncation Truncation) : QueryEvent; diff --git a/windows/src/TablePro.Interop/SavedConnection.cs b/windows/src/TablePro.Interop/SavedConnection.cs index 0ce7a94cae..6b452a301a 100644 --- a/windows/src/TablePro.Interop/SavedConnection.cs +++ b/windows/src/TablePro.Interop/SavedConnection.cs @@ -1,3 +1,5 @@ +using System.Text; + namespace TablePro.Interop; public enum TlsMode @@ -72,6 +74,15 @@ public sealed record ConnectionSecrets( string? SshPassphrase = null) { public static ConnectionSecrets None { get; } = new(); + + // A record's ToString prints every property, so a log line or an assert message would carry the passwords. + private bool PrintMembers(StringBuilder builder) + { + builder.Append("DatabasePassword = ").Append(Redaction.Of(DatabasePassword)); + builder.Append(", SshPassword = ").Append(Redaction.Of(SshPassword)); + builder.Append(", SshPassphrase = ").Append(Redaction.Of(SshPassphrase)); + return true; + } } public sealed record ConnectionGroup(Guid Id, string Name, Guid? ParentId, int Order, string? Color); @@ -102,4 +113,27 @@ public sealed record ParsedConnectionUrl( string? CaFile = null, string? ClientCertificate = null, string? ClientKey = null, - string? ServerName = null); + string? ServerName = null) +{ + private bool PrintMembers(StringBuilder builder) + { + builder.Append("EngineId = ").Append(EngineId); + builder.Append(", Host = ").Append(Host); + builder.Append(", Port = ").Append(Port); + builder.Append(", UserName = ").Append(UserName); + builder.Append(", Password = ").Append(Redaction.Of(Password)); + builder.Append(", Database = ").Append(Database); + builder.Append(", TlsMode = ").Append(TlsMode); + builder.Append(", CaFile = ").Append(CaFile); + builder.Append(", ClientCertificate = ").Append(ClientCertificate); + builder.Append(", ClientKey = ").Append(ClientKey); + builder.Append(", ServerName = ").Append(ServerName); + return true; + } +} + +// Null prints as nothing, as a record prints any null property. +internal static class Redaction +{ + public static string? Of(string? secret) => secret is null ? null : "[REDACTED]"; +} diff --git a/windows/tests/TablePro.App.Model.Tests/RunResultsTests.cs b/windows/tests/TablePro.App.Model.Tests/RunResultsTests.cs index 6b2491e548..407dddb3c7 100644 --- a/windows/tests/TablePro.App.Model.Tests/RunResultsTests.cs +++ b/windows/tests/TablePro.App.Model.Tests/RunResultsTests.cs @@ -50,10 +50,36 @@ public void ASetTheLimitCutShortShowsTheCountedTotalOrAPlus() Assert.True(counted.Truncated); Assert.True(uncounted.Truncated); + Assert.False(counted.StoppedOnServer); + Assert.True(uncounted.StoppedOnServer); Assert.Equal("Result 1 (25000)", results.Label(counted)); Assert.Equal("Result 2 (10000+)", results.Label(uncounted)); } + [Fact] + public void ASetCutForSizeKeepsTheRunsLimitAndShowsAPlusWhenNobodyCounted() + { + var results = New(); + var uncounted = results.Add(0, Columns); + var counted = results.Add(1, Columns); + var rowLimited = results.Add(2, Columns); + + results.Apply(new QueryEvent.RowsAppended(0, 16)); + results.Apply(new QueryEvent.ByteLimitReached(0, 16, 256_000_000)); + results.Apply(new QueryEvent.ResultSetFinished(0, 16, null, Truncation.ByteLimit)); + results.Apply(new QueryEvent.ByteLimitReached(1, 0, 256_000_000)); + results.Apply(new QueryEvent.ResultSetFinished(1, 0, 100, Truncation.ByteLimit)); + results.Apply(new QueryEvent.ResultSetFinished(2, 10_000, 25_000, Truncation.ReadPastLimit)); + + Assert.True(uncounted.Truncated); + Assert.Equal(256_000_000, uncounted.SizeLimit); + Assert.False(uncounted.StoppedOnServer); + Assert.Equal(256_000_000, counted.SizeLimit); + Assert.Null(rowLimited.SizeLimit); + Assert.Equal("Result 1 (16+)", results.Label(uncounted)); + Assert.Equal("Result 2 (100)", results.Label(counted)); + } + [Fact] public void EventsAboutNoListedSetChangeNothing() { diff --git a/windows/tests/TablePro.DataGrid.Gate/BenchData.cs b/windows/tests/TablePro.DataGrid.Gate/BenchData.cs index 572cc2d730..8901006351 100644 --- a/windows/tests/TablePro.DataGrid.Gate/BenchData.cs +++ b/windows/tests/TablePro.DataGrid.Gate/BenchData.cs @@ -41,7 +41,8 @@ await bench.ExecuteAsync("CREATE TABLE pool (k INTEGER PRIMARY KEY, amount TEXT, public async Task<(QueryRun Run, ResultSet Set, ResultSetData Data)> QueryAsync(string sql, long rowLimit) { - var run = _editor.Run(sql, new QueryOptions(RowLimit: rowLimit)); + // The gate times a million wide rows, past the run's default byte limit. + var run = _editor.Run(sql, new QueryOptions(RowLimit: rowLimit, ByteLimit: long.MaxValue)); var outcome = await run.Completion; if (outcome is not QueryOutcome.Completed) { diff --git a/windows/tests/TablePro.Interop.Tests/ByteLimitTests.cs b/windows/tests/TablePro.Interop.Tests/ByteLimitTests.cs new file mode 100644 index 0000000000..b97cf1b7fc --- /dev/null +++ b/windows/tests/TablePro.Interop.Tests/ByteLimitTests.cs @@ -0,0 +1,55 @@ +using Native = TablePro.Interop.Native; + +namespace TablePro.Interop.Tests; + +public sealed class ByteLimitTests +{ + private const string Blobs = + "WITH RECURSIVE c(x) AS (SELECT 1 UNION ALL SELECT x + 1 FROM c WHERE x < 100) " + + "SELECT x, zeroblob(65536) FROM c"; + + [Fact] + public async Task ARunKeepsWholeRowsUpToItsByteLimit() + { + await using var database = await SqliteDatabase.CreateAsync(); + var cancel = TestContext.Current.CancellationToken; + + using var run = database.Editor.Run(Blobs, new QueryOptions(ByteLimit: 1 << 20), cancel); + var events = new List(); + await foreach (var queryEvent in run.ReadEventsAsync(cancel)) + { + events.Add(queryEvent); + } + + Assert.IsType(await run.Completion); + // Each row is two values and 65,536 bytes, so the 16th crosses 1 MiB and is kept whole. + var reached = Assert.Single(events.OfType()); + Assert.Equal(new QueryEvent.ByteLimitReached(0, 16, 1 << 20), reached); + var finished = Assert.Single(events.OfType()); + Assert.Equal(new QueryEvent.ResultSetFinished(0, 16, 100, Truncation.ByteLimit), finished); + Assert.True(events.IndexOf(reached) < events.IndexOf(finished)); + using var set = run.GetResultSet(0)!; + Assert.Equal(2 + (2 * 65_536), set.GetCellText(15, 1)?.Length); + } + + [Fact] + public void TheCoresByteLimitEventAndTruncationKeepTheirFields() + { + var reached = Mapping.Event(new Native.RunEvent.ByteLimitReached(1, 16, 256_000_000)); + var finished = Mapping.Event( + new Native.RunEvent.ResultSetFinished(1, 16, null, Native.TruncationTag.ByteLimit)); + + Assert.Equal(new QueryEvent.ByteLimitReached(1, 16, 256_000_000), reached); + Assert.Equal(new QueryEvent.ResultSetFinished(1, 16, null, Truncation.ByteLimit), finished); + Assert.Equal(256_000_000UL, Mapping.Settings(new QueryOptions(ByteLimit: 256_000_000)).ByteLimit); + Assert.Null(Mapping.Settings(new QueryOptions()).ByteLimit); + } + + [Fact] + public void ATruncationThisBuildDoesNotKnowIsRefused() + { + var unknown = new Native.RunEvent.ResultSetFinished(0, 1, 1, (Native.TruncationTag)99); + + Assert.Throws(() => Mapping.Event(unknown)); + } +} diff --git a/windows/tests/TablePro.Interop.Tests/ConnectionUrlTests.cs b/windows/tests/TablePro.Interop.Tests/ConnectionUrlTests.cs new file mode 100644 index 0000000000..945d4296a6 --- /dev/null +++ b/windows/tests/TablePro.Interop.Tests/ConnectionUrlTests.cs @@ -0,0 +1,28 @@ +namespace TablePro.Interop.Tests; + +public sealed class ConnectionUrlTests +{ + [Fact] + public void AUrlsPasswordComesThroughButItsTextShowsOnlyThatOneIsSet() + { + var parsed = TableProCore.ParseConnectionUrl("postgresql://app:hunter2@db/shop"); + var shown = parsed.ToString(); + + Assert.Equal("hunter2", parsed.Password); + Assert.Equal("app", parsed.UserName); + Assert.Equal("shop", parsed.Database); + Assert.DoesNotContain("hunter2", shown, StringComparison.Ordinal); + Assert.StartsWith("ParsedConnectionUrl { EngineId = postgres, Host = db, ", shown, StringComparison.Ordinal); + Assert.Contains("UserName = app, Password = [REDACTED], Database = shop, ", shown, StringComparison.Ordinal); + } + + [Fact] + public void ConnectionSecretsSayWhichAreSetButNotWhatTheyAre() + { + var secrets = new ConnectionSecrets(DatabasePassword: "hunter2", SshPassphrase: string.Empty); + + Assert.Equal( + "ConnectionSecrets { DatabasePassword = [REDACTED], SshPassword = , SshPassphrase = [REDACTED] }", + secrets.ToString()); + } +}