From 91454c02edb560019ab0ff21b8fa9008ecb2dd6a Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 12 Nov 2026 02:30:52 +0700 Subject: [PATCH 1/2] perf(macos): append the execution log in place, skip unchanged tab saves, read CLI pipes per pipe and release closed agent sessions --- .../Sources/TableProGzip/GzipFile.swift | 3 +- .../File/DownloadedBinary.swift | 9 +- .../Core/AI/CLI/AgentCLIProcess.swift | 102 +++- .../Diagnostics/TableLoadHistoryStore.swift | 31 +- apple/TablePro/Core/LSP/LSPTransport.swift | 37 +- apple/TablePro/Core/Process/PipeReader.swift | 25 +- .../Execution/ExecutionAuditLog.swift | 414 ++++++++++++++-- .../Infrastructure/AgentSessionRegistry.swift | 110 ++--- .../TablePro/Core/Storage/AIChatStorage.swift | 104 ++-- .../TablePro/Core/Storage/TabDiskActor.swift | 123 +++-- .../Core/Storage/TabQueryOverflowStore.swift | 52 +- apple/TablePro/Models/AI/AgentSession.swift | 94 +--- .../AIChatViewModel+Persistence.swift | 122 +++-- .../AIChatViewModel+TableMentions.swift | 20 +- .../TablePro/ViewModels/AIChatViewModel.swift | 164 ++++--- .../TablePro/Views/Import/ImportDialog.swift | 53 ++- .../Core/AI/AgentCLIProcessTests.swift | 143 ++++++ .../Core/LSP/LSPTransportReadTests.swift | 58 +++ .../Core/Process/PipeReaderTests.swift | 89 ++++ .../Services/ExecutionAuditRecordTests.swift | 335 +++++++++++++ .../AgentSessionRegistryTests.swift | 61 ++- .../Storage/AIChatStorageSummaryTests.swift | 64 +++ .../Storage/TabDiskUnchangedSaveTests.swift | 173 +++++++ .../AIChatSessionReleaseTests.swift | 448 ++++++++++++++++++ .../ViewModels/AIChatTableMentionsTests.swift | 38 ++ .../Import/ImportStatementCountTests.swift | 41 ++ 26 files changed, 2382 insertions(+), 531 deletions(-) create mode 100644 apple/TableProTests/Core/LSP/LSPTransportReadTests.swift create mode 100644 apple/TableProTests/Core/Storage/AIChatStorageSummaryTests.swift create mode 100644 apple/TableProTests/Core/Storage/TabDiskUnchangedSaveTests.swift create mode 100644 apple/TableProTests/ViewModels/AIChatSessionReleaseTests.swift create mode 100644 apple/TableProTests/Views/Import/ImportStatementCountTests.swift diff --git a/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift b/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift index 19a4dcad91..d25cea91e8 100644 --- a/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift +++ b/apple/Packages/TableProCore/Sources/TableProGzip/GzipFile.swift @@ -41,7 +41,8 @@ public enum GzipFile { let output = try FileHandle(forWritingTo: destination) defer { try? output.close() } let inflater = try GzipInflater(budget: budget) - while let block = try input.read(upToCount: readSize), !block.isEmpty { + // Each read returns an autoreleased buffer, and this loop never suspends, so nothing drains them until it ends. + while let block = try autoreleasepool(invoking: { try input.read(upToCount: readSize) }), !block.isEmpty { try Task.checkCancellation() try block.withUnsafeBytes { raw in try inflater.inflate(raw) { try output.write(contentsOf: $0) } diff --git a/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/DownloadedBinary.swift b/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/DownloadedBinary.swift index 89f38cd9e9..2a58e8731b 100644 --- a/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/DownloadedBinary.swift +++ b/apple/Packages/TableProMacCore/Sources/TableProUtilities/File/DownloadedBinary.swift @@ -13,7 +13,8 @@ public enum DownloadedBinary { var hasher = SHA256() do { - while let chunk = try handle.read(upToCount: hashChunkSize), !chunk.isEmpty { + // Each read returns an autoreleased buffer, and the caller's job may not drain its pool until the whole file is read. + while let chunk = try autoreleasepool(invoking: { try handle.read(upToCount: hashChunkSize) }), !chunk.isEmpty { hasher.update(data: chunk) } } catch { @@ -23,11 +24,7 @@ public enum DownloadedBinary { return hasher.finalize().map { String(format: "%02x", $0) }.joined() } - /// `ditto -xk` writes `com.apple.quarantine` onto every file it extracts, not just the - /// directory it extracts into, so clearing the bundle root alone leaves the Mach-O inside - /// `Contents/MacOS` still quarantined. Gatekeeper reads the flag on the file it is about - /// to map, and refuses a quarantined bundle with "library load disallowed by system - /// policy" even inside a process that has `com.apple.security.cs.disable-library-validation`. + // `ditto -xk` quarantines every extracted file, and Gatekeeper refuses to map a quarantined Mach-O even with library validation off. public static func stripQuarantine(at url: URL) { stripQuarantineFromSingleItem(at: url) diff --git a/apple/TablePro/Core/AI/CLI/AgentCLIProcess.swift b/apple/TablePro/Core/AI/CLI/AgentCLIProcess.swift index c465866ced..15468e4759 100644 --- a/apple/TablePro/Core/AI/CLI/AgentCLIProcess.swift +++ b/apple/TablePro/Core/AI/CLI/AgentCLIProcess.swift @@ -6,6 +6,7 @@ import Foundation import os import TableProProcess +import TableProUtilities enum AgentCLIError: Error, LocalizedError { case notInstalled(String) @@ -40,6 +41,7 @@ struct AgentCLILaunch: Sendable { } } +// Never FileHandle.bytes: every AsyncBytes read shares one serial queue, so a quiet pipe stalls every other read. struct AgentCLIProcess: Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "AgentCLIProcess") private static let terminationGracePeriod: Duration = .milliseconds(750) @@ -56,20 +58,33 @@ struct AgentCLIProcess: Sendable { let combined = Pipe() process.standardOutput = combined process.standardError = combined + // Read while the child runs: one that fills the pipe never exits. + let output = OSAllocatedUnfairLock(initialState: Data()) + let reader = PipeReader(combined.fileHandleForReading) return try await withTaskCancellationHandler { try await withCheckedThrowingContinuation { continuation in process.terminationHandler = { finished in - let data = (try? combined.fileHandleForReading.readToEnd() ?? Data()) ?? Data() + reader.stopAtEndOfFile() + let data = output.withLock { $0 } continuation.resume( returning: (finished.terminationStatus, String(data: data, encoding: .utf8) ?? "") ) } + reader.start { chunk in + output.withLock { $0.append(chunk) } + } do { try process.run() } catch { process.terminationHandler = nil + reader.stop() continuation.resume(throwing: AgentCLIError.launchFailed(error.localizedDescription)) + return + } + // A cancel that landed before launch found nothing running to stop. + if Task.isCancelled { + Self.terminate(process, label: label) } } } onCancel: { @@ -92,37 +107,45 @@ struct AgentCLIProcess: Sendable { process.standardError = stderr let label = self.label - let stderrDrain = Task { - for try await line in stderr.fileHandleForReading.bytes.lines where !line.isEmpty { - Self.logger.error("\(label, privacy: .public) stderr: \(line, privacy: .private)") + let stdoutLines = OSAllocatedUnfairLock(initialState: LineSplitter()) + let stderrLines = OSAllocatedUnfairLock(initialState: LineSplitter()) + let stdoutReader = PipeReader(stdout.fileHandleForReading) + let stderrReader = PipeReader(stderr.fileHandleForReading) + + continuation.onTermination = { _ in + Self.terminate(process, label: label) + stdoutReader.stop() + stderrReader.stop(drainingUpTo: DescriptorRead.pipeCapacity) + Self.logStderr(stderrLines.withLock { $0.finish() }, label: label) + } + + stderrReader.start { chunk in + Self.logStderr(stderrLines.withLock { $0.append(chunk) }, label: label) + } atEnd: { _ in + Self.logStderr(stderrLines.withLock { $0.finish() }, label: label) + } + stdoutReader.start { chunk in + for line in stdoutLines.withLock({ $0.append(chunk) }) { + continuation.yield(line) } + } atEnd: { failure in + for line in stdoutLines.withLock({ $0.finish() }) { + continuation.yield(line) + } + continuation.finish(throwing: failure) } do { try process.run() } catch { - stderrDrain.cancel() continuation.finish(throwing: AgentCLIError.launchFailed(error.localizedDescription)) - return - } - - let reader = Task { - do { - for try await line in stdout.fileHandleForReading.bytes.lines { - if Task.isCancelled { break } - continuation.yield(line) - } - continuation.finish() - } catch { - continuation.finish(throwing: error) - } } + } + } - continuation.onTermination = { _ in - reader.cancel() - stderrDrain.cancel() - Self.terminate(process, label: label) - } + private static func logStderr(_ lines: [String], label: String) { + for line in lines { + logger.error("\(label, privacy: .public) stderr: \(line, privacy: .private)") } } @@ -147,3 +170,36 @@ struct AgentCLIProcess: Sendable { } } } + +extension AgentCLIProcess { + // LF alone ends a line: stream-json text can hold a raw U+2028, which FileHandle's `lines` also splits on. + struct LineSplitter: Sendable { + private var partial = Data() + + mutating func append(_ chunk: Data) -> [String] { + var lines: [String] = [] + var lineStart = chunk.startIndex + while let newline = chunk[lineStart...].firstIndex(of: 0x0A) { + partial.append(chunk[lineStart ..< newline]) + if let line = Self.decode(partial) { + lines.append(line) + } + partial.removeAll(keepingCapacity: true) + lineStart = chunk.index(after: newline) + } + partial.append(chunk[lineStart...]) + return lines + } + + mutating func finish() -> [String] { + defer { partial = Data() } + return Self.decode(partial).map { [$0] } ?? [] + } + + private static func decode(_ bytes: Data) -> String? { + let line = bytes.last == 0x0D ? bytes.dropLast() : bytes + guard !line.isEmpty else { return nil } + return String(bytes: line, encoding: .utf8) + } + } +} diff --git a/apple/TablePro/Core/Diagnostics/TableLoadHistoryStore.swift b/apple/TablePro/Core/Diagnostics/TableLoadHistoryStore.swift index bc2b9699ab..c5f72e9acf 100644 --- a/apple/TablePro/Core/Diagnostics/TableLoadHistoryStore.swift +++ b/apple/TablePro/Core/Diagnostics/TableLoadHistoryStore.swift @@ -1,22 +1,12 @@ import Foundation import os -/// Where a finished trace goes. The one requirement is that the producer is a synchronous -/// `@MainActor` method, so this cannot be `async`: the call has to return before any disk work -/// happens or the measurement pays for its own recording. +// Not async: the producer is a synchronous main-actor method that must return before any disk work, or the measurement pays for its own recording. internal protocol TableLoadSummarySink: Sendable { func record(_ record: TableLoadPerformanceRecord) } -/// A bounded, device-local history of how long table loads took, kept so an intermittent slowdown -/// can be looked at after it happened rather than only while `log stream` is running. -/// -/// Newline-delimited JSON rather than the single re-encoded array `ExecutionAuditLog` uses. Two -/// reasons, both about the caps this store has and that one does not: rewriting the whole array costs -/// the size of the store on every append, which at the ten thousand records allowed here is megabytes -/// written per table click; and a write torn by a crash takes the entire array with it, where a torn -/// final line costs one record. Nothing is held in memory between appends either, because ten -/// thousand decoded records is a lot of residency for a file the running app never reads. +// Newline-delimited JSON: the retention pass rewrites the file, and a write torn by a crash costs one line, not the file. internal actor TableLoadHistoryStore: TableLoadSummarySink { internal static let shared = TableLoadHistoryStore() @@ -28,8 +18,7 @@ internal actor TableLoadHistoryStore: TableLoadSummarySink { private static let newline = UInt8(ascii: "\n") private static let logger = Logger(subsystem: "com.TablePro", category: "TableLoadHistory") - /// Fractional seconds, because a store that measures milliseconds and stamps whole seconds - /// cannot order a burst of navigation inside one, and the retention pass sorts on this field. + // Fractional seconds: the retention pass sorts on this field, and whole seconds cannot order a burst of navigation. private static let timestamps = Date.ISO8601FormatStyle(includingFractionalSeconds: true) private let fileURL: URL @@ -42,8 +31,6 @@ internal actor TableLoadHistoryStore: TableLoadSummarySink { self.fileURL = fileURL } - /// Resolved through `AppStorageEnvironment` like every other store, so a UI test writes into its - /// own sandbox rather than the history of the person running it. internal static func defaultFileURL() -> URL { let directory = AppStorageEnvironment.shared.supportDirectory try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) @@ -62,8 +49,7 @@ internal actor TableLoadHistoryStore: TableLoadSummarySink { pruneIfNeeded() } - /// The first touch in a process reconciles the file against the retention rules, so a history - /// left behind by a build that quit before pruning cannot grow past its caps. + // A history left by a build that quit before pruning must not grow past the caps. private func loadIfNeeded() { guard !loaded else { return } loaded = true @@ -77,14 +63,7 @@ internal actor TableLoadHistoryStore: TableLoadSummarySink { prune() } - /// An undecodable line is dropped rather than refused. A history whose tail was torn by a crash - /// has to keep working, because it is diagnostic data and refusing to read it would make the - /// store itself the outage. The count is logged rather than swallowed, so a schema change that - /// makes an older record unreadable shows up as a number instead of a file that quietly shrank. - /// - /// The counts are set from the file as read before the rewrite, so a write that fails leaves the - /// caps reflecting what is still on disk. Left at zero they would read as under capacity and the - /// only retry would be the hourly one, which is an hour of appending with no bound enforced. + // A torn line is dropped and counted; the caps come from the file as read, so a failed rewrite leaves them true. private func prune() { lastPrunedAt = Date() diff --git a/apple/TablePro/Core/LSP/LSPTransport.swift b/apple/TablePro/Core/LSP/LSPTransport.swift index 6677bb3748..9ee56f2959 100644 --- a/apple/TablePro/Core/LSP/LSPTransport.swift +++ b/apple/TablePro/Core/LSP/LSPTransport.swift @@ -81,6 +81,7 @@ actor LSPTransport { private var writer: LSPMessageWriter? private var stdoutPipe: Pipe? private var stderrPipe: Pipe? + private var stdoutReader: PipeReader? private var stderrReader: PipeReader? private var nextRequestID: Int = 1 private var pendingRequests: [Int: CheckedContinuation] = [:] @@ -128,9 +129,15 @@ actor LSPTransport { try proc.run() - let handle = stdout.fileHandleForReading + // Not FileHandle.bytes: its reads share one queue, so an idle server would stall every other AsyncBytes read. + let (chunks, chunkSink) = AsyncStream.makeStream(of: Data.self) + let stdoutReader = PipeReader(stdout.fileHandleForReading) + stdoutReader.start { chunkSink.yield($0) } atEnd: { _ in + chunkSink.finish() + } + self.stdoutReader = stdoutReader readerTask = Task { [weak self] in - await self?.runReadLoop(handle: handle) + await self?.runReadLoop(chunks) } Self.logger.info("LSP transport started: \(executablePath)") @@ -148,6 +155,8 @@ actor LSPTransport { writer?.close() writer = nil + stdoutReader?.stop() + stdoutReader = nil if let stdoutHandle = stdoutPipe?.fileHandleForReading { try? stdoutHandle.close() } @@ -270,25 +279,19 @@ actor LSPTransport { continuation.resume(throwing: error) } - private func runReadLoop(handle: FileHandle) async { + private func runReadLoop(_ chunks: AsyncStream) async { var buffer = Data() - do { - for try await byte in handle.bytes { - if Task.isCancelled { return } - buffer.append(byte) - while let (messageData, _) = Self.parseMessageFromBuffer(&buffer) { - dispatchMessage(messageData) - } - } - } catch { - if !Task.isCancelled { - Self.logger.debug("LSP read loop ended: \(error.localizedDescription)") + for await chunk in chunks { + // A stream ended by cancellation still hands over the chunks it buffered. + if Task.isCancelled { return } + buffer.append(chunk) + while let (messageData, _) = Self.parseMessageFromBuffer(&buffer) { + dispatchMessage(messageData) } } } - /// Parse a single LSP message from the buffer. - /// Returns (messageBody, totalBytesConsumed) or nil if buffer is incomplete. + // Nil until the buffer holds a whole message; the Int is the bytes it consumed. private static func parseMessageFromBuffer(_ buffer: inout Data) -> (Data, Int)? { let separator: [UInt8] = [0x0D, 0x0A, 0x0D, 0x0A] guard let separatorRange = buffer.range(of: Data(separator)) else { @@ -363,7 +366,7 @@ actor LSPTransport { if let handler = notificationHandlers[method] { handler(data) } - // Server-initiated request (has both id and method) — reply with handler result or null + // Server-initiated request (has both id and method): reply with the handler's result or null if let id { if let deferred = deferredRequestHandlers[method] { deferred(data, id) diff --git a/apple/TablePro/Core/Process/PipeReader.swift b/apple/TablePro/Core/Process/PipeReader.swift index 916db0a0e0..cdb070f0fa 100644 --- a/apple/TablePro/Core/Process/PipeReader.swift +++ b/apple/TablePro/Core/Process/PipeReader.swift @@ -12,16 +12,25 @@ internal final class PipeReader: @unchecked Sendable { private let handle: FileHandle private let descriptor: Int32 - private let lock = NSLock() + // Recursive: an owner may stop the reader from inside its own consumer or end callback. + private let lock = NSRecursiveLock() private var consumer: (@Sendable (Data) -> Void)? + private var atEnd: (@Sendable ((any Error)?) -> Void)? init(_ handle: FileHandle) { self.handle = handle descriptor = handle.fileDescriptor } - func start(delivering consumer: @escaping @Sendable (Data) -> Void) { - lock.withLock { self.consumer = consumer } + // `atEnd` gets nil at end of file or the read error, after the last chunk, and never once the owner stopped the reader. + func start( + delivering consumer: @escaping @Sendable (Data) -> Void, + atEnd: (@Sendable ((any Error)?) -> Void)? = nil + ) { + lock.withLock { + self.consumer = consumer + self.atEnd = atEnd + } handle.readabilityHandler = { _ in self.readArrivedBytes() } @@ -47,6 +56,7 @@ internal final class PipeReader: @unchecked Sendable { handle.readabilityHandler = nil lock.lock() defer { lock.unlock() } + atEnd = nil guard let consumer else { return } self.consumer = nil drain(consumer) @@ -59,7 +69,7 @@ internal final class PipeReader: @unchecked Sendable { do { let chunk = try DescriptorRead.availableBytes(from: descriptor) guard !chunk.isEmpty else { - endReading() + endReading(failure: nil) return } consumer(chunk) @@ -67,12 +77,15 @@ internal final class PipeReader: @unchecked Sendable { return } catch { Self.logger.error("Reading a pipe failed: \(error.publicLogShape, privacy: .public)") - endReading() + endReading(failure: error) } } - private func endReading() { + private func endReading(failure: (any Error)?) { consumer = nil handle.readabilityHandler = nil + let finished = atEnd + atEnd = nil + finished?(failure) } } diff --git a/apple/TablePro/Core/Services/Execution/ExecutionAuditLog.swift b/apple/TablePro/Core/Services/Execution/ExecutionAuditLog.swift index 15a906ccee..d0fd50ad1b 100644 --- a/apple/TablePro/Core/Services/Execution/ExecutionAuditLog.swift +++ b/apple/TablePro/Core/Services/Execution/ExecutionAuditLog.swift @@ -17,27 +17,93 @@ internal enum ExecutionAuditVerification: Equatable, Sendable { case broken(atSequence: Int) } -/// Append-only log of every authorization decision, written from inside `ExecutionGate`. -/// -/// An actor rather than a lock: the gate is called from many tasks, and the sequence number and the -/// previous hash have to be read and advanced together or two concurrent decisions produce two -/// records claiming the same position. +// The file stays the array older builds read and rewrite, so records go in over its closing bracket. internal actor ExecutionAuditLog: ExecutionAuditLogging { internal static let shared = ExecutionAuditLog() private static let logger = Logger(subsystem: "com.TablePro", category: "ExecutionAudit") + private static let tailWindow: UInt64 = 65_536 + private static let openBracket = UInt8(ascii: "[") + private static let closeBracket = UInt8(ascii: "]") + private static let openBrace = UInt8(ascii: "{") + private static let closeBrace = UInt8(ascii: "}") + private static let comma = UInt8(ascii: ",") + private static let maxRecordLength = 4_096 + + // What the file holds: its last record's link, and where its closing bracket sits. + internal struct ChainHead: Equatable, Sendable { + let nextSequence: Int + let lastHash: String + let closingBracketOffset: UInt64 + let fileSize: UInt64 + } + + private enum Disk { + case noRecords + case chain(ChainHead) + case unavailable + } + + // A decision before it has a place in the chain; it is linked to the head read when it is written. + private struct Draft: Sendable { + let recordedAt: Date + let connectionId: UUID + let kind: String + let caller: String + let outcome: ExecutionAuditRecord.Outcome + let effectiveWrite: Bool + let statementDigest: String? + + func linked(sequence: Int, previousHash: String) -> ExecutionAuditRecord { + ExecutionAuditRecord( + sequence: sequence, + recordedAt: recordedAt, + connectionId: connectionId, + kind: kind, + caller: caller, + outcome: outcome, + effectiveWrite: effectiveWrite, + statementDigest: statementDigest, + previousHash: previousHash + ) + } + + func isWritten(as record: ExecutionAuditRecord) -> Bool { + record.recordedAt == recordedAt && record.connectionId == connectionId && record.kind == kind + && record.caller == caller && record.outcome == outcome && record.effectiveWrite == effectiveWrite + && record.statementDigest == statementDigest + } + } + + // Identifies the file this actor last wrote, so a file replaced or edited since is read again. + private struct FileStamp: Equatable { + let inode: UInt64 + let size: UInt64 + let modifiedSeconds: Int + let modifiedNanoseconds: Int + + init(_ info: stat) { + inode = UInt64(info.st_ino) + size = UInt64(info.st_size) + modifiedSeconds = info.st_mtimespec.tv_sec + modifiedNanoseconds = info.st_mtimespec.tv_nsec + } + } private let fileURL: URL - private var records: [ExecutionAuditRecord] = [] - private var loaded = false + private let retryInterval: Duration + private var head: ChainHead? + private var stamp: FileStamp? + // Decided but not on disk yet, oldest first. + private var pending: [Draft] = [] + private var retryAfter: ContinuousClock.Instant? - internal init(fileURL: URL = ExecutionAuditLog.defaultFileURL()) { + internal init(fileURL: URL = ExecutionAuditLog.defaultFileURL(), retryInterval: Duration = .seconds(5)) { self.fileURL = fileURL + self.retryInterval = retryInterval } - /// Resolved through `AppStorageEnvironment` like every other store, so a UI test appends to its - /// own sandbox. The chain is hash linked, which makes a foreign entry indistinguishable from a - /// real one rather than merely noise. + // Inside the storage environment, so a UI test appends to its own sandbox and never to the real chain. internal static func defaultFileURL() -> URL { let directory = AppStorageEnvironment.shared.supportDirectory try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) @@ -45,8 +111,6 @@ internal actor ExecutionAuditLog: ExecutionAuditLogging { } internal func record(request: OperationRequest, decision: OperationDecision) async { - load() - let outcome: ExecutionAuditRecord.Outcome let effectiveWrite: Bool switch decision { @@ -58,23 +122,21 @@ internal actor ExecutionAuditLog: ExecutionAuditLogging { effectiveWrite = false } - let record = ExecutionAuditRecord( - sequence: records.count, + pending.append(Draft( recordedAt: Date(), connectionId: request.connectionId, kind: String(describing: request.kind), caller: Self.describe(request.caller), outcome: outcome, effectiveWrite: effectiveWrite, - statementDigest: request.sql.map(ExecutionAuditRecord.sha256), - previousHash: records.last?.hash ?? ExecutionAuditRecord.genesisHash - ) - records.append(record) - persist() + statementDigest: request.sql.map(ExecutionAuditRecord.sha256) + )) + // A failure keeps the decision in `pending`; a disk that stays full is not retried on every authorization. + if let retryAfter, ContinuousClock.now < retryAfter { return } + retryAfter = writePending() ? nil : ContinuousClock.now + retryInterval } - /// A label, not the payload. An MCP client's label and an AI session id are caller-supplied and - /// can carry anything, so only the channel is recorded. + // A label, not the payload: an MCP client's label and an AI session id are caller-supplied and can carry anything. private static func describe(_ caller: OperationCaller) -> String { switch caller { case .userInterface: "userInterface" @@ -88,37 +150,317 @@ internal actor ExecutionAuditLog: ExecutionAuditLogging { } internal func entries() -> [ExecutionAuditRecord] { - load() - return records + (try? readAllRecords()) ?? [] } internal func verify() -> ExecutionAuditVerification { - load() + guard FileManager.default.fileExists(atPath: fileURL.path(percentEncoded: false)) else { + return .intact(count: 0) + } + guard let records = try? readAllRecords() else { return .broken(atSequence: 0) } if let broken = ExecutionAuditRecord.firstBrokenSequence(in: records) { return .broken(atSequence: broken) } return .intact(count: records.count) } - private func load() { - guard !loaded else { return } - loaded = true - guard let data = try? Data(contentsOf: fileURL) else { return } + private func readAllRecords() throws -> [ExecutionAuditRecord] { + guard let data = try? Data(contentsOf: fileURL) else { return [] } + return try JSONDecoder().decode([ExecutionAuditRecord].self, from: data) + } + + // MARK: - Appending + + private func writePending() -> Bool { + guard FileManager.default.fileExists(atPath: fileURL.path(percentEncoded: false)) else { + return startChain() + } + let handle: FileHandle + do { + handle = try FileHandle(forUpdating: fileURL) + } catch { + Self.logger.error("Execution audit log could not be opened: \(error.publicLogShape, privacy: .public)") + return false + } + defer { try? handle.close() } + + switch disk(through: handle) { + case .unavailable: + return false + case .noRecords: + return startChain() + case .chain(let current): + return append(after: current, through: handle) + } + } + + private func linkPending(after current: ChainHead?) -> [ExecutionAuditRecord] { + var sequence = current?.nextSequence ?? 0 + var previousHash = current?.lastHash ?? ExecutionAuditRecord.genesisHash + return pending.map { draft in + let record = draft.linked(sequence: sequence, previousHash: previousHash) + sequence += 1 + previousHash = record.hash + return record + } + } + + private func disk(through handle: FileHandle) -> Disk { do { - records = try JSONDecoder().decode([ExecutionAuditRecord].self, from: data) + let current = try Self.stamp(of: handle) + guard current.size > 0 else { return .noRecords } + if let head, current == stamp { + return .chain(head) + } + stamp = nil + head = nil + let read: (head: ChainHead, last: ExecutionAuditRecord?) + if let found = try Self.readHead(from: handle, fileSize: current.size) { + read = found + } else if let repaired = try repairTail(of: handle, fileSize: current.size) { + read = repaired + } else { + return setAsideUnreadableLog() ? .noRecords : .unavailable + } + dropWritten(upTo: read.last) + return .chain(read.head) } catch { - // A log that cannot be read is itself a finding, so it is reported rather than - // silently replaced. Starting a fresh chain over it would destroy the evidence. - Self.logger.error("Execution audit log could not be decoded: \(error.localizedDescription)") + Self.logger.error("Execution audit log could not be read: \(error.publicLogShape, privacy: .public)") + return .unavailable } } - private func persist() { + // An append that failed after its bytes landed must not write its records a second time. + private func dropWritten(upTo last: ExecutionAuditRecord?) { + guard let last, let index = pending.lastIndex(where: { $0.isWritten(as: last) }) else { return } + pending.removeFirst(index + 1) + } + + private func append(after current: ChainHead, through handle: FileHandle) -> Bool { + guard !pending.isEmpty else { return true } + let records = linkPending(after: current) + guard let encoded = Self.encode(records), let last = records.last else { return false } + var tail = Data() + if current.nextSequence > 0 { + tail.append(Self.comma) + } + tail.append(encoded) + tail.append(Self.closeBracket) + + do { + try handle.seek(toOffset: current.closingBracketOffset) + try handle.write(contentsOf: tail) + let newSize = current.closingBracketOffset + UInt64(tail.count) + if newSize < current.fileSize { + try handle.truncate(atOffset: newSize) + } + pending = [] + head = ChainHead( + nextSequence: last.sequence + 1, + lastHash: last.hash, + closingBracketOffset: newSize - 1, + fileSize: newSize + ) + stamp = try? Self.stamp(of: handle) + return true + } catch { + head = nil + stamp = nil + Self.restoreClosingBracket(of: handle, at: current.closingBracketOffset) + Self.logger.error("Execution audit log could not be appended to: \(error.publicLogShape, privacy: .public)") + return false + } + } + + // Puts the array back as it was, so neither this build nor an older one finds an unterminated file. + private static func restoreClosingBracket(of handle: FileHandle, at offset: UInt64) { + do { + try handle.seek(toOffset: offset) + try handle.write(contentsOf: Data([closeBracket])) + try handle.truncate(atOffset: offset + 1) + } catch { + logger.error("Execution audit log could not be restored after a failed append") + } + } + + private func startChain() -> Bool { + head = nil + stamp = nil + guard !pending.isEmpty else { return true } + let records = linkPending(after: nil) + guard let encoded = Self.encode(records), let last = records.last else { return false } + var data = Data([Self.openBracket]) + data.append(encoded) + data.append(Self.closeBracket) do { - let data = try JSONEncoder().encode(records) try data.write(to: fileURL, options: .atomic) + pending = [] + head = ChainHead( + nextSequence: last.sequence + 1, + lastHash: last.hash, + closingBracketOffset: UInt64(data.count - 1), + fileSize: UInt64(data.count) + ) + stamp = try? Self.stamp(atPath: fileURL.path(percentEncoded: false)) + return true + } catch { + Self.logger.error("Execution audit log could not be written: \(error.publicLogShape, privacy: .public)") + return false + } + } + + // A file cut short by a crash keeps the records before the cut; a copy of it as found goes aside first. + private func repairTail(of handle: FileHandle, fileSize: UInt64) throws -> (head: ChainHead, last: ExecutionAuditRecord?)? { + let start = fileSize > Self.tailWindow ? fileSize - Self.tailWindow : 0 + try handle.seek(toOffset: start) + let bytes = [UInt8](try handle.read(upToCount: Int(fileSize - start)) ?? Data()) + guard let last = Self.lastRecord(in: bytes, endingAtOrBefore: bytes.count - 1) else { return nil } + + let destination = LocalSQLiteStoreOpener.setAsideURL(for: fileURL, at: Date()) + do { + try FileManager.default.copyItem(at: fileURL, to: destination) + } catch { + Self.logger.error("Execution audit log could not be copied before its repair: \(error.publicLogShape, privacy: .public)") + return nil + } + + let closingOffset = start + UInt64(last.end) + 1 + try handle.seek(toOffset: closingOffset) + try handle.write(contentsOf: Data([Self.closeBracket])) + try handle.truncate(atOffset: closingOffset + 1) + Self.logger.fault("Execution audit log ended mid-record; repaired after its last whole record, original kept aside") + let head = ChainHead( + nextSequence: last.record.sequence + 1, + lastHash: last.record.hash, + closingBracketOffset: closingOffset, + fileSize: closingOffset + 1 + ) + return (head, last.record) + } + + // The unreadable file is evidence: it is kept under a free name, and nothing is written while it cannot be moved. + private func setAsideUnreadableLog() -> Bool { + let destination = LocalSQLiteStoreOpener.setAsideURL(for: fileURL, at: Date()) + do { + try FileManager.default.moveItem(at: fileURL, to: destination) + Self.logger.fault("Execution audit log was unreadable and was set aside; a new chain starts") + return true } catch { - Self.logger.error("Execution audit log could not be written: \(error.localizedDescription)") + Self.logger.fault("Unreadable execution audit log could not be set aside: \(error.publicLogShape, privacy: .public)") + return false + } + } + + private static func encode(_ records: [ExecutionAuditRecord]) -> Data? { + let encoder = JSONEncoder() + var joined = Data() + do { + for (index, record) in records.enumerated() { + if index > 0 { + joined.append(comma) + } + joined.append(try encoder.encode(record)) + } + return joined + } catch { + logger.error("Execution audit record could not be encoded: \(error.publicLogShape, privacy: .public)") + return nil + } + } + + // MARK: - Reading the tail + + private static func stamp(of handle: FileHandle) throws -> FileStamp { + var info = stat() + guard fstat(handle.fileDescriptor, &info) == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + return FileStamp(info) + } + + private static func stamp(atPath path: String) throws -> FileStamp { + var info = stat() + guard stat(path, &info) == 0 else { + throw POSIXError(POSIXErrorCode(rawValue: errno) ?? .EIO) + } + return FileStamp(info) + } + + private static func readHead( + from handle: FileHandle, + fileSize: UInt64 + ) throws -> (head: ChainHead, last: ExecutionAuditRecord?)? { + let start = fileSize > tailWindow ? fileSize - tailWindow : 0 + try handle.seek(toOffset: start) + let bytes = [UInt8](try handle.read(upToCount: Int(fileSize - start)) ?? Data()) + guard let head = chainHead(inTail: Data(bytes), startingAt: start, fileSize: fileSize) else { return nil } + guard head.nextSequence > 0, + let closing = lastNonWhitespaceIndex(in: bytes, before: bytes.count), + let beforeClosing = lastNonWhitespaceIndex(in: bytes, before: closing) + else { return (head, nil) } + return (head, decodedRecord(in: bytes, endingAt: beforeClosing)) + } + + internal static func chainHead(inTail tail: Data, startingAt start: UInt64, fileSize: UInt64) -> ChainHead? { + let bytes = [UInt8](tail) + guard let closing = lastNonWhitespaceIndex(in: bytes, before: bytes.count), + bytes[closing] == closeBracket, + let beforeClosing = lastNonWhitespaceIndex(in: bytes, before: closing) + else { return nil } + + let closingOffset = start + UInt64(closing) + if bytes[beforeClosing] == openBracket { + guard start == 0, lastNonWhitespaceIndex(in: bytes, before: beforeClosing) == nil else { return nil } + return ChainHead( + nextSequence: 0, + lastHash: ExecutionAuditRecord.genesisHash, + closingBracketOffset: closingOffset, + fileSize: fileSize + ) + } + guard let record = decodedRecord(in: bytes, endingAt: beforeClosing) else { return nil } + return ChainHead( + nextSequence: record.sequence + 1, + lastHash: record.hash, + closingBracketOffset: closingOffset, + fileSize: fileSize + ) + } + + internal static func lastRecord( + in bytes: [UInt8], + endingAtOrBefore limit: Int + ) -> (record: ExecutionAuditRecord, end: Int)? { + var end = limit + while end > 0 { + if let record = decodedRecord(in: bytes, endingAt: end) { + return (record, end) + } + end -= 1 + } + return nil + } + + // Records are flat and short, so one starts at the nearest `{` before its `}`, right after a `,` or the `[`. + private static func decodedRecord(in bytes: [UInt8], endingAt end: Int) -> ExecutionAuditRecord? { + guard end > 0, end < bytes.count, bytes[end] == closeBrace, + let start = bytes[max(0, end - maxRecordLength) ..< end].lastIndex(of: openBrace), + let separator = lastNonWhitespaceIndex(in: bytes, before: start), + bytes[separator] == comma || bytes[separator] == openBracket + else { return nil } + return try? JSONDecoder().decode(ExecutionAuditRecord.self, from: Data(bytes[start ... end])) + } + + private static func lastNonWhitespaceIndex(in bytes: [UInt8], before end: Int) -> Int? { + var index = end - 1 + while index >= 0 { + switch bytes[index] { + case 0x20, 0x09, 0x0A, 0x0D: + index -= 1 + default: + return index + } } + return nil } } diff --git a/apple/TablePro/Core/Services/Infrastructure/AgentSessionRegistry.swift b/apple/TablePro/Core/Services/Infrastructure/AgentSessionRegistry.swift index 42e8bc3191..ba42c1e068 100644 --- a/apple/TablePro/Core/Services/Infrastructure/AgentSessionRegistry.swift +++ b/apple/TablePro/Core/Services/Infrastructure/AgentSessionRegistry.swift @@ -7,12 +7,7 @@ import Combine import Foundation import os -/// Every agent session the app is holding, for as long as the app is running. -/// -/// Sessions used to be reached through a field on the window's trailing pane, which made a -/// session's lifetime the window's: closing a window, disconnecting, or losing a session each -/// destroyed a transcript the user never asked to lose. They live here instead, so a window is one -/// of the places a session can be looked at rather than the thing that owns it. +// Owns every agent session, so closing a window or disconnecting never destroys a transcript. @MainActor internal final class AgentSessionRegistry: ObservableObject { internal static let shared = AgentSessionRegistry() @@ -24,20 +19,10 @@ internal final class AgentSessionRegistry: ObservableObject { private let store: AgentSessionStore private let services: AppServices - /// Serialises writes. `persist()` used to fire an unordered task per mutation, and actor - /// reentrancy let a stale snapshot land last. + // One writer: unordered tasks let a stale snapshot land last. private var writeTask: Task? - /// Which session each connection is showing. - /// - /// Held explicitly rather than derived. Deriving it from `sessions(for:)` answered the oldest - /// one every time, so New Session appended a row nothing switched to and Open Session moved a - /// timestamp nothing read: both panes stayed bound to the first session for ever. - /// - /// Showing nothing is a state of its own. Closing or deleting the session on screen with no other - /// live one to hand to used to fall back to the last session in the list, which was the one just - /// closed, so the column went on drawing a stopped session with a composer that still took - /// messages. + // `.nothing` is a state of its own, or a closed last session would stay drawn with a live composer. private var displayed: [UUID: DisplayedSession] = [:] private enum DisplayedSession: Equatable { @@ -45,14 +30,10 @@ internal final class AgentSessionRegistry: ObservableObject { case nothing } - /// Going to work moves a session up the rail, and the rail observes this registry rather than - /// each session, so the registry has to say so, and write the new stamp so the order survives a - /// relaunch. + // The rail observes the registry, not each session, so a session going to work is announced here. private var activityCancellables: [UUID: AnyCancellable] = [:] - /// Restore runs here, synchronously, which is the whole point of the store being a plain struct. - /// A window that opens while a load is suspended finds nothing, mints a session, and is then - /// joined by the stored one; doing it in the initialiser means no window can exist yet. + // Restored synchronously, before any window exists to mint a second session beside a stored one. internal init(store: AgentSessionStore = AgentSessionStore(), services: AppServices = .live) { self.store = store self.services = services @@ -78,8 +59,7 @@ internal final class AgentSessionRegistry: ObservableObject { } } - /// `@Published` announces a change before it is stored, so the write waits for the next turn to - /// read the new stamp rather than the one it replaces. + // `@Published` announces before it stores, so the write reads the stamp on the next turn. private func observeActivity(of session: AgentSession) { activityCancellables[session.id] = session.$lastActiveAt .dropFirst() @@ -96,8 +76,7 @@ internal final class AgentSessionRegistry: ObservableObject { // MARK: - Reading - /// A connection's sessions, the one that last went to work first, which is the order the rail - /// lists them in. Two stamped in the same instant keep the order they were added in, newest first. + // The rail's order. Two stamped in the same instant list the one added last first. internal func sessions(for connectionId: UUID) -> [AgentSession] { sessions.enumerated() .filter { $0.element.connectionId == connectionId } @@ -114,15 +93,7 @@ internal final class AgentSessionRegistry: ObservableObject { sessions.first { $0.id == id } } - /// The session a connection's two presentations share. - /// - /// Both the trailing-pane chat and the agent-mode conversation column resolve through here, so - /// they cannot end up rendering two different sessions of the same connection. Reading never - /// creates: opening a connection window starts no session and loads no transcript. - /// - /// With nothing named it is the latest live session, then the latest of any, which is how a - /// window opened after a relaunch, where every session comes back stopped, carries on with the - /// last one rather than starting another. + // Falls back to the latest of any, so a window after a relaunch, with every session stopped, carries on. internal func currentSession(for connectionId: UUID) -> AgentSession? { let owned = sessions(for: connectionId) switch displayed[connectionId] { @@ -138,8 +109,6 @@ internal final class AgentSessionRegistry: ObservableObject { return owned.first { !$0.status.isEnded } ?? owned.first } - /// Names the session a connection's two panes render. The pane render key carries it, so a - /// switch repaints rather than comparing equal. internal func setDisplayedSession(_ sessionId: UUID, for connectionId: UUID) { guard sessions.contains(where: { $0.id == sessionId && $0.connectionId == connectionId }) else { return @@ -147,9 +116,7 @@ internal final class AgentSessionRegistry: ObservableObject { displayed[connectionId] = .session(sessionId) } - /// Shows the latest live session in place of one that is ending or going away, and nothing when - /// no other live one is left. A stopped session is shown only once someone opens it, which - /// resumes it. + // A stopped session is shown only once someone opens it, which resumes it. private func handOverDisplay(from session: AgentSession) { let next = sessions(for: session.connectionId).first { $0.id != session.id && !$0.status.isEnded } displayed[session.connectionId] = next.map { .session($0.id) } ?? .nothing @@ -173,7 +140,6 @@ internal final class AgentSessionRegistry: ObservableObject { return session } - /// The session to show for a connection, starting one only when the caller means to. @discardableResult internal func resolveSession(for connectionId: UUID, startingIfNeeded: Bool) -> AgentSession? { if let existing = currentSession(for: connectionId) { @@ -193,11 +159,7 @@ internal final class AgentSessionRegistry: ObservableObject { return startSession(for: connectionId) } - /// Ends one session and keeps its transcript. The conversation stays in the history and the - /// session stays in the rail, to be opened again. - /// - /// The session on screen is handed over as it stops, which is what Close Session means. Nothing - /// used to tell the panes, so they went on drawing a stopped session with a live composer. + // The panes are handed over first, so no window shows the session when its memory is released. internal func stopSession(id: UUID) { guard let session = session(id: id) else { return } let isShown = currentSession(for: session.connectionId) === session @@ -206,32 +168,33 @@ internal final class AgentSessionRegistry: ObservableObject { if isShown { handOverDisplay(from: session) } + session.release() persist() } - /// Ends every session on a connection once nothing is showing it any more. - /// - /// Called from the connection's own teardown rather than from a window's, because a connection - /// can be hosted by two windows and one of them closing is not the connection going away. + // A connection can be hosted by two windows, and one of them closing is not the connection going away. internal func stopSessionsIfUnhosted(for connectionId: UUID) { guard WindowManager.shared.workspaces(for: connectionId).isEmpty else { return } stopSessions(for: connectionId) } - /// Ends every session on a connection. Disconnect and window close reach this; neither is the - /// user discarding a conversation, so nothing is deleted. + // Ended sessions are released too: a failed one, or one stopped while on screen, still holds its transcript. internal func stopSessions(for connectionId: UUID) { - let owned = sessions(for: connectionId).filter { !$0.status.isEnded } - guard !owned.isEmpty else { return } - for session in owned { + let owned = sessions(for: connectionId) + let live = owned.filter { !$0.status.isEnded } + for session in live { session.stop() detachRemoteTools(from: session.id) } - persist() + for session in owned { + session.release() + } + if !live.isEmpty { + persist() + } } - /// Discards a session and the conversation behind it. Only an explicit user action reaches here, - /// and it has been asked to confirm by then. + // Only an explicit, confirmed user action reaches here. internal func removeSession(id: UUID) { guard let index = sessions.firstIndex(where: { $0.id == id }) else { return } let session = sessions[index] @@ -246,35 +209,25 @@ internal final class AgentSessionRegistry: ObservableObject { handOverDisplay(from: session) } if let conversationId { - let storage = services.aiChatStorage - Task { await storage.delete(conversationId) } + session.viewModel.deleteStoredConversations([conversationId]) } persist() } // MARK: - Outside MCP servers - /// Connects the outside MCP servers this session's connection allows, and registers their tools. - /// - /// Started rather than awaited. A server on the other side of a network is not something a - /// session's first paint may wait on, and a session with no remote tools yet is a session the - /// model simply has not been offered them in; they appear on the turn after they land. - /// - /// Attaching an already-attached session is how a resumed one keeps its tools, so this is safe - /// to call on every resolve. + // Not awaited: a first paint never waits on a remote server. Attaching again keeps a resumed session's tools. private func attachRemoteTools(to session: AgentSession) { Task { await MCPRemoteToolCoordinator.shared.attach(session: session) } } - /// Drops this session's claim on every server. The tools stay registered while another session - /// still holds one, so ending one conversation cannot take them from another mid-turn. + // The tools stay registered while another session still holds a server. private func detachRemoteTools(from sessionId: UUID) { Task { await MCPRemoteToolCoordinator.shared.detach(sessionId: sessionId) } } // MARK: - Persistence - /// One writer, so the last snapshot taken is the last one written. internal func persist() { let records = sessions.map(\.record) let store = store @@ -286,10 +239,7 @@ internal final class AgentSessionRegistry: ObservableObject { } } - /// Writes now, on this thread, and returns once it is on disk. - /// - /// `persist()` books the write for the next turn, which is right for an ordinary mutation and - /// wrong for anything that has to know the write landed. + // For a caller that has to know the write landed; `persist()` books it for the next turn. internal func persistNow() { writeTask?.cancel() writeTask = nil @@ -299,11 +249,7 @@ internal final class AgentSessionRegistry: ObservableObject { store.save(sessions.map(\.record)) } - /// Writes on the way out. - /// - /// An actor hop at terminate may never be scheduled, so a session killed mid-reply used to come - /// back with its last turn missing. A session still working when the app went away is recorded - /// as failed, because it was: its reply never finished. + // A hop at terminate may never run. A session still working is recorded as failed: its reply never finished. internal func persistSynchronouslyForTermination() { for session in sessions where session.status == .working { session.mark(.failed) diff --git a/apple/TablePro/Core/Storage/AIChatStorage.swift b/apple/TablePro/Core/Storage/AIChatStorage.swift index 5420eb2f35..0e09c5245d 100644 --- a/apple/TablePro/Core/Storage/AIChatStorage.swift +++ b/apple/TablePro/Core/Storage/AIChatStorage.swift @@ -2,13 +2,10 @@ // AIChatStorage.swift // TablePro // -// File-based persistence for AI chat conversations. -// import Foundation import os -/// Manages persistent storage of AI chat conversations as individual JSON files actor AIChatStorage { static let shared = AIChatStorage() @@ -35,7 +32,6 @@ actor AIChatStorage { .appendingPathComponent("ai_chats", isDirectory: true) directory = dir - // Create directory inline since actor init is nonisolated do { try FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) try FileManager.default.setAttributes( @@ -49,23 +45,16 @@ actor AIChatStorage { // MARK: - Public Methods - /// Maximum encoded size for a single conversation file (500 KB) private static let maxFileSize = 500_000 - /// Maximum number of messages to keep after trimming - private static let trimmedMessageCount = 50 + // Past `maxFileSize` only the last turns are written, so a longer transcript may be on disk in part. + static let turnsKeptWhenTrimmed = 50 - /// Save a conversation to disk func save(_ conversation: AIConversation) { Self.write(conversation, into: directory) } - /// Writes on the calling thread, for the one caller that cannot wait for an actor hop. - /// - /// `applicationWillTerminate` runs to the end of the main run loop turn and the process goes; - /// a hop booked there is never scheduled, so the last turns of a reply were lost while the - /// session record on disk still named the conversation they belonged to. The body touches no - /// actor state beyond `directory`, which is a `let`. + // A hop booked in applicationWillTerminate is never scheduled. Touches nothing but the `let` directory. nonisolated func saveSynchronously(_ conversation: AIConversation) { Self.write(conversation, into: directory) } @@ -80,7 +69,7 @@ actor AIChatStorage { let originalSize = data.count let originalCount = conversation.messages.count var trimmed = conversation - trimmed.messages = Array(trimmed.messages.suffix(Self.trimmedMessageCount)) + trimmed.messages = Array(trimmed.messages.suffix(Self.turnsKeptWhenTrimmed)) let dropped = originalCount - trimmed.messages.count data = try Self.encoder.encode(trimmed) Self.logger.warning( @@ -98,51 +87,48 @@ actor AIChatStorage { } } - /// One connection's conversations, newest first. - /// - /// A conversation stored before conversations carried a connection id has no answer, so it is - /// shown on every connection rather than lost. - func loadAll(connectionId: UUID?) -> [AIConversation] { - guard let connectionId else { return loadAll() } - return loadAll().filter { $0.connectionId == nil || $0.connectionId == connectionId } - } - - /// One conversation by id, without reading the rest of the directory. - func load(id: UUID) -> AIConversation? { - let fileURL = directory.appendingPathComponent("\(id.uuidString).json") - guard let data = try? Data(contentsOf: fileURL) else { return nil } - return try? Self.decoder.decode(AIConversation.self, from: data) + // A conversation stored before conversations carried a connection id is shown everywhere, not lost. + static func isShown(conversationOf owner: UUID?, on connectionId: UUID?) -> Bool { + connectionId == nil || owner == nil || owner == connectionId } - /// Load all conversations, sorted by updatedAt descending - func loadAll() -> [AIConversation] { + // The history list needs titles only, so no stored conversation's messages are held. + func loadSummaries(connectionId: UUID?) -> [AIConversation] { + let files: [URL] do { - let files = try FileManager.default.contentsOfDirectory( + files = try FileManager.default.contentsOfDirectory( at: directory, includingPropertiesForKeys: nil, options: .skipsHiddenFiles ) - - let conversations: [AIConversation] = files - .filter { $0.pathExtension == "json" } - .compactMap { fileURL in - do { - let data = try Data(contentsOf: fileURL) - return try Self.decoder.decode(AIConversation.self, from: data) - } catch { - Self.logger.error("Failed to load conversation from \(fileURL.lastPathComponent): \(error.localizedDescription)") - return nil - } - } - - return conversations.sorted { $0.updatedAt > $1.updatedAt } } catch { Self.logger.error("Failed to list conversations: \(error.localizedDescription)") return [] } + + let summaries: [AIConversation] = files + .filter { $0.pathExtension == "json" } + .compactMap { fileURL in + do { + let data = try Data(contentsOf: fileURL) + let header = try Self.decoder.decode(ConversationHeader.self, from: data) + guard Self.isShown(conversationOf: header.connectionId, on: connectionId) else { return nil } + return header.summary + } catch { + Self.logger.error("Failed to load conversation from \(fileURL.lastPathComponent): \(error.localizedDescription)") + return nil + } + } + + return summaries.sorted { $0.updatedAt > $1.updatedAt } + } + + func load(id: UUID) -> AIConversation? { + let fileURL = directory.appendingPathComponent("\(id.uuidString).json") + guard let data = try? Data(contentsOf: fileURL) else { return nil } + return try? Self.decoder.decode(AIConversation.self, from: data) } - /// Delete a conversation by ID func delete(_ id: UUID) { let fileURL = directory.appendingPathComponent("\(id.uuidString).json") @@ -153,7 +139,6 @@ actor AIChatStorage { } } - /// Delete all conversations func deleteAll() { do { let files = try FileManager.default.contentsOfDirectory( @@ -168,4 +153,27 @@ actor AIChatStorage { Self.logger.error("Failed to delete all conversations: \(error.localizedDescription)") } } + + // Decodes the same keys as `AIConversation` and the same defaults, without `messages`. + private struct ConversationHeader: Decodable { + let id: UUID + let title: String? + let createdAt: Date + let updatedAt: Date + let connectionId: UUID? + let connectionName: String? + let schemaVersion: Int? + + var summary: AIConversation { + AIConversation( + id: id, + title: title ?? "", + createdAt: createdAt, + updatedAt: updatedAt, + connectionId: connectionId, + connectionName: connectionName, + schemaVersion: max(schemaVersion ?? 0, AIConversation.currentSchemaVersion) + ) + } + } } diff --git a/apple/TablePro/Core/Storage/TabDiskActor.swift b/apple/TablePro/Core/Storage/TabDiskActor.swift index a91cc40df4..cfe4a44a76 100644 --- a/apple/TablePro/Core/Storage/TabDiskActor.swift +++ b/apple/TablePro/Core/Storage/TabDiskActor.swift @@ -2,11 +2,8 @@ // TabDiskActor.swift // TablePro // -// Thread-safe actor for tab state persistence. -// Replaces TabStateStorage with actor-based serialization -// to eliminate data races on concurrent file writes. -// +import CryptoKit import Foundation import os @@ -52,9 +49,15 @@ private struct LossyTab: Decodable { } } +// What this process last wrote or read for one connection, so an unchanged save leaves the disk alone. +internal struct TabDiskWrittenState { + var stateDigest: SHA256.Digest? + var sidecars = TabQueryOverflowStore.WrittenSidecars() +} + private final class TabDiskConnectionWriteGate: @unchecked Sendable { private let generation = OSAllocatedUnfairLock(initialState: UInt64.zero) - private let operation = OSAllocatedUnfairLock(initialState: ()) + private let operation = OSAllocatedUnfairLock(initialState: TabDiskWrittenState()) func issueToken() -> UInt64 { generation.withLock { value in @@ -65,17 +68,17 @@ private final class TabDiskConnectionWriteGate: @unchecked Sendable { func performIfCurrent( token: UInt64, - operation work: () throws -> T + operation work: (inout TabDiskWrittenState) throws -> T ) rethrows -> T? { - try operation.withLockUnchecked { _ in + try operation.withLockUnchecked { written in guard generation.withLock({ $0 == token }) else { return nil } - return try work() + return try work(&written) } } - func perform(_ work: () throws -> T) rethrows -> T { - try operation.withLockUnchecked { _ in - try work() + func perform(_ work: (inout TabDiskWrittenState) throws -> T) rethrows -> T { + try operation.withLockUnchecked { written in + try work(&written) } } } @@ -116,7 +119,7 @@ internal actor TabDiskActor { private init() { let directory = Self.resolvedTabStateDirectory() tabStateDirectory = directory - encoder = JSONEncoder() + encoder = Self.makeEncoder() decoder = JSONDecoder() do { @@ -137,26 +140,62 @@ internal actor TabDiskActor { lastActiveSchema: String? = nil, writeToken: UInt64 ) throws -> Bool { - let result = try Self.writeGate(for: connectionId).performIfCurrent(token: writeToken) { - let state = TabDiskState( - tabs: TabQueryOverflowStore.externalize( - tabs, - connectionId: connectionId, - tabStateDirectory: tabStateDirectory - ), + let result = try Self.writeGate(for: connectionId).performIfCurrent(token: writeToken) { written in + try Self.write( + tabs: tabs, selectedTabId: selectedTabId, lastActiveDatabase: lastActiveDatabase, - lastActiveSchema: lastActiveSchema + lastActiveSchema: lastActiveSchema, + connectionId: connectionId, + directory: tabStateDirectory, + encoder: encoder, + written: &written ) - let data = try encoder.encode(state) - let fileURL = tabStateFileURL(for: connectionId) - try data.write(to: fileURL, options: .atomic) } return result != nil } + // Sorted keys, so a dictionary rebuilt in another order encodes to the same bytes and the unchanged check holds. + nonisolated private static func makeEncoder() -> JSONEncoder { + let encoder = JSONEncoder() + encoder.outputFormatting = [.sortedKeys] + return encoder + } + + // Every save funnels here: the 30 s timer, the draft and tab-switch saves, and quit. Most find nothing new. + nonisolated private static func write( + tabs: [PersistedTab], + selectedTabId: UUID?, + lastActiveDatabase: String?, + lastActiveSchema: String?, + connectionId: UUID, + directory: URL, + encoder: JSONEncoder, + written: inout TabDiskWrittenState + ) throws { + let state = TabDiskState( + tabs: TabQueryOverflowStore.externalize( + tabs, + connectionId: connectionId, + tabStateDirectory: directory, + written: &written.sidecars + ), + selectedTabId: selectedTabId, + lastActiveDatabase: lastActiveDatabase, + lastActiveSchema: lastActiveSchema + ) + let data = try encoder.encode(state) + let digest = SHA256.hash(data: data) + let fileURL = directory.appendingPathComponent("\(connectionId.uuidString).json") + if digest == written.stateDigest, FileManager.default.fileExists(atPath: fileURL.path) { + return + } + try data.write(to: fileURL, options: .atomic) + written.stateDigest = digest + } + internal func load(connectionId: UUID) -> TabDiskState? { - Self.writeGate(for: connectionId).perform { + Self.writeGate(for: connectionId).perform { written in let fileURL = tabStateFileURL(for: connectionId) guard FileManager.default.fileExists(atPath: fileURL.path) else { @@ -166,13 +205,20 @@ internal actor TabDiskActor { do { let data = try Data(contentsOf: fileURL) let state = try decoder.decode(TabDiskState.self, from: data) + written.stateDigest = SHA256.hash(data: data) return TabDiskState( - tabs: TabQueryOverflowStore.inline(state.tabs, tabStateDirectory: tabStateDirectory), + tabs: TabQueryOverflowStore.inline( + state.tabs, + tabStateDirectory: tabStateDirectory, + written: &written.sidecars + ), selectedTabId: state.selectedTabId, lastActiveDatabase: state.lastActiveDatabase, lastActiveSchema: state.lastActiveSchema ) } catch { + // The file no longer holds what this process wrote, so the next save must write it. + written = TabDiskWrittenState() Self.logger.error("Failed to load tab state for \(connectionId): \(error.localizedDescription)") return nil } @@ -180,7 +226,8 @@ internal actor TabDiskActor { } internal func clear(connectionId: UUID, writeToken: UInt64) -> Bool { - let result = Self.writeGate(for: connectionId).performIfCurrent(token: writeToken) { + let result = Self.writeGate(for: connectionId).performIfCurrent(token: writeToken) { written in + written = TabDiskWrittenState() TabQueryOverflowStore.removeAll(connectionId: connectionId, tabStateDirectory: tabStateDirectory) let fileURL = tabStateFileURL(for: connectionId) @@ -227,21 +274,18 @@ internal actor TabDiskActor { let writeToken = issueWriteToken(for: connectionId) let directory = resolvedTabStateDirectory() do { - _ = try writeGate(for: connectionId).performIfCurrent(token: writeToken) { - let state = TabDiskState( - tabs: TabQueryOverflowStore.externalize( - tabs, - connectionId: connectionId, - tabStateDirectory: directory - ), + _ = try writeGate(for: connectionId).performIfCurrent(token: writeToken) { written in + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try write( + tabs: tabs, selectedTabId: selectedTabId, lastActiveDatabase: lastActiveDatabase, - lastActiveSchema: lastActiveSchema + lastActiveSchema: lastActiveSchema, + connectionId: connectionId, + directory: directory, + encoder: makeEncoder(), + written: &written ) - let data = try JSONEncoder().encode(state) - try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - let fileURL = tabStateFileURL(for: connectionId) - try data.write(to: fileURL, options: .atomic) } } catch { logger.fault("saveSync failed for \(connectionId): \(error.localizedDescription)") @@ -250,7 +294,8 @@ internal actor TabDiskActor { nonisolated internal static func clearSync(connectionId: UUID) { let writeToken = issueWriteToken(for: connectionId) - _ = writeGate(for: connectionId).performIfCurrent(token: writeToken) { + _ = writeGate(for: connectionId).performIfCurrent(token: writeToken) { written in + written = TabDiskWrittenState() TabQueryOverflowStore.removeAll( connectionId: connectionId, tabStateDirectory: resolvedTabStateDirectory() diff --git a/apple/TablePro/Core/Storage/TabQueryOverflowStore.swift b/apple/TablePro/Core/Storage/TabQueryOverflowStore.swift index b3977473f2..fe2a2fe0a6 100644 --- a/apple/TablePro/Core/Storage/TabQueryOverflowStore.swift +++ b/apple/TablePro/Core/Storage/TabQueryOverflowStore.swift @@ -2,17 +2,21 @@ // TabQueryOverflowStore.swift // TablePro // -// Keeps a query too large to sit inside the tab-state JSON in a sidecar file -// beside it, so restoring a tab returns the text instead of an empty editor. -// Mirrors the sidecar `RecentlyClosedTabStore` already uses for the same limit. -// +import CryptoKit import Foundation import os +// A query too large for the tab-state JSON lives in a sidecar beside it, as in `RecentlyClosedTabStore`. internal enum TabQueryOverflowStore { private static let logger = Logger(subsystem: "com.TablePro", category: "TabDiskActor") + // The sidecars one connection has on disk from this process, so an unchanged query is not rewritten. + internal struct WrittenSidecars { + var digests: [String: SHA256.Digest] = [:] + var keptByLastPrune: Set? + } + static func directory(inside tabStateDirectory: URL) -> URL { tabStateDirectory.appendingPathComponent("Overflow", isDirectory: true) } @@ -21,34 +25,46 @@ internal enum TabQueryOverflowStore { "\(connectionId.uuidString)_\(tabId.uuidString).sql" } - /// Moves any oversized query out to a sidecar and blanks the inline field. - /// Sidecars this connection no longer references are removed in the same pass, - /// so a closed tab does not leave its text behind forever. + // Sidecars the connection no longer references are removed in the same pass, so a closed tab leaves no text behind. static func externalize( _ tabs: [PersistedTab], connectionId: UUID, - tabStateDirectory: URL + tabStateDirectory: URL, + written: inout WrittenSidecars ) -> [PersistedTab] { let overflowDirectory = directory(inside: tabStateDirectory) - var written: Set = [] + var kept: Set = [] let processed = tabs.map { tab -> PersistedTab in guard (tab.query as NSString).length > TabQueryContent.maxPersistableQuerySize else { return tab } let name = fileName(connectionId: connectionId, tabId: tab.id) - guard write(tab.query, named: name, in: overflowDirectory) else { return tab } - written.insert(name) + let digest = digest(of: tab.query) + let path = overflowDirectory.appendingPathComponent(name).path + if written.digests[name] != digest || !FileManager.default.fileExists(atPath: path) { + written.digests[name] = nil + guard write(tab.query, named: name, in: overflowDirectory) else { return tab } + written.digests[name] = digest + } + kept.insert(name) var copy = tab copy.query = "" copy.overflowFileName = name return copy } - pruneOrphans(connectionId: connectionId, keeping: written, in: overflowDirectory) + written.digests = written.digests.filter { kept.contains($0.key) } + if written.keptByLastPrune != kept { + pruneOrphans(connectionId: connectionId, keeping: kept, in: overflowDirectory) + written.keptByLastPrune = kept + } return processed } - /// Reads sidecar text back into the tabs it belongs to. - static func inline(_ tabs: [PersistedTab], tabStateDirectory: URL) -> [PersistedTab] { + static func inline( + _ tabs: [PersistedTab], + tabStateDirectory: URL, + written: inout WrittenSidecars + ) -> [PersistedTab] { let overflowDirectory = directory(inside: tabStateDirectory) return tabs.map { tab -> PersistedTab in guard let name = tab.overflowFileName else { return tab } @@ -57,12 +73,20 @@ internal enum TabQueryOverflowStore { logger.error("Missing overflow sidecar \(name, privacy: .public)") return tab } + written.digests[name] = digest(of: text) var copy = tab copy.query = text return copy } } + private static func digest(of query: String) -> SHA256.Digest { + if let digest = query.utf8.withContiguousStorageIfAvailable({ SHA256.hash(data: UnsafeRawBufferPointer($0)) }) { + return digest + } + return SHA256.hash(data: Data(query.utf8)) + } + static func removeAll(connectionId: UUID, tabStateDirectory: URL) { pruneOrphans(connectionId: connectionId, keeping: [], in: directory(inside: tabStateDirectory)) } diff --git a/apple/TablePro/Models/AI/AgentSession.swift b/apple/TablePro/Models/AI/AgentSession.swift index a2b68570b2..b54d17cb24 100644 --- a/apple/TablePro/Models/AI/AgentSession.swift +++ b/apple/TablePro/Models/AI/AgentSession.swift @@ -6,22 +6,12 @@ import Combine import Foundation -/// One conversation with the assistant about one connection. -/// -/// A session outlives the window that opened it. Closing a window stops its sessions and keeps -/// their transcripts; opening one again continues where it stopped, and nothing is replayed. -/// -/// It owns exactly one `AIChatViewModel`, which is what makes "one session, two presentations" -/// true: the trailing-pane chat in Browse mode and the conversation column in Agent mode render -/// this same view model, so there is one transcript, one composer draft and one scroll position -/// however the user is looking at it. +// Both presentations render this one `AIChatViewModel`, so there is one transcript, draft and scroll position. @MainActor internal final class AgentSession: ObservableObject, Identifiable { internal let id: UUID internal let connectionId: UUID - /// The engine. Built with this session's id so a restored session is the same session rather - /// than a new one wearing its transcript. internal let viewModel: AIChatViewModel @Published internal private(set) var status: AgentSessionStatus @@ -29,32 +19,15 @@ internal final class AgentSession: ObservableObject, Identifiable { internal let startedAt: Date - /// When the session last went to work: the start of the reply it last began, or its own start. - /// The rail lists the latest first. - /// - /// Opening a session is not work, so it does not move one. It used to, and the rail was ordered - /// by start time while it did, so the stamp changed nothing anyone could see; ordered by this, a - /// session double-clicked in the middle of the list would have jumped to the top from under the - /// pointer that opened it. + // The rail's order. Opening a session is not work, or a row would jump from under the pointer. @Published internal private(set) var lastActiveAt: Date - /// Which of the result column's views this session is showing. - /// - /// On the session rather than in the column: the column is one hosting controller per window, - /// drawing whichever session is open, so view state there carried one session's choice into the - /// next. Not stored, so a relaunch opens on the statements. + // The result column is one hosting controller per window, so state kept there leaks between sessions. @Published internal var resultSegment: AgentResultSegment = .sql - /// Text typed before the session could send it, which a connect long enough to notice a typo in - /// needs. Cleared before it is dispatched so three flush sites still send once. internal var pendingPrompt: String? - /// Hands the pending prompt over once the connection is up, and clears it as it goes. - /// - /// The conversation asks from a `task` keyed on the session, the connect and the prompt, and a - /// reparent re-runs every such task on the same view: a mode toggle and a connection switch are - /// both one. Taking rather than reading is what keeps each of those re-runs from sending the - /// prompt a second time. + // A reparent re-runs the task that asks for this, so taking rather than reading sends it once. internal func takePendingPrompt(isConnecting: Bool) -> String? { guard !isConnecting, let prompt = pendingPrompt else { return nil } pendingPrompt = nil @@ -82,27 +55,16 @@ internal final class AgentSession: ObservableObject, Identifiable { observeEngine() } - /// The conversation this session's transcript is stored under, or nil before the first reply. + // A session restored from disk or released holds its conversation's id before its transcript. internal var conversationId: UUID? { - viewModel.activeConversationID + viewModel.activeConversationID ?? viewModel.conversationToRestore } - /// What the rail, the conversation and the delete confirmation call the session. A session names - /// itself from its first question, so one that has not been asked anything goes by the command - /// that made it. internal var displayTitle: String { title.isEmpty ? String(localized: "New Session") : title } - /// Republishes the engine's changes as the session's own, so a rail row bound to the session - /// redraws when the transcript moves. `AIChatViewModel` is an `ObservableObject` of its own and - /// nothing else forwards it. - /// Republishes the engine's changes, and its turns'. - /// - /// A `ChatTurn` is an `ObservableObject` of its own, and appending a tool-use block mutates the - /// turn rather than the view model's `messages` array, so the view model never announced it. - /// The result pane could not show a proposed statement until some later top-level change - /// happened to fire, which for a card waiting on an answer is never. + // A tool-use block mutates its turn rather than `messages`, so each turn is observed as well. private func observeEngine() { viewModel.objectWillChange .sink { [weak self] _ in @@ -132,14 +94,7 @@ internal final class AgentSession: ObservableObject, Identifiable { } } - /// A stopped session keeps the state it ended on; a failed one does not. - /// - /// Retry is offered on a failure and moves the engine back through idle, loading and streaming, - /// so freezing on `.failed` left the rail reporting Failed for the whole of a successful retry - /// and session resolution still treating it as ended. - /// - /// Going to work is what makes a session the latest, so the rail moves it up as a reply starts - /// rather than as someone looks at it. + // A failed session follows the engine, since Retry moves it back through streaming. private func refreshDerivedState() { let engineStatus = derivedStatus() if status != .stopped, status != engineStatus { @@ -148,15 +103,15 @@ internal final class AgentSession: ObservableObject, Identifiable { } status = engineStatus } + // Empty lists say nothing about the title of a transcript that is not read yet. + guard !viewModel.isAwaitingTranscript else { return } let derivedTitle = derivedTitle() if title != derivedTitle { title = derivedTitle } } - /// Waiting is asked before working, because an ordinary tool card leaves `streamingState` on - /// `.streaming`: the stream is parked on the answer, not producing. `.awaitingApproval` is the - /// connection's own AI-access confirmation and is a different question. + // A tool card leaves `streamingState` on `.streaming` while the stream waits on the answer. private func derivedStatus() -> AgentSessionStatus { if ToolApprovalCenter.shared.hasPending(sessionId: id) { return .waitingOnYou } if case .awaitingApproval = viewModel.streamingState { return .waitingOnYou } @@ -190,10 +145,7 @@ internal final class AgentSession: ObservableObject, Identifiable { status = newStatus } - /// Ends the session without touching what it said. - /// - /// Window close, disconnect and a lost session all reach here, and none of them is the user - /// throwing a conversation away, so the transcript is written first and kept. + // None of the callers is the user throwing a conversation away, so the transcript is written first. internal func stop() { viewModel.cancelStream() settlePendingApprovals() @@ -201,23 +153,18 @@ internal final class AgentSession: ObservableObject, Identifiable { mark(.stopped) } - /// Puts a stopped session back to work. Nothing is replayed: a statement that was waiting for an - /// answer when the window closed was cancelled by the stop, and the model is asked again rather - /// than the call being re-issued behind the user's back. - /// - /// Resuming is opening, not working, so the session keeps its place in the rail until it is - /// asked something. + // Only once no window shows the session; its next showing reads back from disk what this drops. + internal func release() { + viewModel.releaseLoadedState() + } + + // Nothing is replayed: a statement waiting for an answer was cancelled by the stop. internal func resume() { guard status.isEnded else { return } mark(.ready) } - /// Settles a card that is still waiting before the transcript is written. - /// - /// `cancelStream()` resumes the continuation, but the suspended task only marks the block - /// `.cancelled` once it is back on the main actor, and the snapshot was taken before that. The - /// restored transcript then held a `.pending` card with no continuation behind it, whose Run - /// button could never work. + // The cancelled task marks its card after the snapshot, which left a dead `.pending` card on disk. private func settlePendingApprovals() { for turn in viewModel.messages { for block in turn.blocks { @@ -242,8 +189,7 @@ internal final class AgentSession: ObservableObject, Identifiable { } } -/// What a session is on disk. It points at a conversation rather than copying its turns, so the -/// transcript has one home and a restored session cannot disagree with it. +// Points at a conversation rather than copying its turns, so the transcript has one home. internal struct AgentSessionRecord: Codable, Equatable, Identifiable, Sendable { internal static let currentSchemaVersion = 1 diff --git a/apple/TablePro/ViewModels/AIChatViewModel+Persistence.swift b/apple/TablePro/ViewModels/AIChatViewModel+Persistence.swift index 108e8c9e2e..f6fced312e 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel+Persistence.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel+Persistence.swift @@ -6,38 +6,52 @@ import Foundation extension AIChatViewModel { - /// Brings in this connection's conversation history, and this session's own transcript. - /// - /// Called when a session is first looked at rather than from `init`, and scoped to the - /// connection. Loading every conversation and restoring whichever happened to be newest meant a - /// second connection's window opened holding the first connection's transcript, and then saved - /// over it. + // Scoped to the connection, so a second connection's window never opens holding the first one's transcript. func restoreConversationsIfNeeded() { - guard !hasRestoredConversation else { return } + trackTableMentions() + // With no connection the history list would hold every connection's conversations. + guard connection != nil, !hasRestoredConversation else { return } markConversationRestored() let storage = chatStorage let connectionId = connection?.id - let wanted = pendingConversationToRestore - Task.detached(priority: .utility) { [weak self] in - let loaded = await storage.loadAll(connectionId: connectionId) - await MainActor.run { - guard let self else { return } - self.conversations = loaded - guard self.messages.isEmpty else { return } - guard let wanted, let stored = loaded.first(where: { $0.id == wanted }) else { return } - self.activeConversationID = stored.id - self.messages = stored.messages.map { ChatTurn(wire: $0) } + let wanted = conversationToRestore + let epoch = transcriptEpoch + let saved = lastConversationWrite + transcriptLoad = Task(priority: .utility) { [weak self] in + await saved?.value + let summaries = await storage.loadSummaries(connectionId: connectionId) + var stored: AIConversation? + if let wanted { + stored = await storage.load(id: wanted) } + guard let self, self.transcriptEpoch == epoch else { return } + self.conversations = Self.merge(summaries, keepingTurnsOf: self.conversations) + guard let wanted, self.conversationToRestore == wanted else { return } + // A message sent before the restore landed started a conversation of its own, which wins. + guard self.messages.isEmpty, self.activeConversationID == nil else { + self.conversationToRestore = nil + return + } + let readable = stored.flatMap { AIChatStorage.isShown(conversationOf: $0.connectionId, on: connectionId) ? $0 : nil } + self.showTranscript(readable) } } + // An entry still carrying turns holds what disk may have trimmed, so a re-read list keeps it. + private static func merge(_ summaries: [AIConversation], keepingTurnsOf current: [AIConversation]) -> [AIConversation] { + let kept = current.filter { !$0.messages.isEmpty } + guard !kept.isEmpty else { return summaries } + let keptByID = Dictionary(kept.map { ($0.id, $0) }, uniquingKeysWith: { first, _ in first }) + let listed = Set(summaries.map(\.id)) + return kept.filter { !listed.contains($0.id) } + summaries.map { keptByID[$0.id] ?? $0 } + } + func clearConversation() { cancelStream() AIProviderFactory.resetCopilotConversation(sessionId: sessionId) - let ids = conversations.map(\.id) - let storage = chatStorage - Task { for id in ids { await storage.delete(id) } } + deleteStoredConversations(conversations.map(\.id)) conversations.removeAll() + conversationToRestore = nil messages.removeAll() activeConversationID = nil clearError() @@ -47,58 +61,72 @@ extension AIChatViewModel { if activeConversationID == id { AIProviderFactory.resetCopilotConversation(sessionId: sessionId) } - Task { await chatStorage.delete(id) } + deleteStoredConversations([id]) conversations.removeAll { $0.id == id } + if conversationToRestore == id { + conversationToRestore = nil + } if activeConversationID == id { activeConversationID = nil messages.removeAll() } } - /// Writes the transcript under the conversation this session already holds. - /// - /// The id is matched against storage rather than against the in-memory list. A session holding - /// an id the list had not caught up with took the new-conversation branch, orphaning the - /// transcript the user was reading and starting a second one beside it. + // By id, not by the in-memory list, so an id the list has not caught up with keeps its own file. func persistCurrentConversation() { guard let conversation = snapshotCurrentConversation() else { return } - Task { await chatStorage.save(conversation) } + let storage = chatStorage + let previous = lastConversationWrite + lastConversationWrite = Task { + await previous?.value + await storage.save(conversation) + } } - /// The same write, finished before this call returns. Termination is the only caller. + // Behind the saves already booked, or one landing late would bring the file back. + func deleteStoredConversations(_ ids: [UUID]) { + guard !ids.isEmpty else { return } + let storage = chatStorage + let previous = lastConversationWrite + lastConversationWrite = Task { + await previous?.value + for id in ids { + await storage.delete(id) + } + } + } + + // Termination is the only caller. func persistCurrentConversationSynchronously() { guard let conversation = snapshotCurrentConversation() else { return } chatStorage.saveSynchronously(conversation) } + // Returns the whole conversation to write and keeps only its summary in `conversations`. private func snapshotCurrentConversation() -> AIConversation? { guard !messages.isEmpty else { return nil } - let wireMessages = messages.map { $0.wireSnapshot } + var conversation: AIConversation if let existingID = activeConversationID { - var conversation = conversations.first(where: { $0.id == existingID }) + conversation = conversations.first(where: { $0.id == existingID }) ?? AIConversation(id: existingID, connectionId: connection?.id, connectionName: connection?.name) - conversation.messages = wireMessages conversation.updatedAt = Date() - conversation.updateTitle() conversation.connectionName = connection?.name - - if let index = conversations.firstIndex(where: { $0.id == existingID }) { - conversations[index] = conversation - } else { - conversations.insert(conversation, at: 0) - } - return conversation + } else { + conversation = AIConversation(connectionId: connection?.id, connectionName: connection?.name) + activeConversationID = conversation.id + conversationToRestore = nil } - - var conversation = AIConversation( - messages: wireMessages, - connectionId: connection?.id, - connectionName: connection?.name - ) + conversation.messages = messages.map { $0.wireSnapshot } conversation.updateTitle() - activeConversationID = conversation.id - conversations.insert(conversation, at: 0) + + var summary = conversation + summary.messages = [] + if let index = conversations.firstIndex(where: { $0.id == conversation.id }) { + conversations[index] = summary + } else { + conversations.insert(summary, at: 0) + } return conversation } } diff --git a/apple/TablePro/ViewModels/AIChatViewModel+TableMentions.swift b/apple/TablePro/ViewModels/AIChatViewModel+TableMentions.swift index e197e82197..02d54057b0 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel+TableMentions.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel+TableMentions.swift @@ -29,8 +29,7 @@ struct TableMentionIndex: Sendable { TableMentionIndex(tables: tables) } - // The names are already sorted, so the first `limit` matches are the ones a sort of every match - // would keep. + // The names are already sorted, so the first `limit` matches are the ones a full sort would keep. func names(matching query: String, limit: Int) -> [String] { guard limit > 0 else { return [] } guard !query.isEmpty else { return Array(names.prefix(limit)) } @@ -49,8 +48,7 @@ struct TableMentionIndex: Sendable { return matches } - // Only ASCII folds to ASCII one character for one, which is where a byte search finds what - // `localizedCaseInsensitiveContains` finds; `ß` folds to `ss` and would match `weis`. + // Only ASCII folds one character for one; `ß` folds to `ss` and a byte search would match `weis`. private static func asciiFolded(_ text: String) -> [UInt8]? { guard text.utf8.allSatisfy({ $0 < 0x80 }) else { return nil } let folded = Array(text.folding(options: .caseInsensitive, locale: .current).utf8) @@ -82,12 +80,15 @@ extension AIChatViewModel { tableMentions.names.count > Self.mentionMenuTableLimit } - // Started by the panel, so a session whose assistant is never shown never indexes its catalog. + // A schema load on another connection leaves this session's tables as they were. func trackTableMentions() { if tableMentionObservation == nil { tableMentionObservation = services.schemaService.$generations - .map { _ in () } - .merge(with: $connection.map { _ in () }) + .combineLatest($connection.map { $0?.id }) + .map { generations, connectionId in + CatalogGeneration(connectionId: connectionId, generation: connectionId.flatMap { generations[$0] }) + } + .removeDuplicates() .receive(on: RunLoop.main) .sink { [weak self] _ in self?.refreshTableMentions() } } @@ -112,3 +113,8 @@ extension AIChatViewModel { lhs.count == rhs.count && lhs.indices.allSatisfy { lhs[$0].name == rhs[$0].name } } } + +private struct CatalogGeneration: Equatable { + let connectionId: UUID? + let generation: Int? +} diff --git a/apple/TablePro/ViewModels/AIChatViewModel.swift b/apple/TablePro/ViewModels/AIChatViewModel.swift index 04b481e4ab..d5f9bf7777 100644 --- a/apple/TablePro/ViewModels/AIChatViewModel.swift +++ b/apple/TablePro/ViewModels/AIChatViewModel.swift @@ -56,7 +56,7 @@ final class AIChatViewModel: ObservableObject { @Published var queryResults: String? var editorTarget: AssistantEditorTarget? - /// The composer's text. It lives on `composerDraft`, which publishes it apart from the conversation. + // Lives on `composerDraft`, which publishes it apart from the conversation. var inputText: String { get { composerDraft.text } set { composerDraft.text = newValue } @@ -117,10 +117,14 @@ final class AIChatViewModel: ObservableObject { var tableMentionSource: [TableInfo] = [] var tableMentionBuild: Task? var tableMentionObservation: AnyCancellable? + // Writes run in the order they were booked, and a transcript load waits for the ones before it. + var lastConversationWrite: Task? + var transcriptLoad: Task? + // A release bumps it, so a load still reading from disk lands in nothing. + var transcriptEpoch = 0 var bulkColumnReadTableCount: Int? var schemaCacheScope: String? - // A custom command's message carries the schema it rendered, so its tables are ranked by the - // user's own words instead. + // A custom command's message carries its rendered schema, so its tables are ranked by the user's words. var schemaQuestionOverride: (turnID: UUID, text: String)? nonisolated(unsafe) var streamingTask: Task? var prepTask: Task? @@ -133,33 +137,21 @@ final class AIChatViewModel: ObservableObject { static let maxMessageCount = 200 - /// The session this engine belongs to. - /// - /// Injected rather than minted here, because a restored session has to be the same session: - /// identity derived inside the engine cannot round-trip, so every guarantee keyed on it - /// (reopening by id, the rail's selection, per-session provider state) silently degraded to - /// "make another one". + // Injected, so a restored session keeps the identity the rail, reopening and provider state key on. let sessionId: UUID - /// The conversation to pull in when this engine is first looked at, if it is resuming one. - /// - /// Restore is lazy on purpose: reading every stored conversation at launch is quadratic in the - /// number of sessions, and `init` used to call `loadConversations()`, so opening any connection - /// window read the whole chat history off disk even with the assistant never revealed. - private var conversationToRestore: UUID? + // Read when the session is shown: reading every stored conversation at launch is quadratic in sessions. + var conversationToRestore: UUID? private var didRestoreConversation = false - var pendingConversationToRestore: UUID? { conversationToRestore } var hasRestoredConversation: Bool { didRestoreConversation } - /// Whether the connection this session names is still being opened. - /// - /// Agent mode draws its composer over a connect on purpose, so a turn can be submitted before - /// there is a session to run its tools against. Every such turn used to open a stream anyway, - /// which reached the tools with no connection behind them and answered the user's first - /// question with a row of failures. The turn is appended to the transcript as usual and the - /// stream is held until the connect lands, which is what a live composer during a connect - /// promises. + // Restored from disk or released: the session names its conversation and holds none of it yet. + var isAwaitingTranscript: Bool { + activeConversationID == nil && conversationToRestore != nil && messages.isEmpty + } + + // A turn sent during a connect is held, so its tools never run with no connection behind them. var isAwaitingConnection = false { didSet { guard oldValue, !isAwaitingConnection else { return } @@ -167,7 +159,6 @@ final class AIChatViewModel: ObservableObject { } } - /// A turn that was submitted during a connect and has not been streamed yet. var heldTurnAwaitsConnection = false private func releaseHeldTurn() { @@ -248,8 +239,7 @@ final class AIChatViewModel: ObservableObject { return Self.acceptsImages(config: config, model: selectedModel ?? config.model, catalog: .shared) } - /// The provider type sets the envelope, and the provider's own model list narrows it: a - /// router lists text-only models beside vision ones, and an image sent to one of those fails. + // A router lists text-only models beside vision ones, so the provider's own list narrows its type. nonisolated static func acceptsImages(config: AIProviderConfig, model: String, catalog: AIModelCatalog) -> Bool { guard let descriptor = AIProviderRegistry.shared.descriptor(for: config.type.rawValue) else { return false } return descriptor.supportsImages(fetched: catalog.fetchedInfo(providerID: config.id, modelID: model)) @@ -342,69 +332,104 @@ final class AIChatViewModel: ObservableObject { AIProviderFactory.resetCopilotConversation(sessionId: sessionId) cancelStream() persistCurrentConversation() + conversationToRestore = nil messages.removeAll() activeConversationID = nil clearError() } + // The transcript on screen stays until the chosen one lands, and when its file is gone or a turn started. func switchConversation(to id: UUID) { - guard let conversation = conversations.first(where: { $0.id == id }) else { return } + guard conversations.contains(where: { $0.id == id }) else { return } AIProviderFactory.resetCopilotConversation(sessionId: sessionId) cancelStream() persistCurrentConversation() - messages = conversation.messages.map { ChatTurn(wire: $0) } - activeConversationID = conversation.id clearError() + guard id != activeConversationID else { + conversationToRestore = nil + return + } + keepTurnsDiskMayTrim() + if let index = conversations.firstIndex(where: { $0.id == id }), !conversations[index].messages.isEmpty { + let kept = conversations[index] + conversations[index].messages = [] + showTranscript(kept) + return + } + conversationToRestore = id + let storage = chatStorage + let epoch = transcriptEpoch + let saved = lastConversationWrite + transcriptLoad = Task { [weak self] in + await saved?.value + let stored = await storage.load(id: id) + guard let self, self.transcriptEpoch == epoch, self.conversationToRestore == id else { return } + guard !self.isBusy else { + self.conversationToRestore = nil + return + } + self.showTranscript(stored) + } } - /// Releases everything this conversation holds, keeping what the user typed. - /// - /// Window close, disconnect and a lost session all reach here, and none of them is the user - /// throwing a conversation away. It used to empty `messages` with nothing written to disk while - /// `cancelStream()` next door persisted first, so the three ordinary ways a window goes away - /// each dropped a reply that was still arriving. - /// - /// Cancelling the task is also not enough on its own to release a turn parked on an approval - /// card: a `CheckedContinuation` is not resumed by cancellation, so the suspended turn held the - /// provider and its open stream for the life of the process. - func clearSessionData() { - ToolApprovalCenter.shared.cancelAll(sessionId: sessionId) - persistCurrentConversation() - AIProviderFactory.resetCopilotConversation(sessionId: sessionId) + // Disk may hold only the last turns of a long transcript, so the ones it may drop stay on its entry. + private func keepTurnsDiskMayTrim() { + guard messages.count > AIChatStorage.turnsKeptWhenTrimmed, + let activeConversationID, + let index = conversations.firstIndex(where: { $0.id == activeConversationID }) + else { return } + conversations[index].messages = messages.map { $0.wireSnapshot } + } + + // Nil when no transcript was found: the one on screen stays. + func showTranscript(_ conversation: AIConversation?) { + conversationToRestore = nil + guard let conversation else { return } + activeConversationID = conversation.id + messages = conversation.messages.map { ChatTurn(wire: $0) } + } + + // For a session no window shows; the next restore reads back what is written here. + func releaseLoadedState() { + if isBusy || streamingTask != nil { + cancelStream() + } else { + persistCurrentConversation() + } + let keepsTranscript = messages.count > AIChatStorage.turnsKeptWhenTrimmed + heldTurnAwaitsConnection = false + transcriptEpoch &+= 1 + transcriptLoad?.cancel() + transcriptLoad = nil + didRestoreConversation = false + if !keepsTranscript { + if let activeConversationID { + conversationToRestore = activeConversationID + } + activeConversationID = nil + messages = [] + } + conversations = conversations.filter { entry in + !entry.messages.isEmpty || (keepsTranscript && entry.id == activeConversationID) + } prepTask?.cancel() prepTask = nil - streamingTask?.cancel() - streamingTask = nil - AIProviderFactory.invalidateCache() - connection = nil - columnsByTable = [:] - foreignKeysByTable = [:] inFlightColumnFetches.values.forEach { $0.cancel() } inFlightColumnFetches.removeAll() inFlightSchemaLoad?.cancel() inFlightSchemaLoad = nil + columnsByTable = [:] + foreignKeysByTable = [:] relevantTableNames = [] tableMentionBuild?.cancel() tableMentionBuild = nil + tableMentionObservation = nil tableMentionSource = [] tableMentions = .empty bulkColumnReadTableCount = nil schemaCacheScope = nil schemaQuestionOverride = nil - currentQuery = nil queryResults = nil - editorTarget = nil - clearPendingWalkthrough() - messages = [] - errorMessage = nil - activeConversationID = nil - streamingState = .idle - for image in attachedImages { - if case .cacheFile(let filename, _) = image.source { - AIImageCache.shared.delete(filename: filename) - } - } - attachedImages = [] } func clearPendingWalkthrough() { @@ -520,8 +545,7 @@ final class AIChatViewModel: ObservableObject { case .failed where modelListSources[id] == entry.source && availableModels[id]?.isEmpty == false: break case .listed, .failed: - // Empty, so the menu shows the saved model as it stands; recorded, so a dead server is - // asked again only after a change or a save. + // Recorded empty, so a dead server is asked again only after a change or a save. availableModels[id] = [] modelListSources[id] = entry.source } @@ -529,12 +553,7 @@ final class AIChatViewModel: ObservableObject { } } - /// The list is read once per connection and then kept current from the app-wide favorites - /// event, the same signal the sidebar, the Quick Switcher and the editor's keyword expansion - /// already follow. Without it a query saved from the editor during a session was offered by - /// every one of those and by nothing in the assistant, for as long as the window stayed open. - /// - /// A nil payload means a global record moved, which is every connection's business. + // A nil payload means a global record moved, which is every connection's business. internal func observeSavedQueryUpdates() { AppEvents.shared.sqlFavoritesDidUpdate .receive(on: RunLoop.main) @@ -552,8 +571,7 @@ final class AIChatViewModel: ObservableObject { return } let favorites = await services.sqlFavoriteManager.fetchFavorites(connectionId: connectionId) - /// The connection can be switched while the read is in flight, and the list belongs to the - /// connection that is on screen now, not the one that asked. + // The connection can be switched while the read is in flight. guard connection?.id == connectionId else { return } savedQueries = favorites for favorite in favorites { diff --git a/apple/TablePro/Views/Import/ImportDialog.swift b/apple/TablePro/Views/Import/ImportDialog.swift index f9627d2e59..3c89d80f34 100644 --- a/apple/TablePro/Views/Import/ImportDialog.swift +++ b/apple/TablePro/Views/Import/ImportDialog.swift @@ -2,8 +2,6 @@ // ImportDialog.swift // TablePro // -// Plugin-aware import dialog. -// import AppKit import Combine @@ -104,7 +102,10 @@ struct ImportDialog: View { } .task { if let initialURL = initialFileURL, fileURL == nil { - await loadFile(initialURL) + // Tracked like a picked file, so a pick made during a long decompression cancels this load. + let load = Task { await loadFile(initialURL) } + loadFileTask = load + await load.value } } .onDisappear { @@ -369,6 +370,7 @@ struct ImportDialog: View { guard let url, case .format(let formatId) = ImportFileFormatResolver.match(url, among: options) else { return } selectedFormatId = formatId + loadFileTask?.cancel() self.loadFileTask = Task { await self.loadFile(url) } @@ -376,6 +378,11 @@ struct ImportDialog: View { @MainActor private func loadFile(_ url: URL) async { + // The previous file's count must not land under this one, even when this one fails before counting. + countStatementsTask?.cancel() + countStatementsTask = nil + statementCount = 0 + isCountingStatements = false cleanupTempFiles() hasPreviewError = false @@ -401,6 +408,13 @@ struct ImportDialog: View { let urlToRead: URL do { urlToRead = try await FileDecompressor.decompressIfNeeded(url) + // A newer pick owns the dialog now; its import must not read this file's copy. + guard !Task.isCancelled else { + if urlToRead != url { + try? FileManager.default.removeItem(at: urlToRead) + } + return + } if urlToRead != url { tempPreviewURL = urlToRead } @@ -466,14 +480,16 @@ struct ImportDialog: View { statementCount = 0 do { - let encoding = selectedEncoding.encoding - let grammar = sessionGrammar - let parser = SQLFileParser() - let count = try await Task.detached { - try await parser.countStatements(url: url, encoding: encoding, grammar: grammar) - }.value + let count = try await ImportStatementCount.count( + url, + encoding: selectedEncoding.encoding, + grammar: sessionGrammar + ) + // A new file or encoding cancelled this count and started its own, which owns the result. + guard !Task.isCancelled else { return } statementCount = count } catch { + guard !Task.isCancelled else { return } Self.logger.warning("Failed to count statements: \(error.publicLogShape, privacy: .public)") statementCount = 0 } @@ -554,3 +570,22 @@ struct ImportDialog: View { } } } + +enum ImportStatementCount { + // Detached keeps the scan off the main actor but ignores the caller's cancel, so the cancel is passed on. + static func count( + _ url: URL, + encoding: String.Encoding, + grammar: SQLLexicalGrammar, + parser: SQLFileParser = SQLFileParser() + ) async throws -> Int { + let counting = Task.detached { + try await parser.countStatements(url: url, encoding: encoding, grammar: grammar) + } + return try await withTaskCancellationHandler { + try await counting.value + } onCancel: { + counting.cancel() + } + } +} diff --git a/apple/TableProTests/Core/AI/AgentCLIProcessTests.swift b/apple/TableProTests/Core/AI/AgentCLIProcessTests.swift index a2480ec0d5..8ae559471d 100644 --- a/apple/TableProTests/Core/AI/AgentCLIProcessTests.swift +++ b/apple/TableProTests/Core/AI/AgentCLIProcessTests.swift @@ -1,14 +1,38 @@ import Foundation @testable import TablePro +import TableProTestSupport import Testing struct AgentCLIProcessTests { + static let perl = "/usr/bin/perl" + + static var hasPerl: Bool { + FileManager.default.isExecutableFile(atPath: perl) + } + + private final class Lines: @unchecked Sendable { + private let lock = NSLock() + private var received: [String] = [] + + func append(_ line: String) { + lock.withLock { received.append(line) } + } + + var all: [String] { + lock.withLock { received } + } + } + private let shellLaunch = AgentCLILaunch( executable: URL(fileURLWithPath: "/bin/sh"), arguments: ["-c", "exit 0"], environment: [:] ) + private static func perlLaunch(_ script: String) -> AgentCLILaunch { + AgentCLILaunch(executable: URL(fileURLWithPath: perl), arguments: ["-e", script], environment: [:]) + } + @Test("A launch the launcher refuses fails as a launch failure instead of running") func refusedLaunchFailsTheRun() async { await #expect(throws: AgentCLIError.self) { @@ -22,4 +46,123 @@ struct AgentCLIProcessTests { for try await _ in AgentCLIProcess(label: "claude").streamLines(shellLaunch) {} } } + + @Test( + "Every stdout line arrives while the child holds stderr open and never writes to it", + .enabled(if: AgentCLIProcessTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func manyLinesArriveBesideSilentStderr() async throws { + let stream = AgentCLIProcess(label: "test").streamLines( + Self.perlLaunch(#"$| = 1; print "line $_\n" for 1 .. 30000;"#) + ) + var lines: [String] = [] + for try await line in stream { + lines.append(line) + } + #expect(lines == (1 ... 30_000).map { "line \($0)" }) + } + + @Test( + "A line arrives while the child keeps running, and ending the stream stops the child", + .enabled(if: AgentCLIProcessTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func lineArrivesBeforeTheChildExits() async throws { + let received = Lines() + let stream = AgentCLIProcess(label: "test").streamLines( + Self.perlLaunch(#"$| = 1; print "$$\n"; select(undef, undef, undef, 0.3); print "running\n"; sleep 30;"#) + ) + let consumer = Task { + for try await line in stream { + received.append(line) + } + } + defer { consumer.cancel() } + + try await waitUntil { received.all.count == 2 } + #expect(received.all.last == "running") + let child = try #require(received.all.first.flatMap { pid_t($0) }) + + consumer.cancel() + _ = await consumer.result + try await waitUntil { kill(child, 0) != 0 } + } + + @Test( + "A last line without a newline still arrives", + .enabled(if: AgentCLIProcessTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func unterminatedLastLineArrives() async throws { + var lines: [String] = [] + for try await line in AgentCLIProcess(label: "test").streamLines(Self.perlLaunch(#"print "first\nlast";"#)) { + lines.append(line) + } + #expect(lines == ["first", "last"]) + } + + @Test( + "A run returns everything a child writes past the pipe's capacity", + .enabled(if: AgentCLIProcessTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func runReturnsOutputPastPipeCapacity() async throws { + let result = try await AgentCLIProcess(label: "test").run( + Self.perlLaunch(#"$| = 1; print "x" x 200000; print STDERR "done"; exit 3;"#) + ) + #expect(result.code == 3) + #expect(result.output == String(repeating: "x", count: 200_000) + "done") + } + + @Test( + "Cancelling a run stops the child, even when the cancel lands before launch", + .enabled(if: AgentCLIProcessTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func cancelledRunStopsTheChild() async { + let cancelled = AsyncGate() + let run = Task { + await cancelled.wait() + return try await AgentCLIProcess(label: "test").run(Self.perlLaunch("sleep 30")) + } + run.cancel() + await cancelled.open() + + let ended = await BoundedCall.result { () async -> Bool in + _ = try? await run.value + return true + } + #expect(ended == true) + } + + @Test("Lines end at LF alone, lose a trailing CR, skip empty lines and keep a character split across chunks") + func splitterJoinsChunks() { + var splitter = AgentCLIProcess.LineSplitter() + let accented = Array("\u{E9}\n".utf8) + let chunks = [ + Data("a\r\nb".utf8), + Data("c\n\n\r\n".utf8), + Data(accented.prefix(1)), + Data(accented.dropFirst()), + Data("x\u{2028}y\r\n".utf8) + ] + + let lines = chunks.map { splitter.append($0) } + + #expect(lines == [["a"], ["bc"], [], ["\u{E9}"], ["x\u{2028}y"]]) + } + + @Test("A line that is not UTF-8 is dropped, and a tail without a newline comes out at the end") + func splitterDropsInvalidLinesAndFlushesTheTail() { + var splitter = AgentCLIProcess.LineSplitter() + + let lines = splitter.append(Data([0x6F, 0xFF, 0x6B, 0x0A]) + Data("ok\n".utf8) + Data("tail".utf8)) + let tail = splitter.finish() + let afterTail = splitter.finish() + + #expect(lines == ["ok"]) + #expect(tail == ["tail"]) + #expect(afterTail.isEmpty) + } } diff --git a/apple/TableProTests/Core/LSP/LSPTransportReadTests.swift b/apple/TableProTests/Core/LSP/LSPTransportReadTests.swift new file mode 100644 index 0000000000..c749a76575 --- /dev/null +++ b/apple/TableProTests/Core/LSP/LSPTransportReadTests.swift @@ -0,0 +1,58 @@ +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +struct LSPTransportReadTests { + static let perl = "/usr/bin/perl" + + static var hasPerl: Bool { + FileManager.default.isExecutableFile(atPath: perl) + } + + // The pause lets the test's AsyncBytes read park before the second message is written. + private static let pausingServer = #""" + $| = 1; + for my $method ('test/first', 'test/second') { + my $body = '{"jsonrpc":"2.0","method":"' . $method . '"}'; + print "Content-Length: " . length($body) . "\r\n\r\n" . $body; + select(undef, undef, undef, 0.3); + } + sleep 30; + """# + + @Test( + "Server messages arrive while an AsyncBytes read elsewhere in the app waits on a quiet pipe", + .enabled(if: LSPTransportReadTests.hasPerl), + .timeLimit(.minutes(1)) + ) + func messagesArriveBesideAParkedAsyncBytesRead() async throws { + let quiet = Pipe() + defer { withExtendedLifetime(quiet) {} } + let quietOutput = quiet.fileHandleForReading + let parked = Task { + for try await _ in quietOutput.bytes {} + } + + let transport = LSPTransport() + let first = AsyncGate() + let second = AsyncGate() + await transport.onNotification(method: "test/first") { _ in + Task { await first.open() } + } + await transport.onNotification(method: "test/second") { _ in + Task { await second.open() } + } + try await transport.start(executablePath: Self.perl, arguments: ["-e", Self.pausingServer]) + + let arrived: Void? = await HeldOpenWriter(quiet.fileHandleForWriting).finished { + await first.wait() + await second.wait() + } + await transport.stop() + + try #require(arrived != nil) + try quiet.fileHandleForWriting.close() + _ = await parked.result + } +} diff --git a/apple/TableProTests/Core/Process/PipeReaderTests.swift b/apple/TableProTests/Core/Process/PipeReaderTests.swift index ca96e68471..585ac9bc1b 100644 --- a/apple/TableProTests/Core/Process/PipeReaderTests.swift +++ b/apple/TableProTests/Core/Process/PipeReaderTests.swift @@ -33,6 +33,19 @@ struct PipeReaderTests { } } + private final class Ends: @unchecked Sendable { + private let lock = NSLock() + private var bytesSeen: [Int] = [] + + func record(_ count: Int) { + lock.withLock { bytesSeen.append(count) } + } + + var all: [Int] { + lock.withLock { bytesSeen } + } + } + @Test("Everything written arrives in order and reading ends by itself at end of file", .timeLimit(.minutes(1))) func deliversInOrderUntilEndOfFile() async throws { let pipe = Pipe() @@ -49,6 +62,82 @@ struct PipeReaderTests { #expect(received.data == payload) } + @Test("The end callback fires once, after the last chunk, when the writer closes", .timeLimit(.minutes(1))) + func endCallbackFiresOnceAfterTheLastChunk() async throws { + let pipe = Pipe() + defer { withExtendedLifetime(pipe) {} } + let received = Received() + let ends = Ends() + let reader = PipeReader(pipe.fileHandleForReading) + reader.start { received.append($0) } atEnd: { failure in + ends.record(failure == nil ? received.data.count : -1) + } + + let payload = Data((0 ..< 3 * DescriptorRead.pipeCapacity).map { UInt8($0 % 251) }) + BackgroundPipeWriter.write([payload], to: pipe.fileHandleForWriting, pausingBeforeEach: 0) + + try await waitUntil { !ends.all.isEmpty } + reader.stop() + #expect(ends.all == [payload.count]) + #expect(received.data == payload) + } + + @Test( + "Stopping the reader keeps the end callback quiet, even for a callback dispatched before the stop", + .timeLimit(.minutes(1)) + ) + func stopSilencesTheEndCallback() async throws { + let pipe = Pipe() + defer { withExtendedLifetime(pipe) {} } + let handle = pipe.fileHandleForReading + let ends = Ends() + let reader = PipeReader(handle) + reader.start { _ in } atEnd: { _ in ends.record(0) } + let dispatched = try #require(handle.readabilityHandler) + + reader.stop() + try pipe.fileHandleForWriting.close() + let returned: Void? = await BoundedCall.resultOnItsOwnThread { dispatched(handle) } + + try #require(returned != nil) + #expect(ends.all.isEmpty) + } + + @Test("Stopping at end of file reads to the end without firing the end callback", .timeLimit(.minutes(1))) + func stopAtEndOfFileSilencesTheEndCallback() async throws { + let pipe = Pipe() + defer { withExtendedLifetime(pipe) {} } + let handle = pipe.fileHandleForReading + let received = Received() + let ends = Ends() + let reader = PipeReader(handle) + reader.start { received.append($0) } atEnd: { _ in ends.record(0) } + handle.readabilityHandler = nil + + try pipe.fileHandleForWriting.write(contentsOf: Data("last".utf8)) + try pipe.fileHandleForWriting.close() + let stopped: Void? = await BoundedCall.resultOnItsOwnThread { reader.stopAtEndOfFile() } + + #expect(stopped != nil) + #expect(received.data == Data("last".utf8)) + #expect(ends.all.isEmpty) + } + + @Test("The owner may stop the reader from inside its end callback", .timeLimit(.minutes(1))) + func stopFromTheEndCallbackReturns() async throws { + let pipe = Pipe() + defer { withExtendedLifetime(pipe) {} } + let ends = Ends() + let reader = PipeReader(pipe.fileHandleForReading) + reader.start { _ in } atEnd: { _ in + reader.stop() + ends.record(0) + } + + try pipe.fileHandleForWriting.close() + try await waitUntil { ends.all == [0] } + } + @Test("A callback dispatched before a drain reads nothing when it runs after it", .timeLimit(.minutes(1))) func dispatchedCallbackAfterDrainReadsNothing() async throws { let pipe = Pipe() diff --git a/apple/TableProTests/Core/Services/ExecutionAuditRecordTests.swift b/apple/TableProTests/Core/Services/ExecutionAuditRecordTests.swift index 50470ab3dd..f2913a8481 100644 --- a/apple/TableProTests/Core/Services/ExecutionAuditRecordTests.swift +++ b/apple/TableProTests/Core/Services/ExecutionAuditRecordTests.swift @@ -195,4 +195,339 @@ struct ExecutionAuditLogTests { #expect(url.deletingLastPathComponent() == AppStorageEnvironment.shared.supportDirectory) #expect(url.path.hasPrefix(AppStorageEnvironment.shared.applicationSupportRoot.path)) } + + private func temporaryURL() -> URL { + FileManager.default.temporaryDirectory.appendingPathComponent("audit-\(UUID().uuidString).json") + } + + private func storedChain(_ count: Int, startingAt first: Int = 0, after previous: String? = nil) -> [ExecutionAuditRecord] { + var records: [ExecutionAuditRecord] = [] + for index in first ..< first + count { + records.append(ExecutionAuditRecord( + sequence: index, + recordedAt: Date(timeIntervalSince1970: 1_700_000_000 + Double(index)), + connectionId: UUID(), + kind: "readQuery", + caller: "userInterface", + outcome: .authorized, + effectiveWrite: false, + statementDigest: ExecutionAuditRecord.sha256("select \(index)"), + previousHash: records.last?.hash ?? previous ?? ExecutionAuditRecord.genesisHash + )) + } + return records + } + + private func decodedArray(at url: URL) throws -> [ExecutionAuditRecord] { + try JSONDecoder().decode([ExecutionAuditRecord].self, from: Data(contentsOf: url)) + } + + @Test("a log an earlier build wrote as one array goes on from its last record") + func continuesAnEarlierBuildsArray() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + let earlier = storedChain(3) + try JSONEncoder().encode(earlier).write(to: url) + + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let entries = await log.entries() + #expect(entries.count == 4) + #expect(entries[3].sequence == 3) + #expect(entries[3].previousHash == earlier[2].hash) + #expect(await log.verify() == .intact(count: 4)) + } + + @Test("after appends the file is still the array an earlier build decodes and rewrites") + func fileStaysAnArray() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + let log = ExecutionAuditLog(fileURL: url) + for _ in 0 ..< 3 { + await log.record(request: request(write: false), decision: .denied(reason: "x")) + } + + let decoded = try decodedArray(at: url) + #expect(decoded.count == 3) + #expect(ExecutionAuditRecord.firstBrokenSequence(in: decoded) == nil) + } + + @Test("a log longer than the tail that is read on launch still chains from its last record") + func longLogChainsFromItsTail() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + let earlier = storedChain(400) + let data = try JSONEncoder().encode(earlier) + #expect(data.count > 65_536) + try data.write(to: url) + + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: true), decision: .denied(reason: "x")) + + let decoded = try decodedArray(at: url) + #expect(decoded.count == 401) + #expect(decoded[400].previousHash == earlier[399].hash) + #expect(await log.verify() == .intact(count: 401)) + } + + @Test("an empty array, padded with whitespace, takes its first record") + func paddedEmptyArrayStartsTheChain() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + try Data("[ ]\n".utf8).write(to: url) + + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + #expect(try decodedArray(at: url).count == 2) + #expect(await log.verify() == .intact(count: 2)) + } + + @Test("a file changed by someone else is read again before the next record goes on") + func externalChangeIsReadBeforeAppending() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let replaced = storedChain(5) + try JSONEncoder().encode(replaced).write(to: url, options: .atomic) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let entries = await log.entries() + #expect(entries.count == 6) + #expect(entries[5].sequence == 5) + #expect(entries[5].previousHash == replaced[4].hash) + } + + @Test("an unreadable log is set aside with its bytes intact and a new chain starts") + func unreadableLogIsSetAsideNotOverwritten() async throws { + let url = temporaryURL() + let directory = url.deletingLastPathComponent() + let stem = url.deletingPathExtension().lastPathComponent + defer { + try? FileManager.default.removeItem(at: url) + for name in (try? FileManager.default.contentsOfDirectory(atPath: directory.path)) ?? [] + where name.hasPrefix("\(stem).damaged-") { + try? FileManager.default.removeItem(at: directory.appendingPathComponent(name)) + } + } + let damaged = Data(#"[{"sequence":0,"recordedAt":1,"#.utf8) + try damaged.write(to: url) + + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let setAside = try FileManager.default.contentsOfDirectory(atPath: directory.path) + .filter { $0.hasPrefix("\(stem).damaged-") } + #expect(setAside.count == 1) + let kept = try Data(contentsOf: directory.appendingPathComponent(try #require(setAside.first))) + #expect(kept == damaged) + #expect(await log.verify() == .intact(count: 1)) + } + + @Test("a log cut short mid-record keeps its whole records, goes on from them, and keeps the cut file aside") + func tornTailIsRepaired() async throws { + let url = temporaryURL() + let directory = url.deletingLastPathComponent() + let stem = url.deletingPathExtension().lastPathComponent + defer { + try? FileManager.default.removeItem(at: url) + for name in (try? FileManager.default.contentsOfDirectory(atPath: directory.path)) ?? [] + where name.hasPrefix("\(stem).damaged-") { + try? FileManager.default.removeItem(at: directory.appendingPathComponent(name)) + } + } + let earlier = storedChain(3) + var torn = try JSONEncoder().encode(earlier) + torn.removeLast() + torn.append(Data(#",{"sequence":3,"recordedAt":17000"#.utf8)) + try torn.write(to: url) + + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let decoded = try decodedArray(at: url) + #expect(decoded.count == 4) + #expect(decoded[3].sequence == 3) + #expect(decoded[3].previousHash == earlier[2].hash) + let kept = try FileManager.default.contentsOfDirectory(atPath: directory.path) + .filter { $0.hasPrefix("\(stem).damaged-") } + #expect(kept.count == 1) + #expect(try Data(contentsOf: directory.appendingPathComponent(try #require(kept.first))) == torn) + } + + @Test("a file replaced by another of the same size is read again rather than written into") + func sameSizeReplacementIsReadAgain() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + let written = try Data(contentsOf: url) + + let first = try decodedArray(at: url)[0] + var replaced = Data("[".utf8) + (try JSONEncoder().encode(first)) + Data("]".utf8) + replaced.append(Data(repeating: UInt8(ascii: " "), count: written.count - replaced.count)) + #expect(replaced.count == written.count) + try replaced.write(to: url) + + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let decoded = try decodedArray(at: url) + #expect(decoded.count == 2) + #expect(decoded[1].previousHash == first.hash) + #expect(await log.verify() == .intact(count: 2)) + } + + @Test("a record that cannot be written is held and written ahead of the next one") + func unwrittenRecordIsHeldNotLost() async throws { + let url = temporaryURL() + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: url.path) + try? FileManager.default.removeItem(at: url) + } + let log = ExecutionAuditLog(fileURL: url, retryInterval: .zero) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + try FileManager.default.setAttributes([.posixPermissions: 0o444], ofItemAtPath: url.path) + await log.record(request: request(write: true), decision: .denied(reason: "x")) + #expect(try decodedArray(at: url).count == 1) + + try FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: url.path) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let decoded = try decodedArray(at: url) + #expect(decoded.map(\.sequence) == [0, 1, 2]) + #expect(decoded[1].kind == "writeQuery") + #expect(await log.verify() == .intact(count: 3)) + } + + @Test("a record held on its first try after launch goes on from the log's last record once written") + func heldRecordLinksToTheHeadAtWriteTime() async throws { + let url = temporaryURL() + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: url.path) + try? FileManager.default.removeItem(at: url) + } + let earlier = storedChain(3) + try JSONEncoder().encode(earlier).write(to: url) + try FileManager.default.setAttributes([.posixPermissions: 0o444], ofItemAtPath: url.path) + + let log = ExecutionAuditLog(fileURL: url, retryInterval: .zero) + await log.record(request: request(write: true), decision: .denied(reason: "x")) + try FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: url.path) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let decoded = try decodedArray(at: url) + #expect(decoded.map(\.sequence) == [0, 1, 2, 3, 4]) + #expect(decoded[3].previousHash == earlier[2].hash) + #expect(decoded[3].kind == "writeQuery") + #expect(await log.verify() == .intact(count: 5)) + } + + @Test("after a failed write the log waits before trying again, holding what was decided meanwhile") + func failedWriteBacksOff() async throws { + let url = temporaryURL() + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: url.path) + try? FileManager.default.removeItem(at: url) + } + let log = ExecutionAuditLog(fileURL: url, retryInterval: .seconds(3_600)) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + try FileManager.default.setAttributes([.posixPermissions: 0o444], ofItemAtPath: url.path) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + try FileManager.default.setAttributes([.posixPermissions: 0o644], ofItemAtPath: url.path) + + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + #expect(try decodedArray(at: url).count == 1) + } + + @Test("a log replaced by another chain of the same size links to the replacement's last record") + func sameSizeChainReplacementIsReadAgain() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + let log = ExecutionAuditLog(fileURL: url) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + let written = try decodedArray(at: url) + + var replacement: [ExecutionAuditRecord] = [] + for record in written { + replacement.append(ExecutionAuditRecord( + sequence: record.sequence, + recordedAt: record.recordedAt, + connectionId: UUID(), + kind: record.kind, + caller: record.caller, + outcome: record.outcome, + effectiveWrite: record.effectiveWrite, + statementDigest: record.statementDigest, + previousHash: replacement.last?.hash ?? ExecutionAuditRecord.genesisHash + )) + } + let replaced = try JSONEncoder().encode(replacement) + #expect(replaced.count == (try Data(contentsOf: url)).count) + try replaced.write(to: url, options: .atomic) + + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + let decoded = try decodedArray(at: url) + #expect(decoded[2].previousHash == replacement[1].hash) + #expect(await log.verify() == .intact(count: 3)) + } + + @Test("an unreadable log that cannot be moved aside is left untouched until it can be") + func unreadableLogThatCannotMoveIsKept() async throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent("audit-dir-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { + try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: directory.path) + try? FileManager.default.removeItem(at: directory) + } + let url = directory.appendingPathComponent("ExecutionAudit.json") + let damaged = Data("not an array".utf8) + try damaged.write(to: url) + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: directory.path) + + let log = ExecutionAuditLog(fileURL: url, retryInterval: .zero) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + #expect(try Data(contentsOf: url) == damaged) + + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: directory.path) + await log.record(request: request(write: false), decision: .denied(reason: "x")) + + #expect(try decodedArray(at: url).map(\.sequence) == [0, 1]) + let setAside = try FileManager.default.contentsOfDirectory(atPath: directory.path) + .filter { $0.hasPrefix("ExecutionAudit.damaged-") } + #expect(setAside.count == 1) + } + + @Test("a damaged log reads as broken, not as an empty intact chain") + func damagedLogVerifiesAsBroken() async throws { + let url = temporaryURL() + defer { try? FileManager.default.removeItem(at: url) } + try Data("not json".utf8).write(to: url) + + let log = ExecutionAuditLog(fileURL: url) + + #expect(await log.verify() == .broken(atSequence: 0)) + } + + @Test("the tail reader takes the last record whose opening brace follows a separator") + func tailReaderFindsTheLastRecord() throws { + let records = storedChain(2) + let data = try JSONEncoder().encode(records) + + let head = try #require(ExecutionAuditLog.chainHead(inTail: data, startingAt: 0, fileSize: UInt64(data.count))) + + #expect(head.nextSequence == 2) + #expect(head.lastHash == records[1].hash) + #expect(head.closingBracketOffset == UInt64(data.count - 1)) + #expect(ExecutionAuditLog.chainHead(inTail: Data("[{}".utf8), startingAt: 0, fileSize: 3) == nil) + #expect(ExecutionAuditLog.chainHead(inTail: Data("x[]".utf8), startingAt: 0, fileSize: 3) == nil) + } } diff --git a/apple/TableProTests/Core/Services/Infrastructure/AgentSessionRegistryTests.swift b/apple/TableProTests/Core/Services/Infrastructure/AgentSessionRegistryTests.swift index f009821b40..54e8fc8f20 100644 --- a/apple/TableProTests/Core/Services/Infrastructure/AgentSessionRegistryTests.swift +++ b/apple/TableProTests/Core/Services/Infrastructure/AgentSessionRegistryTests.swift @@ -26,9 +26,6 @@ struct AgentSessionRegistryTests { #expect(registry.sessions.isEmpty) } - /// One session, two presentations. The trailing-pane chat and the agent-mode conversation column - /// both resolve through here, so they must land on the same object or the feature is two - /// conversations wearing one name. @Test("Both presentations resolve to the same session") func bothPresentationsShareOneSession() { let registry = AgentSessionRegistry(store: makeStore()) @@ -48,9 +45,7 @@ struct AgentSessionRegistryTests { #expect(session.viewModel.sessionId == session.id) } - /// The conversation flushes a held prompt from a `task`, and a reparent re-runs that task on the - /// same view: a mode toggle and a connection switch are both one. Taking the prompt rather than - /// reading it is what keeps each re-run from sending it again. + // A reparent re-runs the task that flushes the prompt, so taking it is what sends it once. @Test("A held prompt is handed over once, and only once the connection is up") func heldPromptIsTakenOnce() throws { let registry = AgentSessionRegistry(store: makeStore()) @@ -65,10 +60,9 @@ struct AgentSessionRegistryTests { #expect(session.pendingPrompt == nil) } - /// Stopping keeps the transcript. Window close, disconnect and a lost session all reach it, and - /// none of them is the user throwing a conversation away. + // The transcript leaves memory and stays on disk under the id the session still names. @Test("Stopping a session keeps it and its transcript") - func stoppingKeepsTheSession() throws { + func stoppingKeepsTheSession() async throws { let registry = AgentSessionRegistry(store: makeStore()) let connectionId = UUID() let session = try #require(registry.resolveSession(for: connectionId, startingIfNeeded: true)) @@ -77,8 +71,14 @@ struct AgentSessionRegistryTests { registry.stopSessions(for: connectionId) #expect(session.status == .stopped) - #expect(session.viewModel.messages.isEmpty == false) #expect(registry.sessions(for: connectionId).count == 1) + let conversationId = try #require(session.conversationId) + await session.viewModel.lastConversationWrite?.value + let stored = await AIChatStorage.shared.load(id: conversationId) + let storedTexts = stored?.messages.map { $0.plainText } + #expect(storedTexts == ["keep me"]) + #expect(session.viewModel.messages.isEmpty) + await AIChatStorage.shared.delete(conversationId) } @Test("Resolving a stopped session resumes it rather than starting another") @@ -120,9 +120,7 @@ struct AgentSessionRegistryTests { #expect(registry.sessions.count == 2) } - /// Restore has to finish before any window exists. A window that opened while a load was - /// suspended found nothing, minted a session, and was then joined by the stored one: two - /// sessions on one conversation, both persisted, both in the rail. + // A window opened while a load was suspended minted a second session beside the stored one. @Test("Sessions come back from disk in the registry's own initialiser") func restoreIsSynchronous() throws { let store = makeStore() @@ -150,6 +148,32 @@ struct AgentSessionRegistryTests { #expect(restored.viewModel.sessionId == sessionId) } + // Its record used to be written with no conversation, so the next launch reopened it empty. + @Test("A restored session that was never opened keeps its conversation across a save") + func unopenedSessionKeepsItsConversation() throws { + let store = makeStore() + let conversationId = UUID() + store.save([ + AgentSessionRecord( + id: UUID(), + connectionId: UUID(), + conversationId: conversationId, + status: .stopped, + title: "Earlier work", + startedAt: Date(timeIntervalSince1970: 1_000), + lastActiveAt: Date(timeIntervalSince1970: 2_000) + ) + ]) + let registry = AgentSessionRegistry(store: store) + + registry.startSession(for: UUID()) + registry.persistNow() + + let records = store.load() + #expect(records.count == 2) + #expect(records.contains { $0.conversationId == conversationId && $0.title == "Earlier work" }) + } + @Test("A restored session keeps its id rather than being minted again") func restorePreservesIdentity() throws { let store = makeStore() @@ -179,9 +203,7 @@ struct AgentSessionRegistryTests { // MARK: - Order - /// The rail lists the latest first, and going to work is what makes a session the latest. Opening - /// one is not work: ordered by that, the row someone double-clicked in the middle of the list - /// would jump to the top from under the pointer that opened it. + // Opening is not work, or the row double-clicked would jump from under the pointer. @Test("The rail lists the session that last went to work first") func sessionsAreOrderedByActivity() throws { let registry = AgentSessionRegistry(store: makeStore()) @@ -208,7 +230,6 @@ struct AgentSessionRegistryTests { #expect(registry.currentSession(for: connectionId) === first) } - /// With nothing named, the latest live session is the one two panes share, not the oldest. @Test("The shared session is the latest live one") func currentSessionPrefersTheLatestLiveOne() { let registry = AgentSessionRegistry(store: makeStore()) @@ -222,8 +243,6 @@ struct AgentSessionRegistryTests { // MARK: - Closing and deleting the session on screen - /// Close used to stop the session and tell nothing, so the conversation column went on drawing it - /// with a composer that still took messages. @Test("Closing the session on screen hands the window the next live one") func closingHandsOverToTheNextLiveSession() { let registry = AgentSessionRegistry(store: makeStore()) @@ -251,8 +270,6 @@ struct AgentSessionRegistryTests { #expect(registry.sessions(for: connectionId).count == 1) } - /// Nothing on screen is a state the user asked for, and entering the mode again is them asking - /// for a session; resolving is what mints it. @Test("A window with no session open starts one when it is asked to") func resolvingAfterClosingStartsAnother() throws { let registry = AgentSessionRegistry(store: makeStore()) @@ -292,8 +309,6 @@ struct AgentSessionRegistryTests { #expect(registry.currentSession(for: connectionId) === open) } - /// A reply that was still arriving when the app went away did not finish, and saying so is more - /// honest than leaving it reading as busy for ever. @Test("A session still working at terminate is recorded as failed") func terminationMarksWorkingSessionsFailed() throws { let store = makeStore() diff --git a/apple/TableProTests/Core/Storage/AIChatStorageSummaryTests.swift b/apple/TableProTests/Core/Storage/AIChatStorageSummaryTests.swift new file mode 100644 index 0000000000..cb077ddcd6 --- /dev/null +++ b/apple/TableProTests/Core/Storage/AIChatStorageSummaryTests.swift @@ -0,0 +1,64 @@ +// +// AIChatStorageSummaryTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import Testing + +struct AIChatStorageSummaryTests { + private let storage = AIChatStorage.shared + + private static func conversation(_ title: String, connectionId: UUID?, updatedAt seconds: TimeInterval) -> AIConversation { + AIConversation( + title: title, + messages: [ + ChatTurnWire(role: .user, blocks: [.text(title)]), + ChatTurnWire(role: .assistant, blocks: [.text("An answer.")]) + ], + createdAt: Date(timeIntervalSince1970: seconds - 500), + updatedAt: Date(timeIntervalSince1970: seconds), + connectionId: connectionId, + connectionName: "Shop" + ) + } + + // A conversation stored before conversations carried a connection id is listed everywhere. + @Test("Summaries carry the stored titles newest first, without messages, for one connection") + func summariesAreFilteredAndOrdered() async throws { + let connectionId = UUID() + let older = Self.conversation("Older question", connectionId: connectionId, updatedAt: 1_000) + let newer = Self.conversation("Newer question", connectionId: connectionId, updatedAt: 3_000) + let legacy = Self.conversation("Legacy question", connectionId: nil, updatedAt: 2_000) + let elsewhere = Self.conversation("Elsewhere", connectionId: UUID(), updatedAt: 4_000) + let written = [older, newer, legacy, elsewhere] + for conversation in written { + await storage.save(conversation) + } + let ids = Set(written.map { $0.id }) + + let summaries = await storage.loadSummaries(connectionId: connectionId) + let listed = summaries.filter { ids.contains($0.id) } + + #expect(listed.map { $0.title } == ["Newer question", "Legacy question", "Older question"]) + #expect(summaries.allSatisfy { $0.messages.isEmpty }) + #expect(summaries.allSatisfy { $0.connectionId == nil || $0.connectionId == connectionId }) + let first = try #require(listed.first) + #expect(first.id == newer.id) + #expect(first.createdAt == newer.createdAt) + #expect(first.updatedAt == newer.updatedAt) + #expect(first.connectionName == "Shop") + #expect(first.schemaVersion == AIConversation.currentSchemaVersion) + + let everywhere = await storage.loadSummaries(connectionId: nil) + #expect(ids.isSubset(of: Set(everywhere.map { $0.id }))) + + let full = await storage.load(id: newer.id) + #expect(full?.messages.count == 2) + + for conversation in written { + await storage.delete(conversation.id) + } + } +} diff --git a/apple/TableProTests/Core/Storage/TabDiskUnchangedSaveTests.swift b/apple/TableProTests/Core/Storage/TabDiskUnchangedSaveTests.swift new file mode 100644 index 0000000000..d028251bd7 --- /dev/null +++ b/apple/TableProTests/Core/Storage/TabDiskUnchangedSaveTests.swift @@ -0,0 +1,173 @@ +import Foundation +@testable import TablePro +import Testing + +struct TabDiskUnchangedSaveTests { + private let tabStateDirectory = AppStorageEnvironment.shared.applicationSupportRoot + .appendingPathComponent("TablePro", isDirectory: true) + .appendingPathComponent("TabState", isDirectory: true) + + private func stateURL(_ connectionId: UUID) -> URL { + tabStateDirectory.appendingPathComponent("\(connectionId.uuidString).json") + } + + private func sidecarURL(_ connectionId: UUID, tabId: UUID) -> URL { + TabQueryOverflowStore.directory(inside: tabStateDirectory) + .appendingPathComponent("\(connectionId.uuidString)_\(tabId.uuidString).sql") + } + + private func fileNumber(_ url: URL) throws -> Int { + try #require(FileManager.default.attributesOfItem(atPath: url.path)[.systemFileNumber] as? Int) + } + + private func tab(id: UUID = UUID(), title: String, query: String) -> PersistedTab { + PersistedTab( + id: id, + title: title, + query: query, + tabType: .query, + tableName: nil, + columnWidths: ["b": 120, "a": 80, "c": 64] + ) + } + + @discardableResult + private func save(_ tabs: [PersistedTab], connectionId: UUID, selected: UUID? = nil) async throws -> Bool { + try await TabDiskActor.shared.save( + connectionId: connectionId, + tabs: tabs, + selectedTabId: selected ?? tabs.first?.id, + writeToken: TabDiskActor.issueWriteToken(for: connectionId) + ) + } + + @Test("A save that changes nothing leaves the tab file in place, and a change replaces it") + func unchangedSaveKeepsTheFile() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabs = [tab(title: "One", query: "SELECT 1"), tab(title: "Two", query: "SELECT 2")] + + try await save(tabs, connectionId: connectionId) + let written = try fileNumber(stateURL(connectionId)) + try await save(tabs, connectionId: connectionId) + #expect(try fileNumber(stateURL(connectionId)) == written) + + try await save(tabs, connectionId: connectionId, selected: tabs[1].id) + #expect(try fileNumber(stateURL(connectionId)) != written) + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.selectedTabId == tabs[1].id) + } + + @Test("A tab file removed behind the store is written again by an unchanged save") + func removedFileIsWrittenAgain() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabs = [tab(title: "One", query: "SELECT 1")] + + try await save(tabs, connectionId: connectionId) + try FileManager.default.removeItem(at: stateURL(connectionId)) + try await save(tabs, connectionId: connectionId) + + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.tabs.map(\.title) == ["One"]) + } + + @Test("A tab file a load found unreadable is written again by the next unchanged save") + func unreadableFileIsRepairedBySave() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabs = [tab(title: "One", query: "SELECT 1")] + + try await save(tabs, connectionId: connectionId) + try Data("{".utf8).write(to: stateURL(connectionId)) + #expect(await TabDiskActor.shared.load(connectionId: connectionId) == nil) + try await save(tabs, connectionId: connectionId) + + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.tabs.map(\.title) == ["One"]) + } + + @Test("An unchanged oversized query keeps its sidecar, and an edited one is rewritten") + func sidecarIsRewrittenOnlyWhenTheQueryChanges() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabId = UUID() + let large = String(repeating: "x", count: TabQueryContent.maxPersistableQuerySize + 1) + + try await save([tab(id: tabId, title: "Dump", query: large)], connectionId: connectionId) + let sidecar = sidecarURL(connectionId, tabId: tabId) + let written = try fileNumber(sidecar) + try await save([tab(id: tabId, title: "Dump", query: large)], connectionId: connectionId) + #expect(try fileNumber(sidecar) == written) + + let edited = large + "y" + try await save([tab(id: tabId, title: "Dump", query: edited)], connectionId: connectionId) + #expect(try fileNumber(sidecar) != written) + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.tabs.first?.query == edited) + } + + @Test("A sidecar removed behind the store is written again by an unchanged save") + func removedSidecarIsWrittenAgain() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabId = UUID() + let large = String(repeating: "z", count: TabQueryContent.maxPersistableQuerySize + 1) + + try await save([tab(id: tabId, title: "Dump", query: large)], connectionId: connectionId) + try FileManager.default.removeItem(at: sidecarURL(connectionId, tabId: tabId)) + try await save([tab(id: tabId, title: "Dump", query: large)], connectionId: connectionId) + + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.tabs.first?.query == large) + } + + @Test("A closed tab's sidecar is removed even though the other sidecars did not change") + func closedTabSidecarIsPruned() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let kept = UUID() + let closed = UUID() + let large = String(repeating: "q", count: TabQueryContent.maxPersistableQuerySize + 1) + + try await save( + [tab(id: kept, title: "Kept", query: large), tab(id: closed, title: "Closed", query: large)], + connectionId: connectionId + ) + try await save([tab(id: kept, title: "Kept", query: large)], connectionId: connectionId) + + #expect(FileManager.default.fileExists(atPath: sidecarURL(connectionId, tabId: kept).path)) + #expect(!FileManager.default.fileExists(atPath: sidecarURL(connectionId, tabId: closed).path)) + } + + @Test("After a clear, saving the same tabs writes them again") + func saveAfterClearWrites() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabs = [tab(title: "One", query: "SELECT 1")] + + try await save(tabs, connectionId: connectionId) + TabDiskActor.clearSync(connectionId: connectionId) + try await save(tabs, connectionId: connectionId) + + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.tabs.map(\.title) == ["One"]) + } + + @Test("The quit-time save skips an unchanged state too, and still writes a change") + func syncSaveSkipsUnchangedState() async throws { + let connectionId = UUID() + defer { TabDiskActor.clearSync(connectionId: connectionId) } + let tabs = [tab(title: "One", query: "SELECT 1")] + + try await save(tabs, connectionId: connectionId) + let written = try fileNumber(stateURL(connectionId)) + TabDiskActor.saveSync(connectionId: connectionId, tabs: tabs, selectedTabId: tabs.first?.id) + #expect(try fileNumber(stateURL(connectionId)) == written) + + let renamed = [tab(id: tabs[0].id, title: "Renamed", query: "SELECT 1")] + TabDiskActor.saveSync(connectionId: connectionId, tabs: renamed, selectedTabId: renamed.first?.id) + let loaded = await TabDiskActor.shared.load(connectionId: connectionId) + #expect(loaded?.tabs.map(\.title) == ["Renamed"]) + } +} diff --git a/apple/TableProTests/ViewModels/AIChatSessionReleaseTests.swift b/apple/TableProTests/ViewModels/AIChatSessionReleaseTests.swift new file mode 100644 index 0000000000..7b9a4447fb --- /dev/null +++ b/apple/TableProTests/ViewModels/AIChatSessionReleaseTests.swift @@ -0,0 +1,448 @@ +// +// AIChatSessionReleaseTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +@MainActor +struct AIChatSessionReleaseTests { + private let storage = AIChatStorage.shared + + private func makeStore() -> AgentSessionStore { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("AIChatSessionReleaseTests-\(UUID().uuidString)", isDirectory: true) + return AgentSessionStore(directory: directory) + } + + private static func turns(_ texts: String...) -> [ChatTurn] { + texts.enumerated().map { offset, text in + ChatTurn(role: offset.isMultiple(of: 2) ? .user : .assistant, blocks: [.text(text)]) + } + } + + private static func turns(count: Int) -> [ChatTurn] { + (0.. AIConversation { + var conversation = AIConversation( + messages: [ChatTurnWire(role: .user, blocks: [.text(text)])], + connectionId: connectionId + ) + conversation.updateTitle() + return conversation + } + + // Sinks deliver through `RunLoop.main.perform` and answer on the main actor; both have run once this returns. + private static func afterQueuedMainWork() async { + await withCheckedContinuation { continuation in + RunLoop.main.perform { continuation.resume() } + } + await withCheckedContinuation { continuation in + DispatchQueue.main.async { continuation.resume() } + } + } + + // MARK: - Release + + @Test("A stopped session drops what it holds and keeps its title, draft and conversation") + func stoppedSessionHoldsNothing() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let store = makeStore() + let registry = AgentSessionRegistry(store: store, services: fixture.services) + let session = registry.startSession(for: connection.id) + let vm = session.viewModel + vm.connection = connection + vm.trackTableMentions() + await vm.tableMentionBuild?.value + await Self.afterQueuedMainWork() + vm.messages = Self.turns("Which orders shipped late?", "Twelve.") + vm.persistCurrentConversation() + vm.columnsByTable["orders"] = [TestFixtures.makeColumnInfo()] + vm.foreignKeysByTable["orders"] = [] + vm.queryResults = "12 rows" + vm.inputText = "And yesterday?" + let conversationId = try #require(vm.activeConversationID) + try await waitUntil { session.title == "Which orders shipped late?" } + + registry.stopSession(id: session.id) + await Self.afterQueuedMainWork() + + #expect(vm.messages.isEmpty) + #expect(vm.conversations.isEmpty) + #expect(vm.columnsByTable.isEmpty) + #expect(vm.foreignKeysByTable.isEmpty) + #expect(vm.queryResults == nil) + #expect(vm.tableMentions.names.isEmpty) + #expect(vm.tableMentionSource.isEmpty) + #expect(vm.tableMentionObservation == nil) + #expect(!vm.hasRestoredConversation) + #expect(vm.inputText == "And yesterday?") + #expect(session.conversationId == conversationId) + #expect(session.title == "Which orders shipped late?") + + // Holding nothing, the session has nothing to write over the file on disk. + await vm.lastConversationWrite?.value + registry.persistNow() + let stored = await storage.load(id: conversationId) + #expect(stored?.messages.count == 2) + #expect(store.load().first?.conversationId == conversationId) + + await storage.delete(conversationId) + } + + // The restore waits for the writes the stop booked, so it reads the transcript the stop wrote. + @Test("Reopening a stopped session reads its transcript back from disk") + func reopeningRestoresTheTranscript() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let registry = AgentSessionRegistry(store: makeStore(), services: fixture.services) + let session = registry.startSession(for: connection.id) + let vm = session.viewModel + vm.connection = connection + vm.messages = Self.turns("Which orders shipped late?", "Twelve.") + vm.persistCurrentConversation() + let conversationId = try #require(vm.activeConversationID) + registry.stopSession(id: session.id) + + session.resume() + registry.setDisplayedSession(session.id, for: connection.id) + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + + let texts = vm.messages.map { $0.plainText } + #expect(texts == ["Which orders shipped late?", "Twelve."]) + #expect(vm.activeConversationID == conversationId) + #expect(vm.conversations.contains { $0.id == conversationId }) + #expect(vm.tableMentionObservation != nil) + await vm.tableMentionBuild?.value + #expect(vm.tableMentions.names == ["orders"]) + + await storage.delete(conversationId) + } + + @Test("A restore that lands after a release is dropped") + func restoreAfterReleaseIsDropped() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let conversation = Self.stored("Earlier work", connectionId: connection.id) + await storage.save(conversation) + let vm = AIChatViewModel(services: fixture.services, restoringConversation: conversation.id) + vm.connection = connection + + vm.restoreConversationsIfNeeded() + let load = try #require(vm.transcriptLoad) + vm.releaseLoadedState() + await load.value + + #expect(vm.messages.isEmpty) + #expect(vm.conversations.isEmpty) + #expect(vm.activeConversationID == nil) + #expect(vm.conversationToRestore == conversation.id) + + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + + #expect(vm.activeConversationID == conversation.id) + #expect(vm.messages.map { $0.plainText } == ["Earlier work"]) + + await storage.delete(conversation.id) + } + + // An engine change on an unread session derived an empty title and renamed it New Session. + @Test("A session restored at launch keeps its title until its transcript is read") + func unopenedSessionKeepsItsTitle() async throws { + let store = makeStore() + let sessionId = UUID() + store.save([ + AgentSessionRecord( + id: sessionId, + connectionId: UUID(), + conversationId: UUID(), + status: .stopped, + title: "Earlier work", + startedAt: Date(timeIntervalSince1970: 1_000), + lastActiveAt: Date(timeIntervalSince1970: 2_000) + ) + ]) + let registry = AgentSessionRegistry(store: store) + let session = try #require(registry.session(id: sessionId)) + + session.viewModel.queryResults = nil + await Self.afterQueuedMainWork() + + #expect(session.title == "Earlier work") + } + + // MARK: - History list + + @Test("The history list holds no transcripts after a restore or a save") + func conversationsHoldSummaries() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let earlier = Self.stored("Earlier work", connectionId: connection.id) + await storage.save(earlier) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + + #expect(vm.conversations.contains { $0.id == earlier.id && $0.title == "Earlier work" }) + #expect(vm.conversations.allSatisfy { $0.messages.isEmpty }) + + vm.messages = Self.turns("Which orders shipped late?", "Twelve.") + vm.persistCurrentConversation() + let current = try #require(vm.activeConversationID) + + #expect(vm.conversations.first?.id == current) + #expect(vm.conversations.first?.title == "Which orders shipped late?") + #expect(vm.conversations.allSatisfy { $0.messages.isEmpty }) + await vm.lastConversationWrite?.value + let saved = await storage.load(id: current) + #expect(saved?.messages.count == 2) + + await storage.delete(earlier.id) + await storage.delete(current) + } + + // MARK: - Switching + + @Test("Switching reads the chosen transcript and keeps the one it left on disk") + func switchingLoadsTheChosenTranscript() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let earlier = Self.stored("Earlier work", connectionId: connection.id) + await storage.save(earlier) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + vm.messages = Self.turns("Current work") + vm.persistCurrentConversation() + let current = try #require(vm.activeConversationID) + + vm.switchConversation(to: earlier.id) + await vm.transcriptLoad?.value + + #expect(vm.activeConversationID == earlier.id) + #expect(vm.messages.map { $0.plainText } == ["Earlier work"]) + #expect(vm.conversations.allSatisfy { $0.messages.isEmpty }) + await vm.lastConversationWrite?.value + let left = await storage.load(id: current) + #expect(left?.messages.map { $0.plainText } == ["Current work"]) + + await storage.delete(earlier.id) + await storage.delete(current) + } + + // The second file is gone, so only a late first load could change the transcript, whichever lands first. + @Test("A switch that lands after a second switch is dropped") + func lateSwitchIsDropped() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let first = Self.stored("First", connectionId: connection.id) + let second = Self.stored("Second", connectionId: connection.id) + await storage.save(first) + await storage.save(second) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + vm.messages = Self.turns("Current work") + vm.persistCurrentConversation() + let current = try #require(vm.activeConversationID) + await storage.delete(second.id) + + vm.switchConversation(to: first.id) + let firstLoad = try #require(vm.transcriptLoad) + vm.switchConversation(to: second.id) + let secondLoad = try #require(vm.transcriptLoad) + await firstLoad.value + await secondLoad.value + + #expect(vm.activeConversationID == current) + #expect(vm.messages.map { $0.plainText } == ["Current work"]) + #expect(vm.conversationToRestore == nil) + + await vm.lastConversationWrite?.value + await storage.delete(first.id) + await storage.delete(current) + } + + @Test("A switch that lands after New Conversation is dropped") + func switchAfterNewConversationIsDropped() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let earlier = Self.stored("Earlier work", connectionId: connection.id) + await storage.save(earlier) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + + vm.switchConversation(to: earlier.id) + let load = try #require(vm.transcriptLoad) + vm.startNewConversation() + await load.value + + #expect(vm.activeConversationID == nil) + #expect(vm.messages.isEmpty) + + await storage.delete(earlier.id) + } + + // MARK: - Transcripts disk may have trimmed + + @Test( + "A release keeps a transcript longer than disk keeps whole and drops one that fits", + arguments: [AIChatStorage.turnsKeptWhenTrimmed + 10, 10] + ) + func releaseKeepsWhatDiskMayTrim(turnCount: Int) async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + await vm.tableMentionBuild?.value + vm.messages = Self.turns(count: turnCount) + vm.persistCurrentConversation() + vm.columnsByTable["orders"] = [TestFixtures.makeColumnInfo()] + let conversationId = try #require(vm.activeConversationID) + + vm.releaseLoadedState() + + let keeps = turnCount > AIChatStorage.turnsKeptWhenTrimmed + #expect(vm.messages.count == (keeps ? turnCount : 0)) + #expect(vm.activeConversationID == (keeps ? conversationId : nil)) + #expect(vm.conversationToRestore == (keeps ? nil : conversationId)) + #expect(vm.columnsByTable.isEmpty) + #expect(vm.tableMentions.names.isEmpty) + #expect(vm.tableMentionObservation == nil) + + await vm.lastConversationWrite?.value + await storage.delete(conversationId) + } + + // The first file is deleted while the second is shown, so only memory can bring every turn back. + @Test("Switching away from a long transcript and back keeps every turn") + func longTranscriptSurvivesASwitch() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let other = Self.stored("Other work", connectionId: connection.id) + await storage.save(other) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + let turnCount = AIChatStorage.turnsKeptWhenTrimmed + 10 + vm.messages = Self.turns(count: turnCount) + vm.persistCurrentConversation() + let long = try #require(vm.activeConversationID) + + vm.switchConversation(to: other.id) + await vm.transcriptLoad?.value + #expect(vm.activeConversationID == other.id) + #expect(vm.conversations.first { $0.id == long }?.messages.count == turnCount) + #expect(vm.conversations.first { $0.id == other.id }?.messages.isEmpty == true) + await vm.lastConversationWrite?.value + await storage.delete(long) + + vm.switchConversation(to: long) + + #expect(vm.activeConversationID == long) + #expect(vm.messages.count == turnCount) + #expect(vm.conversations.allSatisfy { $0.messages.isEmpty }) + + await vm.lastConversationWrite?.value + await storage.delete(other.id) + await storage.delete(long) + } + + // A read from disk would rebuild every turn, so the same turn objects show nothing was read. + @Test("Picking the conversation on screen keeps its transcript and reads nothing") + func repickingTheActiveConversationReadsNothing() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + vm.messages = Self.turns("Which orders shipped late?", "Twelve.", "And yesterday?") + vm.persistCurrentConversation() + let current = try #require(vm.activeConversationID) + let shown = vm.messages + + vm.switchConversation(to: current) + await vm.transcriptLoad?.value + + #expect(vm.messages.count == shown.count) + #expect(zip(vm.messages, shown).allSatisfy { $0 === $1 }) + #expect(vm.activeConversationID == current) + + await vm.lastConversationWrite?.value + await storage.delete(current) + } + + // MARK: - Restore + + // The message is set straight on the engine, standing in for a send whose reply has not come yet. + @Test("A message sent before the restore lands keeps its own conversation") + func messageBeforeRestoreWins() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let earlier = Self.stored("Earlier work", connectionId: connection.id) + await storage.save(earlier) + let vm = AIChatViewModel(services: fixture.services, restoringConversation: earlier.id) + vm.connection = connection + + vm.restoreConversationsIfNeeded() + let load = try #require(vm.transcriptLoad) + vm.messages = Self.turns("A new question") + await load.value + + #expect(vm.messages.map { $0.plainText } == ["A new question"]) + #expect(vm.activeConversationID == nil) + #expect(vm.conversationToRestore == nil) + #expect(vm.conversations.contains { $0.id == earlier.id }) + + await storage.delete(earlier.id) + } + + // A restore with no connection would list every connection's conversations. + @Test("A restore with no connection waits for one") + func restoreWaitsForTheConnection() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let earlier = Self.stored("Earlier work", connectionId: connection.id) + let elsewhere = Self.stored("Elsewhere", connectionId: UUID()) + await storage.save(earlier) + await storage.save(elsewhere) + let vm = AIChatViewModel(services: fixture.services, restoringConversation: earlier.id) + + vm.restoreConversationsIfNeeded() + + #expect(vm.transcriptLoad == nil) + #expect(!vm.hasRestoredConversation) + + vm.connection = connection + vm.restoreConversationsIfNeeded() + await vm.transcriptLoad?.value + + #expect(vm.hasRestoredConversation) + #expect(vm.messages.map { $0.plainText } == ["Earlier work"]) + #expect(vm.conversations.contains { $0.id == earlier.id }) + #expect(!vm.conversations.contains { $0.id == elsewhere.id }) + + await storage.delete(earlier.id) + await storage.delete(elsewhere.id) + } +} diff --git a/apple/TableProTests/ViewModels/AIChatTableMentionsTests.swift b/apple/TableProTests/ViewModels/AIChatTableMentionsTests.swift index 4799cc73f1..16971297b8 100644 --- a/apple/TableProTests/ViewModels/AIChatTableMentionsTests.swift +++ b/apple/TableProTests/ViewModels/AIChatTableMentionsTests.swift @@ -88,6 +88,44 @@ struct AIChatTableMentionsTests { try await waitUntil { vm.tableMentions.names == ["accounts", "invoices"] } #expect(!vm.hasUnlistedMentionTables) } + + // A cleared source makes any refresh build again, so a build is how a refresh shows. + @Test("Only a schema load on the session's own connection refreshes its index") + func indexFollowsOnlyItsOwnConnection() async throws { + let connection = TestFixtures.makeConnection(type: .postgresql) + let fixture = await AIChatSchemaFixture.loaded(connection: connection, names: ["orders"]) + let vm = AIChatViewModel(services: fixture.services) + vm.connection = connection + vm.trackTableMentions() + await vm.tableMentionBuild?.value + await Self.afterQueuedMainWork() + await vm.tableMentionBuild?.value + #expect(vm.tableMentions.names == ["orders"]) + vm.tableMentionSource = [] + + let other = TestFixtures.makeConnection(type: .postgresql) + let otherDriver = MockDatabaseDriver(connection: other) + otherDriver.tablesToReturn = [TestFixtures.makeTableInfo(name: "invoices")] + await fixture.schemaService.load(connectionId: other.id, driver: otherDriver, connection: other) + await Self.afterQueuedMainWork() + + #expect(vm.tableMentionSource.isEmpty) + #expect(vm.tableMentionBuild == nil) + + await fixture.schemaService.reload(connectionId: connection.id, driver: fixture.driver, connection: connection) + + try await waitUntil { vm.tableMentionSource.map { $0.name } == ["orders"] } + } + + // Sinks deliver through `RunLoop.main.perform` and answer on the main actor; both have run once this returns. + private static func afterQueuedMainWork() async { + await withCheckedContinuation { continuation in + RunLoop.main.perform { continuation.resume() } + } + await withCheckedContinuation { continuation in + DispatchQueue.main.async { continuation.resume() } + } + } } @MainActor diff --git a/apple/TableProTests/Views/Import/ImportStatementCountTests.swift b/apple/TableProTests/Views/Import/ImportStatementCountTests.swift new file mode 100644 index 0000000000..9bb6aabc11 --- /dev/null +++ b/apple/TableProTests/Views/Import/ImportStatementCountTests.swift @@ -0,0 +1,41 @@ +import Foundation +@testable import TablePro +import TableProSQLGrammar +import Testing + +struct ImportStatementCountTests { + private func script(statements: Int) throws -> URL { + let url = FileManager.default.temporaryDirectory.appendingPathComponent("count-\(UUID().uuidString).sql") + try Data(String(repeating: "SELECT 1;\n", count: statements).utf8).write(to: url) + return url + } + + @Test("A count reaches every statement in the file") + func countsEveryStatement() async throws { + let url = try script(statements: 1_000) + defer { try? FileManager.default.removeItem(at: url) } + + let count = try await ImportStatementCount.count(url, encoding: .utf8, grammar: TestGrammar.postgres) + + #expect(count == 1_000) + } + + @Test("Cancelling the dialog's count stops the scan before the end of the file", .timeLimit(.minutes(1))) + func cancelReachesTheScan() async throws { + let total = 2_000_000 + let url = try script(statements: total) + defer { try? FileManager.default.removeItem(at: url) } + + let counting = Task { + try await ImportStatementCount.count(url, encoding: .utf8, grammar: TestGrammar.postgres) + } + counting.cancel() + + switch await counting.result { + case .success(let count): + #expect(count < total) + case .failure(let error): + #expect(error is CancellationError) + } + } +} From a7dbf09a9dfd58c24571c386058af1c09ec031aa Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 12 Nov 2026 04:24:39 +0700 Subject: [PATCH 2/2] docs(changelog): add the lane L1 entries --- apple/changelog.d/4212.changed.md | 1 + apple/changelog.d/4212.fixed.1.md | 1 + apple/changelog.d/4212.fixed.2.md | 1 + apple/changelog.d/4212.fixed.3.md | 1 + apple/changelog.d/4212.fixed.4.md | 1 + apple/changelog.d/4212.fixed.5.md | 1 + apple/changelog.d/4212.fixed.6.md | 1 + apple/changelog.d/4212.fixed.md | 1 + 8 files changed, 8 insertions(+) create mode 100644 apple/changelog.d/4212.changed.md create mode 100644 apple/changelog.d/4212.fixed.1.md create mode 100644 apple/changelog.d/4212.fixed.2.md create mode 100644 apple/changelog.d/4212.fixed.3.md create mode 100644 apple/changelog.d/4212.fixed.4.md create mode 100644 apple/changelog.d/4212.fixed.5.md create mode 100644 apple/changelog.d/4212.fixed.6.md create mode 100644 apple/changelog.d/4212.fixed.md diff --git a/apple/changelog.d/4212.changed.md b/apple/changelog.d/4212.changed.md new file mode 100644 index 0000000000..446e2b3b8f --- /dev/null +++ b/apple/changelog.d/4212.changed.md @@ -0,0 +1 @@ +Queries start sooner: running one no longer rewrites the whole execution log first. diff --git a/apple/changelog.d/4212.fixed.1.md b/apple/changelog.d/4212.fixed.1.md new file mode 100644 index 0000000000..94ce9e289f --- /dev/null +++ b/apple/changelog.d/4212.fixed.1.md @@ -0,0 +1 @@ +Open tabs were written to disk every 30 seconds and on every tab switch even when nothing had changed. diff --git a/apple/changelog.d/4212.fixed.2.md b/apple/changelog.d/4212.fixed.2.md new file mode 100644 index 0000000000..7aad7d83bf --- /dev/null +++ b/apple/changelog.d/4212.fixed.2.md @@ -0,0 +1 @@ +Closed agent sessions kept their transcripts and schema in memory until TablePro quit. diff --git a/apple/changelog.d/4212.fixed.3.md b/apple/changelog.d/4212.fixed.3.md new file mode 100644 index 0000000000..81e16ab59c --- /dev/null +++ b/apple/changelog.d/4212.fixed.3.md @@ -0,0 +1 @@ +An agent session restored at launch could lose its conversation the next time another session was saved. diff --git a/apple/changelog.d/4212.fixed.4.md b/apple/changelog.d/4212.fixed.4.md new file mode 100644 index 0000000000..30c58520b5 --- /dev/null +++ b/apple/changelog.d/4212.fixed.4.md @@ -0,0 +1 @@ +Opening a .gz file, or checking the Copilot and Cloud SQL Proxy helpers, held the whole file in memory. diff --git a/apple/changelog.d/4212.fixed.5.md b/apple/changelog.d/4212.fixed.5.md new file mode 100644 index 0000000000..a324f1b9ff --- /dev/null +++ b/apple/changelog.d/4212.fixed.5.md @@ -0,0 +1 @@ +An execution log TablePro could not read was overwritten by a new one instead of being kept. diff --git a/apple/changelog.d/4212.fixed.6.md b/apple/changelog.d/4212.fixed.6.md new file mode 100644 index 0000000000..9fdbb1a84a --- /dev/null +++ b/apple/changelog.d/4212.fixed.6.md @@ -0,0 +1 @@ +Picking another file in the Import dialog while a .gz file was still opening could import the first file. diff --git a/apple/changelog.d/4212.fixed.md b/apple/changelog.d/4212.fixed.md new file mode 100644 index 0000000000..91e2ecdee3 --- /dev/null +++ b/apple/changelog.d/4212.fixed.md @@ -0,0 +1 @@ +Claude Agent replies appeared only once finished, and a long reply never finished.