diff --git a/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHListenerAcceptor.swift b/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHListenerAcceptor.swift new file mode 100644 index 0000000000..9fff8db54a --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProSSHTransport/SSHListenerAcceptor.swift @@ -0,0 +1,103 @@ +// +// SSHListenerAcceptor.swift +// TableProSSHTransport +// + +import Foundation +import os + +public struct SSHAcceptedClient: Sendable { + public let descriptor: Int32 + // Taken at accept, before any queue hop, since the client's own connect timeout is already running. + public let acceptedAt: Date +} + +// Accepts on a tunnel's listening socket only when the kernel reports a pending connection, so an +// idle listener costs no wakeups and parks no thread. GCD runs the cancel handler only after a +// running event handler has returned, so once `waitForCancellation()` returns nothing can accept on +// the descriptor and its owner may close it. The acceptor never closes it. +public final class SSHListenerAcceptor: @unchecked Sendable { + private let source: any DispatchSourceRead + private let isCancelled: OSAllocatedUnfairLock + private let cancelled: DispatchGroup + + public init( + listenFD: Int32, + queue: DispatchQueue, + onAccept: @escaping @Sendable (SSHAcceptedClient) -> Void + ) { + // A drain then ends on EAGAIN instead of blocking the queue in accept. + let flags = fcntl(listenFD, F_GETFL, 0) + if flags >= 0 { + _ = fcntl(listenFD, F_SETFL, flags | O_NONBLOCK) + } + + let isCancelled = OSAllocatedUnfairLock(initialState: false) + let cancelled = DispatchGroup() + let source = DispatchSource.makeReadSource(fileDescriptor: listenFD, queue: queue) + source.setEventHandler { @Sendable in + SSHListenerAcceptor.acceptPending( + listenFD: listenFD, + isCancelled: { isCancelled.withLock { $0 } }, + onAccept: onAccept + ) + } + cancelled.enter() + source.setCancelHandler { @Sendable in + cancelled.leave() + } + + self.source = source + self.isCancelled = isCancelled + self.cancelled = cancelled + } + + // libdispatch crashes on releasing a source that was never activated. + deinit { + cancel() + } + + public func start() { + source.activate() + } + + public func cancel() { + isCancelled.withLock { $0 = true } + source.cancel() + // A source cancelled before start runs its cancel handler only once active. + source.activate() + } + + public func waitForCancellation() async { + await withCheckedContinuation { (continuation: CheckedContinuation) in + cancelled.notify(queue: .global(qos: .utility)) { + continuation.resume() + } + } + } + + private static func acceptPending( + listenFD: Int32, + isCancelled: () -> Bool, + onAccept: (SSHAcceptedClient) -> Void + ) { + while !isCancelled() { + let descriptor = accept(listenFD, nil, nil) + guard descriptor >= 0 else { + switch errno { + case EINTR, ECONNABORTED: + continue + default: + return + } + } + let acceptedAt = Date() + // Darwin hands the listener's O_NONBLOCK to the accepted socket, and the relays block on it. + let flags = fcntl(descriptor, F_GETFL, 0) + if flags >= 0 { + _ = fcntl(descriptor, F_SETFL, flags & ~O_NONBLOCK) + } + onAccept(SSHAcceptedClient(descriptor: descriptor, acceptedAt: acceptedAt)) + } + } +} diff --git a/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/SSHListenerAcceptorTests.swift b/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/SSHListenerAcceptorTests.swift new file mode 100644 index 0000000000..17a5e1236a --- /dev/null +++ b/apple/Packages/TableProCore/Tests/TableProSSHTransportTests/SSHListenerAcceptorTests.swift @@ -0,0 +1,233 @@ +// +// SSHListenerAcceptorTests.swift +// TableProSSHTransportTests +// + +import Foundation +import os +import TableProSSHTransport +import TableProTestSupport +import Testing + +struct SSHListenerAcceptorTests { + @Test("A client that connects is reported once, with a blocking descriptor") + func reportsAConnectedClientOnce() async throws { + let listener = try LoopbackListener() + defer { listener.close() } + let clients = AcceptedClients() + let acceptor = SSHListenerAcceptor(listenFD: listener.descriptor, queue: Self.acceptQueue()) { clients.record($0) } + acceptor.start() + + let peer = try listener.connect() + defer { Darwin.close(peer) } + + #expect(await Self.finishes { await clients.arrived(1) }) + acceptor.cancel() + #expect(await Self.finishes { await acceptor.waitForCancellation() }) + + let accepted = clients.descriptors + defer { accepted.forEach { _ = Darwin.close($0) } } + #expect(accepted.count == 1) + #expect(accepted.allSatisfy { Self.isBlocking($0) }) + } + + @Test("Two clients that connect back to back are both reported") + func reportsBackToBackClients() async throws { + let listener = try LoopbackListener() + defer { listener.close() } + let clients = AcceptedClients() + let acceptor = SSHListenerAcceptor(listenFD: listener.descriptor, queue: Self.acceptQueue()) { clients.record($0) } + acceptor.start() + + let first = try listener.connect() + defer { Darwin.close(first) } + let second = try listener.connect() + defer { Darwin.close(second) } + + #expect(await Self.finishes { await clients.arrived(2) }) + acceptor.cancel() + #expect(await Self.finishes { await acceptor.waitForCancellation() }) + + let accepted = clients.descriptors + defer { accepted.forEach { _ = Darwin.close($0) } } + #expect(accepted.count == 2) + #expect(Set(accepted).count == 2) + #expect(accepted.allSatisfy { Self.isBlocking($0) }) + } + + @Test("After cancel the completion fires, and a client that connects later is left for someone else") + func cancelStopsAccepting() async throws { + let listener = try LoopbackListener() + defer { listener.close() } + let clients = AcceptedClients() + let acceptor = SSHListenerAcceptor(listenFD: listener.descriptor, queue: Self.acceptQueue()) { clients.record($0) } + acceptor.start() + + acceptor.cancel() + #expect(await Self.finishes { await acceptor.waitForCancellation() }) + + let late = try listener.connect() + defer { Darwin.close(late) } + // Accepting it here proves the cancelled acceptor did not take it and never can. + let taken = try listener.acceptWaiting() + defer { Darwin.close(taken) } + #expect(clients.descriptors.isEmpty) + } + + @Test("Cancelling more than once, from several threads, is safe") + func cancelIsIdempotent() async throws { + let listener = try LoopbackListener() + defer { listener.close() } + let acceptor = SSHListenerAcceptor(listenFD: listener.descriptor, queue: Self.acceptQueue()) { _ = Darwin.close($0.descriptor) } + acceptor.start() + + DispatchQueue.concurrentPerform(iterations: 8) { _ in acceptor.cancel() } + acceptor.cancel() + + #expect(await Self.finishes { await acceptor.waitForCancellation() }) + #expect(await Self.finishes { await acceptor.waitForCancellation() }) + } + + @Test("An acceptor cancelled before it started still completes") + func cancelBeforeStartCompletes() async throws { + let listener = try LoopbackListener() + defer { listener.close() } + let acceptor = SSHListenerAcceptor(listenFD: listener.descriptor, queue: Self.acceptQueue()) { _ = Darwin.close($0.descriptor) } + + acceptor.cancel() + + #expect(await Self.finishes { await acceptor.waitForCancellation() }) + } + + private static func acceptQueue() -> DispatchQueue { + DispatchQueue(label: "com.TablePro.tests.ssh.accept") + } + + private static func finishes(_ work: @escaping @Sendable () async -> Void) async -> Bool { + await BoundedCall.result(of: work) != nil + } + + private static func isBlocking(_ descriptor: Int32) -> Bool { + let flags = fcntl(descriptor, F_GETFL, 0) + return flags >= 0 && flags & O_NONBLOCK == 0 + } +} + +private struct SocketFailure: Error, CustomStringConvertible { + let call: String + let code: Int32 + + var description: String { "\(call) failed: \(String(cString: strerror(code)))" } +} + +private struct LoopbackListener { + let descriptor: Int32 + let port: UInt16 + + init() throws { + let fd = socket(AF_INET, SOCK_STREAM, 0) + guard fd >= 0 else { throw SocketFailure(call: "socket", code: errno) } + + var address = Self.loopback(port: 0) + let bound = withUnsafePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + bind(fd, $0, socklen_t(MemoryLayout.size)) + } + } + guard bound == 0, listen(fd, 8) == 0 else { + let failure = SocketFailure(call: "bind", code: errno) + Darwin.close(fd) + throw failure + } + + var assigned = sockaddr_in() + var length = socklen_t(MemoryLayout.size) + let named = withUnsafeMutablePointer(to: &assigned) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + getsockname(fd, $0, &length) + } + } + guard named == 0 else { + let failure = SocketFailure(call: "getsockname", code: errno) + Darwin.close(fd) + throw failure + } + + descriptor = fd + port = UInt16(bigEndian: assigned.sin_port) + } + + func close() { + Darwin.close(descriptor) + } + + func connect() throws -> Int32 { + let fd = socket(AF_INET, SOCK_STREAM, 0) + guard fd >= 0 else { throw SocketFailure(call: "socket", code: errno) } + + var address = Self.loopback(port: port) + let connected = withUnsafePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { + Darwin.connect(fd, $0, socklen_t(MemoryLayout.size)) + } + } + guard connected == 0 else { + let failure = SocketFailure(call: "connect", code: errno) + Darwin.close(fd) + throw failure + } + return fd + } + + // Returns as soon as a connection is pending; the timeout only bounds a failing run. + func acceptWaiting() throws -> Int32 { + var pollFD = pollfd(fd: descriptor, events: Int16(POLLIN), revents: 0) + guard poll(&pollFD, 1, 10_000) == 1 else { throw SocketFailure(call: "poll", code: errno) } + let fd = accept(descriptor, nil, nil) + guard fd >= 0 else { throw SocketFailure(call: "accept", code: errno) } + return fd + } + + private static func loopback(port: UInt16) -> sockaddr_in { + var address = sockaddr_in() + address.sin_family = sa_family_t(AF_INET) + address.sin_port = port.bigEndian + address.sin_addr.s_addr = inet_addr("127.0.0.1") + return address + } +} + +private final class AcceptedClients: Sendable { + private struct State { + var clients: [SSHAcceptedClient] = [] + var waiters: [(count: Int, continuation: CheckedContinuation)] = [] + } + + private let state = OSAllocatedUnfairLock(initialState: State()) + + var descriptors: [Int32] { + state.withLock { $0.clients.map { $0.descriptor } } + } + + func record(_ client: SSHAcceptedClient) { + let ready = state.withLock { state -> [CheckedContinuation] in + state.clients.append(client) + let total = state.clients.count + let due = state.waiters.filter { $0.count <= total }.map { $0.continuation } + state.waiters.removeAll { $0.count <= total } + return due + } + ready.forEach { $0.resume() } + } + + func arrived(_ count: Int) async { + await withCheckedContinuation { (continuation: CheckedContinuation) in + let isReady = state.withLock { state -> Bool in + guard state.clients.count < count else { return true } + state.waiters.append((count, continuation)) + return false + } + if isReady { continuation.resume() } + } + } +} diff --git a/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift b/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift index 439acba77e..30560e4b83 100644 --- a/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift +++ b/apple/Plugins/SQLiteDriverPlugin/SQLitePlugin.swift @@ -802,6 +802,11 @@ final class SQLitePluginDriver: PluginDatabaseDriver, @unchecked Sendable { ) } + // The schema cookie in the main database header, which SQLite bumps on every schema change. + func schemaChangeToken() async throws -> String? { + try await execute(query: "PRAGMA main.schema_version").rows.first?.first?.asText + } + func columnReorderSchemaFingerprint(table: String, schema: String?) async throws -> String? { try Self.requireMainDatabase(schema) return try await SQLiteColumnReorderPlanner.schemaFingerprint( diff --git a/apple/Plugins/SQLiteDriverTests/SQLiteSchemaChangeTokenTests.swift b/apple/Plugins/SQLiteDriverTests/SQLiteSchemaChangeTokenTests.swift new file mode 100644 index 0000000000..73cb0107f9 --- /dev/null +++ b/apple/Plugins/SQLiteDriverTests/SQLiteSchemaChangeTokenTests.swift @@ -0,0 +1,43 @@ +import Foundation +import TableProPluginKit +import Testing + +struct SQLiteSchemaChangeTokenTests { + private let directory: URL + + init() throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("sqlite-schema-token-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + } + + private func connect() async throws -> SQLitePluginDriver { + let driver = SQLitePluginDriver(config: DriverConnectionConfig( + host: "", port: 0, username: "", password: "", + database: directory.appendingPathComponent("app.db").path + )) + try await driver.connect() + return driver + } + + @Test("Writing rows leaves the token alone, and another connection's schema change moves it") + func tokenFollowsSchemaNotData() async throws { + let watcher = try await connect() + let writer = try await connect() + defer { + watcher.disconnect() + writer.disconnect() + try? FileManager.default.removeItem(at: directory) + } + _ = try await writer.execute(query: "CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)") + + let before = try #require(try await watcher.schemaChangeToken()) + _ = try await writer.execute(query: "INSERT INTO users (name) VALUES ('a')") + _ = try await writer.execute(query: "UPDATE users SET name = 'b'") + #expect(try await watcher.schemaChangeToken() == before) + + _ = try await writer.execute(query: "CREATE INDEX users_name ON users (name)") + let after = try #require(try await watcher.schemaChangeToken()) + #expect(after != before) + } +} diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift index 6ad5f5bbf2..42a037a4b2 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnection.swift @@ -34,8 +34,9 @@ final class SnowflakeConnection: @unchecked Sendable { let params: ResolvedParameters private let connectionIdentifier: String - private let session: URLSession - private let connectSession: URLSession + private let sessions: SnowflakeURLSessions + private var session: URLSession { sessions.query } + private var connectSession: URLSession { sessions.connect } private let lock = NSLock() private let heartbeat = SnowflakeHeartbeat() private var sessionToken: String? @@ -46,6 +47,14 @@ final class SnowflakeConnection: @unchecked Sendable { private var renewalTask: Task? var sessionFingerprint: String { + Self.sessionFingerprint(params: params, account: account, connectionIdentifier: connectionIdentifier) + } + + static func sessionFingerprint( + params: ResolvedParameters, + account: SnowflakeAccount?, + connectionIdentifier: String + ) -> String { SnowflakeSessionKey.fingerprint( connectionId: connectionIdentifier, host: account?.host ?? params.account, @@ -55,6 +64,18 @@ final class SnowflakeConnection: @unchecked Sendable { ) } + static func connectionIdentifier(of config: DriverConnectionConfig) -> String { + config.additionalFields["connectionId"] ?? "" + } + + private var _serverVersion: String? + + // Read once per shared session: every pooled driver reports the same server. + var serverVersion: String? { + get { lock.withLock { _serverVersion } } + set { lock.withLock { _serverVersion = newValue } } + } + /// Statements the connection issues for itself, such as the connect-time `USE` calls, belong to /// no driver and are never the target of a Stop. static let sessionOwner = "session" @@ -76,22 +97,28 @@ final class SnowflakeConnection: @unchecked Sendable { var currentWarehouse: String? { lock.withLock { _currentWarehouse } } var currentRole: String? { lock.withLock { _currentRole } } - init(config: DriverConnectionConfig) { - self.params = Self.resolveParameters(from: config) + convenience init(config: DriverConnectionConfig) { + self.init(config: config, params: Self.resolveParameters(from: config)) + } + + init(config: DriverConnectionConfig, params: ResolvedParameters) { + self.params = params self.account = SnowflakeAccount(params.account) - self.connectionIdentifier = config.additionalFields["connectionId"] ?? "" + self.connectionIdentifier = Self.connectionIdentifier(of: config) let configuration = URLSessionConfiguration.ephemeral configuration.timeoutIntervalForRequest = Self.requestTimeout configuration.timeoutIntervalForResource = 600 - self.session = URLSession( - configuration: configuration, delegate: PluginRedirectRefusingDelegate(), delegateQueue: nil - ) let connectConfiguration = URLSessionConfiguration.ephemeral connectConfiguration.timeoutIntervalForRequest = TimeInterval(PluginConnectTimeout.maximumMilliseconds) / 1_000 connectConfiguration.timeoutIntervalForResource = TimeInterval(PluginConnectTimeout.maximumMilliseconds) / 1_000 - self.connectSession = URLSession( - configuration: connectConfiguration, delegate: PluginRedirectRefusingDelegate(), delegateQueue: nil + self.sessions = SnowflakeURLSessions( + query: URLSession( + configuration: configuration, delegate: PluginRedirectRefusingDelegate(), delegateQueue: nil + ), + connect: URLSession( + configuration: connectConfiguration, delegate: PluginRedirectRefusingDelegate(), delegateQueue: nil + ) ) self._currentDatabase = params.database.isEmpty ? nil : params.database @@ -102,7 +129,7 @@ final class SnowflakeConnection: @unchecked Sendable { // MARK: - Parameter Resolution - private static func resolveParameters(from config: DriverConnectionConfig) -> ResolvedParameters { + static func resolveParameters(from config: DriverConnectionConfig) -> ResolvedParameters { let fields = config.additionalFields func field(_ key: String) -> String { fields[key]?.trimmingCharacters(in: .whitespaces) ?? "" @@ -736,7 +763,7 @@ final class SnowflakeConnection: @unchecked Sendable { continuation.finish() return } - let task = Task { [session] in + let task = Task { [sessions] in do { var buffer: [Int: [[PluginCellValueBox]]] = [:] var nextToYield = 0 @@ -746,7 +773,7 @@ final class SnowflakeConnection: @unchecked Sendable { let index = nextIndex group.addTask { (index, try await Self.downloadChunk( - urls[index], headers: headers, session: session, columns: columns + urls[index], headers: headers, session: sessions.query, columns: columns )) } nextIndex += 1 @@ -762,7 +789,7 @@ final class SnowflakeConnection: @unchecked Sendable { let next = nextIndex group.addTask { (next, try await Self.downloadChunk( - urls[next], headers: headers, session: session, columns: columns + urls[next], headers: headers, session: sessions.query, columns: columns )) } nextIndex += 1 @@ -800,9 +827,9 @@ final class SnowflakeConnection: @unchecked Sendable { var nextIndex = 0 while nextIndex < min(Self.chunkDownloadWorkers, urls.count) { let index = nextIndex - group.addTask { [session] in + group.addTask { [sessions] in (index, try await Self.downloadChunk( - urls[index], headers: headers, session: session, columns: columns + urls[index], headers: headers, session: sessions.query, columns: columns )) } nextIndex += 1 @@ -811,9 +838,9 @@ final class SnowflakeConnection: @unchecked Sendable { rowsByChunk[index] = rows if nextIndex < urls.count { let next = nextIndex - group.addTask { [session] in + group.addTask { [sessions] in (next, try await Self.downloadChunk( - urls[next], headers: headers, session: session, columns: columns + urls[next], headers: headers, session: sessions.query, columns: columns )) } nextIndex += 1 @@ -1073,3 +1100,21 @@ private extension SnowflakeConnection.ResolvedParameters { } } } + +// A URLSession keeps its delegate until it is invalidated, so each connection's two sessions are +// invalidated when the last request path holding them lets go. Invalidating any earlier would make a +// late request raise instead of fail. +final class SnowflakeURLSessions: Sendable { + let query: URLSession + let connect: URLSession + + init(query: URLSession, connect: URLSession) { + self.query = query + self.connect = connect + } + + deinit { + query.finishTasksAndInvalidate() + connect.finishTasksAndInvalidate() + } +} diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnectionRegistry.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnectionRegistry.swift index 74f97edf4a..ff486dc8ba 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnectionRegistry.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakeConnectionRegistry.swift @@ -10,6 +10,7 @@ import Foundation import TableProPluginKit +import TableProSnowflakeCore final class SnowflakeConnectionRegistry: @unchecked Sendable { static let shared = SnowflakeConnectionRegistry() @@ -17,16 +18,22 @@ final class SnowflakeConnectionRegistry: @unchecked Sendable { private let lock = NSLock() private var entries: [String: (connection: SnowflakeConnection, refCount: Int)] = [:] + // Keyed before anything is built: a connection owns two URLSessions, which a discarded copy would leak. func acquire(config: DriverConnectionConfig) -> SnowflakeConnection { - lock.withLock { - let candidate = SnowflakeConnection(config: config) - let key = candidate.sessionFingerprint + let params = SnowflakeConnection.resolveParameters(from: config) + let key = SnowflakeConnection.sessionFingerprint( + params: params, + account: SnowflakeAccount(params.account), + connectionIdentifier: SnowflakeConnection.connectionIdentifier(of: config) + ) + return lock.withLock { if let entry = entries[key] { entries[key] = (entry.connection, entry.refCount + 1) return entry.connection } - entries[key] = (candidate, 1) - return candidate + let connection = SnowflakeConnection(config: config, params: params) + entries[key] = (connection, 1) + return connection } } diff --git a/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift b/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift index a2e8ec8c9d..aebb75e0fa 100644 --- a/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift +++ b/apple/Plugins/SnowflakeDriverPlugin/SnowflakePluginDriver.swift @@ -90,9 +90,15 @@ final class SnowflakePluginDriver: PluginDatabaseDriver, @unchecked Sendable { } lock.withLock { _connection = conn } + if let known = conn.serverVersion { + lock.withLock { _serverVersion = known } + return + } if let result = try? await conn.query("SELECT CURRENT_VERSION()", owner: queryOwner), let first = result.rows.first?.first, case .text(let version) = first { - lock.withLock { _serverVersion = "Snowflake \(version)" } + let serverVersion = "Snowflake \(version)" + conn.serverVersion = serverVersion + lock.withLock { _serverVersion = serverVersion } } else { lock.withLock { _serverVersion = "Snowflake" } } diff --git a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift index 1453ffef39..688981124a 100644 --- a/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift +++ b/apple/Plugins/TableProPluginKit/PluginDatabaseDriver.swift @@ -506,6 +506,10 @@ public protocol PluginDatabaseDriver: AnyObject, Sendable { /// answer, which stands the check down for an engine TablePro never runs a rebuild on anyway. func columnReorderSchemaFingerprint(table: String, schema: String?) async throws -> String? + // A cheap value that changes whenever the catalog does, so a write to a database file that left the schema + // alone needs no catalog reload. Nil where the engine keeps none; the host then reloads on every change. + func schemaChangeToken() async throws -> String? + var unsupportedStructureColumnFields: Set { get } var unsupportedIndexTypes: Set { get } func schemaOperationRefusal(_ operation: PluginSchemaOperation) -> String? @@ -1334,6 +1338,8 @@ public extension PluginDatabaseDriver { func columnReorderSchemaFingerprint(table: String, schema: String?) async throws -> String? { nil } + func schemaChangeToken() async throws -> String? { nil } + var unsupportedStructureColumnFields: Set { [] } var unsupportedIndexTypes: Set { [] } func schemaOperationRefusal(_ operation: PluginSchemaOperation) -> String? { nil } diff --git a/apple/TablePro/Core/MCP/Lifecycle/MCPServerManager.swift b/apple/TablePro/Core/MCP/Lifecycle/MCPServerManager.swift index 4c5574605d..85a95d0096 100644 --- a/apple/TablePro/Core/MCP/Lifecycle/MCPServerManager.swift +++ b/apple/TablePro/Core/MCP/Lifecycle/MCPServerManager.swift @@ -31,7 +31,7 @@ internal enum MCPServerStartError: Error, LocalizedError, Equatable { @MainActor internal final class MCPServerManager: ObservableObject { - internal struct SessionSnapshot: Sendable, Identifiable { + internal struct SessionSnapshot: Sendable, Identifiable, Equatable { internal let id: String internal let clientName: String internal let clientVersion: String? @@ -71,7 +71,6 @@ internal final class MCPServerManager: ObservableObject { @Published private var revocationObserverId: UUID? @Published private var dispatchTask: Task? @Published private var stateTask: Task? - @Published private var clientRefreshTask: Task? @Published private var tokenRenewalTask: Task? internal var isRunning: Bool { @@ -293,7 +292,6 @@ internal final class MCPServerManager: ObservableObject { await built.start() startDispatchLoop(built, generation: generation) startStateLoop(built, generation: generation) - startClientRefresh(built, generation: generation) startBridgeCredentialRenewal(generation: generation) await registerRevocationObserver(built, store: store, generation: generation) @@ -321,8 +319,6 @@ internal final class MCPServerManager: ObservableObject { dispatchTask = nil stateTask?.cancel() stateTask = nil - clientRefreshTask?.cancel() - clientRefreshTask = nil tokenRenewalTask?.cancel() tokenRenewalTask = nil supersededBridgeCredentials = [] @@ -340,7 +336,7 @@ internal final class MCPServerManager: ObservableObject { await releaseBridgeCredential() tokenStore = nil - connectedClients = [] + publishClients([]) state = .stopped MCPAuditLogger.logServerStopped() } @@ -448,13 +444,11 @@ internal final class MCPServerManager: ObservableObject { } } - private func startClientRefresh(_ built: MCPServerComposition, generation: Int) { - clientRefreshTask = Task { @MainActor [weak self] in - while !Task.isCancelled { - guard let self, self.isCurrent(generation) else { return } - await self.refreshClients() - try? await Task.sleep(for: Self.clientRefreshInterval) - } + // Only a view showing the list reads it between changes, so the poll lives as long as that view's task. + internal func refreshClientsWhileShown() async { + while !Task.isCancelled { + await refreshClients() + try? await Task.sleep(for: Self.clientRefreshInterval) } } @@ -513,15 +507,15 @@ internal final class MCPServerManager: ObservableObject { self.generation == generation } - private func refreshClients() async { + internal func refreshClients() async { guard let composition else { - connectedClients = [] + publishClients([]) return } let now = Date() await composition.activityLedger.prune(now: now) let entries = await composition.activityLedger.snapshot(now: now) - connectedClients = entries.map { entry in + publishClients(entries.map { entry in SessionSnapshot( id: entry.id, clientName: entry.clientName, @@ -531,6 +525,11 @@ internal final class MCPServerManager: ObservableObject { tokenName: entry.tokenName, remoteAddress: entry.address ) - } + }) + } + + private func publishClients(_ clients: [SessionSnapshot]) { + guard clients != connectedClients else { return } + connectedClients = clients } } diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter+SchemaChangeChecks.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter+SchemaChangeChecks.swift index 8f28aa10e1..5dcb69f862 100644 --- a/apple/TablePro/Core/Plugins/PluginDriverAdapter+SchemaChangeChecks.swift +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter+SchemaChangeChecks.swift @@ -7,6 +7,10 @@ import Foundation import TableProPluginKit internal extension PluginDriverAdapter { + func schemaChangeToken() async throws -> String? { + try await schemaPluginDriver.schemaChangeToken() + } + func reviewSchemaChange( table: String, schema: String?, diff --git a/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteTunnel.swift b/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteTunnel.swift index 5d7e3fb833..4e406585bb 100644 --- a/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteTunnel.swift +++ b/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteTunnel.swift @@ -9,15 +9,13 @@ import os import TableProSSHTransport -/// A loopback listener whose accepted clients each get a fresh exec channel running the SQLite -/// agent on the SSH server, so the SQLite plugin can reach a live database on that server as if it -/// were a local port. -/// -/// It mirrors `LibSSH2Tunnel`, which forwards a TCP port over one authenticated session: the same -/// accept loop, keepalive, teardown latch and byte counter. What differs is the cargo. A forwarding -/// tunnel opens a direct-tcpip channel per client; this opens a session channel and runs a command -/// on it, and it admits a client only after the client presents the per-connection token, because -/// nothing behind this listener authenticates the way a database port does. +// A loopback listener whose accepted clients each get a fresh exec channel running the SQLite agent +// on the SSH server, so the SQLite plugin can reach a live database there as if it were a local port. +// +// It mirrors `LibSSH2Tunnel`: the same acceptor, keepalive, teardown latch and byte counter. What +// differs is the cargo. A forwarding tunnel opens a direct-tcpip channel per client; this opens a +// session channel and runs a command on it, and it admits a client only after the client presents the +// per-connection token, because nothing behind this listener authenticates the way a database port does. final class RemoteSQLiteTunnel: @unchecked Sendable { let connectionId: UUID let localPort: Int @@ -35,7 +33,7 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { private var session: OpaquePointer { chain.session } private var socketFD: Int32 { chain.socketFD } - private var forwardingTask: Task? + private var acceptor: SSHListenerAcceptor? private var keepAliveTask: Task? private let aliveLatch = TeardownLatch() @@ -56,7 +54,6 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { private let byteCounter = TransportByteCounter() private static let relayBufferSize = 32_768 - private static let acceptPollTimeoutMs: Int32 = 200 private static let admissionTimeoutSeconds = 5 init( @@ -98,26 +95,18 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { func startForwarding() { sessionQueue.sync { libssh2_session_set_blocking(session, 0) } - forwardingTask = Task.detached { [weak self] in - guard let self else { return } - await withCheckedContinuation { (continuation: CheckedContinuation) in - self.acceptQueue.async { [weak self] in - defer { continuation.resume() } - guard let self else { return } - Self.logger.info("Remote SQLite listener started on 127.0.0.1:\(self.localPort)") - while self.isRunning { - guard let clientFD = self.acceptClient() else { - if self.isRunning { continue } - break - } - self.spawnClient( - clientFD: clientFD, - budget: self.relayDeadlines.next() - ) - } - } + // A handler already running when the latch is claimed may still spawn one relay; teardown + // waits for the cancel handler before the relay group, so it waits for that relay too. + let acceptor = SSHListenerAcceptor(listenFD: listenFD, queue: acceptQueue) { [weak self] client in + guard let self, self.isRunning else { + Darwin.close(client.descriptor) + return } + self.spawnClient(clientFD: client.descriptor, budget: self.relayDeadlines.next()) } + self.acceptor = acceptor + acceptor.start() + Self.logger.info("Remote SQLite listener started on 127.0.0.1:\(self.localPort)") } func startKeepAlive() { @@ -149,12 +138,11 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { performTeardown() } - /// Synchronous cleanup for app termination, which waits for nothing because the process is - /// exiting. No descriptor is closed and the session is not freed: the relays and the accept - /// loop may still be polling them, and the OS reclaims every one of them anyway. + // App termination: waits for nothing, closes no descriptor and frees no session, since a relay or + // the accept handler may still be using them and the OS reclaims them all. func closeSync() { guard aliveLatch.claim() else { return } - forwardingTask?.cancel() + acceptor?.cancel() keepAliveTask?.cancel() shutdown(socketFD, SHUT_RDWR) } @@ -165,17 +153,12 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { onDeath?(connectionId) } - /// Breaks every blocking wait first, then frees the session and the descriptors only once - /// every task that could still be polling them has exited. `shutdown` unblocks a poll at - /// once without releasing the descriptor number, which another thread would otherwise be - /// free to receive from the kernel and poll by mistake. `shutdown` does not wake a poll on a - /// listening socket on Darwin, so the accept loop ends on the latch instead, inside one - /// `acceptPollTimeoutMs`, and its descriptor closes once it has. - /// - /// The relays are waited on after the accept loop, not alongside it, because the accept loop - /// is the only thing that starts one: once it has ended, the group can only empty. + // `shutdown` wakes every poll on the transport but keeps the descriptor number, so no other + // socket can take it mid-poll. The listening socket closes only after the acceptor's cancel + // handler ran, which GCD runs once no accept handler is running. The relays are awaited after + // that, because the accept handler is the only thing that starts one: the group can only empty. private func performTeardown() { - forwardingTask?.cancel() + acceptor?.cancel() keepAliveTask?.cancel() shutdown(socketFD, SHUT_RDWR) @@ -184,11 +167,11 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { let relayQueue = self.relayQueue let clientRelays = self.clientRelays let listenFD = self.listenFD - let forwardingTask = self.forwardingTask + let acceptor = self.acceptor let keepAliveTask = self.keepAliveTask let connectionId = self.connectionId Task.detached { - await forwardingTask?.value + await acceptor?.waitForCancellation() await keepAliveTask?.value await withCheckedContinuation { (continuation: CheckedContinuation) in clientRelays.notify(queue: relayQueue) { continuation.resume() } @@ -202,19 +185,6 @@ final class RemoteSQLiteTunnel: @unchecked Sendable { // MARK: - Accept - private func acceptClient() -> Int32? { - var pollFD = pollfd(fd: listenFD, events: Int16(POLLIN), revents: 0) - guard poll(&pollFD, 1, Self.acceptPollTimeoutMs) > 0, pollFD.revents & Int16(POLLIN) != 0 else { - return nil - } - var clientAddr = sockaddr_in() - var addrLen = socklen_t(MemoryLayout.size) - let clientFD = withUnsafeMutablePointer(to: &clientAddr) { - $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { accept(listenFD, $0, &addrLen) } - } - return clientFD >= 0 ? clientFD : nil - } - private func spawnClient( clientFD: Int32, budget: ConnectionRelayDeadlineProvider.Budget diff --git a/apple/TablePro/Core/SSH/LibSSH2SFTPSession.swift b/apple/TablePro/Core/SSH/LibSSH2SFTPSession.swift index e62fb7a0dc..5dfb190016 100644 --- a/apple/TablePro/Core/SSH/LibSSH2SFTPSession.swift +++ b/apple/TablePro/Core/SSH/LibSSH2SFTPSession.swift @@ -143,21 +143,28 @@ internal final class SFTPSerialSessionGate: @unchecked Sendable { } } -/// An authenticated SSH session carrying the SFTP subsystem and nothing else. -/// -/// A sibling of `LibSSH2Tunnel` rather than a mode of it: a tunnel owns a listening socket and an -/// accept loop, both meaningless here. Both types build their session through -/// `LibSSH2TunnelFactory.buildAuthenticatedChain`, so SFTP inherits password, public key, agent, -/// keyboard-interactive, jump hosts, TOTP and `~/.ssh/config` resolution without restating any of -/// it, and tears down through the same `cleanupChain`. -/// -/// libssh2 is not thread-safe per session, so every call runs on one serial queue. +// An authenticated SSH session carrying the SFTP subsystem and nothing else. +// +// A sibling of `LibSSH2Tunnel` rather than a mode of it: a tunnel owns a listening socket and its +// acceptor, both meaningless here. Both types build their session through +// `LibSSH2TunnelFactory.buildAuthenticatedChain`, so SFTP inherits password, public key, agent, +// keyboard-interactive, jump hosts, TOTP and `~/.ssh/config` resolution without restating any of +// it, and tears down through the same `cleanupChain`. +// +// libssh2 is not thread-safe per session, so every call runs on one serial queue. final class LibSSH2SFTPSession: @unchecked Sendable { private static let logger = Logger(subsystem: "com.TablePro", category: "SFTP") - /// Measured against OpenSSH 10.3: 32 KiB requests give ~94 MiB/s up and ~320 MiB/s down on - /// loopback while keeping each `libssh2_sftp_read` inside one SFTP packet. - private static let chunkSize = 32 * 1_024 + // libssh2_sftp_read keeps up to four times its buffer in flight (`max_read_ahead` in sftp.c, + // itself capped at 8 MiB) and returns once some bytes are in, so the buffer sets how much one + // round trip carries, not how long a read waits. 2 MiB reaches libssh2's cap; 32 KiB is a floor + // so a tiny or empty file still reads in a normal buffer. + private static let minimumChunkSize = 32 * 1_024 + private static let maximumChunkSize = 2 * 1_024 * 1_024 + + static func chunkSize(forFileSize fileSize: UInt64) -> Int { + Int(min(max(fileSize, UInt64(minimumChunkSize)), UInt64(maximumChunkSize))) + } private let chain: LibSSH2TunnelFactory.AuthenticatedChain private let sftp: OpaquePointer @@ -479,7 +486,7 @@ final class LibSSH2SFTPSession: @unchecked Sendable { } var hasher = SHA256() - var buffer = [UInt8](repeating: 0, count: Self.chunkSize) + var buffer = [UInt8](repeating: 0, count: Self.chunkSize(forFileSize: expected.size)) var received: UInt64 = 0 while true { diff --git a/apple/TablePro/Core/SSH/LibSSH2Tunnel.swift b/apple/TablePro/Core/SSH/LibSSH2Tunnel.swift index 5b8ca05346..a339bfdf6b 100644 --- a/apple/TablePro/Core/SSH/LibSSH2Tunnel.swift +++ b/apple/TablePro/Core/SSH/LibSSH2Tunnel.swift @@ -10,9 +10,8 @@ import CLibSSH2 import TableProSSHTransport -/// Represents an active SSH tunnel backed by libssh2. -/// Each instance owns a TCP socket, libssh2 session, a local listening socket, -/// and the forwarding/keep-alive tasks. +// An SSH tunnel backed by libssh2. It owns the TCP socket, the session, the local listening socket +// with its acceptor, and the keep-alive task. internal final class LibSSH2Tunnel: @unchecked Sendable { let connectionId: UUID let localPort: Int @@ -28,7 +27,7 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { // Jump host chain (in connection order) private let jumpChain: [JumpHop] - private var forwardingTask: Task? + private var acceptor: SSHListenerAcceptor? private var keepAliveTask: Task? private let aliveLatch = TeardownLatch() @@ -48,7 +47,7 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { /// Individual libssh2 calls within each relay are dispatched to `sessionQueue`. private let relayQueue: DispatchQueue - /// Dedicated queue for the accept loop (poll + accept only, no libssh2 calls). + // Serial: the acceptor's handler, which accepts and makes no libssh2 call. private let acceptQueue: DispatchQueue /// Callback invoked when the tunnel dies (keep-alive failure, etc.) @@ -73,10 +72,6 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { private static let relayBufferSize = 32_768 // 32KB - /// How long the accept loop waits per poll before rechecking `isRunning`. Small enough that - /// noticing a client the kernel already accepted costs a slim part of its connect budget. - internal static let acceptPollTimeoutMs: Int32 = 200 - init( connectionId: UUID, localPort: Int, @@ -123,39 +118,24 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { func startForwarding(destination: SSHForwardDestination) { sessionQueue.sync { libssh2_session_set_blocking(session, 0) } - forwardingTask = Task.detached { [weak self] in - guard let self else { return } - - await withCheckedContinuation { (continuation: CheckedContinuation) in - self.acceptQueue.async { [weak self] in - defer { continuation.resume() } - guard let self else { return } - - let target = destination.logDescription - - Self.logger.info( - "Forwarding started on port \(self.localPort) -> \(target)" - ) - - while self.isRunning { - guard let client = self.acceptClient() else { - if self.isRunning { - continue - } - break - } - - self.spawnClient( - clientFD: client, - budget: self.relayDeadlines.next(), - destination: destination - ) - } - - Self.logger.info("Forwarding loop ended for port \(self.localPort)") - } + // A handler already running when the latch is claimed may still spawn one relay; teardown + // waits for the cancel handler before the relay group, so it waits for that relay too. + let acceptor = SSHListenerAcceptor(listenFD: listenFD, queue: acceptQueue) { [weak self] client in + guard let self, self.isRunning else { + Darwin.close(client.descriptor) + return } + self.spawnClient( + clientFD: client.descriptor, + budget: self.relayDeadlines.next(), + destination: destination + ) } + self.acceptor = acceptor + acceptor.start() + + let target = destination.logDescription + Self.logger.info("Forwarding started on port \(self.localPort) -> \(target)") } // MARK: - Keep-Alive @@ -202,17 +182,12 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { aliveLatch.claim() } - /// Breaks every blocking wait first, then frees the session and the descriptors only once - /// every task that could still be polling them has exited. `shutdown` unblocks a poll at - /// once without releasing the descriptor number, which another thread would otherwise be - /// free to receive from the kernel and poll by mistake. `shutdown` does not wake a poll on a - /// listening socket on Darwin, so the accept loop ends on the latch instead, inside one - /// `acceptPollTimeoutMs`, and its descriptor closes once it has. - /// - /// The relays are waited on after the accept loop, not alongside it, because the accept loop - /// is the only thing that starts one: once it has ended, the group can only empty. + // `shutdown` wakes every poll on the transport but keeps the descriptor number, so no other + // socket can take it mid-poll. The listening socket closes only after the acceptor's cancel + // handler ran, which GCD runs once no accept handler is running. The relays are awaited after + // that, because the accept handler is the only thing that starts one: the group can only empty. private func performTeardown() { - forwardingTask?.cancel() + acceptor?.cancel() keepAliveTask?.cancel() shutdown(socketFD, SHUT_RDWR) @@ -225,10 +200,10 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { let listenFD = self.listenFD let jumpChain = self.jumpChain let connectionId = self.connectionId - let forwardingTask = self.forwardingTask + let acceptor = self.acceptor let keepAliveTask = self.keepAliveTask Task.detached { - await forwardingTask?.value + await acceptor?.waitForCancellation() await keepAliveTask?.value await withCheckedContinuation { (continuation: CheckedContinuation) in clientRelays.notify(queue: relayQueue) { continuation.resume() } @@ -262,13 +237,12 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { } } - /// Synchronous cleanup for app termination, which waits for nothing because the process is - /// exiting. No descriptor is closed and the session is not freed: the relays and the accept - /// loop may still be polling them, and the OS reclaims every one of them anyway. + // App termination: waits for nothing, closes no descriptor and frees no session, since a relay or + // the accept handler may still be using them and the OS reclaims them all. func closeSync() { guard consumeAliveLatch() else { return } - forwardingTask?.cancel() + acceptor?.cancel() keepAliveTask?.cancel() shutdown(socketFD, SHUT_RDWR) @@ -287,30 +261,8 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { onDeath?(connectionId) } - private func acceptClient() -> Int32? { - var pollFD = pollfd(fd: listenFD, events: Int16(POLLIN), revents: 0) - let pollResult = poll(&pollFD, 1, Self.acceptPollTimeoutMs) - - guard pollResult > 0, pollFD.revents & Int16(POLLIN) != 0 else { - return nil - } - - var clientAddr = sockaddr_in() - var addrLen = socklen_t(MemoryLayout.size) - - let clientFD = withUnsafeMutablePointer(to: &clientAddr) { - $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { - accept(listenFD, $0, &addrLen) - } - } - - guard clientFD >= 0 else { return nil } - return clientFD - } - - /// Open the channel and relay the client, off the accept loop so a slow open cannot - /// stall the next accept, and off `sessionQueue` between attempts so it cannot stall - /// the relays and keep-alive that share the session. + // Off `sessionQueue` between open attempts, so a slow open cannot stall the relays and the + // keep-alive that share the session. private func openAndRelay( clientFD: Int32, budget: ConnectionRelayDeadlineProvider.Budget, @@ -411,9 +363,7 @@ internal final class LibSSH2Tunnel: @unchecked Sendable { } } - /// Opens the channel and relays one accepted client, off the accept loop so a slow - /// open cannot delay the next accept. The loop runs on `relayQueue` (concurrent); - /// individual libssh2 calls are dispatched to `sessionQueue` (serial) for thread safety. + // On `relayQueue`, off the accept handler, so a slow open cannot delay the next accept. private func spawnClient( clientFD: Int32, budget: ConnectionRelayDeadlineProvider.Budget, diff --git a/apple/TablePro/Core/SSH/LibSSH2TunnelFactory.swift b/apple/TablePro/Core/SSH/LibSSH2TunnelFactory.swift index 08f6239f36..a1802d8c42 100644 --- a/apple/TablePro/Core/SSH/LibSSH2TunnelFactory.swift +++ b/apple/TablePro/Core/SSH/LibSSH2TunnelFactory.swift @@ -29,9 +29,8 @@ internal struct LocalTunnelListener: Sendable { internal enum LibSSH2TunnelFactory { internal static let logger = Logger(subsystem: "com.TablePro", category: "LibSSH2TunnelFactory") - /// A single session opens more than one connection through the tunnel: the query - /// connection plus the metadata pool. A backlog that cannot hold them resets the - /// overflow before the accept loop reaches it. + // One session opens more than one connection through the tunnel: the query connection plus the + // metadata pool. A backlog that cannot hold them resets the overflow before the acceptor reaches it. private static let listenBacklogSize: Int32 = 16 // MARK: - Global Init diff --git a/apple/TablePro/Core/Services/Query/SchemaRefreshService.swift b/apple/TablePro/Core/Services/Query/SchemaRefreshService.swift index 384a97a6b7..af9f368966 100644 --- a/apple/TablePro/Core/Services/Query/SchemaRefreshService.swift +++ b/apple/TablePro/Core/Services/Query/SchemaRefreshService.swift @@ -22,7 +22,8 @@ final class SchemaRefreshService { private struct InFlightRefresh { let id: UUID - let task: Task + // Whether the refresh committed a fresh tables list. + let task: Task } nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "SchemaRefreshService") @@ -62,21 +63,23 @@ final class SchemaRefreshService { SchemaForeignKeyStore.shared.invalidate(connectionId: connection.id) let key = RefreshKey(connectionId: connection.id, database: database) if let existing = inFlight[key] { - await existing.task.value + _ = await existing.task.value return } let entry = InFlightRefresh(id: UUID(), task: Task { @MainActor [weak self] in - guard let self else { return } - await self.performRefresh(connection: connection) + guard let self else { return false } + return await self.performRefresh(connection: connection) }) inFlight[key] = entry - await entry.task.value + _ = await entry.task.value finish(entry, for: key) } - /// Never joins a load that began before the write, which would hand back the old catalog. The - /// entry is replaced before anything awaits, so every later plain refresh joins this one instead. - func refreshAfterWrite(connection: DatabaseConnection) async { + // Never joins a load that began before the write, which would hand back the old catalog. The entry + // is replaced before anything awaits, so every later plain refresh joins this one instead. Returns + // whether a fresh tables list was committed: false when the load failed or another refresh replaced it. + @discardableResult + func refreshAfterWrite(connection: DatabaseConnection) async -> Bool { let connectionId = connection.id SchemaForeignKeyStore.shared.invalidate(connectionId: connectionId) for key in Array(inFlight.keys) where key.connectionId == connectionId { @@ -84,13 +87,14 @@ final class SchemaRefreshService { } let key = RefreshKey(connectionId: connectionId, database: nil) let entry = InFlightRefresh(id: UUID(), task: Task { @MainActor [weak self] in - guard let self else { return } + guard let self else { return false } await self.schemaService.prepareForReload(connectionId: connectionId) - await self.performRefresh(connection: connection) + return await self.performRefresh(connection: connection) }) inFlight[key] = entry - await entry.task.value + let committed = await entry.task.value finish(entry, for: key) + return committed } private func finish(_ entry: InFlightRefresh, for key: RefreshKey) { @@ -103,7 +107,7 @@ final class SchemaRefreshService { key.connectionId == connectionId ? entry.task : nil } for task in tasks { - await task.value + _ = await task.value } } @@ -277,8 +281,9 @@ final class SchemaRefreshService { await syncAutocompleteProvider(connectionId: connectionId) } - private func performRefresh(connection: DatabaseConnection) async { + private func performRefresh(connection: DatabaseConnection) async -> Bool { let connectionId = connection.id + var committed = false if pluginManager.databaseGroupingStrategy(for: connection.type) == .hierarchicalSchema { await schemaService.prepareForReload(connectionId: connectionId) @@ -290,11 +295,11 @@ final class SchemaRefreshService { throw DatabaseError.notConnected } let awaitedSchemas = schemasAwaitingJudgement(in: scope) - try await metadataDriverProvider.withMetadataDriver( + committed = try await metadataDriverProvider.withMetadataDriver( scope: scope, workload: .bulk ) { [schemaService] driver in - await schemaService.reload( + let didLoad = await schemaService.reload( connectionId: connectionId, driver: driver, connection: connection, @@ -305,9 +310,10 @@ final class SchemaRefreshService { fetchingNow: awaitedSchemas, driver: driver ) + return didLoad } } catch is CancellationError { - return + return false } catch { Self.logger.warning( "[schema] refresh failed connId=\(connectionId, privacy: .public) error=\(error.publicLogShape, privacy: .public)" @@ -320,6 +326,7 @@ final class SchemaRefreshService { } await syncAutocompleteProvider(connectionId: connectionId) + return committed } /// The schemas judged against the refreshed catalog as soon as it settles: the browsed one, and diff --git a/apple/TablePro/Core/Services/Query/SchemaService.swift b/apple/TablePro/Core/Services/Query/SchemaService.swift index 5cf3d3befd..4914fd012e 100644 --- a/apple/TablePro/Core/Services/Query/SchemaService.swift +++ b/apple/TablePro/Core/Services/Query/SchemaService.swift @@ -739,16 +739,19 @@ final class SchemaService: ObservableObject { && isCatalogCurrent(for: connectionId): return case .idle, .loading, .failed, .loaded: - await runLoad(connectionId: connectionId, driver: driver, connection: connection, scope: scope) + _ = await runLoad(connectionId: connectionId, driver: driver, connection: connection, scope: scope) } } + // Returns whether a fresh tables list was committed, so a caller can tell a reload from a failed one. + // A hierarchical engine loads its tables per schema later, so its reload always returns true. + @discardableResult func reload( connectionId: UUID, driver: DatabaseDriver, connection: DatabaseConnection, scope: DatabaseScope? = nil - ) async { + ) async -> Bool { await runLoad(connectionId: connectionId, driver: driver, connection: connection, scope: scope) } @@ -858,7 +861,7 @@ final class SchemaService: ObservableObject { await prepareForReload(connectionId: connectionId) let connection = session.connection do { - try await DatabaseManager.shared.withMetadataDriver(scope: scope, workload: .bulk) { [self] driver in + _ = try await DatabaseManager.shared.withMetadataDriver(scope: scope, workload: .bulk) { [self] driver in await reload( connectionId: connectionId, driver: driver, @@ -914,7 +917,7 @@ final class SchemaService: ObservableObject { driver: DatabaseDriver, connection: DatabaseConnection, scope: DatabaseScope? - ) async { + ) async -> Bool { let type = connection.type let key = scope.map { catalogKey(for: $0, type: type) } ?? shownKey(connectionId) let generation = beginLoadGeneration(for: key) @@ -948,7 +951,7 @@ final class SchemaService: ObservableObject { generation: generation, type: type ) - return + return true } async let tablesTask: [TableInfo] = loadDedup.execute(key: loadKey) { @@ -990,24 +993,26 @@ final class SchemaService: ObservableObject { /// Tables are committed the moment they arrive and the other kinds settle behind them, so a /// failed tables fetch still settles routines, triggers and types instead of leaving each /// section on a spinner no load is coming to replace. + var committed = false do { let tables = try await tablesTask guard isCurrentLoadGeneration(generation, for: key, phase: "tables-loaded") else { - return + return false } adoptShown(key, scope: scope, generation: generation, type: type) states[key] = .loaded(tables) unrefreshedCatalogs.remove(key) bumpGeneration(connectionId) + committed = true } catch is CancellationError { abandonLoad(key, generation: generation) - return + return false } catch { guard isCurrentLoadGeneration(generation, for: key, phase: "tables-failed") else { if loadGenerations[key] == nil, case .loading = states[key] { states[key] = .idle } - return + return false } Self.logger.warning( "[schema] load failed connId=\(connectionId, privacy: .public) error=\(error.publicLogShape, privacy: .public)" @@ -1022,13 +1027,13 @@ final class SchemaService: ObservableObject { let triggersOutcome = await triggersTask let typesOutcome = await typesTask guard isCurrentLoadGeneration(generation, for: key, phase: "side-objects-loaded") else { - return + return committed } commitSideObjects(key, routines: routinesOutcome, triggers: triggersOutcome, types: typesOutcome) if let loadedSchemas = await schemasTask { guard isCurrentLoadGeneration(generation, for: key, phase: "schemas-loaded") else { - return + return committed } if listsAttached { attachedSchemasInOrder[key] = loadedSchemas @@ -1037,6 +1042,7 @@ final class SchemaService: ObservableObject { } } bumpGeneration(connectionId) + return committed } private func runHierarchicalLoad( diff --git a/apple/TablePro/Core/Services/SQL/FileSystemEventWatcher.swift b/apple/TablePro/Core/Services/SQL/FileSystemEventWatcher.swift new file mode 100644 index 0000000000..022d0d7e48 --- /dev/null +++ b/apple/TablePro/Core/Services/SQL/FileSystemEventWatcher.swift @@ -0,0 +1,152 @@ +// +// FileSystemEventWatcher.swift +// TablePro +// + +import CoreServices +import Foundation +import os + +internal struct FileSystemEvent: Equatable, Sendable { + let path: String + let flags: FSEventStreamEventFlags + + func has(_ flag: Int) -> Bool { + flags & FSEventStreamEventFlags(flag) != 0 + } +} + +@MainActor +internal final class FileSystemEventWatcher { + nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "FileSystemEventWatcher") + + var onChange: (() -> Void)? + + private let latency: CFTimeInterval + private let reportsFileEvents: Bool + private let accepts: @Sendable ([FileSystemEvent]) -> Bool + private var watchedPaths: Set = [] + private var stream: EventStream? + + // `accepts` runs on the event queue, so a batch it refuses never wakes the main actor. + init( + latency: CFTimeInterval = 0.5, + reportsFileEvents: Bool = false, + accepts: @escaping @Sendable ([FileSystemEvent]) -> Bool = { _ in true } + ) { + self.latency = latency + self.reportsFileEvents = reportsFileEvents + self.accepts = accepts + } + + func watch(_ paths: Set) { + guard paths != watchedPaths else { return } + cancel() + watchedPaths = paths + guard !paths.isEmpty else { return } + + var flags = kFSEventStreamCreateFlagNoDefer | kFSEventStreamCreateFlagWatchRoot | kFSEventStreamCreateFlagUseCFTypes + if reportsFileEvents { + flags |= kFSEventStreamCreateFlagFileEvents + } + let sink = EventSink(watcher: self, accepts: accepts) + guard let created = EventStream( + paths: paths, + latency: latency, + flags: FSEventStreamCreateFlags(flags), + sink: sink + ) else { + Self.logger.error("Failed to start a file system event stream") + return + } + stream = created + } + + func cancel() { + watchedPaths = [] + // A batch already on its way to the main actor finds no watcher and is dropped. + stream?.sink.watcher = nil + stream = nil + } +} + +// Retained by the stream through its context, so a callback never reads freed memory, and it holds +// the watcher weakly, so a released watcher is never called. +@MainActor +private final class EventSink { + weak var watcher: FileSystemEventWatcher? + nonisolated let accepts: @Sendable ([FileSystemEvent]) -> Bool + + init(watcher: FileSystemEventWatcher, accepts: @escaping @Sendable ([FileSystemEvent]) -> Bool) { + self.watcher = watcher + self.accepts = accepts + } + + func deliver() { + watcher?.onChange?() + } +} + +// Stops and releases the stream when it goes away, whether the watcher cancelled it or was released. +private final class EventStream { + let sink: EventSink + private let stream: FSEventStreamRef + + init?(paths: Set, latency: CFTimeInterval, flags: FSEventStreamCreateFlags, sink: EventSink) { + var context = FSEventStreamContext( + version: 0, + info: Unmanaged.passUnretained(sink).toOpaque(), + retain: EventStream.retainSink, + release: EventStream.releaseSink, + copyDescription: nil + ) + guard let stream = FSEventStreamCreate( + kCFAllocatorDefault, + EventStream.callback, + &context, + Array(paths) as CFArray, + FSEventStreamEventId(kFSEventStreamEventIdSinceNow), + latency, + flags + ) else { + return nil + } + FSEventStreamSetDispatchQueue(stream, .global(qos: .utility)) + guard FSEventStreamStart(stream) else { + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + return nil + } + self.sink = sink + self.stream = stream + } + + deinit { + FSEventStreamStop(stream) + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + } + + nonisolated private static let retainSink: CFAllocatorRetainCallBack = { info in + guard let info else { return nil } + _ = Unmanaged.fromOpaque(info).retain() + return info + } + + nonisolated private static let releaseSink: CFAllocatorReleaseCallBack = { info in + guard let info else { return } + Unmanaged.fromOpaque(info).release() + } + + nonisolated private static let callback: FSEventStreamCallback = { _, info, count, eventPaths, eventFlags, _ in + guard let info else { return } + let sink = Unmanaged.fromOpaque(info).takeUnretainedValue() + let paths = (Unmanaged.fromOpaque(eventPaths).takeUnretainedValue() as NSArray) as? [String] ?? [] + let flags = UnsafeBufferPointer(start: eventFlags, count: count) + let events = zip(paths, flags).map { FileSystemEvent(path: $0, flags: $1) } + guard sink.accepts(events) else { return } + Task { @MainActor in + sink.deliver() + } + } +} diff --git a/apple/TablePro/Core/Services/SQL/GitRepositoryWatcher.swift b/apple/TablePro/Core/Services/SQL/GitRepositoryWatcher.swift deleted file mode 100644 index 3c2cb7be52..0000000000 --- a/apple/TablePro/Core/Services/SQL/GitRepositoryWatcher.swift +++ /dev/null @@ -1,65 +0,0 @@ -// -// GitRepositoryWatcher.swift -// TablePro -// - -import CoreServices -import Foundation -import os - -@MainActor -internal final class GitRepositoryWatcher { - nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "GitRepositoryWatcher") - - var onChange: (() -> Void)? - - private var eventStream: FSEventStreamRef? - private var watchedPaths: Set = [] - - nonisolated private static let eventCallback: FSEventStreamCallback = { _, info, _, _, _, _ in - guard let info else { return } - let watcher = Unmanaged.fromOpaque(info).takeUnretainedValue() - Task { @MainActor in - watcher.onChange?() - } - } - - func watch(_ paths: Set) { - guard paths != watchedPaths else { return } - cancel() - watchedPaths = paths - guard !paths.isEmpty else { return } - - var context = FSEventStreamContext( - version: 0, - info: Unmanaged.passUnretained(self).toOpaque(), - retain: nil, - release: nil, - copyDescription: nil - ) - guard let stream = FSEventStreamCreate( - kCFAllocatorDefault, - Self.eventCallback, - &context, - Array(paths) as CFArray, - FSEventStreamEventId(kFSEventStreamEventIdSinceNow), - 0.5, - UInt32(kFSEventStreamCreateFlagNoDefer | kFSEventStreamCreateFlagWatchRoot) - ) else { - Self.logger.error("Failed to create the Git repository event stream") - return - } - FSEventStreamSetDispatchQueue(stream, .global(qos: .utility)) - FSEventStreamStart(stream) - eventStream = stream - } - - func cancel() { - watchedPaths = [] - guard let stream = eventStream else { return } - FSEventStreamStop(stream) - FSEventStreamInvalidate(stream) - FSEventStreamRelease(stream) - eventStream = nil - } -} diff --git a/apple/TablePro/Core/Services/SQL/LinkedFolderGitStatusStore.swift b/apple/TablePro/Core/Services/SQL/LinkedFolderGitStatusStore.swift index dec2fb4a1d..21eddbd047 100644 --- a/apple/TablePro/Core/Services/SQL/LinkedFolderGitStatusStore.swift +++ b/apple/TablePro/Core/Services/SQL/LinkedFolderGitStatusStore.swift @@ -49,9 +49,10 @@ internal final class LinkedFolderGitStatusStore: ObservableObject { private let clientFactory: @Sendable () -> GitClient? private let foldersProvider: @MainActor () -> [LinkedSQLFolder] - private let repositoryWatcher = GitRepositoryWatcher() + private let repositoryWatcher = FileSystemEventWatcher() private var cancellables: Set = [] private var refreshTask: Task? + private var repositorySearch: Task? private var hasStarted = false init( @@ -68,13 +69,35 @@ internal final class LinkedFolderGitStatusStore: ObservableObject { AppEvents.shared.linkedSQLFoldersDidUpdate .sink { [weak self] _ in self?.scheduleRefresh() } .store(in: &cancellables) + // A folder in a repository is followed by the folder and `.git` watchers. One that is not can join a + // repository made in a parent folder, which neither watcher sees, so only those folders are asked again. NotificationCenter.default.publisher(for: NSApplication.didBecomeActiveNotification) - .sink { [weak self] _ in self?.scheduleRefresh() } + .sink { [weak self] _ in self?.lookForNewRepositories() } .store(in: &cancellables) repositoryWatcher.onChange = { [weak self] in self?.scheduleRefresh() } scheduleRefresh(after: .zero) } + @discardableResult + func lookForNewRepositories() -> Task? { + let outside = foldersProvider().filter { $0.isEnabled && snapshots[$0.id] == nil } + guard !outside.isEmpty, repositorySearch == nil else { return nil } + let client = clientFactory() + let search = Task { [weak self] in + defer { self?.repositorySearch = nil } + guard let client else { return } + for folder in outside { + let repository = try? await client.repositoryInfo(in: folder.expandedURL) + if repository != nil { + self?.scheduleRefresh() + return + } + } + } + repositorySearch = search + return search + } + func state(for favorite: LinkedSQLFavorite) -> LinkedFileGitState? { snapshots[favorite.folderId]?.state(forRelativePath: favorite.relativePath) } diff --git a/apple/TablePro/Core/Services/SQL/LinkedSQLFolderScanner.swift b/apple/TablePro/Core/Services/SQL/LinkedSQLFolderScanner.swift new file mode 100644 index 0000000000..ede46d884c --- /dev/null +++ b/apple/TablePro/Core/Services/SQL/LinkedSQLFolderScanner.swift @@ -0,0 +1,107 @@ +// +// LinkedSQLFolderScanner.swift +// TablePro +// + +import Foundation +import TableProUtilities + +internal struct LinkedSQLFolderScanner: Sendable { + typealias HeaderLoader = @Sendable (URL) -> FileTextLoader.LoadedText? + + private let index: LinkedSQLIndex + private let loadHeader: HeaderLoader + + init( + index: LinkedSQLIndex = .shared, + loadHeader: @escaping HeaderLoader = { FileTextLoader.loadHeader($0) } + ) { + self.index = index + self.loadHeader = loadHeader + } + + // Returns whether the folder's rows changed. A file whose size and modification time match its + // row keeps the row without reading the file. + @concurrent + func scan(folderId: UUID, at folderURL: URL) async -> Bool { + guard FileManager.default.fileExists(atPath: folderURL.path) else { + return await index.removeFolder(folderId: folderId) + } + let stored = await index.storedFiles(folderId: folderId) + let files = indexedFiles(in: folderURL, stored: stored) + return await index.synchronize(folderId: folderId, files: files) + } + + // Returns whether any folder was dropped from the index. + @concurrent + func pruneFolders(keeping known: Set) async -> Bool { + var pruned = false + let stale = await index.allFolderIds().subtracting(known) + for id in stale { + if await index.removeFolder(folderId: id) { + pruned = true + } + } + return pruned + } + + private func indexedFiles( + in folderURL: URL, + stored: [String: LinkedSQLIndex.IndexedFile] + ) -> [LinkedSQLIndex.IndexedFile] { + let keys: [URLResourceKey] = [.isRegularFileKey, .contentModificationDateKey, .fileSizeKey] + guard let enumerator = FileManager.default.enumerator( + at: folderURL, + includingPropertiesForKeys: keys, + options: [.skipsHiddenFiles] + ) else { + return [] + } + + var files: [LinkedSQLIndex.IndexedFile] = [] + for case let url as URL in enumerator { + guard SQLFileService.supportedExtensions.contains(url.pathExtension.lowercased()), + let values = try? url.resourceValues(forKeys: Set(keys)), + values.isRegularFile == true, + let relativePath = Self.relativePath(of: url, in: folderURL) else { + continue + } + let mtime = (values.contentModificationDate ?? Date()).timeIntervalSince1970 + let fileSize = Int64(values.fileSize ?? 0) + if let known = stored[relativePath], known.mtime == mtime, known.fileSize == fileSize { + files.append(known) + } else { + files.append(readFile(at: url, relativePath: relativePath, mtime: mtime, fileSize: fileSize)) + } + } + return files + } + + private func readFile( + at url: URL, + relativePath: String, + mtime: TimeInterval, + fileSize: Int64 + ) -> LinkedSQLIndex.IndexedFile { + let header = loadHeader(url) + let metadata = header.map { SQLFrontmatter.parse($0.content) } ?? SQLFrontmatter.Metadata() + let baseName = (url.lastPathComponent as NSString).deletingPathExtension + let name = metadata.name?.trimmingCharacters(in: .whitespaces) + return LinkedSQLIndex.IndexedFile( + relativePath: relativePath, + name: name.flatMap { $0.isEmpty ? nil : $0 } ?? baseName, + keyword: metadata.keyword, + description: metadata.description, + mtime: mtime, + fileSize: fileSize, + encodingName: (header?.encoding ?? .utf8).ianaName + ) + } + + private static func relativePath(of url: URL, in base: URL) -> String? { + let urlPath = url.standardizedFileURL.path + let basePath = base.standardizedFileURL.path + guard urlPath.hasPrefix(basePath + "/") else { return nil } + return String(urlPath.dropFirst(basePath.count + 1)) + } +} diff --git a/apple/TablePro/Core/Services/SQL/SQLFolderWatcher.swift b/apple/TablePro/Core/Services/SQL/SQLFolderWatcher.swift index ba46a5a216..3aba27af14 100644 --- a/apple/TablePro/Core/Services/SQL/SQLFolderWatcher.swift +++ b/apple/TablePro/Core/Services/SQL/SQLFolderWatcher.swift @@ -7,48 +7,86 @@ import Combine import CoreServices import Foundation import os -import TableProUtilities @MainActor internal final class SQLFolderWatcher: ObservableObject { static let shared = SQLFolderWatcher() - nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "SQLFolderWatcher") @Published private(set) var lastScanCompletedAt: Date? - private var eventStream: FSEventStreamRef? + private let eventWatcher = FileSystemEventWatcher( + latency: 1.0, + reportsFileEvents: true, + accepts: { events in + if SQLFolderWatcher.touchesGitState(events) { + SQLFolderWatcher.gitStateTouched.withLock { $0 = true } + return true + } + return SQLFolderWatcher.shouldRescan(after: events) + } + ) private var debounceTask: Task? private var hasStarted = false - nonisolated private static let eventCallback: FSEventStreamCallback = { _, info, _, eventPaths, _, _ in - guard let info else { return } - let paths = (Unmanaged.fromOpaque(eventPaths).takeUnretainedValue() as NSArray) as? [String] ?? [] - guard !touchesOnlyRepositoryMetadata(paths) else { return } - let watcher = Unmanaged.fromOpaque(info).takeUnretainedValue() - Task { @MainActor in - watcher.scheduleDebouncedRescan() - } - } + // Flags after which the paths in the batch no longer tell what changed. + nonisolated private static let rescanFlags = FSEventStreamEventFlags( + kFSEventStreamEventFlagMustScanSubDirs + | kFSEventStreamEventFlagRootChanged + | kFSEventStreamEventFlagUserDropped + | kFSEventStreamEventFlagKernelDropped + | kFSEventStreamEventFlagEventIdsWrapped + ) nonisolated static func touchesOnlyRepositoryMetadata(_ paths: [String]) -> Bool { guard !paths.isEmpty else { return false } - return paths.allSatisfy { path in - path.split(separator: "/").contains(".git") + return paths.allSatisfy(isRepositoryMetadata) + } + + // A batch is skipped only when every path is inside `.git` or is a file that cannot be a linked query. + nonisolated static func shouldRescan(after events: [FileSystemEvent]) -> Bool { + guard !events.isEmpty else { return true } + if events.contains(where: { $0.flags & rescanFlags != 0 }) { + return true + } + return !events.allSatisfy { event in + if isRepositoryMetadata(event.path) { + return true + } + guard event.has(kFSEventStreamEventFlagItemIsFile) else { return false } + let pathExtension = (event.path as NSString).pathExtension.lowercased() + return !SQLFileService.supportedExtensions.contains(pathExtension) + } + } + + // A repository created or removed here, or an ignore rule edited, changes Git status without touching + // a query, so the update is sent even when no row changed. + nonisolated private static let gitStateTouched = OSAllocatedUnfairLock(initialState: false) + + nonisolated static func touchesGitState(_ events: [FileSystemEvent]) -> Bool { + events.contains { event in + let name = (event.path as NSString).lastPathComponent + return name == ".git" || name == ".gitignore" } } - private init() {} + nonisolated private static func isRepositoryMetadata(_ path: String) -> Bool { + path.split(separator: "/").contains(".git") + } + + private init() { + eventWatcher.onChange = { [weak self] in self?.scheduleDebouncedRescan() } + } func start() { guard !hasStarted else { return } hasStarted = true let folders = LinkedSQLFolderStorage.shared.loadFolders().filter(\.isEnabled) scheduleFullRescan(folders: folders) - setupEventStream(for: folders) + eventWatcher.watch(Set(folders.map(\.expandedURL.path))) } func stop() { - cancelEventStream() + eventWatcher.cancel() debounceTask?.cancel() debounceTask = nil hasStarted = false @@ -59,60 +97,12 @@ internal final class SQLFolderWatcher: ObservableObject { start() } - // MARK: - Event stream - - private func setupEventStream(for folders: [LinkedSQLFolder]) { - cancelEventStream() - guard !folders.isEmpty else { return } - - let paths = folders.map(\.expandedURL.path) as CFArray - var context = FSEventStreamContext( - version: 0, - info: Unmanaged.passUnretained(self).toOpaque(), - retain: nil, - release: nil, - copyDescription: nil - ) - - let flags = UInt32( - kFSEventStreamCreateFlagFileEvents - | kFSEventStreamCreateFlagNoDefer - | kFSEventStreamCreateFlagWatchRoot - | kFSEventStreamCreateFlagUseCFTypes - ) - - guard let stream = FSEventStreamCreate( - kCFAllocatorDefault, - Self.eventCallback, - &context, - paths, - FSEventStreamEventId(kFSEventStreamEventIdSinceNow), - 1.0, - flags - ) else { - Self.logger.error("Failed to create FSEventStream") - return - } - - FSEventStreamSetDispatchQueue(stream, .global(qos: .utility)) - FSEventStreamStart(stream) - eventStream = stream - } - - private func cancelEventStream() { - guard let stream = eventStream else { return } - FSEventStreamStop(stream) - FSEventStreamInvalidate(stream) - FSEventStreamRelease(stream) - eventStream = nil - } - // MARK: - Scan scheduling private func scheduleFullRescan(folders: [LinkedSQLFolder]) { debounceTask?.cancel() debounceTask = Task { @MainActor [weak self] in - await Self.rescan(folders: folders) + _ = await Self.rescan(folders: folders) self?.lastScanCompletedAt = Date() AppEvents.shared.linkedSQLFoldersDidUpdate.send(nil) } @@ -127,100 +117,31 @@ internal final class SQLFolderWatcher: ObservableObject { return } let folders = LinkedSQLFolderStorage.shared.loadFolders().filter(\.isEnabled) - await Self.rescan(folders: folders) + let changed = await Self.rescan(folders: folders) + let gitStateTouched = Self.gitStateTouched.withLock { touched in + defer { touched = false } + return touched + } self?.lastScanCompletedAt = Date() + guard changed || gitStateTouched else { return } AppEvents.shared.linkedSQLFoldersDidUpdate.send(nil) } } - private static func rescan(folders: [LinkedSQLFolder]) async { - await Task.detached(priority: .utility) { - for folder in folders { - await scanFolder(folder) + // Returns whether any folder's rows changed or a folder was dropped from the index. + @concurrent + nonisolated private static func rescan(folders: [LinkedSQLFolder]) async -> Bool { + let scanner = LinkedSQLFolderScanner() + var changed = false + for folder in folders { + if await scanner.scan(folderId: folder.id, at: folder.expandedURL) { + changed = true } - let allKnownIds = Set(LinkedSQLFolderStorage.shared.loadFolders().map(\.id)) - await pruneRemovedFolders(stillKnownIds: allKnownIds) - }.value - } - - // MARK: - Per-folder scan (background) - - private static func scanFolder(_ folder: LinkedSQLFolder) async { - let folderURL = folder.expandedURL - let fileManager = FileManager.default - guard fileManager.fileExists(atPath: folderURL.path) else { - await LinkedSQLIndex.shared.removeFolder(folderId: folder.id) - return } - - var indexed: [LinkedSQLIndex.IndexedFile] = [] - - enumerateSQLFileURLs(in: folderURL, fileManager: fileManager) { url in - let resourceValues = try? url.resourceValues(forKeys: [ - .isRegularFileKey, .contentModificationDateKey, .fileSizeKey - ]) - guard resourceValues?.isRegularFile == true else { return } - let mtime = resourceValues?.contentModificationDate ?? Date() - let fileSize = Int64(resourceValues?.fileSize ?? 0) - - guard let relativePath = relativePathFor(url: url, base: folderURL) else { return } - let header = FileTextLoader.loadHeader(url) - let metadata = header.map { SQLFrontmatter.parse($0.content) } ?? SQLFrontmatter.Metadata() - let encoding = header?.encoding ?? .utf8 - - let baseName = (url.lastPathComponent as NSString).deletingPathExtension - let displayName = metadata.name?.trimmingCharacters(in: .whitespaces).nonEmpty - ?? baseName - - indexed.append(LinkedSQLIndex.IndexedFile( - relativePath: relativePath, - name: displayName, - keyword: metadata.keyword, - description: metadata.description, - mtime: mtime, - fileSize: fileSize, - encoding: encoding - )) + let known = Set(LinkedSQLFolderStorage.shared.loadFolders().map(\.id)) + if await scanner.pruneFolders(keeping: known) { + changed = true } - - await LinkedSQLIndex.shared.replaceAll(folderId: folder.id, files: indexed, folderURL: folderURL) + return changed } - - private static func enumerateSQLFileURLs( - in folderURL: URL, - fileManager: FileManager, - handle: (URL) -> Void - ) { - guard let enumerator = fileManager.enumerator( - at: folderURL, - includingPropertiesForKeys: [.isRegularFileKey, .contentModificationDateKey, .fileSizeKey], - options: [.skipsHiddenFiles] - ) else { - return - } - - for case let url as URL in enumerator { - guard SQLFileService.supportedExtensions.contains(url.pathExtension.lowercased()) else { continue } - handle(url) - } - } - - private static func pruneRemovedFolders(stillKnownIds: Set) async { - let indexedIds = await LinkedSQLIndex.shared.allFolderIds() - let stale = indexedIds.subtracting(stillKnownIds) - for id in stale { - await LinkedSQLIndex.shared.removeFolder(folderId: id) - } - } - - private static func relativePathFor(url: URL, base: URL) -> String? { - let urlPath = url.standardizedFileURL.path - let basePath = base.standardizedFileURL.path - guard urlPath.hasPrefix(basePath + "/") else { return nil } - return String(urlPath.dropFirst(basePath.count + 1)) - } -} - -private extension String { - var nonEmpty: String? { isEmpty ? nil : self } } diff --git a/apple/TablePro/Core/Storage/LinkedSQLIndex.swift b/apple/TablePro/Core/Storage/LinkedSQLIndex.swift index 7c37703c7a..79b78740b2 100644 --- a/apple/TablePro/Core/Storage/LinkedSQLIndex.swift +++ b/apple/TablePro/Core/Storage/LinkedSQLIndex.swift @@ -6,7 +6,6 @@ import Foundation import os import SQLite3 -import TableProUtilities internal actor LinkedSQLIndex { static let shared = LinkedSQLIndex() @@ -132,55 +131,120 @@ internal actor LinkedSQLIndex { // MARK: - Mutations - func replaceAll(folderId: UUID, files: [IndexedFile], folderURL: URL) { - guard sqlite3_exec(db, "BEGIN IMMEDIATE;", nil, nil, nil) == SQLITE_OK else { return } + // Writes only what differs from the stored rows, in one transaction, and returns whether any row changed. + @discardableResult + func synchronize(folderId: UUID, files: [IndexedFile]) -> Bool { + let stored = storedFiles(folderId: folderId) + let listed = Dictionary(files.map { ($0.relativePath, $0) }, uniquingKeysWith: { _, last in last }) + let removed = stored.keys.filter { listed[$0] == nil }.sorted() + let changed = listed.values.filter { stored[$0.relativePath] != $0 }.sorted { $0.relativePath < $1.relativePath } + guard !removed.isEmpty || !changed.isEmpty else { return false } + + guard sqlite3_exec(db, "BEGIN IMMEDIATE;", nil, nil, nil) == SQLITE_OK else { + Self.logger.error("Could not begin a linked SQL index write: \(String(cString: sqlite3_errmsg(self.db)))") + return false + } + guard deleteRows(folderId: folderId, relativePaths: removed), upsertRows(folderId: folderId, files: changed) else { + sqlite3_exec(db, "ROLLBACK;", nil, nil, nil) + return false + } + guard sqlite3_exec(db, "COMMIT;", nil, nil, nil) == SQLITE_OK else { + Self.logger.error("Could not commit a linked SQL index write: \(String(cString: sqlite3_errmsg(self.db)))") + sqlite3_exec(db, "ROLLBACK;", nil, nil, nil) + return false + } + return true + } - let deleteSQL = "DELETE FROM linked_sql_files WHERE folder_id = ?;" - var deleteStatement: OpaquePointer? - let folderIdString = folderId.uuidString - if sqlite3_prepare_v2(db, deleteSQL, -1, &deleteStatement, nil) == SQLITE_OK { - sqlite3_bind_text(deleteStatement, 1, folderIdString, -1, Self.transient) - sqlite3_step(deleteStatement) + private func deleteRows(folderId: UUID, relativePaths: [String]) -> Bool { + guard !relativePaths.isEmpty else { return true } + let sql = "DELETE FROM linked_sql_files WHERE folder_id = ? AND relative_path = ?;" + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return false } + for relativePath in relativePaths { + sqlite3_reset(statement) + sqlite3_bind_text(statement, 1, folderId.uuidString, -1, Self.transient) + sqlite3_bind_text(statement, 2, relativePath, -1, Self.transient) + guard sqlite3_step(statement) == SQLITE_DONE else { + Self.logger.error("Failed to delete a linked file: \(String(cString: sqlite3_errmsg(self.db)))") + return false + } } - sqlite3_finalize(deleteStatement) + return true + } - let insertSQL = """ + private func upsertRows(folderId: UUID, files: [IndexedFile]) -> Bool { + guard !files.isEmpty else { return true } + let sql = """ INSERT INTO linked_sql_files (folder_id, relative_path, name, keyword, description, mtime, file_size, encoding) - VALUES (?, ?, ?, ?, ?, ?, ?, ?); + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(folder_id, relative_path) DO UPDATE SET + name = excluded.name, + keyword = excluded.keyword, + description = excluded.description, + mtime = excluded.mtime, + file_size = excluded.file_size, + encoding = excluded.encoding; """ - var insertStatement: OpaquePointer? - guard sqlite3_prepare_v2(db, insertSQL, -1, &insertStatement, nil) == SQLITE_OK else { - sqlite3_exec(db, "ROLLBACK;", nil, nil, nil) - return - } - defer { sqlite3_finalize(insertStatement) } - + var statement: OpaquePointer? + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return false } for file in files { - sqlite3_reset(insertStatement) - sqlite3_bind_text(insertStatement, 1, folderIdString, -1, Self.transient) - sqlite3_bind_text(insertStatement, 2, file.relativePath, -1, Self.transient) - sqlite3_bind_text(insertStatement, 3, file.name, -1, Self.transient) - if let keyword = file.keyword { - sqlite3_bind_text(insertStatement, 4, keyword, -1, Self.transient) - } else { - sqlite3_bind_null(insertStatement, 4) - } - if let description = file.description { - sqlite3_bind_text(insertStatement, 5, description, -1, Self.transient) - } else { - sqlite3_bind_null(insertStatement, 5) + sqlite3_reset(statement) + sqlite3_bind_text(statement, 1, folderId.uuidString, -1, Self.transient) + sqlite3_bind_text(statement, 2, file.relativePath, -1, Self.transient) + sqlite3_bind_text(statement, 3, file.name, -1, Self.transient) + bindOptionalText(statement, 4, file.keyword) + bindOptionalText(statement, 5, file.description) + sqlite3_bind_double(statement, 6, file.mtime) + sqlite3_bind_int64(statement, 7, file.fileSize) + sqlite3_bind_text(statement, 8, file.encodingName, -1, Self.transient) + guard sqlite3_step(statement) == SQLITE_DONE else { + Self.logger.error("Failed to write a linked file: \(String(cString: sqlite3_errmsg(self.db)))") + return false } - sqlite3_bind_double(insertStatement, 6, file.mtime.timeIntervalSince1970) - sqlite3_bind_int64(insertStatement, 7, file.fileSize) - sqlite3_bind_text(insertStatement, 8, file.encoding.ianaName, -1, Self.transient) + } + return true + } - if sqlite3_step(insertStatement) != SQLITE_DONE { - Self.logger.error("Failed to insert linked file: \(String(cString: sqlite3_errmsg(self.db)))") - } + private func bindOptionalText(_ statement: OpaquePointer?, _ index: Int32, _ value: String?) { + if let value { + sqlite3_bind_text(statement, index, value, -1, Self.transient) + } else { + sqlite3_bind_null(statement, index) } + } - sqlite3_exec(db, "COMMIT;", nil, nil, nil) + func storedFiles(folderId: UUID) -> [String: IndexedFile] { + let sql = """ + SELECT relative_path, name, keyword, description, mtime, file_size, encoding + FROM linked_sql_files + WHERE folder_id = ?; + """ + var statement: OpaquePointer? + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return [:] } + defer { sqlite3_finalize(statement) } + sqlite3_bind_text(statement, 1, folderId.uuidString, -1, Self.transient) + + var files: [String: IndexedFile] = [:] + while sqlite3_step(statement) == SQLITE_ROW { + guard let relativePath = sqlite3_column_text(statement, 0).map({ String(cString: $0) }), + let name = sqlite3_column_text(statement, 1).map({ String(cString: $0) }) else { + continue + } + files[relativePath] = IndexedFile( + relativePath: relativePath, + name: name, + keyword: sqlite3_column_text(statement, 2).map { String(cString: $0) }, + description: sqlite3_column_text(statement, 3).map { String(cString: $0) }, + mtime: sqlite3_column_double(statement, 4), + fileSize: sqlite3_column_int64(statement, 5), + encodingName: sqlite3_column_text(statement, 6).map { String(cString: $0) } ?? "utf-8" + ) + } + return files } func allFolderIds() -> Set { @@ -199,14 +263,16 @@ internal actor LinkedSQLIndex { return ids } - func removeFolder(folderId: UUID) { + // Returns whether the folder had any rows. + @discardableResult + func removeFolder(folderId: UUID) -> Bool { let sql = "DELETE FROM linked_sql_files WHERE folder_id = ?;" var statement: OpaquePointer? - if sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK { - sqlite3_bind_text(statement, 1, folderId.uuidString, -1, Self.transient) - sqlite3_step(statement) - } - sqlite3_finalize(statement) + defer { sqlite3_finalize(statement) } + guard sqlite3_prepare_v2(db, sql, -1, &statement, nil) == SQLITE_OK else { return false } + sqlite3_bind_text(statement, 1, folderId.uuidString, -1, Self.transient) + guard sqlite3_step(statement) == SQLITE_DONE else { return false } + return sqlite3_changes(db) > 0 } // MARK: - Queries @@ -286,13 +352,14 @@ internal actor LinkedSQLIndex { } internal extension LinkedSQLIndex { - struct IndexedFile { + struct IndexedFile: Equatable, Sendable { let relativePath: String let name: String let keyword: String? let description: String? - let mtime: Date + // Seconds since 1970 as the column stores them, so a row read back compares exactly. + let mtime: TimeInterval let fileSize: Int64 - let encoding: String.Encoding + let encodingName: String } } diff --git a/apple/TablePro/Core/Sync/SyncCoordinator.swift b/apple/TablePro/Core/Sync/SyncCoordinator.swift index a1734725af..389ee50e76 100644 --- a/apple/TablePro/Core/Sync/SyncCoordinator.swift +++ b/apple/TablePro/Core/Sync/SyncCoordinator.swift @@ -62,6 +62,13 @@ final class SyncCoordinator: ObservableObject { /// suspended across the network can tell whether its outcome is still the current answer. private var statusGeneration = 0 + // An activation asks only for what other Macs sent, so one download per interval answers every activation in it. + nonisolated static let activationInterval: Duration = .seconds(60) + private let activationClock: any Clock + private var activationCooldown: Task? + private var activationHeldForCooldown = false + private var downloadIsAheadInRun = false + init( services: AppServices = .live, recordCache: SyncRecordCache = SyncCoordinator.makeRecordCache(), @@ -70,9 +77,11 @@ final class SyncCoordinator: ObservableObject { columnLayouts: @escaping @autoclosure () -> FileColumnLayoutPersister = .shared, themeCatalog: ThemeCatalog = .shared, licenseGate: (@MainActor () -> DisableReason?)? = nil, - recordCreationDate: @escaping (CKRecord) -> Date? = { $0.creationDate } + recordCreationDate: @escaping (CKRecord) -> Date? = { $0.creationDate }, + activationClock: any Clock = ContinuousClock() ) { self.services = services + self.activationClock = activationClock self.makeTransport = transport self.makeNetworkMonitor = makeNetworkMonitor self.recordCreationDate = recordCreationDate @@ -99,6 +108,7 @@ final class SyncCoordinator: ObservableObject { deinit { debounceTask?.cancel() retryTask?.cancel() + activationCooldown?.cancel() } // MARK: - Lifecycle @@ -136,6 +146,9 @@ final class SyncCoordinator: ObservableObject { Self.logger.info("Sync skipped: not allowed for \(String(describing: trigger), privacy: .public)") return } + if trigger == .activation, holdsActivationBack() { + return + } guard !isRunning else { if !pendingTriggers.contains(trigger) { pendingTriggers.append(trigger) @@ -173,6 +186,40 @@ final class SyncCoordinator: ObservableObject { await runPendingTrigger() } + // A run whose download has not started brings what the activation asks for. After a download, activations + // wait for the interval to end and then run once, so a change made on another Mac still arrives. + private func holdsActivationBack() -> Bool { + if isRunning { + return downloadIsAheadInRun + } + guard activationCooldown != nil else { return false } + activationHeldForCooldown = true + return true + } + + var isHoldingActivations: Bool { + activationCooldown != nil + } + + private func startActivationCooldown() { + activationCooldown?.cancel() + activationHeldForCooldown = false + let clock = activationClock + activationCooldown = Task { [weak self] in + do { + try await clock.sleep(for: Self.activationInterval) + } catch { + return + } + // A newer download may have replaced this hold after the sleep ended. + guard !Task.isCancelled, let self else { return } + activationCooldown = nil + guard activationHeldForCooldown else { return } + activationHeldForCooldown = false + await sync(.activation) + } + } + internal func runSyncCycle(_ admission: SyncAdmission = .full) async -> SyncError? { await accountThenPushThenPull(admission).failure?.error } @@ -184,6 +231,8 @@ final class SyncCoordinator: ObservableObject { private func accountThenPushThenPull(_ requested: SyncAdmission) async -> CycleResult { var admission = requested + downloadIsAheadInRun = true + defer { downloadIsAheadInRun = false } switch await confirmAccount() { case .unavailable(let failure): return CycleResult(admission: admission, failure: failure) @@ -208,6 +257,9 @@ final class SyncCoordinator: ObservableObject { let push = admission == .full ? await performPush() : PushReport() let pullFailure = await performPull(echoGuard: push.echoGuard) + if pullFailure == nil { + startActivationCooldown() + } let zone = metadataStorage.zoneState let download = zone.reconciled(downloadFailure: pullFailure) @@ -357,8 +409,13 @@ final class SyncCoordinator: ObservableObject { guard !pendingTriggers.isEmpty else { return } let error = syncStatus.error /// The network coming back mid-run matters only if the run ended unable to reach iCloud. - let triggers = pendingTriggers.filter { $0 != .networkRestored || error == .offline } + var triggers = pendingTriggers.filter { $0 != .networkRestored || error == .offline } pendingTriggers = [] + // Taken out before the pick, so a held activation never stands in for an edit queued behind it. + if activationCooldown != nil, triggers.contains(.activation) { + triggers.removeAll { $0 == .activation } + activationHeldForCooldown = true + } let attempt = nextAttempt let throttle = throttledUntil let reach: (SyncTrigger) -> Int = { trigger in @@ -531,6 +588,9 @@ final class SyncCoordinator: ObservableObject { pendingTriggers = [] debounceTask?.cancel() debounceTask = nil + activationCooldown?.cancel() + activationCooldown = nil + activationHeldForCooldown = false } // Exhaustive, so a paying customer the app could not reach the server about is never told to buy. @@ -690,6 +750,7 @@ final class SyncCoordinator: ObservableObject { let token = metadataStorage.loadToken() let tokenStatus = token == nil ? "nil (full fetch)" : "present (delta)" Self.logger.info("Pull starting, token: \(tokenStatus)") + downloadIsAheadInRun = false do { let result = try await transport.pull(since: token) diff --git a/apple/TablePro/Models/ServerDashboard/ServerDashboardModels.swift b/apple/TablePro/Models/ServerDashboard/ServerDashboardModels.swift index d21d7ca284..bf613d325b 100644 --- a/apple/TablePro/Models/ServerDashboard/ServerDashboardModels.swift +++ b/apple/TablePro/Models/ServerDashboard/ServerDashboardModels.swift @@ -35,7 +35,7 @@ enum DashboardRefreshInterval: Double, CaseIterable, Identifiable { // MARK: - Dashboard Session -struct DashboardSession: Identifiable { +struct DashboardSession: Identifiable, Equatable { let id: String let user: String let database: String @@ -49,7 +49,7 @@ struct DashboardSession: Identifiable { // MARK: - Dashboard Metric -struct DashboardMetric: Identifiable { +struct DashboardMetric: Identifiable, Equatable { let id: String let label: String let value: String @@ -65,4 +65,8 @@ struct DashboardSlowQuery: Identifiable { let query: String let user: String let database: String + + func hasSameContent(as other: DashboardSlowQuery) -> Bool { + duration == other.duration && query == other.query && user == other.user && database == other.database + } } diff --git a/apple/TablePro/Theme/ThemeCatalog.swift b/apple/TablePro/Theme/ThemeCatalog.swift index e53c4be15e..e6bf3897f5 100644 --- a/apple/TablePro/Theme/ThemeCatalog.swift +++ b/apple/TablePro/Theme/ThemeCatalog.swift @@ -1,4 +1,3 @@ -import AppKit import Combine import Foundation import os @@ -62,14 +61,18 @@ internal struct ThemeFileTrash: Sendable { } } -/// Owns the list. A user theme is identified by its JSON `id` but lives in whatever file it was -/// loaded from, so a save, rename or move to the Trash acts on that file rather than on `.json`; -/// the two differ for any file the user named themselves. Every mutation updates the in-memory list -/// synchronously, and the list reloads whenever TablePro becomes active, so a theme fixed, dropped -/// or removed in Finder is picked up without a view asking. +// Owns the list. A user theme is identified by its JSON `id` but lives in whatever file it was +// loaded from, so a save, rename or move to the Trash acts on that file rather than on `.json`; +// the two differ for any file the user named themselves. Every mutation updates the in-memory list +// synchronously, and the shared catalog reloads when its folder changes on disk, so a theme fixed, +// dropped or removed in Finder is picked up without a view asking. @MainActor internal final class ThemeCatalog: ObservableObject { - internal static let shared = ThemeCatalog() + internal static let shared: ThemeCatalog = { + let catalog = ThemeCatalog() + catalog.startWatchingUserDirectory() + return catalog + }() @Published internal private(set) var themes: [ThemeDefinition] = BuiltInThemes.all @Published internal private(set) var rejected: [RejectedThemeRecord] = [] @@ -81,30 +84,37 @@ internal final class ThemeCatalog: ObservableObject { /// new file, which would bring back a theme deleted in Finder or on another Mac. private var vanished: [String: URL] = [:] private let userDirectory: URL - private let bundleDirectory: URL? + private let bundled: [ThemeDefinition] private let fileTrash: ThemeFileTrash private let syncTracker: SyncChangeTracker - private var activationCancellable: AnyCancellable? + private let directoryWatcher = FileSystemEventWatcher() nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "ThemeCatalog") - /// Loaded before the first activation, so a user theme in a slot is in the list the launch - /// activation resolves against. + // Loads here, so a user theme a slot names is listed before the engine first resolves it. + // Bundled themes are read only here: they cannot change while TablePro runs. internal init( userDirectory: URL = ThemeCatalog.defaultUserDirectory, bundleDirectory: URL? = Bundle.main.resourceURL, - notificationCenter: NotificationCenter = .default, fileTrash: ThemeFileTrash = .resolved(in: .shared), syncTracker: SyncChangeTracker = .shared ) { self.userDirectory = userDirectory - self.bundleDirectory = bundleDirectory + self.bundled = ThemeCatalog.loadBundled(from: bundleDirectory) self.fileTrash = fileTrash self.syncTracker = syncTracker reloadSynchronously() - activationCancellable = notificationCenter - .publisher(for: NSApplication.didBecomeActiveNotification) - .sink { [weak self] _ in self?.reloadSynchronously() } + } + + // The catalog's own writes report too; their reload finds what the list already holds and + // publishes nothing. + internal func startWatchingUserDirectory() { + directoryWatcher.onChange = { [weak self] in self?.reloadSynchronously() } + directoryWatcher.watch([userDirectory.path]) + } + + internal func stopWatchingUserDirectory() { + directoryWatcher.cancel() } internal func theme(id: String) -> ThemeDefinition? { @@ -122,7 +132,7 @@ internal final class ThemeCatalog: ObservableObject { } internal func reloadSynchronously() { - adopt(Self.loadContents(userDirectory: userDirectory, bundleDirectory: bundleDirectory)) + adopt(Self.loadContents(userDirectory: userDirectory, bundled: bundled)) } /// Writes the theme to the file it came from, or a new theme to `.json`. The list then holds @@ -361,25 +371,28 @@ internal final class ThemeCatalog: ObservableObject { // MARK: - Disk - nonisolated private static func loadContents(userDirectory: URL, bundleDirectory: URL?) -> ThemeCatalogContents { - var themes = BuiltInThemes.all + nonisolated private static func loadBundled(from directory: URL?) -> [ThemeDefinition] { + guard let directory else { return [] } + var themes: [ThemeDefinition] = [] + for file in jsonFiles(in: directory) where file.lastPathComponent.hasPrefix(ThemeDefinition.builtInPrefix) { + do { + let theme = try ThemeFileFormat.decode(try Data(contentsOf: file)) + guard !(BuiltInThemes.all + themes).contains(where: { $0.id == theme.id }) else { continue } + themes.append(theme) + } catch { + logger.error("Bundled theme \(file.lastPathComponent) rejected: \(error.publicLogShape, privacy: .public)") + } + } + return themes + } + + nonisolated private static func loadContents(userDirectory: URL, bundled: [ThemeDefinition]) -> ThemeCatalogContents { + var themes = BuiltInThemes.all + bundled var rejected: [RejectedThemeRecord] = [] var userFiles: [String: ThemeFile] = [:] ensureDirectory(userDirectory) - if let bundle = bundleDirectory { - for file in jsonFiles(in: bundle) where file.lastPathComponent.hasPrefix(ThemeDefinition.builtInPrefix) { - do { - let theme = try ThemeFileFormat.decode(try Data(contentsOf: file)) - guard !themes.contains(where: { $0.id == theme.id }) else { continue } - themes.append(theme) - } catch { - logger.error("Bundled theme \(file.lastPathComponent) rejected: \(error.publicLogShape, privacy: .public)") - } - } - } - let files = jsonFiles(in: userDirectory) .sorted { $0.lastPathComponent.localizedStandardCompare($1.lastPathComponent) == .orderedAscending } diff --git a/apple/TablePro/ViewModels/ServerDashboardViewModel.swift b/apple/TablePro/ViewModels/ServerDashboardViewModel.swift index 2ad773a298..d077040d9a 100644 --- a/apple/TablePro/ViewModels/ServerDashboardViewModel.swift +++ b/apple/TablePro/ViewModels/ServerDashboardViewModel.swift @@ -25,10 +25,7 @@ final class ServerDashboardViewModel: ObservableObject { @Published var refreshInterval: DashboardRefreshInterval = .fiveSeconds { didSet { guard oldValue != refreshInterval else { return } - if refreshTask != nil || refreshInterval != .off { - startAutoRefresh() - } - updateTileActivity() + restartRefreshLoop() } } @@ -66,6 +63,7 @@ final class ServerDashboardViewModel: ObservableObject { nonisolated(unsafe) private var refreshTask: Task? private var isOnScreen = false + private var isWindowVisible = true private let services: AppServices private var providerServerVersion: String? private var hasAdoptedServerVersion = false @@ -122,14 +120,28 @@ final class ServerDashboardViewModel: ObservableObject { // MARK: - Auto Refresh func startAutoRefresh() { - refreshTask?.cancel() isOnScreen = true - updateTileActivity() + restartRefreshLoop() + } - guard refreshInterval != .off else { - refreshTask = nil - return - } + func stopAutoRefresh() { + isOnScreen = false + restartRefreshLoop() + isRefreshing = false + } + + // A hidden, minimized or fully covered window shows nothing, so the server is not asked; coming back refreshes at once. + func windowVisibilityChanged(_ isVisible: Bool) { + guard isWindowVisible != isVisible else { return } + isWindowVisible = isVisible + restartRefreshLoop() + } + + private func restartRefreshLoop() { + refreshTask?.cancel() + refreshTask = nil + updateTileActivity() + guard isOnScreen, isWindowVisible, refreshInterval != .off else { return } refreshTask = Task { [weak self] in while !Task.isCancelled { @@ -144,16 +156,12 @@ final class ServerDashboardViewModel: ObservableObject { } } - func stopAutoRefresh() { - refreshTask?.cancel() - refreshTask = nil - isRefreshing = false - isOnScreen = false - updateTileActivity() + var isPolling: Bool { + refreshTask != nil } private func updateTileActivity() { - tileBoard.isActive = isOnScreen && !isPaused && refreshInterval != .off + tileBoard.isActive = isOnScreen && isWindowVisible && !isPaused && refreshInterval != .off } // MARK: - Data Fetching @@ -222,8 +230,11 @@ final class ServerDashboardViewModel: ObservableObject { if provider.supportedPanels.contains(.activeSessions) { do { - sessions = try await provider.fetchSessions(execute: execute) - sessions.sort(using: sessionSortOrder) + var fetched = try await provider.fetchSessions(execute: execute) + fetched.sort(using: sessionSortOrder) + if fetched != sessions { + sessions = fetched + } } catch { Self.logger.warning("Failed to fetch sessions: \(error.localizedDescription)") newPanelErrors[.activeSessions] = error.localizedDescription @@ -232,7 +243,10 @@ final class ServerDashboardViewModel: ObservableObject { if provider.supportedPanels.contains(.serverMetrics) { do { - metrics = try await provider.fetchMetrics(execute: execute) + let fetched = try await provider.fetchMetrics(execute: execute) + if fetched != metrics { + metrics = fetched + } } catch { Self.logger.warning("Failed to fetch metrics: \(error.localizedDescription)") newPanelErrors[.serverMetrics] = error.localizedDescription @@ -241,14 +255,19 @@ final class ServerDashboardViewModel: ObservableObject { if provider.supportedPanels.contains(.slowQueries) { do { - slowQueries = try await provider.fetchSlowQueries(execute: execute) + let fetched = try await provider.fetchSlowQueries(execute: execute) + if !fetched.elementsEqual(slowQueries, by: { $0.hasSameContent(as: $1) }) { + slowQueries = fetched + } } catch { Self.logger.warning("Failed to fetch slow queries: \(error.localizedDescription)") newPanelErrors[.slowQueries] = error.localizedDescription } } - panelErrors = newPanelErrors + if panelErrors != newPanelErrors { + panelErrors = newPanelErrors + } await tileBoard.refresh(force: forcingTiles) lastRefreshDate = Date() } diff --git a/apple/TablePro/Views/Components/WindowVisibilityReader.swift b/apple/TablePro/Views/Components/WindowVisibilityReader.swift new file mode 100644 index 0000000000..0685ffcb49 --- /dev/null +++ b/apple/TablePro/Views/Components/WindowVisibilityReader.swift @@ -0,0 +1,58 @@ +import AppKit +import SwiftUI + +// Reports whether any part of the hosting window is on screen. A minimized, hidden or fully covered window, +// a background window tab, and a view taken out of its window all report false; `onDisappear` misses most of these. +struct WindowVisibilityReader: NSViewRepresentable { + var onChange: (Bool) -> Void + + func makeNSView(context: Context) -> WindowVisibilityView { + let view = WindowVisibilityView() + view.onChange = onChange + return view + } + + func updateNSView(_ nsView: WindowVisibilityView, context: Context) { + nsView.onChange = onChange + } + + static func dismantleNSView(_ nsView: WindowVisibilityView, coordinator: ()) { + nsView.stopObserving() + } +} + +final class WindowVisibilityView: NSView { + var onChange: ((Bool) -> Void)? + private var observer: NSObjectProtocol? + private var reported: Bool? + + override func viewDidMoveToWindow() { + super.viewDidMoveToWindow() + stopObserving() + if let window { + observer = NotificationCenter.default.addObserver( + forName: NSWindow.didChangeOcclusionStateNotification, + object: window, + queue: .main + ) { [weak self] _ in + MainActor.assumeIsolated { self?.report() } + } + } + // Moving into a window happens inside a SwiftUI update, which must not publish. + DispatchQueue.main.async { [weak self] in self?.report() } + } + + func stopObserving() { + if let observer { + NotificationCenter.default.removeObserver(observer) + } + observer = nil + } + + private func report() { + let isVisible = window?.occlusionState.contains(.visible) ?? false + guard isVisible != reported else { return } + reported = isVisible + onChange?(isVisible) + } +} diff --git a/apple/TablePro/Views/Integrations/IntegrationsConnectedClientsPane.swift b/apple/TablePro/Views/Integrations/IntegrationsConnectedClientsPane.swift index 6e75c0694c..27f00bd25f 100644 --- a/apple/TablePro/Views/Integrations/IntegrationsConnectedClientsPane.swift +++ b/apple/TablePro/Views/Integrations/IntegrationsConnectedClientsPane.swift @@ -33,6 +33,7 @@ struct IntegrationsConnectedClientsPane: View { } .navigationTitle(IntegrationsActivitySection.connectedClients.title) .navigationSubtitle(subtitle) + .task { await manager.refreshClientsWhileShown() } .toolbar(content: toolbar) .alert( String(localized: "Disconnect client?"), @@ -190,14 +191,20 @@ private struct ConnectedClientsTable: View { @ViewBuilder private func connectedCell(for client: MCPServerManager.SessionSnapshot) -> some View { - Text(client.connectedSince, format: .relative(presentation: .named)) - .help(client.connectedSince.formatted(date: .complete, time: .standard)) + relativeText(client.connectedSince) } @ViewBuilder private func lastActivityCell(for client: MCPServerManager.SessionSnapshot) -> some View { - Text(client.lastActivityAt, format: .relative(presentation: .named)) - .help(client.lastActivityAt.formatted(date: .complete, time: .standard)) + relativeText(client.lastActivityAt) + } + + // The list publishes only when it changes, so the elapsed time redraws on its own schedule. + private func relativeText(_ date: Date) -> some View { + TimelineView(.periodic(from: .now, by: 5)) { _ in + Text(date.formatted(.relative(presentation: .named))) + } + .help(date.formatted(date: .complete, time: .standard)) } @ViewBuilder diff --git a/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+DatabaseFileChanges.swift b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+DatabaseFileChanges.swift new file mode 100644 index 0000000000..6ca180a9ce --- /dev/null +++ b/apple/TablePro/Views/Main/Extensions/MainContentCoordinator+DatabaseFileChanges.swift @@ -0,0 +1,42 @@ +// +// MainContentCoordinator+DatabaseFileChanges.swift +// TablePro +// + +import Foundation + +// Every commit rewrites a database file, so a file event alone says nothing about the catalog. The engine's +// schema token does: the catalog reloads only when it moved, or when the engine has none. +internal struct SchemaChangeTokenGate: Equatable { + // Read before the catalog it stands for, so it never claims a schema newer than the one loaded. + private(set) var loadedToken: String? + + mutating func admits(_ token: String?) -> Bool { + if let token, token == loadedToken { return false } + loadedToken = token + return true + } + + mutating func forget() { + loadedToken = nil + } +} + +extension MainContentCoordinator { + func refreshTablesIfSchemaChanged() async { + let token = try? await services.databaseManager.withBrowseMetadataDriver(connectionId: connectionId) { driver in + try await (driver as? PluginDriverAdapter)?.schemaChangeToken() + } + guard schemaTokenGate.admits(token) else { return } + DatabaseTreeMetadataService.shared.markListsStale(connectionId: connectionId) + schemaColumns.removeAll() + // Never joins a load that began before this write: that load's catalog predates the new token. + let committed = await services.schemaRefreshService.refreshAfterWrite(connection: connection) + // Another window's refresh may have replaced this one; prune against the catalog it leaves. + await services.schemaRefreshService.waitForRefresh(connectionId: connectionId) + pruneStaleSidebarState() + if !committed { + schemaTokenGate.forget() + } + } +} diff --git a/apple/TablePro/Views/Main/MainContentCoordinator.swift b/apple/TablePro/Views/Main/MainContentCoordinator.swift index 896bc3d933..a2a9bfd85a 100644 --- a/apple/TablePro/Views/Main/MainContentCoordinator.swift +++ b/apple/TablePro/Views/Main/MainContentCoordinator.swift @@ -366,6 +366,7 @@ final class MainContentCoordinator: ObservableObject { let diskContent: String } internal private(set) var fileWatcher: DatabaseFileWatcher? + var schemaTokenGate = SchemaChangeTokenGate() // Set during handleTabChange to suppress redundant column-change reconfiguration. internal var isHandlingTabSwitch = false @@ -754,8 +755,7 @@ final class MainContentCoordinator: ObservableObject { watcher.watch(filePath: filePath, connectionId: connectionId) { [weak self] in guard let self else { return } if case .loading = services.schemaService.state(for: self.connectionId) { return } - DatabaseTreeMetadataService.shared.markListsStale(connectionId: self.connectionId) - Task { await self.refreshTables() } + Task { await self.refreshTablesIfSchemaChanged() } } fileWatcher = watcher } diff --git a/apple/TablePro/Views/ServerDashboard/ServerDashboardView.swift b/apple/TablePro/Views/ServerDashboard/ServerDashboardView.swift index 81c418131c..9916ea38c9 100644 --- a/apple/TablePro/Views/ServerDashboard/ServerDashboardView.swift +++ b/apple/TablePro/Views/ServerDashboard/ServerDashboardView.swift @@ -19,6 +19,7 @@ struct ServerDashboardView: View { } } .frame(maxWidth: .infinity, maxHeight: .infinity) + .background { WindowVisibilityReader { viewModel.windowVisibilityChanged($0) } } .onAppear { viewModel.startAutoRefresh() } .onDisappear { viewModel.stopAutoRefresh() } .alert( diff --git a/apple/TablePro/Views/Settings/Appearance/ThemeEditorModel.swift b/apple/TablePro/Views/Settings/Appearance/ThemeEditorModel.swift index b79452e486..18132bc678 100644 --- a/apple/TablePro/Views/Settings/Appearance/ThemeEditorModel.swift +++ b/apple/TablePro/Views/Settings/Appearance/ThemeEditorModel.swift @@ -12,10 +12,10 @@ internal protocol ThemePreviewing: AnyObject { extension ThemeEngine: ThemePreviewing {} -/// One open theme. The draft is saved a moment after it settles and on close, previewed in every -/// window while it is edited, and undone one settled step at a time. The catalog reloads whenever -/// TablePro becomes active, so a file changed or removed elsewhere is noticed here before a save -/// could write over it. +// One open theme. The draft is saved a moment after it settles and on close, previewed in every +// window while it is edited, and undone one settled step at a time. The catalog reloads when its +// folder changes on disk, so a file changed or removed elsewhere is noticed before a save could +// write over it. @MainActor internal final class ThemeEditorModel: ObservableObject { internal enum Alert: Identifiable, Equatable { diff --git a/apple/TableProMobile/TableProMobile/SSH/SSHTunnel.swift b/apple/TableProMobile/TableProMobile/SSH/SSHTunnel.swift index afacbf7bce..2177ce2d69 100644 --- a/apple/TableProMobile/TableProMobile/SSH/SSHTunnel.swift +++ b/apple/TableProMobile/TableProMobile/SSH/SSHTunnel.swift @@ -3,18 +3,17 @@ import Foundation import os import TableProSSHTransport -/// One authenticated libssh2 session forwarding a local port, built by `SSHTunnelFactory`. -/// -/// A class rather than an actor, the shape `LibSSH2Tunnel` already has on macOS. Every libssh2 -/// call is serialized on `sessionQueue`, the accept loop has a serial queue of its own and the -/// relays share a concurrent one, so nothing here blocks a thread of Swift's cooperative pool. -/// An actor gives mutual exclusion but never thread transfer, so a tunnel parked in `poll` held -/// a cooperative thread for its whole life: measured, 48 tunnels ran 12 at a time in 4,005ms -/// where queues ran all 48 in 1,003ms. `Task.detached` measured the same 12, because it is the -/// same pool. -/// -/// Its socket, session and listening socket are `let`, established before construction, so there -/// is no mutable state to protect and `close()` is synchronous. +// One authenticated libssh2 session forwarding a local port, built by `SSHTunnelFactory`. +// +// A class rather than an actor, the shape `LibSSH2Tunnel` already has on macOS. Every libssh2 call +// is serialized on `sessionQueue`, the acceptor has a serial queue of its own and the relays share a +// concurrent one, so nothing here blocks a thread of Swift's cooperative pool. An actor gives mutual +// exclusion but never thread transfer, so a tunnel parked in `poll` held a cooperative thread for its +// whole life: measured, 48 tunnels ran 12 at a time in 4,005ms where queues ran all 48 in 1,003ms. +// `Task.detached` measured the same 12, because it is the same pool. +// +// Its socket, session and listening socket are `let`, established before construction, so `close()` +// is synchronous. nonisolated final class SSHTunnel: @unchecked Sendable { let localPort: Int @@ -31,7 +30,7 @@ nonisolated final class SSHTunnel: @unchecked Sendable { /// Concurrent: relay loops, which poll, send and recv but make no libssh2 call directly. private let relayQueue: DispatchQueue - /// Serial: the accept loop, which polls the listening socket and nothing else. + // Serial: the acceptor's handler, which accepts and makes no libssh2 call. private let acceptQueue: DispatchQueue private let aliveLatch = TeardownLatch() @@ -44,7 +43,7 @@ nonisolated final class SSHTunnel: @unchecked Sendable { /// here, since the relay runs on `relayQueue` outside the task's cancellation scope and stops /// on `aliveLatch` rather than on `Task.isCancelled`. private let clientRelays = DispatchGroup() - private var forwardingTask: Task? + private var acceptor: SSHListenerAcceptor? private var keepAliveTask: Task? private static let relayBufferSize = 32_768 @@ -56,10 +55,6 @@ nonisolated final class SSHTunnel: @unchecked Sendable { /// before the driver reports its own timeout, which names no cause. static let channelOpenDeadlineSeconds: TimeInterval = 6 - /// How long the accept loop waits per poll before rechecking the latch. Small enough that - /// noticing a client the kernel already accepted costs a slim part of the margin above. - static let acceptPollTimeoutMs: Int32 = 200 - private static let keepAliveIntervalSeconds = 10 init( @@ -90,30 +85,24 @@ nonisolated final class SSHTunnel: @unchecked Sendable { func startForwarding(destination: SSHForwardDestination) { sessionQueue.sync { libssh2_session_set_blocking(session, 0) } - forwardingTask = Task.detached { [weak self] in - guard let self else { return } - - await withCheckedContinuation { (continuation: CheckedContinuation) in - self.acceptQueue.async { [weak self] in - defer { continuation.resume() } - guard let self else { return } - - let target = destination.logDescription - Self.logger.info("Forwarding started on port \(self.localPort) -> \(target)") - - while self.isRunning { - guard let client = self.acceptClient() else { continue } - self.spawnClient( - clientFD: client.fd, - acceptedAt: client.acceptedAt, - destination: destination - ) - } - - Self.logger.info("Forwarding loop ended for port \(self.localPort)") - } + // A handler already running when the latch is claimed may still spawn one relay; teardown + // waits for the cancel handler before the relay group, so it waits for that relay too. + let acceptor = SSHListenerAcceptor(listenFD: listenFD, queue: acceptQueue) { [weak self] client in + guard let self, self.isRunning else { + Darwin.close(client.descriptor) + return } + self.spawnClient( + clientFD: client.descriptor, + acceptedAt: client.acceptedAt, + destination: destination + ) } + self.acceptor = acceptor + acceptor.start() + + let target = destination.logDescription + Self.logger.info("Forwarding started on port \(self.localPort) -> \(target)") } // MARK: - Keep-Alive @@ -163,11 +152,11 @@ nonisolated final class SSHTunnel: @unchecked Sendable { } // shutdown wakes every poll on the transport but keeps the number, so no other socket can take it mid-poll. - // Darwin's shutdown does not wake a poll on a listening socket, so the accept loop ends on the latch instead. - // The relays are awaited after the accept loop, the only thing that starts one, so the group can only empty. + // The listening socket closes only after the acceptor's cancel handler ran, once no accept handler is running. + // The relays are awaited after that, since the accept handler is the only thing that starts one. // The hops go last, in the Mac's order. private func performTeardown() { - forwardingTask?.cancel() + acceptor?.cancel() keepAliveTask?.cancel() shutdown(socketFD, SHUT_RDWR) @@ -180,10 +169,10 @@ nonisolated final class SSHTunnel: @unchecked Sendable { let listenFD = self.listenFD let jumpHops = self.jumpHops let localPort = self.localPort - let forwardingTask = self.forwardingTask + let acceptor = self.acceptor let keepAliveTask = self.keepAliveTask Task.detached { - await forwardingTask?.value + await acceptor?.waitForCancellation() await keepAliveTask?.value await withCheckedContinuation { (continuation: CheckedContinuation) in clientRelays.notify(queue: relayQueue) { continuation.resume() } @@ -205,31 +194,7 @@ nonisolated final class SSHTunnel: @unchecked Sendable { // MARK: - Private - /// The accept timestamp is taken here, not once the open reaches `openAndRelay`, because the - /// client's own connect timeout is already running by then and the scheduling hops in between - /// would push the deadline past it. - private func acceptClient() -> (fd: Int32, acceptedAt: Date)? { - var pollFD = pollfd(fd: listenFD, events: Int16(POLLIN), revents: 0) - let pollResult = poll(&pollFD, 1, Self.acceptPollTimeoutMs) - - guard pollResult > 0, pollFD.revents & Int16(POLLIN) != 0 else { return nil } - - var clientAddr = sockaddr_in() - var addrLen = socklen_t(MemoryLayout.size) - - let clientFD = withUnsafeMutablePointer(to: &clientAddr) { - $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { - accept(listenFD, $0, &addrLen) - } - } - - guard clientFD >= 0 else { return nil } - return (clientFD, Date()) - } - - /// Opens the channel and relays one accepted client, off the accept loop so a slow open - /// cannot delay the next accept. The loop runs on `relayQueue` (concurrent); individual - /// libssh2 calls are dispatched to `sessionQueue` (serial) for thread safety. + // On `relayQueue`, off the accept handler, so a slow open cannot delay the next accept. private func spawnClient(clientFD: Int32, acceptedAt: Date, destination: SSHForwardDestination) { let clientRelays = self.clientRelays clientRelays.enter() diff --git a/apple/TableProTests/Core/MCP/Lifecycle/MCPServerClientListTests.swift b/apple/TableProTests/Core/MCP/Lifecycle/MCPServerClientListTests.swift new file mode 100644 index 0000000000..5ffa228d2e --- /dev/null +++ b/apple/TableProTests/Core/MCP/Lifecycle/MCPServerClientListTests.swift @@ -0,0 +1,41 @@ +import Combine +import Foundation +import Testing + +@testable import TablePro +import TableProMCP + +@MainActor +struct MCPServerClientListTests { + private let directory: URL + + init() throws { + directory = FileManager.default.temporaryDirectory + .appendingPathComponent("mcp-clients-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: directory.path) + } + + @Test("A client list refresh that finds nothing new publishes nothing") + func unchangedRefreshPublishesNothing() async { + let manager = MCPServerManager(handshake: MCPHandshakeFile(directory: directory), isEnabled: { false }) + var published = 0 + let subscription = manager.objectWillChange.sink { _ in published += 1 } + defer { subscription.cancel() } + + await manager.refreshClients() + await manager.refreshClients() + + #expect(published == 0) + #expect(manager.connectedClients.isEmpty) + } + + @Test("The client list refresh ends with the task of the view that shows it") + func refreshEndsWithItsTask() async { + let manager = MCPServerManager(handshake: MCPHandshakeFile(directory: directory), isEnabled: { false }) + let shown = Task { await manager.refreshClientsWhileShown() } + shown.cancel() + await shown.value + #expect(manager.connectedClients.isEmpty) + } +} diff --git a/apple/TableProTests/Core/SSH/SFTPDownloadChunkSizeTests.swift b/apple/TableProTests/Core/SSH/SFTPDownloadChunkSizeTests.swift new file mode 100644 index 0000000000..e15517ad52 --- /dev/null +++ b/apple/TableProTests/Core/SSH/SFTPDownloadChunkSizeTests.swift @@ -0,0 +1,31 @@ +// +// SFTPDownloadChunkSizeTests.swift +// TableProTests +// + +@testable import TablePro +import Testing + +struct SFTPDownloadChunkSizeTests { + private static let minimum = 32 * 1_024 + private static let maximum = 2 * 1_024 * 1_024 + + @Test("A file under 32 KiB still reads into a 32 KiB buffer") + func smallFileGetsTheFloor() { + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: 0) == Self.minimum) + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: 4_096) == Self.minimum) + } + + @Test("A file over 2 MiB reads into a 2 MiB buffer, which fills libssh2's 8 MiB read-ahead") + func largeFileGetsTheCap() { + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: 500 * 1_024 * 1_024) == Self.maximum) + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: .max) == Self.maximum) + } + + @Test("A file between the floor and the cap reads into a buffer of its own size") + func midSizeFileGetsItsOwnSize() { + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: UInt64(Self.minimum)) == Self.minimum) + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: 300_000) == 300_000) + #expect(LibSSH2SFTPSession.chunkSize(forFileSize: UInt64(Self.maximum)) == Self.maximum) + } +} diff --git a/apple/TableProTests/Core/SSH/SSHTunnelDeadlineMarginTests.swift b/apple/TableProTests/Core/SSH/SSHTunnelDeadlineMarginTests.swift index 1c250164d7..62a6a3dada 100644 --- a/apple/TableProTests/Core/SSH/SSHTunnelDeadlineMarginTests.swift +++ b/apple/TableProTests/Core/SSH/SSHTunnelDeadlineMarginTests.swift @@ -27,10 +27,4 @@ struct SSHTunnelDeadlineMarginTests { #expect(second.deadline.configuredSeconds == 60) #expect(second.deadline.instant == secondStart.advanced(by: .seconds(60))) } - - @Test("The accept loop notices a waiting client well inside the margin") - func acceptPollIsShort() { - #expect(LibSSH2Tunnel.acceptPollTimeoutMs <= 200) - #expect(LibSSH2Tunnel.acceptPollTimeoutMs > 0) - } } diff --git a/apple/TableProTests/Core/Services/Query/SchemaRefreshAfterWriteTests.swift b/apple/TableProTests/Core/Services/Query/SchemaRefreshAfterWriteTests.swift index d42c6456b9..b1625b8da6 100644 --- a/apple/TableProTests/Core/Services/Query/SchemaRefreshAfterWriteTests.swift +++ b/apple/TableProTests/Core/Services/Query/SchemaRefreshAfterWriteTests.swift @@ -93,7 +93,7 @@ struct SchemaRefreshAfterWriteTests { // The main queue runs jobs in order, so the second refresh has joined before the fetch resumes. await Task { @MainActor in }.value driver.resumeFetchTables() - await postWrite.value + _ = await postWrite.value await joined.value #expect(driver.fetchTablesCallCount == 1) @@ -113,4 +113,20 @@ struct SchemaRefreshAfterWriteTests { #expect(driver.fetchTablesCallCount == 2) } + + @Test("a refresh after a write says whether it committed a fresh table list") + func refreshAfterWriteReportsWhetherItCommitted() async { + let driver = MockDatabaseDriver() + driver.tablesToReturn = [orders] + let schemaService = SchemaService() + let service = makeService(schemaService: schemaService, driver: driver) + let connection = TestFixtures.makeConnection() + + #expect(await service.refreshAfterWrite(connection: connection)) + + driver.tablesToReturn = [orders, probe] + driver.fetchTablesError = DatabaseError.queryFailed("database is locked") + #expect(await service.refreshAfterWrite(connection: connection) == false) + #expect(schemaService.state(for: connection.id) == .loaded([orders])) + } } diff --git a/apple/TableProTests/Core/Services/SQL/FileSystemEventWatcherTests.swift b/apple/TableProTests/Core/Services/SQL/FileSystemEventWatcherTests.swift new file mode 100644 index 0000000000..1f5de5306f --- /dev/null +++ b/apple/TableProTests/Core/Services/SQL/FileSystemEventWatcherTests.swift @@ -0,0 +1,45 @@ +// +// FileSystemEventWatcherTests.swift +// TableProTests +// +// The stream used to hold its watcher unretained, so an event that arrived after the watcher +// was released read freed memory. +// + +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +@MainActor +struct FileSystemEventWatcherTests { + @Test("A change in the folder reaches a live watcher, and none reaches one cancelled or released before delivery") + func onlyALiveWatcherIsCalled() async throws { + let folder = FileManager.default.temporaryDirectory + .appendingPathComponent("FileSystemEventWatcherTests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: folder, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: folder) } + + var silenced = 0 + let cancelled = FileSystemEventWatcher(latency: 0.05) + cancelled.onChange = { silenced += 1 } + cancelled.watch([folder.path]) + var released: FileSystemEventWatcher? = FileSystemEventWatcher(latency: 0.05) + released?.onChange = { silenced += 1 } + released?.watch([folder.path]) + + var reported = 0 + let live = FileSystemEventWatcher(latency: 0.05) + live.onChange = { reported += 1 } + live.watch([folder.path]) + defer { live.cancel() } + + try Data("{}".utf8).write(to: folder.appendingPathComponent("theme.json")) + // Stopped before FSEvents delivers, so this pins that a stopped or released watcher is never called. + cancelled.cancel() + released = nil + + try await waitUntil { reported > 0 } + #expect(silenced == 0) + } +} diff --git a/apple/TableProTests/Core/Services/SQL/LinkedFolderGitStatusStoreTests.swift b/apple/TableProTests/Core/Services/SQL/LinkedFolderGitStatusStoreTests.swift new file mode 100644 index 0000000000..fd9544165c --- /dev/null +++ b/apple/TableProTests/Core/Services/SQL/LinkedFolderGitStatusStoreTests.swift @@ -0,0 +1,44 @@ +import Foundation +@testable import TablePro +import Testing + +@MainActor +@Suite(.enabled(if: GitClient.make() != nil)) +struct LinkedFolderGitStatusStoreTests { + @Test("Switching back asks Git only about folders outside a repository, once each") + func activationAsksOnlyFoldersOutsideARepository() async throws { + let real = try #require(GitClient.make()) + let root = FileManager.default.temporaryDirectory + .appendingPathComponent("LinkedFolderGitStatusStoreTests-\(UUID().uuidString)", isDirectory: true) + .resolvingSymlinksInPath() + let repository = root.appendingPathComponent("repository", isDirectory: true) + let plain = root.appendingPathComponent("plain", isDirectory: true) + try FileManager.default.createDirectory(at: repository, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: plain, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: root) } + let initialized = try await real.runner.run( + GitCommand(arguments: ["init", "-q"], workingDirectory: repository, access: .write) + ) + #expect(initialized.succeeded) + + let log = root.appendingPathComponent("invocations.log") + let wrapper = root.appendingPathComponent("git") + let script = "#!/bin/sh\npwd -P >> '\(log.path)'\nexec '\(real.runner.executableURL.path)' \"$@\"\n" + try Data(script.utf8).write(to: wrapper) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: wrapper.path) + let client = GitClient(runner: GitProcessRunner(executableURL: wrapper)) + let folders = [LinkedSQLFolder(path: repository.path), LinkedSQLFolder(path: plain.path)] + let store = LinkedFolderGitStatusStore(clientFactory: { client }, foldersProvider: { folders }) + + await store.refresh() + #expect(store.snapshots.count == 1) + try Data().write(to: log) + + await store.lookForNewRepositories()?.value + + // `pwd -P` prints the /private/var path; resolving strips that prefix again. + let asked = try String(contentsOf: log, encoding: .utf8).split(separator: "\n") + .map { URL(fileURLWithPath: String($0)).resolvingSymlinksInPath().path } + #expect(asked == [plain.path]) + } +} diff --git a/apple/TableProTests/Core/Services/SQL/LinkedSQLFolderScannerTests.swift b/apple/TableProTests/Core/Services/SQL/LinkedSQLFolderScannerTests.swift new file mode 100644 index 0000000000..c9e87738d3 --- /dev/null +++ b/apple/TableProTests/Core/Services/SQL/LinkedSQLFolderScannerTests.swift @@ -0,0 +1,172 @@ +// +// LinkedSQLFolderScannerTests.swift +// TableProTests +// +// Any file event under a linked folder used to read every SQL file's header again and rewrite +// the folder's whole index. A scan now reads only files whose size or modification time moved, +// and writes only rows that differ. +// + +import Foundation +import os +@testable import TablePro +import TableProUtilities +import Testing + +struct LinkedSQLFolderScannerTests { + private final class Fixture { + let root: URL + let folder: URL + let folderId = UUID() + let index: LinkedSQLIndex + private let reads = OSAllocatedUnfairLock<[String]>(initialState: []) + + init() throws { + root = FileManager.default.temporaryDirectory + .appendingPathComponent("LinkedSQLFolderScannerTests-\(UUID().uuidString)", isDirectory: true) + folder = root.appendingPathComponent("Queries", isDirectory: true) + try FileManager.default.createDirectory( + at: folder.appendingPathComponent("reports", isDirectory: true), + withIntermediateDirectories: true + ) + index = LinkedSQLIndex(databaseURL: root.appendingPathComponent("index.db"), removeDatabaseOnDeinit: true) + } + + var scanner: LinkedSQLFolderScanner { + let reads = reads + return LinkedSQLFolderScanner(index: index) { url in + reads.withLock { $0.append(url.lastPathComponent) } + return FileTextLoader.loadHeader(url) + } + } + + func scan() async -> Bool { + await scanner.scan(folderId: folderId, at: folder) + } + + // The header reads since the last call. + func takeReads() -> [String] { + reads.withLock { recorded in + defer { recorded = [] } + return recorded.sorted() + } + } + + func write(_ relativePath: String, _ text: String, modified: Date? = nil) throws { + let url = folder.appendingPathComponent(relativePath) + try Data(text.utf8).write(to: url) + if let modified { + try FileManager.default.setAttributes([.modificationDate: modified], ofItemAtPath: url.path) + } + } + + func stored() async -> [String: LinkedSQLIndex.IndexedFile] { + await index.storedFiles(folderId: folderId) + } + + func remove() { + try? FileManager.default.removeItem(at: root) + } + } + + private static func seed(_ fixture: Fixture) throws { + try fixture.write("revenue.sql", "-- @name: Revenue\n-- @keyword: rev\nSELECT 1;") + try fixture.write("users.sql", "-- @description: Active users\nSELECT 2;") + try fixture.write("reports/daily.psql", "SELECT 3;") + try fixture.write("notes.txt", "not a query") + } + + @Test("The first scan reads every SQL file and indexes its header") + func firstScanReadsEverySQLFile() async throws { + let fixture = try Fixture() + defer { fixture.remove() } + try Self.seed(fixture) + + #expect(await fixture.scan()) + + #expect(fixture.takeReads() == ["daily.psql", "revenue.sql", "users.sql"]) + let stored = await fixture.stored() + #expect(Set(stored.keys) == ["revenue.sql", "users.sql", "reports/daily.psql"]) + #expect(stored["revenue.sql"]?.name == "Revenue") + #expect(stored["revenue.sql"]?.keyword == "rev") + #expect(stored["users.sql"]?.name == "users") + #expect(stored["users.sql"]?.description == "Active users") + } + + @Test("A second scan of an unchanged folder reads no file and changes no row") + func unchangedFolderReadsNothing() async throws { + let fixture = try Fixture() + defer { fixture.remove() } + try Self.seed(fixture) + #expect(await fixture.scan()) + _ = fixture.takeReads() + let before = await fixture.stored() + + #expect(await fixture.scan() == false) + + #expect(fixture.takeReads().isEmpty) + #expect(await fixture.stored() == before) + } + + @Test("A file whose modification time moved is the only one read again") + func touchedFileIsReadAgain() async throws { + let fixture = try Fixture() + defer { fixture.remove() } + try Self.seed(fixture) + #expect(await fixture.scan()) + _ = fixture.takeReads() + let before = await fixture.stored() + + let modified = Date(timeIntervalSince1970: 1_800_000_000) + try fixture.write("users.sql", "-- @keyword: usr\nSELECT 2;", modified: modified) + + #expect(await fixture.scan()) + + #expect(fixture.takeReads() == ["users.sql"]) + let after = await fixture.stored() + #expect(after["users.sql"]?.keyword == "usr") + #expect(after["users.sql"]?.mtime == modified.timeIntervalSince1970) + #expect(after["revenue.sql"] == before["revenue.sql"]) + #expect(after["reports/daily.psql"] == before["reports/daily.psql"]) + } + + @Test("A deleted file's row is removed and a new file's row is added") + func deletedAndAddedFiles() async throws { + let fixture = try Fixture() + defer { fixture.remove() } + try Self.seed(fixture) + #expect(await fixture.scan()) + _ = fixture.takeReads() + let before = await fixture.stored() + + try FileManager.default.removeItem(at: fixture.folder.appendingPathComponent("revenue.sql")) + try fixture.write("reports/weekly.sql", "-- @name: Weekly\nSELECT 4;") + + #expect(await fixture.scan()) + + #expect(fixture.takeReads() == ["weekly.sql"]) + let after = await fixture.stored() + #expect(Set(after.keys) == ["users.sql", "reports/daily.psql", "reports/weekly.sql"]) + #expect(after["reports/weekly.sql"]?.name == "Weekly") + #expect(after["users.sql"] == before["users.sql"]) + } + + @Test("A folder no longer linked, or gone from disk, is dropped and reported once") + func removedFoldersAreDropped() async throws { + let fixture = try Fixture() + defer { fixture.remove() } + try Self.seed(fixture) + #expect(await fixture.scan()) + + #expect(await fixture.scanner.pruneFolders(keeping: [fixture.folderId]) == false) + #expect(await fixture.scanner.pruneFolders(keeping: [])) + #expect(await fixture.stored().isEmpty) + #expect(await fixture.scanner.pruneFolders(keeping: []) == false) + + #expect(await fixture.scan()) + try FileManager.default.removeItem(at: fixture.folder) + #expect(await fixture.scan()) + #expect(await fixture.stored().isEmpty) + #expect(await fixture.scan() == false) + } +} diff --git a/apple/TableProTests/Core/Services/SQL/SQLFolderWatcherTests.swift b/apple/TableProTests/Core/Services/SQL/SQLFolderWatcherTests.swift new file mode 100644 index 0000000000..b3877acf9e --- /dev/null +++ b/apple/TableProTests/Core/Services/SQL/SQLFolderWatcherTests.swift @@ -0,0 +1,73 @@ +// +// SQLFolderWatcherTests.swift +// TableProTests +// +// Every event under a linked folder used to rescan it, so a `.DS_Store` write or a build tool's +// output in a linked repository read every SQL file again. +// + +import CoreServices +import Foundation +@testable import TablePro +import Testing + +struct SQLFolderWatcherTests { + private static func event(_ path: String, _ flags: Int) -> FileSystemEvent { + FileSystemEvent(path: path, flags: FSEventStreamEventFlags(flags)) + } + + private static func file(_ path: String) -> FileSystemEvent { + event(path, kFSEventStreamEventFlagItemIsFile | kFSEventStreamEventFlagItemModified) + } + + @Test("A batch of only non-SQL files and repository metadata is skipped") + func nonSQLBatchIsSkipped() { + let batch = [ + Self.file("/repo/.DS_Store"), + Self.file("/repo/build/output.log"), + Self.file("/repo/queries/.revenue.sql.swp"), + Self.file("/repo/.git/index"), + Self.event("/repo/.git/refs/heads", kFSEventStreamEventFlagItemIsDir | kFSEventStreamEventFlagItemCreated), + ] + + #expect(SQLFolderWatcher.shouldRescan(after: batch) == false) + } + + @Test("A repository made or removed in the folder, or an ignore file edited, counts as a Git change") + func gitStateChanges() { + let created = Self.event("/repo/.git", kFSEventStreamEventFlagItemIsDir | kFSEventStreamEventFlagItemCreated) + #expect(SQLFolderWatcher.touchesGitState([created])) + #expect(SQLFolderWatcher.touchesGitState([Self.file("/repo/.DS_Store"), Self.file("/repo/queries/.gitignore")])) + #expect(!SQLFolderWatcher.touchesGitState([Self.file("/repo/.git/index"), Self.file("/repo/.DS_Store")])) + } + + @Test("A batch holding a SQL file is kept, whatever the case of its extension") + func sqlFileIsKept() { + #expect(SQLFolderWatcher.shouldRescan(after: [Self.file("/repo/notes.txt"), Self.file("/repo/q/Revenue.SQL")])) + #expect(SQLFolderWatcher.shouldRescan(after: [Self.file("/repo/q/daily.pgsql")])) + } + + @Test("A directory event is kept, a renamed directory included") + func directoryEventIsKept() { + let created = Self.event("/repo/queries", kFSEventStreamEventFlagItemIsDir | kFSEventStreamEventFlagItemCreated) + let renamed = Self.event("/repo/reports", kFSEventStreamEventFlagItemIsDir | kFSEventStreamEventFlagItemRenamed) + + #expect(SQLFolderWatcher.shouldRescan(after: [Self.file("/repo/.DS_Store"), created])) + #expect(SQLFolderWatcher.shouldRescan(after: [renamed])) + } + + @Test("A batch whose paths no longer say what changed is kept") + func coalescedBatchIsKept() { + let flags = [ + kFSEventStreamEventFlagMustScanSubDirs, + kFSEventStreamEventFlagRootChanged, + kFSEventStreamEventFlagUserDropped, + kFSEventStreamEventFlagKernelDropped, + kFSEventStreamEventFlagEventIdsWrapped, + ] + for flag in flags { + let batch = [Self.event("/repo/build.log", kFSEventStreamEventFlagItemIsFile | flag)] + #expect(SQLFolderWatcher.shouldRescan(after: batch), "flag \(flag)") + } + } +} diff --git a/apple/TableProTests/Core/Storage/AppSettingsValueColorsTests.swift b/apple/TableProTests/Core/Storage/AppSettingsValueColorsTests.swift index 314bccfa82..f6cdbc6cc1 100644 --- a/apple/TableProTests/Core/Storage/AppSettingsValueColorsTests.swift +++ b/apple/TableProTests/Core/Storage/AppSettingsValueColorsTests.swift @@ -31,7 +31,6 @@ struct AppSettingsValueColorsTests { let catalog = ThemeCatalog( userDirectory: environment.directory.appendingPathComponent("Themes", isDirectory: true), bundleDirectory: nil, - notificationCenter: NotificationCenter(), fileTrash: .folder(environment.directory.appendingPathComponent("Trash", isDirectory: true)), syncTracker: environment.tracker ) diff --git a/apple/TableProTests/Core/Storage/AppearanceModeStorageTests.swift b/apple/TableProTests/Core/Storage/AppearanceModeStorageTests.swift index 88944ae2a0..3a2d06c3ed 100644 --- a/apple/TableProTests/Core/Storage/AppearanceModeStorageTests.swift +++ b/apple/TableProTests/Core/Storage/AppearanceModeStorageTests.swift @@ -100,7 +100,6 @@ struct AppearanceModeStorageTests { let catalog = ThemeCatalog( userDirectory: root.appendingPathComponent("Themes", isDirectory: true), bundleDirectory: nil, - notificationCenter: NotificationCenter(), fileTrash: .folder(root.appendingPathComponent("Trash", isDirectory: true)), syncTracker: tracker ) diff --git a/apple/TableProTests/Core/Storage/LinkedSQLIndexDamageTests.swift b/apple/TableProTests/Core/Storage/LinkedSQLIndexDamageTests.swift index ff79aaa9a0..ca4bd4c8e1 100644 --- a/apple/TableProTests/Core/Storage/LinkedSQLIndexDamageTests.swift +++ b/apple/TableProTests/Core/Storage/LinkedSQLIndexDamageTests.swift @@ -18,12 +18,12 @@ struct LinkedSQLIndexDamageTests { name: "revenue", keyword: "rev", description: nil, - mtime: Date(timeIntervalSince1970: 1_800_000_000), + mtime: 1_800_000_000, fileSize: 8, - encoding: .utf8 + encodingName: "utf-8" ) - await index.replaceAll(folderId: folderId, files: [file], folderURL: scratch) + await index.synchronize(folderId: folderId, files: [file]) #expect(await index.allFolderIds() == [folderId]) let leftovers = try FileManager.default.contentsOfDirectory(atPath: scratch.path(percentEncoded: false)) diff --git a/apple/TableProTests/Core/Sync/SyncCoordinatorActivationTests.swift b/apple/TableProTests/Core/Sync/SyncCoordinatorActivationTests.swift new file mode 100644 index 0000000000..9eaf3ecb93 --- /dev/null +++ b/apple/TableProTests/Core/Sync/SyncCoordinatorActivationTests.swift @@ -0,0 +1,143 @@ +import CloudKit +import Foundation +@testable import TablePro +import TableProSyncTransport +import TableProTestSupport +import Testing + +@MainActor +struct SyncCoordinatorActivationTests { + private let environment: SyncTestEnvironment + private let clock = ManualClock() + + init() throws { + environment = try SyncTestEnvironment(label: "sync-activation") + var settings = SyncSettings.default + settings.enabled = true + AppSettingsStorage(userDefaults: environment.defaults).saveSync(settings) + } + + private func makeCoordinator(_ transport: ScriptedSyncTransport) -> SyncCoordinator { + environment.makeCoordinator(transport: transport, activationClock: clock) + } + + @Test("An activation while the launch run has yet to download adds no second run") + func activationDuringLaunchRunIsDropped() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + let activated = ActivationLatch() + await transport.whileZoneIsRead { + guard activated.claim() else { return } + await coordinator.sync(.activation) + } + + await coordinator.sync(.launch) + #expect(activated.isClaimed) + + // Held instead of dropped, it would run once the interval ends. + await clock.waitUntilSleeping() + clock.advance(by: SyncCoordinator.activationInterval) + try await waitUntil { !coordinator.isHoldingActivations } + try await waitUntil { coordinator.syncStatus != .syncing } + #expect(await transport.pullCount == 1) + } + + @Test("Activations inside the interval wait for its end and then download once") + func activationsInsideTheIntervalRunOnceAtItsEnd() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + await coordinator.sync(.launch) + #expect(await transport.pullCount == 1) + + await coordinator.sync(.activation) + await coordinator.sync(.activation) + #expect(await transport.pullCount == 1) + + await clock.waitUntilSleeping() + clock.advance(by: SyncCoordinator.activationInterval) + try await waitUntil { await transport.pullCount == 2 } + try await waitUntil { coordinator.syncStatus != .syncing } + #expect(await transport.pullCount == 2) + } + + @Test("An activation after the interval downloads at once") + func activationAfterTheIntervalRuns() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + await coordinator.sync(.launch) + + await clock.waitUntilSleeping() + clock.advance(by: SyncCoordinator.activationInterval) + try await waitUntil { !coordinator.isHoldingActivations } + await coordinator.sync(.activation) + #expect(await transport.pullCount == 2) + } + + @Test("An edit queued behind an activation during a download is not held with it") + func queuedEditIsNotHeldBehindAnActivation() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + await coordinator.sync(.launch) + await clock.waitUntilSleeping() + clock.advance(by: SyncCoordinator.activationInterval) + try await waitUntil { !coordinator.isHoldingActivations } + + // The second zone read of a run comes after its download, where an activation is queued, not dropped. + let reads = ZoneReadCounter() + await transport.whileZoneIsRead { + guard reads.next() == 2 else { return } + await coordinator.sync(.activation) + await coordinator.sync(.localChange) + } + await coordinator.sync(.userRequest) + try await waitUntil { await transport.pullCount == 3 } + } + + @Test("Turning sync off ends the activation hold") + func syncOffEndsTheHold() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + let coordinator = makeCoordinator(transport) + await coordinator.sync(.launch) + #expect(coordinator.isHoldingActivations) + + var settings = SyncSettings.default + settings.enabled = false + AppSettingsStorage(userDefaults: environment.defaults).saveSync(settings) + coordinator.start() + #expect(!coordinator.isHoldingActivations) + } + + @Test("A failed download leaves the next activation free to run") + func failedDownloadDoesNotHoldActivation() async throws { + let transport = ScriptedSyncTransport(zoneID: SyncTestEnvironment.zoneID) + await transport.failPulls(with: CKError(.networkFailure)) + let coordinator = makeCoordinator(transport) + await coordinator.sync(.launch) + #expect(await transport.pullCount == 1) + + await transport.failPulls(with: nil) + await coordinator.sync(.activation) + #expect(await transport.pullCount == 2) + } +} + +@MainActor +private final class ActivationLatch { + private(set) var isClaimed = false + + func claim() -> Bool { + guard !isClaimed else { return false } + isClaimed = true + return true + } +} + +@MainActor +private final class ZoneReadCounter { + private var count = 0 + + func next() -> Int { + count += 1 + return count + } +} diff --git a/apple/TableProTests/Core/Sync/SyncCoordinatorThemeTests.swift b/apple/TableProTests/Core/Sync/SyncCoordinatorThemeTests.swift index 33d20c49a2..e386631c31 100644 --- a/apple/TableProTests/Core/Sync/SyncCoordinatorThemeTests.swift +++ b/apple/TableProTests/Core/Sync/SyncCoordinatorThemeTests.swift @@ -47,7 +47,6 @@ struct SyncCoordinatorThemeTests { catalog = ThemeCatalog( userDirectory: themes, bundleDirectory: nil, - notificationCenter: NotificationCenter(), fileTrash: .folder(trash), syncTracker: environment.tracker ) diff --git a/apple/TableProTests/Helpers/SyncTestEnvironment.swift b/apple/TableProTests/Helpers/SyncTestEnvironment.swift index 2655e6683a..395ff56e49 100644 --- a/apple/TableProTests/Helpers/SyncTestEnvironment.swift +++ b/apple/TableProTests/Helpers/SyncTestEnvironment.swift @@ -83,7 +83,8 @@ final class SyncTestEnvironment { transport: ScriptedSyncTransport, favorites: SQLFavoriteManager? = nil, makeNetworkMonitor: @escaping () -> SyncNetworkMonitor? = { nil }, - licenseGate: @escaping @MainActor () -> DisableReason? = { nil } + licenseGate: @escaping @MainActor () -> DisableReason? = { nil }, + activationClock: any Clock = ContinuousClock() ) -> SyncCoordinator { let live = AppServices.live let services = AppServices( @@ -137,7 +138,8 @@ final class SyncTestEnvironment { transport: transport, makeNetworkMonitor: makeNetworkMonitor, columnLayouts: self.columnLayouts, - licenseGate: licenseGate + licenseGate: licenseGate, + activationClock: activationClock ) } } diff --git a/apple/TableProTests/Services/SchemaServiceRefreshTests.swift b/apple/TableProTests/Services/SchemaServiceRefreshTests.swift index baae278511..4a0fca0fde 100644 --- a/apple/TableProTests/Services/SchemaServiceRefreshTests.swift +++ b/apple/TableProTests/Services/SchemaServiceRefreshTests.swift @@ -139,7 +139,7 @@ struct SchemaServiceRefreshTests { } driver.resumeFetchTables() - await reload.value + _ = await reload.value #expect(service.tables(for: connectionId).map(\.name) == ["orders", "invoices"]) #expect(!service.isRefreshing(connectionId: connectionId)) @@ -182,8 +182,8 @@ struct SchemaServiceRefreshTests { } try await waitUntil { driver.tablesCallCount == 2 } driver.resumeFetchTables() - await first.value - await second.value + _ = await first.value + _ = await second.value #expect(driver.tablesCallCount == 2, "Each browse scope must run its own fetch") #expect(service.loadedScope(for: connectionId) == analytics) @@ -241,7 +241,7 @@ struct SchemaServiceRefreshTests { driver.resumeFetchTables() await waiter.value - await reload.value + _ = await reload.value #expect(service.loadedScope(for: connectionId) == secondScope) #expect(service.tables(for: connectionId).map(\.name) == ["events"]) diff --git a/apple/TableProTests/Theme/ThemeCatalogSyncTests.swift b/apple/TableProTests/Theme/ThemeCatalogSyncTests.swift index 29ec2a9024..393168ff3b 100644 --- a/apple/TableProTests/Theme/ThemeCatalogSyncTests.swift +++ b/apple/TableProTests/Theme/ThemeCatalogSyncTests.swift @@ -7,7 +7,6 @@ // anything, after treating it as untrusted input. // -import AppKit import Foundation import os @testable import TablePro @@ -20,7 +19,6 @@ struct ThemeCatalogSyncTests { @MainActor private final class Harness { let root: URL - let center = NotificationCenter() let suite: EphemeralDefaults let tracker: SyncChangeTracker let fileTrash: ThemeFileTrash @@ -47,7 +45,6 @@ struct ThemeCatalogSyncTests { catalog = ThemeCatalog( userDirectory: userDirectory ?? folders[0], bundleDirectory: nil, - notificationCenter: center, fileTrash: self.fileTrash, syncTracker: tracker ) @@ -64,7 +61,6 @@ struct ThemeCatalogSyncTests { catalog = ThemeCatalog( userDirectory: themes, bundleDirectory: nil, - notificationCenter: center, fileTrash: fileTrash, syncTracker: tracker ) @@ -136,14 +132,13 @@ struct ThemeCatalogSyncTests { #expect(harness.tombstoned.isEmpty) } - @Test("Loading the folder and reloading on activation mark nothing") + @Test("Loading the folder and reloading it mark nothing") func loadsMarkNothing() throws { let harness = try Harness() defer { harness.remove() } try harness.write(try Self.theme("user.existing")) harness.reopen() - harness.center.post(name: NSApplication.didBecomeActiveNotification, object: nil) harness.catalog.reloadSynchronously() #expect(harness.catalog.theme(id: "user.existing") != nil) @@ -646,7 +641,6 @@ struct ThemeCatalogSyncTests { let catalog = ThemeCatalog( userDirectory: folder, bundleDirectory: nil, - notificationCenter: NotificationCenter(), syncTracker: SyncChangeTracker(metadataStorage: SyncMetadataStorage(userDefaults: suite.defaults)) ) diff --git a/apple/TableProTests/Theme/ThemeCatalogTests.swift b/apple/TableProTests/Theme/ThemeCatalogTests.swift index 2fbee6b303..02cf1822f0 100644 --- a/apple/TableProTests/Theme/ThemeCatalogTests.swift +++ b/apple/TableProTests/Theme/ThemeCatalogTests.swift @@ -8,9 +8,10 @@ // A file removed outside TablePro is a conflict too: a save must never bring it back. // -import AppKit +import Combine import Foundation @testable import TablePro +import TableProTestSupport import TableProTheme import Testing @@ -18,7 +19,6 @@ import Testing struct ThemeCatalogTests { private struct Sandbox { let root: URL - let center = NotificationCenter() var themes: URL { root.appendingPathComponent("Themes", isDirectory: true) } var incoming: URL { root.appendingPathComponent("Incoming", isDirectory: true) } var trash: URL { root.appendingPathComponent("Trash", isDirectory: true) } @@ -43,7 +43,7 @@ struct ThemeCatalogTests { @MainActor func catalog() -> ThemeCatalog { - ThemeCatalog(userDirectory: themes, bundleDirectory: nil, notificationCenter: center, fileTrash: fileTrash) + ThemeCatalog(userDirectory: themes, bundleDirectory: nil, fileTrash: fileTrash) } @discardableResult @@ -153,7 +153,7 @@ struct ThemeCatalogTests { #expect(try sandbox.fileNames().isEmpty) } - @Test("After an activation reload drops a deleted theme, a stale save is still refused") + @Test("After a reload drops a deleted theme, a stale save is still refused") func saveAfterReloadRefusesADeletedTheme() throws { let sandbox = try Sandbox() defer { sandbox.remove() } @@ -162,7 +162,7 @@ struct ThemeCatalogTests { let stale = try #require(catalog.theme(id: "user.mine")) try sandbox.delete("Mine.json") - sandbox.center.post(name: NSApplication.didBecomeActiveNotification, object: nil) + catalog.reloadSynchronously() #expect(catalog.theme(id: "user.mine") == nil) #expect(throws: ThemeLoadError.deletedOnDisk("Mine.json")) { try catalog.save(stale) } @@ -199,22 +199,60 @@ struct ThemeCatalogTests { #expect(catalog.theme(id: "user.mine") == nil) } - // MARK: - Activation + // MARK: - Watching the folder - @Test("The catalog reloads when TablePro becomes active") - func reloadsOnActivation() throws { + @Test("A watching catalog picks up a theme dropped into its folder, edited there and removed from it") + func picksUpChangesInItsFolder() async throws { + let sandbox = try Sandbox() + defer { sandbox.remove() } + let catalog = sandbox.catalog() + catalog.startWatchingUserDirectory() + defer { catalog.stopWatchingUserDirectory() } + + try sandbox.write(try Self.userTheme(id: "user.dropped", name: "Dropped"), as: "Dropped.json") + try await waitUntil { catalog.theme(id: "user.dropped")?.name == "Dropped" } + + try sandbox.write(try Self.userTheme(id: "user.dropped", name: "Edited"), as: "Dropped.json") + try await waitUntil { catalog.theme(id: "user.dropped")?.name == "Edited" } + + try sandbox.delete("Dropped.json") + try await waitUntil { catalog.theme(id: "user.dropped") == nil } + } + + @Test("A catalog that is not watching keeps its list until it reloads") + func unwatchedCatalogWaitsForAReload() throws { let sandbox = try Sandbox() defer { sandbox.remove() } let catalog = sandbox.catalog() - #expect(catalog.theme(id: "user.dropped") == nil) try sandbox.write(try Self.userTheme(id: "user.dropped", name: "Dropped"), as: "Dropped.json") #expect(catalog.theme(id: "user.dropped") == nil) - sandbox.center.post(name: NSApplication.didBecomeActiveNotification, object: nil) + catalog.reloadSynchronously() #expect(catalog.theme(id: "user.dropped")?.name == "Dropped") } + // The catalog's own save reports a change in its folder too. + @Test("A reload after the catalog's own save publishes nothing") + func reloadAfterOwnSavePublishesNothing() throws { + let sandbox = try Sandbox() + defer { sandbox.remove() } + let catalog = sandbox.catalog() + var published = 0 + let subscription = catalog.$themes.dropFirst().sink { _ in published += 1 } + defer { subscription.cancel() } + + try catalog.save(try Self.userTheme(id: "user.saved", name: "Saved", overrides: [.editorBackground: "#112233"])) + #expect(published == 1) + let rejected = catalog.rejected + + catalog.reloadSynchronously() + + #expect(published == 1) + #expect(catalog.rejected == rejected) + #expect(catalog.storedTheme(id: "user.saved")?.overrides[.editorBackground]?.hex == "#112233") + } + // MARK: - Trash @Test("Move to Trash moves the file it was loaded from, and restore brings it back") @@ -485,7 +523,7 @@ struct ThemeCatalogTests { try sandbox.write(try Self.userTheme(id: "user.aardvark", name: "Aardvark"), as: "aardvark.json") let bundle = try #require(Bundle.main.resourceURL) - let catalog = ThemeCatalog(userDirectory: sandbox.themes, bundleDirectory: bundle, notificationCenter: sandbox.center) + let catalog = ThemeCatalog(userDirectory: sandbox.themes, bundleDirectory: bundle) let ids = catalog.themes.map(\.id) #expect(Array(ids.prefix(2)) == [BuiltInThemes.defaultLightId, BuiltInThemes.defaultDarkId]) @@ -495,6 +533,32 @@ struct ThemeCatalogTests { #expect(catalog.rejected.isEmpty) } + @Test("Bundled themes are read once: a reload keeps them as first read, whatever their files hold now") + func bundledThemesAreReadOnce() throws { + let sandbox = try Sandbox() + defer { sandbox.remove() } + let bundle = sandbox.root.appendingPathComponent("Bundle", isDirectory: true) + try FileManager.default.createDirectory(at: bundle, withIntermediateDirectories: true) + let original = try Self.userTheme(id: "tablepro.fixture", name: "Fixture", overrides: [.editorBackground: "#102030"]) + let file = try sandbox.write(original, as: "tablepro.fixture.json", in: bundle) + + let catalog = ThemeCatalog(userDirectory: sandbox.themes, bundleDirectory: bundle, fileTrash: sandbox.fileTrash) + #expect(catalog.theme(id: "tablepro.fixture") == original) + + try sandbox.write( + try Self.userTheme(id: "tablepro.fixture", name: "Changed", overrides: [.editorBackground: "#FFFFFF"]), + as: "tablepro.fixture.json", + in: bundle + ) + catalog.reloadSynchronously() + #expect(catalog.theme(id: "tablepro.fixture") == original) + + try FileManager.default.removeItem(at: file) + catalog.reloadSynchronously() + #expect(catalog.theme(id: "tablepro.fixture") == original) + #expect(catalog.rejected.isEmpty) + } + // MARK: - Duplicate @Test("Duplicate gives a new user id and a Finder-style name") diff --git a/apple/TableProTests/Theme/ThemeEngineTests.swift b/apple/TableProTests/Theme/ThemeEngineTests.swift index 623bbb49bb..c029caca69 100644 --- a/apple/TableProTests/Theme/ThemeEngineTests.swift +++ b/apple/TableProTests/Theme/ThemeEngineTests.swift @@ -8,7 +8,6 @@ // change subject, so nothing the app draws with moves. // -import AppKit import Combine import Foundation @testable import TablePro @@ -21,7 +20,6 @@ struct ThemeEngineTests { @MainActor private final class Harness { let root: URL - let center = NotificationCenter() let changes = PassthroughSubject() let catalog: ThemeCatalog let engine: ThemeEngine @@ -44,7 +42,6 @@ struct ThemeEngineTests { catalog = ThemeCatalog( userDirectory: folder, bundleDirectory: nil, - notificationCenter: center, fileTrash: ThemeFileTrash { url in let destination = trash.appendingPathComponent(url.lastPathComponent) try FileManager.default.moveItem(at: url, to: destination) @@ -113,17 +110,18 @@ struct ThemeEngineTests { try await waitUntil { harness.engine.pair.dark == night } } - @Test("A file changed on disk repaints once TablePro becomes active") - func activationReloadRepaints() async throws { + @Test("A file changed on disk repaints once the watching catalog sees the change") + func diskChangeRepaints() async throws { let night = try Self.theme("user.night", .dark, background: "#101010") let harness = try Harness(themes: [night]) defer { harness.remove() } harness.engine.select(lightThemeId: BuiltInThemes.defaultLightId, darkThemeId: night.id) + harness.catalog.startWatchingUserDirectory() + defer { harness.catalog.stopWatchingUserDirectory() } var fixed = night fixed.overrides[.editorBackground] = try ThemeColor(validating: "#303030") try ThemeFileFormat.encode(fixed).write(to: harness.themes.appendingPathComponent("\(night.id).json")) - harness.center.post(name: NSApplication.didBecomeActiveNotification, object: nil) try await waitUntil { harness.engine.pair.dark == fixed } } diff --git a/apple/TableProTests/ViewModels/ServerDashboardViewModelTests.swift b/apple/TableProTests/ViewModels/ServerDashboardViewModelTests.swift index 1c394ac653..aedc5fa001 100644 --- a/apple/TableProTests/ViewModels/ServerDashboardViewModelTests.swift +++ b/apple/TableProTests/ViewModels/ServerDashboardViewModelTests.swift @@ -257,6 +257,41 @@ struct ServerDashboardViewModelTests { #expect(vm.sessionSelection.ids == ["10"]) } + @Test("A hidden, minimized or covered window stops the polling, and showing it again resumes") + func hiddenWindowStopsPolling() { + let vm = makeViewModel(databaseType: .mysql) + vm.startAutoRefresh() + #expect(vm.isPolling) + #expect(vm.tileBoard.isActive) + + vm.windowVisibilityChanged(false) + #expect(!vm.isPolling) + #expect(!vm.tileBoard.isActive) + + vm.windowVisibilityChanged(true) + #expect(vm.isPolling) + #expect(vm.tileBoard.isActive) + vm.stopAutoRefresh() + #expect(!vm.isPolling) + } + + @Test("A window hidden before the dashboard appears keeps it from polling") + func hiddenBeforeAppearStaysIdle() { + let vm = makeViewModel(databaseType: .mysql) + vm.windowVisibilityChanged(false) + vm.startAutoRefresh() + #expect(!vm.isPolling) + #expect(!vm.tileBoard.isActive) + vm.stopAutoRefresh() + } + + @Test("Changing the interval while the dashboard is off screen starts no polling") + func intervalChangeOffScreenStaysIdle() { + let vm = makeViewModel(databaseType: .mysql) + vm.refreshInterval = .tenSeconds + #expect(!vm.isPolling) + } + @Test("stopAutoRefresh clears the refreshing flag") func stopAutoRefreshClearsRefreshingFlag() { let vm = makeViewModel(databaseType: .mysql) diff --git a/apple/TableProTests/Views/Main/SchemaChangeTokenGateTests.swift b/apple/TableProTests/Views/Main/SchemaChangeTokenGateTests.swift new file mode 100644 index 0000000000..6856d43f14 --- /dev/null +++ b/apple/TableProTests/Views/Main/SchemaChangeTokenGateTests.swift @@ -0,0 +1,31 @@ +@testable import TablePro +import Testing + +struct SchemaChangeTokenGateTests { + private static func admissions(_ tokens: [String?], forgettingAfter forgetIndex: Int? = nil) -> [Bool] { + var gate = SchemaChangeTokenGate() + var admitted: [Bool] = [] + for (index, token) in tokens.enumerated() { + admitted.append(gate.admits(token)) + if index == forgetIndex { + gate.forget() + } + } + return admitted + } + + @Test("A write that leaves the schema token alone reloads nothing after the first reload") + func unchangedTokenIsSkipped() { + #expect(Self.admissions(["12", "12", "12"]) == [true, false, false]) + } + + @Test("A moved token reloads, and so does every event of an engine with no token") + func movedOrMissingTokenReloads() { + #expect(Self.admissions(["12", "13", nil, nil, "13"]) == [true, true, true, true, true]) + } + + @Test("A failed reload forgets the token, so the next write reloads again") + func forgottenTokenReloads() { + #expect(Self.admissions(["12", "12"], forgettingAfter: 0) == [true, true]) + } +} diff --git a/apple/TableProTests/Views/Settings/ThemeLibraryTestFixture.swift b/apple/TableProTests/Views/Settings/ThemeLibraryTestFixture.swift index e109e82e9a..81d11ac545 100644 --- a/apple/TableProTests/Views/Settings/ThemeLibraryTestFixture.swift +++ b/apple/TableProTests/Views/Settings/ThemeLibraryTestFixture.swift @@ -38,7 +38,6 @@ final class ThemeLibraryTestFixture { catalog = ThemeCatalog( userDirectory: folders[0], bundleDirectory: nil, - notificationCenter: center, fileTrash: .folder(folders[1]), syncTracker: tracker ) diff --git a/apple/changelog.d/4229.changed.1.md b/apple/changelog.d/4229.changed.1.md new file mode 100644 index 0000000000..13afcbf59e --- /dev/null +++ b/apple/changelog.d/4229.changed.1.md @@ -0,0 +1 @@ +Switching back to TablePro syncs with iCloud at most once a minute. diff --git a/apple/changelog.d/4229.changed.2.md b/apple/changelog.d/4229.changed.2.md new file mode 100644 index 0000000000..dd119f8563 --- /dev/null +++ b/apple/changelog.d/4229.changed.2.md @@ -0,0 +1 @@ +A theme edited in Finder or another app loads as soon as it is saved. diff --git a/apple/changelog.d/4229.changed.md b/apple/changelog.d/4229.changed.md new file mode 100644 index 0000000000..2465f5018b --- /dev/null +++ b/apple/changelog.d/4229.changed.md @@ -0,0 +1 @@ +Opening a remote database file over SFTP downloads it several times faster on high-latency links. diff --git a/apple/changelog.d/4229.fixed.1.md b/apple/changelog.d/4229.fixed.1.md new file mode 100644 index 0000000000..1c91c84e53 --- /dev/null +++ b/apple/changelog.d/4229.fixed.1.md @@ -0,0 +1 @@ +Every write another app made to a SQLite file reloaded the sidebar, even when the schema stayed the same. diff --git a/apple/changelog.d/4229.fixed.2.md b/apple/changelog.d/4229.fixed.2.md new file mode 100644 index 0000000000..30c8bf9d2e --- /dev/null +++ b/apple/changelog.d/4229.fixed.2.md @@ -0,0 +1 @@ +The Server Dashboard kept querying the server while its window was minimized, hidden or covered. diff --git a/apple/changelog.d/4229.fixed.3.md b/apple/changelog.d/4229.fixed.3.md new file mode 100644 index 0000000000..04050f3906 --- /dev/null +++ b/apple/changelog.d/4229.fixed.3.md @@ -0,0 +1 @@ +Each switch back to TablePro re-read every theme file and ran Git in each linked SQL folder. diff --git a/apple/changelog.d/4229.fixed.4.md b/apple/changelog.d/4229.fixed.4.md new file mode 100644 index 0000000000..c9a3d7f483 --- /dev/null +++ b/apple/changelog.d/4229.fixed.4.md @@ -0,0 +1 @@ +Saving any file in a linked SQL folder re-read every SQL file in it. diff --git a/apple/changelog.d/4229.fixed.5.md b/apple/changelog.d/4229.fixed.5.md new file mode 100644 index 0000000000..29824bd092 --- /dev/null +++ b/apple/changelog.d/4229.fixed.5.md @@ -0,0 +1 @@ +Snowflake leaked two network sessions each time a pooled connection opened. diff --git a/apple/changelog.d/4229.fixed.md b/apple/changelog.d/4229.fixed.md new file mode 100644 index 0000000000..6d24895ce0 --- /dev/null +++ b/apple/changelog.d/4229.fixed.md @@ -0,0 +1 @@ +An open SSH tunnel woke the Mac five times a second while TablePro sat idle. diff --git a/apple/scripts/probes/check-sftp-read-ahead.sh b/apple/scripts/probes/check-sftp-read-ahead.sh new file mode 100755 index 0000000000..ba322b0560 --- /dev/null +++ b/apple/scripts/probes/check-sftp-read-ahead.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# Measures how the buffer passed to libssh2_sftp_read sets SFTP download speed over a link with +# latency, against the vendored libssh2 and a real OpenSSH server. +# +# libssh2 keeps buffer_size * 4 bytes of FXP_READ requests in flight (sftp_read in src/sftp.c, +# capped at four channel windows), so a download moves about four buffers per round trip. On +# loopback that never shows. LibSSH2SFTPSession sizes its download buffer on this; rerun after a +# libssh2 bump. +# +# Usage: +# scripts/probes/check-sftp-read-ahead.sh [delay-ms] [size-mb] # default 50 ms, 32 MB +# +# Needs Docker. Starts a throwaway OpenSSH container with NET_ADMIN, delays every packet the server +# sends with tc netem, which adds that much to each round trip, and removes the container on exit. Exits non-zero if a 2 MB buffer is not at least three times +# faster than a 32 KB one. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=../lib/openssh-fixture.sh +source "$ROOT/scripts/lib/openssh-fixture.sh" + +DELAY_MS="${1:-50}" +SIZE_MB="${2:-32}" +CONTAINER="tp-sftp-read-ahead" +PORT="${TABLEPRO_SFTP_READ_AHEAD_PORT:-22023}" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"; docker rm -f -v "$CONTAINER" > /dev/null 2>&1 || true' EXIT + +docker rm -f -v "$CONTAINER" > /dev/null 2>&1 || true +docker run -d --name "$CONTAINER" --cap-add NET_ADMIN \ + -p "127.0.0.1:$PORT:2222" \ + -e USER_NAME=tp -e USER_PASSWORD=tppass -e PASSWORD_ACCESS=true -e SUDO_ACCESS=false \ + "$OPENSSH_FIXTURE_IMAGE_10" > /dev/null +openssh_wait_for_banner 127.0.0.1 "$PORT" 60 || { echo "no SSH banner within 60s" >&2; exit 2; } + +docker exec "$CONTAINER" sh -c "apk add --no-cache iproute2 > /dev/null && tc qdisc add dev eth0 root netem delay ${DELAY_MS}ms" +docker exec "$CONTAINER" sh -c "head -c $((SIZE_MB * 1024 * 1024)) /dev/urandom > /config/read-ahead.bin && chmod 644 /config/read-ahead.bin" + +cat > "$WORK/probe.c" << 'PROBE' +#include +#include +#include +#include +#include +#include +#include +#include +#include "libssh2.h" +#include "libssh2_sftp.h" + +static double download(int port, size_t buffer_size, size_t *bytes) { + int sock = socket(AF_INET, SOCK_STREAM, 0); + struct sockaddr_in a; + memset(&a, 0, sizeof(a)); + a.sin_family = AF_INET; + a.sin_port = htons((unsigned short)port); + a.sin_addr.s_addr = inet_addr("127.0.0.1"); + if (connect(sock, (struct sockaddr *)&a, sizeof(a))) { puts("tcp connect failed"); exit(2); } + LIBSSH2_SESSION *ses = libssh2_session_init(); + if (libssh2_session_handshake(ses, sock)) { puts("handshake failed"); exit(2); } + if (libssh2_userauth_password(ses, "tp", "tppass")) { puts("auth failed"); exit(2); } + LIBSSH2_SFTP *sftp = libssh2_sftp_init(ses); + if (!sftp) { puts("sftp refused"); exit(2); } + const char *path = "/config/read-ahead.bin"; + LIBSSH2_SFTP_HANDLE *h = libssh2_sftp_open_ex(sftp, path, (unsigned)strlen(path), LIBSSH2_FXF_READ, 0, LIBSSH2_SFTP_OPENFILE); + if (!h) { puts("open failed"); exit(2); } + char *buffer = malloc(buffer_size); + struct timespec start, end; + clock_gettime(CLOCK_MONOTONIC, &start); + *bytes = 0; + for (;;) { + ssize_t n = libssh2_sftp_read(h, buffer, buffer_size); + if (n == 0) break; + if (n < 0) { printf("read failed %zd\n", n); exit(2); } + *bytes += (size_t)n; + } + clock_gettime(CLOCK_MONOTONIC, &end); + free(buffer); + libssh2_sftp_close_handle(h); + libssh2_sftp_shutdown(sftp); + libssh2_session_disconnect(ses, "done"); + libssh2_session_free(ses); + close(sock); + return (double)(end.tv_sec - start.tv_sec) + (double)(end.tv_nsec - start.tv_nsec) / 1e9; +} + +int main(int argc, char **argv) { + int port = atoi(argv[1]); + libssh2_init(0); + size_t sizes[] = { 32 * 1024, 256 * 1024, 1024 * 1024, 2 * 1024 * 1024 }; + double rates[4]; + for (int i = 0; i < 4; i++) { + size_t bytes = 0; + double seconds = download(port, sizes[i], &bytes); + rates[i] = (double)bytes / seconds / (1024.0 * 1024.0); + printf("buffer %5zu KB: %zu bytes in %.2f s, %.1f MB/s\n", sizes[i] / 1024, bytes, seconds, rates[i]); + } + libssh2_exit(); + return rates[3] >= rates[0] * 3 ? 0 : 1; +} +PROBE + +clang -w -arch arm64 -o "$WORK/probe" "$WORK/probe.c" \ + -I "$ROOT/TablePro/Core/SSH/CLibSSH2/include" \ + "$ROOT/Libs/libssh2_arm64.a" "$ROOT/Libs/libssl_arm64.a" "$ROOT/Libs/libcrypto_arm64.a" \ + -lz -framework Security -framework CoreFoundation + +echo "libssh2 SFTP download, ${DELAY_MS} ms added to each round trip, ${SIZE_MB} MB file:" +"$WORK/probe" "$PORT"