Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
2 changes: 1 addition & 1 deletion .claude/rules/data-sync-storage.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ paths:
- "apple/TablePro/Core/RoutineDebugging/RoutineDebugOptionStore.swift"
- "apple/TablePro/Core/RowGeneration/RowGenerationModel.swift"
- "apple/TablePro/Core/SSH/HostKeyStore.swift"
- "apple/TablePro/Core/Services/Export/ImportColumnMatcher.swift"
- "apple/TablePro/Core/Transfer/Shared/ImportColumnMatcher.swift"
- "apple/TablePro/Core/Services/Formatting/ValueDisplayFormat.swift"
- "apple/TablePro/Core/Services/TeamLibrary/TeamLibraryState.swift"
- "apple/TablePro/Core/Services/Account/*Store.swift"
Expand Down
14 changes: 14 additions & 0 deletions apple/Packages/TableProCore/Package.swift
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ let package = Package(
.library(name: "TableProR2SQLCore", targets: ["TableProR2SQLCore"]),
.library(name: "TableProConnectionLibrary", targets: ["TableProConnectionLibrary"]),
.library(name: "TableProSQLGrammar", targets: ["TableProSQLGrammar"]),
.library(name: "TableProSQLFile", targets: ["TableProSQLFile"]),
.library(name: "TableProSQLQuoting", targets: ["TableProSQLQuoting"]),
.library(name: "TableProProcess", targets: ["TableProProcess"]),
.library(name: "TableProSQLVocabulary", targets: ["TableProSQLVocabulary"]),
Expand Down Expand Up @@ -268,6 +269,13 @@ let package = Package(
dependencies: ["TableProSQLQuoting"],
path: "Sources/TableProSQLGrammar"
),
// Reads a .sql file for both apps' imports, so never TableProPluginKit.
.target(
name: "TableProSQLFile",
dependencies: ["TableProSQLGrammar", "TableProTabularIO"],
path: "Sources/TableProSQLFile",
swiftSettings: pluginCoreSettings
),
// Linked by PluginKit, plugin bundles and both apps, so it depends on nothing.
.target(
name: "TableProSQLQuoting",
Expand Down Expand Up @@ -527,6 +535,12 @@ let package = Package(
dependencies: ["TableProSQLGrammar"],
path: "Tests/TableProSQLGrammarTests"
),
.testTarget(
name: "TableProSQLFileTests",
dependencies: ["TableProSQLFile", "TableProSQLGrammar", "TableProTabularIO"],
path: "Tests/TableProSQLFileTests",
swiftSettings: pluginCoreSettings
),
.testTarget(
name: "TableProSQLQuotingTests",
dependencies: ["TableProSQLQuoting", "TableProSQLGrammar", "CSQLite"],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -154,4 +154,9 @@ public enum CassandraObjectQueries {
USING '\(escapeLiteral(className))';
"""
}

// A CQL trigger is named per table, so the drop needs its ON clause.
public static func dropTriggerStatement(name: String, keyspace: String, table: String) -> String {
"DROP TRIGGER IF EXISTS \(quote(name)) ON \(quote(keyspace)).\(quote(table))"
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ public final class ConnectionManager: @unchecked Sendable {
private var dials: [UUID: SessionDial] = [:]
private var catalogs: [UUID: CatalogConnection] = [:]
private var catalogHolds: [UUID: Int] = [:]
private var transferDrivers: [UUID: [ObjectIdentifier: any DatabaseDriver]] = [:]
private var transferHolds: [UUID: Int] = [:]
private var transferHoldObservers: [@Sendable (UUID) -> Void] = []
private let now: @Sendable () -> Date

// The Mac's metadata pool values: an idle server connection is not held for good, and a failure is tried again.
Expand Down Expand Up @@ -235,13 +238,139 @@ public final class ConnectionManager: @unchecked Sendable {
return ids
}

// A connection a transfer holds keeps its file: the transfer's own background task keeps the process alive.
private func suspensionBlockingIds() -> [UUID] {
lock.lock()
defer { lock.unlock() }
let connected = sessions.filter { $0.value.driver.holdsSuspensionBlockingResource }.map(\.key)
let connected = sessions.filter { id, session in
session.driver.holdsSuspensionBlockingResource && transferHolds[id] == nil
}.map(\.key)
return Array(blockingTeardowns.union(connected))
}

// MARK: - Transfers

public func holdForTransfer(_ connectionId: UUID) -> TransferHold {
lock.lock()
transferHolds[connectionId, default: 0] += 1
lock.unlock()
return TransferHold { [weak self] in self?.releaseTransferHold(connectionId) }
}

// Called once the last hold on a connection ends, so a pass that skipped it can release it now.
public func observeTransferHoldsEnded(_ observer: @escaping @Sendable (UUID) -> Void) {
lock.lock()
defer { lock.unlock() }
transferHoldObservers.append(observer)
}

private func releaseTransferHold(_ connectionId: UUID) {
lock.lock()
let left = transferHolds[connectionId, default: 1] - 1
transferHolds[connectionId] = left > 0 ? left : nil
let observers = left > 0 ? [] : transferHoldObservers
lock.unlock()
for observer in observers {
observer(connectionId)
}
}

// A driver of the transfer's own, so a long read never shares the session the grid streams on. It is the session's
// driver itself when the engine has no second connection to give now (a SQLite file with a transaction open, an
// in-memory database). `closeDedicatedDriver` ends it; a disconnect or a catalog hold ends it first.
public func openDedicatedDriver(for connectionId: UUID, database: String?) async throws -> any DatabaseDriver {
guard let (session, dial) = sessionAndDial(for: connectionId) else { throw ConnectionError.notConnected }
guard !holdsCatalogs(connectionId) else { throw TransferConnectionError.connectionHeld }
let driver: any DatabaseDriver
if let owner = session.driver as? DedicatedTransferConnection {
driver = try await owner.openTransferDriver()
guard driver !== session.driver else { return driver }
} else {
driver = try await Self.openTransferDriver(
dial: dial, database: database ?? "", factory: driverFactory, secureStore: secureStore
)
}
guard registerTransferDriver(driver, for: connectionId, session: session) else {
try? await driver.disconnect()
throw holdsCatalogs(connectionId) ? TransferConnectionError.connectionHeld : ConnectionError.notConnected
}
return driver
}

// Leaves the session's own driver alone.
public func closeDedicatedDriver(_ driver: any DatabaseDriver, for connectionId: UUID) async {
guard takeTransferDriver(driver, for: connectionId) else { return }
try? await driver.disconnect()
}

private func sessionAndDial(for connectionId: UUID) -> (ConnectionSession, SessionDial)? {
lock.lock()
defer { lock.unlock() }
guard let session = sessions[connectionId], let dial = dials[connectionId] else { return nil }
return (session, dial)
}

private func holdsCatalogs(_ connectionId: UUID) -> Bool {
lock.lock()
defer { lock.unlock() }
return catalogHolds[connectionId] != nil
}

// Refused once the session it was opened for has gone or a hold began while it opened.
private func registerTransferDriver(
_ driver: any DatabaseDriver,
for connectionId: UUID,
session: ConnectionSession
) -> Bool {
lock.lock()
defer { lock.unlock() }
guard catalogHolds[connectionId] == nil, let current = sessions[connectionId],
current.driver === session.driver else { return false }
transferDrivers[connectionId, default: [:]][ObjectIdentifier(driver)] = driver
return true
}

private func takeTransferDriver(_ driver: any DatabaseDriver, for connectionId: UUID) -> Bool {
lock.lock()
defer { lock.unlock() }
guard transferDrivers[connectionId]?.removeValue(forKey: ObjectIdentifier(driver)) != nil else { return false }
if transferDrivers[connectionId]?.isEmpty == true {
transferDrivers.removeValue(forKey: connectionId)
}
return true
}

private func takeTransferDrivers(for connectionId: UUID) -> [any DatabaseDriver] {
lock.lock()
defer { lock.unlock() }
return transferDrivers.removeValue(forKey: connectionId).map { Array($0.values) } ?? []
}

// A transfer reads as long as it takes: no catalog preparation (SQL Server's LOCK_TIMEOUT) and no query timeout,
// as the Mac lifts the timeout for an export.
private static func openTransferDriver(
dial: SessionDial,
database: String,
factory: DriverFactory,
secureStore: SecureStore
) async throws -> any DatabaseDriver {
var target = dial.effective
if !database.isEmpty {
target.database = database
}
target.queryTimeoutSeconds = nil
let stored = try secureStore.retrieve(forKey: passwordKey(for: dial.saved.id))
let password = try await factory.resolvePassword(for: dial.saved, stored: stored, secureStore: secureStore)
let driver = try factory.createDriver(
for: target,
password: password,
preTunnelHost: dial.preTunnelHost,
refreshPassword: passwordRefresh(for: dial.saved, stored: stored, factory: factory, secureStore: secureStore)
)
try await driver.connect()
return driver
}

/// Closes the tunnel before the driver, because a driver blocked reading through a tunnel the server
/// dropped only returns once its socket's peer closes; the reverse order queues `disconnect()` behind
/// a read that never completes. The lock is held past the `teardowns` write, so the task's own
Expand All @@ -262,10 +391,15 @@ public final class ConnectionManager: @unchecked Sendable {
if let catalog {
Task { await catalog.close() }
}
let transfers = transferDrivers.removeValue(forKey: connectionId).map { Array($0.values) } ?? []
Task { [weak self] in
if let sshProvider {
try? await sshProvider.closeTunnel(for: connectionId)
}
// Before the session: a DuckDB transfer connection reads the database the session's close ends.
for transfer in transfers {
try? await transfer.disconnect()
}
try? await session.driver.disconnect()
self?.retireTeardown(teardown.id, for: connectionId)
teardown.finish()
Expand Down Expand Up @@ -345,8 +479,12 @@ public final class ConnectionManager: @unchecked Sendable {

// SQL Server cannot drop or take single-user a database another connection sits in, so a run that may do either
// closes the catalog connection first and keeps it closed until the run ends.
// A transfer's own connection sits in the database too, so the hold ends it.
public func holdCatalogConnections(for connectionId: UUID) async {
await takeCatalogForHold(for: connectionId)?.close()
for transfer in takeTransferDrivers(for: connectionId) {
try? await transfer.disconnect()
}
}

public func releaseCatalogConnections(for connectionId: UUID) {
Expand Down Expand Up @@ -495,6 +633,46 @@ public final class ConnectionManager: @unchecked Sendable {
}
}

// A session driver whose engine gives no second connection through the factory makes the transfer's own: a SQLite
// reader beside a WAL session or over a snapshot, a DuckDB connection on the open database. It answers with itself
// when it has none to give now.
public protocol DedicatedTransferConnection: DatabaseDriver {
func openTransferDriver() async throws -> any DatabaseDriver
}

// Keeps the connection's file open through a background release while a transfer runs.
public final class TransferHold: @unchecked Sendable {
private let lock = NSLock()
private var onRelease: (@Sendable () -> Void)?

init(onRelease: @escaping @Sendable () -> Void) {
self.onRelease = onRelease
}

deinit {
release()
}

public func release() {
lock.lock()
let release = onRelease
onRelease = nil
lock.unlock()
release?()
}
}

public enum TransferConnectionError: Error, Equatable, Sendable, LocalizedError {
case connectionHeld

public var errorDescription: String? {
switch self {
case .connectionHeld:
return String(localized: "Another task needs this connection alone. Try again when it ends.")
}
}
}

/// One teardown, and the waiters parked on it.
///
/// `await task.value` on a `Task<_, Never>` ignores the awaiting task's cancellation entirely: measured,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,15 @@ public protocol DatabaseDriver: AnyObject, Sendable {
var settledLexicalFacts: SQLSessionLexicalFacts? { get }

func escapeStringLiteral(_ value: String) -> String

// One statement with bound values, sent the way this engine's macOS driver sends them. The default refuses:
// a literal fallback would quote binary and typed values differently from the Mac.
func executeParameterized(query: String, parameters: [PluginCellValue], typeNames: [String?]?) async throws
-> QueryResult

// The statements that turn the engine's foreign key checks off or back on for this session, empty where it
// has none.
func foreignKeyCheckStatements(enabled: Bool) -> [String]
}

public extension DatabaseDriver {
Expand Down Expand Up @@ -105,4 +114,11 @@ public extension DatabaseDriver {
func executeStreaming(query: String, options: StreamOptions = .default) -> AsyncThrowingStream<StreamElement, Error> {
QueryResultStreaming.stream(options: options) { try await self.execute(query: query) }
}

func executeParameterized(query: String, parameters: [PluginCellValue], typeNames: [String?]?) async throws
-> QueryResult {
throw DatabaseDriverCapabilityError.unsupported(.parameterizedStatements)
}

func foreignKeyCheckStatements(enabled: Bool) -> [String] { [] }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import Foundation
import TableProPluginKit

public enum DatabaseDriverCapabilityError: Error, Equatable, Sendable, LocalizedError {
public enum Capability: Sendable, Equatable {
case parameterizedStatements
case catalogQueries
case exportRows
}

case unsupported(Capability)

public var errorDescription: String? {
switch self {
case .unsupported(.parameterizedStatements):
return String(localized: "This connection cannot send statements with bound values.")
case .unsupported(.catalogQueries):
return String(localized: "This connection cannot read its catalog for an export.")
case .unsupported(.exportRows):
return String(localized: "This connection cannot stream rows to a file.")
}
}
}

// Raw catalog reads for the shared export catalogs: every row, bytes kept, no display truncation.
public protocol CatalogQuerying: DatabaseDriver {
func catalogQuery(_ sql: String, parameters: [PluginCellValue]) async throws -> PluginQueryResult
}

public extension CatalogQuerying {
func catalogQuery(_ sql: String) async throws -> PluginQueryResult {
try await catalogQuery(sql, parameters: [])
}
}

// Rows for a file export: pulled one batch at a time, bytes kept, no row cap, and a cancel ends the server read.
public protocol ExportRowStreaming: DatabaseDriver {
func exportRows(query: String, readOnly: Bool) -> AsyncThrowingStream<PluginStreamElement, Error>
}
Loading
Loading