diff --git a/apple/Plugins/MSSQLDriverPlugin/FreeTDSConnection.swift b/apple/Plugins/MSSQLDriverPlugin/FreeTDSConnection.swift index c2a30356ce..d59d08201a 100644 --- a/apple/Plugins/MSSQLDriverPlugin/FreeTDSConnection.swift +++ b/apple/Plugins/MSSQLDriverPlugin/FreeTDSConnection.swift @@ -143,7 +143,13 @@ nonisolated private func freetdsRecordMessage(_ message: MSSQLServerMessage, for return } let key = freetdsConnectionKey(dbproc) + // db-lib hands a login's messages the DBPROCESS it is still building, which has no gate until it is adopted and is + // freed when the login fails, so the first refusal goes where a failed open reads its reason. + let refusesLogin = message.isError && freetdsGate(forKey: key) == nil freetdsErrors.withLock { state in + if refusesLogin, state.global.isEmpty { + state.global = message.text + } state.perConnection[key, default: FreeTDSDiagnostics()].record(message) } } @@ -342,7 +348,7 @@ nonisolated final class FreeTDSConnection: @unchecked Sendable { defer { dbloginfree(login) } try configure(login) - let opened: UnsafeMutablePointer? + let opened: (proc: UnsafeMutablePointer?, refusal: String) do { opened = try freetdsConfigFile.withEntry( entry, @@ -350,7 +356,6 @@ nonisolated final class FreeTDSConnection: @unchecked Sendable { givingUpWhen: { attempt.isSettled } ) { guard !attempt.isSettled else { throw CancellationError() } - freetdsClearError(for: nil) return try open(login, serverName: entry.name, deadline: deadline) } } catch MSSQLFreeTDSConfigError.nameInUse { @@ -358,8 +363,8 @@ nonisolated final class FreeTDSConnection: @unchecked Sendable { } catch let error as MSSQLFreeTDSConfigError { throw MSSQLCoreError.connectionFailed(error.localizedDescription) } - guard let proc = opened else { - throw openFailure() + guard let proc = opened.proc else { + throw openFailure(detail: opened.refusal) } return proc } @@ -373,14 +378,16 @@ nonisolated final class FreeTDSConnection: @unchecked Sendable { } } + // A refused login's reason lands in process-wide state the next login clears, so it is read under the same lock. private func open( _ login: UnsafeMutablePointer, serverName: String, deadline: MSSQLConnectDeadline - ) throws -> UnsafeMutablePointer? { + ) throws -> (proc: UnsafeMutablePointer?, refusal: String) { Self.loginStartLock.lock() defer { Self.loginStartLock.unlock() } - return try withKerberosEnvironmentIfNeeded(deadline: deadline) { + freetdsClearError(for: nil) + let proc = try withKerberosEnvironmentIfNeeded(deadline: deadline) { let timeoutMilliseconds = deadline.remainingMilliseconds guard timeoutMilliseconds > 0 else { throw MSSQLCoreError.connectionTimedOut(isKerberos: options.authMethod == .windows) @@ -389,6 +396,7 @@ nonisolated final class FreeTDSConnection: @unchecked Sendable { _ = dbsetlogintime(seconds) return dbopen(login, serverName) } + return (proc, proc == nil ? freetdsGetError(for: nil) : "") } private func discardKerberosCache() { @@ -426,8 +434,7 @@ nonisolated final class FreeTDSConnection: @unchecked Sendable { } } - private func openFailure() -> MSSQLCoreError { - let detail = freetdsGetError(for: nil) + private func openFailure(detail: String) -> MSSQLCoreError { let msg = detail.isEmpty ? "Check host, port, credentials, and TLS settings" : detail if let kind = MSSQLTLSClassifier.classifySSLError(detail) { return .tlsHandshakeFailed(kind: kind, serverMessage: detail) diff --git a/apple/TablePro/AppDelegate.swift b/apple/TablePro/AppDelegate.swift index 87a96c7c4c..3d818d47ac 100644 --- a/apple/TablePro/AppDelegate.swift +++ b/apple/TablePro/AppDelegate.swift @@ -186,6 +186,8 @@ class AppDelegate: NSObject, NSApplicationDelegate { await AppModuleTermination.stopModuleWork() await MCPServerManager.shared.stop() await DatabaseManager.shared.runPostDisconnectScriptsBeforeQuit() + // A Vault lease outlives the app on the server until its TTL, so its database user goes now. + await VaultDatabaseCredentials.shared.revokeAllLeases() NSApp.reply(toApplicationShouldTerminate: true) } return .terminateLater diff --git a/apple/TablePro/Core/Database/ConnectionCredentialResolver.swift b/apple/TablePro/Core/Database/ConnectionCredentialResolver.swift index ee7d29c157..9cf22a2c33 100644 --- a/apple/TablePro/Core/Database/ConnectionCredentialResolver.swift +++ b/apple/TablePro/Core/Database/ConnectionCredentialResolver.swift @@ -29,12 +29,14 @@ enum ConnectionCredentialResolver { } // A nil username keeps the connection's own; Redshift IAM and a Vault database role hand back another. - // A sign-in that carries a lease belongs to whoever uses it, which releases it when done. + // A sign-in that carries a lease belongs to whoever uses it, which releases it when done. `leaseOwner` replaces + // `credentialOwner(for:)`, for a Vault user that must not be shared. static func resolveSignIn( for connection: DatabaseConnection, fields: [String: String], override: String? = nil, - deadline: ConnectionDeadline? = nil + deadline: ConnectionDeadline? = nil, + leaseOwner: UUID? = nil ) async throws -> ResolvedSignIn { if let minter = iamPasswordMinter(for: connection, fields: fields) { return try await minter.mint(deadline: deadline) @@ -43,7 +45,11 @@ enum ConnectionCredentialResolver { if KerberosAuthFields.isSelected(in: connection.additionalFields) { return ResolvedSignIn(username: nil, password: "") } - return try await resolveStoredSignIn(for: connection, override: override) + return try await resolveStoredSignIn( + for: connection, + override: override, + leaseOwner: leaseOwner ?? credentialOwner(for: connection) + ) } // A lease is a database user Vault creates, so a caller that only wants to know whether a password exists asks this. @@ -60,17 +66,21 @@ enum ConnectionCredentialResolver { return connection.passwordSource } - private static func resolveStoredSignIn(for connection: DatabaseConnection, override: String?) async throws -> ResolvedSignIn { + private static func resolveStoredSignIn( + for connection: DatabaseConnection, + override: String?, + leaseOwner: UUID + ) async throws -> ResolvedSignIn { if let override { return ResolvedSignIn(username: nil, password: override) } if case .profile(let profileId) = connection.credentialMode, let profile = CredentialProfileStorage.shared.profile(for: profileId) { - return try await resolveProfileSignIn(profile, for: connection) + return try await resolveProfileSignIn(profile, for: connection, leaseOwner: leaseOwner) } if let passwordSource = connection.passwordSource { guard ConnectionStorage.shared.storeIsTrusted else { throw PasswordSourceResolver.ResolutionError.storeNotTrusted } - return try await resolveSource(passwordSource, for: connection) + return try await resolveSource(passwordSource, for: connection, leaseOwner: leaseOwner) } if connection.usePgpass, readsPgpass(connection) { return ResolvedSignIn(username: nil, password: pgpassPassword(for: connection, username: resolveUsername(for: connection))) @@ -78,9 +88,18 @@ enum ConnectionCredentialResolver { return ResolvedSignIn(username: nil, password: ConnectionStorage.shared.loadPassword(for: connection.id) ?? "") } - private static func resolveSource(_ source: PasswordSource, for connection: DatabaseConnection) async throws -> ResolvedSignIn { + private static func resolveSource( + _ source: PasswordSource, + for connection: DatabaseConnection, + leaseOwner: UUID + ) async throws -> ResolvedSignIn { if case .vaultDatabaseRole(let role) = source { - return try await VaultDatabaseCredentials.shared.signIn(role, interactive: false) + return try await VaultDatabaseCredentials.shared.signIn( + role, + owner: leaseOwner, + for: connection.id, + interactive: false + ) } let password = try await PasswordSourceResolver.resolve(source, for: ConnectionScriptContext(connection: connection)) return ResolvedSignIn(username: nil, password: password) @@ -99,10 +118,8 @@ enum ConnectionCredentialResolver { return profile.username } - /// Which record a prompted password is remembered against. A profile answers for every - /// connection linked to it, so one profile in "ask every time" mode asks once per launch - /// rather than once per connection. - static func promptCacheKey(for connection: DatabaseConnection) -> UUID { + // A profile answers for every connection linked to it: one prompt per launch, one Vault user while its lease lives. + static func credentialOwner(for connection: DatabaseConnection) -> UUID { connection.credentialMode.profileId ?? connection.id } @@ -125,7 +142,8 @@ enum ConnectionCredentialResolver { private static func resolveProfileSignIn( _ profile: CredentialProfile, - for connection: DatabaseConnection + for connection: DatabaseConnection, + leaseOwner: UUID ) async throws -> ResolvedSignIn { switch profile.passwordMode { case .stored: @@ -141,7 +159,7 @@ enum ConnectionCredentialResolver { guard CredentialProfileStorage.shared.storeIsTrusted else { throw PasswordSourceResolver.ResolutionError.profileStoreNotTrusted } - return try await resolveSource(source, for: connection) + return try await resolveSource(source, for: connection, leaseOwner: leaseOwner) } } @@ -214,9 +232,9 @@ enum ConnectionCredentialResolver { struct ResolvedSignIn: Equatable, Sendable { let username: String? let password: String - let lease: VaultLease? + let lease: VaultLeaseHold? - init(username: String?, password: String, lease: VaultLease? = nil) { + init(username: String?, password: String, lease: VaultLeaseHold? = nil) { self.username = username self.password = password self.lease = lease diff --git a/apple/TablePro/Core/Database/DatabaseManager+Health.swift b/apple/TablePro/Core/Database/DatabaseManager+Health.swift index a82634ef22..676c27efac 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Health.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Health.swift @@ -216,7 +216,16 @@ extension DatabaseManager { || error is LocalCredentialConsentError { return .stop(ConnectionFailureClassifier.info(for: error)) } - guard isAuthenticationFailure(error) else { return .retry(nil) } + // The lease renewal stops on the same Vault errors; an unreachable or failing Vault is worth another try. + if error is VaultError || error is VaultRequestError { + let info = ConnectionFailureClassifier.info(for: error) + return VaultLease.isDefinitive(error) ? .stop(info) : .retry(info) + } + // The new user is kept, and the next attempt may find it loaded. + if error is VaultUserNotLoaded { + return .retry(ConnectionFailureClassifier.info(for: error)) + } + guard ConnectionFailureClassifier.isAuthenticationFailure(error) else { return .retry(nil) } let message = String(format: String(localized: "Reconnect failed: %@"), error.localizedDescription) return .stop(ConnectionFailureInfo(message: message)) } @@ -631,7 +640,7 @@ extension DatabaseManager { guard allowsCredentialPrompt else { return .fail } guard ConnectionCredentialResolver.promptsForPassword(session.connection), !pluginManager.hidesPassword(for: session.connection), - isAuthenticationFailure(error) + ConnectionFailureClassifier.isAuthenticationFailure(error) else { return .fail } @@ -652,29 +661,6 @@ extension DatabaseManager { return .retry(prompted) } - private static let invalidAuthorizationSQLState = "28000" - private static let mysqlAccessDeniedErrorCode = 1_045 - - internal func isAuthenticationFailure(_ error: Error) -> Bool { - if let pluginError = error as? any PluginDriverError { - if pluginError.pluginSqlState == Self.invalidAuthorizationSQLState { - return true - } - if pluginError.pluginErrorCode == Self.mysqlAccessDeniedErrorCode { - return true - } - return messageIndicatesAuthenticationFailure(pluginError.pluginErrorMessage) - } - return messageIndicatesAuthenticationFailure(error.localizedDescription) - } - - private func messageIndicatesAuthenticationFailure(_ message: String) -> Bool { - let lowered = message.lowercased() - return lowered.contains("access denied") - || lowered.contains("authentication failed") - || lowered.contains("invalid credentials") - } - // A superseded attempt leaves the slot alone: the tunnel under this connection's id is the newer attempt's. private func closeReconnectTunnels(for connection: DatabaseConnection, ownsAttempt: () -> Bool) async { guard ownsAttempt(), let tunnelManager = activeTunnelManager(for: connection) else { return } diff --git a/apple/TablePro/Core/Database/DatabaseManager+Queries.swift b/apple/TablePro/Core/Database/DatabaseManager+Queries.swift index 0dd30526dd..c22ebed391 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Queries.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Queries.swift @@ -80,6 +80,13 @@ extension DatabaseManager { } let tunnelManager = activeTunnelManager(for: connection) + // The form tests under a throwaway id, so a role of the connection's own leases a user no connect reuses. + defer { + let owner = ConnectionCredentialResolver.credentialOwner(for: connection) + if owner == connection.id, activeSessions[connection.id] == nil { + VaultDatabaseCredentials.shared.endUse(connection: connection.id, live: Set(activeSessions.keys)) + } + } let result: ConnectionTestResult do { diff --git a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift index 36c19de2ac..554da8ddd5 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Sessions.swift @@ -245,7 +245,7 @@ extension DatabaseManager { static func promptCacheKey(for connection: DatabaseConnection) -> PromptedPasswordKey { let sharedByProfile = connection.credentialMode.profileId != nil return PromptedPasswordKey( - owner: ConnectionCredentialResolver.promptCacheKey(for: connection), + owner: ConnectionCredentialResolver.credentialOwner(for: connection), destination: sharedByProfile ? "" : connection.destination.digest ) } @@ -833,6 +833,7 @@ extension DatabaseManager { "[close] disconnectSession driver.disconnect done connId=\(sessionId, privacy: .public) elapsedMs=\(Int(Date().timeIntervalSince(driverStart) * 1_000))" ) removeSessionEntry(for: sessionId) + VaultDatabaseCredentials.shared.endUse(connection: sessionId, live: Set(activeSessions.keys)) await SchemaService.shared.invalidate(connectionId: sessionId) await DatabaseTreeMetadataService.shared.handleDisconnect(connectionId: sessionId) diff --git a/apple/TablePro/Core/Database/DatabaseManager+Tunnel.swift b/apple/TablePro/Core/Database/DatabaseManager+Tunnel.swift index 93aead5136..7ab723f1bd 100644 --- a/apple/TablePro/Core/Database/DatabaseManager+Tunnel.swift +++ b/apple/TablePro/Core/Database/DatabaseManager+Tunnel.swift @@ -74,6 +74,7 @@ extension DatabaseManager { type: connection.type, sshConfig: SSHConfiguration(), sslConfig: tunnelSSL, + credentialMode: connection.credentialMode, passwordSource: connection.passwordSource, additionalFields: effectiveFields ) diff --git a/apple/TablePro/Core/Database/NativeDumpService.swift b/apple/TablePro/Core/Database/NativeDumpService.swift index 6e8bee1bfa..577d3e9b85 100644 --- a/apple/TablePro/Core/Database/NativeDumpService.swift +++ b/apple/TablePro/Core/Database/NativeDumpService.swift @@ -238,7 +238,8 @@ final class NativeDumpService: ObservableObject { private var cancelRequested = false private var preparation: Task? // The Vault lease the tool signs in with, held until the tool exits. - private var credentialLease: VaultLease? + private var credentialLease: VaultLeaseHold? + private var endsCredentialLease = false // The roots file the tool checks the server with, removed when the tool exits. private var systemTrustAnchors: SystemTrustAnchorFile? private let runningWork: RunningWorkRegistry @@ -397,11 +398,15 @@ final class NativeDumpService: ObservableObject { var effective = session?.effectiveConnection ?? connection let resolvedFields = DatabaseDriverFactory.resolvedAdditionalFields(for: effective) ?? effective.additionalFields + // A password on the tool's command line is readable while it runs, so that Vault user is the tool's alone and + // is revoked when it exits. + let exposesPassword = descriptor.exposesPasswordInArguments // The connect's own resolution, errors included: a Keychain read misses every other source. let signIn = try await ConnectionCredentialResolver.resolveSignIn( for: effective, fields: resolvedFields, - override: session?.cachedPassword + override: session?.cachedPassword, + leaseOwner: exposesPassword ? UUID() : nil ) // A tool signs in with the Keychain secrets the driver gets beside the password. if let token = resolvedFields[ConnectionSecretOptions.mongoSessionTokenFieldId], !token.isEmpty { @@ -416,6 +421,7 @@ final class NativeDumpService: ObservableObject { let password = signIn.password releaseRunResources() credentialLease = signIn.lease + endsCredentialLease = exposesPassword signIn.lease?.start { _ in } var jobStarted = false defer { @@ -479,7 +485,11 @@ final class NativeDumpService: ObservableObject { } private func releaseRunResources() { - credentialLease?.release() + if endsCredentialLease { + credentialLease?.end() + } else { + credentialLease?.release() + } credentialLease = nil systemTrustAnchors?.remove() systemTrustAnchors = nil diff --git a/apple/TablePro/Core/Database/Vault/VaultAuthenticator.swift b/apple/TablePro/Core/Database/Vault/VaultAuthenticator.swift index 1aabae8779..1108b7eb85 100644 --- a/apple/TablePro/Core/Database/Vault/VaultAuthenticator.swift +++ b/apple/TablePro/Core/Database/Vault/VaultAuthenticator.swift @@ -149,7 +149,7 @@ actor VaultAuthenticator { do { grant = try await VaultClient(endpoint: endpoint, transport: transport).lookupSelf(token: value) } catch let error as VaultRequestError { - throw VaultError.tokenRejected(source, error.message) + throw VaultError.signInFailure(error, at: endpoint, otherwise: .tokenRejected(source, error.message)) } // The user owns this token, so it is never renewed here. return VaultToken( @@ -176,7 +176,7 @@ actor VaultAuthenticator { grant = try await VaultClient(endpoint: endpoint, transport: transport) .appRoleLogin(mount: mount, roleId: roleId, secretId: secretId) } catch let error as VaultRequestError { - throw VaultError.tokenRejected(.appRole, error.message) + throw VaultError.signInFailure(error, at: endpoint, otherwise: .tokenRejected(.appRole, error.message)) } return token(from: grant, source: .appRole) } diff --git a/apple/TablePro/Core/Database/Vault/VaultClient.swift b/apple/TablePro/Core/Database/Vault/VaultClient.swift index cbc60ae066..8e8ac88fcc 100644 --- a/apple/TablePro/Core/Database/Vault/VaultClient.swift +++ b/apple/TablePro/Core/Database/Vault/VaultClient.swift @@ -74,6 +74,9 @@ struct VaultRequestError: Error, Equatable { var isPermissionDenied: Bool { status == 403 } + // Rate limited, sealed, or a proxy in front of it that cannot reach it: the same request can work in a moment. + var isTransient: Bool { [429, 502, 503, 504].contains(status) } + // Vault wraps several causes as "2 errors occurred:\n\t* permission denied\n\t* invalid token". var message: String { let lines = messages.flatMap { $0.components(separatedBy: .newlines) } diff --git a/apple/TablePro/Core/Database/Vault/VaultDatabaseCredentials.swift b/apple/TablePro/Core/Database/Vault/VaultDatabaseCredentials.swift index bdc0d04403..66ffd7b1cb 100644 --- a/apple/TablePro/Core/Database/Vault/VaultDatabaseCredentials.swift +++ b/apple/TablePro/Core/Database/Vault/VaultDatabaseCredentials.swift @@ -6,31 +6,41 @@ import Foundation import os -// Every call reads a new lease. The credential lives only in the returned value and the driver it configures. +// The credential lives only in memory: in the cached lease and the drivers that sign in with it. struct VaultDatabaseCredentials: Sendable { static let shared = VaultDatabaseCredentials() // A lease dies with the token that created it, so a minted token close to its end is renewed or replaced first. static let minimumTokenLife: Duration = .seconds(300) + // Quit waits this long for Vault to revoke the users still leased. + static let quitRevokeLimit: Duration = .seconds(3) private let authenticator: VaultAuthenticator private let transport: any VaultTransport private let clock: any Clock + private let leases: VaultLeaseCache - init( + init>( authenticator: VaultAuthenticator = .shared, transport: any VaultTransport = URLSessionVaultTransport.shared, - clock: any Clock = ContinuousClock() + clock: C = ContinuousClock() ) { self.authenticator = authenticator self.transport = transport self.clock = clock + self.leases = VaultLeaseCache(clock: clock) } - func signIn(_ role: VaultDatabaseRole, interactive: Bool) async throws -> ResolvedSignIn { + // `owner` is the credential profile, or the connection for a role of its own. One owner signs in as one user + // while its lease lives. `connection` is the one signing in, nil for a test of the source alone. + func signIn( + _ role: VaultDatabaseRole, + owner: UUID, + for connection: UUID?, + interactive: Bool + ) async throws -> ResolvedSignIn { let endpoint = try VaultEndpoint.resolve(address: role.address, environment: authenticator.currentEnvironment) let client = VaultClient(endpoint: endpoint, transport: transport) - let path = role.credentialsPath var token = try await authenticator.token( for: role.auth, @@ -42,6 +52,31 @@ struct VaultDatabaseCredentials: Sendable { await authenticator.forget(role.auth, endpoint: endpoint) token = try await authenticator.token(for: role.auth, endpoint: endpoint, interactive: interactive) } + let key = VaultLeaseCache.Key(owner: owner, role: role, endpoint: endpoint, token: token.value) + let issuing = token + let hold = try await leases.acquire(key, for: connection) { + try await readLease(role, endpoint: endpoint, client: client, token: issuing, interactive: interactive) + } + return ResolvedSignIn(username: hold.username, password: hold.password, lease: hold) + } + + func endUse(connection: UUID, live: Set) { + leases.endUse(connection: connection, live: live) + } + + func revokeAllLeases() async { + await leases.revokeAll(within: Self.quitRevokeLimit) + } + + private func readLease( + _ role: VaultDatabaseRole, + endpoint: VaultEndpoint, + client: VaultClient, + token initialToken: VaultToken, + interactive: Bool + ) async throws -> VaultLeaseCache.Issued { + let path = role.credentialsPath + var token = initialToken let credential: VaultLeasedCredential let issuedAt: ContinuousClock.Instant do { @@ -54,7 +89,7 @@ struct VaultDatabaseCredentials: Sendable { credential = try await Self.read(path: path, token: token.value, client: client) issuedAt = authenticator.currentInstant } catch let error as VaultRequestError { - throw VaultError.credentialsFailed(path: path, message: error.message) + throw Self.failure(error, path: path, client: client) } if token.source.isMinted, @@ -88,7 +123,12 @@ struct VaultDatabaseCredentials: Sendable { renewer: renewer, clock: clock ) - return ResolvedSignIn(username: credential.username, password: credential.password, lease: lease) + return VaultLeaseCache.Issued( + username: credential.username, + password: credential.password, + lease: lease, + token: token.value + ) } // Only a token TablePro minted bounds the lease's schedule. The user's own token is theirs to renew; rotating @@ -101,9 +141,13 @@ struct VaultDatabaseCredentials: Sendable { do { return try await client.readCredentials(path: path, token: token) } catch let error as VaultRequestError { - throw VaultError.credentialsFailed(path: path, message: error.message) + throw failure(error, path: path, client: client) } } + + private static func failure(_ error: VaultRequestError, path: String, client: VaultClient) -> VaultError { + .signInFailure(error, at: client.endpoint, otherwise: .credentialsFailed(path: path, message: error.message)) + } } // Renews and revokes with the token that issued the lease: Vault revokes a lease with that token, whatever token diff --git a/apple/TablePro/Core/Database/Vault/VaultError.swift b/apple/TablePro/Core/Database/Vault/VaultError.swift index b19321fa91..123b39465e 100644 --- a/apple/TablePro/Core/Database/Vault/VaultError.swift +++ b/apple/TablePro/Core/Database/Vault/VaultError.swift @@ -45,8 +45,21 @@ enum VaultError: LocalizedError, Equatable { case oidcCallbackPortInUse case credentialsFailed(path: String, message: String) case unreachable(String, String) + // Vault's own failure, such as a standby with no active node: worth another try. + case serverFailed(String) case malformedResponse + // A sign-in's Vault HTTP failure: a busy, sealed or failing Vault is worth another try, anything else is `refusal`. + static func signInFailure( + _ error: VaultRequestError, + at endpoint: VaultEndpoint, + otherwise refusal: VaultError + ) -> VaultError { + if error.isTransient { return .unreachable(endpoint.address.absoluteString, error.message) } + if error.status >= 500 { return .serverFailed(error.message) } + return refusal + } + var errorDescription: String? { switch self { case .noAddress: @@ -94,6 +107,8 @@ enum VaultError: LocalizedError, Equatable { return String(format: String(localized: "Vault could not issue credentials from %@: %@"), path, message) case let .unreachable(address, message): return String(format: String(localized: "Could not reach Vault at %@: %@"), address, message) + case let .serverFailed(message): + return String(format: String(localized: "Vault could not complete the request: %@"), message) case .malformedResponse: return String(localized: "Vault returned a response TablePro could not read.") } diff --git a/apple/TablePro/Core/Database/Vault/VaultLease.swift b/apple/TablePro/Core/Database/Vault/VaultLease.swift index 5547b63fdb..c27991a8af 100644 --- a/apple/TablePro/Core/Database/Vault/VaultLease.swift +++ b/apple/TablePro/Core/Database/Vault/VaultLease.swift @@ -58,8 +58,7 @@ struct VaultLeaseSchedule: Equatable, Sendable { } } -// Owned by whatever signed in with it, a driver or a backup job. Dropping the last reference revokes it too, so a -// lease a timed-out connect never adopted does not outlive it. +// Owned by `VaultLeaseCache`, which hands out holds. Dropping the last reference revokes it too. final class VaultLease: Sendable { let leaseId: String @@ -98,6 +97,9 @@ final class VaultLease: Sendable { var hasEnded: Bool { control.hasEnded } + // What `onEnding` gets when a renewal finds the lease gone: the time a running statement has to finish in. + var endingLead: Duration { schedule.lead } + // `onEnding` gets how long the lease has left, which is the time to sign in again in. func start(onEnding: @escaping @Sendable (Duration) -> Void) { let leaseId = leaseId @@ -130,6 +132,23 @@ final class VaultLease: Sendable { } } + func revoke() async { + guard control.release() else { return } + await renewer.revoke(leaseId: leaseId) + } + + // Vault Agent drops a cached lease once renewing it fails; this asks the same question on demand. Nil when the + // lease cannot be renewed or Vault gave no definite answer. + func stillExists(timeout: Duration) async -> Bool? { + guard schedule.renewable, !hasEnded else { return nil } + do { + _ = try await renewer.renew(leaseId: leaseId, increment: schedule.initialDuration, timeout: timeout) + return true + } catch { + return Self.isDefinitive(error) ? false : nil + } + } + // Returns when the lease is about to end and can no longer be renewed. private static func keepAlive( leaseId: String, @@ -227,8 +246,10 @@ final class VaultLease: Sendable { if let request = error as? VaultRequestError { return (400 ..< 500).contains(request.status) && request.status != 429 } - if case .unreachable = error as? VaultError { return false } - return error is VaultError + switch error as? VaultError { + case .unreachable, .serverFailed: return false + default: return error is VaultError + } } private final class Control: Sendable { diff --git a/apple/TablePro/Core/Database/Vault/VaultLeaseCache.swift b/apple/TablePro/Core/Database/Vault/VaultLeaseCache.swift new file mode 100644 index 0000000000..f38a31ad29 --- /dev/null +++ b/apple/TablePro/Core/Database/Vault/VaultLeaseCache.swift @@ -0,0 +1,499 @@ +// +// VaultLeaseCache.swift +// TablePro +// + +import CryptoKit +import Foundation +import os +import TableProPluginKit +import TableProUtilities + +// Every sign-in of one owner and role shares a lease, so a database still loading a new user meets the same user +// on the next attempt instead of yet another new one. +final class VaultLeaseCache: Sendable { + struct Key: Hashable, Sendable { + let owner: UUID + let role: VaultDatabaseRole + let endpoint: VaultEndpoint + // Vault Agent reuses a lease only for the token that read it; so does this. + let tokenDigest: Data + + init(owner: UUID, role: VaultDatabaseRole, endpoint: VaultEndpoint, token: String) { + self.owner = owner + self.role = role + self.endpoint = endpoint + self.tokenDigest = Data(SHA256.hash(data: Data(token.utf8))) + } + + func reading(with token: String) -> Key { + Key(owner: owner, role: role, endpoint: endpoint, token: token) + } + } + + struct Issued: Sendable { + let username: String + let password: String + let lease: VaultLease + // A re-login while reading leaves the lease with another token than the one the request was keyed by. + let token: String + } + + // An unused lease goes when the pool lets an unused connection go. + static let idleRetention: Duration = .seconds(MetadataConnectionPool.idleTimeout) + // Vitess, ProxySQL and replicas refuse a user Vault just created until they load it. HashiCorp's Terraform + // provider retries a new credential for the same 30 seconds, then gives up on it. + static let acceptanceWindow: Duration = .seconds(30) + // Vault's own default request timeout. + static let vaultRequestTimeout: Duration = .seconds(30) + + let clock: any Clock + // Starts timing a lease's age on `clock`. + private let stopwatch: @Sendable () -> @Sendable () -> Duration + private let state = OSAllocatedUnfairLock(initialState: State()) + + init>(clock: C = ContinuousClock()) { + self.clock = clock + stopwatch = { + let start = clock.now + return { start.duration(to: clock.now) } + } + } + + // `connection` is the connection that signs in, nil for a test of the source alone. + func acquire( + _ key: Key, + for connection: UUID?, + issue: @escaping @Sendable () async throws -> Issued + ) async throws -> VaultLeaseHold { + let lookup = state.withLock { state -> Lookup in + if let id = state.listed[key], let hold = attach(to: id, for: connection, joinedRead: false, in: &state) { + return .held(hold) + } + if var running = state.minting[key] { + running.waiters += 1 + state.minting[key] = running + return .reading(running.task) + } + let task = read(key, issue: issue) + state.minting[key] = Minting(task: task, waiters: 1) + return .reading(task) + } + switch lookup { + case .held(let hold): + return hold + case .reading(let task): + let id = try await task.value + // Only a quit that revoked everything takes a slot from under a sign-in that joined its read. + guard let hold = state.withLock({ attach(to: id, for: connection, joinedRead: true, in: &$0) }) else { + throw CancellationError() + } + return hold + } + } + + // The user ended `connection`. A lease no connection in `live` signed in with is no longer handed out, and goes + // once its last driver lets go, which a pooled read or a backup can do after this returns. + func endUse(connection: UUID, live: Set) { + let leases = state.withLock { state -> [VaultLease] in + let ended = state.slots.compactMap { id, slot -> UInt64? in + slot.users.contains(connection) ? id : nil + } + return ended.compactMap { id in + state.slots[id]?.users.remove(connection) + guard state.slots[id]?.users.isDisjoint(with: live) == true else { return nil } + unlist(id, in: &state) + return settle(id, in: &state) + } + } + leases.forEach { $0.release() } + } + + // A revoke sent from a detached task at quit never leaves the process, so these are awaited, within `limit`. + func revokeAll(within limit: Duration) async { + let leases = state.withLock { state -> [VaultLease] in + state.closed = true + return Array(state.slots.keys).compactMap { drop($0, in: &state) } + } + guard !leases.isEmpty else { return } + let clock = clock + await withTaskGroup(of: Void.self) { group in + group.addTask { + await withTaskGroup(of: Void.self) { revokes in + for lease in leases { + revokes.addTask { await lease.revoke() } + } + } + } + group.addTask { + try? await clock.sleep(for: limit) + } + await group.next() + group.cancelAll() + } + } + + // MARK: - Holds + + fileprivate func subscribe(slot id: UInt64, hold: UInt64, onEnding: @escaping @Sendable (Duration) -> Void) { + let ended = state.withLock { state -> Duration? in + guard state.slots[id]?.holds[hold] != nil else { return nil } + state.slots[id]?.holds[hold]?.onEnding = onEnding + return state.slots[id]?.ending + } + if let ended { onEnding(ended) } + } + + // `ending` revokes the lease too, unless another hold still uses it. + fileprivate func release(slot id: UInt64, hold: UInt64, ending: Bool) { + let lease = state.withLock { state -> VaultLease? in + guard state.slots[id]?.holds.removeValue(forKey: hold) != nil else { return nil } + if ending, state.slots[id]?.isIdle == true { + unlist(id, in: &state) + } + return settle(id, in: &state) + } + lease?.release() + } + + // A refusal alone cannot say whether Vault revoked the user or the database turned it away for its own reasons + // (another database, a privilege, a user it has not loaded), so Vault is asked. Only a lease it no longer has ends. + fileprivate func signInAccepted(slot id: UInt64) { + state.withLock { $0.slots[id]?.accepted = true } + } + + fileprivate func signInRefused(slot id: UInt64, credentials: Bool) { + let lease = state.withLock { state -> VaultLease? in + guard let slot = state.slots[id], slot.ending == nil else { return nil } + if credentials { + state.slots[id]?.credentialsRefused = true + } + guard !slot.asking else { return nil } + state.slots[id]?.asking = true + return slot.issued.lease + } + guard let lease else { return } + Task { [weak self] in + let exists = await lease.stillExists(timeout: Self.vaultRequestTimeout) + self?.answered(id, exists: exists) + } + } + + fileprivate func hasEnded(slot id: UInt64) -> Bool { + state.withLock { $0.slots[id]?.ending != nil } + } + + fileprivate func redialDelay(slot id: UInt64, afterFailures failures: Int) -> Duration? { + state.withLock { state -> Duration? in + guard let slot = state.slots[id], slot.ending == nil, !slot.issued.lease.hasEnded else { return nil } + let left = Self.acceptanceWindow - slot.age() + guard left > .zero else { return nil } + return min(.seconds(ExponentialBackoff.delay(for: failures)), left) + } + } + + // MARK: - State + + private struct Holder: Sendable { + var onEnding: (@Sendable (Duration) -> Void)? + } + + private struct Slot: Sendable { + let key: Key + let issued: Issued + let age: @Sendable () -> Duration + // Sign-ins that joined the read and have not taken their hold yet. + var joinedReads: Int + var holds: [UInt64: Holder] = [:] + // Connections that signed in with it and were not ended since. + var users: Set = [] + // Vault is being asked whether it still has the lease. + var asking = false + var accepted = false + var credentialsRefused = false + var ending: Duration? + var idleRelease: Task? + + var isIdle: Bool { holds.isEmpty && joinedReads == 0 } + } + + private struct Minting: Sendable { + let task: Task + var waiters: Int + } + + private enum Lookup: Sendable { + case held(VaultLeaseHold) + case reading(Task) + } + + private struct State: Sendable { + var slots: [UInt64: Slot] = [:] + var listed: [Key: UInt64] = [:] + var minting: [Key: Minting] = [:] + var nextId: UInt64 = 0 + // Set at quit: a lease read after the revoke would outlive the app. + var closed = false + } + + private func read(_ key: Key, issue: @escaping @Sendable () async throws -> Issued) -> Task { + Task { + let issued: Issued + do { + issued = try await issue() + } catch { + state.withLock { _ = $0.minting.removeValue(forKey: key) } + throw error + } + guard let (id, replaced) = register(issued, requestedAs: key) else { + issued.lease.release() + throw CancellationError() + } + replaced?.release() + hearEnding(of: issued.lease, slot: id) + return id + } + } + + private func hearEnding(of lease: VaultLease, slot id: UInt64) { + lease.start { [weak self] remaining in + self?.leaseEnding(id, remaining: remaining) + } + } + + private func register(_ issued: Issued, requestedAs requested: Key) -> (UInt64, replaced: VaultLease?)? { + let age = stopwatch() + return state.withLock { state in + let waiters = state.minting.removeValue(forKey: requested)?.waiters ?? 0 + guard !state.closed else { return nil } + let key = requested.reading(with: issued.token) + var replaced: VaultLease? + if let previous = state.listed[key] { + unlist(previous, in: &state) + replaced = settle(previous, in: &state) + } + state.nextId &+= 1 + let id = state.nextId + state.slots[id] = Slot(key: key, issued: issued, age: age, joinedReads: waiters) + state.listed[key] = id + return (id, replaced) + } + } + + // Nil for a slot that is gone, or, outside a joined read, for a lease that ended before its ending was handled. + private func attach(to id: UInt64, for connection: UUID?, joinedRead: Bool, in state: inout State) -> VaultLeaseHold? { + guard var slot = state.slots[id] else { return nil } + if joinedRead { + slot.joinedReads -= 1 + } else if slot.issued.lease.hasEnded { + return nil + } + slot.idleRelease?.cancel() + slot.idleRelease = nil + state.nextId &+= 1 + let hold = state.nextId + slot.holds[hold] = Holder() + if let connection { + slot.users.insert(connection) + } + state.slots[id] = slot + return VaultLeaseHold(cache: self, slot: id, hold: hold, issued: slot.issued) + } + + private func answered(_ id: UInt64, exists: Bool?) { + let (lead, dropped) = state.withLock { state -> (Duration?, VaultLease?) in + guard let slot = state.slots[id] else { return (nil, nil) } + state.slots[id]?.asking = false + state.slots[id]?.credentialsRefused = false + // Vault renews most engines' users without asking the database, so a user dropped there still exists to + // Vault. Once a server took it and then refused its password, new sign-ins read another; sessions keep it. + if exists != false, slot.accepted, slot.credentialsRefused { + unlist(id, in: &state) + return (nil, settle(id, in: &state)) + } + return (slot.issued.lease.endingLead, nil) + } + dropped?.release() + if exists == false, let lead { + leaseEnding(id, remaining: lead) + } + } + + private func leaseEnding(_ id: UInt64, remaining: Duration) { + let (handlers, lease) = state.withLock { state -> ([@Sendable (Duration) -> Void], VaultLease?) in + guard let slot = state.slots[id], slot.ending == nil else { return ([], nil) } + state.slots[id]?.ending = remaining + unlist(id, in: &state) + let handlers = state.slots[id]?.holds.values.compactMap(\.onEnding) ?? [] + return (handlers, settle(id, in: &state)) + } + handlers.forEach { $0(remaining) } + lease?.release() + } + + private func expireIdle(_ id: UInt64) { + let lease = state.withLock { state -> VaultLease? in + guard state.slots[id]?.isIdle == true else { return nil } + return drop(id, in: &state) + } + lease?.release() + } + + // For a slot that may have lost its last user: an unlisted one goes now, a listed one after the retention. + private func settle(_ id: UInt64, in state: inout State) -> VaultLease? { + guard let slot = state.slots[id], slot.isIdle else { return nil } + guard state.listed[slot.key] == id else { return drop(id, in: &state) } + slot.idleRelease?.cancel() + let clock = clock + state.slots[id]?.idleRelease = Task { [weak self] in + do { + try await clock.sleep(for: Self.idleRetention) + } catch { + return + } + self?.expireIdle(id) + } + return nil + } + + private func unlist(_ id: UInt64, in state: inout State) { + guard let key = state.slots[id]?.key, state.listed[key] == id else { return } + state.listed[key] = nil + } + + private func drop(_ id: UInt64, in state: inout State) -> VaultLease? { + guard let slot = state.slots.removeValue(forKey: id) else { return nil } + slot.idleRelease?.cancel() + if state.listed[slot.key] == id { + state.listed[slot.key] = nil + } + return slot.issued.lease + } +} + +// One user of a cached lease. Releasing it, or dropping the last reference to it, hands the lease back. +final class VaultLeaseHold: Sendable { + let username: String + let password: String + let leaseId: String + + private let lease: VaultLease + private let cache: VaultLeaseCache + private let slot: UInt64 + private let hold: UInt64 + private let released = OSAllocatedUnfairLock(initialState: false) + + fileprivate init(cache: VaultLeaseCache, slot: UInt64, hold: UInt64, issued: VaultLeaseCache.Issued) { + self.cache = cache + self.slot = slot + self.hold = hold + self.username = issued.username + self.password = issued.password + self.leaseId = issued.lease.leaseId + self.lease = issued.lease + } + + deinit { + release() + } + + var hasEnded: Bool { lease.hasEnded || cache.hasEnded(slot: slot) } + + var clock: any Clock { cache.clock } + + // `onEnding` gets how long the lease has left, which is the time to sign in again in. + func start(onEnding: @escaping @Sendable (Duration) -> Void) { + cache.subscribe(slot: slot, hold: hold, onEnding: onEnding) + } + + func release() { + release(ending: false) + } + + // For a user whose password left the process, such as a backup tool's arguments: revoked as soon as nothing + // else holds it, not kept for the next sign-in. + func end() { + release(ending: true) + } + + private func release(ending: Bool) { + let first = released.withLock { released -> Bool in + defer { released = true } + return !released + } + guard first else { return } + cache.release(slot: slot, hold: hold, ending: ending) + } + + func signInAccepted() { + cache.signInAccepted(slot: slot) + } + + // `credentials`: the server named the password or user as the reason, as MySQL's 1045 does. + func signInRefused(credentials: Bool) { + cache.signInRefused(slot: slot, credentials: credentials) + } + + // Nil once the lease is past the acceptance window or has ended: a refusal then is final. + func redialDelay(afterFailures failures: Int) -> Duration? { + cache.redialDelay(slot: slot, afterFailures: failures) + } + + // Dials until the database takes this user, while the lease is young. `waiting` hears each wait before it + // starts; `isCurrent` says whether the connect still wants a dial once it ends. + func dial( + _ connect: () async throws -> Void, + waiting: (Duration) -> Void, + isCurrent: () -> Bool + ) async throws { + let startedYoung = redialDelay(afterFailures: 1) != nil + var failures = 0 + while true { + do { + try await connect() + return + } catch { + guard let refusal = error as? any PluginDriverError, Self.isRefusedSignIn(refusal) else { throw error } + failures += 1 + guard let delay = redialDelay(afterFailures: failures) else { + throw startedYoung && !hasEnded ? VaultUserNotLoaded(refusal: refusal) : error + } + waiting(delay) + try await clock.sleep(for: delay) + guard isCurrent() else { throw CancellationError() } + guard !hasEnded else { throw error } + } + } + } + + // MySQL answers 1045 for a user it has not loaded yet, the same as for a wrong password. + static func isRefusedSignIn(_ error: Error) -> Bool { + guard let error = error as? any PluginDriverError else { return false } + return error.pluginErrorCode == 1_045 && error.pluginSqlState == "28000" + } +} + +extension VaultLeaseHold: Equatable { + static func == (lhs: VaultLeaseHold, rhs: VaultLeaseHold) -> Bool { lhs === rhs } +} + +// A new Vault user the server refused for the whole acceptance window. The lease is kept, so the next attempt meets +// the same user, which the server may have loaded by then. +struct VaultUserNotLoaded: PluginDriverError { + let refusal: any PluginDriverError + + var pluginErrorMessage: String { refusal.pluginErrorMessage } + var pluginErrorCode: Int? { refusal.pluginErrorCode } + var pluginSqlState: String? { refusal.pluginSqlState } + var pluginErrorDetail: String? { refusal.pluginErrorDetail } + + var recoverySuggestion: String? { + String( + format: String(localized: """ + The database refused the new user Vault created for it for %lld seconds. Some databases load new \ + users on a timer, so try again in a moment. + """), + Int64(VaultLeaseCache.acceptanceWindow.components.seconds) + ) + } +} diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter+CellValue.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter+CellValue.swift new file mode 100644 index 0000000000..b8361d0ed5 --- /dev/null +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter+CellValue.swift @@ -0,0 +1,52 @@ +import Foundation +import os +import TableProNumberFormatting +import TableProPluginKit +import TableProUtilities + +extension PluginDriverAdapter { + private static let iso8601Formatter = OSAllocatedUnfairLock( + uncheckedState: { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return formatter + }() + ) + + static func cellValue(for parameter: Any?) -> PluginCellValue { + guard let parameter else { return .null } + if let typed = parameter as? TypedParameterValue { return .text(typed.text) } + if let data = parameter as? Data { return .bytes(data) } + if let f = parameter as? Float { + guard f.isFinite else { return .null } + return .text(NumberText.text(for: f)) + } + if let f = parameter as? any BinaryFloatingPoint { + let d = Double(f) + guard d.isFinite else { return .null } + return .text(NumberText.text(for: d)) + } + return .text(stringValue(for: parameter)) + } + + private static func stringValue(for parameter: Any) -> String { + switch parameter { + case let s as String: + return s + case let b as Bool: + return b ? "1" : "0" + case let i as any BinaryInteger: + return String(i) + case let f as any BinaryFloatingPoint: + return NumberText.text(for: Double(f)) + case let d as Date: + return Self.iso8601Formatter.withLockUnchecked { $0.string(from: d) } + case let data as Data: + return data.hexEncoded + case let uuid as UUID: + return uuid.uuidString + default: + return String(describing: parameter) + } + } +} diff --git a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift index 7360b71037..557bcf6116 100644 --- a/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift +++ b/apple/TablePro/Core/Plugins/PluginDriverAdapter.swift @@ -16,6 +16,7 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor private struct ConnectAttempt { let generation: UInt64 let gate: ConnectionSingleResumeGate + var deadline: ContinuousClock.Instant var task: Task? var isAbandoned = false } @@ -38,7 +39,7 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor let effectiveQueryTimeoutSeconds: Int private let pluginDriver: any PluginDatabaseDriver // The Vault lease this driver signed in with. Its database user goes away when it ends. - private let credentialLease: VaultLease? + private let credentialLease: VaultLeaseHold? private let connectionDeadline: ConnectionDeadline private let timeoutEndpoint: ConnectionTimeoutEndpoint private let requiresHostDeadline: Bool @@ -96,51 +97,6 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor private static let logger = Logger(subsystem: "com.TablePro", category: "PluginDriverAdapter") - private static let iso8601Formatter = OSAllocatedUnfairLock( - uncheckedState: { - let formatter = ISO8601DateFormatter() - formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] - return formatter - }() - ) - - static func cellValue(for parameter: Any?) -> PluginCellValue { - guard let parameter else { return .null } - if let typed = parameter as? TypedParameterValue { return .text(typed.text) } - if let data = parameter as? Data { return .bytes(data) } - if let f = parameter as? Float { - guard f.isFinite else { return .null } - return .text(NumberText.text(for: f)) - } - if let f = parameter as? any BinaryFloatingPoint { - let d = Double(f) - guard d.isFinite else { return .null } - return .text(NumberText.text(for: d)) - } - return .text(stringValue(for: parameter)) - } - - private static func stringValue(for parameter: Any) -> String { - switch parameter { - case let s as String: - return s - case let b as Bool: - return b ? "1" : "0" - case let i as any BinaryInteger: - return String(i) - case let f as any BinaryFloatingPoint: - return NumberText.text(for: Double(f)) - case let d as Date: - return Self.iso8601Formatter.withLockUnchecked { $0.string(from: d) } - case let data as Data: - return data.hexEncoded - case let uuid as UUID: - return uuid.uuidString - default: - return String(describing: parameter) - } - } - init( connection: DatabaseConnection, pluginDriver: any PluginDatabaseDriver, @@ -148,7 +104,7 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor timeoutEndpoint: ConnectionTimeoutEndpoint? = nil, effectiveQueryTimeoutSeconds: Int? = nil, requiresHostDeadline: Bool? = nil, - credentialLease: VaultLease? = nil + credentialLease: VaultLeaseHold? = nil ) { self.connection = connection self.pluginDriver = pluginDriver @@ -202,7 +158,7 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor let generation = state.nextConnectGeneration state.status = .connecting state.connectionWarning = nil - state.connectAttempt = ConnectAttempt(generation: generation, gate: gate) + state.connectAttempt = ConnectAttempt(generation: generation, gate: gate, deadline: connectionDeadline.instant) return generation } guard let generation else { @@ -216,12 +172,17 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor let operation = Task { [self] in do { - try await pluginDriver.connect(reportingStage: guardedReport) + try await connectSigningIn(generation: generation, report: guardedReport) + credentialLease?.signInAccepted() let adopted = gate.resume(with: .success(())) finishConnect(generation: generation, succeeded: true, adopted: adopted, error: nil) } catch { // Every sign-in fails here, pooled and per-database ones included, so a rejected reused password goes. PasswordCommandCache.shared.forget(connectionId: connection.id) + // A user refused through the whole window is still on its way: the server may load it by the next try. + if !(error is VaultUserNotLoaded), ConnectionFailureClassifier.isAuthenticationFailure(error) { + credentialLease?.signInRefused(credentials: VaultLeaseHold.isRefusedSignIn(error)) + } let adopted = gate.resume(with: .failure(error)) finishConnect(generation: generation, succeeded: false, adopted: adopted, error: error) } @@ -231,10 +192,15 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor let timeout: Task? if requiresHostDeadline { timeout = Task { [self] in - do { - try await ContinuousClock().sleep(until: connectionDeadline.instant) - } catch { - return + var instant = connectionDeadline.instant + while true { + do { + try await ContinuousClock().sleep(until: instant) + } catch { + return + } + guard let later = hostDeadline(of: generation), later > instant else { break } + instant = later } abandonConnect( generation: generation, @@ -301,6 +267,39 @@ final class PluginDriverAdapter: DatabaseDriver, SchemaSwitchable, DatabaseRepor state.withLock { $0.disconnectInProgress = false } } + // A database can refuse a user Vault created moments ago until it loads it, so a young lease dials again. + private func connectSigningIn(generation: UInt64, report: @escaping ConnectionStageReporter) async throws { + guard let credentialLease else { + try await pluginDriver.connect(reportingStage: report) + return + } + try await credentialLease.dial( + { try await pluginDriver.connect(reportingStage: report) }, + waiting: { delay in + // The wait is the database's, not the connect's, so the host deadline moves out by it. + moveHostDeadline(of: generation, past: delay) + report(.custom(String(localized: "Waiting for the database to accept the new Vault user"))) + }, + isCurrent: { isCurrentConnect(generation) } + ) + } + + private func moveHostDeadline(of generation: UInt64, past wait: Duration) { + let instant = ContinuousClock.now + wait + .seconds(connectionDeadline.configuredSeconds) + state.withLock { state in + guard var attempt = state.connectAttempt, attempt.generation == generation else { return } + attempt.deadline = max(attempt.deadline, instant) + state.connectAttempt = attempt + } + } + + private func hostDeadline(of generation: UInt64) -> ContinuousClock.Instant? { + state.withLock { state in + guard let attempt = state.connectAttempt, attempt.generation == generation else { return nil } + return attempt.deadline + } + } + private func attach(_ task: Task, to generation: UInt64) { let shouldCancel = state.withLock { state in guard var attempt = state.connectAttempt, attempt.generation == generation else { diff --git a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift index 9de54b604c..29dfc758d7 100644 --- a/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift +++ b/apple/TablePro/Core/Services/Query/MetadataConnectionPool.swift @@ -180,7 +180,7 @@ final class MetadataConnectionPool { /// Ten minutes rather than something tighter, because re-taking one is a whole connect, /// measured at 1.7-5.8ms on loopback but 800-1900ms across the internet, and someone still /// working must never pay that. Ten minutes of silence is nobody still working. - static let idleTimeout: TimeInterval = 600 + nonisolated static let idleTimeout: TimeInterval = 600 /// The sweeper wakes more often than the timeout so a connection that goes idle just after a /// tick does not wait a whole second timeout, and never so often that an idle app spends the diff --git a/apple/TablePro/Core/Utilities/Connection/PasswordSourceResolver.swift b/apple/TablePro/Core/Utilities/Connection/PasswordSourceResolver.swift index d6749cb279..fd1e34cc15 100644 --- a/apple/TablePro/Core/Utilities/Connection/PasswordSourceResolver.swift +++ b/apple/TablePro/Core/Utilities/Connection/PasswordSourceResolver.swift @@ -81,12 +81,23 @@ enum PasswordSourceResolver { } } - // A test reports what the source returns right now, so it neither reads nor fills the reuse cache. - // A Vault role test may open the browser to sign in, and revokes the lease it was given at once. - static func test(_ source: PasswordSource, for context: ConnectionScriptContext) async throws -> String { + // A command test reports what the command prints right now, so it neither reads nor fills the reuse cache. + // A Vault role test may open the browser to sign in. It shows the user `leaseOwner` signs in as, and leaves the + // lease for that owner's next connect, so a database slow to accept a new user has it ready by then. With no + // owner, as for an unsaved connection, nothing would reuse it, so it is revoked at once. + static func test(_ source: PasswordSource, for context: ConnectionScriptContext, leaseOwner: UUID?) async throws -> String { if case .vaultDatabaseRole(let role) = source { - let signIn = try await VaultDatabaseCredentials.shared.signIn(role, interactive: true) - signIn.lease?.release() + let signIn = try await VaultDatabaseCredentials.shared.signIn( + role, + owner: leaseOwner ?? UUID(), + for: nil, + interactive: true + ) + if leaseOwner == nil { + signIn.lease?.end() + } else { + signIn.lease?.release() + } return signIn.username.map { "\($0) / \(signIn.password)" } ?? signIn.password } return try await resolve(source.withoutReuse, for: context) diff --git a/apple/TablePro/Models/Connection/ConnectionFailureClassifier.swift b/apple/TablePro/Models/Connection/ConnectionFailureClassifier.swift index bf327fcfe4..83b77f45bb 100644 --- a/apple/TablePro/Models/Connection/ConnectionFailureClassifier.swift +++ b/apple/TablePro/Models/Connection/ConnectionFailureClassifier.swift @@ -77,6 +77,37 @@ internal enum ConnectionFailureClassifier { return nsError.domain == NSCocoaErrorDomain && nsError.code == NSUserCancelledError } + private static let invalidAuthorizationSQLState = "28000" + private static let mysqlAccessDeniedErrorCode = 1_045 + + internal static func isAuthenticationFailure(_ error: Error) -> Bool { + if let pluginError = error as? any PluginDriverError { + if pluginError.pluginSqlState == invalidAuthorizationSQLState { + return true + } + if pluginError.pluginErrorCode == mysqlAccessDeniedErrorCode { + return true + } + return messageIndicatesAuthenticationFailure(pluginError.pluginErrorMessage) + } + return messageIndicatesAuthenticationFailure(error.localizedDescription) + } + + // SQL Server, Oracle, Redis, Cassandra and Snowflake word a refused login their own way. + private static let refusedLoginPhrases = [ + "access denied", "authentication failed", "invalid credentials", "login failed for user", "ora-01017", + "wrongpass", "password are incorrect", "incorrect username or password", + ] + + // SQL Server says "Login failed for user" while it is upgrading or in single-user mode too, and connects again after. + private static let passingLoginRefusals = ["script upgrade mode", "single user mode"] + + private static func messageIndicatesAuthenticationFailure(_ message: String) -> Bool { + let lowered = message.lowercased() + guard !passingLoginRefusals.contains(where: { lowered.contains($0) }) else { return false } + return refusedLoginPhrases.contains { lowered.contains($0) } + } + internal static func info(for error: Error) -> ConnectionFailureInfo { if error is SSLHandshakeError { return ConnectionFailureInfo(message: SSLHandshakeError.formatted(error)) diff --git a/apple/TablePro/Resources/Localizable.xcstrings b/apple/TablePro/Resources/Localizable.xcstrings index 0a90bab5dd..7234eab691 100644 --- a/apple/TablePro/Resources/Localizable.xcstrings +++ b/apple/TablePro/Resources/Localizable.xcstrings @@ -49780,6 +49780,9 @@ } } } + }, + "Connections on this profile share one database user. TablePro renews its lease while in use, revokes it once the last connection disconnects or after 10 minutes unused, and never saves the generated password." : { + }, "Connections the client may use, ordered by name" : { "localizations" : { @@ -188107,6 +188110,9 @@ } } } + }, + "The database refused the new user Vault created for it for %lld seconds. Some databases load new users on a timer, so try again in a moment." : { + }, "The database server enforced this, not TablePro's Safe Mode. You are most likely connected to a read replica, or the server sets new transactions to read-only. Connect to the primary server to write.\n\nServer response: %@" : { "localizations" : { @@ -213611,6 +213617,9 @@ }, "Vault Address" : { + }, + "Vault could not complete the request: %@" : { + }, "Vault could not issue credentials from %@: %@" : { "localizations" : { @@ -214846,6 +214855,9 @@ }, "Waiting for a compatible build" : { + }, + "Waiting for the database to accept the new Vault user" : { + }, "Waiting for the tunnel" : { "localizations" : { diff --git a/apple/TablePro/Views/Connection/PasswordPromptToggle.swift b/apple/TablePro/Views/Connection/PasswordPromptToggle.swift index e93cdc358a..db527b7f46 100644 --- a/apple/TablePro/Views/Connection/PasswordPromptToggle.swift +++ b/apple/TablePro/Views/Connection/PasswordPromptToggle.swift @@ -26,6 +26,7 @@ struct PasswordPromptToggle: View { PasswordSourceTestView( source: source, context: { coordinator.passwordSourceContext() }, + leaseOwner: { coordinator.connectionId }, refusal: { coordinator.storage.storeIsTrusted ? nil : PasswordSourceResolver.ResolutionError.storeNotTrusted } diff --git a/apple/TablePro/Views/Connection/PasswordSourceTest.swift b/apple/TablePro/Views/Connection/PasswordSourceTest.swift index fdf9a8081e..5ad7c12df6 100644 --- a/apple/TablePro/Views/Connection/PasswordSourceTest.swift +++ b/apple/TablePro/Views/Connection/PasswordSourceTest.swift @@ -16,11 +16,11 @@ final class PasswordSourceTestModel: ObservableObject { @Published private(set) var isRunning = false @Published private var latest: (source: PasswordSource, environment: [String: String], outcome: Outcome)? - private let resolve: @Sendable (PasswordSource, ConnectionScriptContext) async throws -> String + private let resolve: @Sendable (PasswordSource, ConnectionScriptContext, UUID?) async throws -> String init( - resolve: @escaping @Sendable (PasswordSource, ConnectionScriptContext) async throws -> String = { source, context in - try await PasswordSourceResolver.test(source, for: context) + resolve: @escaping @Sendable (PasswordSource, ConnectionScriptContext, UUID?) async throws -> String = { source, context, owner in + try await PasswordSourceResolver.test(source, for: context, leaseOwner: owner) } ) { self.resolve = resolve @@ -32,7 +32,7 @@ final class PasswordSourceTestModel: ObservableObject { return latest.outcome } - func run(_ source: PasswordSource, context: ConnectionScriptContext, refusal: (any Error)?) async { + func run(_ source: PasswordSource, context: ConnectionScriptContext, leaseOwner: UUID?, refusal: (any Error)?) async { guard !isRunning else { return } if let refusal { latest = (source, context.environment, .failed(refusal.localizedDescription)) @@ -41,7 +41,7 @@ final class PasswordSourceTestModel: ObservableObject { isRunning = true defer { isRunning = false } do { - let value = try await resolve(source, context) + let value = try await resolve(source, context, leaseOwner) latest = (source, context.environment, .returned(value)) } catch { latest = (source, context.environment, .failed(error.localizedDescription)) @@ -52,6 +52,9 @@ final class PasswordSourceTestModel: ObservableObject { struct PasswordSourceTestView: View { let source: PasswordSource let context: () -> ConnectionScriptContext + // Whose Vault user a database role test signs in as: the profile, or the connection for a source of its own. + // Nil for an unsaved connection, whose test user nothing would reuse. + let leaseOwner: () -> UUID? let refusal: () -> (any Error)? @StateObject private var model = PasswordSourceTestModel() @@ -61,8 +64,9 @@ struct PasswordSourceTestView: View { HStack { Button(String(localized: "Test Source")) { let context = context() + let leaseOwner = leaseOwner() let refusal = refusal() - Task { await model.run(source, context: context, refusal: refusal) } + Task { await model.run(source, context: context, leaseOwner: leaseOwner, refusal: refusal) } } .disabled(model.isRunning) .accessibilityIdentifier("password-source-test") diff --git a/apple/TablePro/Views/ConnectionForm/Components/CredentialProfilePicker.swift b/apple/TablePro/Views/ConnectionForm/Components/CredentialProfilePicker.swift index 029f4797d7..69fa97b307 100644 --- a/apple/TablePro/Views/ConnectionForm/Components/CredentialProfilePicker.swift +++ b/apple/TablePro/Views/ConnectionForm/Components/CredentialProfilePicker.swift @@ -42,6 +42,7 @@ struct CredentialProfilePicker: View { PasswordSourceTestView( source: source, context: { coordinator.passwordSourceContext() }, + leaseOwner: { profile.id }, refusal: { CredentialProfileStorage.shared.storeIsTrusted ? nil : PasswordSourceResolver.ResolutionError.profileStoreNotTrusted diff --git a/apple/TablePro/Views/Settings/Profiles/CredentialProfileEditorSheet.swift b/apple/TablePro/Views/Settings/Profiles/CredentialProfileEditorSheet.swift index cea90a444d..7d59dadc49 100644 --- a/apple/TablePro/Views/Settings/Profiles/CredentialProfileEditorSheet.swift +++ b/apple/TablePro/Views/Settings/Profiles/CredentialProfileEditorSheet.swift @@ -29,6 +29,8 @@ struct CredentialProfileEditorSheet: View { @State private var sourceField: String = "" @State private var commandReuseSeconds: Int? @State private var vaultAccess = VaultAccessDraft() + // A new profile keeps this id when saved, so its first connect signs in as the user Test Source leased. + @State private var draftProfileId = UUID() @State private var secureFieldType: DatabaseType = .mysql @State private var secureFieldValues: [String: String] = [:] @@ -176,6 +178,7 @@ struct CredentialProfileEditorSheet: View { PasswordSourceTestView( source: testedSource, context: { testContext }, + leaseOwner: { profileId }, refusal: { Self.testRefusal( for: testedSource, @@ -233,7 +236,10 @@ struct CredentialProfileEditorSheet: View { .font(.caption) .foregroundStyle(.secondary) } - Text("Each connection signs in as a new database user. TablePro renews the lease while connected, revokes it on disconnect, and never saves the generated password.") + Text(String(localized: """ + Connections on this profile share one database user. TablePro renews its lease while in use, revokes it \ + once the last connection disconnects or after 10 minutes unused, and never saves the generated password. + """)) .font(.caption) .foregroundStyle(.secondary) } @@ -253,9 +259,13 @@ struct CredentialProfileEditorSheet: View { return source } + private var profileId: UUID { + existingProfile?.id ?? draftProfileId + } + private var testContext: ConnectionScriptContext { ConnectionScriptContext( - connectionId: existingProfile?.id ?? UUID(), + connectionId: profileId, connectionName: "", host: "", port: 0, @@ -418,7 +428,6 @@ struct CredentialProfileEditorSheet: View { } private func saveProfile() { - let profileId = existingProfile?.id ?? UUID() let liveFields = secureFieldValues.filter { !$0.value.isEmpty } let profile = CredentialProfile( id: profileId, diff --git a/apple/TableProTests/Core/Database/DatabaseManagerTunnelTests.swift b/apple/TableProTests/Core/Database/DatabaseManagerTunnelTests.swift index 84bed140b7..b006fc473f 100644 --- a/apple/TableProTests/Core/Database/DatabaseManagerTunnelTests.swift +++ b/apple/TableProTests/Core/Database/DatabaseManagerTunnelTests.swift @@ -60,6 +60,18 @@ struct DatabaseManagerTunnelTests { #expect(tunneled.passwordSource == .env(variable: "DB_PASS")) } + @Test("A tunneled connection keeps its credential profile, so it signs in with the profile's credentials") + func tunnelPreservesCredentialProfile() { + let profileId = UUID() + var connection = DatabaseConnection(name: "tunneled", host: "db.internal", port: 3_306, type: .mysql) + connection.credentialMode = .profile(id: profileId) + + let tunneled = DatabaseManager.shared.tunneledConnection(from: connection, localPort: 61_235) + + #expect(tunneled.credentialMode.profileId == profileId) + #expect(ConnectionCredentialResolver.credentialOwner(for: tunneled) == profileId) + } + @Test("A tunneled DuckDB Remote row dials the forward through its own Host and Port fields") func tunnelRewritesServerEndpointFields() { let connection = DatabaseConnection( diff --git a/apple/TableProTests/Core/Database/PostgreSQLDriverErrorClassificationTests.swift b/apple/TableProTests/Core/Database/PostgreSQLDriverErrorClassificationTests.swift index 87f7872eeb..9a26a2b864 100644 --- a/apple/TableProTests/Core/Database/PostgreSQLDriverErrorClassificationTests.swift +++ b/apple/TableProTests/Core/Database/PostgreSQLDriverErrorClassificationTests.swift @@ -30,7 +30,7 @@ struct PostgreSQLDriverErrorClassificationTests { pluginSqlState: "28P01" ) - #expect(DatabaseManager.shared.isAuthenticationFailure(error)) + #expect(ConnectionFailureClassifier.isAuthenticationFailure(error)) } @Test("a read-only refusal lost with the connection is still classified, with the server's words") @@ -67,6 +67,6 @@ struct PostgreSQLDriverErrorClassificationTests { """, pluginSqlState: nil ) - #expect(DatabaseManager.shared.isAuthenticationFailure(error)) + #expect(ConnectionFailureClassifier.isAuthenticationFailure(error)) } } diff --git a/apple/TableProTests/Core/Database/ReconnectCredentialRecoveryTests.swift b/apple/TableProTests/Core/Database/ReconnectCredentialRecoveryTests.swift index a6604f2766..3c41f37e29 100644 --- a/apple/TableProTests/Core/Database/ReconnectCredentialRecoveryTests.swift +++ b/apple/TableProTests/Core/Database/ReconnectCredentialRecoveryTests.swift @@ -13,7 +13,7 @@ struct ReconnectCredentialRecoveryTests { pluginSqlState: "28000" ) - #expect(DatabaseManager.shared.isAuthenticationFailure(error)) + #expect(ConnectionFailureClassifier.isAuthenticationFailure(error)) } @Test("An authentication failure stops an unattended reconnect with the server's message") @@ -30,6 +30,32 @@ struct ReconnectCredentialRecoveryTests { ) } + @Test("A Vault refusal stops an unattended reconnect with Vault's reason") + func vaultRefusalStopsTheReconnect() { + let error = VaultError.credentialsFailed(path: "database/creds/ro", message: "permission denied") + + #expect(DatabaseManager.shared.reconnectVerdict(for: error) == .stop(ConnectionFailureClassifier.info(for: error))) + } + + @Test("A new Vault user the database is still loading keeps the reconnect going") + func userNotLoadedKeepsRetrying() { + let error = VaultUserNotLoaded(refusal: FakePluginAuthError( + pluginErrorMessage: "Access denied for user 'v-token-ro-1'", + pluginErrorCode: 1_045, + pluginSqlState: "28000" + )) + + #expect(DatabaseManager.shared.reconnectVerdict(for: error) == .retry(ConnectionFailureClassifier.info(for: error))) + } + + @Test("An unreachable or failing Vault keeps the reconnect going and says why", arguments: [ + VaultError.unreachable("https://vault.test:8200", "The request timed out."), + VaultError.serverFailed("local node not active but active cluster node not found"), + ]) + func unreachableVaultKeepsRetrying(error: VaultError) { + #expect(DatabaseManager.shared.reconnectVerdict(for: error) == .retry(ConnectionFailureClassifier.info(for: error))) + } + @Test("A connection that is merely gone keeps the reconnect going with no reason of its own") func lostConnectionKeepsRetrying() { #expect(DatabaseManager.shared.reconnectVerdict(for: DatabaseError.notConnected) == .retry(nil)) diff --git a/apple/TableProTests/Core/Database/Vault/VaultDatabaseCredentialsTests.swift b/apple/TableProTests/Core/Database/Vault/VaultDatabaseCredentialsTests.swift index 7c81dd214d..30d3d516a2 100644 --- a/apple/TableProTests/Core/Database/Vault/VaultDatabaseCredentialsTests.swift +++ b/apple/TableProTests/Core/Database/Vault/VaultDatabaseCredentialsTests.swift @@ -21,6 +21,7 @@ private final class TokenRecorder: @unchecked Sendable { struct VaultDatabaseCredentialsTests { private let role = VaultDatabaseRole(address: "https://vault.test:8200", mount: "database", role: "readonly") + private let owner = UUID() private func credentials(_ transport: VaultFakeTransport, files: [String: String] = [:]) -> VaultDatabaseCredentials { VaultDatabaseCredentials( @@ -36,7 +37,7 @@ struct VaultDatabaseCredentialsTests { transport.reply("GET", "auth/token/lookup-self", VaultFixtures.lookupReply(ttl: 0)) transport.reply("GET", "database/creds/readonly", VaultFixtures.credsReply) - let signIn = try await credentials(transport).signIn(role, interactive: false) + let signIn = try await credentials(transport).signIn(role, owner: owner, for: nil, interactive: false) #expect(signIn.username == "v-token-readonly-1") #expect(signIn.password == "s3cret") @@ -45,15 +46,33 @@ struct VaultDatabaseCredentialsTests { lease.release() } - @Test("Each sign-in reads a new lease") - func newLeasePerSignIn() async throws { + @Test("Sign-ins for one owner and role share one lease") + func signInsShareTheLease() async throws { let transport = VaultFakeTransport() transport.reply("GET", "auth/token/lookup-self", VaultFixtures.lookupReply(ttl: 0)) transport.reply("GET", "database/creds/readonly", VaultFixtures.credsReply) let credentials = credentials(transport) - let first = try await credentials.signIn(role, interactive: false) - let second = try await credentials.signIn(role, interactive: false) + let first = try await credentials.signIn(role, owner: owner, for: nil, interactive: false) + first.lease?.release() + let second = try await credentials.signIn(role, owner: owner, for: nil, interactive: false) + + #expect(transport.requests(to: "database/creds/readonly").count == 1) + #expect(second.username == first.username) + #expect(second.lease?.leaseId == first.lease?.leaseId) + second.lease?.release() + } + + @Test("Another owner signs in as a user of its own") + func otherOwnerReadsItsOwnLease() async throws { + let transport = VaultFakeTransport() + transport.reply("GET", "auth/token/lookup-self", VaultFixtures.lookupReply(ttl: 0)) + transport.reply("GET", "database/creds/readonly", VaultFixtures.credsReply) + transport.reply("GET", "database/creds/readonly", VaultFixtures.credsReply) + let credentials = credentials(transport) + + let first = try await credentials.signIn(role, owner: owner, for: nil, interactive: false) + let second = try await credentials.signIn(role, owner: UUID(), for: nil, interactive: false) #expect(transport.requests(to: "database/creds/readonly").count == 2) #expect(first.lease != second.lease) @@ -68,7 +87,29 @@ struct VaultDatabaseCredentialsTests { transport.reply("GET", "database/creds/readonly", status: 403, #"{"errors":["1 error occurred:\n\t* permission denied\n\n"]}"#) await #expect(throws: VaultError.credentialsFailed(path: "database/creds/readonly", message: "permission denied")) { - try await credentials(transport).signIn(role, interactive: false) + try await credentials(transport).signIn(role, owner: owner, for: nil, interactive: false) + } + } + + @Test("A sealed Vault reads as unreachable, so an unattended reconnect tries again", arguments: [503, 429]) + func sealedVaultIsUnreachable(status: Int) async { + let transport = VaultFakeTransport() + transport.reply("GET", "auth/token/lookup-self", VaultFixtures.lookupReply(ttl: 0)) + transport.reply("GET", "database/creds/readonly", status: status, #"{"errors":["Vault is sealed"]}"#) + + await #expect(throws: VaultError.unreachable("https://vault.test:8200", "Vault is sealed")) { + try await credentials(transport).signIn(role, owner: owner, for: nil, interactive: false) + } + } + + @Test("Vault's own failure names its reason and is worth another try") + func vaultFailureNamesItsReason() async { + let transport = VaultFakeTransport() + transport.reply("GET", "auth/token/lookup-self", VaultFixtures.lookupReply(ttl: 0)) + transport.reply("GET", "database/creds/readonly", status: 500, #"{"errors":["local node not active"]}"#) + + await #expect(throws: VaultError.serverFailed("local node not active")) { + try await credentials(transport).signIn(role, owner: owner, for: nil, interactive: false) } } @@ -82,22 +123,26 @@ struct VaultDatabaseCredentialsTests { var appRole = role appRole.auth = .appRole(mount: "approle", roleId: "r", secretIdPath: "/keys/secret-id") - let signIn = try await credentials(transport, files: ["secret-id": "s"]).signIn(appRole, interactive: false) + let signIn = try await credentials(transport, files: ["secret-id": "s"]) + .signIn(appRole, owner: owner, for: nil, interactive: false) #expect(transport.requests(to: "database/creds/readonly").map(\.token) == ["old", "new"]) #expect(signIn.username == "v-token-readonly-1") signIn.lease?.release() } - @Test("Releasing the sign-in's lease revokes it with the same token") - func releaseRevokes() async throws { + @Test("Ending the last connection that used a lease revokes it, with the token that issued it") + func endUseRevokes() async throws { let transport = VaultFakeTransport() transport.reply("GET", "auth/token/lookup-self", VaultFixtures.lookupReply(ttl: 0)) transport.reply("GET", "database/creds/readonly", VaultFixtures.credsReply) transport.reply("PUT", "sys/leases/revoke", status: 204, "") + let credentials = credentials(transport) + let connection = UUID() - let signIn = try await credentials(transport).signIn(role, interactive: false) + let signIn = try await credentials.signIn(role, owner: owner, for: connection, interactive: false) signIn.lease?.release() + credentials.endUse(connection: connection, live: []) try await waitUntil { !transport.requests(to: "sys/leases/revoke").isEmpty } let revoke = try #require(transport.requests(to: "sys/leases/revoke").first) diff --git a/apple/TableProTests/Core/Database/Vault/VaultLeaseCacheTests.swift b/apple/TableProTests/Core/Database/Vault/VaultLeaseCacheTests.swift new file mode 100644 index 0000000000..9fb1ad33b3 --- /dev/null +++ b/apple/TableProTests/Core/Database/Vault/VaultLeaseCacheTests.swift @@ -0,0 +1,426 @@ +// +// VaultLeaseCacheTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProTestSupport +import Testing + +private final class Revocations: VaultLeaseRenewing, @unchecked Sendable { + private let lock = NSLock() + private var revoked: [String] = [] + private var renewed = 0 + // What Vault answers a renewal: the lease is still there, or it is gone. + var leaseExists = false + + var leases: [String] { lock.withLock { revoked } } + var renewals: Int { lock.withLock { renewed } } + + func renew(leaseId: String, increment: Duration, timeout: Duration) async throws -> VaultLeaseGrant { + let exists = lock.withLock { () -> Bool in + renewed += 1 + return leaseExists + } + guard exists else { throw VaultRequestError(status: 400, messages: ["lease not found"]) } + return VaultLeaseGrant(leaseDuration: increment, tokenRemaining: nil) + } + + func revoke(leaseId: String) async { + lock.withLock { revoked.append(leaseId) } + } +} + +private final class Names: @unchecked Sendable { + private let lock = NSLock() + private var recorded: [String] = [] + + var values: [String] { lock.withLock { recorded } } + + func record(_ name: String) { + lock.withLock { recorded.append(name) } + } +} + +// Vault's side: each read issues the next lease. +private final class LeaseIssuer: @unchecked Sendable { + private let lock = NSLock() + private var issued = 0 + private let clock: ManualClock + private let seconds: Int64 + let revocations = Revocations() + var gate: AsyncGate? + + private let renewable: Bool + + init(clock: ManualClock, seconds: Int64 = 3_600, renewable: Bool = false) { + self.clock = clock + self.seconds = seconds + self.renewable = renewable + } + + var reads: Int { lock.withLock { issued } } + + func issue() async throws -> VaultLeaseCache.Issued { + await gate?.wait() + let number = lock.withLock { () -> Int in + issued += 1 + return issued + } + return VaultLeaseCache.Issued( + username: "v-token-ro-\(number)", + password: "s3cret", + lease: VaultLease( + leaseId: "database/creds/ro/\(number)", + grant: VaultLeaseGrant(leaseDuration: .seconds(seconds), tokenRemaining: nil), + renewable: renewable, + renewer: revocations, + clock: clock + ), + token: "t0" + ) + } +} + +struct VaultLeaseCacheTests { + private let owner = UUID() + private let role = VaultDatabaseRole(mount: "database", role: "ro") + + private func key(owner: UUID? = nil, role: VaultDatabaseRole? = nil, token: String = "t0") throws -> VaultLeaseCache.Key { + VaultLeaseCache.Key( + owner: owner ?? self.owner, + role: role ?? self.role, + endpoint: try VaultEndpoint.resolve(address: "https://vault.test:8200", environment: [:]), + token: token + ) + } + + private func acquire( + _ cache: VaultLeaseCache, + _ issuer: LeaseIssuer, + key: VaultLeaseCache.Key, + for connection: UUID? = nil + ) async throws -> VaultLeaseHold { + try await cache.acquire(key, for: connection) { try await issuer.issue() } + } + + @Test("One owner and role read one lease") + func oneReadPerOwnerAndRole() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + + let first = try await acquire(cache, issuer, key: try key()) + let second = try await acquire(cache, issuer, key: try key()) + + #expect(issuer.reads == 1) + #expect(first.leaseId == second.leaseId) + #expect(first != second) + } + + @Test("Another owner, role or token reads a lease of its own") + func otherKeysReadTheirOwn() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + + let holds = [ + try await acquire(cache, issuer, key: try key()), + try await acquire(cache, issuer, key: try key(owner: UUID())), + try await acquire(cache, issuer, key: try key(role: VaultDatabaseRole(mount: "database", role: "rw"))), + try await acquire(cache, issuer, key: try key(token: "someone-else")), + ] + + #expect(issuer.reads == 4) + #expect(Set(holds.map(\.leaseId)).count == 4) + } + + @Test("Concurrent sign-ins for one key read one lease, whichever order they reach the cache in") + func concurrentSignInsReadOneLease() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let gate = AsyncGate() + issuer.gate = gate + let key = try key() + + async let first = acquire(cache, issuer, key: key) + async let second = acquire(cache, issuer, key: key) + await gate.arrived(1) + await gate.open() + let holds = try await [first, second] + + #expect(issuer.reads == 1) + #expect(holds[0].leaseId == holds[1].leaseId) + } + + @Test("A failed read leaves nothing behind, so the next sign-in reads again") + func failedReadIsNotKept() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let key = try key() + + await #expect(throws: VaultError.self) { + try await cache.acquire(key, for: nil) { throw VaultError.noAddress } + } + _ = try await acquire(cache, issuer, key: key) + + #expect(issuer.reads == 1) + } + + @Test("A sign-in whose caller gave up leaves its lease for the next one, unrevoked") + func abandonedSignInIsKept() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + + _ = try await acquire(cache, issuer, key: try key()) + let next = try await acquire(cache, issuer, key: try key()) + + #expect(issuer.reads == 1) + #expect(next.leaseId == "database/creds/ro/1") + #expect(issuer.revocations.leases.isEmpty) + } + + @Test("A lease nothing holds is revoked after the idle retention, unless a sign-in takes it first") + func idleRetention() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + + try await acquire(cache, issuer, key: try key()).release() + try await waitUntil { clock.pendingSleeperCount == 2 } + clock.advance(by: VaultLeaseCache.idleRetention - .seconds(1)) + let again = try await acquire(cache, issuer, key: try key()) + // Only the lease's own schedule is left: the idle timer was cancelled. + try await waitUntil { clock.pendingSleeperCount == 1 } + clock.advance(by: .seconds(1)) + #expect(issuer.revocations.leases.isEmpty) + + again.release() + try await waitUntil { clock.pendingSleeperCount == 2 } + clock.advance(by: VaultLeaseCache.idleRetention) + + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + #expect(issuer.reads == 1) + } + + @Test("An ended lease is not handed out again, each hold hears its ending, and the last one revokes it") + func endedLease() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock, seconds: 60) + let endings = Names() + let first = try await acquire(cache, issuer, key: try key()) + let second = try await acquire(cache, issuer, key: try key()) + first.start { _ in endings.record("first") } + second.start { _ in endings.record("second") } + + await clock.waitUntilSleeping() + clock.advance(by: .seconds(50)) + + try await waitUntil { Set(endings.values) == ["first", "second"] } + let next = try await acquire(cache, issuer, key: try key()) + #expect(next.leaseId == "database/creds/ro/2") + first.release() + #expect(issuer.revocations.leases.isEmpty) + second.release() + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + } + + @Test("A refused sign-in on a lease Vault no longer has ends it for every hold and replaces it") + func refusalOfARevokedLease() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock, renewable: true) + let endings = Names() + let session = try await acquire(cache, issuer, key: try key()) + session.start { _ in endings.record("session") } + let refused = try await acquire(cache, issuer, key: try key()) + + refused.signInRefused(credentials: false) + + try await waitUntil { endings.values == ["session"] && refused.hasEnded } + #expect(try await acquire(cache, issuer, key: try key()).leaseId == "database/creds/ro/2") + session.release() + refused.release() + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + } + + @Test( + "A refused sign-in on a lease Vault still has keeps it, unless a server took it and then refused its password", + arguments: [(false, true), (true, false)] + ) + func refusalOfALiveLease(accepted: Bool, credentials: Bool) async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock, renewable: true) + issuer.revocations.leaseExists = true + let refused = try await acquire(cache, issuer, key: try key()) + if accepted { + refused.signInAccepted() + } + + refused.signInRefused(credentials: credentials) + try await waitUntil { issuer.revocations.renewals == 1 } + + #expect(try await acquire(cache, issuer, key: try key()).leaseId == "database/creds/ro/1") + #expect(!refused.hasEnded) + #expect(issuer.revocations.leases.isEmpty) + } + + @Test("A user a server took and then refused for its password is not handed out again, and sessions keep it") + func refusalOfAUserDroppedInTheDatabase() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock, renewable: true) + issuer.revocations.leaseExists = true + let endings = Names() + let session = try await acquire(cache, issuer, key: try key()) + session.start { _ in endings.record("session") } + session.signInAccepted() + let refused = try await acquire(cache, issuer, key: try key()) + + refused.signInRefused(credentials: true) + refused.release() + try await waitUntil { issuer.revocations.renewals == 1 } + session.release() + + // A listed lease would wait out the idle retention instead. + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + #expect(endings.values.isEmpty) + #expect(try await acquire(cache, issuer, key: try key()).leaseId == "database/creds/ro/2") + } + + @Test("Ending a connection revokes a lease only it used, including after a late release") + func endUseRevokesTheConnectionsLease() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let connection = UUID() + let session = try await acquire(cache, issuer, key: try key(), for: connection) + let pooledRead = try await acquire(cache, issuer, key: try key(), for: connection) + session.release() + + cache.endUse(connection: connection, live: []) + // Still there while the read holds it. + #expect(pooledRead.redialDelay(afterFailures: 1) != nil) + pooledRead.release() + + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + #expect(try await acquire(cache, issuer, key: try key()).leaseId == "database/creds/ro/2") + } + + @Test("A read left over from a connection ended earlier does not keep the lease past the last disconnect") + func leftoverReadDoesNotKeepTheLease() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let first = UUID() + let second = UUID() + let firstsRead = try await acquire(cache, issuer, key: try key(), for: first) + let seconds = try await acquire(cache, issuer, key: try key(), for: second) + + cache.endUse(connection: first, live: [second]) + seconds.release() + cache.endUse(connection: second, live: []) + firstsRead.release() + + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + } + + @Test("Ending a connection leaves a lease another live connection is reconnecting with") + func endUseKeepsALeaseBetweenDrivers() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let ended = UUID() + let reconnecting = UUID() + try await acquire(cache, issuer, key: try key(), for: ended).release() + try await acquire(cache, issuer, key: try key(), for: reconnecting).release() + + cache.endUse(connection: ended, live: [reconnecting]) + + #expect(try await acquire(cache, issuer, key: try key(), for: reconnecting).leaseId == "database/creds/ro/1") + #expect(issuer.reads == 1) + } + + @Test("Ending a hold whose password left the process revokes the lease at once") + func endRevokesAtOnce() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + + try await acquire(cache, issuer, key: try key(owner: UUID())).end() + + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + } + + @Test("Ending one connection leaves a lease another connection on the profile still holds") + func endUseLeavesSharedLease() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let ended = UUID() + let other = UUID() + let mine = try await acquire(cache, issuer, key: try key(), for: ended) + let theirs = try await acquire(cache, issuer, key: try key(), for: other) + mine.release() + + cache.endUse(connection: ended, live: [other]) + + #expect(try await acquire(cache, issuer, key: try key()).leaseId == "database/creds/ro/1") + #expect(issuer.revocations.leases.isEmpty) + theirs.release() + } + + @Test("Quit revokes every lease, held or not, and waits for Vault to answer") + func revokeAll() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let held = try await acquire(cache, issuer, key: try key()) + try await acquire(cache, issuer, key: try key(owner: UUID())).release() + + await cache.revokeAll(within: .seconds(3)) + + #expect(Set(issuer.revocations.leases) == ["database/creds/ro/1", "database/creds/ro/2"]) + held.release() + } + + @Test("A lease read after the quit revoke is revoked at once and the sign-in is cancelled") + func readFinishingAfterQuit() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let issuer = LeaseIssuer(clock: clock) + let gate = AsyncGate() + issuer.gate = gate + + let signIn = Task { try await acquire(cache, issuer, key: try key()) } + await gate.arrived(1) + await cache.revokeAll(within: .seconds(3)) + await gate.open() + + await #expect(throws: CancellationError.self) { try await signIn.value } + try await waitUntil { issuer.revocations.leases == ["database/creds/ro/1"] } + } + + @Test("A refused new user is dialed again after 2, 4 and 8 seconds, then only until the window closes") + func redialDelays() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let hold = try await acquire(cache, LeaseIssuer(clock: clock), key: try key()) + + #expect(hold.redialDelay(afterFailures: 1) == .seconds(2)) + #expect(hold.redialDelay(afterFailures: 2) == .seconds(4)) + #expect(hold.redialDelay(afterFailures: 3) == .seconds(8)) + clock.advance(by: .seconds(14)) + #expect(hold.redialDelay(afterFailures: 4) == .seconds(16)) + clock.advance(by: .seconds(6)) + #expect(hold.redialDelay(afterFailures: 4) == .seconds(10)) + clock.advance(by: .seconds(10)) + #expect(hold.redialDelay(afterFailures: 5) == nil) + } +} diff --git a/apple/TableProTests/Core/Database/Vault/VaultLeaseOwnershipTests.swift b/apple/TableProTests/Core/Database/Vault/VaultLeaseOwnershipTests.swift index 9c305ab03e..66798cdae1 100644 --- a/apple/TableProTests/Core/Database/Vault/VaultLeaseOwnershipTests.swift +++ b/apple/TableProTests/Core/Database/Vault/VaultLeaseOwnershipTests.swift @@ -11,14 +11,63 @@ import Testing private final class SilentPluginDriver: PluginDriverStub, @unchecked Sendable {} +private struct ServerRefusal: PluginDriverError { + let pluginErrorMessage: String + let pluginErrorCode: Int? + let pluginSqlState: String? + + static let unknownUser = ServerRefusal( + pluginErrorMessage: "Access denied for user 'v-token-ro-1'", + pluginErrorCode: 1_045, + pluginSqlState: "28000" + ) +} + +// Refuses the first `refusals` dials the way a database that has not loaded a new user does. +private final class SlowToAcceptDriver: PluginDriverStub, @unchecked Sendable { + private let lock = NSLock() + private var dials = 0 + private let refusals: Int + private let refusal: Error + + init(refusals: Int, refusal: Error = ServerRefusal.unknownUser) { + self.refusals = refusals + self.refusal = refusal + } + + var attempts: Int { lock.withLock { dials } } + + func connect() async throws { + let attempt = lock.withLock { () -> Int in + dials += 1 + return dials + } + if attempt <= refusals { throw refusal } + } +} + +private final class Reads: @unchecked Sendable { + private let lock = NSLock() + private var reads = 0 + + var value: Int { lock.withLock { reads } } + + func count() { + lock.withLock { reads += 1 } + } +} + private final class RevokeRecorder: VaultLeaseRenewing, @unchecked Sendable { private let lock = NSLock() private var revoked: [String] = [] + // What Vault answers a renewal: the lease is still there, or it is gone. + var leaseExists = false var revokedLeases: [String] { lock.withLock { revoked } } func renew(leaseId: String, increment: Duration, timeout: Duration) async throws -> VaultLeaseGrant { - throw VaultRequestError(status: 400, messages: ["lease not found"]) + guard lock.withLock({ leaseExists }) else { throw VaultRequestError(status: 400, messages: ["lease not found"]) } + return VaultLeaseGrant(leaseDuration: increment, tokenRemaining: nil) } func revoke(leaseId: String) async { @@ -28,28 +77,55 @@ private final class RevokeRecorder: VaultLeaseRenewing, @unchecked Sendable { @MainActor struct VaultLeaseOwnershipTests { - private func lease(_ recorder: RevokeRecorder, clock: ManualClock, seconds: Int64 = 3_600) -> VaultLease { - VaultLease( - leaseId: "database/creds/ro/1", - grant: VaultLeaseGrant(leaseDuration: .seconds(seconds), tokenRemaining: nil), - renewable: false, - renewer: recorder, - clock: clock - ) + private let owner = UUID() + private let role = VaultDatabaseRole(mount: "database", role: "ro") + + private func hold( + _ cache: VaultLeaseCache, + recorder: RevokeRecorder, + clock: ManualClock, + seconds: Int64 = 3_600, + renewable: Bool = false, + for connection: UUID? = nil, + reads: Reads = Reads() + ) async throws -> VaultLeaseHold { + let endpoint = try VaultEndpoint.resolve(address: "https://vault.test:8200", environment: [:]) + let key = VaultLeaseCache.Key(owner: owner, role: role, endpoint: endpoint, token: "t0") + return try await cache.acquire(key, for: connection) { + reads.count() + return VaultLeaseCache.Issued( + username: "v-token-ro-1", + password: "s3cret", + lease: VaultLease( + leaseId: "database/creds/ro/1", + grant: VaultLeaseGrant(leaseDuration: .seconds(seconds), tokenRemaining: nil), + renewable: renewable, + renewer: recorder, + clock: clock + ), + token: "t0" + ) + } } - @Test("A driver revokes the lease it signed in with when it disconnects") - func disconnectRevokes() async throws { + @Test("A driver's disconnect hands the lease back, and the cache revokes it once it sits unused") + func disconnectHandsTheLeaseBack() async throws { let recorder = RevokeRecorder() + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) let adapter = PluginDriverAdapter( connection: TestFixtures.makeConnection(), pluginDriver: SilentPluginDriver(), - credentialLease: lease(recorder, clock: ManualClock()) + credentialLease: try await hold(cache, recorder: recorder, clock: clock) ) adapter.disconnect() adapter.disconnect() + // The lease's own end and the idle retention. + try await waitUntil { clock.pendingSleeperCount == 2 } + #expect(recorder.revokedLeases.isEmpty) + clock.advance(by: VaultLeaseCache.idleRetention) try await waitUntil { recorder.revokedLeases == ["database/creds/ro/1"] } } @@ -57,10 +133,11 @@ struct VaultLeaseOwnershipTests { func endedLeaseIsLostConnection() async throws { let recorder = RevokeRecorder() let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) let adapter = PluginDriverAdapter( connection: TestFixtures.makeConnection(), pluginDriver: SilentPluginDriver(), - credentialLease: lease(recorder, clock: clock, seconds: 60) + credentialLease: try await hold(cache, recorder: recorder, clock: clock, seconds: 60) ) #expect(!adapter.hasLostConnection) @@ -70,4 +147,182 @@ struct VaultLeaseOwnershipTests { try await waitUntil { adapter.hasLostConnection } adapter.disconnect() } + + @Test("A refused sign-in on a new lease dials again until the database accepts the user") + func youngLeaseRedials() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let driver = SlowToAcceptDriver(refusals: 2) + let adapter = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: driver, + credentialLease: try await hold(cache, recorder: RevokeRecorder(), clock: clock) + ) + + let connect = Task { try await adapter.connect() } + try await waitUntil { driver.attempts == 1 && clock.pendingSleeperCount == 2 } + clock.advance(by: .seconds(2)) + try await waitUntil { driver.attempts == 2 && clock.pendingSleeperCount == 2 } + clock.advance(by: .seconds(4)) + try await connect.value + + #expect(driver.attempts == 3) + #expect(adapter.status == .connected) + adapter.disconnect() + } + + @Test("A user still refused when the window closes fails with the reason and is kept for the next attempt") + func refusedThroughTheWindow() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let reads = Reads() + let recorder = RevokeRecorder() + let driver = SlowToAcceptDriver(refusals: 10) + let adapter = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: driver, + credentialLease: try await hold(cache, recorder: recorder, clock: clock, reads: reads) + ) + + let connect = Task { try await adapter.connect() } + for (attempt, wait) in [(1, 2), (2, 4), (3, 8), (4, 16)] { + try await waitUntil { driver.attempts == attempt && clock.pendingSleeperCount == 2 } + clock.advance(by: .seconds(wait)) + } + + let error = await #expect(throws: VaultUserNotLoaded.self) { try await connect.value } + #expect(driver.attempts == 5) + #expect(error?.pluginErrorCode == 1_045) + #expect(error?.recoverySuggestion != nil) + adapter.disconnect() + _ = try await hold(cache, recorder: recorder, clock: clock, reads: reads) + #expect(reads.value == 1) + } + + @Test("A refusal of a dial that started past the window fails at once and keeps a user Vault still has") + func oldLeaseFailsAtOnce() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let reads = Reads() + let recorder = RevokeRecorder() + let driver = SlowToAcceptDriver(refusals: 1) + let lease = try await hold(cache, recorder: recorder, clock: clock, reads: reads) + clock.advance(by: VaultLeaseCache.acceptanceWindow) + let adapter = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: driver, + credentialLease: lease + ) + + await #expect(throws: ServerRefusal.self) { try await adapter.connect() } + #expect(driver.attempts == 1) + adapter.disconnect() + _ = try await hold(cache, recorder: recorder, clock: clock, reads: reads) + #expect(reads.value == 1) + } + + @Test("A refused sign-in on any engine asks Vault, and a user Vault revoked is replaced, without dialing again") + func otherEngineRefusalReplacesTheUser() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let reads = Reads() + let recorder = RevokeRecorder() + let refusal = ServerRefusal( + pluginErrorMessage: "password authentication failed for user \"v-token-ro-1\"", + pluginErrorCode: nil, + pluginSqlState: nil + ) + let driver = SlowToAcceptDriver(refusals: 1, refusal: refusal) + let lease = try await hold(cache, recorder: recorder, clock: clock, renewable: true, reads: reads) + let adapter = PluginDriverAdapter( + connection: TestFixtures.makeConnection(type: .postgresql), + pluginDriver: driver, + credentialLease: lease + ) + + await #expect(throws: ServerRefusal.self) { try await adapter.connect() } + #expect(driver.attempts == 1) + try await waitUntil { lease.hasEnded } + adapter.disconnect() + _ = try await hold(cache, recorder: recorder, clock: clock, reads: reads) + #expect(reads.value == 2) + } + + @Test("A MySQL user a connection signed in with and another is refused for is not handed out again") + func acceptedUserRefusedLater() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let recorder = RevokeRecorder() + recorder.leaseExists = true + let first = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: SlowToAcceptDriver(refusals: 0), + credentialLease: try await hold(cache, recorder: recorder, clock: clock, renewable: true) + ) + try await first.connect() + clock.advance(by: VaultLeaseCache.acceptanceWindow) + let second = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: SlowToAcceptDriver(refusals: 1), + credentialLease: try await hold(cache, recorder: recorder, clock: clock, renewable: true) + ) + + await #expect(throws: ServerRefusal.self) { try await second.connect() } + second.disconnect() + first.disconnect() + + // A lease still handed out would wait out the idle retention instead. + try await waitUntil { recorder.revokedLeases == ["database/creds/ro/1"] } + } + + @Test("A refused database is not dialed again and keeps the user the profile signs in with") + func otherErrorFailsAtOnce() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let reads = Reads() + let recorder = RevokeRecorder() + let wrongDatabase = ServerRefusal( + pluginErrorMessage: "Access denied for user 'v-token-ro-1' to database 'orders'", + pluginErrorCode: 1_044, + pluginSqlState: "42000" + ) + let driver = SlowToAcceptDriver(refusals: 1, refusal: wrongDatabase) + let adapter = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: driver, + credentialLease: try await hold(cache, recorder: recorder, clock: clock, reads: reads) + ) + + await #expect(throws: ServerRefusal.self) { try await adapter.connect() } + #expect(driver.attempts == 1) + adapter.disconnect() + _ = try await hold(cache, recorder: recorder, clock: clock, reads: reads) + #expect(reads.value == 1) + } + + @Test("A connect cancelled while it waits for the database dials no more") + func cancelledWaitStopsDialing() async throws { + let clock = ManualClock() + let cache = VaultLeaseCache(clock: clock) + let driver = SlowToAcceptDriver(refusals: 5) + let ended = UUID() + let lease = try await hold(cache, recorder: RevokeRecorder(), clock: clock, for: ended) + // Unlisted, so letting go of it revokes it and leaves no idle timer among the sleepers counted below. + cache.endUse(connection: ended, live: []) + let adapter = PluginDriverAdapter( + connection: TestFixtures.makeConnection(), + pluginDriver: driver, + credentialLease: lease + ) + + let connect = Task { try await adapter.connect() } + try await waitUntil { driver.attempts == 1 && clock.pendingSleeperCount == 2 } + connect.cancel() + await #expect(throws: CancellationError.self) { try await connect.value } + // Neither the redial's wait nor the lease's schedule is left. + try await waitUntil { clock.pendingSleeperCount == 0 } + clock.advance(by: VaultLeaseCache.acceptanceWindow) + + #expect(driver.attempts == 1) + } } diff --git a/apple/TableProTests/Core/Utilities/PasswordSourceResolverTests.swift b/apple/TableProTests/Core/Utilities/PasswordSourceResolverTests.swift index 0cb1ce7b76..78b83b244f 100644 --- a/apple/TableProTests/Core/Utilities/PasswordSourceResolverTests.swift +++ b/apple/TableProTests/Core/Utilities/PasswordSourceResolverTests.swift @@ -259,8 +259,8 @@ struct PasswordSourceResolverTests { ) #expect(try await PasswordSourceResolver.resolve(source, for: context, cache: cache) == "1") - #expect(try await PasswordSourceResolver.test(source, for: context) == "2") - #expect(try await PasswordSourceResolver.test(source, for: context) == "3") + #expect(try await PasswordSourceResolver.test(source, for: context, leaseOwner: context.connectionId) == "2") + #expect(try await PasswordSourceResolver.test(source, for: context, leaseOwner: context.connectionId) == "3") #expect(try await PasswordSourceResolver.resolve(source, for: context, cache: cache) == "1") } @@ -270,7 +270,11 @@ struct PasswordSourceResolverTests { connectionId: UUID(), connectionName: "c", host: "db.example.com", port: 5432, user: "u", database: "d" ) - let output = try await PasswordSourceResolver.test(.command(shell: "echo {host}:$TABLEPRO_PORT"), for: context) + let output = try await PasswordSourceResolver.test( + .command(shell: "echo {host}:$TABLEPRO_PORT"), + for: context, + leaseOwner: context.connectionId + ) #expect(output == "db.example.com:5432") } diff --git a/apple/TableProTests/Models/Connection/ConnectionFailureClassifierTests.swift b/apple/TableProTests/Models/Connection/ConnectionFailureClassifierTests.swift index 01e828927e..04017ad967 100644 --- a/apple/TableProTests/Models/Connection/ConnectionFailureClassifierTests.swift +++ b/apple/TableProTests/Models/Connection/ConnectionFailureClassifierTests.swift @@ -11,7 +11,34 @@ import Foundation import TableProPluginKit import Testing +private struct ServerError: PluginDriverError { + let pluginErrorMessage: String + var pluginErrorCode: Int? + var pluginSqlState: String? +} + struct ConnectionFailureClassifierTests { + @Test("A refused login reads as one in each engine's own words", arguments: [ + "Login failed for user 'v-token-ro-1'.", + "ORA-01017: invalid username/password; logon denied", + "WRONGPASS invalid username-password pair or user is disabled.", + "Provided username v-token-ro-1 and/or password are incorrect", + "Incorrect username or password was specified.", + "password authentication failed for user \"v-token-ro-1\"", + ]) + func refusedLoginInEachEngine(message: String) { + #expect(ConnectionFailureClassifier.isAuthenticationFailure(ServerError(pluginErrorMessage: message))) + } + + @Test("A SQL Server login refused only while it upgrades, restarts or runs single-user is not a refused login", arguments: [ + "Login failed for user 'v-token-ro-1'. Reason: Server is in script upgrade mode. Only administrator can connect at this time.", + "Login failed for user 'v-token-ro-1'. Reason: Server is in single user mode. Only one administrator can connect at this time.", + "Cannot open database \"inventory\" requested by the login. The login failed.", + ]) + func passingSQLServerRefusals(message: String) { + #expect(!ConnectionFailureClassifier.isAuthenticationFailure(ServerError(pluginErrorMessage: message))) + } + @Test("A Swift cancellation is a cancel, not a failure") func swiftCancellationIsCancelled() { #expect(ConnectionFailureClassifier.isUserCancelled(CancellationError())) diff --git a/apple/TableProTests/Views/Connection/PasswordSourceTestModelTests.swift b/apple/TableProTests/Views/Connection/PasswordSourceTestModelTests.swift index a99dbec0aa..1b6250f752 100644 --- a/apple/TableProTests/Views/Connection/PasswordSourceTestModelTests.swift +++ b/apple/TableProTests/Views/Connection/PasswordSourceTestModelTests.swift @@ -22,13 +22,18 @@ struct PasswordSourceTestModelTests { @Test("A refused test reports why and never runs the source") func refusalSkipsTheRun() async { let runs = RunCounter() - let model = PasswordSourceTestModel { _, _ in + let model = PasswordSourceTestModel { _, _, _ in runs.increment() return "secret" } let source = PasswordSource.command(shell: "echo secret") - await model.run(source, context: context, refusal: PasswordSourceResolver.ResolutionError.storeNotTrusted) + await model.run( + source, + context: context, + leaseOwner: context.connectionId, + refusal: PasswordSourceResolver.ResolutionError.storeNotTrusted + ) #expect(runs.value == 0) #expect(model.outcome(for: source, context: context) == .failed( @@ -41,7 +46,7 @@ struct PasswordSourceTestModelTests { let model = PasswordSourceTestModel() let source = PasswordSource.command(shell: "echo hunter2") - await model.run(source, context: context, refusal: nil) + await model.run(source, context: context, leaseOwner: context.connectionId, refusal: nil) #expect(model.outcome(for: source, context: context) == .returned("hunter2")) #expect(!model.isRunning) @@ -52,7 +57,7 @@ struct PasswordSourceTestModelTests { let model = PasswordSourceTestModel() let source = PasswordSource.command(shell: "echo 'op: not signed in' >&2; exit 3") - await model.run(source, context: context, refusal: nil) + await model.run(source, context: context, leaseOwner: context.connectionId, refusal: nil) guard case .failed(let message) = model.outcome(for: source, context: context) else { Issue.record("Expected a failure") @@ -64,10 +69,10 @@ struct PasswordSourceTestModelTests { @Test("A result is shown only for the source that produced it") func outcomeBelongsToItsSource() async { - let model = PasswordSourceTestModel { _, _ in "secret" } + let model = PasswordSourceTestModel { _, _, _ in "secret" } let tested = PasswordSource.env(variable: "A") - await model.run(tested, context: context, refusal: nil) + await model.run(tested, context: context, leaseOwner: context.connectionId, refusal: nil) #expect(model.outcome(for: tested, context: context) == .returned("secret")) #expect(model.outcome(for: .env(variable: "B"), context: context) == nil) @@ -75,13 +80,13 @@ struct PasswordSourceTestModelTests { @Test("Editing the connection's details hides a result that used the old ones") func outcomeBelongsToItsConnectionDetails() async { - let model = PasswordSourceTestModel { _, context in context.host } + let model = PasswordSourceTestModel { _, context, _ in context.host } let source = PasswordSource.command(shell: "echo {host}") let moved = ConnectionScriptContext( connectionId: context.connectionId, connectionName: "c", host: "other", port: 1, user: "u", database: "d" ) - await model.run(source, context: context, refusal: nil) + await model.run(source, context: context, leaseOwner: context.connectionId, refusal: nil) #expect(model.outcome(for: source, context: context) == .returned("h")) #expect(model.outcome(for: source, context: moved) == nil) @@ -89,13 +94,13 @@ struct PasswordSourceTestModelTests { @Test("A new connection's details match from one render to the next") func outcomeIgnoresTheThrowawayId() async { - let model = PasswordSourceTestModel { _, _ in "secret" } + let model = PasswordSourceTestModel { _, _, _ in "secret" } let source = PasswordSource.env(variable: "A") let rerendered = ConnectionScriptContext( connectionId: UUID(), connectionName: "c", host: "h", port: 1, user: "u", database: "d" ) - await model.run(source, context: context, refusal: nil) + await model.run(source, context: context, leaseOwner: context.connectionId, refusal: nil) #expect(model.outcome(for: source, context: rerendered) == .returned("secret")) } diff --git a/apple/changelog.d/4230.fixed.1.md b/apple/changelog.d/4230.fixed.1.md new file mode 100644 index 0000000000..5a08d62022 --- /dev/null +++ b/apple/changelog.d/4230.fixed.1.md @@ -0,0 +1 @@ +A connection with a credential profile signed in without the profile's credentials when it used a tunnel. diff --git a/apple/changelog.d/4230.fixed.2.md b/apple/changelog.d/4230.fixed.2.md new file mode 100644 index 0000000000..00a761099b --- /dev/null +++ b/apple/changelog.d/4230.fixed.2.md @@ -0,0 +1 @@ +A refused SQL Server login showed a generic message instead of the server's reason. diff --git a/apple/changelog.d/4230.fixed.3.md b/apple/changelog.d/4230.fixed.3.md new file mode 100644 index 0000000000..2612d2ff99 --- /dev/null +++ b/apple/changelog.d/4230.fixed.3.md @@ -0,0 +1 @@ +Quitting TablePro left every Vault database lease alive until it expired. diff --git a/apple/changelog.d/4230.fixed.md b/apple/changelog.d/4230.fixed.md new file mode 100644 index 0000000000..57611a3efa --- /dev/null +++ b/apple/changelog.d/4230.fixed.md @@ -0,0 +1 @@ +A Vault database role leased a new user on every sign-in, so a database slow to load users never accepted one. diff --git a/apple/scripts/probes/check-freetds-login-messages.sh b/apple/scripts/probes/check-freetds-login-messages.sh new file mode 100755 index 0000000000..22dd78d255 --- /dev/null +++ b/apple/scripts/probes/check-freetds-login-messages.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# +# Check how FreeTDS db-lib reports a refused login to the SQL Server driver, through the shipped Libs/libsybdb.a +# and the stub headers in Plugins/MSSQLDriverPlugin/CFreeTDS that the driver compiles against. +# +# db-lib hands the message and error handlers the DBPROCESS it is still building, not NULL, and frees it when the +# login fails. FreeTDSConnection filed those messages under that handle, so a failed open read an empty reason and +# reported only "Check host, port, credentials, and TLS settings", and nothing could tell a refused login apart. +# freetdsRecordMessage now sends an error for a handle with no gate yet to the open's own reason, first one first. +# A 4060 (the database cannot be opened) arrives before its 18456, so the reason stays the 4060. Each server message +# is followed by db-lib's 20018 notice, which the driver skips (MSSQLLibraryError.serverMessageNotice). +# +# No SQL Server or Docker: freetds-login-messages/fake_login.py answers PRELOGIN and refuses LOGIN7 with the given +# ERROR tokens, and freetds-login-messages/client.c logs in the way the driver does. Run it after a FreeTDS bump. +# +# Usage: +# scripts/probes/check-freetds-login-messages.sh +# +# Exits 0 when db-lib answers as expected, 1 when it differs, 3 when it cannot run. + +set -uo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +HELPERS="$ROOT/scripts/probes/freetds-login-messages" +HEADERS="$ROOT/Plugins/MSSQLDriverPlugin/CFreeTDS/include" +DYLIBS="$ROOT/Libs/dylibs" +WORK="$(mktemp -d)" +SERVER_PID="" +trap '[ -z "$SERVER_PID" ] || { kill "$SERVER_PID"; wait "$SERVER_PID"; } 2> /dev/null; rm -rf "$WORK"' EXIT + +cannot_run() { + echo "$1" >&2 + exit 3 +} + +command -v python3 > /dev/null 2>&1 || cannot_run "no python3" +command -v cc > /dev/null 2>&1 || cannot_run "no cc" + +LIB="" +for candidate in "$ROOT/Libs/libsybdb.a" "$ROOT/Libs/libsybdb_$(uname -m).a"; do + if [ -f "$candidate" ]; then + LIB="$candidate" + break + fi +done +[ -n "$LIB" ] || cannot_run "not found: Libs/libsybdb.a (run scripts/download-libs.sh)" +for dylib in libssl.3.dylib libcrypto.3.dylib; do + [ -f "$DYLIBS/$dylib" ] || cannot_run "not found: Libs/dylibs/$dylib (run scripts/download-libs.sh)" +done + +# The MSSQLDriver target's link line in project.yml. +if ! cc -I "$HEADERS" "$HELPERS/client.c" "$LIB" -L "$DYLIBS" -lssl.3 -lcrypto.3 -liconv -framework GSS -lcom_err \ + -Wl,-rpath,"$DYLIBS" -o "$WORK/client" > "$WORK/cc.log" 2>&1; then + cat "$WORK/cc.log" >&2 + cannot_run "client.c failed to build" +fi + +python3 -I "$HELPERS/fake_login.py" \ + "4060:11:Cannot open database \"inventory\" requested by the login. The login failed." \ + "18456:14:Login failed for user 'v-token-ro-1'." \ + > "$WORK/port" 2> "$WORK/server.log" & +SERVER_PID=$! +PORT="" +for _ in $(seq 1 50); do + PORT="$(head -n 1 "$WORK/port")" + [ -n "$PORT" ] && break + sleep 0.1 +done +if [ -z "$PORT" ]; then + cat "$WORK/server.log" >&2 + cannot_run "fake_login.py never reported a port" +fi + +printf '[fake]\n\thost = 127.0.0.1\n\tport = %s\n\ttds version = 7.4\n\tconnect timeout = 5\n\tencryption = off\n' \ + "$PORT" > "$WORK/freetds.conf" +"$WORK/client" "$WORK/freetds.conf" fake > "$WORK/actual" 2> "$WORK/client.log" + +cat > "$WORK/expected" << 'EXPECTED' +msg proc=SET 4060 sev 11: Cannot open database "inventory" requested by the login. The login failed. +err proc=SET dberr=20018: General SQL Server error: Check messages from the SQL Server +msg proc=SET 18456 sev 14: Login failed for user 'v-token-ro-1'. +err proc=SET dberr=20018: General SQL Server error: Check messages from the SQL Server +err proc=SET dberr=20002: Adaptive Server connection failed (fake) +dbopen=NULL +EXPECTED + +cat "$WORK/actual" +if diff -u "$WORK/expected" "$WORK/actual" > "$WORK/diff"; then + echo "OK: db-lib reports the login refusal as expected" + exit 0 +fi + +echo "" >&2 +echo "FAIL: db-lib's answers differ (- expected, + read):" >&2 +tail -n +3 "$WORK/diff" >&2 +cat "$WORK/client.log" "$WORK/server.log" >&2 +exit 1 diff --git a/apple/scripts/probes/freetds-login-messages/client.c b/apple/scripts/probes/freetds-login-messages/client.c new file mode 100644 index 0000000000..7082e8e5f6 --- /dev/null +++ b/apple/scripts/probes/freetds-login-messages/client.c @@ -0,0 +1,33 @@ +// Prints whether db-lib hands the message and error handlers a DBPROCESS during a refused login. +#include +#include +#include +#include + +static int on_error(DBPROCESS *proc, int severity, int dberr, int oserr, char *text, char *ostext) +{ + printf("err proc=%s dberr=%d: %s\n", proc ? "SET" : "NULL", dberr, text ? text : ""); + return INT_CANCEL; +} + +static int on_message(DBPROCESS *proc, DBINT number, int state, int severity, char *text, char *server, + char *procedure, int line) +{ + printf("msg proc=%s %d sev %d: %s\n", proc ? "SET" : "NULL", (int) number, severity, text ? text : ""); + return 0; +} + +int main(int argc, char **argv) +{ + dbinit(); + dbsetifile(argv[1]); + dberrhandle((EHANDLEFUNC) on_error); + dbmsghandle((MHANDLEFUNC) on_message); + LOGINREC *login = dblogin(); + DBSETLUSER(login, "v-token-ro-1"); + DBSETLPWD(login, "secret"); + dbsetlname(login, "inventory", DBSETDBNAME); + DBPROCESS *proc = dbopen(login, argv[2]); + printf("dbopen=%s\n", proc ? "SET" : "NULL"); + return 0; +} diff --git a/apple/scripts/probes/freetds-login-messages/fake_login.py b/apple/scripts/probes/freetds-login-messages/fake_login.py new file mode 100644 index 0000000000..ee80922a2a --- /dev/null +++ b/apple/scripts/probes/freetds-login-messages/fake_login.py @@ -0,0 +1,88 @@ +# Answers PRELOGIN, then refuses LOGIN7 with the ERROR tokens given on the command line. +# Usage: python3 -I fake_login.py "4060:11:Cannot open database ..." "18456:14:Login failed for user 'v'." +import socket +import struct +import sys + + +def b_varchar(text): + return bytes([len(text)]) + text.encode("utf-16-le") + + +def us_varchar(text): + return struct.pack("BHH", 0, offset, len(version)) + + struct.pack(">BHH", 1, offset + len(version), len(enc)) + + b"\xff" + version + enc) + + +def error(number, severity, text): + body = (struct.pack("BBH", header[:4]) + body = receive(conn, length - 8) + if body is None: + return None + payload += body + if status & 0x01: + return kind + + +def packet(payload): + return struct.pack(">BBHHBB", 0x04, 1, len(payload) + 8, 0, 1, 0) + payload + + +def main(): + server = socket.create_server(("127.0.0.1", 0)) + server.settimeout(20) + print(server.getsockname()[1], flush=True) + conn, _ = server.accept() + conn.settimeout(20) + with conn: + while True: + kind = read_message(conn) + if kind is None: + return + if kind == 0x12: + conn.sendall(packet(prelogin())) + elif kind == 0x10: + conn.sendall(packet(refusal())) + else: + return + + +if __name__ == "__main__": + main()