From 7d2e898dac741bf42f7bc417c9e5b9df9706fdd8 Mon Sep 17 00:00:00 2001 From: Foad Kesheh Date: Wed, 16 Sep 2026 14:49:14 -0300 Subject: [PATCH] fix(ui): preserve local terminals during cloud outages --- docs/cloud-outage-recovery.md | 29 +++++++ .../components/CloudConnectionNotice.test.ts | 36 +++++++++ ui/src/components/CloudConnectionNotice.tsx | 35 +++++++++ ui/src/components/ConnectionDiagnostics.tsx | 6 +- ui/src/components/Dashboard.tsx | 27 +++++-- .../hybrid-terminal-transport.test.ts | 77 +++++++++++++++++++ .../hybrid-terminal-transport.ts | 18 +++-- 7 files changed, 214 insertions(+), 14 deletions(-) create mode 100644 docs/cloud-outage-recovery.md create mode 100644 ui/src/components/CloudConnectionNotice.test.ts create mode 100644 ui/src/components/CloudConnectionNotice.tsx diff --git a/docs/cloud-outage-recovery.md b/docs/cloud-outage-recovery.md new file mode 100644 index 0000000..a88d1bf --- /dev/null +++ b/docs/cloud-outage-recovery.md @@ -0,0 +1,29 @@ +# Cloud outage recovery + +The dashboard keeps existing Local/P2P terminals available while Centrifugo is +unreachable. A nonblocking notice replaces the automatic full-screen error panel; +diagnostics are available on demand and can be dismissed without reloading. +Centrifuge continues its existing reconnect/resubscribe cycle. Once connected, +the notice disappears and cloud subscriptions resume without replacing healthy +direct terminal connections. + +The terminal transport remembers bridge loopback advertisements learned through +authenticated cloud presence. If presence subsequently fails, times out, or loses +the bridge, the normal retry loop can still attempt a nonce-authenticated localhost +connection. A successful presence response refreshes advertisements, including +rotated credentials. Output switches paths only after receiving the new terminal +screen, using the existing transport switchover logic. + +Advertisements remain in memory within the authenticated transport instance and +are cleared on disposal. They are not written to browser storage. This supports +outages in an already-loaded dashboard, not cold offline startup or page reloads. +A bridge restarted with a different port/nonce needs cloud discovery again. + +Session creation, loops, and other management RPCs still require Centrifugo. +Local terminal input, output, and resize use the direct connection. A remote bridge +without a reachable direct path remains unavailable until a path recovers. + +Coverage: `CloudConnectionNotice.test.ts` checks nonblocking/dismissible diagnostics +and recovery; `hybrid-terminal-transport.test.ts` checks reconnection during rejected, +hanging, and empty presence responses, continued terminal I/O, and refreshed +credentials after cloud recovery. diff --git a/ui/src/components/CloudConnectionNotice.test.ts b/ui/src/components/CloudConnectionNotice.test.ts new file mode 100644 index 0000000..0c70038 --- /dev/null +++ b/ui/src/components/CloudConnectionNotice.test.ts @@ -0,0 +1,36 @@ +// @vitest-environment jsdom +import { createElement } from 'react'; +import { cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { afterEach, expect, it, vi } from 'vitest'; +import { CloudConnectionNotice } from './CloudConnectionNotice'; + +afterEach(cleanup); + +it('leaves terminals interactive during an outage and hides the notice on recovery', () => { + const input = vi.fn(); + const props = { + connectionStatus: 'error' as const, + connectionError: 'transport closed', + centrifugoUrl: 'wss://example.com/connection/websocket', + token: '', + onRetry: vi.fn(), + }; + const view = render(createElement('div', null, + createElement('button', { onClick: input }, 'Terminal'), + createElement(CloudConnectionNotice, props), + )); + expect(screen.queryByText('Connection Failed')).toBeNull(); + expect(screen.getByRole('status').textContent).toContain('reconnecting automatically'); + fireEvent.click(screen.getByText('Terminal')); + expect(input).toHaveBeenCalledTimes(1); + fireEvent.click(screen.getByText('Connection details')); + expect(screen.getByText('Connection Failed')).toBeTruthy(); + fireEvent.click(screen.getByText('Back to terminals')); + expect(screen.queryByText('Connection Failed')).toBeNull(); + fireEvent.click(screen.getByText('Connection details')); + view.rerender(createElement(CloudConnectionNotice, { ...props, connectionStatus: 'connected' })); + expect(screen.queryByRole('status')).toBeNull(); + view.rerender(createElement(CloudConnectionNotice, props)); + expect(screen.queryByText('Connection Failed')).toBeNull(); + expect(props.onRetry).not.toHaveBeenCalled(); +}); diff --git a/ui/src/components/CloudConnectionNotice.tsx b/ui/src/components/CloudConnectionNotice.tsx new file mode 100644 index 0000000..2400896 --- /dev/null +++ b/ui/src/components/CloudConnectionNotice.tsx @@ -0,0 +1,35 @@ +"use client"; + +import { useEffect, useState } from "react"; +import type { ConnectionStatus } from "@/hooks/useCentrifugo"; +import { ConnectionDiagnostics } from "./ConnectionDiagnostics"; + +interface Props { + connectionStatus: ConnectionStatus; + connectionError: string | null; + centrifugoUrl: string; + token: string; + onRetry: () => void; +} + +/** Cloud availability must never cover or unmount working local terminals. */ +export function CloudConnectionNotice(props: Props) { + const [details, setDetails] = useState(false); + useEffect(() => { + if (props.connectionStatus === "connected") setDetails(false); + }, [props.connectionStatus]); + + if (props.connectionStatus === "connected") return null; + + return ( + <> +
+
Cloud disconnected — reconnecting automatically. Local/P2P terminals remain available when reachable. Cloud controls resume after reconnecting.
+ +
+ {details && ( + setDetails(false)} /> + )} + + ); +} diff --git a/ui/src/components/ConnectionDiagnostics.tsx b/ui/src/components/ConnectionDiagnostics.tsx index a29fb6c..1d056eb 100644 --- a/ui/src/components/ConnectionDiagnostics.tsx +++ b/ui/src/components/ConnectionDiagnostics.tsx @@ -15,6 +15,7 @@ interface ConnectionDiagnosticsProps { centrifugoUrl: string; token: string; onRetry: () => void; + onDismiss?: () => void; } function decodeJwtPayload(token: string): Record | null { @@ -51,7 +52,7 @@ function StatusIcon({ status }: { status: DiagnosticCheck["status"] }) { ); } -export function ConnectionDiagnostics({ connectionStatus, connectionError, centrifugoUrl, token, onRetry }: ConnectionDiagnosticsProps) { +export function ConnectionDiagnostics({ connectionStatus, connectionError, centrifugoUrl, token, onRetry, onDismiss }: ConnectionDiagnosticsProps) { const [checks, setChecks] = useState([]); const [running, setRunning] = useState(false); const [hasRun, setHasRun] = useState(false); @@ -201,7 +202,7 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr }, [token, centrifugoUrl, updateCheck]); if (connectionStatus === "connected") return null; - if (connectionStatus === "connecting" && !hasRun) return null; + if (connectionStatus === "connecting" && !hasRun && !onDismiss) return null; const hasFail = checks.some((c) => c.status === "fail"); const allPass = checks.length > 0 && checks.every((c) => c.status === "pass"); @@ -233,6 +234,7 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr
Connection Failed
+ {onDismiss && } {connectionError && (
{connectionError} diff --git a/ui/src/components/Dashboard.tsx b/ui/src/components/Dashboard.tsx index 5e25278..a1b8e2e 100644 --- a/ui/src/components/Dashboard.tsx +++ b/ui/src/components/Dashboard.tsx @@ -23,7 +23,7 @@ import { NewFactoryModal } from "./factory/NewFactoryModal"; import { deriveFactories } from "./factory/useFactory"; import type { FactoryInfo, NewFactoryInput } from "./factory/types"; import { factoryInitPrompt, factoryKey, factoryWorkerOf } from "./factory/types"; -import { ConnectionDiagnostics } from "./ConnectionDiagnostics"; +import { CloudConnectionNotice } from "./CloudConnectionNotice"; import { SessionAttentionAlert } from "./SessionAttentionAlert"; import { mergeBridgeOrder } from "@/lib/bridge-order"; import { StatusDot } from "@/lib/StatusDot"; @@ -860,18 +860,31 @@ PY`; {/* Connection status */} -
- +
+ {directlyReachableBridgeIds.size > 0 && ( + <> + + + Local/P2P connected + + + + )} + 0 ? "connecting" : connectionStatus} + title={connectionStatus === "connected" ? "Cloud connected" : "Cloud reconnecting"} + /> 0 + ? "var(--status-pending)" + : connectionStatus === "error" || connectionStatus === "disconnected" ? "var(--status-error)" : "var(--text-muted)", - textTransform: "capitalize", }} > - {connectionStatus} + {connectionStatus === "connected" ? "Cloud connected" : "Cloud reconnecting"}
@@ -1410,7 +1423,7 @@ PY`; /> )} - { + it.each(['reject', 'hang', 'empty'])('reconnects locally when presence is %s, then refreshes credentials on cloud recovery', async (failure) => { + vi.useFakeTimers(); + let state = 'online'; + let nonce = 'first-nonce'; + class PresenceSub extends FakeSubscription { + presence(): Promise<{ clients: Record }> { + if (state === 'reject') return Promise.reject(new Error('cloud unavailable')); + if (state === 'hang') return new Promise(() => {}); + const clients: Record = state === 'empty' ? {} : { + bridge: { connInfo: { bridgeId: 'bridge-1', localPort: 41999, localNonce: nonce } }, + }; + return Promise.resolve({ clients }); + } + } + class Client extends FakeCentrifugeClient { + override newSubscription(channel: string): FakeSubscription { + if (channel.startsWith('bridges:presence')) { + const sub = new PresenceSub(channel); + this.subs.set(channel, sub); + return sub; + } + return super.newSubscription(channel); + } + } + const localPeers: FakePeer[] = []; + const loopbackPeerFactory = vi.fn((opts: { bridgeId: string }) => { + const p = new FakePeer(opts.bridgeId); + localPeers.push(p); + return p; + }); + const transport = new HybridTerminalTransport({ + centrifuge: new Client(), userId: 'user', clientId: 'tab', + publishCommand: vi.fn(), loopbackPeerFactory, + peerFactory: (opts: PeerFactoryOpts) => { + const p = new FakePeer(opts.bridgeId); + p.connectMode = 'reject'; + return p; + }, + upgradeBackoffMs: [1000], upgradeJitter: 0, + } as unknown as ConstructorParameters[0]); + try { + const output = vi.fn(); + transport.subscribeTerminal('session', 'bridge-1', { onOutput: output, onScreen: vi.fn() }); + await vi.advanceTimersByTimeAsync(0); + expect(transport.getMode('session')).toBe('local'); + state = failure; + transport.sendInput('session', 'still-local'); + expect(localPeers[0].sendInputCalls).toContainEqual(['session', 'still-local']); + localPeers[0].simulateClose(); + await vi.advanceTimersByTimeAsync(1000 + LOOPBACK_TIMEOUT_MS); + expect(localPeers).toHaveLength(2); + expect(loopbackPeerFactory).toHaveBeenLastCalledWith(expect.objectContaining({ nonce: 'first-nonce' })); + localPeers[1].attachCalls[0][1].onScreen('reconnected'); + expect(transport.getMode('session')).toBe('local'); + localPeers[1].attachCalls[0][1].onOutput('working offline'); + expect(output).toHaveBeenCalledWith('working offline'); + transport.sendResize('session', 100, 30); + expect(localPeers[1].sendResizeCalls).toContainEqual(['session', 100, 30]); + state = 'online'; + nonce = 'rotated-nonce'; + // Cloud recovery alone must not replace a working direct peer. + await vi.advanceTimersByTimeAsync(3000); + expect(localPeers).toHaveLength(2); + localPeers[1].simulateClose(); + await vi.advanceTimersByTimeAsync(1000); + expect(loopbackPeerFactory).toHaveBeenLastCalledWith(expect.objectContaining({ nonce: 'rotated-nonce' })); + localPeers[2].attachCalls[0][1].onScreen('fresh screen'); + expect(transport.getMode('session')).toBe('local'); + } finally { + transport.dispose(); + vi.useRealTimers(); + } + }); +}); diff --git a/ui/src/lib/direct-transport/hybrid-terminal-transport.ts b/ui/src/lib/direct-transport/hybrid-terminal-transport.ts index cb1bb3a..008f607 100644 --- a/ui/src/lib/direct-transport/hybrid-terminal-transport.ts +++ b/ui/src/lib/direct-transport/hybrid-terminal-transport.ts @@ -228,6 +228,9 @@ export class HybridTerminalTransport implements TerminalTransportApi { private readonly upgradeJitter: number; private readonly peers = new Map(); + // Scoped to this authenticated transport's lifetime; never persist nonces. + // Cloud presence can disappear while the loopback server is still healthy. + private readonly localAdverts = new Map(); private readonly sessions = new Map(); private readonly modeChangeCbs = new Set<(sessionId: string, mode: TerminalTransportMode) => void>(); private readonly bridgeReachabilityCbs = new Set< @@ -396,6 +399,7 @@ export class HybridTerminalTransport implements TerminalTransportApi { pe.peer?.close(); } this.peers.clear(); + this.localAdverts.clear(); this.modeChangeCbs.clear(); this.bridgeReachabilityCbs.clear(); @@ -785,6 +789,9 @@ export class HybridTerminalTransport implements TerminalTransportApi { */ private async isBridgeOnlineForUpgrade(bridgeId: string): Promise { if (this.advertInjected) return true; + // Let the authenticated loopback handshake determine local reachability, + // even when Fly cannot answer presence queries. + if (this.localAdverts.has(bridgeId)) return true; const sub = this.ensurePresenceSub(); if (typeof sub.presence !== 'function') return false; const result = await this.readPresenceBounded(sub); @@ -1191,24 +1198,25 @@ export class HybridTerminalTransport implements TerminalTransportApi { } private async defaultGetLocalAdvert(bridgeId: string): Promise { + const cached = this.localAdverts.get(bridgeId) ?? null; const sub = this.ensurePresenceSub(); - if (typeof sub.presence !== 'function') return null; + if (typeof sub.presence !== 'function') return cached; const result = await this.readPresenceBounded(sub); - if (!result) return null; + if (!result) return cached; for (const info of Object.values(result.clients)) { const ci = info.connInfo as | { bridgeId?: string; localPort?: number; localNonce?: string } | undefined; if ( ci && - ci.bridgeId === bridgeId && + typeof ci.bridgeId === 'string' && typeof ci.localPort === 'number' && typeof ci.localNonce === 'string' ) { - return { localPort: ci.localPort, localNonce: ci.localNonce }; + this.localAdverts.set(ci.bridgeId, { localPort: ci.localPort, localNonce: ci.localNonce }); } } - return null; + return this.localAdverts.get(bridgeId) ?? null; } /**