diff --git a/docs/cloud-outage-recovery.md b/docs/cloud-outage-recovery.md
new file mode 100644
index 0000000..a88d1bf
--- /dev/null
+++ b/docs/cloud-outage-recovery.md
@@ -0,0 +1,29 @@
+# Cloud outage recovery
+
+The dashboard keeps existing Local/P2P terminals available while Centrifugo is
+unreachable. A nonblocking notice replaces the automatic full-screen error panel;
+diagnostics are available on demand and can be dismissed without reloading.
+Centrifuge continues its existing reconnect/resubscribe cycle. Once connected,
+the notice disappears and cloud subscriptions resume without replacing healthy
+direct terminal connections.
+
+The terminal transport remembers bridge loopback advertisements learned through
+authenticated cloud presence. If presence subsequently fails, times out, or loses
+the bridge, the normal retry loop can still attempt a nonce-authenticated localhost
+connection. A successful presence response refreshes advertisements, including
+rotated credentials. Output switches paths only after receiving the new terminal
+screen, using the existing transport switchover logic.
+
+Advertisements remain in memory within the authenticated transport instance and
+are cleared on disposal. They are not written to browser storage. This supports
+outages in an already-loaded dashboard, not cold offline startup or page reloads.
+A bridge restarted with a different port/nonce needs cloud discovery again.
+
+Session creation, loops, and other management RPCs still require Centrifugo.
+Local terminal input, output, and resize use the direct connection. A remote bridge
+without a reachable direct path remains unavailable until a path recovers.
+
+Coverage: `CloudConnectionNotice.test.ts` checks nonblocking/dismissible diagnostics
+and recovery; `hybrid-terminal-transport.test.ts` checks reconnection during rejected,
+hanging, and empty presence responses, continued terminal I/O, and refreshed
+credentials after cloud recovery.
diff --git a/ui/src/components/CloudConnectionNotice.test.ts b/ui/src/components/CloudConnectionNotice.test.ts
new file mode 100644
index 0000000..0c70038
--- /dev/null
+++ b/ui/src/components/CloudConnectionNotice.test.ts
@@ -0,0 +1,36 @@
+// @vitest-environment jsdom
+import { createElement } from 'react';
+import { cleanup, fireEvent, render, screen } from '@testing-library/react';
+import { afterEach, expect, it, vi } from 'vitest';
+import { CloudConnectionNotice } from './CloudConnectionNotice';
+
+afterEach(cleanup);
+
+it('leaves terminals interactive during an outage and hides the notice on recovery', () => {
+ const input = vi.fn();
+ const props = {
+ connectionStatus: 'error' as const,
+ connectionError: 'transport closed',
+ centrifugoUrl: 'wss://example.com/connection/websocket',
+ token: '',
+ onRetry: vi.fn(),
+ };
+ const view = render(createElement('div', null,
+ createElement('button', { onClick: input }, 'Terminal'),
+ createElement(CloudConnectionNotice, props),
+ ));
+ expect(screen.queryByText('Connection Failed')).toBeNull();
+ expect(screen.getByRole('status').textContent).toContain('reconnecting automatically');
+ fireEvent.click(screen.getByText('Terminal'));
+ expect(input).toHaveBeenCalledTimes(1);
+ fireEvent.click(screen.getByText('Connection details'));
+ expect(screen.getByText('Connection Failed')).toBeTruthy();
+ fireEvent.click(screen.getByText('Back to terminals'));
+ expect(screen.queryByText('Connection Failed')).toBeNull();
+ fireEvent.click(screen.getByText('Connection details'));
+ view.rerender(createElement(CloudConnectionNotice, { ...props, connectionStatus: 'connected' }));
+ expect(screen.queryByRole('status')).toBeNull();
+ view.rerender(createElement(CloudConnectionNotice, props));
+ expect(screen.queryByText('Connection Failed')).toBeNull();
+ expect(props.onRetry).not.toHaveBeenCalled();
+});
diff --git a/ui/src/components/CloudConnectionNotice.tsx b/ui/src/components/CloudConnectionNotice.tsx
new file mode 100644
index 0000000..2400896
--- /dev/null
+++ b/ui/src/components/CloudConnectionNotice.tsx
@@ -0,0 +1,35 @@
+"use client";
+
+import { useEffect, useState } from "react";
+import type { ConnectionStatus } from "@/hooks/useCentrifugo";
+import { ConnectionDiagnostics } from "./ConnectionDiagnostics";
+
+interface Props {
+ connectionStatus: ConnectionStatus;
+ connectionError: string | null;
+ centrifugoUrl: string;
+ token: string;
+ onRetry: () => void;
+}
+
+/** Cloud availability must never cover or unmount working local terminals. */
+export function CloudConnectionNotice(props: Props) {
+ const [details, setDetails] = useState(false);
+ useEffect(() => {
+ if (props.connectionStatus === "connected") setDetails(false);
+ }, [props.connectionStatus]);
+
+ if (props.connectionStatus === "connected") return null;
+
+ return (
+ <>
+
+
Cloud disconnected — reconnecting automatically. Local/P2P terminals remain available when reachable. Cloud controls resume after reconnecting.
+
+
+ {details && (
+ setDetails(false)} />
+ )}
+ >
+ );
+}
diff --git a/ui/src/components/ConnectionDiagnostics.tsx b/ui/src/components/ConnectionDiagnostics.tsx
index a29fb6c..1d056eb 100644
--- a/ui/src/components/ConnectionDiagnostics.tsx
+++ b/ui/src/components/ConnectionDiagnostics.tsx
@@ -15,6 +15,7 @@ interface ConnectionDiagnosticsProps {
centrifugoUrl: string;
token: string;
onRetry: () => void;
+ onDismiss?: () => void;
}
function decodeJwtPayload(token: string): Record | null {
@@ -51,7 +52,7 @@ function StatusIcon({ status }: { status: DiagnosticCheck["status"] }) {
);
}
-export function ConnectionDiagnostics({ connectionStatus, connectionError, centrifugoUrl, token, onRetry }: ConnectionDiagnosticsProps) {
+export function ConnectionDiagnostics({ connectionStatus, connectionError, centrifugoUrl, token, onRetry, onDismiss }: ConnectionDiagnosticsProps) {
const [checks, setChecks] = useState([]);
const [running, setRunning] = useState(false);
const [hasRun, setHasRun] = useState(false);
@@ -201,7 +202,7 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr
}, [token, centrifugoUrl, updateCheck]);
if (connectionStatus === "connected") return null;
- if (connectionStatus === "connecting" && !hasRun) return null;
+ if (connectionStatus === "connecting" && !hasRun && !onDismiss) return null;
const hasFail = checks.some((c) => c.status === "fail");
const allPass = checks.length > 0 && checks.every((c) => c.status === "pass");
@@ -233,6 +234,7 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr
Connection Failed
+ {onDismiss && }
{connectionError && (
{connectionError}
diff --git a/ui/src/components/Dashboard.tsx b/ui/src/components/Dashboard.tsx
index 5e25278..a1b8e2e 100644
--- a/ui/src/components/Dashboard.tsx
+++ b/ui/src/components/Dashboard.tsx
@@ -23,7 +23,7 @@ import { NewFactoryModal } from "./factory/NewFactoryModal";
import { deriveFactories } from "./factory/useFactory";
import type { FactoryInfo, NewFactoryInput } from "./factory/types";
import { factoryInitPrompt, factoryKey, factoryWorkerOf } from "./factory/types";
-import { ConnectionDiagnostics } from "./ConnectionDiagnostics";
+import { CloudConnectionNotice } from "./CloudConnectionNotice";
import { SessionAttentionAlert } from "./SessionAttentionAlert";
import { mergeBridgeOrder } from "@/lib/bridge-order";
import { StatusDot } from "@/lib/StatusDot";
@@ -860,18 +860,31 @@ PY`;
{/* Connection status */}
-
-
+
+ {directlyReachableBridgeIds.size > 0 && (
+ <>
+
+
+ Local/P2P connected
+
+ ·
+ >
+ )}
+ 0 ? "connecting" : connectionStatus}
+ title={connectionStatus === "connected" ? "Cloud connected" : "Cloud reconnecting"}
+ />
0
+ ? "var(--status-pending)"
+ : connectionStatus === "error" || connectionStatus === "disconnected"
? "var(--status-error)"
: "var(--text-muted)",
- textTransform: "capitalize",
}}
>
- {connectionStatus}
+ {connectionStatus === "connected" ? "Cloud connected" : "Cloud reconnecting"}
@@ -1410,7 +1423,7 @@ PY`;
/>
)}
-
{
+ it.each(['reject', 'hang', 'empty'])('reconnects locally when presence is %s, then refreshes credentials on cloud recovery', async (failure) => {
+ vi.useFakeTimers();
+ let state = 'online';
+ let nonce = 'first-nonce';
+ class PresenceSub extends FakeSubscription {
+ presence(): Promise<{ clients: Record }> {
+ if (state === 'reject') return Promise.reject(new Error('cloud unavailable'));
+ if (state === 'hang') return new Promise(() => {});
+ const clients: Record = state === 'empty' ? {} : {
+ bridge: { connInfo: { bridgeId: 'bridge-1', localPort: 41999, localNonce: nonce } },
+ };
+ return Promise.resolve({ clients });
+ }
+ }
+ class Client extends FakeCentrifugeClient {
+ override newSubscription(channel: string): FakeSubscription {
+ if (channel.startsWith('bridges:presence')) {
+ const sub = new PresenceSub(channel);
+ this.subs.set(channel, sub);
+ return sub;
+ }
+ return super.newSubscription(channel);
+ }
+ }
+ const localPeers: FakePeer[] = [];
+ const loopbackPeerFactory = vi.fn((opts: { bridgeId: string }) => {
+ const p = new FakePeer(opts.bridgeId);
+ localPeers.push(p);
+ return p;
+ });
+ const transport = new HybridTerminalTransport({
+ centrifuge: new Client(), userId: 'user', clientId: 'tab',
+ publishCommand: vi.fn(), loopbackPeerFactory,
+ peerFactory: (opts: PeerFactoryOpts) => {
+ const p = new FakePeer(opts.bridgeId);
+ p.connectMode = 'reject';
+ return p;
+ },
+ upgradeBackoffMs: [1000], upgradeJitter: 0,
+ } as unknown as ConstructorParameters[0]);
+ try {
+ const output = vi.fn();
+ transport.subscribeTerminal('session', 'bridge-1', { onOutput: output, onScreen: vi.fn() });
+ await vi.advanceTimersByTimeAsync(0);
+ expect(transport.getMode('session')).toBe('local');
+ state = failure;
+ transport.sendInput('session', 'still-local');
+ expect(localPeers[0].sendInputCalls).toContainEqual(['session', 'still-local']);
+ localPeers[0].simulateClose();
+ await vi.advanceTimersByTimeAsync(1000 + LOOPBACK_TIMEOUT_MS);
+ expect(localPeers).toHaveLength(2);
+ expect(loopbackPeerFactory).toHaveBeenLastCalledWith(expect.objectContaining({ nonce: 'first-nonce' }));
+ localPeers[1].attachCalls[0][1].onScreen('reconnected');
+ expect(transport.getMode('session')).toBe('local');
+ localPeers[1].attachCalls[0][1].onOutput('working offline');
+ expect(output).toHaveBeenCalledWith('working offline');
+ transport.sendResize('session', 100, 30);
+ expect(localPeers[1].sendResizeCalls).toContainEqual(['session', 100, 30]);
+ state = 'online';
+ nonce = 'rotated-nonce';
+ // Cloud recovery alone must not replace a working direct peer.
+ await vi.advanceTimersByTimeAsync(3000);
+ expect(localPeers).toHaveLength(2);
+ localPeers[1].simulateClose();
+ await vi.advanceTimersByTimeAsync(1000);
+ expect(loopbackPeerFactory).toHaveBeenLastCalledWith(expect.objectContaining({ nonce: 'rotated-nonce' }));
+ localPeers[2].attachCalls[0][1].onScreen('fresh screen');
+ expect(transport.getMode('session')).toBe('local');
+ } finally {
+ transport.dispose();
+ vi.useRealTimers();
+ }
+ });
+});
diff --git a/ui/src/lib/direct-transport/hybrid-terminal-transport.ts b/ui/src/lib/direct-transport/hybrid-terminal-transport.ts
index cb1bb3a..008f607 100644
--- a/ui/src/lib/direct-transport/hybrid-terminal-transport.ts
+++ b/ui/src/lib/direct-transport/hybrid-terminal-transport.ts
@@ -228,6 +228,9 @@ export class HybridTerminalTransport implements TerminalTransportApi {
private readonly upgradeJitter: number;
private readonly peers = new Map();
+ // Scoped to this authenticated transport's lifetime; never persist nonces.
+ // Cloud presence can disappear while the loopback server is still healthy.
+ private readonly localAdverts = new Map();
private readonly sessions = new Map();
private readonly modeChangeCbs = new Set<(sessionId: string, mode: TerminalTransportMode) => void>();
private readonly bridgeReachabilityCbs = new Set<
@@ -396,6 +399,7 @@ export class HybridTerminalTransport implements TerminalTransportApi {
pe.peer?.close();
}
this.peers.clear();
+ this.localAdverts.clear();
this.modeChangeCbs.clear();
this.bridgeReachabilityCbs.clear();
@@ -785,6 +789,9 @@ export class HybridTerminalTransport implements TerminalTransportApi {
*/
private async isBridgeOnlineForUpgrade(bridgeId: string): Promise {
if (this.advertInjected) return true;
+ // Let the authenticated loopback handshake determine local reachability,
+ // even when Fly cannot answer presence queries.
+ if (this.localAdverts.has(bridgeId)) return true;
const sub = this.ensurePresenceSub();
if (typeof sub.presence !== 'function') return false;
const result = await this.readPresenceBounded(sub);
@@ -1191,24 +1198,25 @@ export class HybridTerminalTransport implements TerminalTransportApi {
}
private async defaultGetLocalAdvert(bridgeId: string): Promise {
+ const cached = this.localAdverts.get(bridgeId) ?? null;
const sub = this.ensurePresenceSub();
- if (typeof sub.presence !== 'function') return null;
+ if (typeof sub.presence !== 'function') return cached;
const result = await this.readPresenceBounded(sub);
- if (!result) return null;
+ if (!result) return cached;
for (const info of Object.values(result.clients)) {
const ci = info.connInfo as
| { bridgeId?: string; localPort?: number; localNonce?: string }
| undefined;
if (
ci &&
- ci.bridgeId === bridgeId &&
+ typeof ci.bridgeId === 'string' &&
typeof ci.localPort === 'number' &&
typeof ci.localNonce === 'string'
) {
- return { localPort: ci.localPort, localNonce: ci.localNonce };
+ this.localAdverts.set(ci.bridgeId, { localPort: ci.localPort, localNonce: ci.localNonce });
}
}
- return null;
+ return this.localAdverts.get(bridgeId) ?? null;
}
/**