Keep local terminals usable during cloud outages

docs/cloud-outage-recovery.md

@@ -0,0 +1,29 @@
+# Cloud outage recovery
+
+The dashboard keeps existing Local/P2P terminals available while Centrifugo is
+unreachable. A nonblocking notice replaces the automatic full-screen error panel;
+diagnostics are available on demand and can be dismissed without reloading.
+Centrifuge continues its existing reconnect/resubscribe cycle. Once connected,
+the notice disappears and cloud subscriptions resume without replacing healthy
+direct terminal connections.
+
+The terminal transport remembers bridge loopback advertisements learned through
+authenticated cloud presence. If presence subsequently fails, times out, or loses
+the bridge, the normal retry loop can still attempt a nonce-authenticated localhost
+connection. A successful presence response refreshes advertisements, including
+rotated credentials. Output switches paths only after receiving the new terminal
+screen, using the existing transport switchover logic.
+
+Advertisements remain in memory within the authenticated transport instance and
+are cleared on disposal. They are not written to browser storage. This supports
+outages in an already-loaded dashboard, not cold offline startup or page reloads.
+A bridge restarted with a different port/nonce needs cloud discovery again.
+
+Session creation, loops, and other management RPCs still require Centrifugo.
+Local terminal input, output, and resize use the direct connection. A remote bridge
+without a reachable direct path remains unavailable until a path recovers.
+
+Coverage: `CloudConnectionNotice.test.ts` checks nonblocking/dismissible diagnostics
+and recovery; `hybrid-terminal-transport.test.ts` checks reconnection during rejected,
+hanging, and empty presence responses, continued terminal I/O, and refreshed
+credentials after cloud recovery.

ui/src/components/CloudConnectionNotice.test.ts

@@ -0,0 +1,36 @@
+// @vitest-environment jsdom
+import { createElement } from 'react';
+import { cleanup, fireEvent, render, screen } from '@testing-library/react';
+import { afterEach, expect, it, vi } from 'vitest';
+import { CloudConnectionNotice } from './CloudConnectionNotice';
+
+afterEach(cleanup);
+
+it('leaves terminals interactive during an outage and hides the notice on recovery', () => {
+  const input = vi.fn();
+  const props = {
+    connectionStatus: 'error' as const,
+    connectionError: 'transport closed',
+    centrifugoUrl: 'wss://example.com/connection/websocket',
+    token: '',
+    onRetry: vi.fn(),
+  };
+  const view = render(createElement('div', null,
+    createElement('button', { onClick: input }, 'Terminal'),
+    createElement(CloudConnectionNotice, props),
+  ));
+  expect(screen.queryByText('Connection Failed')).toBeNull();
+  expect(screen.getByRole('status').textContent).toContain('reconnecting automatically');
+  fireEvent.click(screen.getByText('Terminal'));
+  expect(input).toHaveBeenCalledTimes(1);
+  fireEvent.click(screen.getByText('Connection details'));
+  expect(screen.getByText('Connection Failed')).toBeTruthy();
+  fireEvent.click(screen.getByText('Back to terminals'));
+  expect(screen.queryByText('Connection Failed')).toBeNull();
+  fireEvent.click(screen.getByText('Connection details'));
+  view.rerender(createElement(CloudConnectionNotice, { ...props, connectionStatus: 'connected' }));
+  expect(screen.queryByRole('status')).toBeNull();
+  view.rerender(createElement(CloudConnectionNotice, props));
+  expect(screen.queryByText('Connection Failed')).toBeNull();
+  expect(props.onRetry).not.toHaveBeenCalled();
+});

ui/src/components/CloudConnectionNotice.tsx

@@ -0,0 +1,35 @@
+"use client";
+
+import { useEffect, useState } from "react";
+import type { ConnectionStatus } from "@/hooks/useCentrifugo";
+import { ConnectionDiagnostics } from "./ConnectionDiagnostics";
+
+interface Props {
+  connectionStatus: ConnectionStatus;
+  connectionError: string | null;
+  centrifugoUrl: string;
+  token: string;
+  onRetry: () => void;
+}
+
+/** Cloud availability must never cover or unmount working local terminals. */
+export function CloudConnectionNotice(props: Props) {
+  const [details, setDetails] = useState(false);
+  useEffect(() => {
+    if (props.connectionStatus === "connected") setDetails(false);
+  }, [props.connectionStatus]);
+
+  if (props.connectionStatus === "connected") return null;
+
+  return (
+    <>
+      <div style={{ position: "absolute", top: 8, right: 8, zIndex: 90, maxWidth: "min(420px, calc(100% - 16px))", padding: "8px 12px", borderRadius: 8, background: "var(--bg-surface)", border: "1px solid var(--border-muted)", fontSize: 12 }}>
+        <div role="status">Cloud disconnected — reconnecting automatically. Local/P2P terminals remain available when reachable. Cloud controls resume after reconnecting.</div>
+        <button className="btn-ghost" onClick={() => setDetails(true)}>Connection details</button>
+      </div>
+      {details && (
+        <ConnectionDiagnostics {...props} onDismiss={() => setDetails(false)} />
+      )}
+    </>
+  );
+}

ui/src/components/ConnectionDiagnostics.tsx

@@ -15,6 +15,7 @@ interface ConnectionDiagnosticsProps {
   centrifugoUrl: string;
   token: string;
   onRetry: () => void;
+  onDismiss?: () => void;
 }
 
 function decodeJwtPayload(token: string): Record<string, unknown> | null {
@@ -51,7 +52,7 @@ function StatusIcon({ status }: { status: DiagnosticCheck["status"] }) {
   );
 }
 
-export function ConnectionDiagnostics({ connectionStatus, connectionError, centrifugoUrl, token, onRetry }: ConnectionDiagnosticsProps) {
+export function ConnectionDiagnostics({ connectionStatus, connectionError, centrifugoUrl, token, onRetry, onDismiss }: ConnectionDiagnosticsProps) {
   const [checks, setChecks] = useState<DiagnosticCheck[]>([]);
   const [running, setRunning] = useState(false);
   const [hasRun, setHasRun] = useState(false);
@@ -201,7 +202,7 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr
   }, [token, centrifugoUrl, updateCheck]);
 
   if (connectionStatus === "connected") return null;
-  if (connectionStatus === "connecting" && !hasRun) return null;
+  if (connectionStatus === "connecting" && !hasRun && !onDismiss) return null;
 
   const hasFail = checks.some((c) => c.status === "fail");
   const allPass = checks.length > 0 && checks.every((c) => c.status === "pass");
@@ -233,6 +234,7 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr
         <div style={{ fontSize: 13, fontWeight: 700, color: "var(--text-primary)", marginBottom: 4 }}>
           Connection Failed
         </div>
+        {onDismiss && <button className="btn-ghost" onClick={onDismiss}>Back to terminals</button>}
         {connectionError && (
           <div style={{ fontSize: 11, color: "var(--status-error)", marginBottom: 12, lineHeight: 1.5 }}>
             {connectionError}

ui/src/components/Dashboard.tsx

@@ -23,7 +23,7 @@ import { NewFactoryModal } from "./factory/NewFactoryModal";
 import { deriveFactories } from "./factory/useFactory";
 import type { FactoryInfo, NewFactoryInput } from "./factory/types";
 import { factoryInitPrompt, factoryKey, factoryWorkerOf } from "./factory/types";
-import { ConnectionDiagnostics } from "./ConnectionDiagnostics";
+import { CloudConnectionNotice } from "./CloudConnectionNotice";
 import { SessionAttentionAlert } from "./SessionAttentionAlert";
 import { mergeBridgeOrder } from "@/lib/bridge-order";
 import { StatusDot } from "@/lib/StatusDot";
@@ -860,18 +860,31 @@ PY`;
           <span style={{ width: 1, height: 12, background: "var(--border-muted)" }} />
 
           {/* Connection status */}
-          <div className="flex items-center gap-1.5">
-            <StatusDot kind={connectionStatus} title={connectionStatus} />
+          <div className="flex items-center gap-1.5" role="status" aria-live="polite">
+            {directlyReachableBridgeIds.size > 0 && (
+              <>
+                <StatusDot kind="connected" title="Local/P2P connected" />
+                <span style={{ fontSize: 11, color: "var(--status-running)" }}>
+                  Local/P2P connected
+                </span>
+                <span aria-hidden="true" style={{ color: "var(--text-muted)" }}>·</span>
+              </>
+            )}
+            <StatusDot
+              kind={connectionStatus === "connected" ? "connected" : directlyReachableBridgeIds.size > 0 ? "connecting" : connectionStatus}
+              title={connectionStatus === "connected" ? "Cloud connected" : "Cloud reconnecting"}
+            />
             <span
               style={{
                 fontSize: 11,
-                color: connectionStatus === "error" || connectionStatus === "disconnected"
+                color: connectionStatus !== "connected" && directlyReachableBridgeIds.size > 0
+                  ? "var(--status-pending)"
+                  : connectionStatus === "error" || connectionStatus === "disconnected"
                   ? "var(--status-error)"
                   : "var(--text-muted)",
-                textTransform: "capitalize",
               }}
             >
-              {connectionStatus}
+              {connectionStatus === "connected" ? "Cloud connected" : "Cloud reconnecting"}
             </span>
           </div>
 
@@ -1410,7 +1423,7 @@ PY`;
         />
       )}
 
-      <ConnectionDiagnostics
+      <CloudConnectionNotice
         connectionStatus={connectionStatus}
         connectionError={connectionError}
         centrifugoUrl={centrifugoUrl}

ui/src/lib/direct-transport/hybrid-terminal-transport.test.ts

@@ -2172,3 +2172,80 @@ describe('HybridTerminalTransport — transport upgrade (transport-upgrade-adden
     expect(toPeer.indexOf('bc')).toBeLessThan(toPeer.indexOf('z'));
   });
 });
+
+
+describe('cloud outage recovery with remembered loopback discovery', () => {
+  it.each(['reject', 'hang', 'empty'])('reconnects locally when presence is %s, then refreshes credentials on cloud recovery', async (failure) => {
+    vi.useFakeTimers();
+    let state = 'online';
+    let nonce = 'first-nonce';
+    class PresenceSub extends FakeSubscription {
+      presence(): Promise<{ clients: Record<string, { connInfo: unknown }> }> {
+        if (state === 'reject') return Promise.reject(new Error('cloud unavailable'));
+        if (state === 'hang') return new Promise(() => {});
+        const clients: Record<string, { connInfo: unknown }> = state === 'empty' ? {} : {
+          bridge: { connInfo: { bridgeId: 'bridge-1', localPort: 41999, localNonce: nonce } },
+        };
+        return Promise.resolve({ clients });
+      }
+    }
+    class Client extends FakeCentrifugeClient {
+      override newSubscription(channel: string): FakeSubscription {
+        if (channel.startsWith('bridges:presence')) {
+          const sub = new PresenceSub(channel);
+          this.subs.set(channel, sub);
+          return sub;
+        }
+        return super.newSubscription(channel);
+      }
+    }
+    const localPeers: FakePeer[] = [];
+    const loopbackPeerFactory = vi.fn((opts: { bridgeId: string }) => {
+      const p = new FakePeer(opts.bridgeId);
+      localPeers.push(p);
+      return p;
+    });
+    const transport = new HybridTerminalTransport({
+      centrifuge: new Client(), userId: 'user', clientId: 'tab',
+      publishCommand: vi.fn(), loopbackPeerFactory,
+      peerFactory: (opts: PeerFactoryOpts) => {
+        const p = new FakePeer(opts.bridgeId);
+        p.connectMode = 'reject';
+        return p;
+      },
+      upgradeBackoffMs: [1000], upgradeJitter: 0,
+    } as unknown as ConstructorParameters<typeof HybridTerminalTransport>[0]);
+    try {
+      const output = vi.fn();
+      transport.subscribeTerminal('session', 'bridge-1', { onOutput: output, onScreen: vi.fn() });
+      await vi.advanceTimersByTimeAsync(0);
+      expect(transport.getMode('session')).toBe('local');
+      state = failure;
+      transport.sendInput('session', 'still-local');
+      expect(localPeers[0].sendInputCalls).toContainEqual(['session', 'still-local']);
+      localPeers[0].simulateClose();
+      await vi.advanceTimersByTimeAsync(1000 + LOOPBACK_TIMEOUT_MS);
+      expect(localPeers).toHaveLength(2);
+      expect(loopbackPeerFactory).toHaveBeenLastCalledWith(expect.objectContaining({ nonce: 'first-nonce' }));
+      localPeers[1].attachCalls[0][1].onScreen('reconnected');
+      expect(transport.getMode('session')).toBe('local');
+      localPeers[1].attachCalls[0][1].onOutput('working offline');
+      expect(output).toHaveBeenCalledWith('working offline');
+      transport.sendResize('session', 100, 30);
+      expect(localPeers[1].sendResizeCalls).toContainEqual(['session', 100, 30]);
+      state = 'online';
+      nonce = 'rotated-nonce';
+      // Cloud recovery alone must not replace a working direct peer.
+      await vi.advanceTimersByTimeAsync(3000);
+      expect(localPeers).toHaveLength(2);
+      localPeers[1].simulateClose();
+      await vi.advanceTimersByTimeAsync(1000);
+      expect(loopbackPeerFactory).toHaveBeenLastCalledWith(expect.objectContaining({ nonce: 'rotated-nonce' }));
+      localPeers[2].attachCalls[0][1].onScreen('fresh screen');
+      expect(transport.getMode('session')).toBe('local');
+    } finally {
+      transport.dispose();
+      vi.useRealTimers();
+    }
+  });
+});

ui/src/lib/direct-transport/hybrid-terminal-transport.ts

@@ -228,6 +228,9 @@ export class HybridTerminalTransport implements TerminalTransportApi {
   private readonly upgradeJitter: number;
 
   private readonly peers = new Map<string, PeerEntry>();
+  // Scoped to this authenticated transport's lifetime; never persist nonces.
+  // Cloud presence can disappear while the loopback server is still healthy.
+  private readonly localAdverts = new Map<string, BridgeLocalAdvert>();
   private readonly sessions = new Map<string, SessionEntry>();
   private readonly modeChangeCbs = new Set<(sessionId: string, mode: TerminalTransportMode) => void>();
   private readonly bridgeReachabilityCbs = new Set<
@@ -396,6 +399,7 @@ export class HybridTerminalTransport implements TerminalTransportApi {
       pe.peer?.close();
     }
     this.peers.clear();
+    this.localAdverts.clear();
     this.modeChangeCbs.clear();
     this.bridgeReachabilityCbs.clear();
 
@@ -785,6 +789,9 @@ export class HybridTerminalTransport implements TerminalTransportApi {
    */
   private async isBridgeOnlineForUpgrade(bridgeId: string): Promise<boolean> {
     if (this.advertInjected) return true;
+    // Let the authenticated loopback handshake determine local reachability,
+    // even when Fly cannot answer presence queries.
+    if (this.localAdverts.has(bridgeId)) return true;
     const sub = this.ensurePresenceSub();
     if (typeof sub.presence !== 'function') return false;
     const result = await this.readPresenceBounded(sub);
@@ -1191,24 +1198,25 @@ export class HybridTerminalTransport implements TerminalTransportApi {
   }
 
   private async defaultGetLocalAdvert(bridgeId: string): Promise<BridgeLocalAdvert | null> {
+    const cached = this.localAdverts.get(bridgeId) ?? null;
     const sub = this.ensurePresenceSub();
-    if (typeof sub.presence !== 'function') return null;
+    if (typeof sub.presence !== 'function') return cached;
     const result = await this.readPresenceBounded(sub);
-    if (!result) return null;
+    if (!result) return cached;
     for (const info of Object.values(result.clients)) {
       const ci = info.connInfo as
         | { bridgeId?: string; localPort?: number; localNonce?: string }
         | undefined;
       if (
         ci &&
-        ci.bridgeId === bridgeId &&
+        typeof ci.bridgeId === 'string' &&
         typeof ci.localPort === 'number' &&
         typeof ci.localNonce === 'string'
       ) {
-        return { localPort: ci.localPort, localNonce: ci.localNonce };
+        this.localAdverts.set(ci.bridgeId, { localPort: ci.localPort, localNonce: ci.localNonce });
       }
     }
-    return null;
+    return this.localAdverts.get(bridgeId) ?? null;
   }
 
   /**