fmktech · fkesheh · Sep 17, 2026 · Sep 17, 2026
diff --git a/docs/cloud-outage-recovery.md b/docs/cloud-outage-recovery.md
@@ -1,10 +1,10 @@
 # Cloud outage recovery
 
 The dashboard keeps existing Local/P2P terminals available while Centrifugo is
-unreachable. A nonblocking notice replaces the automatic full-screen error panel;
-diagnostics are available on demand and can be dismissed without reloading.
+unreachable. The connection status indicator opens a hover/focus/click panel instead of
+printing an outage notice over the terminal. Diagnostics can be dismissed without reloading.
 Centrifuge continues its existing reconnect/resubscribe cycle. Once connected,
-the notice disappears and cloud subscriptions resume without replacing healthy
+the indicator updates and cloud subscriptions resume without replacing healthy
 direct terminal connections.
 
 The terminal transport remembers bridge loopback advertisements learned through
@@ -27,3 +27,23 @@ Coverage: `CloudConnectionNotice.test.ts` checks nonblocking/dismissible diagnos
 and recovery; `hybrid-terminal-transport.test.ts` checks reconnection during rejected,
 hanging, and empty presence responses, continued terminal I/O, and refreshed
 credentials after cloud recovery.
+
+## Incident diagnostics
+
+The status panel remains available after recovery. It shows the sanitized WebSocket
+endpoint, browser online hint, current direct bridge count, and recent UTC events.
+A per-tab, 100-event in-memory history records cloud event/error codes, browser
+online/offline and visibility changes, direct reachability counts, token-refresh
+HTTP status, and observed outage duration. Account changes clear the history.
+
+During cloud unavailability, bounded WebSocket-open and same-origin website probes
+run at most once every 30 seconds. Each records start time, result, and duration.
+Diagnostic sockets close on completion, timeout, or cancellation. Manual network
+checks are available while connected as well. Download report exports JSON without
+JWTs, nonces, user IDs, raw error payloads, or URL credentials/query parameters.
+Reports are downloaded locally; there is no telemetry upload.
+
+A browser WebSocket failure does not reveal whether DNS, TCP, TLS, a proxy, or an
+ISP failed. The browser online flag is a hint, not an internet reachability check.
+Website probes can be answered by a service worker. These limits are included in
+the report. History does not survive reloads or a new tab.
diff --git a/ui/src/components/CloudConnectionNotice.test.ts b/ui/src/components/CloudConnectionNotice.test.ts
@@ -1,36 +1,53 @@
 // @vitest-environment jsdom
 import { createElement } from 'react';
-import { cleanup, fireEvent, render, screen } from '@testing-library/react';
-import { afterEach, expect, it, vi } from 'vitest';
+import { act, cleanup, fireEvent, render, screen } from '@testing-library/react';
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
 import { CloudConnectionNotice } from './CloudConnectionNotice';
+import { clearConnectionHistory, getConnectionHistory, probeWebSocket } from '@/lib/connection-history';
 
-afterEach(cleanup);
+vi.mock('@/lib/connection-history', async (original) => ({
+  ...(await original<typeof import('@/lib/connection-history')>()),
+  probeWebSocket: vi.fn().mockResolvedValue({ result: 'timeout', durationMs: 5000 }),
+  probeWebsite: vi.fn().mockResolvedValue({ result: 'HTTP response', status: 200, durationMs: 80 }),
+}));
+const props = {
+  connectionStatus: 'error' as const, connectionError: 'transport closed',
+  centrifugoUrl: 'wss://example.com/connection/websocket?token=secret',
+  token: 'private-token', onRetry: vi.fn(), directBridgeCount: 1,
+};
+beforeEach(() => { clearConnectionHistory(); vi.clearAllMocks(); });
+afterEach(() => { cleanup(); vi.useRealTimers(); });
 
-it('leaves terminals interactive during an outage and hides the notice on recovery', () => {
-  const input = vi.fn();
-  const props = {
-    connectionStatus: 'error' as const,
-    connectionError: 'transport closed',
-    centrifugoUrl: 'wss://example.com/connection/websocket',
-    token: '',
-    onRetry: vi.fn(),
-  };
-  const view = render(createElement('div', null,
-    createElement('button', { onClick: input }, 'Terminal'),
-    createElement(CloudConnectionNotice, props),
-  ));
-  expect(screen.queryByText('Connection Failed')).toBeNull();
-  expect(screen.getByRole('status').textContent).toContain('reconnecting automatically');
-  fireEvent.click(screen.getByText('Terminal'));
-  expect(input).toHaveBeenCalledTimes(1);
-  fireEvent.click(screen.getByText('Connection details'));
-  expect(screen.getByText('Connection Failed')).toBeTruthy();
-  fireEvent.click(screen.getByText('Back to terminals'));
-  expect(screen.queryByText('Connection Failed')).toBeNull();
-  fireEvent.click(screen.getByText('Connection details'));
+it('shows the explanation only on interaction and retains history after recovery', async () => {
+  const view = render(createElement(CloudConnectionNotice, props));
+  expect(screen.queryByText(/Reachable Local/)).toBeNull();
+  await act(async () => {});
+  fireEvent.click(screen.getByRole('button', { name: 'Connection status and diagnostics' }));
+  expect(screen.getByText(/Reachable Local/)).toBeTruthy();
+  expect(screen.getByText(/Endpoint:/).textContent).not.toContain('secret');
+  expect(getConnectionHistory().some((e) => e.event === 'WebSocket probe')).toBe(true);
   view.rerender(createElement(CloudConnectionNotice, { ...props, connectionStatus: 'connected' }));
-  expect(screen.queryByRole('status')).toBeNull();
-  view.rerender(createElement(CloudConnectionNotice, props));
-  expect(screen.queryByText('Connection Failed')).toBeNull();
+  expect(screen.getByText(/Recent outage history/)).toBeTruthy();
+  expect(getConnectionHistory().some((e) => e.event === 'cloud recovered')).toBe(true);
+  expect(JSON.stringify(getConnectionHistory())).not.toMatch(/secret|private-token/);
+  fireEvent.keyDown(screen.getByRole('button', { name: 'Connection status and diagnostics' }), { key: 'Escape' });
+  expect(screen.queryByRole('region')).toBeNull();
   expect(props.onRetry).not.toHaveBeenCalled();
 });
+
+it('throttles automatic outage checks and records browser connectivity changes', async () => {
+  vi.useFakeTimers();
+  const view = render(createElement(CloudConnectionNotice, props));
+  await act(async () => {});
+  expect(probeWebSocket).toHaveBeenCalledTimes(1);
+  view.rerender(createElement(CloudConnectionNotice, { ...props, connectionStatus: 'connecting' }));
+  await act(async () => { await vi.advanceTimersByTimeAsync(29_000); });
+  expect(probeWebSocket).toHaveBeenCalledTimes(1);
+  fireEvent(window, new Event('offline'));
+  expect(getConnectionHistory().at(-1)?.event).toBe('browser offline');
+  await act(async () => { await vi.advanceTimersByTimeAsync(1000); });
+  expect(probeWebSocket).toHaveBeenCalledTimes(2);
+  view.rerender(createElement(CloudConnectionNotice, { ...props, connectionStatus: 'connected' }));
+  await act(async () => { await vi.advanceTimersByTimeAsync(60_000); });
+  expect(probeWebSocket).toHaveBeenCalledTimes(2);
+});
diff --git a/ui/src/components/CloudConnectionNotice.tsx b/ui/src/components/CloudConnectionNotice.tsx
@@ -1,35 +1,143 @@
 "use client";
 
-import { useEffect, useState } from "react";
+import { useCallback, useEffect, useRef, useState, type ReactNode } from "react";
 import type { ConnectionStatus } from "@/hooks/useCentrifugo";
 import { ConnectionDiagnostics } from "./ConnectionDiagnostics";
+import { diagnosticEndpoint, getConnectionHistory, probeWebSocket, probeWebsite, recordConnectionEvent } from "@/lib/connection-history";
 
 interface Props {
   connectionStatus: ConnectionStatus;
   connectionError: string | null;
   centrifugoUrl: string;
   token: string;
   onRetry: () => void;
+  directBridgeCount?: number;
+  children?: ReactNode;
 }
 
-/** Cloud availability must never cover or unmount working local terminals. */
+/** Diagnostics are opt-in UI; passive recording survives cloud recovery. */
 export function CloudConnectionNotice(props: Props) {
+  const [hover, setHover] = useState(false);
+  const [pinned, setPinned] = useState(false);
   const [details, setDetails] = useState(false);
+  const [busy, setBusy] = useState(false);
+  const [, refresh] = useState(0);
+  const activeProbe = useRef<AbortController | null>(null);
+  const lastProbe = useRef(-Infinity);
+  const outage = useRef<number | null>(null);
+  const hoverTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
+  const connected = props.connectionStatus === "connected";
+  const open = hover || pinned;
+
+  const runChecks = useCallback(async () => {
+    if (activeProbe.current) return;
+    const controller = new AbortController();
+    activeProbe.current = controller;
+    lastProbe.current = Date.now();
+    setBusy(true);
+    const startedAt = new Date().toISOString();
+    const [socket, website] = await Promise.all([
+      probeWebSocket(props.centrifugoUrl, controller.signal),
+      probeWebsite(controller.signal),
+    ]);
+    if (!controller.signal.aborted) {
+      recordConnectionEvent("WebSocket probe", { startedAt, ...socket });
+      recordConnectionEvent("website probe", { startedAt, ...website });
+      setBusy(false);
+      refresh((n) => n + 1);
+    }
+    if (activeProbe.current === controller) activeProbe.current = null;
+  }, [props.centrifugoUrl]);
+
+  useEffect(() => () => {
+    activeProbe.current?.abort();
+    if (hoverTimer.current) clearTimeout(hoverTimer.current);
+  }, [props.centrifugoUrl]);
+  useEffect(() => {
+    recordConnectionEvent("dashboard state", {
+      cloud: props.connectionStatus, directBridges: props.directBridgeCount ?? 0,
+      browserOnline: navigator.onLine, visibility: document.visibilityState,
+    });
+  }, [props.connectionStatus, props.directBridgeCount]);
+  useEffect(() => {
+    if (connected) {
+      if (outage.current !== null) {
+        recordConnectionEvent("cloud recovered", { outageMs: Date.now() - outage.current });
+        outage.current = null;
+      }
+      return;
+    }
+    if (outage.current === null) {
+      outage.current = Date.now();
+      recordConnectionEvent("cloud unavailable");
+    }
+    // Sample during an outage without a reconnect storm creating probe storms.
+    const sample = () => { if (Date.now() - lastProbe.current >= 30_000) void runChecks(); };
+    sample();
+    const interval = setInterval(sample, 30_000);
+    return () => clearInterval(interval);
+  }, [connected, runChecks]);
+  useEffect(() => {
+    const online = () => recordConnectionEvent("browser online");
+    const offline = () => recordConnectionEvent("browser offline");
+    const visibility = () => recordConnectionEvent("visibility", { state: document.visibilityState });
+    window.addEventListener("online", online);
+    window.addEventListener("offline", offline);
+    document.addEventListener("visibilitychange", visibility);
+    return () => {
+      window.removeEventListener("online", online);
+      window.removeEventListener("offline", offline);
+      document.removeEventListener("visibilitychange", visibility);
+    };
+  }, []);
   useEffect(() => {
-    if (props.connectionStatus === "connected") setDetails(false);
-  }, [props.connectionStatus]);
+    if (!open) return;
+    const interval = setInterval(() => refresh((n) => n + 1), 1000);
+    return () => clearInterval(interval);
+  }, [open]);
 
-  if (props.connectionStatus === "connected") return null;
+  const download = () => {
+    const report = {
+      capturedAt: new Date().toISOString(), endpoint: diagnosticEndpoint(props.centrifugoUrl),
+      cloud: props.connectionStatus, directBridges: props.directBridgeCount ?? 0,
+      browserOnline: navigator.onLine, visibility: document.visibilityState,
+      timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone,
+      note: "Browser WebSocket failures cannot distinguish DNS, TCP, TLS, proxy, or ISP causes. Website checks may be served by a service worker. History is limited to this tab lifetime (100 events).",
+      events: getConnectionHistory(),
+    };
+    const url = URL.createObjectURL(new Blob([JSON.stringify(report, null, 2)], { type: "application/json" }));
+    const link = document.createElement("a");
+    link.href = url;
+    link.download = `ftown-connection-${Date.now()}.json`;
+    link.click();
+    setTimeout(() => URL.revokeObjectURL(url), 1000);
+  };
 
   return (
-    <>
-      <div style={{ position: "absolute", top: 8, right: 8, zIndex: 90, maxWidth: "min(420px, calc(100% - 16px))", padding: "8px 12px", borderRadius: 8, background: "var(--bg-surface)", border: "1px solid var(--border-muted)", fontSize: 12 }}>
-        <div role="status">Cloud disconnected — reconnecting automatically. Local/P2P terminals remain available when reachable. Cloud controls resume after reconnecting.</div>
-        <button className="btn-ghost" onClick={() => setDetails(true)}>Connection details</button>
-      </div>
-      {details && (
-        <ConnectionDiagnostics {...props} onDismiss={() => setDetails(false)} />
-      )}
-    </>
+    <div onMouseEnter={() => {
+      if (hoverTimer.current) clearTimeout(hoverTimer.current);
+      setHover(true);
+    }} onMouseLeave={() => { hoverTimer.current = setTimeout(() => setHover(false), 180); }}
+      onFocus={() => setHover(true)} onBlur={(event) => { if (!event.currentTarget.contains(event.relatedTarget)) setHover(false); }}
+      onKeyDown={(event) => { if (event.key === "Escape") { setPinned(false); setHover(false); } }}>
+      <button type="button" className="flex items-center gap-1.5" aria-label="Connection status and diagnostics"
+        aria-expanded={open} onClick={() => { setPinned(!pinned); setHover(false); }}>
+        {props.children ?? (connected ? "Cloud connected" : "Cloud reconnecting")}
+      </button>
+      {open && <div role="region" aria-label="Connection diagnostics" style={{ position: "fixed", top: 52, right: 12, zIndex: 100, width: "min(440px, calc(100vw - 24px))", maxHeight: "70vh", overflowY: "auto", padding: 16, borderRadius: 8, background: "var(--bg-surface)", border: "1px solid var(--border-muted)", fontSize: 12 }}>
+        <p>{connected ? "Cloud connected. Recent outage history is available below." : "Cloud disconnected — reconnecting automatically. Reachable Local/P2P terminals remain available; cloud controls resume after reconnecting."}</p>
+        <p style={{ overflowWrap: "anywhere" }}>Endpoint: {diagnosticEndpoint(props.centrifugoUrl)}</p>
+        <p>Browser: {navigator.onLine ? "online" : "offline"} · Direct bridges: {props.directBridgeCount ?? 0}</p>
+        <div className="flex flex-wrap gap-2" style={{ margin: "8px 0" }}>
+          <button className="btn-ghost" disabled={busy} onClick={() => void runChecks()}>{busy ? "Checking…" : "Run network checks"}</button>
+          <button className="btn-ghost" onClick={download}>Download report</button>
+          <button className="btn-ghost" onClick={() => { setDetails(true); setPinned(false); setHover(false); }}>Authentication checks</button>
+          <button className="btn-ghost" onClick={() => { setPinned(false); setHover(false); }}>Close</button>
+        </div>
+        <p>Times below are UTC. WebSocket failure alone cannot identify an ISP problem.</p>
+        <pre style={{ whiteSpace: "pre-wrap", overflowWrap: "anywhere", fontSize: 11 }}>{getConnectionHistory().slice(-20).map((entry) => `${entry.at} ${entry.event} ${JSON.stringify(entry.details)}`).join("\n")}</pre>
+      </div>}
+      {details && <ConnectionDiagnostics {...props} onDismiss={() => setDetails(false)} />}
+    </div>
   );
 }
diff --git a/ui/src/components/ConnectionDiagnostics.tsx b/ui/src/components/ConnectionDiagnostics.tsx
@@ -119,7 +119,10 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr
     updateCheck(3, { status: "running" });
     try {
       const wsResult = await new Promise<{ connected: boolean; code?: number; reason?: string }>((resolve) => {
-        const timeout = setTimeout(() => resolve({ connected: false, reason: "Timeout (5s)" }), 5000);
+        const timeout = setTimeout(() => {
+          resolve({ connected: false, reason: "Timeout (5s) before WebSocket open — DNS/TCP/TLS cannot be distinguished by this browser check" });
+          ws.close();
+        }, 5000);
         const ws = new WebSocket(centrifugoUrl);
         ws.onopen = () => {
           clearTimeout(timeout);
@@ -153,7 +156,10 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr
     updateCheck(4, { status: "running" });
     try {
       const handshakeResult = await new Promise<{ ok: boolean; detail: string }>((resolve) => {
-        const timeout = setTimeout(() => resolve({ ok: false, detail: "Timeout (8s)" }), 8000);
+        const timeout = setTimeout(() => {
+          resolve({ ok: false, detail: "Timeout (8s)" });
+          ws.close();
+        }, 8000);
         const ws = new WebSocket(centrifugoUrl);
         let gotResponse = false;
 
@@ -201,7 +207,7 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr
     setRunning(false);
   }, [token, centrifugoUrl, updateCheck]);
 
-  if (connectionStatus === "connected") return null;
+  if (connectionStatus === "connected" && !onDismiss) return null;
   if (connectionStatus === "connecting" && !hasRun && !onDismiss) return null;
 
   const hasFail = checks.some((c) => c.status === "fail");
@@ -232,7 +238,7 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr
         }}
       >
         <div style={{ fontSize: 13, fontWeight: 700, color: "var(--text-primary)", marginBottom: 4 }}>
-          Connection Failed
+          Connection diagnostics
         </div>
         {onDismiss && <button className="btn-ghost" onClick={onDismiss}>Back to terminals</button>}
         {connectionError && (
@@ -280,7 +286,7 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr
                 : checks.some((c) => c.name === "WebSocket reachable" && c.status === "fail")
                   ? "Cannot reach the WebSocket server. Check your network connection, VPN, or firewall settings."
                   : checks.some((c) => c.name === "Centrifugo handshake" && c.status === "fail")
-                    ? "WebSocket connects but the server rejected authentication. Try reloading the page for a fresh token."
+                    ? "The protocol check failed. A timeout or closed socket does not prove authentication was rejected; inspect the result above."
                     : "Check the details above for more information."}
           </div>
         )}

diff --git a/ui/src/components/Dashboard.tsx b/ui/src/components/Dashboard.tsx
@@ -860,7 +860,9 @@ PY`;
           <span style={{ width: 1, height: 12, background: "var(--border-muted)" }} />
 
           {/* Connection status */}
-          <div className="flex items-center gap-1.5" role="status" aria-live="polite">
+          <CloudConnectionNotice connectionStatus={connectionStatus} connectionError={connectionError}
+            centrifugoUrl={centrifugoUrl} token={token} onRetry={() => window.location.reload()}
+            directBridgeCount={directlyReachableBridgeIds.size}>
             {directlyReachableBridgeIds.size > 0 && (
               <>
                 <StatusDot kind="connected" title="Local/P2P connected" />
@@ -886,7 +888,7 @@ PY`;
             >
               {connectionStatus === "connected" ? "Cloud connected" : "Cloud reconnecting"}
             </span>
-          </div>
+          </CloudConnectionNotice>
 
           <span style={{ width: 1, height: 12, background: "var(--border-muted)" }} />
 
@@ -1423,13 +1425,6 @@ PY`;
         />
       )}
 
-      <CloudConnectionNotice
-        connectionStatus={connectionStatus}
-        connectionError={connectionError}
-        centrifugoUrl={centrifugoUrl}
-        token={token}
-        onRetry={() => window.location.reload()}
-      />
 
       {latestAttention && (
         <SessionAttentionAlert