diff --git a/docs/cloud-outage-recovery.md b/docs/cloud-outage-recovery.md index a88d1bf..92edac9 100644 --- a/docs/cloud-outage-recovery.md +++ b/docs/cloud-outage-recovery.md @@ -1,10 +1,10 @@ # Cloud outage recovery The dashboard keeps existing Local/P2P terminals available while Centrifugo is -unreachable. A nonblocking notice replaces the automatic full-screen error panel; -diagnostics are available on demand and can be dismissed without reloading. +unreachable. The connection status indicator opens a hover/focus/click panel instead of +printing an outage notice over the terminal. Diagnostics can be dismissed without reloading. Centrifuge continues its existing reconnect/resubscribe cycle. Once connected, -the notice disappears and cloud subscriptions resume without replacing healthy +the indicator updates and cloud subscriptions resume without replacing healthy direct terminal connections. The terminal transport remembers bridge loopback advertisements learned through @@ -27,3 +27,23 @@ Coverage: `CloudConnectionNotice.test.ts` checks nonblocking/dismissible diagnos and recovery; `hybrid-terminal-transport.test.ts` checks reconnection during rejected, hanging, and empty presence responses, continued terminal I/O, and refreshed credentials after cloud recovery. + +## Incident diagnostics + +The status panel remains available after recovery. It shows the sanitized WebSocket +endpoint, browser online hint, current direct bridge count, and recent UTC events. +A per-tab, 100-event in-memory history records cloud event/error codes, browser +online/offline and visibility changes, direct reachability counts, token-refresh +HTTP status, and observed outage duration. Account changes clear the history. + +During cloud unavailability, bounded WebSocket-open and same-origin website probes +run at most once every 30 seconds. Each records start time, result, and duration. +Diagnostic sockets close on completion, timeout, or cancellation. Manual network +checks are available while connected as well. Download report exports JSON without +JWTs, nonces, user IDs, raw error payloads, or URL credentials/query parameters. +Reports are downloaded locally; there is no telemetry upload. + +A browser WebSocket failure does not reveal whether DNS, TCP, TLS, a proxy, or an +ISP failed. The browser online flag is a hint, not an internet reachability check. +Website probes can be answered by a service worker. These limits are included in +the report. History does not survive reloads or a new tab. diff --git a/ui/src/components/CloudConnectionNotice.test.ts b/ui/src/components/CloudConnectionNotice.test.ts index 0c70038..652eb9a 100644 --- a/ui/src/components/CloudConnectionNotice.test.ts +++ b/ui/src/components/CloudConnectionNotice.test.ts @@ -1,36 +1,53 @@ // @vitest-environment jsdom import { createElement } from 'react'; -import { cleanup, fireEvent, render, screen } from '@testing-library/react'; -import { afterEach, expect, it, vi } from 'vitest'; +import { act, cleanup, fireEvent, render, screen } from '@testing-library/react'; +import { afterEach, beforeEach, expect, it, vi } from 'vitest'; import { CloudConnectionNotice } from './CloudConnectionNotice'; +import { clearConnectionHistory, getConnectionHistory, probeWebSocket } from '@/lib/connection-history'; -afterEach(cleanup); +vi.mock('@/lib/connection-history', async (original) => ({ + ...(await original()), + probeWebSocket: vi.fn().mockResolvedValue({ result: 'timeout', durationMs: 5000 }), + probeWebsite: vi.fn().mockResolvedValue({ result: 'HTTP response', status: 200, durationMs: 80 }), +})); +const props = { + connectionStatus: 'error' as const, connectionError: 'transport closed', + centrifugoUrl: 'wss://example.com/connection/websocket?token=secret', + token: 'private-token', onRetry: vi.fn(), directBridgeCount: 1, +}; +beforeEach(() => { clearConnectionHistory(); vi.clearAllMocks(); }); +afterEach(() => { cleanup(); vi.useRealTimers(); }); -it('leaves terminals interactive during an outage and hides the notice on recovery', () => { - const input = vi.fn(); - const props = { - connectionStatus: 'error' as const, - connectionError: 'transport closed', - centrifugoUrl: 'wss://example.com/connection/websocket', - token: '', - onRetry: vi.fn(), - }; - const view = render(createElement('div', null, - createElement('button', { onClick: input }, 'Terminal'), - createElement(CloudConnectionNotice, props), - )); - expect(screen.queryByText('Connection Failed')).toBeNull(); - expect(screen.getByRole('status').textContent).toContain('reconnecting automatically'); - fireEvent.click(screen.getByText('Terminal')); - expect(input).toHaveBeenCalledTimes(1); - fireEvent.click(screen.getByText('Connection details')); - expect(screen.getByText('Connection Failed')).toBeTruthy(); - fireEvent.click(screen.getByText('Back to terminals')); - expect(screen.queryByText('Connection Failed')).toBeNull(); - fireEvent.click(screen.getByText('Connection details')); +it('shows the explanation only on interaction and retains history after recovery', async () => { + const view = render(createElement(CloudConnectionNotice, props)); + expect(screen.queryByText(/Reachable Local/)).toBeNull(); + await act(async () => {}); + fireEvent.click(screen.getByRole('button', { name: 'Connection status and diagnostics' })); + expect(screen.getByText(/Reachable Local/)).toBeTruthy(); + expect(screen.getByText(/Endpoint:/).textContent).not.toContain('secret'); + expect(getConnectionHistory().some((e) => e.event === 'WebSocket probe')).toBe(true); view.rerender(createElement(CloudConnectionNotice, { ...props, connectionStatus: 'connected' })); - expect(screen.queryByRole('status')).toBeNull(); - view.rerender(createElement(CloudConnectionNotice, props)); - expect(screen.queryByText('Connection Failed')).toBeNull(); + expect(screen.getByText(/Recent outage history/)).toBeTruthy(); + expect(getConnectionHistory().some((e) => e.event === 'cloud recovered')).toBe(true); + expect(JSON.stringify(getConnectionHistory())).not.toMatch(/secret|private-token/); + fireEvent.keyDown(screen.getByRole('button', { name: 'Connection status and diagnostics' }), { key: 'Escape' }); + expect(screen.queryByRole('region')).toBeNull(); expect(props.onRetry).not.toHaveBeenCalled(); }); + +it('throttles automatic outage checks and records browser connectivity changes', async () => { + vi.useFakeTimers(); + const view = render(createElement(CloudConnectionNotice, props)); + await act(async () => {}); + expect(probeWebSocket).toHaveBeenCalledTimes(1); + view.rerender(createElement(CloudConnectionNotice, { ...props, connectionStatus: 'connecting' })); + await act(async () => { await vi.advanceTimersByTimeAsync(29_000); }); + expect(probeWebSocket).toHaveBeenCalledTimes(1); + fireEvent(window, new Event('offline')); + expect(getConnectionHistory().at(-1)?.event).toBe('browser offline'); + await act(async () => { await vi.advanceTimersByTimeAsync(1000); }); + expect(probeWebSocket).toHaveBeenCalledTimes(2); + view.rerender(createElement(CloudConnectionNotice, { ...props, connectionStatus: 'connected' })); + await act(async () => { await vi.advanceTimersByTimeAsync(60_000); }); + expect(probeWebSocket).toHaveBeenCalledTimes(2); +}); diff --git a/ui/src/components/CloudConnectionNotice.tsx b/ui/src/components/CloudConnectionNotice.tsx index 2400896..4782f8b 100644 --- a/ui/src/components/CloudConnectionNotice.tsx +++ b/ui/src/components/CloudConnectionNotice.tsx @@ -1,8 +1,9 @@ "use client"; -import { useEffect, useState } from "react"; +import { useCallback, useEffect, useRef, useState, type ReactNode } from "react"; import type { ConnectionStatus } from "@/hooks/useCentrifugo"; import { ConnectionDiagnostics } from "./ConnectionDiagnostics"; +import { diagnosticEndpoint, getConnectionHistory, probeWebSocket, probeWebsite, recordConnectionEvent } from "@/lib/connection-history"; interface Props { connectionStatus: ConnectionStatus; @@ -10,26 +11,133 @@ interface Props { centrifugoUrl: string; token: string; onRetry: () => void; + directBridgeCount?: number; + children?: ReactNode; } -/** Cloud availability must never cover or unmount working local terminals. */ +/** Diagnostics are opt-in UI; passive recording survives cloud recovery. */ export function CloudConnectionNotice(props: Props) { + const [hover, setHover] = useState(false); + const [pinned, setPinned] = useState(false); const [details, setDetails] = useState(false); + const [busy, setBusy] = useState(false); + const [, refresh] = useState(0); + const activeProbe = useRef(null); + const lastProbe = useRef(-Infinity); + const outage = useRef(null); + const hoverTimer = useRef | null>(null); + const connected = props.connectionStatus === "connected"; + const open = hover || pinned; + + const runChecks = useCallback(async () => { + if (activeProbe.current) return; + const controller = new AbortController(); + activeProbe.current = controller; + lastProbe.current = Date.now(); + setBusy(true); + const startedAt = new Date().toISOString(); + const [socket, website] = await Promise.all([ + probeWebSocket(props.centrifugoUrl, controller.signal), + probeWebsite(controller.signal), + ]); + if (!controller.signal.aborted) { + recordConnectionEvent("WebSocket probe", { startedAt, ...socket }); + recordConnectionEvent("website probe", { startedAt, ...website }); + setBusy(false); + refresh((n) => n + 1); + } + if (activeProbe.current === controller) activeProbe.current = null; + }, [props.centrifugoUrl]); + + useEffect(() => () => { + activeProbe.current?.abort(); + if (hoverTimer.current) clearTimeout(hoverTimer.current); + }, [props.centrifugoUrl]); + useEffect(() => { + recordConnectionEvent("dashboard state", { + cloud: props.connectionStatus, directBridges: props.directBridgeCount ?? 0, + browserOnline: navigator.onLine, visibility: document.visibilityState, + }); + }, [props.connectionStatus, props.directBridgeCount]); + useEffect(() => { + if (connected) { + if (outage.current !== null) { + recordConnectionEvent("cloud recovered", { outageMs: Date.now() - outage.current }); + outage.current = null; + } + return; + } + if (outage.current === null) { + outage.current = Date.now(); + recordConnectionEvent("cloud unavailable"); + } + // Sample during an outage without a reconnect storm creating probe storms. + const sample = () => { if (Date.now() - lastProbe.current >= 30_000) void runChecks(); }; + sample(); + const interval = setInterval(sample, 30_000); + return () => clearInterval(interval); + }, [connected, runChecks]); + useEffect(() => { + const online = () => recordConnectionEvent("browser online"); + const offline = () => recordConnectionEvent("browser offline"); + const visibility = () => recordConnectionEvent("visibility", { state: document.visibilityState }); + window.addEventListener("online", online); + window.addEventListener("offline", offline); + document.addEventListener("visibilitychange", visibility); + return () => { + window.removeEventListener("online", online); + window.removeEventListener("offline", offline); + document.removeEventListener("visibilitychange", visibility); + }; + }, []); useEffect(() => { - if (props.connectionStatus === "connected") setDetails(false); - }, [props.connectionStatus]); + if (!open) return; + const interval = setInterval(() => refresh((n) => n + 1), 1000); + return () => clearInterval(interval); + }, [open]); - if (props.connectionStatus === "connected") return null; + const download = () => { + const report = { + capturedAt: new Date().toISOString(), endpoint: diagnosticEndpoint(props.centrifugoUrl), + cloud: props.connectionStatus, directBridges: props.directBridgeCount ?? 0, + browserOnline: navigator.onLine, visibility: document.visibilityState, + timeZone: Intl.DateTimeFormat().resolvedOptions().timeZone, + note: "Browser WebSocket failures cannot distinguish DNS, TCP, TLS, proxy, or ISP causes. Website checks may be served by a service worker. History is limited to this tab lifetime (100 events).", + events: getConnectionHistory(), + }; + const url = URL.createObjectURL(new Blob([JSON.stringify(report, null, 2)], { type: "application/json" })); + const link = document.createElement("a"); + link.href = url; + link.download = `ftown-connection-${Date.now()}.json`; + link.click(); + setTimeout(() => URL.revokeObjectURL(url), 1000); + }; return ( - <> -
-
Cloud disconnected — reconnecting automatically. Local/P2P terminals remain available when reachable. Cloud controls resume after reconnecting.
- -
- {details && ( - setDetails(false)} /> - )} - +
{ + if (hoverTimer.current) clearTimeout(hoverTimer.current); + setHover(true); + }} onMouseLeave={() => { hoverTimer.current = setTimeout(() => setHover(false), 180); }} + onFocus={() => setHover(true)} onBlur={(event) => { if (!event.currentTarget.contains(event.relatedTarget)) setHover(false); }} + onKeyDown={(event) => { if (event.key === "Escape") { setPinned(false); setHover(false); } }}> + + {open &&
+

{connected ? "Cloud connected. Recent outage history is available below." : "Cloud disconnected — reconnecting automatically. Reachable Local/P2P terminals remain available; cloud controls resume after reconnecting."}

+

Endpoint: {diagnosticEndpoint(props.centrifugoUrl)}

+

Browser: {navigator.onLine ? "online" : "offline"} · Direct bridges: {props.directBridgeCount ?? 0}

+
+ + + + +
+

Times below are UTC. WebSocket failure alone cannot identify an ISP problem.

+ 
{getConnectionHistory().slice(-20).map((entry) => `${entry.at} ${entry.event} ${JSON.stringify(entry.details)}`).join("\n")}
+
} + {details && setDetails(false)} />} +
); } diff --git a/ui/src/components/ConnectionDiagnostics.tsx b/ui/src/components/ConnectionDiagnostics.tsx index 1d056eb..de81a4e 100644 --- a/ui/src/components/ConnectionDiagnostics.tsx +++ b/ui/src/components/ConnectionDiagnostics.tsx @@ -119,7 +119,10 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr updateCheck(3, { status: "running" }); try { const wsResult = await new Promise<{ connected: boolean; code?: number; reason?: string }>((resolve) => { - const timeout = setTimeout(() => resolve({ connected: false, reason: "Timeout (5s)" }), 5000); + const timeout = setTimeout(() => { + resolve({ connected: false, reason: "Timeout (5s) before WebSocket open — DNS/TCP/TLS cannot be distinguished by this browser check" }); + ws.close(); + }, 5000); const ws = new WebSocket(centrifugoUrl); ws.onopen = () => { clearTimeout(timeout); @@ -153,7 +156,10 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr updateCheck(4, { status: "running" }); try { const handshakeResult = await new Promise<{ ok: boolean; detail: string }>((resolve) => { - const timeout = setTimeout(() => resolve({ ok: false, detail: "Timeout (8s)" }), 8000); + const timeout = setTimeout(() => { + resolve({ ok: false, detail: "Timeout (8s)" }); + ws.close(); + }, 8000); const ws = new WebSocket(centrifugoUrl); let gotResponse = false; @@ -201,7 +207,7 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr setRunning(false); }, [token, centrifugoUrl, updateCheck]); - if (connectionStatus === "connected") return null; + if (connectionStatus === "connected" && !onDismiss) return null; if (connectionStatus === "connecting" && !hasRun && !onDismiss) return null; const hasFail = checks.some((c) => c.status === "fail"); @@ -232,7 +238,7 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr }} >
- Connection Failed + Connection diagnostics
{onDismiss && } {connectionError && ( @@ -280,7 +286,7 @@ export function ConnectionDiagnostics({ connectionStatus, connectionError, centr : checks.some((c) => c.name === "WebSocket reachable" && c.status === "fail") ? "Cannot reach the WebSocket server. Check your network connection, VPN, or firewall settings." : checks.some((c) => c.name === "Centrifugo handshake" && c.status === "fail") - ? "WebSocket connects but the server rejected authentication. Try reloading the page for a fresh token." + ? "The protocol check failed. A timeout or closed socket does not prove authentication was rejected; inspect the result above." : "Check the details above for more information."} )} diff --git a/ui/src/components/Dashboard.tsx b/ui/src/components/Dashboard.tsx index a1b8e2e..5da9466 100644 --- a/ui/src/components/Dashboard.tsx +++ b/ui/src/components/Dashboard.tsx @@ -860,7 +860,9 @@ PY`; {/* Connection status */} -
+ window.location.reload()} + directBridgeCount={directlyReachableBridgeIds.size}> {directlyReachableBridgeIds.size > 0 && ( <> @@ -886,7 +888,7 @@ PY`; > {connectionStatus === "connected" ? "Cloud connected" : "Cloud reconnecting"} -
+ @@ -1423,13 +1425,6 @@ PY`; /> )} - window.location.reload()} - /> {latestAttention && ( { headers: { "Content-Type": "application/json" }, }); + recordConnectionEvent("token refresh HTTP", { status: response.status }); + if (response.status === 401) { throw new UnauthorizedError("session expired"); } @@ -112,6 +115,7 @@ export function useCentrifugo( // Stable per-tab id for the lifetime of this hook instance (survives // reconnects; a fresh page load gets a fresh id). const clientIdRef = useRef(""); + useEffect(() => { clearConnectionHistory(); }, [userId]); if (!clientIdRef.current) clientIdRef.current = uuidv4(); const cleanup = useCallback(() => { @@ -145,17 +149,20 @@ export function useCentrifugo( getToken: tokenRefresher ?? fetchCentrifugoToken, }); - client.on("connecting", () => { + client.on("connecting", (ctx) => { + recordConnectionEvent("cloud connecting", { code: ctx.code }); setStatus("connecting"); setError(null); }); client.on("connected", () => { + recordConnectionEvent("cloud connected"); setStatus("connected"); setError(null); }); client.on("disconnected", (ctx) => { + recordConnectionEvent("cloud disconnected", { code: ctx.code, unauthorized: ctx.reason === "unauthorized" }); // getToken threw UnauthorizedError (401 from the token route): the // underlying credential itself is gone — NextAuth session (hosted) or // solo access key — not just the Centrifugo token. centrifuge-js stops @@ -180,6 +187,7 @@ export function useCentrifugo( }); client.on("error", (ctx) => { + recordConnectionEvent("cloud error", { code: ctx.error.code, type: ctx.type }); setStatus("error"); setError(`Connection error: ${ctx.error.message}`); }); diff --git a/ui/src/lib/connection-history.test.ts b/ui/src/lib/connection-history.test.ts new file mode 100644 index 0000000..98d6aed --- /dev/null +++ b/ui/src/lib/connection-history.test.ts @@ -0,0 +1,27 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { clearConnectionHistory, diagnosticEndpoint, getConnectionHistory, probeWebSocket, recordConnectionEvent } from './connection-history'; + +afterEach(() => { vi.useRealTimers(); vi.unstubAllGlobals(); clearConnectionHistory(); }); + +it('bounds history and strips credentials and query parameters from report endpoints', () => { + for (let i = 0; i < 110; i++) recordConnectionEvent('attempt', { code: i }); + expect(getConnectionHistory()).toHaveLength(100); + expect(getConnectionHistory()[0].details.code).toBe(10); + expect(diagnosticEndpoint('wss://user:password@example.com/ws?token=secret#secret')).toBe('wss://example.com/ws'); +}); + +it.each(['timeout', 'cancel', 'open', 'closed'])('closes diagnostic sockets on %s', async (outcome) => { + vi.useFakeTimers(); + const socket = { onopen: null as null | (() => void), onclose: null as null | ((event: { code: number; wasClean: boolean }) => void), onerror: null, close: vi.fn() }; + vi.stubGlobal('WebSocket', class { constructor() { return socket; } }); + const controller = new AbortController(); + const result = probeWebSocket('wss://example.com/ws', controller.signal); + if (outcome === 'timeout') await vi.advanceTimersByTimeAsync(5000); + else if (outcome === 'cancel') controller.abort(); + else if (outcome === 'open') socket.onopen!(); + else socket.onclose!({ code: 1006, wasClean: false }); + expect((await result).result).toBe(outcome === 'cancel' ? 'cancelled' : outcome); + expect(socket.close).toHaveBeenCalledTimes(1); + expect(socket.onopen).toBeNull(); + expect(vi.getTimerCount()).toBe(0); +}); diff --git a/ui/src/lib/connection-history.ts b/ui/src/lib/connection-history.ts new file mode 100644 index 0000000..0fbe3b2 --- /dev/null +++ b/ui/src/lib/connection-history.ts @@ -0,0 +1,75 @@ +export interface ConnectionEvent { + at: string; + event: string; + details: Record; +} + +// Per-tab memory only. Never store tokens, nonces, user IDs, or raw server payloads. +const history: ConnectionEvent[] = []; +export function recordConnectionEvent(event: string, details: ConnectionEvent['details'] = {}) { + history.push({ at: new Date().toISOString(), event, details }); + if (history.length > 100) history.shift(); +} +export function getConnectionHistory(): ConnectionEvent[] { + return history.map((entry) => ({ ...entry, details: { ...entry.details } })); +} +export function clearConnectionHistory() { history.length = 0; } + +export function diagnosticEndpoint(raw: string): string { + try { + const url = new URL(raw); + return `${url.protocol}//${url.host}${url.pathname}`; + } catch { return 'invalid endpoint'; } +} + +/** No authentication needed: measures opening the socket, not server login. */ +export function probeWebSocket(url: string, signal: AbortSignal): Promise> { + const start = performance.now(); + return new Promise((resolve) => { + let ws: WebSocket | undefined; + let done = false; + let sawError = false; + const finish = (result: Record) => { + if (done) return; + done = true; + clearTimeout(timer); + signal.removeEventListener('abort', abort); + if (ws) { + ws.onopen = ws.onclose = ws.onerror = null; + // Closing a CONNECTING socket may still emit an error in browsers. + ws.onerror = () => {}; + try { ws.close(); } catch { /* already closed */ } + } + resolve({ ...result, durationMs: Math.round(performance.now() - start) }); + }; + const abort = () => finish({ result: 'cancelled' }); + const timer = setTimeout(() => finish({ result: sawError ? 'error' : 'timeout', stage: 'before WebSocket open' }), 5000); + if (signal.aborted) { abort(); return; } + signal.addEventListener('abort', abort, { once: true }); + try { + ws = new WebSocket(url); + ws.onopen = () => finish({ result: 'open' }); + // Browsers normally emit error then close. Wait for close to retain its code. + ws.onerror = () => { sawError = true; }; + ws.onclose = (event) => finish({ result: 'closed', code: event.code, clean: event.wasClean, sawError }); + } catch { finish({ result: 'invalid URL or browser restriction' }); } + }); +} + +export async function probeWebsite(signal: AbortSignal): Promise> { + const start = performance.now(); + const controller = new AbortController(); + const abort = () => controller.abort(); + const timer = setTimeout(abort, 5000); + signal.addEventListener('abort', abort, { once: true }); + if (signal.aborted) abort(); + try { + const response = await fetch('/favicon.ico', { method: 'HEAD', cache: 'no-store', signal: controller.signal }); + return { result: 'HTTP response', status: response.status, durationMs: Math.round(performance.now() - start) }; + } catch { + return { result: signal.aborted ? 'cancelled' : controller.signal.aborted ? 'timeout' : 'network error', durationMs: Math.round(performance.now() - start) }; + } finally { + clearTimeout(timer); + signal.removeEventListener('abort', abort); + } +}