From c96979360e260e915257c05f5b3034c273189901 Mon Sep 17 00:00:00 2001 From: Alade YESSOUFOU Date: Fri, 2 Oct 2026 15:19:37 +0200 Subject: [PATCH 1/4] feat(init): shorten init and make the first deploy work end to end - init asks at most six questions; --host/--domain allow a complete non-interactive config, and DB passwords read process.env.DB_PASSWORD - default health check accepts any HTTP answer below 500; a configured path stays 2xx/3xx, healthCheck.strict overrides either way - setup falls back to root when the deploy user does not exist yet - first deploy uploads (or creates) the remote env file outside CI --- .claude/skills/shipnode/SKILL.md | 9 +- CHANGELOG.md | 8 + README.md | 27 +- docs/adr/0006-ci-environment-ownership.md | 4 + src/cli/commands/config.ts | 2 +- src/cli/commands/deploy.ts | 4 +- src/cli/commands/env.ts | 25 +- src/cli/commands/init.ts | 365 ++++++------------ src/cli/commands/setup.ts | 21 +- src/cli/index.ts | 2 + src/cli/runner.ts | 53 ++- src/config/schema.ts | 13 +- src/domain/deploy/backend-strategy.ts | 33 +- src/domain/deploy/dotenv.ts | 21 + src/services/health.service.ts | 28 +- src/shared/types.ts | 6 + tests/unit/backend-strategy.test.ts | 66 +++- tests/unit/builder.test.ts | 2 +- tests/unit/env.test.ts | 2 +- tests/unit/health.test.ts | 43 +++ tests/unit/init.test.ts | 76 ++++ tests/unit/runner.test.ts | 62 ++- .../src/content/docs/docs/commands/init.md | 9 +- website/src/content/docs/docs/quick-start.md | 25 +- 24 files changed, 571 insertions(+), 335 deletions(-) create mode 100644 tests/unit/init.test.ts diff --git a/.claude/skills/shipnode/SKILL.md b/.claude/skills/shipnode/SKILL.md index 7dd8770..28cb0d2 100644 --- a/.claude/skills/shipnode/SKILL.md +++ b/.claude/skills/shipnode/SKILL.md @@ -85,10 +85,11 @@ Target one app: `shipnode deploy --app api`, `shipnode logs --app web`. `rollbac ## Workflows ### First deploy -1. `shipnode init` -2. `shipnode setup` — creates `deploy` user by default (`--no-deploy-user` to skip) -3. `shipnode env` (per app if multi-app: `--app api`) -4. `shipnode deploy` +1. `shipnode init` — asks only for the server IP and an optional domain; `--host --domain --non-interactive` skips prompts +2. `shipnode setup` — creates the `deploy` user (`--no-deploy-user` to skip). When the config says `user: 'deploy'` and that user does not exist yet, it logs in as `root` for this run +3. `shipnode deploy` — the first deploy uploads the local `envFile` when the server has none (outside CI). With no local `.env`, it starts from an empty one. Use `shipnode env` to push later changes + +The health check passes on any response below 500 until `.healthCheck(path)` is set; a configured path must return 2xx/3xx. ### CI/CD diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a6e0a0..5769dbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to `@devalade/shipnode` will be documented here. ## [Unreleased] +### Changed +- **`init` asks at most six questions instead of up to fifteen:** what you're deploying (pre-selected from detection), server IP, port, domain, and for backends whether to install a database or Redis. SSH user and port, deploy path, PM2 name, runtime, health check path and extra users take defaults you can edit in the file. `--host` and `--domain` let `init --non-interactive` write a complete config. A DB password is no longer written into the config as a literal; it reads `process.env.DB_PASSWORD`. If `shipnode.config.ts` already exists, `init` says so before asking anything. +- **The default health check accepts any HTTP answer below 500.** Apps without a `/health` route no longer fail their first deploy on a 404. A path set with `.healthCheck(path)` (or `healthCheck.path`) is still held to 2xx/3xx; `healthCheck.strict` overrides either way. Failures now say whether nothing answered on the port or the configured route returned 404. + +### Fixed +- **`setup` could not reach a fresh server with the config `init` wrote.** `init` sets `user: 'deploy'`, which only exists after `setup` creates it. When the server refuses `deploy`, `setup` now logs in as `root` for that run. If the config uses another user, the "switch ssh.user" hint is shown as before. +- **The first deploy failed with "Remote environment file is missing".** When the server has no env file, `deploy` now uploads the local one (what `shipnode env` would do). With no local `.env` either, it creates an empty one. A custom `envFile` name that is missing locally still fails. None of this happens in CI, where the env belongs to `ci env-sync` (ADR-0006). + ## [3.2.0-beta.3] - 2026-10-01 ### Changed diff --git a/README.md b/README.md index c548b5d..fb1ac95 100644 --- a/README.md +++ b/README.md @@ -11,22 +11,17 @@ npm install -g @devalade/shipnode ## Quick start ```bash -# 1. Generate config -shipnode init - -# 2. Provision the server. Installs Node, PM2, Caddy, mise, DB/Redis if -# configured, and bootstraps a `deploy` user (sudo, NOPASSWD) keyed off -# ${ssh.identityFile}.pub so subsequent runs don't need root SSH. -shipnode setup - -# 3. Switch ssh.user in shipnode.config.ts to 'deploy', then lock down SSH: -shipnode harden - -# 4. Deploy -shipnode deploy +shipnode init # asks for your server's IP (and a domain, if you have one) +shipnode setup # once per server: installs Node, PM2, Caddy, creates a 'deploy' user +shipnode deploy # every time you ship ``` -Pass `--no-deploy-user` to `setup` if you want to manage users yourself. +That's it. All you need is a fresh Ubuntu/Debian VPS you can `ssh root@` into with your key. + +- `setup` logs in as `root` the first time, because the `deploy` user it creates doesn't exist yet. After that, everything runs as `deploy`. +- On the first deploy, your local `.env` is uploaded to the server. After you change it, run `shipnode env` to push it again. +- The health check passes once your app answers HTTP on its port. Add `.healthCheck('/health')` to require a real 2xx from a specific route. +- `shipnode harden` (optional) disables root and password SSH and turns on a firewall. ## Configuration @@ -329,9 +324,9 @@ See [ADR-0007](docs/adr/0007-fleet-replication.md) and [ADR-0008](docs/adr/0008- | `.zeroDowntime(altPort?)` | automatic for Caddy backends | Force blue-green releases and optionally choose the green port | | `.blueGreenRetention('warm' \| 'rollback' \| 'none')` | `'warm'` | What happens to the old colour after the switch: stop it but keep its release so `rollback` can start it again (`warm`), keep it running for an instant flip (`rollback`), or stop it with no rollback (`none`) | | `.noZeroDowntime()` | — | Opt out and recreate PM2 processes during deploy | -| `.healthCheck(path, opts?)` | `/health`, 30s, 3 retries | Post-deploy health check | +| `.healthCheck(path, opts?)` | `/health`, 30s, 3 retries; any response below 500 passes until you set a path | Post-deploy health check — a configured path must return 2xx/3xx | | `.noHealthCheck()` | — | Skip health check | -| `.envFile(f)` | `.env` | Local .env file to upload | +| `.envFile(f)` | `.env` | Local .env file to upload (automatically on the first deploy) | | `.sharedDirs(dirs)` | — | Dirs persisted across releases | | `.sharedFiles(files)` | — | Files persisted across releases | | `.database(opts)` | — | Database connection config | diff --git a/docs/adr/0006-ci-environment-ownership.md b/docs/adr/0006-ci-environment-ownership.md index a4057cb..94fcdd8 100644 --- a/docs/adr/0006-ci-environment-ownership.md +++ b/docs/adr/0006-ci-environment-ownership.md @@ -24,6 +24,10 @@ Generated workflows do not run a generic repository-root build. Frontend strateg Production jobs use GitHub Environment protection, least-privilege repository permissions, a timeout, and serialized concurrency with `cancel-in-progress: false`. Root-level monorepo changes continue to trigger deployments; no `paths` filter is generated. +## First deploy from a developer machine + +When the server has no env file, `deploy` uploads the local one, which is what `shipnode env` would have done. With no local file and the default `.env` name, it creates an empty one. This never happens when `CI` is set: a missing file there means a misconfigured workflow, and the preflight still fails. + ## Trade-offs - Server-managed env minimizes secret exposure but requires an out-of-band env upload or rotation step. diff --git a/src/cli/commands/config.ts b/src/cli/commands/config.ts index 05c20b8..1fd047d 100644 --- a/src/cli/commands/config.ts +++ b/src/cli/commands/config.ts @@ -46,7 +46,7 @@ function showApp(app: ShipnodeApp, nodeVersion: string): void { if (app.appType === 'backend') { ui.section('Health Check', [ ['enabled', String(app.healthCheck.enabled)], - ['path', app.healthCheck.path], + ['path', app.healthCheck.strict === false ? `${app.healthCheck.path} (any response below 500)` : app.healthCheck.path], ['timeout', String(app.healthCheck.timeout)], ['retries', String(app.healthCheck.retries)], ['startupDelay', String(app.healthCheck.startupDelay)], diff --git a/src/cli/commands/deploy.ts b/src/cli/commands/deploy.ts index 0c59a64..18839a7 100644 --- a/src/cli/commands/deploy.ts +++ b/src/cli/commands/deploy.ts @@ -390,7 +390,9 @@ function renderAppPlan( if (app.hooks?.preDeploy) steps.push('Run preDeploy hook'); steps.push('Switch symlink (atomic)'); if (app.appType === 'backend') steps.push('Reload PM2'); - if (app.healthCheck.enabled) steps.push(`Health check ${app.healthCheck.path}`); + if (app.healthCheck.enabled) { + steps.push(`Health check ${app.healthCheck.path}${app.healthCheck.strict === false ? ' (any response below 500)' : ''}`); + } steps.push('Record release'); if (app.hooks?.postDeploy) steps.push('Run postDeploy hook'); if (app.hooks?.afterFleet) steps.push('Run afterFleet hook (last replica only)'); diff --git a/src/cli/commands/env.ts b/src/cli/commands/env.ts index c96c353..3a63ec7 100644 --- a/src/cli/commands/env.ts +++ b/src/cli/commands/env.ts @@ -5,30 +5,7 @@ import { runRemoteCommandForTargets } from '../runner.js'; import { ui } from '../ui.js'; import { getDeploymentName } from '../../domain/pm2/apps.js'; import { isWatt, resolveWattUnits, restartUnitCommand } from '../../domain/runtime/watt.js'; -import type { RemoteExecutor } from '../../domain/remote/executor.js'; - -function shellSingleQuote(value: string): string { - return `'${value.replace(/'/g, `'"'"'`)}'`; -} - -/** Atomically replace a remote environment file without exposing its raw content to shell parsing. */ -export async function uploadEnvironmentFile( - executor: RemoteExecutor, - remotePath: string, - content: Buffer, -): Promise { - const b64 = content.toString('base64'); - await executor.execOrThrow(`mkdir -p "$(dirname ${shellSingleQuote(remotePath)})"`); - const temporaryEnv = `${remotePath}.shipnode.XXXXXX`; - await executor.execOrThrow([ - `tmp=$(mktemp ${shellSingleQuote(temporaryEnv)})`, - `trap 'rm -f "$tmp"' EXIT`, - `printf '%s' ${shellSingleQuote(b64)} | base64 -d > "$tmp"`, - 'chmod 600 "$tmp"', - `mv -f "$tmp" ${shellSingleQuote(remotePath)}`, - 'trap - EXIT', - ].join(' && ')); -} +import { uploadEnvironmentFile } from '../../domain/deploy/dotenv.js'; export async function cmdEnv( cwd: string, diff --git a/src/cli/commands/init.ts b/src/cli/commands/init.ts index acf885e..4cde322 100644 --- a/src/cli/commands/init.ts +++ b/src/cli/commands/init.ts @@ -1,7 +1,7 @@ import { writeFile, readFile as readFileNode } from 'node:fs/promises'; import { ensureDir, pathExists } from 'fs-extra'; import { resolve } from 'path'; -import { text, select, confirm, isCancel, group } from '@clack/prompts'; +import { text, select, confirm, isCancel } from '@clack/prompts'; import { detectFramework, detectPkgManager } from '../../domain/framework/detector.js'; import { isValidIpOrHostname, isValidPort } from '../../domain/validation/ip.js'; import { ui } from '../ui.js'; @@ -14,18 +14,44 @@ function cancelIfNeeded(v: unknown): asserts v is string | boolean | number { } } -export async function cmdInit(cwd: string, options: { nonInteractive?: boolean; print?: boolean }): Promise { +interface InitOptions { + nonInteractive?: boolean; + print?: boolean; + host?: string; + domain?: string; +} + +/** + * Ask only what shipnode cannot work out by itself: where the server is, and + * the domain if there is one. Everything else — app type, port, package + * manager — is detected, and the rest takes a default the developer can change + * in the generated file. Power-user options (workers, watt, fleets, users) live + * in the config, not in a questionnaire every newcomer has to sit through. + */ +export async function cmdInit(cwd: string, options: InitOptions): Promise { + const configPath = resolve(cwd, 'shipnode.config.ts'); + if (!options.print && (await pathExists(configPath))) { + ui.warn('shipnode.config.ts already exists — edit it, or delete it to start over.'); + return; + } + const detection = await detectFramework(cwd); const pkgManager = await detectPkgManager(cwd); - const isBackend = detection.appType === 'backend'; const appName = await getAppName(cwd); const defaultPort = detection.port ?? 3000; + if (options.host !== undefined && !isValidIpOrHostname(options.host)) { + ui.error('Invalid --host. Must be an IP address or hostname.'); + process.exit(1); + } + if (options.nonInteractive || options.print) { const config = generateConfig({ - app: isBackend ? 'backend' : 'frontend', + app: detection.appType === 'backend' ? 'backend' : 'frontend', appName, + sshHost: options.host, backendPort: defaultPort, + domain: options.domain, pkgManager: pkgManager ?? undefined, }); @@ -34,14 +60,9 @@ export async function cmdInit(cwd: string, options: { nonInteractive?: boolean; return; } - const configPath = resolve(cwd, 'shipnode.config.ts'); - if (await pathExists(configPath)) { - ui.warn('shipnode.config.ts already exists. Use --force to overwrite.'); - return; - } - await writeFile(configPath, config, 'utf-8'); ui.success('Created shipnode.config.ts'); + if (!options.host) ui.warn("Set your server's IP in .ssh({ host }) before running shipnode setup."); await generateShipnodeDir(cwd); return; } @@ -53,201 +74,88 @@ export async function cmdInit(cwd: string, options: { nonInteractive?: boolean; 'Project', ); - // ── Server ───────────────────────────────────────────────────── const appTypeVal = await select({ - message: 'App type', - initialValue: isBackend ? 'backend' : 'frontend', + message: 'What are you deploying?', + initialValue: detection.appType === 'backend' ? 'backend' : 'frontend', options: [ - { value: 'backend', label: 'Backend', hint: 'Node.js server, PM2 managed' }, - { value: 'frontend', label: 'Frontend', hint: 'Static files, served by Caddy' }, + { value: 'backend', label: 'A Node.js server', hint: 'API, SSR, Express, Fastify, Nest…' }, + { value: 'frontend', label: 'A static site', hint: 'Vite, SPA, docs — built locally, served by Caddy' }, ], }); cancelIfNeeded(appTypeVal); - const appType = appTypeVal as string; + const appType = appTypeVal as 'backend' | 'frontend'; - const sshHost = await text({ message: 'SSH host', placeholder: '1.2.3.4' }); + const sshHost = options.host ?? await text({ + message: "Server IP address", + placeholder: '1.2.3.4', + validate: (value) => (isValidIpOrHostname(value ?? '') ? undefined : 'Enter an IP address or hostname'), + }); cancelIfNeeded(sshHost); - if (!isValidIpOrHostname(sshHost as string)) { - ui.error('Invalid host. Must be an IP address or hostname.'); - process.exit(1); - } - - const sshUser = await text({ message: 'SSH user', initialValue: 'deploy' }); - cancelIfNeeded(sshUser); - const sshPortStr = await text({ message: 'SSH port', initialValue: '22' }); - cancelIfNeeded(sshPortStr); - const sshPort = parseInt(sshPortStr as string, 10); - if (!isValidPort(sshPort)) { - ui.error('Invalid port number.'); - process.exit(1); + let backendPort = defaultPort; + if (appType === 'backend') { + const portVal = await text({ + message: 'Which port does your app listen on?', + initialValue: String(defaultPort), + validate: (value) => (isValidPort(parseInt(value ?? '', 10)) ? undefined : 'Enter a port number'), + }); + cancelIfNeeded(portVal); + backendPort = parseInt(portVal as string, 10); } - const remotePath = await text({ message: 'Remote deploy path', initialValue: `/var/www/${appName}` }); - cancelIfNeeded(remotePath); + const domainVal = options.domain ?? await text({ + message: 'Domain (leave empty to skip — you get HTTPS automatically when set)', + placeholder: appType === 'backend' ? 'api.example.com' : 'example.com', + }); + cancelIfNeeded(domainVal); + const domain = (domainVal as string | undefined)?.trim() || undefined; - let pm2Name = ''; - let wattMain: string | undefined; - let backendPort = defaultPort; - let domain = ''; - let healthCheckPath = '/health'; let dbType: DatabaseType | undefined; - let dbHost: string | undefined; - let dbPort: number | undefined; - let dbName: string | undefined; - let dbUser: string | undefined; - let dbPassword: string | undefined; - let redisHost: string | undefined; - let redisPort: number | undefined; - let redisPassword: string | undefined; - + let hasRedis = false; if (appType === 'backend') { - const pm2Val = await text({ message: 'PM2 app name', initialValue: appName }); - cancelIfNeeded(pm2Val); - pm2Name = pm2Val as string; - - const runtimeVal = await select({ - message: 'Process runtime', + const dbVal = await select({ + message: 'Install a database on the server?', + initialValue: 'none', options: [ - { value: 'pm2', label: 'PM2 (default)' }, - { value: 'watt', label: 'wattpm — worker threads sharing one port (Linux, opt-in)' }, + { value: 'none', label: 'No' }, + { value: 'postgres', label: 'PostgreSQL' }, + { value: 'mysql', label: 'MySQL' }, + { value: 'mongodb', label: 'MongoDB' }, + { value: 'sqlite', label: 'SQLite', hint: 'a file, nothing to install' }, ], - initialValue: 'pm2', }); - cancelIfNeeded(runtimeVal); - if (runtimeVal === 'watt') { - const mainVal = await text({ - message: 'Entry file each worker thread loads (must listen on process.env.PORT)', - initialValue: 'dist/server.js', - }); - cancelIfNeeded(mainVal); - wattMain = mainVal as string; - } - - const portVal = await text({ message: 'Backend port', initialValue: String(defaultPort) }); - cancelIfNeeded(portVal); - backendPort = parseInt(portVal as string, 10); - - const domainVal = await text({ message: 'Domain', placeholder: 'api.example.com (optional)' }); - cancelIfNeeded(domainVal); - domain = domainVal as string; - - const hcVal = await text({ message: 'Health check path', initialValue: '/health' }); - cancelIfNeeded(hcVal); - healthCheckPath = hcVal as string; + cancelIfNeeded(dbVal); + dbType = dbVal === 'none' ? undefined : (dbVal as DatabaseType); - const hasDb = await confirm({ message: 'Configure a database?', initialValue: false }); - cancelIfNeeded(hasDb); - - if (hasDb) { - const dbTypeVal = await select({ - message: 'Database type', - options: [ - { value: 'postgres', label: 'PostgreSQL', hint: 'port 5432' }, - { value: 'mysql', label: 'MySQL', hint: 'port 3306' }, - { value: 'sqlite', label: 'SQLite', hint: 'file-based' }, - { value: 'mongodb', label: 'MongoDB', hint: 'port 27017' }, - ], - }); - cancelIfNeeded(dbTypeVal); - dbType = dbTypeVal as DatabaseType; - - if (dbType !== 'sqlite') { - const defaultDbPort = dbType === 'postgres' ? '5432' : dbType === 'mysql' ? '3306' : '27017'; - const dbVals = await group({ - dbHost: () => text({ message: 'Database host', initialValue: 'localhost' }), - dbPort: () => text({ message: 'Database port', initialValue: defaultDbPort }), - dbName: () => text({ message: 'Database name', initialValue: appName }), - dbUser: () => text({ message: 'Database user', initialValue: appName }), - dbPassword: () => text({ message: 'Database password', placeholder: 'optional' }), - }); - dbHost = dbVals.dbHost as string; - dbPort = parseInt(dbVals.dbPort as string, 10); - dbName = dbVals.dbName as string; - dbUser = dbVals.dbUser as string; - dbPassword = (dbVals.dbPassword as string) || undefined; - } else { - const sqliteVal = await text({ message: 'SQLite file path', initialValue: './data.db' }); - cancelIfNeeded(sqliteVal); - dbName = sqliteVal as string; - } - } - - const hasRedis = await confirm({ message: 'Configure Redis?', initialValue: false }); - cancelIfNeeded(hasRedis); - - if (hasRedis) { - const redisVals = await group({ - redisHost: () => text({ message: 'Redis host', initialValue: 'localhost' }), - redisPort: () => text({ message: 'Redis port', initialValue: '6379' }), - redisPassword: () => text({ message: 'Redis password', placeholder: 'optional' }), - }); - redisHost = redisVals.redisHost as string; - redisPort = parseInt(redisVals.redisPort as string, 10); - redisPassword = (redisVals.redisPassword as string) || undefined; - } - } else { - const domainVal = await text({ message: 'Domain', placeholder: 'example.com (optional)' }); - cancelIfNeeded(domainVal); - domain = domainVal as string; - } - - // ── Users ────────────────────────────────────────────────────── - const users: Array<{ username: string; publicKey: string; sudo: boolean }> = []; - const addUsers = await confirm({ message: 'Add SSH users to the server?', initialValue: false }); - cancelIfNeeded(addUsers); - - if (addUsers) { - let addMore = true; - while (addMore) { - const username = await text({ message: 'Username' }); - cancelIfNeeded(username); - if (!(username as string)) break; - const publicKey = await text({ message: `Public key for ${username as string}`, placeholder: 'ssh-ed25519 AAAA...' }); - cancelIfNeeded(publicKey); - const sudo = await confirm({ message: `Grant sudo to ${username as string}?`, initialValue: false }); - cancelIfNeeded(sudo); - users.push({ username: username as string, publicKey: publicKey as string, sudo: sudo as boolean }); - const more = await confirm({ message: 'Add another user?', initialValue: false }); - cancelIfNeeded(more); - addMore = more as boolean; - } + const redisVal = await confirm({ message: 'Install Redis on the server?', initialValue: false }); + cancelIfNeeded(redisVal); + hasRedis = redisVal as boolean; } const config = generateConfig({ - app: appType as 'backend' | 'frontend', + app: appType, appName, sshHost: sshHost as string, - sshUser: sshUser as string, - sshPort, - remotePath: remotePath as string, - pm2Name, - wattMain, backendPort, - domain: domain || undefined, - healthCheckPath, + domain, pkgManager: pkgManager ?? undefined, dbType, - dbHost, - dbPort, - dbName, - dbUser, - dbPassword: dbPassword || undefined, - redisHost, - redisPort, - redisPassword: redisPassword || undefined, + redis: hasRedis, }); - const configPath = resolve(cwd, 'shipnode.config.ts'); - if (await pathExists(configPath)) { - ui.warn('shipnode.config.ts already exists. Use --force to overwrite.'); - return; - } - await writeFile(configPath, config, 'utf-8'); - await generateShipnodeDir(cwd, users); + ui.success('Created shipnode.config.ts'); + await generateShipnodeDir(cwd); - ui.outro('Ready! Run shipnode setup to provision your server.'); + ui.note( + [ + 'shipnode setup # once: installs Node, PM2, Caddy… on the server', + 'shipnode deploy # every time you ship', + ].join('\n'), + 'Next', + ); + if (domain) ui.info(`Point ${domain}'s DNS A record at ${sshHost as string} before deploying.`); + ui.outro('Ready!'); } async function getAppName(cwd: string): Promise { @@ -268,50 +176,21 @@ interface ConfigOptions { app: 'backend' | 'frontend'; appName: string; sshHost?: string; - sshUser?: string; - sshPort?: number; - remotePath?: string; - pm2Name?: string; - wattMain?: string; backendPort?: number; domain?: string; - healthCheckPath?: string; pkgManager?: string; dbType?: DatabaseType; - dbHost?: string; - dbPort?: number; - dbName?: string; - dbUser?: string; - dbPassword?: string; - redisHost?: string; - redisPort?: number; - redisPassword?: string; + redis?: boolean; } -async function generateShipnodeDir( - cwd: string, - users: Array<{ username: string; publicKey: string; sudo: boolean }> = [], -): Promise { - const dir = resolve(cwd, '.shipnode'); - await ensureDir(dir); +async function generateShipnodeDir(cwd: string): Promise { + await ensureDir(resolve(cwd, '.shipnode')); const ignorePath = resolve(cwd, '.shipnodeignore'); if (!(await pathExists(ignorePath))) { await writeFile(ignorePath, generateShipnodeIgnore(), 'utf-8'); + ui.success('Generated .shipnodeignore'); } - - if (users.length > 0) { - const usersPath = resolve(dir, 'users.yml'); - const yaml = users.map((u) => [ - `- username: ${u.username}`, - ` publicKey: ${u.publicKey}`, - ` sudo: ${u.sudo}`, - ].join('\n')).join('\n') + '\n'; - await writeFile(usersPath, yaml, 'utf-8'); - ui.success(`Created .shipnode/users.yml with ${users.length} user(s)`); - } - - ui.success('Generated .shipnodeignore'); } function generateShipnodeIgnore(): string { @@ -328,74 +207,48 @@ coverage/ `; } -function generateConfig(opts: ConfigOptions): string { +const DEFAULT_DB_PORT: Record, number> = { + postgres: 5432, + mysql: 3306, + mongodb: 27017, +}; + +export function generateConfig(opts: ConfigOptions): string { const lines = [ "import { shipnode } from '@devalade/shipnode';", '', 'export default shipnode', ` .${opts.app}()`, + // 'deploy' is the user `shipnode setup` creates. Setup itself logs in as + // root the first time, while that user does not exist yet. + ` .ssh({ host: '${opts.sshHost ?? 'YOUR_SERVER_IP'}', user: 'deploy' })`, + ` .deployTo('/var/www/${opts.appName}')`, ]; - const sshOpts: string[] = []; - if (opts.sshHost) sshOpts.push(`host: '${opts.sshHost}'`); - if (opts.sshUser) sshOpts.push(`user: '${opts.sshUser}'`); - sshOpts.push(`port: ${opts.sshPort ?? 22}`); - - if (sshOpts.length > 0) { - lines.push(` .ssh({ ${sshOpts.join(', ')} })`); - } - - if (opts.remotePath) { - lines.push(` .deployTo('${opts.remotePath}')`); - } - - if (opts.pm2Name) { - lines.push(` .pm2('${opts.pm2Name}')`); - } - if (opts.app === 'backend') { + lines.push(` .pm2('${opts.appName}')`); lines.push(` .port(${opts.backendPort ?? 3000})`); } - if (opts.wattMain) { - lines.push(` .runtime('watt', { main: '${opts.wattMain}' })`); - } - if (opts.domain) { lines.push(` .domain('${opts.domain}')`); } - if (opts.healthCheckPath) { - lines.push(` .healthCheck('${opts.healthCheckPath}')`); - } - if (opts.pkgManager) { lines.push(` .pkgManager('${opts.pkgManager}')`); } - if (opts.dbType) { - if (opts.dbType === 'sqlite') { - lines.push(` .database({ type: 'sqlite', name: '${opts.dbName ?? './data.db'}' })`); - } else { - const dbOpts = [ - `type: '${opts.dbType}'`, - `host: '${opts.dbHost ?? 'localhost'}'`, - `port: ${opts.dbPort ?? 5432}`, - `name: '${opts.dbName ?? ''}'`, - `user: '${opts.dbUser ?? ''}'`, - ...(opts.dbPassword ? [`password: process.env.DB_PASSWORD ?? '${opts.dbPassword}'`] : [`// password: process.env.DB_PASSWORD`]), - ]; - lines.push(` .database({ ${dbOpts.join(', ')} })`); - } + if (opts.dbType === 'sqlite') { + lines.push(` .database({ type: 'sqlite', name: './data.db' })`); + } else if (opts.dbType) { + lines.push( + ` .database({ type: '${opts.dbType}', host: 'localhost', port: ${DEFAULT_DB_PORT[opts.dbType]}, ` + + `name: '${opts.appName}', user: '${opts.appName}', password: process.env.DB_PASSWORD })`, + ); } - if (opts.redisHost !== undefined || opts.redisPort !== undefined) { - const redisOpts = [ - `host: '${opts.redisHost ?? 'localhost'}'`, - `port: ${opts.redisPort ?? 6379}`, - ...(opts.redisPassword ? [`password: process.env.REDIS_PASSWORD ?? '${opts.redisPassword}'`] : [`// password: process.env.REDIS_PASSWORD`]), - ]; - lines.push(` .redis({ ${redisOpts.join(', ')} })`); + if (opts.redis) { + lines.push(` .redis({ host: 'localhost', port: 6379, password: process.env.REDIS_PASSWORD })`); } lines.push(' .build();'); diff --git a/src/cli/commands/setup.ts b/src/cli/commands/setup.ts index 2f05636..306e67d 100644 --- a/src/cli/commands/setup.ts +++ b/src/cli/commands/setup.ts @@ -29,18 +29,18 @@ interface SetupOptions { export async function cmdSetup(cwd: string, options: SetupOptions): Promise { await runRemoteCommandForTargets( cwd, - async ({ config, executor, serverName }) => { + async ({ config, executor, serverName, ssh }) => { ui.banner(); - ui.step(`Setting up ${serverName} (${config.ssh.user}@${config.ssh.host})`); + ui.step(`Setting up ${serverName} (${ssh.user}@${ssh.host})`); const created = !options.noDeployUser && (await bootstrapDeployUser(cwd, config, executor)); await buildTasks(executor, config, created ? DEPLOY_USER : null).run(); - if (created) { + if (created && config.ssh.user !== DEPLOY_USER) { ui.note( [ `A '${DEPLOY_USER}' user was created and owns ${config.remotePath}.`, `Switch ssh.user in shipnode.config.ts to '${DEPLOY_USER}', then:`, - ` shipnode harden # disable root SSH`, ` shipnode deploy`, + ` shipnode harden # optional: disable root and password SSH`, ].join('\n'), 'Next steps', ); @@ -48,7 +48,18 @@ export async function cmdSetup(cwd: string, options: SetupOptions): Promise (ssh.user === DEPLOY_USER ? { ...ssh, user: 'root' } : undefined), + }, ); } diff --git a/src/cli/index.ts b/src/cli/index.ts index 7e33c20..7f24be1 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -55,6 +55,8 @@ program program .command('init') .description('Initialize a new shipnode.config.ts') + .option('--host ', "Your server's IP address or hostname") + .option('--domain ', 'Domain to serve the app on (HTTPS is automatic)') .option('--non-interactive', 'Generate config without prompts') .option('--print', 'Print config to stdout without writing file') .action((opts) => cmdInit(process.cwd(), opts)); diff --git a/src/cli/runner.ts b/src/cli/runner.ts index eb3062f..77ff6ed 100644 --- a/src/cli/runner.ts +++ b/src/cli/runner.ts @@ -1,4 +1,4 @@ -import type { ShipnodeConfig } from '../shared/types.js'; +import type { ShipnodeConfig, SshConfig } from '../shared/types.js'; import { SshConnection } from '../infrastructure/ssh/connection.js'; import type { RemoteExecutor } from '../domain/remote/executor.js'; import { loadConfig } from '../config/loader.js'; @@ -56,8 +56,15 @@ export async function runRemoteCommand( */ export async function runRemoteCommandForTargets( cwd: string, - command: (ctx: { config: ShipnodeConfig; executor: RemoteExecutor; serverName: string }) => Promise, - options: { configPath?: string; includeEmpty?: boolean; appName?: string; serverName?: string } = {}, + command: (ctx: { config: ShipnodeConfig; executor: RemoteExecutor; serverName: string; ssh: SshConfig }) => Promise, + options: { + configPath?: string; + includeEmpty?: boolean; + appName?: string; + serverName?: string; + /** Who to log in as when the server refuses the configured user. See `connectWithFallback`. */ + sshFallback?: (ssh: SshConfig) => SshConfig | undefined; + } = {}, ): Promise { const workspace = await loadConfig(cwd, options.configPath); @@ -95,8 +102,8 @@ export async function runRemoteCommandForTargets( visited += 1; const ssh = new SshConnection(); try { - await ssh.connect(target.ssh); - await command({ config: targetConfig, executor: ssh, serverName: target.name }); + const connectedAs = await connectWithFallback(ssh, target.ssh, options.sshFallback); + await command({ config: targetConfig, executor: ssh, serverName: target.name, ssh: connectedAs }); } catch (error) { const message = error instanceof Error ? error.message : String(error); ui.error(`${target.name}: ${message}`); @@ -115,6 +122,42 @@ export async function runRemoteCommandForTargets( } } +/** + * Connect, and when the server turns the configured user away, try once more as + * the user `fallback` names. This is how `setup` reaches a fresh server whose + * `deploy` user it has not created yet. Only an authentication refusal triggers + * the retry: an unreachable host would only fail again, twice as slowly. + */ +async function connectWithFallback( + ssh: SshConnection, + config: SshConfig, + fallback?: (ssh: SshConfig) => SshConfig | undefined, +): Promise { + try { + await ssh.connect(config); + return config; + } catch (error) { + const retry = fallback?.(config); + if (!retry || !isAuthenticationRefusal(error)) throw error; + + ui.info(`Can't log in as ${config.user} on ${config.host} yet — trying ${retry.user}`); + try { + await ssh.connect(retry); + } catch (retryError) { + if (!isAuthenticationRefusal(retryError)) throw retryError; + throw new Error( + `Could not log in to ${config.host} as ${config.user} or ${retry.user}. ` + + `For a server's first setup, set ssh.user to the account your provider gave you (often root or ubuntu).`, + ); + } + return retry; + } +} + +function isAuthenticationRefusal(error: unknown): boolean { + return error instanceof Error && /authentication methods failed/i.test(error.message); +} + export async function runRemoteCommandForConfig( config: ShipnodeConfig, command: (ctx: { config: ShipnodeConfig; executor: RemoteExecutor }) => Promise, diff --git a/src/config/schema.ts b/src/config/schema.ts index 1211dd5..e327825 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -180,13 +180,22 @@ export const WattConfigSchema = z.object({ maxHeapUsed: z.string().regex(/^\d+[KMG]?$/i, 'maxHeapUsed must look like 512M or 1G').optional(), }); +// A path the user chose is a promise that the route exists, so the probe holds +// it to a 2xx/3xx. A path shipnode picked is only a guess: most apps have no +// `/health` route, and failing their first deploy on a 404 says nothing about +// whether the app is up. Then any answer below 500 means the app is listening. export const HealthCheckConfigSchema = z.object({ enabled: z.boolean().default(true), - path: z.string().default('/health'), + path: z.string().optional(), timeout: z.number().int().min(1).default(30), retries: z.number().int().min(1).default(3), startupDelay: z.number().int().min(0).default(3), -}).default({}); + strict: z.boolean().optional(), +}).transform(({ path, strict, ...rest }) => ({ + ...rest, + path: path ?? '/health', + strict: strict ?? path !== undefined, +})).default({}); const networkDbFields = { on: z.string().min(1).optional(), diff --git a/src/domain/deploy/backend-strategy.ts b/src/domain/deploy/backend-strategy.ts index 6a84fc1..588e892 100644 --- a/src/domain/deploy/backend-strategy.ts +++ b/src/domain/deploy/backend-strategy.ts @@ -1,5 +1,6 @@ import chalk from 'chalk'; import { execa } from 'execa'; +import { readFile } from 'node:fs/promises'; import { pathExists } from 'fs-extra'; import { resolve } from 'path'; import type { ShipnodeConfig, ShipnodeApp, Pm2App, PkgManager } from '../../shared/types.js'; @@ -10,7 +11,7 @@ import { getInstallCommand, getRunCommand, detectPkgManager } from '../framework import { RSYNC_DEFAULT_EXCLUDES } from '../../shared/constants.js'; import { DeployError } from '../../shared/errors.js'; import type { DeploymentStrategy, StrategyContext } from './strategy.js'; -import { runWithDotenv } from './dotenv.js'; +import { runWithDotenv, uploadEnvironmentFile } from './dotenv.js'; import { envSymlinkCommand } from './env-links.js'; import { WATT_APP_FILE, WATT_RUNTIME_FILE, WATT_START_COMMAND, @@ -66,7 +67,37 @@ export class BackendStrategy implements DeploymentStrategy { await execa('rsync', args, { stdio: 'inherit' }); } + /** + * The first deploy finds no env file on the server. Failing there only sends + * the developer off to run `shipnode env`, which uploads the same local file — + * so upload it here instead. With no local file and the default `.env` name, + * start from an empty one: plenty of apps need no variables at all. A custom + * `envFile` name is a promise that the file exists, so its absence still fails. + * + * Never in CI. There the env belongs to `ci env-sync` (ADR-0006), and a missing + * file is a misconfigured workflow rather than a first deploy. + */ + private async provisionFirstEnv(ctx: StrategyContext): Promise { + if (!this.app.envFile || process.env['CI']) return; + + const sharedEnvPath = `${this.appPath}/shared/${this.app.envFile}`; + const probe = await ctx.executor.exec(`[ -f ${shellSingleQuote(sharedEnvPath)} ]`); + if (probe.exitCode === 0) return; + + const localEnvPath = resolve(this.cwd, this.app.envFile); + if (await pathExists(localEnvPath)) { + await uploadEnvironmentFile(ctx.executor, sharedEnvPath, await readFile(localEnvPath)); + console.log(chalk.dim(` uploaded ${this.app.envFile} (first deploy) — after editing it, run: shipnode env`)); + return; + } + + if (this.app.envFile !== '.env') return; + await uploadEnvironmentFile(ctx.executor, sharedEnvPath, Buffer.alloc(0)); + console.log(chalk.yellow(` no .env found locally — the app starts with no variables. Add them, then run: shipnode env`)); + } + async setupEnvironment(ctx: StrategyContext): Promise { + await this.provisionFirstEnv(ctx); const pkgManager = await this.resolvePkgManager(); const installCmd = this.workspace.installCommand ?? getInstallCommand(pkgManager); const runCmd = getRunCommand(pkgManager); diff --git a/src/domain/deploy/dotenv.ts b/src/domain/deploy/dotenv.ts index 9b1e3af..538f30c 100644 --- a/src/domain/deploy/dotenv.ts +++ b/src/domain/deploy/dotenv.ts @@ -1,3 +1,5 @@ +import type { RemoteExecutor } from '../remote/executor.js'; + function shellSingleQuote(value: string): string { return `'${value.replace(/'/g, "'\"'\"'")}'`; } @@ -24,3 +26,22 @@ export function runWithDotenv( return `node -e "$(printf '%s' ${shellSingleQuote(encodedRunner)} | base64 --decode)" -- ` + `${shellSingleQuote(envFile)} ${shellSingleQuote(encodedEnvironment)} bash -c ${shellSingleQuote(command)}`; } + +/** Atomically replace a remote environment file without exposing its raw content to shell parsing. */ +export async function uploadEnvironmentFile( + executor: RemoteExecutor, + remotePath: string, + content: Buffer, +): Promise { + const b64 = content.toString('base64'); + await executor.execOrThrow(`mkdir -p "$(dirname ${shellSingleQuote(remotePath)})"`); + const temporaryEnv = `${remotePath}.shipnode.XXXXXX`; + await executor.execOrThrow([ + `tmp=$(mktemp ${shellSingleQuote(temporaryEnv)})`, + `trap 'rm -f "$tmp"' EXIT`, + `printf '%s' ${shellSingleQuote(b64)} | base64 -d > "$tmp"`, + 'chmod 600 "$tmp"', + `mv -f "$tmp" ${shellSingleQuote(remotePath)}`, + 'trap - EXIT', + ].join(' && ')); +} diff --git a/src/services/health.service.ts b/src/services/health.service.ts index a74706c..3093fe6 100644 --- a/src/services/health.service.ts +++ b/src/services/health.service.ts @@ -18,6 +18,28 @@ function retryDelayMs(attempt: number, backoff?: RetryBackoff): number { return Math.min(backoff.maxMs, backoff.initialMs * 2 ** (attempt - 1)); } +/** + * Whether one probe's HTTP status means the app is up. Strict checks need a + * 2xx/3xx; lenient ones accept any answer below 500, because a 404 from an app + * with no health route still proves it is listening. Status 0 means nothing + * answered at all. + */ +export function isHealthyStatus(status: number, strict: boolean): boolean { + if (strict) return status >= 200 && status < 400; + return status >= 100 && status < 500; +} + +/** One line telling the developer what the last status most likely means. */ +function healthFailureHint(status: number, port: number | undefined, path: string): string { + if (!status) { + return `\nNothing answered on port ${port}. Make sure your app listens on process.env.PORT (or port ${port}).`; + } + if (status === 404) { + return `\nYour app has no route at ${path}. Add one, or drop .healthCheck(...) to accept any response.`; + } + return ''; +} + interface Pm2JlistEntry { name: string; pm2_env?: { @@ -115,7 +137,7 @@ export class HealthCheckService { lastStatus = parseInt(parts[0], 10); lastResponseMs = parseInt(parts[1], 10); - if (lastStatus >= 200 && lastStatus < 400) { + if (isHealthyStatus(lastStatus, healthCheck.strict ?? true)) { return { attempts: attempt, responseMs: lastResponseMs }; } @@ -126,7 +148,9 @@ export class HealthCheckService { const diagnostics = unit ? await this.collectUnitLogs(unit) : await this.collectPm2Logs(webApp.name); throw new HealthCheckError( - `Health check failed after ${retries} attempts. Last status: ${lastStatus}` + diagnostics, + `Health check failed after ${retries} attempts. Last status: ${lastStatus}` + + healthFailureHint(lastStatus, port, path) + + diagnostics, retries, lastStatus, ); diff --git a/src/shared/types.ts b/src/shared/types.ts index 6a939fa..a68e97c 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -159,6 +159,12 @@ export interface HealthCheckConfig { timeout: number; retries: number; startupDelay: number; + /** + * Whether the probe needs a 2xx/3xx. When false, any answer below 500 counts + * as up. The config schema sets it to false when no path was configured, so an + * app without a `/health` route still deploys. Omitted means strict. + */ + strict?: boolean; } export type DatabaseConfig = SqliteDatabaseConfig | NetworkDatabaseConfig; diff --git a/tests/unit/backend-strategy.test.ts b/tests/unit/backend-strategy.test.ts index 31ef487..519f29b 100644 --- a/tests/unit/backend-strategy.test.ts +++ b/tests/unit/backend-strategy.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { BackendStrategy } from '../../src/domain/deploy/backend-strategy.js'; import { FakeRemoteExecutor } from '../testing/fake-executor.js'; import type { ShipnodeConfig } from '../../src/shared/types.js'; @@ -13,6 +13,11 @@ vi.mock('fs-extra', () => ({ pathExists: vi.fn().mockResolvedValue(false), })); +vi.mock('node:fs/promises', async (importOriginal) => ({ + ...(await importOriginal()), + readFile: vi.fn().mockResolvedValue(Buffer.from('DATABASE_URL=postgres://local\n')), +})); + const { execa } = await import('execa'); const { pathExists } = await import('fs-extra'); const mockedExeca = vi.mocked(execa); @@ -702,3 +707,62 @@ describe('BackendStrategy.startApp — blue-green', () => { expect(afterCmds.some((c) => c.includes('pm2 reload') && c.includes('ecosystem.workers.config.cjs'))).toBe(true); }); }); + +describe('BackendStrategy — first deploy env', () => { + const missingRemoteEnv = (executor: FakeRemoteExecutor) => + executor.when((c) => c.startsWith('[ -f '), { stdout: '', stderr: '', exitCode: 1 }); + const uploads = (executor: FakeRemoteExecutor) => + executor.getHistory().filter((e) => e.command.includes('mv -f "$tmp"')).map((e) => e.command); + + beforeEach(() => vi.stubEnv('CI', '')); + afterEach(() => vi.unstubAllEnvs()); + + it('uploads the local .env when the server has none yet', async () => { + mockedPathExists.mockResolvedValue(true as never); + const executor = new FakeRemoteExecutor(); + missingRemoteEnv(executor); + await makeStrategy(makeConfig(), '/local/project').setupEnvironment!(makeCtx(executor)); + + const [upload] = uploads(executor); + expect(upload).toContain("'/var/www/app/myapp/shared/.env'"); + expect(upload).toContain(Buffer.from('DATABASE_URL=postgres://local\n').toString('base64')); + }); + + it('starts from an empty .env when there is none locally either', async () => { + const executor = new FakeRemoteExecutor(); + missingRemoteEnv(executor); + await makeStrategy(makeConfig(), '/local/project').setupEnvironment!(makeCtx(executor)); + + const [upload] = uploads(executor); + expect(upload).toContain("printf '%s' '' | base64 -d"); + expect(upload).toContain("'/var/www/app/myapp/shared/.env'"); + }); + + it('leaves a missing custom env file to the preflight — a named file is meant to exist', async () => { + const config = makeConfig({ envFile: '.env.production' }); + const executor = new FakeRemoteExecutor(); + missingRemoteEnv(executor); + await makeStrategy(config, '/local/project').setupEnvironment!(makeCtx(executor, { config })); + + expect(uploads(executor)).toEqual([]); + }); + + it('never touches the server env when it already exists', async () => { + mockedPathExists.mockResolvedValue(true as never); + const executor = new FakeRemoteExecutor(); + await makeStrategy(makeConfig(), '/local/project').setupEnvironment!(makeCtx(executor)); + + expect(uploads(executor)).toEqual([]); + }); + + it('does nothing in CI, where the env belongs to ci env-sync', async () => { + vi.stubEnv('CI', 'true'); + mockedPathExists.mockResolvedValue(true as never); + const executor = new FakeRemoteExecutor(); + missingRemoteEnv(executor); + await makeStrategy(makeConfig(), '/local/project').setupEnvironment!(makeCtx(executor)); + + expect(executor.getHistory().some((e) => e.command.startsWith('[ -f '))).toBe(false); + expect(uploads(executor)).toEqual([]); + }); +}); diff --git a/tests/unit/builder.test.ts b/tests/unit/builder.test.ts index 4ee02e3..3ce209b 100644 --- a/tests/unit/builder.test.ts +++ b/tests/unit/builder.test.ts @@ -49,7 +49,7 @@ describe('ShipnodeBuilder', () => { expect(config.apps[0].blueGreenRetention).toBe('none'); expect(config.apps[0].sharedDirs).toEqual(['storage', 'uploads']); expect(config.apps[0].sharedFiles).toEqual(['.htpasswd']); - expect(config.apps[0].healthCheck).toEqual({ enabled: true, path: '/healthz', timeout: 60, retries: 5, startupDelay: 10 }); + expect(config.apps[0].healthCheck).toEqual({ enabled: true, path: '/healthz', timeout: 60, retries: 5, startupDelay: 10, strict: true }); expect(config.apps[0].envFile).toBe('.env.production'); expect(config.nodeVersion).toBe('22'); expect(config.pkgManager).toBe('pnpm'); diff --git a/tests/unit/env.test.ts b/tests/unit/env.test.ts index e7fbf5c..7193d10 100644 --- a/tests/unit/env.test.ts +++ b/tests/unit/env.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from 'vitest'; import { FakeRemoteExecutor } from '../testing/fake-executor.js'; -import { uploadEnvironmentFile } from '../../src/cli/commands/env.js'; +import { uploadEnvironmentFile } from '../../src/domain/deploy/dotenv.js'; describe('env upload — executor contract', () => { it('atomically uploads via base64 with restrictive permissions', async () => { diff --git a/tests/unit/health.test.ts b/tests/unit/health.test.ts index 856a79d..261800b 100644 --- a/tests/unit/health.test.ts +++ b/tests/unit/health.test.ts @@ -106,3 +106,46 @@ describe('HealthCheckService.perform — PM2 status check', () => { expect(executor.getHistory()).toHaveLength(0); }); }); + +describe('HealthCheckService.perform — default vs. configured path', () => { + function configWith(healthCheck: Record) { + return assembleConfig({ + app: 'backend', + ssh: { host: '1.2.3.4', user: 'deploy', port: 22 }, + remotePath: '/var/www/app', + pm2: { apps: [{ name: 'api', port: 3000 }] }, + healthCheck: { timeout: 5, retries: 1, startupDelay: 0, ...healthCheck }, + }); + } + + function respond(status: string) { + const executor = new FakeRemoteExecutor(); + executor.when((c) => c.includes('curl'), { stdout: `${status} 5`, stderr: '', exitCode: 0 }); + executor.when((c) => c.includes('pm2 jlist'), { stdout: pm2JlistOnline(['api']), stderr: '', exitCode: 0 }); + return executor; + } + + it('accepts a 404 when no path was configured — the app is listening, it just has no /health route', async () => { + const config = configWith({}); + expect(config.apps[0].healthCheck).toMatchObject({ path: '/health', strict: false }); + + const result = await new HealthCheckService(respond('404'), config).perform(config.apps[0]); + expect(result.attempts).toBe(1); + }); + + it('still fails a 5xx when no path was configured', async () => { + const config = configWith({}); + await expect(new HealthCheckService(respond('502'), config).perform(config.apps[0])).rejects.toThrow(/Last status: 502/); + }); + + it('fails when nothing answers, and says to listen on PORT', async () => { + const config = configWith({}); + await expect(new HealthCheckService(respond('000'), config).perform(config.apps[0])).rejects.toThrow(/Nothing answered on port 3000/); + }); + + it('holds a configured path to a 2xx/3xx and names the missing route', async () => { + const config = configWith({ path: '/healthz' }); + expect(config.apps[0].healthCheck.strict).toBe(true); + await expect(new HealthCheckService(respond('404'), config).perform(config.apps[0])).rejects.toThrow(/no route at \/healthz/); + }); +}); diff --git a/tests/unit/init.test.ts b/tests/unit/init.test.ts new file mode 100644 index 0000000..534ff68 --- /dev/null +++ b/tests/unit/init.test.ts @@ -0,0 +1,76 @@ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { mkdtemp, writeFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { generateConfig } from '../../src/cli/commands/init.js'; +import { loadConfig } from '../../src/config/loader.js'; + +// Every config `init` can write must load as-is: a generated file that fails +// validation is the first thing a newcomer sees go wrong. +describe('init — generated config', () => { + let dir: string; + const builder = join(process.cwd(), 'src/config/builder.ts').replace(/\\/g, '/'); + + beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), 'shipnode-init-')); + }); + + afterAll(async () => { + await rm(dir, { recursive: true, force: true }); + }); + + // The exported `shipnode` is a single mutable builder, so loading several + // configs in one process would leak state between them. Each gets its own. + async function load(source: string, name: string) { + const isolated = source.replace( + "import { shipnode } from '@devalade/shipnode';", + `import { ShipnodeBuilder } from '${builder}';\nconst shipnode = new ShipnodeBuilder();`, + ); + expect(isolated).not.toBe(source); + await writeFile(join(dir, name), isolated); + return loadConfig(dir, name); + } + + it('writes a backend that deploys as the deploy user and probes leniently', async () => { + const source = generateConfig({ app: 'backend', appName: 'shop', sshHost: '1.2.3.4', backendPort: 4000, pkgManager: 'pnpm' }); + const config = await load(source, 'backend.config.ts'); + + expect(config.ssh).toMatchObject({ host: '1.2.3.4', user: 'deploy', port: 22 }); + expect(config.remotePath).toBe('/var/www/shop'); + expect(config.apps[0].pm2?.apps[0]).toMatchObject({ name: 'shop', port: 4000 }); + expect(config.apps[0].envFile).toBe('.env'); + expect(config.apps[0].healthCheck).toMatchObject({ enabled: true, strict: false }); + expect(source).not.toContain('healthCheck'); + }); + + it('writes a backend with a domain, a database and Redis', async () => { + const source = generateConfig({ + app: 'backend', appName: 'shop', sshHost: '1.2.3.4', backendPort: 3000, + domain: 'api.example.com', dbType: 'postgres', redis: true, + }); + const config = await load(source, 'full.config.ts'); + + expect(config.apps[0].domain).toBe('api.example.com'); + expect(config.database).toMatchObject({ type: 'postgres', host: 'localhost', port: 5432, name: 'shop', user: 'shop' }); + expect(config.redis).toMatchObject({ host: 'localhost', port: 6379 }); + }); + + it('writes a static site', async () => { + const config = await load( + generateConfig({ app: 'frontend', appName: 'site', sshHost: '1.2.3.4', domain: 'example.com' }), + 'frontend.config.ts', + ); + + expect(config.apps[0].appType).toBe('frontend'); + expect(config.apps[0].domain).toBe('example.com'); + }); + + it('writes SQLite without a server to install', async () => { + const config = await load( + generateConfig({ app: 'backend', appName: 'shop', sshHost: '1.2.3.4', dbType: 'sqlite' }), + 'sqlite.config.ts', + ); + + expect(config.database).toMatchObject({ type: 'sqlite', name: './data.db' }); + }); +}); diff --git a/tests/unit/runner.test.ts b/tests/unit/runner.test.ts index ed38ad5..1133e62 100644 --- a/tests/unit/runner.test.ts +++ b/tests/unit/runner.test.ts @@ -8,13 +8,21 @@ const mocks = vi.hoisted(() => ({ /** Hosts whose connection is still open — must be empty once a run returns. */ open: new Set(), config: undefined as unknown as ShipnodeConfig, + /** Users the fake server refuses, as ssh2 does when no key is authorised. */ + refusedUsers: new Set(), + /** `user@host` for every connect attempt, accepted or not. */ + attempts: [] as string[], })); vi.mock('../../src/infrastructure/ssh/connection.js', () => { return { SshConnection: class MockSshConnection { private host = ''; - async connect(cfg?: { host: string }) { + async connect(cfg?: { host: string; user?: string }) { + mocks.attempts.push(`${cfg?.user}@${cfg?.host}`); + if (cfg?.user && mocks.refusedUsers.has(cfg.user)) { + throw new Error('SSH connection failed: All configured authentication methods failed'); + } this.host = cfg?.host ?? ''; mocks.connected.push(this.host); mocks.open.add(this.host); @@ -62,6 +70,8 @@ function workspace(): ShipnodeConfig { } beforeEach(() => { + mocks.refusedUsers.clear(); + mocks.attempts.length = 0; mocks.connected.length = 0; mocks.open.clear(); mocks.config = { @@ -248,6 +258,56 @@ describe('runRemoteCommandForTargets', () => { }); }); +describe('runRemoteCommandForTargets — sshFallback', () => { + const asRoot = (ssh: { user: string }) => (ssh.user === 'deploy' ? { ...ssh, user: 'root' } as never : undefined); + + it('logs in as the fallback user when the configured one is refused, and says who it is', async () => { + mocks.refusedUsers.add('deploy'); + const infoSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + const seen: string[] = []; + + await runRemoteCommandForTargets('/test', async ({ ssh, config }) => { + seen.push(`${ssh.user} (config ${config.ssh.user})`); + }, { sshFallback: asRoot }); + + expect(mocks.attempts).toEqual(['deploy@1.2.3.4', 'root@1.2.3.4']); + expect(seen).toEqual(['root (config deploy)']); + infoSpy.mockRestore(); + }); + + it('does not retry without a fallback', async () => { + mocks.refusedUsers.add('deploy'); + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + + await runRemoteCommandForTargets('/test', async () => {}); + + expect(mocks.attempts).toEqual(['deploy@1.2.3.4']); + expect(exitSpy).toHaveBeenCalledWith(1); + exitSpy.mockRestore(); + errorSpy.mockRestore(); + }); + + it('names both users when the fallback is refused too', async () => { + mocks.refusedUsers.add('deploy'); + mocks.refusedUsers.add('root'); + const exitSpy = vi.spyOn(process, 'exit').mockImplementation(() => undefined as never); + const failures: string[] = []; + const errorSpy = vi.spyOn(console, 'error').mockImplementation((...args) => { failures.push(args.join(' ')); }); + const logSpy = vi.spyOn(console, 'log').mockImplementation((...args) => { failures.push(args.join(' ')); }); + const writeSpy = vi.spyOn(process.stdout, 'write').mockImplementation((chunk) => { failures.push(String(chunk)); return true; }); + + await runRemoteCommandForTargets('/test', async () => {}, { sshFallback: asRoot }); + + expect(exitSpy).toHaveBeenCalledWith(1); + expect(failures.join('\n')).toContain('as deploy or root'); + exitSpy.mockRestore(); + errorSpy.mockRestore(); + logSpy.mockRestore(); + writeSpy.mockRestore(); + }); +}); + describe('runLocalCommand', () => { it('executes the command without SSH connection', async () => { const command = vi.fn(async (config: ShipnodeConfig) => { diff --git a/website/src/content/docs/docs/commands/init.md b/website/src/content/docs/docs/commands/init.md index 420f325..70fa6ff 100644 --- a/website/src/content/docs/docs/commands/init.md +++ b/website/src/content/docs/docs/commands/init.md @@ -3,15 +3,20 @@ title: shipnode init description: Create a shipnode.config.ts in the current project. --- -Interactive prompt that detects the framework, package manager, and app type, then writes `shipnode.config.ts`. +Detects the framework, package manager, app type and port. It asks only for your server's IP, an optional domain, and whether to install a database or Redis, then writes `shipnode.config.ts`. Everything else is a sensible default you can change in the file. ```bash npx shipnode init + +# or with no prompts at all +npx shipnode init --host 203.0.113.10 --domain api.example.com --non-interactive ``` ## Options | Flag | Purpose | |---|---| -| `--non-interactive` | Generate config from defaults without prompts. | +| `--host ` | Server IP or hostname. | +| `--domain ` | Domain to serve the app on (HTTPS is automatic). | +| `--non-interactive` | Generate config from detection, flags and defaults without prompts. | | `--print` | Print the config to stdout instead of writing a file. | diff --git a/website/src/content/docs/docs/quick-start.md b/website/src/content/docs/docs/quick-start.md index b7dfaf2..c9005bc 100644 --- a/website/src/content/docs/docs/quick-start.md +++ b/website/src/content/docs/docs/quick-start.md @@ -59,22 +59,20 @@ bun add -d @devalade/shipnode npx shipnode init ``` -This prompts for framework, package manager, app type, SSH target, domain, and port — then writes `shipnode.config.ts`. You can rerun `init` or edit the file by hand at any time. See the [configuration reference](/docs/configuration/) for every option. +`init` detects your framework, package manager and port. It asks for your server's IP, an optional domain, and whether to install a database or Redis, then writes `shipnode.config.ts`. You can edit the file by hand at any time. See the [configuration reference](/docs/configuration/) for every option. -A minimal backend config: +What it writes for a typical API: ```ts import { shipnode } from '@devalade/shipnode'; export default shipnode .backend() - .ssh({ host: '203.0.113.10', user: 'root' }) + .ssh({ host: '203.0.113.10', user: 'deploy' }) .deployTo('/var/www/api') - .pm2('api', { instances: 2 }) + .pm2('api') .port(3000) .domain('api.example.com') - .healthCheck('/health') - .nodeVersion('22') .pkgManager('pnpm') .build(); ``` @@ -85,7 +83,9 @@ export default shipnode npx shipnode setup ``` -One-time, idempotent. Installs **mise**, **Node.js**, **PM2** (+ `pm2-logrotate`), **Caddy**, and your package manager. Re-running it is safe — it skips anything already present. +One-time, idempotent. Installs **mise**, **Node.js**, **PM2**, **Caddy**, and your package manager, and creates a `deploy` user keyed with your SSH key. Re-running it is safe — it skips anything already present. + +The config says `user: 'deploy'`, but that user doesn't exist on a fresh server yet. On that first run, `setup` logs in as `root` to create it, and you don't have to edit anything. Verify with: @@ -95,9 +95,9 @@ npx shipnode doctor If anything is red, fix it before deploying. -## 4. Upload secrets +## 4. Secrets -Most apps need environment variables in production — database URLs, API keys, signing secrets. Keep them in a local file (do **not** commit it) and push it to the server once: +On the first deploy, shipnode uploads your local `.env` to the server for you. If you don't have one, the server gets an empty one. You only need this step to push a different file or to update values later: ```bash npx shipnode env --file .env.production @@ -131,7 +131,7 @@ install pnpm install --frozen-lockfile build pnpm run build symlink current -> releases/20260524160000 pm2 reload api --update-env -health GET /health 200 OK 47ms +health GET /health answered 47ms deployed https://api.example.com ``` @@ -225,8 +225,9 @@ npx shipnode logs Most common reasons: -- The `healthCheck` path in `shipnode.config.ts` doesn't exist in your app (404). -- The app didn't bind to the port from `.port(...)`. +- The app didn't bind to the port from `.port(...)` — the error says "Nothing answered on port …". +- The app answered with a 5xx. +- You set `.healthCheck(path)` and that route doesn't exist (404). Without a configured path, any response below 500 counts as healthy. - A required env var isn't set — re-check `npx shipnode env --file .env.production`. ### `Deploy is locked` even though nothing is running From b87316ee7292ed029956e19b9b9f4e07d35ceb83 Mon Sep 17 00:00:00 2001 From: Alade YESSOUFOU Date: Fri, 2 Oct 2026 17:27:45 +0200 Subject: [PATCH 2/4] feat(monitor): fleet-wide live monitor with streaming, filterable logs - monitor connects to every server by default and opens a fleet overview (release skew, unreachable replicas, per-server load) with drill-down to the existing per-replica panels; --on/--app still narrow it - real log streaming (PM2, systemd, Caddy) over the existing SSH connections, merged across servers, with reconnect/backoff and replay de-duplication, replacing the 2s `pm2 logs --nostream` poll - filter by server, app, process, level and text//regex//!exclude, in hide or dim mode, with live ERR/WARN counts, pause and scrollback - `logs --follow`, `--level`, `--grep` share the same stream and filters - ExecOptions gains `signal` and `pty` so a follow ends cleanly remotely - rollback from the monitor is refused for multi-server apps --- .claude/skills/shipnode/SKILL.md | 7 +- CHANGELOG.md | 5 + src/cli/commands/logs.ts | 165 +++++- src/cli/commands/monitor.ts | 55 +- src/cli/index.ts | 24 +- src/cli/monitor/App.tsx | 516 ++++++++-------- src/cli/monitor/app-selector.tsx | 64 -- src/cli/monitor/components/HelpOverlay.tsx | 76 ++- src/cli/monitor/fleet-model.ts | 147 +++++ src/cli/monitor/hooks/use-fleet.ts | 33 ++ src/cli/monitor/hooks/use-live-logs.ts | 70 --- src/cli/monitor/hooks/use-log-stream.ts | 90 +++ src/cli/monitor/hooks/use-monitor-data.ts | 126 ---- src/cli/monitor/index.tsx | 13 +- src/cli/monitor/layout/HeaderBar.tsx | 43 +- src/cli/monitor/layout/MonitorFrame.tsx | 49 +- src/cli/monitor/layout/StatusBar.tsx | 29 +- src/cli/monitor/log-color.ts | 15 + src/cli/monitor/log-view-state.ts | 103 ++++ src/cli/monitor/monitor-session.ts | 30 +- src/cli/monitor/panels/EventsPanel.tsx | 29 +- src/cli/monitor/panels/FleetPanel.tsx | 138 +++++ src/cli/monitor/panels/LogPanel.tsx | 108 ---- src/cli/monitor/panels/LogViewer.tsx | 140 +++++ src/cli/observe.ts | 82 ++- src/domain/observe/log-filter.ts | 194 ++++++ src/domain/observe/log-line.ts | 205 +++++++ src/domain/observe/log-source.ts | 92 +++ src/domain/remote/executor.ts | 11 + src/infrastructure/ssh/connection.ts | 9 +- src/services/observe/log-stream.ts | 246 ++++++++ tests/unit/monitor-fleet.test.ts | 183 ++++++ tests/unit/monitor.test.ts | 55 +- tests/unit/observe-logs.test.ts | 556 ++++++++++++++++++ website/src/components/Commands.astro | 3 +- .../src/content/docs/docs/commands/logs.md | 31 +- .../src/content/docs/docs/commands/monitor.md | 65 ++ 37 files changed, 2924 insertions(+), 883 deletions(-) delete mode 100644 src/cli/monitor/app-selector.tsx create mode 100644 src/cli/monitor/fleet-model.ts create mode 100644 src/cli/monitor/hooks/use-fleet.ts delete mode 100644 src/cli/monitor/hooks/use-live-logs.ts create mode 100644 src/cli/monitor/hooks/use-log-stream.ts delete mode 100644 src/cli/monitor/hooks/use-monitor-data.ts create mode 100644 src/cli/monitor/log-color.ts create mode 100644 src/cli/monitor/log-view-state.ts create mode 100644 src/cli/monitor/panels/FleetPanel.tsx delete mode 100644 src/cli/monitor/panels/LogPanel.tsx create mode 100644 src/cli/monitor/panels/LogViewer.tsx create mode 100644 src/domain/observe/log-filter.ts create mode 100644 src/domain/observe/log-line.ts create mode 100644 src/domain/observe/log-source.ts create mode 100644 src/services/observe/log-stream.ts create mode 100644 tests/unit/monitor-fleet.test.ts create mode 100644 tests/unit/observe-logs.test.ts create mode 100644 website/src/content/docs/docs/commands/monitor.md diff --git a/.claude/skills/shipnode/SKILL.md b/.claude/skills/shipnode/SKILL.md index 28cb0d2..3a8fe83 100644 --- a/.claude/skills/shipnode/SKILL.md +++ b/.claude/skills/shipnode/SKILL.md @@ -171,8 +171,10 @@ shipnode cloudflare init # one tunnel, ingress per app domain ### Monitor ```bash -shipnode monitor # live TUI: PM2, system, health, logs -shipnode monitor --app api +shipnode monitor # live TUI across every server: fleet overview, replica detail, streaming logs +shipnode monitor --app api # one app on all its servers +shipnode monitor --on web-2 +shipnode monitor --once # one snapshot; --json for machine-readable ``` ## Day-to-day @@ -180,6 +182,7 @@ shipnode monitor --app api ```bash shipnode status [--app name] shipnode logs [--app name] [--lines 500] +shipnode logs --follow [--level error] [--grep '/timeout/i'] [--on server] # live, merged across servers shipnode restart [--app name] shipnode stop [--app name] shipnode run "pnpm db:apply" [--app name] diff --git a/CHANGELOG.md b/CHANGELOG.md index 5769dbd..9b7414b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to `@devalade/shipnode` will be documented here. ## [Unreleased] +### Added +- **`monitor` watches the whole fleet.** With no `--on` it connects to every server and opens a fleet overview: each app on each server it runs on, the release each replica serves, process and health state, and a one-line verdict per app (a half-finished roll reads `split across 2 releases` and marks the replica that is `behind`; an unreachable server keeps its row). `Enter` opens the per-replica view the old single-server monitor showed. `--on` and `--app` still narrow it. Rollback from the monitor is refused for an app that runs on several servers, since it would split the fleet; use `shipnode rollback`. +- **Live log streaming with filters.** `f` opens a merged, streaming view of every server's logs (PM2, systemd and Caddy access logs) over the existing SSH connections, replacing the 2-second `pm2 logs --nostream` poll and its duplicated or dropped lines. Filter by server (`s`), app (`a`), process (`p`), level (`v`: all, warn+, error) and text, `/regex/` or `!exclude` search (`/`), in hide or dim mode (`m`), with live `ERR`/`WARN` counts. Pause, scroll back, and clear are supported. A dropped connection is retried with backoff without replaying lines already shown. +- **`shipnode logs --follow`** streams the same merged logs to the terminal, with `--level`, `--grep`, `--app`, `--on` and `--process`. `--level` and `--grep` also filter the one-shot `logs` output. + ### Changed - **`init` asks at most six questions instead of up to fifteen:** what you're deploying (pre-selected from detection), server IP, port, domain, and for backends whether to install a database or Redis. SSH user and port, deploy path, PM2 name, runtime, health check path and extra users take defaults you can edit in the file. `--host` and `--domain` let `init --non-interactive` write a complete config. A DB password is no longer written into the config as a literal; it reads `process.env.DB_PASSWORD`. If `shipnode.config.ts` already exists, `init` says so before asking anything. - **The default health check accepts any HTTP answer below 500.** Apps without a `/health` route no longer fail their first deploy on a 404. A path set with `.healthCheck(path)` (or `healthCheck.path`) is still held to 2xx/3xx; `healthCheck.strict` overrides either way. Failures now say whether nothing answered on the port or the configured route returned 404. diff --git a/src/cli/commands/logs.ts b/src/cli/commands/logs.ts index 9b54454..7fb1bbc 100644 --- a/src/cli/commands/logs.ts +++ b/src/cli/commands/logs.ts @@ -1,5 +1,33 @@ +import chalk from 'chalk'; import { runRemoteCommandForTargets } from '../runner.js'; import { isWatt, logsCommand, resolveWattUnits } from '../../domain/runtime/watt.js'; +import { loadConfig } from '../../config/loader.js'; +import { + compileQuery, + EMPTY_LOG_FILTER, + matchesLogFilter, + parseLevelOption, + type LogFilter, +} from '../../domain/observe/log-filter.js'; +import { detectLevel, LOG_LEVEL_RANK, type LogLine } from '../../domain/observe/log-line.js'; +import { planLogSources } from '../../domain/observe/log-source.js'; +import { getPm2Name } from '../../domain/pm2/apps.js'; +import { LogStream, type LogBinding } from '../../services/observe/log-stream.js'; +import { connectFleet, planObserveHosts } from '../observe.js'; +import { ui } from '../ui.js'; + +export interface LogsOptions { + lines?: number; + config?: string; + process?: string; + app?: string; + on?: string; + follow?: boolean; + /** Show this level and above: warn or error. */ + level?: string; + /** Plain text, `/regex/`, or `!not`. */ + grep?: string; +} /** * Prefix every line, not every block. @@ -8,15 +36,47 @@ import { isWatt, logsCommand, resolveWattUnits } from '../../domain/runtime/watt * leaves the reader guessing which box a stack trace came from once the second * server's output starts. */ -function prefixLines(text: string, label: string): string { +function prefixLines(text: string, label: string, keep: (line: string) => boolean): string { return text .split('\n') - .filter((line) => line.length > 0) + .filter((line) => line.length > 0 && keep(line)) .map((line) => `${label} ${line}`) .join('\n'); } -export async function cmdLogs(cwd: string, options: { lines?: number; config?: string; process?: string; app?: string; on?: string }): Promise { +/** Level and text filtering for raw text, where no stream has classified the line yet. */ +function textFilter(filter: LogFilter): (line: string) => boolean { + const { matcher } = compileQuery(filter.query); + const min = filter.minLevel === null ? 0 : LOG_LEVEL_RANK[filter.minLevel]; + return (line) => (LOG_LEVEL_RANK[detectLevel(line) ?? 'info'] >= min) && (matcher === null || matcher(line)); +} + +function filterFromOptions(options: LogsOptions): LogFilter | null { + const level = parseLevelOption(options.level); + if (level.error !== undefined) { + ui.error(level.error); + return null; + } + const { error } = compileQuery(options.grep ?? ''); + if (error !== undefined) { + ui.error(`Invalid --grep: ${error}`); + return null; + } + return { ...EMPTY_LOG_FILTER, minLevel: level.level, query: options.grep ?? '' }; +} + +export async function cmdLogs(cwd: string, options: LogsOptions): Promise { + const filter = filterFromOptions(options); + if (filter === null) { + process.exit(1); + return; + } + if (options.follow) { + await followLogs(cwd, options, filter); + return; + } + + const keep = textFilter(filter); await runRemoteCommandForTargets( cwd, async ({ config, executor, serverName }) => { @@ -45,8 +105,8 @@ export async function cmdLogs(cwd: string, options: { lines?: number; config?: s for (const unit of units) { const result = await executor.exec(logsCommand(unit, { lines })); const label = `[${serverName} ${app.name} ${unit}]`; - if (result.stdout) process.stdout.write(`${prefixLines(result.stdout, label)}\n`); - if (result.stderr) process.stderr.write(`${prefixLines(result.stderr, label)}\n`); + if (result.stdout) process.stdout.write(`${prefixLines(result.stdout, label, keep)}\n`); + if (result.stderr) process.stderr.write(`${prefixLines(result.stderr, label, keep)}\n`); } continue; } @@ -58,10 +118,101 @@ export async function cmdLogs(cwd: string, options: { lines?: number; config?: s `${mise}; mise exec "node@${nodeVersion}" -- pm2 logs ${target} --lines ${lines} --nostream`, ); const label = `[${serverName} ${app.name}]`; - if (result.stdout) process.stdout.write(`${prefixLines(result.stdout, label)}\n`); - if (result.stderr) process.stderr.write(`${prefixLines(result.stderr, label)}\n`); + if (result.stdout) process.stdout.write(`${prefixLines(result.stdout, label, keep)}\n`); + if (result.stderr) process.stderr.write(`${prefixLines(result.stderr, label, keep)}\n`); } }, { configPath: options.config, appName: options.app, serverName: options.on }, ); } + +const SERVER_COLORS = [chalk.cyan, chalk.magenta, chalk.blue, chalk.greenBright, chalk.whiteBright]; + +/** + * `shipnode logs --follow`: every server at once, merged, until Ctrl-C. + * + * The same stream the monitor uses, so what counts as an error and how a + * reconnect avoids replaying lines are not re-implemented for the CLI. Lines go + * to stdout and status to stderr, so `shipnode logs -f | grep ...` stays clean. + */ +async function followLogs(cwd: string, options: LogsOptions, initialFilter: LogFilter): Promise { + let filter = initialFilter; + const config = await loadConfig(cwd, options.config); + const plan = planObserveHosts(config, { app: options.app, on: options.on }); + if (plan.isErr()) { + ui.error(plan.error.message); + process.exit(1); + return; + } + + const fleet = await connectFleet(config, plan.value); + try { + // A process name only means something within one app; the stream labels + // lines with the full PM2-style name, so translate the short one once. + if (options.process !== undefined) { + const candidates = [...new Map(fleet.hosts.flatMap((host) => host.apps).map((app) => [app.name, app])).values()]; + if (candidates.length !== 1) { + ui.error('--process requires --app to target a specific app'); + process.exit(1); + return; + } + const app = candidates[0]; + const declared = app.pm2?.apps ?? []; + if (!declared.some((p) => p.name === options.process)) { + ui.error(`No process named '${options.process}'. Known: ${declared.map((p) => p.name).join(', ') || '(none)'}`); + process.exit(1); + return; + } + filter = { ...filter, process: getPm2Name(declared[0].name, options.process) }; + } + + const bindings: LogBinding[] = []; + for (const host of fleet.hosts) { + if (host.executor === null) { + ui.warn(`${host.name}: ${host.error ?? 'unreachable'}`); + continue; + } + const apps = options.app === undefined ? host.apps : host.apps.filter((app) => app.name === options.app); + for (const app of apps) { + try { + const sources = await planLogSources(host.executor, host.name, host.config.remotePath, app); + for (const source of sources) bindings.push({ executor: host.executor, source }); + } catch (cause: unknown) { + ui.warn(`${host.name}/${app.name}: ${cause instanceof Error ? cause.message : String(cause)}`); + } + } + } + + if (bindings.length === 0) { + ui.error('Nothing to follow: no reachable server runs a matching app.'); + process.exit(1); + return; + } + + const serverNames = [...new Set(bindings.map((binding) => binding.source.server))].sort(); + const multiServer = serverNames.length > 1; + const multiApp = new Set(bindings.map((binding) => binding.source.app)).size > 1; + + const print = (line: LogLine): void => { + if (!matchesLogFilter(line, filter)) return; + const who = line.process === null || line.process === line.app ? '' : `:${line.process}`; + const head = [multiServer ? line.server : null, multiApp ? line.app : null].filter((p) => p !== null).join(' '); + const label = head === '' && who === '' ? '' : `[${head}${who}] `; + const color = SERVER_COLORS[serverNames.indexOf(line.server) % SERVER_COLORS.length]; + const text = line.level === 'error' ? chalk.red(line.text) : line.level === 'warn' ? chalk.yellow(line.text) : line.text; + process.stdout.write(`${label === '' ? '' : color(label)}${text}\n`); + }; + + const stream = new LogStream({ bindings, backlog: options.lines ?? 100, onLine: print }); + stream.start(); + ui.info(`Following ${bindings.length} source(s) on ${serverNames.join(', ')} — Ctrl-C to stop`); + + await new Promise((resolve) => { + process.once('SIGINT', resolve); + process.once('SIGTERM', resolve); + }); + await stream.stop(); + } finally { + fleet.close(); + } +} diff --git a/src/cli/commands/monitor.ts b/src/cli/commands/monitor.ts index 3c16903..2b03885 100644 --- a/src/cli/commands/monitor.ts +++ b/src/cli/commands/monitor.ts @@ -1,14 +1,7 @@ import { loadConfig } from '../../config/loader.js'; -import { SshConnection } from '../../infrastructure/ssh/connection.js'; -import { configForServer } from '../../domain/servers.js'; import type { ShipnodeConfig } from '../../shared/types.js'; import { runMonitor } from '../monitor/index.js'; -import { - getAccessoriesForMonitorTarget, - getAppsForMonitorTarget, - resolveMonitorSession, -} from '../monitor/monitor-session.js'; -import { observeStateJson, printObserveStatus, takeSnapshot } from '../observe.js'; +import { connectFleet, observeStateJson, planObserveHosts, printObserveStatus, takeSnapshot } from '../observe.js'; import { ui } from '../ui.js'; export async function cmdMonitor( @@ -49,50 +42,24 @@ async function runLiveDashboard( config: ShipnodeConfig, options: { app?: string; on?: string; interval: number }, ): Promise { - const session = resolveMonitorSession(config, options.app, options.on); - if (session.isErr()) { - ui.error(session.error.message); + // The whole workspace by default; --app and --on narrow it. A fleet is + // exactly what this view is for, so nothing here insists on a single replica. + const plan = planObserveHosts(config, { app: options.app, on: options.on }); + if (plan.isErr()) { + ui.error(plan.error.message); process.exit(1); return; } - - const apps = getAppsForMonitorTarget(config, session.value.target.name); - if (apps.isErr()) { - ui.error(apps.error.message); - process.exit(1); - return; - } - const accessoryNames = getAccessoriesForMonitorTarget(config, session.value.target.name); - if (accessoryNames.isErr()) { - ui.error(accessoryNames.error.message); - process.exit(1); - return; - } - - const { target, app } = session.value; - const host = `${target.ssh.user}@${target.ssh.host}:${target.ssh.port}`; - const ssh = new SshConnection(); - try { - await ssh.connect(target.ssh); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - ui.error(`Failed to connect to ${host}: ${msg}`); + if (plan.value.length === 0) { + ui.error('Nothing to monitor: no server runs an app or accessory.'); process.exit(1); return; } + const fleet = await connectFleet(config, plan.value); try { - await runMonitor({ - executor: ssh, - config: configForServer(config, target.name), - app, - apps: apps.value, - accessoryNames: accessoryNames.value, - targetName: target.name, - host, - interval: options.interval, - }); + await runMonitor({ fleet, interval: options.interval, focusApp: options.app }); } finally { - ssh.disconnect(); + fleet.close(); } } diff --git a/src/cli/index.ts b/src/cli/index.ts index 7f24be1..a69c226 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -141,13 +141,25 @@ program program .command('logs') - .description('Show application logs') - .option('--lines ', 'Number of log lines to show', '100') + .description('Show application logs (add --follow to stream every server live)') + .option('--lines ', 'Number of log lines to show (also the backlog for --follow)', '100') + .option('-f, --follow', 'Stream new lines from every server until Ctrl-C') + .option('--level ', 'Only this level and above: warn or error') + .option('--grep ', 'Only lines matching text, /regex/, or !text to exclude') .option('--process ', 'Target a specific process (PM2 app or watt unit)') .option('--app ', 'Target a specific app') .option('--config ', 'Use a specific config file') .option('--on ', 'Target a specific server') - .action((opts) => cmdLogs(process.cwd(), { lines: parseInt(opts.lines, 10), process: opts.process, app: opts.app, config: opts.config, on: opts.on })); + .action((opts) => cmdLogs(process.cwd(), { + lines: parseInt(opts.lines, 10), + follow: opts.follow, + level: opts.level, + grep: opts.grep, + process: opts.process, + app: opts.app, + config: opts.config, + on: opts.on, + })); program .command('restart') @@ -177,10 +189,10 @@ program program .command('monitor') - .description('Live TUI dashboard with process stats (PM2 or systemd), system metrics, and logs') + .description('Live TUI dashboard across every server: fleet overview, per-replica detail, and filterable streaming logs') .option('--interval ', 'Polling interval in seconds (default: 2)', '2') - .option('--app ', 'Target a specific app') - .option('--on ', 'Watch one replica (required for a live TUI on a fleet)') + .option('--app ', 'Watch a specific app') + .option('--on ', 'Watch one server instead of the whole fleet') .option('--once', 'Collect one snapshot and exit') .option('--json', 'Print one snapshot as JSON (implies --once)') .option('--config ', 'Use a specific config file') diff --git a/src/cli/monitor/App.tsx b/src/cli/monitor/App.tsx index 09d422a..f8d6b12 100644 --- a/src/cli/monitor/App.tsx +++ b/src/cli/monitor/App.tsx @@ -1,116 +1,162 @@ -import { Box, useInput, useApp, useStdout } from 'ink'; -import { useState } from 'react'; -import type { RemoteExecutor } from '../../domain/remote/executor.js'; -import type { ShipnodeConfig, ShipnodeApp } from '../../shared/types.js'; +import { Box, useApp, useInput, useStdout } from 'ink'; +import { useEffect, useReducer, useRef, useState } from 'react'; +import { HEALTH_ALERT_THRESHOLD } from '../../services/observe/session.js'; +import type { ObserveEvent } from '../../services/observe/events.js'; +import { applyLogFilter, countLevels, logFacets } from '../../domain/observe/log-filter.js'; +import type { FleetConnection } from '../observe.js'; import { Pm2Panel } from './panels/Pm2Panel.js'; import { SystemPanel } from './panels/SystemPanel.js'; import { ReleasePanel } from './panels/ReleasePanel.js'; -import { LogPanel } from './panels/LogPanel.js'; import { EventsPanel } from './panels/EventsPanel.js'; +import { FleetPanel } from './panels/FleetPanel.js'; +import { LogViewer } from './panels/LogViewer.js'; import { StaticFrontendPanel } from './panels/StaticFrontendPanel.js'; import { AccessoriesPanel } from './panels/AccessoriesPanel.js'; -import { AppSelector } from './app-selector.js'; import { HelpOverlay } from './components/HelpOverlay.js'; import { ConfirmDialog } from './components/ConfirmDialog.js'; import { restartProcess, rollbackToRelease } from './actions.js'; -import chalk from 'chalk'; -import { useMonitorData, HEALTH_ALERT_THRESHOLD } from './hooks/use-monitor-data.js'; -import { useLiveLogs } from './hooks/use-live-logs.js'; import { MonitorFrame, WaitingPanel } from './layout/MonitorFrame.js'; +import type { HeaderAlert } from './layout/HeaderBar.js'; +import { buildFleetRows, toMetricsSnapshot } from './fleet-model.js'; +import { INITIAL_LOG_VIEW, logViewReducer } from './log-view-state.js'; +import { useFleet } from './hooks/use-fleet.js'; +import { useLogStream } from './hooks/use-log-stream.js'; -type View = 'dashboard' | 'logs'; -type Overlay = 'none' | 'selector' | 'help' | 'confirmRestart' | 'confirmRollback'; +type View = 'fleet' | 'detail' | 'logs'; +type Overlay = 'none' | 'help' | 'confirmRestart' | 'confirmRollback'; + +/** Rows the logs view spends on everything but log lines: header, status, border, title, filter bar, source warning. */ +const LOGS_CHROME_ROWS = 9; +const STRIP_ROWS = 5; +const MAX_ALERTS = 3; interface AppProps { - executor: RemoteExecutor; - config: ShipnodeConfig; - app: ShipnodeApp; - apps: ShipnodeApp[]; - accessoryNames: string[]; - targetName: string; - host: string; + fleet: FleetConnection; interval: number; + /** Open on this app instead of the fleet overview. */ + focusApp?: string; } -export function App({ executor, config, app: initialApp, apps, accessoryNames, targetName, host, interval }: AppProps) { +export function App({ fleet: connection, interval, focusApp }: AppProps) { const { exit } = useApp(); const { stdout } = useStdout(); - const [currentApp, setCurrentApp] = useState(initialApp); - const [view, setView] = useState('dashboard'); + const { state, session, refresh } = useFleet(connection.targets, interval); + const [view, setView] = useState('fleet'); + const [returnTo, setReturnTo] = useState>('fleet'); const [overlay, setOverlay] = useState('none'); - const [liveMode, setLiveMode] = useState(false); - const [logFilter, setLogFilter] = useState(null); - const [logSearch, setLogSearch] = useState(''); - const [searchTyping, setSearchTyping] = useState(false); - const [logsPaused, setLogsPaused] = useState(false); - const [selectedRow, setSelectedRow] = useState(0); - const monitor = useMonitorData(executor, config, currentApp, interval, accessoryNames); - const liveActive = liveMode || view === 'logs'; - const liveLogs = useLiveLogs(executor, currentApp, liveActive && !logsPaused, interval, logFilter); - - // On the 24-row terminals this layout was tuned for, the releases box has - // room for exactly one row; taller terminals get a few more so a rollback - // selection made with ↑/↓ is actually visible before confirming. + const [stripOn, setStripOn] = useState(false); + const [selectedKey, setSelectedKey] = useState(null); + const [detailKey, setDetailKey] = useState(null); + const [detailRow, setDetailRow] = useState(0); + const [logView, dispatchLog] = useReducer(logViewReducer, INITIAL_LOG_VIEW); + + const rows = buildFleetRows(state.fleets); + const logs = useLogStream(connection.hosts, stripOn || view === 'logs'); const terminalRows = stdout?.rows ?? 24; + + // ── Fleet selection ────────────────────────────────────────────── + const selectedAt = Math.max(0, rows.findIndex((row) => row.key === selectedKey)); + const selected = rows[selectedAt]; + + // A fleet of one is not worth an overview: open it directly, once. + const autoOpened = useRef(false); + useEffect(() => { + if (autoOpened.current || rows.length === 0) return; + autoOpened.current = true; + const focus = focusApp === undefined ? undefined : rows.find((row) => row.app === focusApp); + const target = rows.length === 1 ? rows[0] : focus; + if (target !== undefined && rows.length === 1) { + setDetailKey(target.key); + setView('detail'); + setReturnTo('detail'); + } + setSelectedKey((focus ?? rows[0]).key); + }, [rows.length]); + + // ── Detail target ──────────────────────────────────────────────── + const detail = view === 'detail' || overlay.startsWith('confirm') ? rows.find((row) => row.key === detailKey) : undefined; + const detailHost = detail === undefined ? undefined : connection.hosts.find((host) => host.name === detail.server); + const detailApp = detail === undefined ? undefined : detailHost?.apps.find((app) => app.name === detail.app); + const detailServer = detail === undefined ? undefined : state.servers.find((server) => server.server === detail.server); + const detailSnapshot = detailServer !== undefined && detail?.snapshot !== undefined ? toMetricsSnapshot(detailServer, detail.snapshot) : null; + const detailHistory = detail === undefined ? undefined : session.history(detail.server, detail.app); + + const processes = detailSnapshot?.processes ?? []; + const maxReleases = (detailHost?.accessoryNames.length ?? 0) > 0 ? 4 : 5; + const releaseRows = (detailSnapshot?.releases ?? []).slice(0, maxReleases); + const totalDetailRows = processes.length + releaseRows.length; + const detailIndex = totalDetailRows === 0 ? 0 : Math.min(detailRow, totalDetailRows - 1); + const selectedProcess = detailIndex < processes.length ? processes[detailIndex] : undefined; + const selectedRelease = detailIndex >= processes.length ? releaseRows[detailIndex - processes.length] : undefined; + const currentTimestamp = detailSnapshot?.currentRelease?.split('/').pop() ?? null; const releaseBoxExtra = Math.max(0, Math.min(6, terminalRows - 30)); - // Selection runs over processes first, then the visible release rows, so - // ↓ walks straight from the PM2 panel into the Releases panel. - const processes = monitor.snapshot?.processes ?? []; - const maxReleases = accessoryNames.length > 0 ? 4 : 5; - const releaseRows = (monitor.snapshot?.releases ?? []).slice(0, maxReleases); - const totalRows = processes.length + releaseRows.length; - const selectedIndex = totalRows === 0 ? 0 : Math.min(selectedRow, totalRows - 1); - const selectedInfo = selectedIndex < processes.length ? processes[selectedIndex] : undefined; - const selectedRelease = - selectedIndex >= processes.length ? releaseRows[selectedIndex - processes.length] : undefined; - const currentTimestamp = monitor.snapshot?.currentRelease?.split('/').pop() ?? null; - const alertStreak = - monitor.healthFailStreak >= HEALTH_ALERT_THRESHOLD ? monitor.healthFailStreak : 0; + // ── Logs ───────────────────────────────────────────────────────── + const source = logView.frozen ?? logs.lines; + const filtered = applyLogFilter(source, logView.filter); + const counts = countLevels(source, logView.filter); + const facets = logFacets(logs.lines, logView.filter); + const serverNames = connection.hosts.map((host) => host.name).sort(); + const showServer = serverNames.length > 1; + const showApp = new Set(logs.lines.map((line) => line.app)).size > 1; + const troubled = logs.health.filter((entry) => entry.state !== 'live'); + const logHeight = Math.max(3, terminalRows - LOGS_CHROME_ROWS); + const maxScroll = Math.max(0, filtered.lines.length - logHeight); + // ── Alerts ─────────────────────────────────────────────────────── + const alerts: HeaderAlert[] = []; + for (const server of state.servers) { + if (server.error !== undefined) alerts.push({ text: `${server.server} DOWN` }); + else if (server.deployLock != null) alerts.push({ text: `DEPLOY LOCK ${server.server} (${server.deployLock.ageSeconds}s)` }); + } + for (const row of state.fleets.flatMap((f) => f.replicas.map((r) => ({ app: f.app, server: r.server })))) { + const streak = session.healthFailStreak(row.server, row.app); + if (streak >= HEALTH_ALERT_THRESHOLD) alerts.push({ text: `HEALTH ${row.app}@${row.server} ×${streak}` }); + } + const shownAlerts = alerts.slice(0, MAX_ALERTS); + if (alerts.length > MAX_ALERTS) shownAlerts.push({ text: `+${alerts.length - MAX_ALERTS} more` }); + + // Ring the terminal bell once when a health check starts failing. + const lastEvent = useRef(undefined); + useEffect(() => { + const events = state.events; + const from = lastEvent.current === undefined ? 0 : events.indexOf(lastEvent.current) + 1; + if (events.slice(from).some((event) => event.kind === 'health-failing')) stdout?.write('\x07'); + lastEvent.current = events[events.length - 1]; + }, [state.events]); + + // ── Actions ────────────────────────────────────────────────────── const confirmRestart = async (): Promise => { setOverlay('none'); - if (selectedInfo === undefined) return; - monitor.appendEvent(chalk.yellow(`Restarting ${selectedInfo.pm2Name}…`)); - const result = await restartProcess(executor, selectedInfo.pm2Name, selectedInfo.supervisor); - if (result.isOk()) { - monitor.appendEvent(chalk.green(`Restarted ${chalk.bold(selectedInfo.pm2Name)}`)); - } else { - monitor.appendEvent(chalk.red(result.error.message)); - } - void monitor.refresh(); + if (selectedProcess === undefined || detailHost?.executor == null) return; + session.notice(`Restarting ${selectedProcess.pm2Name} on ${detailHost.name}…`); + const result = await restartProcess(detailHost.executor, selectedProcess.pm2Name, selectedProcess.supervisor); + session.notice(result.isOk() ? `Restarted ${selectedProcess.pm2Name} on ${detailHost.name}` : result.error.message); + void refresh(); }; const confirmRollback = async (): Promise => { setOverlay('none'); - if (selectedRelease === undefined) return; - monitor.appendEvent(chalk.yellow(`Rolling back to ${selectedRelease.timestamp}…`)); - const result = await rollbackToRelease(executor, config, currentApp, selectedRelease.timestamp); - if (result.isOk()) { - monitor.appendEvent(chalk.green(`Rolled back to ${chalk.bold(selectedRelease.timestamp)}`)); - } else { - monitor.appendEvent(chalk.red(result.error.message)); - } - void monitor.refresh(); + if (selectedRelease === undefined || detailHost?.executor == null || detailApp === undefined) return; + session.notice(`Rolling back ${detailApp.name} on ${detailHost.name} to ${selectedRelease.timestamp}…`); + const result = await rollbackToRelease(detailHost.executor, detailHost.config, detailApp, selectedRelease.timestamp); + session.notice(result.isOk() ? `Rolled back to ${selectedRelease.timestamp}` : result.error.message); + void refresh(); }; - const exitLogsView = (): void => { - setView('dashboard'); - setSearchTyping(false); - setLogSearch(''); - setLogsPaused(false); + const openLogs = (from: Exclude): void => { + const scope = from === 'detail' ? detail : selected; + dispatchLog({ type: 'enter', server: from === 'detail' ? (scope?.server ?? null) : null, app: scope?.app ?? null }); + setReturnTo(from); + setView('logs'); }; - const cycleLogFilter = (direction: 1 | -1): void => { - if (currentApp.appType === 'frontend') return; - const names = monitor.snapshot?.processes.map((p) => p.pm2Name) ?? []; - if (names.length === 0) return; - const cycle: Array = [null, ...names]; - const index = cycle.indexOf(logFilter); - const next = cycle[(index + direction + cycle.length) % cycle.length]; - setLogFilter(next); + const closeLogs = (): void => { + dispatchLog({ type: 'resume' }); + setView(returnTo); }; + // ── Keys ───────────────────────────────────────────────────────── useInput((input, key) => { if (overlay === 'help') { setOverlay('none'); @@ -119,24 +165,12 @@ export function App({ executor, config, app: initialApp, apps, accessoryNames, t if (overlay !== 'none') return; // While typing a search query every printable key belongs to the query, - // including q/f/r — this branch must stay ahead of the global bindings. - if (view === 'logs' && searchTyping) { - if (key.return) { - setSearchTyping(false); - return; - } - if (key.escape) { - setSearchTyping(false); - setLogSearch(''); - return; - } - if (key.backspace || key.delete) { - setLogSearch((s) => s.slice(0, -1)); - return; - } - if (input !== '' && !key.ctrl && !key.meta) { - setLogSearch((s) => s + input); - } + // including q/f/r - this branch must stay ahead of the global bindings. + if (view === 'logs' && logView.typing) { + if (key.return) dispatchLog({ type: 'commit-search' }); + else if (key.escape) dispatchLog({ type: 'cancel-search' }); + else if (key.backspace || key.delete) dispatchLog({ type: 'backspace' }); + else if (input !== '' && !key.ctrl && !key.meta) dispatchLog({ type: 'type', text: input }); return; } @@ -149,90 +183,90 @@ export function App({ executor, config, app: initialApp, apps, accessoryNames, t return; } if (input === 'r' || input === 'R') { - void monitor.refresh(); - monitor.appendEvent(chalk.green('Refresh triggered')); - return; - } - if (input === 'f' || input === 'F') { - if (view === 'logs') exitLogsView(); - else setView('logs'); + void refresh(); + session.notice('Refresh triggered'); return; } if (view === 'logs') { - if (key.escape) exitLogsView(); - if (key.leftArrow) cycleLogFilter(-1); - if (key.rightArrow) cycleLogFilter(1); - if (input === '/') { - setSearchTyping(true); - setLogSearch(''); + if (input === 'f' || input === 'F' || key.escape) return closeLogs(); + if (input === 's' || input === 'S') return dispatchLog({ type: 'cycle', dimension: 'server', direction: input === 's' ? 1 : -1, facets }); + if (input === 'a' || input === 'A') return dispatchLog({ type: 'cycle', dimension: 'app', direction: input === 'a' ? 1 : -1, facets }); + if (input === 'p' || input === 'P') return dispatchLog({ type: 'cycle', dimension: 'process', direction: input === 'p' ? 1 : -1, facets }); + if (input === 'v') return dispatchLog({ type: 'cycle-level' }); + if (input === 'm') return dispatchLog({ type: 'toggle-mode' }); + if (input === '/') return dispatchLog({ type: 'start-search' }); + if (input === 'c') return dispatchLog({ type: 'clear-filters' }); + if (input === 'C') { + dispatchLog({ type: 'resume' }); + return logs.clear(); } - if (input === ' ') setLogsPaused((p) => !p); + if (input === ' ') return dispatchLog({ type: 'toggle-pause', live: logs.lines }); + if (input === 'G' || key.end) return dispatchLog({ type: 'resume' }); + if (key.upArrow) return dispatchLog({ type: 'scroll', delta: 1, live: logs.lines, max: maxScroll }); + if (key.downArrow) return dispatchLog({ type: 'scroll', delta: -1, live: logs.lines, max: maxScroll }); + if (key.pageUp) return dispatchLog({ type: 'scroll', delta: logHeight - 1, live: logs.lines, max: maxScroll }); + if (key.pageDown) return dispatchLog({ type: 'scroll', delta: -(logHeight - 1), live: logs.lines, max: maxScroll }); return; } - if (key.tab) { - setOverlay('selector'); + if (input === 'f' || input === 'F') return openLogs(view); + if (input === 'l' || input === 'L') { + setStripOn((on) => !on); return; } - if (key.upArrow) { - setSelectedRow(Math.max(0, selectedIndex - 1)); + + if (view === 'fleet') { + if (key.upArrow) setSelectedKey(rows[Math.max(0, selectedAt - 1)]?.key ?? null); + else if (key.downArrow) setSelectedKey(rows[Math.min(rows.length - 1, selectedAt + 1)]?.key ?? null); + else if (key.return && selected !== undefined) { + setDetailKey(selected.key); + setDetailRow(0); + setReturnTo('fleet'); + setView('detail'); + } return; } - if (key.downArrow) { - setSelectedRow(Math.min(Math.max(totalRows - 1, 0), selectedIndex + 1)); + + // detail + if (key.escape) { + setView('fleet'); return; } + if (key.upArrow) return setDetailRow(Math.max(0, detailIndex - 1)); + if (key.downArrow) return setDetailRow(Math.min(Math.max(totalDetailRows - 1, 0), detailIndex + 1)); if (key.return || input === 'x' || input === 'X') { - if (selectedInfo !== undefined) { - if (monitor.snapshot?.deployLock != null) { - monitor.appendEvent(chalk.red('Restart blocked: a deploy is in progress (lock held)')); - return; - } - setOverlay('confirmRestart'); + if (detailSnapshot?.deployLock != null) { + session.notice(`Blocked: a deploy is in progress on ${detail?.server} (lock held)`); return; } + if (selectedProcess !== undefined) return setOverlay('confirmRestart'); if (selectedRelease !== undefined) { - if (monitor.snapshot?.deployLock != null) { - monitor.appendEvent(chalk.red('Rollback blocked: a deploy is in progress (lock held)')); + if (detail?.replicated === true) { + session.notice(`Rollback on one replica of ${detail.app} would split the fleet - use 'shipnode rollback'`); return; } if (selectedRelease.status !== 'success') { - monitor.appendEvent(chalk.red('Cannot roll back to a failed release')); + session.notice('Cannot roll back to a failed release'); return; } if (selectedRelease.timestamp === currentTimestamp) { - monitor.appendEvent(chalk.yellow(`${selectedRelease.timestamp} is already the current release`)); + session.notice(`${selectedRelease.timestamp} is already the current release`); return; } setOverlay('confirmRollback'); - return; } - return; - } - if (input === 'l' || input === 'L') { - setLiveMode((v) => { - const next = !v; - if (next) { - monitor.appendEvent(chalk.yellow('Live logs enabled')); - } else { - liveLogs.clearLogs(); - monitor.appendEvent(chalk.yellow('Live logs disabled')); - } - return next; - }); } }); - if (overlay === 'help') { - return ; - } + // ── Overlays ───────────────────────────────────────────────────── + if (overlay === 'help') return ; - if (overlay === 'confirmRestart' && selectedInfo !== undefined) { - const dropsRequests = selectedInfo.execMode !== 'cluster' || selectedInfo.instances <= 1; + if (overlay === 'confirmRestart' && selectedProcess !== undefined) { + const dropsRequests = selectedProcess.execMode !== 'cluster' || selectedProcess.instances <= 1; return ( { void confirmRestart(); @@ -242,12 +276,12 @@ export function App({ executor, config, app: initialApp, apps, accessoryNames, t ); } - if (overlay === 'confirmRollback' && selectedRelease !== undefined) { + if (overlay === 'confirmRollback' && selectedRelease !== undefined && detailApp !== undefined) { return ( server.error !== undefined).length; + const frame = { + interval, + liveMode: stripOn || view === 'logs', + alerts: shownAlerts, + lastUpdate: state.lastUpdate, + polling: state.polling, + }; + + if (view === 'logs') { + const scope = [logView.filter.server, logView.filter.app].filter((part) => part !== null).join(' / '); return ( - { - setCurrentApp(app); - setOverlay('none'); - exitLogsView(); - setLogFilter(null); - setSelectedRow(0); - monitor.reset(); - liveLogs.clearLogs(); - setLiveMode(false); - monitor.appendEvent(chalk.green(`Switched to ${chalk.bold(app.name)}`)); - }} - onCancel={() => setOverlay('none')} - /> + 0 ? `no logs from ${logs.skipped.join(', ')}` : null} + > + + + + ); } - if (view === 'logs') { - const rows = stdout?.rows ?? 24; - const title = - currentApp.appType === 'frontend' - ? `Caddy Access Log — ${currentApp.name}` - : `Live Logs — ${logFilter ?? 'all processes'} (←/→ filter, / search, space pause, F back)`; + const bottom = stripOn ? ( + + ) : ( + + ); + + if (view === 'fleet') { return ( 0 ? ` · ${unreachable} down` : ''}`} + hints="↑/↓ select Enter open f logs l strip" > - - + + {bottom} ); } + // detail + const unavailable = detail?.reachable === false ? (detail.error ?? 'server unreachable') : null; return ( - {currentApp.appType === 'frontend' ? ( - - ) : monitor.snapshot ? ( + {detailApp?.appType === 'frontend' ? ( + + ) : detailSnapshot && detailHistory ? ( ) : ( )} - {monitor.snapshot ? ( + {detailSnapshot && detailHistory ? ( <> - - {accessoryNames.length > 0 && ( - + + {(detailHost?.accessoryNames.length ?? 0) > 0 && ( + )} ) : ( @@ -364,26 +420,18 @@ export function App({ executor, config, app: initialApp, apps, accessoryNames, t - {monitor.snapshot && ( - 0 ? 6 : 7) + releaseBoxExtra}> + {detailSnapshot && ( + 0 ? 6 : 7) + releaseBoxExtra}> )} - - {liveMode ? ( - - ) : ( - - )} - + {bottom} ); } diff --git a/src/cli/monitor/app-selector.tsx b/src/cli/monitor/app-selector.tsx deleted file mode 100644 index 2a7cc11..0000000 --- a/src/cli/monitor/app-selector.tsx +++ /dev/null @@ -1,64 +0,0 @@ -import { Box, Text, useInput } from 'ink'; -import { useState } from 'react'; -import type { ShipnodeApp } from '../../shared/types.js'; - -const BG = '#0d1117'; -const BG_OVERLAY = '#161b22'; -const BG_SELECTED = '#1f2937'; - -interface AppSelectorProps { - apps: ShipnodeApp[]; - targetName: string; - onSelect: (app: ShipnodeApp) => void; - onCancel: () => void; -} - -export function AppSelector({ apps, targetName, onSelect, onCancel }: AppSelectorProps) { - const [selected, setSelected] = useState(0); - - useInput((input, key) => { - if (key.return) { - onSelect(apps[selected]); - } else if (key.escape || input === 'q') { - onCancel(); - } else if (key.upArrow) { - setSelected(Math.max(0, selected - 1)); - } else if (key.downArrow) { - setSelected(Math.min(apps.length - 1, selected + 1)); - } - }); - - return ( - - - Select App ({apps.length} on {targetName}) - - {apps.map((app, i) => { - const typeLabel = app.appType === 'backend' ? 'backend' : 'frontend'; - const isSelected = i === selected; - return ( - - - {isSelected ? '❯' : ' '} {app.name} - - ({typeLabel}) - {app.domain && {app.domain}} - - ); - })} - ↑↓ navigate Enter select q/Escape cancel - - ); -} diff --git a/src/cli/monitor/components/HelpOverlay.tsx b/src/cli/monitor/components/HelpOverlay.tsx index 53148fc..93e194a 100644 --- a/src/cli/monitor/components/HelpOverlay.tsx +++ b/src/cli/monitor/components/HelpOverlay.tsx @@ -3,18 +3,51 @@ import { Box, Text } from 'ink'; const BG = '#0d1117'; const ACCENT = '#d6a85d'; -const BINDINGS: Array<[string, string]> = [ - ['q', 'quit'], - ['Tab', 'switch app on this server'], - ['r', 'refresh now'], - ['l', 'toggle live log strip'], - ['f', 'fullscreen logs'], - ['←/→', 'filter logs by process (fullscreen)'], - ['/', 'search logs (fullscreen)'], - ['space', 'pause/resume logs (fullscreen)'], - ['↑/↓', 'select process or release'], - ['Enter/x', 'restart process / rollback to release'], - ['?', 'this help'], +type Section = { title: string; bindings: Array<[string, string]> }; + +const SECTIONS: Section[] = [ + { + title: 'Everywhere', + bindings: [ + ['q', 'quit'], + ['r', 'refresh now'], + ['f', 'full-screen logs (scoped to what is selected)'], + ['l', 'toggle the live log strip'], + ['?', 'this help'], + ], + }, + { + title: 'Fleet overview', + bindings: [ + ['↑/↓', 'select an app on a server'], + ['Enter', 'open that replica'], + ], + }, + { + title: 'Replica detail', + bindings: [ + ['Esc', 'back to the fleet'], + ['↑/↓', 'select process or release'], + ['Enter/x', 'restart process / roll back (single-server apps)'], + ], + }, + { + title: 'Logs', + bindings: [ + ['s / S', 'filter by server (next / previous)'], + ['a / A', 'filter by app'], + ['p / P', 'filter by process'], + ['v', 'level: all → warn+ → error'], + ['/', 'search: text, /regex/, !not'], + ['m', 'search mode: hide others / dim others'], + ['c', 'clear filters'], + ['space', 'pause / resume'], + ['↑/↓ PgUp/PgDn', 'scroll back (pauses)'], + ['G', 'jump to newest and follow'], + ['C', 'clear the buffer'], + ['Esc / f', 'back'], + ], + }, ]; export function HelpOverlay() { @@ -28,14 +61,17 @@ export function HelpOverlay() { alignItems="center" > Keybindings - - {BINDINGS.map(([keys, description]) => ( - - {keys.padEnd(9)} - {description} - - ))} - + {SECTIONS.map((section) => ( + + {section.title} + {section.bindings.map(([keys, description]) => ( + + {keys.padEnd(15)} + {description} + + ))} + + ))} press any key to close diff --git a/src/cli/monitor/fleet-model.ts b/src/cli/monitor/fleet-model.ts new file mode 100644 index 0000000..55a33d4 --- /dev/null +++ b/src/cli/monitor/fleet-model.ts @@ -0,0 +1,147 @@ +import type { AppSnapshot, FleetView, ServerSnapshot } from '../../domain/observe/snapshot.js'; +import { releaseNameOf } from '../../domain/observe/snapshot.js'; +import type { HealthInfo, SystemInfo } from '../../domain/observe/types.js'; +import type { ObserveEvent } from '../../services/observe/events.js'; +import type { MetricsSnapshot } from './state.js'; + +/** + * One line of the fleet table: one app on one server. + * + * Derived from `FleetView`s, so a replica the poll could not reach still gets a + * row - it is exactly the row the operator needs to see. + */ +export interface FleetRow { + /** Stable across ticks, so a selection survives a refresh. */ + key: string; + app: string; + appType: 'backend' | 'frontend'; + server: string; + reachable: boolean; + error?: string; + release: string | null; + /** `behind` marks a replica not on the newest release while the fleet is split. */ + releaseState: 'current' | 'behind' | 'none' | 'unknown'; + online: number; + total: number; + cpu: number; + memoryMb: number; + restarts: number; + health?: HealthInfo; + /** Whether the app runs on more than one server, which decides if skew is meaningful. */ + replicated: boolean; + snapshot?: AppSnapshot; + system?: SystemInfo; +} + +export function buildFleetRows(fleets: readonly FleetView[]): FleetRow[] { + const rows: FleetRow[] = []; + for (const fleet of fleets) { + const replicated = fleet.replicas.length + fleet.unreachable.length > 1; + const newest = fleet.convergence.releases[0]; + + for (const replica of fleet.replicas) { + const snapshot = replica.snapshot; + const release = releaseNameOf(snapshot); + const processes = snapshot.processes; + rows.push({ + key: rowKey(fleet.app, replica.server), + app: fleet.app, + appType: fleet.appType, + server: replica.server, + reachable: replica.reachable, + error: snapshot.error, + release, + releaseState: + release === null ? 'none' : fleet.convergence.converged || release === newest ? 'current' : 'behind', + online: processes.filter((p) => p.status === 'online').length, + total: processes.length, + cpu: processes.reduce((sum, p) => sum + p.cpu, 0), + memoryMb: processes.reduce((sum, p) => sum + p.memory, 0), + restarts: processes.reduce((sum, p) => sum + p.restarts, 0), + health: snapshot.health, + replicated, + snapshot, + system: replica.system, + }); + } + + for (const server of fleet.unreachable) { + rows.push({ + key: rowKey(fleet.app, server), + app: fleet.app, + appType: fleet.appType, + server, + reachable: false, + release: null, + releaseState: 'unknown', + online: 0, + total: 0, + cpu: 0, + memoryMb: 0, + restarts: 0, + replicated, + }); + } + } + return rows; +} + +/** NUL-joined: app and server names are free-form, so a printable separator could collide. */ +export function rowKey(app: string, server: string): string { + return `${app}\u0000${server}`; +} + +export interface FleetHeadline { + tone: 'ok' | 'warn' | 'bad'; + text: string; +} + +/** One sentence per app: is the fleet agreed on a release, and can every replica be seen? */ +export function describeFleet(fleet: FleetView): FleetHeadline { + const { convergence, unreachable, replicas } = fleet; + const total = replicas.length + unreachable.length; + if (unreachable.length > 0) { + return { tone: 'bad', text: `${unreachable.length}/${total} unreachable: ${unreachable.join(', ')}` }; + } + if (convergence.releases.length > 1) { + return { tone: 'bad', text: `split across ${convergence.releases.length} releases - a roll stopped partway` }; + } + if (convergence.undeployed.length > 0) { + return { tone: 'warn', text: `no release on ${convergence.undeployed.join(', ')}` }; + } + return { tone: 'ok', text: total > 1 ? `converged on ${convergence.releases[0] ?? 'no release'}` : 'ok' }; +} + +/** The flattened shape the per-app panels still take. */ +export function toMetricsSnapshot(server: ServerSnapshot, app: AppSnapshot): MetricsSnapshot { + return { + timestamp: server.timestamp, + processes: app.processes, + system: server.system, + currentRelease: app.currentRelease, + releases: app.releases, + deployLock: server.deployLock, + health: app.health, + accessories: server.accessories, + caddy: app.caddy, + error: server.error ?? app.error, + }; +} + +/** Wording for an event; colour is the renderer's business. */ +export function describeEvent(event: ObserveEvent): { tone: 'ok' | 'warn' | 'bad' | 'info'; text: string } { + switch (event.kind) { + case 'health-failing': + return { tone: 'bad', text: `${event.app} on ${event.server}: health check failing (${event.streak} consecutive probes)` }; + case 'health-recovered': + return { tone: 'ok', text: `${event.app} on ${event.server}: health check recovered` }; + case 'server-unreachable': + return { tone: 'bad', text: `${event.server} unreachable: ${event.message}` }; + case 'server-recovered': + return { tone: 'ok', text: `${event.server} reachable again` }; + case 'app-error': + return { tone: 'bad', text: `${event.app} on ${event.server}: ${event.message}` }; + case 'notice': + return { tone: 'info', text: event.message }; + } +} diff --git a/src/cli/monitor/hooks/use-fleet.ts b/src/cli/monitor/hooks/use-fleet.ts new file mode 100644 index 0000000..9b05a55 --- /dev/null +++ b/src/cli/monitor/hooks/use-fleet.ts @@ -0,0 +1,33 @@ +import { useEffect, useRef, useState } from 'react'; +import { ObserveSession, type ObserveState, type ObserveTarget } from '../../../services/observe/session.js'; + +export interface FleetData { + state: ObserveState; + session: ObserveSession; + refresh: () => Promise; +} + +/** + * Own one `ObserveSession` for the life of the dashboard and mirror its state + * into React. The session does the scheduling, history and event detection; this + * hook only subscribes and renders what it publishes. + */ +export function useFleet(targets: ObserveTarget[], intervalSeconds: number): FleetData { + const sessionRef = useRef(null); + if (sessionRef.current === null) { + sessionRef.current = new ObserveSession({ targets, intervalSeconds }); + } + const session = sessionRef.current; + const [state, setState] = useState(() => session.getState()); + + useEffect(() => { + const unsubscribe = session.subscribe(setState); + session.start(); + return () => { + unsubscribe(); + session.stop(); + }; + }, [session]); + + return { state, session, refresh: () => session.tick() }; +} diff --git a/src/cli/monitor/hooks/use-live-logs.ts b/src/cli/monitor/hooks/use-live-logs.ts deleted file mode 100644 index 0d0d20c..0000000 --- a/src/cli/monitor/hooks/use-live-logs.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { useEffect, useRef, useState } from 'react'; -import type { RemoteExecutor } from '../../../domain/remote/executor.js'; -import type { ShipnodeApp } from '../../../shared/types.js'; -import { collectLogs, collectCaddyLogs } from '../poller.js'; -import { isWatt } from '../../../domain/runtime/watt.js'; - -const MAX_BUFFER_LINES = 500; - -export interface LiveLogsState { - logBuffer: string; - clearLogs: () => void; -} - -export function useLiveLogs( - executor: RemoteExecutor, - app: ShipnodeApp, - liveMode: boolean, - interval: number, - /** Exact process (pm2 name or systemd unit) to tail, or null for the whole app namespace. */ - filter: string | null = null, -): LiveLogsState { - const [logBuffer, setLogBuffer] = useState(''); - const lastChunkRef = useRef(''); - const inFlightRef = useRef(false); - - const clearLogs = (): void => { - lastChunkRef.current = ''; - setLogBuffer(''); - }; - - const fetchLogs = async (): Promise => { - if (app.appType === 'frontend') { - return collectCaddyLogs(executor, app.name, 50); - } - const target = filter ?? app.pm2?.apps[0]?.name; - if (target === undefined) return ''; - return collectLogs(executor, target, 20, isWatt(app) ? 'systemd' : undefined); - }; - - const pollLogs = async (): Promise => { - if (inFlightRef.current) return; - inFlightRef.current = true; - try { - const freshLogs = await fetchLogs(); - if (!freshLogs || freshLogs === '(no logs)' || freshLogs === lastChunkRef.current) return; - lastChunkRef.current = freshLogs; - setLogBuffer((prev) => { - const combined = prev ? `${prev}\n${freshLogs}` : freshLogs; - return combined.split('\n').slice(-MAX_BUFFER_LINES).join('\n'); - }); - } finally { - inFlightRef.current = false; - } - }; - - useEffect(() => { - clearLogs(); - }, [app, filter]); - - useEffect(() => { - if (!liveMode) return; - void pollLogs(); - const timer = setInterval(() => { - void pollLogs(); - }, interval * 1000); - return () => clearInterval(timer); - }, [liveMode, app, interval, filter]); - - return { logBuffer, clearLogs }; -} diff --git a/src/cli/monitor/hooks/use-log-stream.ts b/src/cli/monitor/hooks/use-log-stream.ts new file mode 100644 index 0000000..c94c070 --- /dev/null +++ b/src/cli/monitor/hooks/use-log-stream.ts @@ -0,0 +1,90 @@ +import { useEffect, useRef, useState } from 'react'; +import { planLogSources } from '../../../domain/observe/log-source.js'; +import type { LogLine } from '../../../domain/observe/log-line.js'; +import { LogStream, type LogBinding, type SourceHealth } from '../../../services/observe/log-stream.js'; +import type { FleetHost } from '../../observe.js'; + +export interface LogStreamData { + lines: readonly LogLine[]; + health: readonly SourceHealth[]; + /** Total sources being followed; 0 once planning finished with nothing to follow. */ + sources: number; + /** Servers whose sources could not be planned (host unreachable, units unresolved). */ + skipped: string[]; + clear: () => void; +} + +/** + * Follow every server's logs from the moment they are first wanted. + * + * Streaming starts lazily - a dashboard that never opens logs never opens the + * channels - and then stays on for the rest of the session, so toggling the + * view off and on does not drop the buffer or replay the backlog. + */ +export function useLogStream(hosts: readonly FleetHost[], wanted: boolean): LogStreamData { + const streamRef = useRef(null); + const startedRef = useRef(false); + const [lines, setLines] = useState([]); + const [health, setHealth] = useState([]); + const [skipped, setSkipped] = useState([]); + + useEffect(() => { + if (!wanted || startedRef.current) return; + startedRef.current = true; + let cancelled = false; + + void (async () => { + const bindings: LogBinding[] = []; + const failed: string[] = []; + await Promise.all( + hosts.map(async (host) => { + if (host.executor === null) { + failed.push(host.name); + return; + } + for (const app of host.apps) { + try { + const sources = await planLogSources(host.executor, host.name, host.config.remotePath, app); + for (const source of sources) bindings.push({ executor: host.executor, source }); + } catch { + failed.push(host.name); + } + } + }), + ); + if (cancelled) return; + + // Promise.all finished in completion order; sort so colours and cycling are stable. + bindings.sort((a, b) => a.source.server.localeCompare(b.source.server) || a.source.app.localeCompare(b.source.app)); + const stream = new LogStream({ bindings }); + streamRef.current = stream; + stream.subscribe(() => { + setLines(stream.lines()); + setHealth(stream.health()); + }); + setSkipped([...new Set(failed)]); + setHealth(stream.health()); + stream.start(); + })(); + + return () => { + cancelled = true; + }; + }, [wanted, hosts]); + + // Stop only when the dashboard itself goes away. + useEffect( + () => () => { + void streamRef.current?.stop(); + }, + [], + ); + + return { + lines, + health, + sources: health.length, + skipped, + clear: () => streamRef.current?.clear(), + }; +} diff --git a/src/cli/monitor/hooks/use-monitor-data.ts b/src/cli/monitor/hooks/use-monitor-data.ts deleted file mode 100644 index 9c68137..0000000 --- a/src/cli/monitor/hooks/use-monitor-data.ts +++ /dev/null @@ -1,126 +0,0 @@ -import { useEffect, useRef, useState } from 'react'; -import { useStdout } from 'ink'; -import chalk from 'chalk'; -import type { RemoteExecutor } from '../../../domain/remote/executor.js'; -import type { ShipnodeApp, ShipnodeConfig } from '../../../shared/types.js'; -import { collectMetrics } from '../poller.js'; -import { MetricsHistory, nextHealthFailStreak, type MetricsSnapshot } from '../state.js'; - -/** Sample docker accessory state roughly every this many seconds, not every poll. */ -const ACCESSORY_SAMPLE_SECONDS = 10; - -/** Consecutive failed health probes before the monitor raises an alert. */ -export const HEALTH_ALERT_THRESHOLD = 3; - -export interface MonitorDataState { - snapshot: MetricsSnapshot | null; - history: MetricsHistory; - events: string[]; - error: string | null; - lastUpdate: string; - polling: boolean; - /** Consecutive failed health probes; >= HEALTH_ALERT_THRESHOLD means alerting. */ - healthFailStreak: number; - refresh: () => Promise; - appendEvent: (message: string) => void; - reset: () => void; -} - -export function useMonitorData( - executor: RemoteExecutor, - config: ShipnodeConfig, - app: ShipnodeApp, - interval: number, - accessoryNames: string[] = [], -): MonitorDataState { - const [snapshot, setSnapshot] = useState(null); - const [events, setEvents] = useState([]); - const [error, setError] = useState(null); - const [lastUpdate, setLastUpdate] = useState(''); - const [polling, setPolling] = useState(false); - const [healthFailStreak, setHealthFailStreak] = useState(0); - const { stdout } = useStdout(); - const historyRef = useRef(new MetricsHistory()); - const inFlightRef = useRef(false); - const pollCountRef = useRef(0); - const healthStreakRef = useRef(0); - - const appendEvent = (message: string): void => { - setEvents((prev) => [...prev.slice(-100), message]); - }; - - const refresh = async (): Promise => { - if (inFlightRef.current) return; - inFlightRef.current = true; - setPolling(true); - try { - const cadence = Math.max(1, Math.ceil(ACCESSORY_SAMPLE_SECONDS / interval)); - const sampleAccessories = pollCountRef.current % cadence === 0; - pollCountRef.current += 1; - - const nextSnapshot = await collectMetrics(executor, app, config, { - intervalSeconds: interval, - accessoryNames: sampleAccessories ? accessoryNames : undefined, - }); - setSnapshot((prev) => - nextSnapshot.accessories === undefined && prev !== null - ? { ...nextSnapshot, accessories: prev.accessories } - : nextSnapshot, - ); - historyRef.current.push(nextSnapshot); - - const streak = nextHealthFailStreak(healthStreakRef.current, nextSnapshot.health); - if (streak === HEALTH_ALERT_THRESHOLD && healthStreakRef.current < HEALTH_ALERT_THRESHOLD) { - appendEvent(chalk.bold.red(`Health check failing (${streak} consecutive probes)`)); - stdout?.write('\x07'); - } - if (streak === 0 && healthStreakRef.current >= HEALTH_ALERT_THRESHOLD) { - appendEvent(chalk.green('Health check recovered')); - } - healthStreakRef.current = streak; - setHealthFailStreak(streak); - - setLastUpdate(new Date().toLocaleTimeString()); - setError(nextSnapshot.error ?? null); - if (nextSnapshot.error) appendEvent(chalk.red(nextSnapshot.error)); - } catch (cause: unknown) { - const message = cause instanceof Error ? cause.message : String(cause); - setError(message); - appendEvent(chalk.red(`Poll failed: ${message}`)); - } finally { - inFlightRef.current = false; - setPolling(false); - } - }; - - const reset = (): void => { - historyRef.current.clear(); - pollCountRef.current = 0; - healthStreakRef.current = 0; - setHealthFailStreak(0); - setSnapshot(null); - setError(null); - setLastUpdate(''); - }; - - useEffect(() => { - void refresh(); - const timer = setInterval(() => { - void refresh(); - }, interval * 1000); - return () => clearInterval(timer); - }, [app, interval]); - - return { - snapshot, - history: historyRef.current, - events, - error, - lastUpdate, - polling, - healthFailStreak, - refresh, - appendEvent, - reset, - }; -} diff --git a/src/cli/monitor/index.tsx b/src/cli/monitor/index.tsx index 3d95be9..9aff5e1 100644 --- a/src/cli/monitor/index.tsx +++ b/src/cli/monitor/index.tsx @@ -1,17 +1,12 @@ import { render } from 'ink'; -import type { RemoteExecutor } from '../../domain/remote/executor.js'; -import type { ShipnodeConfig, ShipnodeApp } from '../../shared/types.js'; +import type { FleetConnection } from '../observe.js'; import { App } from './App.js'; interface MonitorOptions { - executor: RemoteExecutor; - config: ShipnodeConfig; - app: ShipnodeApp; - apps: ShipnodeApp[]; - accessoryNames: string[]; - targetName: string; - host: string; + fleet: FleetConnection; interval: number; + /** Start with this app selected in the overview. */ + focusApp?: string; } export async function runMonitor(options: MonitorOptions): Promise { diff --git a/src/cli/monitor/layout/HeaderBar.tsx b/src/cli/monitor/layout/HeaderBar.tsx index 7c95187..d9f12f8 100644 --- a/src/cli/monitor/layout/HeaderBar.tsx +++ b/src/cli/monitor/layout/HeaderBar.tsx @@ -1,40 +1,33 @@ import { Box, Text } from 'ink'; -import type { ShipnodeApp } from '../../../shared/types.js'; -import type { DeployLockInfo } from '../state.js'; const ACCENT = '#d6a85d'; const BG_HEADER = '#161b22'; +export interface HeaderAlert { + text: string; +} + interface HeaderBarProps { - app: ShipnodeApp; - targetName: string; - host: string; + /** What is being watched: "fleet · 3 servers", or "api on b". */ + scope: string; interval: number; liveMode: boolean; - deployLock?: DeployLockInfo | null; - /** Consecutive failed health probes; alerting once it reaches the threshold. */ - healthFailStreak?: number; + alerts: readonly HeaderAlert[]; } -export function HeaderBar({ app, targetName, host, interval, liveMode, deployLock, healthFailStreak = 0 }: HeaderBarProps) { +export function HeaderBar({ scope, interval, liveMode, alerts }: HeaderBarProps) { return ( - ShipNode Monitor - — {app.name} ({app.appType}) - — {targetName} {host} - {' │'} interval: {interval}s - {liveMode ? ( - {' │'} logs: ON - ) : ( - │ logs: off - )} - {deployLock != null && ( - {' │ '}DEPLOY LOCK ({deployLock.ageSeconds}s) - )} - {healthFailStreak > 0 && ( - {' │ '}HEALTH FAILING ×{healthFailStreak} - )} - {' │'} [?] help + + ShipNode Monitor + — {scope} + {' │'} interval: {interval}s + {liveMode ? {' │'} logs: ON : │ logs: off} + {alerts.map((alert) => ( + {' │ '}{alert.text} + ))} + {' │'} [?] help + ); } diff --git a/src/cli/monitor/layout/MonitorFrame.tsx b/src/cli/monitor/layout/MonitorFrame.tsx index cf9fa17..4562870 100644 --- a/src/cli/monitor/layout/MonitorFrame.tsx +++ b/src/cli/monitor/layout/MonitorFrame.tsx @@ -1,59 +1,30 @@ import { Box, Text } from 'ink'; import type { ReactNode } from 'react'; -import type { ShipnodeApp } from '../../../shared/types.js'; -import type { MetricsSnapshot } from '../state.js'; -import { HeaderBar } from './HeaderBar.js'; +import { HeaderBar, type HeaderAlert } from './HeaderBar.js'; import { StatusBar } from './StatusBar.js'; const BG = '#0d1117'; const BORDER = '#30363d'; interface MonitorFrameProps { - app: ShipnodeApp; - targetName: string; - host: string; + scope: string; interval: number; liveMode: boolean; - lastUpdate: string; - snapshot: MetricsSnapshot | null; + alerts: readonly HeaderAlert[]; + lastUpdate: string | null; polling: boolean; - error: string | null; - /** Streak to surface in the header; pass 0 until it crosses the alert threshold. */ - healthFailStreak?: number; + summary: string; + hints?: string; + error?: string | null; children: ReactNode; } -export function MonitorFrame({ - app, - targetName, - host, - interval, - liveMode, - lastUpdate, - snapshot, - polling, - error, - healthFailStreak = 0, - children, -}: MonitorFrameProps) { +export function MonitorFrame({ scope, interval, liveMode, alerts, lastUpdate, polling, summary, hints, error, children }: MonitorFrameProps) { return ( - + {children} - + ); } diff --git a/src/cli/monitor/layout/StatusBar.tsx b/src/cli/monitor/layout/StatusBar.tsx index 30d9ee0..bc59265 100644 --- a/src/cli/monitor/layout/StatusBar.tsx +++ b/src/cli/monitor/layout/StatusBar.tsx @@ -1,30 +1,27 @@ import { Box, Text } from 'ink'; -import type { MetricsSnapshot } from '../state.js'; const BG_HEADER = '#161b22'; interface StatusBarProps { - lastUpdate: string; - snapshot: MetricsSnapshot | null; + lastUpdate: string | null; polling: boolean; - error: string | null; + /** "3 servers · 5 apps", or whatever summarises what is on screen. */ + summary: string; + /** Hotkeys for the current view. */ + hints?: string; + error?: string | null; } -export function StatusBar({ lastUpdate, snapshot, polling, error }: StatusBarProps) { - const health = snapshot?.health; +export function StatusBar({ lastUpdate, polling, summary, hints, error }: StatusBarProps) { + const updated = lastUpdate === null ? '-' : new Date(lastUpdate).toLocaleTimeString(); return ( - - {' '}Status: {polling ? 'polling' : snapshot ? 'ready' : 'waiting'}{' │ '} - Last update: {lastUpdate || '-'}{' │ '} - {snapshot ? `${snapshot.processes.length} process(es)` : '-'} + + {' '}Status: {polling ? 'polling' : lastUpdate === null ? 'waiting' : 'ready'}{' │ '} + Last update: {updated}{' │ '}{summary} + {hints !== undefined && ` │ ${hints}`} - {health !== undefined && ( - - {' │ '}hc:{health.httpCode || 'down'} {health.responseMs}ms - - )} - {error && {' │ '}{error}} + {error != null && {' │ '}{error}} ); } diff --git a/src/cli/monitor/log-color.ts b/src/cli/monitor/log-color.ts new file mode 100644 index 0000000..30643ec --- /dev/null +++ b/src/cli/monitor/log-color.ts @@ -0,0 +1,15 @@ +import { detectLevel, type LogLevel } from '../../domain/observe/log-line.js'; + +/** + * Severity color for a raw log line; the classification itself lives in the + * domain so the viewer, the filters and the CLI agree on what an error is. + */ +export function logLineColor(line: string): 'red' | 'yellow' | undefined { + return levelColor(detectLevel(line)); +} + +export function levelColor(level: LogLevel | undefined): 'red' | 'yellow' | undefined { + if (level === 'error') return 'red'; + if (level === 'warn') return 'yellow'; + return undefined; +} diff --git a/src/cli/monitor/log-view-state.ts b/src/cli/monitor/log-view-state.ts new file mode 100644 index 0000000..ab8a2f9 --- /dev/null +++ b/src/cli/monitor/log-view-state.ts @@ -0,0 +1,103 @@ +import { + cycleMinLevel, + cycleOption, + EMPTY_LOG_FILTER, + type LogFacets, + type LogFilter, +} from '../../domain/observe/log-filter.js'; +import type { LogLine } from '../../domain/observe/log-line.js'; + +/** + * Everything the log viewer remembers between keystrokes. + * + * A reducer rather than a bag of `useState`s: the interactions have rules that + * span fields (choosing a server resets an app that does not run there; + * scrolling up freezes the view) and those rules deserve tests that do not need + * a terminal. + */ +export interface LogViewState { + filter: LogFilter; + /** True while keystrokes are building the search query. */ + typing: boolean; + /** The buffer as it was when the view was frozen; null means follow the live tail. */ + frozen: readonly LogLine[] | null; + /** Lines scrolled back from the newest. Only meaningful while frozen. */ + offset: number; +} + +export const INITIAL_LOG_VIEW: LogViewState = { + filter: EMPTY_LOG_FILTER, + typing: false, + frozen: null, + offset: 0, +}; + +export type LogViewAction = + | { type: 'cycle'; dimension: 'server' | 'app' | 'process'; direction: 1 | -1; facets: LogFacets } + | { type: 'cycle-level' } + | { type: 'toggle-mode' } + | { type: 'start-search' } + | { type: 'type'; text: string } + | { type: 'backspace' } + | { type: 'commit-search' } + | { type: 'cancel-search' } + | { type: 'clear-filters' } + | { type: 'toggle-pause'; live: readonly LogLine[] } + | { type: 'resume' } + | { type: 'scroll'; delta: number; live: readonly LogLine[]; max: number } + /** Enter the view scoped to what the user was looking at, or unscoped. */ + | { type: 'enter'; server: string | null; app: string | null }; + +export function logViewReducer(state: LogViewState, action: LogViewAction): LogViewState { + switch (action.type) { + case 'cycle': { + const { dimension, direction, facets } = action; + if (dimension === 'server') { + const server = cycleOption(facets.servers, state.filter.server, direction); + // The new server may not run the chosen app, and the process belongs to the app. + return { ...state, filter: { ...state.filter, server, app: null, process: null } }; + } + if (dimension === 'app') { + const app = cycleOption(facets.apps, state.filter.app, direction); + return { ...state, filter: { ...state.filter, app, process: null } }; + } + const process = cycleOption(facets.processes, state.filter.process, direction); + return { ...state, filter: { ...state.filter, process } }; + } + case 'cycle-level': + return { ...state, filter: { ...state.filter, minLevel: cycleMinLevel(state.filter.minLevel) } }; + case 'toggle-mode': + return { ...state, filter: { ...state.filter, mode: state.filter.mode === 'hide' ? 'dim' : 'hide' } }; + case 'start-search': + return { ...state, typing: true, filter: { ...state.filter, query: '' } }; + case 'type': + return { ...state, filter: { ...state.filter, query: state.filter.query + action.text } }; + case 'backspace': + return { ...state, filter: { ...state.filter, query: state.filter.query.slice(0, -1) } }; + case 'commit-search': + return { ...state, typing: false }; + case 'cancel-search': + return { ...state, typing: false, filter: { ...state.filter, query: '' } }; + case 'clear-filters': + return { ...state, filter: { ...EMPTY_LOG_FILTER, mode: state.filter.mode } }; + case 'toggle-pause': + return state.frozen === null + ? { ...state, frozen: action.live, offset: 0 } + : { ...state, frozen: null, offset: 0 }; + case 'resume': + return { ...state, frozen: null, offset: 0 }; + case 'scroll': { + // Scrolling back through history while lines keep arriving would move the + // page under the reader, so the first scroll up freezes the view. + const frozen = state.frozen ?? (action.delta > 0 ? action.live : null); + if (frozen === null) return state; + const offset = Math.min(Math.max(0, state.offset + action.delta), Math.max(0, action.max)); + return { ...state, frozen, offset }; + } + case 'enter': + return { + ...INITIAL_LOG_VIEW, + filter: { ...EMPTY_LOG_FILTER, server: action.server, app: action.app }, + }; + } +} diff --git a/src/cli/monitor/monitor-session.ts b/src/cli/monitor/monitor-session.ts index fdd22c6..cd1b4da 100644 --- a/src/cli/monitor/monitor-session.ts +++ b/src/cli/monitor/monitor-session.ts @@ -1,33 +1,7 @@ import { Result, type Result as ResultType } from 'better-result'; -import { getServerTargetResult, resolveServerNamesResult, type ServerTarget } from '../../domain/servers.js'; +import { resolveServerNamesResult } from '../../domain/servers.js'; import type { ShipnodeApp, ShipnodeConfig } from '../../shared/types.js'; -import { UnknownAppError, type AppTargetError, type ServerTargetError } from '../../shared/result-errors.js'; - -export interface MonitorSession { - config: ShipnodeConfig; - app: ShipnodeApp; - target: ServerTarget; -} - -export function resolveMonitorSession( - config: ShipnodeConfig, - appName?: string, - serverName?: string, -): ResultType { - const app = appName === undefined - ? config.apps[0] - : config.apps.find((candidate) => candidate.name === appName); - - if (app === undefined) return Result.err(new UnknownAppError({ name: appName ?? '(default)' })); - - // The live TUI holds one connection. `--once` / `--json` observe the whole - // fleet; this path still needs a replica. `--on` picks it; otherwise the - // app's `on` must already name a single server. - const target = getServerTargetResult(config, serverName ?? app.on, `App '${app.name}'`); - if (target.isErr()) return Result.err(target.error); - - return Result.ok({ config, app, target: target.value }); -} +import type { ServerTargetError } from '../../shared/result-errors.js'; export function getAppsForMonitorTarget( config: ShipnodeConfig, diff --git a/src/cli/monitor/panels/EventsPanel.tsx b/src/cli/monitor/panels/EventsPanel.tsx index 9e096c8..77375a5 100644 --- a/src/cli/monitor/panels/EventsPanel.tsx +++ b/src/cli/monitor/panels/EventsPanel.tsx @@ -1,28 +1,33 @@ import { Box, Text } from 'ink'; +import type { ObserveEvent } from '../../../services/observe/events.js'; +import { describeEvent } from '../fleet-model.js'; const ACCENT = '#d6a85d'; const BG = '#0d1117'; const BORDER = '#30363d'; +const TONE = { ok: 'green', warn: 'yellow', bad: 'red', info: 'yellow' } as const; interface EventsPanelProps { - events: string[]; + events: readonly ObserveEvent[]; + rows?: number; } -export function EventsPanel({ events }: EventsPanelProps) { +export function EventsPanel({ events, rows = 6 }: EventsPanelProps) { return ( - + Events {events.length === 0 ? ( - No events yet. [L] live logs · [F] fullscreen · [?] help + No events yet. [L] live log strip · [F] fullscreen logs · [?] help ) : ( - events.slice(-6).map((event, index) => {event}) + events.slice(-rows).map((event, index) => { + const { tone, text } = describeEvent(event); + return ( + + {new Date(event.at).toLocaleTimeString()} + {text} + + ); + }) )} ); diff --git a/src/cli/monitor/panels/FleetPanel.tsx b/src/cli/monitor/panels/FleetPanel.tsx new file mode 100644 index 0000000..03b706d --- /dev/null +++ b/src/cli/monitor/panels/FleetPanel.tsx @@ -0,0 +1,138 @@ +import { Box, Text } from 'ink'; +import type { FleetView, ServerSnapshot } from '../../../domain/observe/snapshot.js'; +import { systemCpuPercent } from '../../../domain/observe/types.js'; +import { describeFleet, type FleetRow } from '../fleet-model.js'; +import { formatBytes } from '../charts.js'; +import { Gauge } from '../components/charts.js'; + +const BG = '#0d1117'; +const BG_SELECTED = '#1f2937'; +const ACCENT = '#d6a85d'; +const TONE = { ok: 'green', warn: 'yellow', bad: 'red' } as const; + +interface FleetPanelProps { + fleets: readonly FleetView[]; + rows: readonly FleetRow[]; + servers: readonly ServerSnapshot[]; + selectedKey: string | null; + /** Lines available; the list scrolls to keep the selection in view. */ + height: number; +} + +type Item = + | { kind: 'app'; fleet: FleetView } + | { kind: 'row'; row: FleetRow } + | { kind: 'servers-head' } + | { kind: 'server'; server: ServerSnapshot }; + +function itemsOf(fleets: readonly FleetView[], rows: readonly FleetRow[], servers: readonly ServerSnapshot[]): Item[] { + const items: Item[] = []; + for (const fleet of fleets) { + items.push({ kind: 'app', fleet }); + for (const row of rows) if (row.app === fleet.app) items.push({ kind: 'row', row }); + } + items.push({ kind: 'servers-head' }); + for (const server of servers) items.push({ kind: 'server', server }); + return items; +} + +function shortRelease(release: string | null): string { + return release === null ? 'none' : release.length > 19 ? release.slice(0, 19) : release; +} + +function ReplicaLine({ row, selected }: { row: FleetRow; selected: boolean }) { + const releaseColor = + row.releaseState === 'behind' ? 'red' : row.releaseState === 'none' ? 'yellow' : row.releaseState === 'unknown' ? 'gray' : undefined; + const allOnline = row.total > 0 && row.online === row.total; + return ( + + + {selected ? '❯' : ' '} + {row.reachable ? '●' : '○'} + {row.server.padEnd(12).slice(0, 12)} + {!row.reachable ? ( + unreachable + ) : row.error !== undefined ? ( + {row.error} + ) : ( + <> + {shortRelease(row.release).padEnd(19)} + {row.releaseState === 'behind' && behind} + {row.appType === 'backend' && ( + {row.online}/{row.total} up + )} + {row.appType === 'backend' && ( + cpu {row.cpu.toFixed(0)}% mem {formatBytes(row.memoryMb)} + )} + {row.restarts > 0 && ↻{row.restarts}} + {row.health !== undefined && ( + + {' '}hc {row.health.httpCode || 'down'} {row.health.responseMs}ms + + )} + + )} + + + ); +} + +function ServerLine({ server }: { server: ServerSnapshot }) { + if (server.error !== undefined) { + return ( + + ○ {server.server.padEnd(12).slice(0, 12)} {server.error} + + ); + } + const mem = server.system.totalMem > 0 ? server.system.usedMem / server.system.totalMem : 0; + const disk = server.system.totalDisk > 0 ? server.system.usedDisk / server.system.totalDisk : 0; + return ( + + + ● + {server.server.padEnd(12).slice(0, 12)} + cpu + mem + disk {(disk * 100).toFixed(0)}% + {server.deployLock != null && DEPLOY LOCK {server.deployLock.ageSeconds}s} + + + ); +} + +export function FleetPanel({ fleets, rows, servers, selectedKey, height }: FleetPanelProps) { + const items = itemsOf(fleets, rows, servers); + const selectedAt = items.findIndex((item) => item.kind === 'row' && item.row.key === selectedKey); + + // Keep the selection in view; with nothing selected, show the top. + const room = Math.max(3, height); + const start = selectedAt < 0 ? 0 : Math.min(Math.max(0, selectedAt - Math.floor(room / 2)), Math.max(0, items.length - room)); + const visible = items.slice(start, start + room); + + return ( + + Fleet + {fleets.length === 0 && Waiting for data...} + {visible.map((item, index) => { + if (item.kind === 'app') { + const headline = describeFleet(item.fleet); + return ( + + {item.fleet.app} + ({item.fleet.appType}) + {headline.tone === 'ok' ? '✓' : '⚠'} {headline.text} + + ); + } + if (item.kind === 'row') { + return ; + } + if (item.kind === 'servers-head') { + return Servers; + } + return ; + })} + + ); +} diff --git a/src/cli/monitor/panels/LogPanel.tsx b/src/cli/monitor/panels/LogPanel.tsx deleted file mode 100644 index 1dde91f..0000000 --- a/src/cli/monitor/panels/LogPanel.tsx +++ /dev/null @@ -1,108 +0,0 @@ -import { Box, Text } from 'ink'; - -const BG = '#0d1117'; - -const RED_TAG = /\[(fatal|error)\]/i; -const YELLOW_TAG = /\[warn(ing)?\]/i; -const RED_PREFIX = /(^|\s)(fatal|error)\s*[:\]]/i; -const CAMEL_ERROR = /[a-z](Error|Exception)\b/; -const RED_WORD = /\b(fatal|exception|uncaught|unhandled|panic|crit(ical)?)\b/i; -const YELLOW_WORD = /\bwarn(ing)?\b/i; -const PLAIN_ERROR = /\berror\b/i; - -/** Numeric pino levels: trace10 debug20 info30 warn40 error50 fatal60. */ -function jsonLogSeverity(line: string): 'red' | 'yellow' | undefined { - const trimmed = line.trim(); - if (!trimmed.startsWith('{')) return undefined; - let parsed: unknown; - try { - parsed = JSON.parse(trimmed); - } catch { - return undefined; - } - if (typeof parsed !== 'object' || parsed === null) return undefined; - const level = (parsed as Record).level ?? (parsed as Record).severity; - - if (typeof level === 'number') { - if (level >= 50) return 'red'; - if (level >= 40) return 'yellow'; - return undefined; - } - if (typeof level === 'string') { - const lvl = level.toLowerCase(); - if (['error', 'fatal', 'panic', 'crit', 'critical'].includes(lvl)) return 'red'; - if (['warn', 'warning'].includes(lvl)) return 'yellow'; - } - return undefined; -} - -/** - * Severity color for a raw log line. Structured JSON logs (pino/winston-style - * `level`/`severity` fields) win first; plain text falls back to bracket - * tags, "ERROR:"-style prefixes, camelCase Error/Exception class names, and - * finally whole-word matches — never a bare substring, so "mirror" or - * "Terror" don't false-positive. - */ -export function logLineColor(line: string): 'red' | 'yellow' | undefined { - const structured = jsonLogSeverity(line); - if (structured !== undefined) return structured; - - if (RED_TAG.test(line) || RED_PREFIX.test(line) || CAMEL_ERROR.test(line) || RED_WORD.test(line) || PLAIN_ERROR.test(line)) { - return 'red'; - } - if (YELLOW_TAG.test(line) || YELLOW_WORD.test(line)) return 'yellow'; - return undefined; -} - -interface LogPanelProps { - logBuffer: string; - maxLines?: number; - title?: string; - /** Case-insensitive substring: matching lines pop, the rest fade. */ - search?: string; - /** True while the user is still typing the search query. */ - searchTyping?: boolean; - paused?: boolean; -} - -export function LogPanel({ - logBuffer, - maxLines = 30, - title = 'Live Logs', - search = '', - searchTyping = false, - paused = false, -}: LogPanelProps) { - const lines = logBuffer ? logBuffer.split('\n').slice(-maxLines) : []; - const query = search.toLowerCase(); - return ( - - - {title} - {paused && ⏸ paused (space resumes)} - {(search !== '' || searchTyping) && ( - /{search}{searchTyping ? '▌' : ''} - )} - - {lines.length === 0 ? ( - Waiting for logs... - ) : ( - lines.map((line, i) => { - if (query !== '') { - return line.toLowerCase().includes(query) ? ( - {line} - ) : ( - {line} - ); - } - const severity = logLineColor(line); - return severity !== undefined ? ( - {line} - ) : ( - {line} - ); - }) - )} - - ); -} diff --git a/src/cli/monitor/panels/LogViewer.tsx b/src/cli/monitor/panels/LogViewer.tsx new file mode 100644 index 0000000..0e0fb18 --- /dev/null +++ b/src/cli/monitor/panels/LogViewer.tsx @@ -0,0 +1,140 @@ +import { Box, Text } from 'ink'; +import type { LevelCounts, LogFilter } from '../../../domain/observe/log-filter.js'; +import type { LogLine } from '../../../domain/observe/log-line.js'; +import type { SourceHealth } from '../../../services/observe/log-stream.js'; +import { levelColor } from '../log-color.js'; + +const BG = '#0d1117'; +const BORDER = '#30363d'; +const ACCENT = '#d6a85d'; +const MAX_LABEL = 28; + +/** Servers keep a colour for the session, kept clear of red/yellow which mean severity. */ +const SERVER_COLORS = ['cyan', 'magenta', 'blue', 'greenBright', 'whiteBright'] as const; + +export function serverColor(server: string, known: readonly string[]): (typeof SERVER_COLORS)[number] { + const index = Math.max(0, known.indexOf(server)); + return SERVER_COLORS[index % SERVER_COLORS.length]; +} + +/** `server app:process`, showing only the parts that actually vary in what is on screen. */ +export function sourceLabel(line: LogLine, showServer: boolean, showApp: boolean): string { + const head = [showServer ? line.server : null, showApp ? line.app : null].filter((part) => part !== null).join(' '); + const who = line.process === null || line.process === line.app ? null : line.process; + const label = who === null ? head : head === '' ? who : `${head}:${who}`; + return label.length > MAX_LABEL ? `${label.slice(0, MAX_LABEL - 1)}…` : label; +} + +interface LogViewerProps { + lines: readonly LogLine[]; + matched: ReadonlySet; + filter: LogFilter; + /** Every server seen, in a stable order, for colour assignment. */ + servers: readonly string[]; + showServer: boolean; + showApp: boolean; + /** Rows available for log lines. */ + height: number; + /** Lines scrolled back from the newest (0 follows the tail). */ + offset: number; + frozen: boolean; + typing: boolean; + queryError?: string; + counts?: LevelCounts; + /** Sources not currently live, for the status line. */ + troubled: readonly SourceHealth[]; + totalSources: number; + title: string; + /** Strip mode: lines only, no filter bar. */ + compact?: boolean; +} + +function FilterBar({ filter, typing, queryError, counts, frozen }: Pick) { + const chip = (label: string, value: string | null, active = value !== null) => ( + + {label} + {value ?? 'all'} + {' '} + + ); + const level = filter.minLevel === null ? null : filter.minLevel === 'error' ? 'error' : `${filter.minLevel}+`; + return ( + + + {chip('server', filter.server)} + {chip('app', filter.app)} + {chip('proc', filter.process)} + {chip('level', level)} + mode + {filter.mode} + {frozen && {' '}⏸ paused} + + + search + {filter.query === '' && !typing ? ( + (/ to search; /regex/ and !not supported) + ) : ( + + {filter.query}{typing ? '▌' : ''} + + )} + {queryError !== undefined && {' '}{queryError}} + {counts !== undefined && ( + + {' '} + 0} dimColor={counts.error === 0}>ERR {counts.error} + {' '} + 0} dimColor={counts.warn === 0}>WARN {counts.warn} + + )} + + + ); +} + +export function LogViewer(props: LogViewerProps) { + const { lines, matched, filter, servers, showServer, showApp, height, offset, compact = false } = props; + const end = lines.length - offset; + const visible = lines.slice(Math.max(0, end - height), Math.max(0, end)); + const labels = visible.map((line) => sourceLabel(line, showServer, showApp)); + const labelWidth = Math.max(0, ...labels.map((label) => label.length)); + const dimMode = filter.mode === 'dim' && filter.query !== ''; + + return ( + + + {props.title} + {props.frozen && compact && {' '}⏸} + {offset > 0 && {' '}↑ {offset} newer below (G to follow)} + + {!compact && } + {props.troubled.length > 0 && ( + + ⚠ {props.troubled.length}/{props.totalSources} source(s) not live:{' '} + {props.troubled.slice(0, 2).map((t) => `${t.server}/${t.app} ${t.state}${t.error ? ` (${t.error})` : ''}`).join('; ')} + + )} + {visible.length === 0 ? ( + {props.totalSources === 0 ? 'Nothing to follow.' : 'Waiting for logs...'} + ) : ( + visible.map((line, i) => { + const hit = matched.has(line.id); + const faded = dimMode && !hit; + const color = faded ? 'gray' : dimMode && hit ? 'cyan' : levelColor(line.level); + return ( + + {labelWidth > 0 && ( + + {labels[i].padEnd(labelWidth)}{' │ '} + + )} + + {line.text} + + + ); + }) + )} + + ); +} diff --git a/src/cli/observe.ts b/src/cli/observe.ts index abbf435..5ba2342 100644 --- a/src/cli/observe.ts +++ b/src/cli/observe.ts @@ -5,6 +5,7 @@ import { parseSystemStats } from '../domain/observe/parse.js'; import { releaseNameOf, type FleetView, type ServerSnapshot } from '../domain/observe/snapshot.js'; import { configForAppResult, configForServer, getServerTargets, type ServerTarget } from '../domain/servers.js'; import { SshConnection } from '../infrastructure/ssh/connection.js'; +import type { RemoteExecutor } from '../domain/remote/executor.js'; import type { AppTargetError, ServerTargetError } from '../shared/result-errors.js'; import { UnknownServerTargetError } from '../shared/result-errors.js'; import type { ShipnodeApp, ShipnodeConfig, SshConfig } from '../shared/types.js'; @@ -38,23 +39,54 @@ export async function takeSnapshot( const hosts = planObserveHosts(config, filter); if (hosts.isErr()) return Result.err(hosts.error); - const connections: SshConnection[] = []; - const targets: ObserveTarget[] = []; + const fleet = await connectFleet(config, hosts.value); try { - for (const host of hosts.value) { - targets.push(await connectHost(config, host, connections)); - } const session = new ObserveSession({ - targets, + targets: fleet.targets, intervalSeconds: filter.intervalSeconds ?? 2, }); await session.tick(); return Result.ok(session.getState()); } finally { - for (const ssh of connections) ssh.disconnect(); + fleet.close(); } } +/** One planned host, connected (or not) and ready for observing, acting on, and streaming from. */ +export interface FleetHost extends ObserveHostPlan { + /** `user@host:port`, for display. */ + address: string; + /** This server's slice of the workspace, for actions that read config. */ + config: ShipnodeConfig; + /** Null when the connection failed; `error` says why. */ + executor: RemoteExecutor | null; + error?: string; +} + +export interface FleetConnection { + hosts: FleetHost[]; + targets: ObserveTarget[]; + close(): void; +} + +/** + * Connect every host once, in parallel, and keep the connections. + * + * A host that cannot be reached is not an error here: it becomes an observer + * that reports the failure every tick, so the fleet view can show it as down + * instead of the whole command refusing to start. + */ +export async function connectFleet(config: ShipnodeConfig, plans: ObserveHostPlan[]): Promise { + const connected = await Promise.all(plans.map((plan) => connectHost(config, plan))); + return { + hosts: connected.map((entry) => entry.host), + targets: connected.map((entry) => entry.target), + close() { + for (const entry of connected) entry.ssh?.disconnect(); + }, + }; +} + export function planObserveHosts( config: ShipnodeConfig, filter: ObserveFilter = {}, @@ -99,25 +131,37 @@ function selectServers( async function connectHost( config: ShipnodeConfig, - host: ObserveHostPlan, - connections: SshConnection[], -): Promise { + plan: ObserveHostPlan, +): Promise<{ host: FleetHost; target: ObserveTarget; ssh: SshConnection | null }> { + const serverConfig = configForServer(config, plan.name); + const base = { + ...plan, + address: `${plan.ssh.user}@${plan.ssh.host}:${plan.ssh.port}`, + config: serverConfig, + }; const ssh = new SshConnection(); try { - await ssh.connect(host.ssh); - connections.push(ssh); + await ssh.connect(plan.ssh); return { - observer: new MetricsCollector(ssh, host.name, configForServer(config, host.name)), - apps: host.apps, - accessoryNames: host.accessoryNames, + host: { ...base, executor: ssh }, + target: { + observer: new MetricsCollector(ssh, plan.name, serverConfig), + apps: plan.apps, + accessoryNames: plan.accessoryNames, + }, + ssh, }; } catch (cause: unknown) { ssh.disconnect(); - const message = cause instanceof Error ? cause.message : String(cause); + const message = `Failed to connect: ${cause instanceof Error ? cause.message : String(cause)}`; return { - observer: unreachableObserver(host.name, `Failed to connect: ${message}`, host.apps), - apps: host.apps, - accessoryNames: host.accessoryNames, + host: { ...base, executor: null, error: message }, + target: { + observer: unreachableObserver(plan.name, message, plan.apps), + apps: plan.apps, + accessoryNames: plan.accessoryNames, + }, + ssh: null, }; } } diff --git a/src/domain/observe/log-filter.ts b/src/domain/observe/log-filter.ts new file mode 100644 index 0000000..7d315ae --- /dev/null +++ b/src/domain/observe/log-filter.ts @@ -0,0 +1,194 @@ +import { LOG_LEVEL_RANK, type LogLevel, type LogLine } from './log-line.js'; + +/** + * What the viewer wants to see. Applied locally to the buffer, so changing a + * filter never reconnects anything and never loses a line that was merely + * hidden. + * + * `null` means "all" for the single-select dimensions. + */ +export interface LogFilter { + server: string | null; + app: string | null; + process: string | null; + /** Show this level and above; null shows everything. */ + minLevel: LogLevel | null; + /** Raw query as typed: plain text, or `/regex/` with optional `i` flag. */ + query: string; + /** Matching lines are kept and the rest dropped (`hide`), or the rest only faded (`dim`). */ + mode: 'hide' | 'dim'; +} + +export const EMPTY_LOG_FILTER: LogFilter = { + server: null, + app: null, + process: null, + minLevel: null, + query: '', + mode: 'hide', +}; + +export type LogMatcher = (text: string) => boolean; + +/** + * Compile a query. `/pattern/` and `/pattern/i` are regular expressions; + * anything else is a case-insensitive substring, because that is what a person + * typing a word into a log viewer means. A leading `!` negates either form. + * An unfinished or invalid regex is reported, not thrown, since it is + * re-evaluated on every keystroke. + */ +export function compileQuery(raw: string): { matcher: LogMatcher | null; error?: string } { + const negated = raw.startsWith('!') && raw.length > 1; + const body = negated ? raw.slice(1) : raw; + if (body === '') return { matcher: null }; + + const asRegex = /^\/(.+)\/([i]?)$/.exec(body); + let test: LogMatcher; + if (asRegex !== null) { + try { + const pattern = new RegExp(asRegex[1], asRegex[2] === 'i' ? 'i' : ''); + test = (text) => pattern.test(text); + } catch (cause: unknown) { + return { matcher: null, error: cause instanceof Error ? cause.message : 'invalid regex' }; + } + } else { + const needle = body.toLowerCase(); + test = (text) => text.toLowerCase().includes(needle); + } + return { matcher: negated ? (text) => !test(text) : test }; +} + +/** A line's verdict against every filter dimension except the text query. */ +function matchesScope(line: LogLine, filter: LogFilter): boolean { + if (filter.server !== null && line.server !== filter.server) return false; + if (filter.app !== null && line.app !== filter.app) return false; + if (filter.process !== null && line.process !== filter.process) return false; + if (filter.minLevel !== null && LOG_LEVEL_RANK[line.level] < LOG_LEVEL_RANK[filter.minLevel]) return false; + return true; +} + +export interface FilteredLog { + lines: LogLine[]; + /** Lines the text query matched (all of `lines` in hide mode). Drives highlighting in dim mode. */ + matched: Set; + queryError?: string; +} + +/** + * Apply a filter. In `dim` mode the query does not remove lines, it only marks + * which ones matched, so the surrounding context stays readable. + */ +export function applyLogFilter(lines: readonly LogLine[], filter: LogFilter): FilteredLog { + const { matcher, error } = compileQuery(filter.query); + const kept: LogLine[] = []; + const matched = new Set(); + + for (const line of lines) { + if (!matchesScope(line, filter)) continue; + const hit = matcher === null || matcher(line.text); + if (hit) matched.add(line.id); + if (hit || filter.mode === 'dim') kept.push(line); + } + return { lines: kept, matched, queryError: error }; +} + +export type LevelCounts = Record; + +/** + * Per-level totals under every filter except the level itself, so the counts + * answer "how many errors would I see if I switched to error" instead of + * collapsing to zero the moment the filter is set to something else. + */ +export function countLevels(lines: readonly LogLine[], filter: LogFilter): LevelCounts { + const counts: LevelCounts = { debug: 0, info: 0, warn: 0, error: 0 }; + const scope: LogFilter = { ...filter, minLevel: null }; + const { matcher } = compileQuery(filter.query); + for (const line of lines) { + if (!matchesScope(line, scope)) continue; + if (filter.mode === 'hide' && matcher !== null && !matcher(line.text)) continue; + // An inherited level (a stack frame) is shown at that level but not tallied again. + if (!line.levelKnown && line.level !== 'info') continue; + counts[line.level] += 1; + } + return counts; +} + +/** + * Step through `[null, ...options]` - null being "all" - in either direction, + * wrapping. A current value that has since vanished (a server that stopped + * logging) restarts the cycle from "all". + */ +export function cycleOption(options: readonly string[], current: string | null, direction: 1 | -1): string | null { + const ring: Array = [null, ...options]; + const index = ring.indexOf(current); + const from = index === -1 ? 0 : index; + return ring[(from + direction + ring.length) % ring.length]; +} + +const LEVEL_CYCLE: Array = [null, 'warn', 'error']; + +/** all -> warn+ -> error only -> all. Debug-only is not a useful place to rest. */ +export function cycleMinLevel(current: LogLevel | null): LogLevel | null { + const index = LEVEL_CYCLE.indexOf(current); + return LEVEL_CYCLE[(index + 1) % LEVEL_CYCLE.length]; +} + +/** Whether any dimension narrows the view, for the "filters active" badge. */ +export function isFiltering(filter: LogFilter): boolean { + return ( + filter.server !== null || + filter.app !== null || + filter.process !== null || + filter.minLevel !== null || + filter.query !== '' + ); +} + +/** What can be picked in each dimension, derived from the lines actually seen. */ +export interface LogFacets { + servers: string[]; + apps: string[]; + processes: string[]; +} + +/** + * Facets narrow each other: once an app is chosen, only its processes are + * offered. Derived from the buffer, not config, so a process that has never + * logged does not clutter the cycle. + */ +export function logFacets(lines: readonly LogLine[], filter: Pick): LogFacets { + const servers = new Set(); + const apps = new Set(); + const processes = new Set(); + for (const line of lines) { + servers.add(line.server); + if (filter.server === null || line.server === filter.server) apps.add(line.app); + if ( + (filter.server === null || line.server === filter.server) && + (filter.app === null || line.app === filter.app) && + line.process !== null + ) { + processes.add(line.process); + } + } + const sorted = (set: Set): string[] => [...set].sort(); + return { servers: sorted(servers), apps: sorted(apps), processes: sorted(processes) }; +} + +/** + * Whether one line passes. Used where lines are consumed as they arrive + * (`logs --follow`) rather than filtered out of a buffer. + */ +export function matchesLogFilter(line: LogLine, filter: LogFilter): boolean { + return applyLogFilter([line], { ...filter, mode: 'hide' }).lines.length > 0; +} + +/** Parse a `--level` value; the error names what is accepted. */ +export function parseLevelOption(raw: string | undefined): { level: LogLevel | null; error?: string } { + if (raw === undefined) return { level: null }; + const name = raw.toLowerCase(); + if (name === 'debug' || name === 'info' || name === 'warn' || name === 'error') { + return { level: name === 'debug' ? null : name }; + } + return { level: null, error: `Unknown level '${raw}'. Use one of: debug, info, warn, error.` }; +} diff --git a/src/domain/observe/log-line.ts b/src/domain/observe/log-line.ts new file mode 100644 index 0000000..bc2f2b0 --- /dev/null +++ b/src/domain/observe/log-line.ts @@ -0,0 +1,205 @@ +/** + * Log lines as values. + * + * A streamed line carries where it came from (server, app, process) and what it + * says, so filtering is a pure function over lines rather than a property of + * which connection happened to deliver them. + */ + +export type LogLevel = 'debug' | 'info' | 'warn' | 'error'; + +/** Higher is more severe. Lines with no recognisable level rank as `info`. */ +export const LOG_LEVEL_RANK: Record = { debug: 0, info: 1, warn: 2, error: 3 }; + +export interface LogLine { + /** Monotonic per stream; a stable React key and a tie-break for equal timestamps. */ + id: number; + /** Receipt time, ms. Remote clocks are not trusted, so this orders lines across servers. */ + at: number; + server: string; + app: string; + /** PM2 process or systemd unit; null when the source cannot say (Caddy access log). */ + process: string | null; + /** + * True when the level was read from this line's own text. False for lines + * that defaulted to info or inherited an error from the line above (a stack + * frame), so counts do not tally one exception once per frame. + */ + levelKnown: boolean; + level: LogLevel; + text: string; +} + +const ANSI = /\u001b\[[0-9;?]*[ -/]*[@-~]/g; + +export function stripAnsi(text: string): string { + return text.replace(ANSI, ''); +} + +// ── Level classification ────────────────────────────────────────── + +const RED_TAG = /\[(fatal|error)\]/i; +const YELLOW_TAG = /\[warn(ing)?\]/i; +const RED_PREFIX = /(^|\s)(fatal|error)\s*[:\]]/i; +const CAMEL_ERROR = /[a-z](Error|Exception)\b/; +const RED_WORD = /\b(fatal|exception|uncaught|unhandled|panic|crit(ical)?)\b/i; +const YELLOW_WORD = /\bwarn(ing)?\b/i; +const PLAIN_ERROR = /\berror\b/i; + +/** Numeric pino levels: trace10 debug20 info30 warn40 error50 fatal60. */ +function jsonLevel(line: string): LogLevel | undefined { + const trimmed = line.trim(); + if (!trimmed.startsWith('{')) return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch { + return undefined; + } + if (typeof parsed !== 'object' || parsed === null) return undefined; + const record = parsed as Record; + + // Caddy access log: no useful level (always "info"), but the status says it. + if (typeof record.status === 'number' && 'request' in record) { + if (record.status >= 500) return 'error'; + if (record.status >= 400) return 'warn'; + return 'info'; + } + + const level = record.level ?? record.severity; + if (typeof level === 'number') { + if (level >= 50) return 'error'; + if (level >= 40) return 'warn'; + if (level >= 30) return 'info'; + return 'debug'; + } + if (typeof level === 'string') { + const name = level.toLowerCase(); + if (['error', 'fatal', 'panic', 'crit', 'critical'].includes(name)) return 'error'; + if (['warn', 'warning'].includes(name)) return 'warn'; + if (['info', 'notice'].includes(name)) return 'info'; + if (['debug', 'trace', 'verbose'].includes(name)) return 'debug'; + } + return undefined; +} + +/** + * The level a raw line announces, or undefined when it announces none. + * + * Structured JSON logs (pino/winston `level`/`severity`) win first; plain text + * falls back to bracket tags, "ERROR:"-style prefixes, camelCase Error class + * names, then whole-word matches - never a bare substring, so "mirror" or + * "Terror" do not false-positive. + */ +export function detectLevel(line: string): LogLevel | undefined { + const structured = jsonLevel(line); + if (structured !== undefined) return structured; + + if ( + RED_TAG.test(line) || + RED_PREFIX.test(line) || + CAMEL_ERROR.test(line) || + RED_WORD.test(line) || + PLAIN_ERROR.test(line) + ) { + return 'error'; + } + if (YELLOW_TAG.test(line) || YELLOW_WORD.test(line)) return 'warn'; + return undefined; +} + +/** A stack-trace frame belongs to the error line above it, not to a level of its own. */ +export function isContinuation(text: string): boolean { + return /^\s+at\s/.test(text) || /^\s*\.\.\.\s*\d+ more/.test(text); +} + +// ── Per-supervisor line parsing ─────────────────────────────────── + +export interface ParsedLine { + process: string | null; + text: string; +} + +/** + * `pm2 logs` prefixes each line with `| | `, and prints a + * ` last N lines:` banner per log file ahead of the backlog. Banners are + * noise; unprefixed lines (pm2's own notices) keep a null process. + */ +export function parsePm2Line(raw: string): ParsedLine | null { + const line = stripAnsi(raw); + if (/^\S+\.log last \d+ lines:\s*$/.test(line)) return null; + const match = /^(\d+)\|([^|\s]+)\s*\|\s?(.*)$/.exec(line); + if (match === null) return { process: null, text: line }; + return { process: match[2], text: match[3] }; +} + +/** One systemd unit or Caddy file per stream, so the source already knows the process. */ +export function parsePlainLine(process: string | null): (raw: string) => ParsedLine | null { + return (raw) => ({ process, text: stripAnsi(raw) }); +} + +// ── Chunks to lines ─────────────────────────────────────────────── + +/** + * Re-cut arbitrary stream chunks into whole lines. + * + * SSH delivers data in whatever sizes the network picked, so a chunk can end + * mid-line. The unfinished tail is held until its newline arrives. CR is + * dropped because a pty turns every LF into CRLF. + */ +export class LineAssembler { + private tail = ''; + + push(chunk: string): string[] { + const parts = (this.tail + chunk).split('\n'); + this.tail = parts.pop() ?? ''; + return parts.map((part) => part.replace(/\r$/, '')).filter((part) => part.length > 0); + } + + /** The partial line left when the stream ended without a final newline. */ + flush(): string[] { + const rest = this.tail.replace(/\r$/, ''); + this.tail = ''; + return rest.length > 0 ? [rest] : []; + } +} + +// ── Reconnect de-duplication ────────────────────────────────────── + +/** + * Drops the backlog a reconnect replays. + * + * Every (re)connect asks the server for its last N lines so a fresh viewer has + * context. After a dropped connection those lines were already shown, so the + * guard holds the tail of what this source delivered and swallows incoming + * lines for as long as they keep matching it in order. The first line that + * does not match ends the replay; everything after it is new. + */ +export class ReplayGuard { + private seen: string[] = []; + private replay: string[] = []; + private replaying = false; + + constructor(private readonly window: number = 50) {} + + /** Arm for a reconnect: the next lines may repeat what was already delivered. */ + arm(): void { + this.replay = [...this.seen]; + this.replaying = true; + } + + /** True when the line should be shown. */ + accept(text: string): boolean { + if (this.replaying) { + const at = this.replay.indexOf(text); + if (at !== -1) { + this.replay = this.replay.slice(at + 1); + return false; + } + this.replaying = false; + } + this.seen.push(text); + if (this.seen.length > this.window) this.seen.shift(); + return true; + } +} diff --git a/src/domain/observe/log-source.ts b/src/domain/observe/log-source.ts new file mode 100644 index 0000000..0028c0b --- /dev/null +++ b/src/domain/observe/log-source.ts @@ -0,0 +1,92 @@ +import type { RemoteExecutor } from '../remote/executor.js'; +import type { ShipnodeApp } from '../../shared/types.js'; +import { getPm2Name } from '../pm2/apps.js'; +import { isWatt, resolveWattUnits } from '../runtime/watt.js'; +import { MISE, shellQuote } from './script.js'; +import { parsePlainLine, parsePm2Line, type ParsedLine } from './log-line.js'; + +/** + * One thing worth following: a command that prints log lines forever, plus how + * to read what it prints. + * + * PM2 gives one stream per app (it multiplexes processes and prefixes each + * line); systemd gives one per unit; Caddy one per access-log file. Each + * source tags its lines with the server and app it belongs to, so a stream + * never has to guess. + */ +export interface LogSource { + server: string; + app: string; + kind: 'pm2' | 'systemd' | 'caddy'; + /** Follow command, replaying `backlog` lines first. */ + command(backlog: number): string; + parse(raw: string): ParsedLine | null; +} + +export function pm2FollowCommand(namespace: string, backlog: number): string { + return `${MISE} && pm2 logs ${shellQuote(namespace)} --lines ${backlog} --no-color 2>&1`; +} + +export function journalFollowCommand(unit: string, backlog: number): string { + return ( + `S=$([ "$(id -u)" = 0 ] || echo sudo); ` + + `$S journalctl -u ${shellQuote(unit)} -n ${backlog} -f --no-pager -o cat 2>&1` + ); +} + +export function caddyFollowCommand(appName: string, backlog: number): string { + const file = shellQuote(`/var/log/caddy/${appName}.log`); + // -F keeps following across log rotation; `sudo -n` fails fast rather than prompting. + return `sudo -n tail -n ${backlog} -F ${file} 2>/dev/null || tail -n ${backlog} -F ${file} 2>&1`; +} + +/** + * The sources for one app on one server. Watt units are resolved by asking the + * host, because a blue-green unit carries a colour only the host knows. + */ +export async function planLogSources( + executor: RemoteExecutor, + server: string, + remotePath: string, + app: ShipnodeApp, +): Promise { + if (app.appType === 'frontend') { + return [ + { + server, + app: app.name, + kind: 'caddy', + command: (backlog) => caddyFollowCommand(app.name, backlog), + parse: parsePlainLine(null), + }, + ]; + } + + const processes = app.pm2?.apps ?? []; + if (processes.length === 0) return []; + + if (isWatt(app)) { + // One unit per declared process, in declaration order. The lines are labelled + // with the PM2-style name so a process filter means the same on either supervisor. + const units = await resolveWattUnits(executor, `${remotePath}/${app.name}`, app, { colors: 'active' }); + return units.map((unit, index) => ({ + server, + app: app.name, + kind: 'systemd' as const, + command: (backlog: number) => journalFollowCommand(unit, backlog), + parse: parsePlainLine(getPm2Name(processes[0].name, processes[index].name)), + })); + } + + // Same namespace `shipnode logs` and the collector use: the first process names the deployment. + const namespace = processes[0].name; + return [ + { + server, + app: app.name, + kind: 'pm2', + command: (backlog) => pm2FollowCommand(namespace, backlog), + parse: parsePm2Line, + }, + ]; +} diff --git a/src/domain/remote/executor.ts b/src/domain/remote/executor.ts index 1a0b135..f30a3f4 100644 --- a/src/domain/remote/executor.ts +++ b/src/domain/remote/executor.ts @@ -12,6 +12,17 @@ import type { ExecResult } from '../../shared/types.js'; export interface ExecOptions { timeout?: number; onData?: (chunk: string, fd: 'stdout' | 'stderr') => void; + /** + * Ends a long-running command (a log follow). `exec` then resolves with + * whatever the command had produced - an abort is a normal way to finish. + */ + signal?: AbortSignal; + /** + * Allocate a pseudo-terminal. The remote side hangs up the command when the + * channel closes, so an abandoned `tail -f` does not outlive the session. + * stderr is merged into stdout, and line endings arrive as CRLF. + */ + pty?: boolean; } export abstract class RemoteExecutor { diff --git a/src/infrastructure/ssh/connection.ts b/src/infrastructure/ssh/connection.ts index e597c58..01fb000 100644 --- a/src/infrastructure/ssh/connection.ts +++ b/src/infrastructure/ssh/connection.ts @@ -133,7 +133,7 @@ export class SshConnection extends RemoteExecutor { } return new Promise((resolve, reject) => { - this.client.exec(command, { pty: false }, (err, stream: ClientChannel) => { + this.client.exec(command, { pty: options?.pty === true }, (err, stream: ClientChannel) => { if (err) { return reject(new SshError(`Failed to execute command: ${err.message}`)); } @@ -154,7 +154,14 @@ export class SshConnection extends RemoteExecutor { options?.onData?.(chunk, 'stderr'); }); + const onAbort = (): void => { + stream.close(); + }; + if (options?.signal?.aborted) onAbort(); + else options?.signal?.addEventListener('abort', onAbort, { once: true }); + stream.on('close', (code: number | undefined) => { + options?.signal?.removeEventListener('abort', onAbort); exitCode = code ?? 1; resolve({ stdout: stdout.trim(), stderr: stderr.trim(), exitCode }); }); diff --git a/src/services/observe/log-stream.ts b/src/services/observe/log-stream.ts new file mode 100644 index 0000000..4976f0c --- /dev/null +++ b/src/services/observe/log-stream.ts @@ -0,0 +1,246 @@ +import type { RemoteExecutor } from '../../domain/remote/executor.js'; +import type { LogSource } from '../../domain/observe/log-source.js'; +import { + detectLevel, + isContinuation, + LineAssembler, + ReplayGuard, + type LogLevel, + type LogLine, +} from '../../domain/observe/log-line.js'; + +export type SourceState = 'connecting' | 'live' | 'reconnecting'; + +export interface LogBinding { + executor: RemoteExecutor; + source: LogSource; +} + +/** How one followed source is doing, for the viewer's status line. */ +export interface SourceHealth { + server: string; + app: string; + kind: LogSource['kind']; + state: SourceState; + /** Why the last attempt ended; cleared once data flows again. */ + error?: string; +} + +export interface LogStreamOptions { + bindings: LogBinding[]; + /** Lines kept across all sources; the oldest fall off first. */ + capacity?: number; + /** Lines of history requested on every (re)connect. */ + backlog?: number; + /** A quiet app sends nothing, so a connection that has not failed by now is called live. */ + liveAfterMs?: number; + /** Coalesce bursts: subscribers are told at most this often. */ + flushMs?: number; + backoffMs?: (attempt: number) => number; + /** Called synchronously for every accepted line, before batching - for consumers that print rather than render. */ + onLine?: (line: LogLine) => void; + now?: () => number; + /** Injected for tests; must resolve early when the signal aborts. */ + sleep?: (ms: number, signal: AbortSignal) => Promise; +} + +type Subscriber = () => void; + +const DEFAULT_BACKOFF = (attempt: number): number => Math.min(30_000, 1000 * 2 ** attempt); + +function abortableSleep(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal.aborted) return resolve(); + const timer = setTimeout(done, ms); + function done(): void { + clearTimeout(timer); + signal.removeEventListener('abort', done); + resolve(); + } + signal.addEventListener('abort', done, { once: true }); + }); +} + +/** + * Follows every source at once and merges them into one bounded buffer. + * + * Streaming is per source and holds one long-lived channel each over the SSH + * connection the host already has, so adding a server adds no connection. + * Filtering is deliberately not here: the buffer holds everything and the + * viewer filters it, which is why changing a filter never reconnects and never + * loses a line that was only hidden. + */ +export class LogStream { + private readonly capacity: number; + private readonly backlog: number; + private readonly liveAfterMs: number; + private readonly flushMs: number; + private readonly backoffMs: (attempt: number) => number; + private readonly now: () => number; + private readonly sleep: (ms: number, signal: AbortSignal) => Promise; + + private readonly subscribers = new Set(); + private readonly healths: SourceHealth[]; + private buffer: LogLine[] = []; + private snapshot: readonly LogLine[] = []; + private nextId = 1; + private controller: AbortController | null = null; + private running: Promise[] = []; + private flushTimer: ReturnType | null = null; + + constructor(private readonly options: LogStreamOptions) { + this.capacity = options.capacity ?? 2000; + this.backlog = options.backlog ?? 20; + this.liveAfterMs = options.liveAfterMs ?? 1500; + this.flushMs = options.flushMs ?? 100; + this.backoffMs = options.backoffMs ?? DEFAULT_BACKOFF; + this.now = options.now ?? Date.now; + this.sleep = options.sleep ?? abortableSleep; + this.healths = options.bindings.map(({ source }) => ({ + server: source.server, + app: source.app, + kind: source.kind, + state: 'connecting', + })); + } + + get isRunning(): boolean { + return this.controller !== null; + } + + lines(): readonly LogLine[] { + return this.snapshot; + } + + health(): readonly SourceHealth[] { + return this.healths.map((health) => ({ ...health })); + } + + subscribe(subscriber: Subscriber): () => void { + this.subscribers.add(subscriber); + return () => { + this.subscribers.delete(subscriber); + }; + } + + /** Forget what has been seen. Followed sources keep following. */ + clear(): void { + this.buffer = []; + this.snapshot = []; + this.notify(); + } + + start(): void { + if (this.controller !== null) return; + const controller = new AbortController(); + this.controller = controller; + this.running = this.options.bindings.map((binding, index) => this.follow(binding, index, controller.signal)); + } + + /** Stop following and wait for every channel to close, so nothing is left running remotely. */ + async stop(): Promise { + const controller = this.controller; + if (controller === null) return; + this.controller = null; + controller.abort(); + if (this.flushTimer !== null) { + clearTimeout(this.flushTimer); + this.flushTimer = null; + } + await Promise.allSettled(this.running); + this.running = []; + } + + private async follow(binding: LogBinding, index: number, signal: AbortSignal): Promise { + const { executor, source } = binding; + const assembler = new LineAssembler(); + const guard = new ReplayGuard(); + let lastLevel: LogLevel | undefined; + let attempt = 0; + let firstConnect = true; + + const ingest = (raw: string): void => { + const parsed = source.parse(raw); + if (parsed === null || parsed.text.trim() === '') return; + if (!guard.accept(parsed.text)) return; + + const detected = detectLevel(parsed.text); + // A stack frame continues the error above it rather than reading as info. + const inherited = detected === undefined && isContinuation(parsed.text) ? lastLevel : undefined; + if (detected !== undefined) lastLevel = detected; + + this.push({ + id: this.nextId++, + at: this.now(), + server: source.server, + app: source.app, + process: parsed.process, + levelKnown: detected !== undefined, + level: detected ?? inherited ?? 'info', + text: parsed.text, + }); + }; + + while (!signal.aborted) { + this.setState(index, firstConnect ? 'connecting' : 'reconnecting'); + if (!firstConnect) guard.arm(); + firstConnect = false; + + let reason: string; + const liveTimer = setTimeout(() => this.setState(index, 'live'), this.liveAfterMs); + try { + const result = await executor.exec(source.command(this.backlog), { + pty: true, + signal, + onData: (chunk) => { + if (attempt !== 0 || this.healths[index].state !== 'live') { + attempt = 0; + this.setState(index, 'live', null); + } + for (const line of assembler.push(chunk)) ingest(line); + }, + }); + for (const line of assembler.flush()) ingest(line); + reason = `stream ended (exit ${result.exitCode})`; + } catch (cause: unknown) { + reason = cause instanceof Error ? cause.message : String(cause); + } finally { + clearTimeout(liveTimer); + } + + if (signal.aborted) break; + this.setState(index, 'reconnecting', reason); + await this.sleep(this.backoffMs(attempt), signal); + attempt += 1; + } + } + + private setState(index: number, state: SourceState, error?: string | null): void { + const health = this.healths[index]; + const nextError = error === undefined ? health.error : (error ?? undefined); + if (health.state === state && health.error === nextError) return; + health.state = state; + health.error = nextError; + this.schedule(); + } + + private push(line: LogLine): void { + this.options.onLine?.(line); + this.buffer.push(line); + if (this.buffer.length > this.capacity) this.buffer.splice(0, this.buffer.length - this.capacity); + this.schedule(); + } + + private schedule(): void { + if (this.flushTimer !== null || this.controller === null) return; + this.flushTimer = setTimeout(() => { + this.flushTimer = null; + this.snapshot = this.buffer.slice(); + this.notify(); + }, this.flushMs); + } + + private notify(): void { + for (const subscriber of this.subscribers) subscriber(); + } +} diff --git a/tests/unit/monitor-fleet.test.ts b/tests/unit/monitor-fleet.test.ts new file mode 100644 index 0000000..fe4fa03 --- /dev/null +++ b/tests/unit/monitor-fleet.test.ts @@ -0,0 +1,183 @@ +import { describe, it, expect } from 'vitest'; +import { buildFleetRows, describeEvent, describeFleet, toMetricsSnapshot } from '../../src/cli/monitor/fleet-model.js'; +import { INITIAL_LOG_VIEW, logViewReducer, type LogViewState } from '../../src/cli/monitor/log-view-state.js'; +import { pivotByApp } from '../../src/domain/observe/pivot.js'; +import type { AppSnapshot, ServerSnapshot } from '../../src/domain/observe/snapshot.js'; +import type { ProcessInfo } from '../../src/domain/observe/types.js'; +import { parseSystemStats } from '../../src/domain/observe/parse.js'; +import type { LogLine } from '../../src/domain/observe/log-line.js'; +import { sourceLabel } from '../../src/cli/monitor/panels/LogViewer.js'; + +function proc(overrides: Partial = {}): ProcessInfo { + return { + name: 'web', pm2Name: 'api-web', pid: 1, status: 'online', cpu: 10, memory: 100, uptime: 60, + restarts: 0, execMode: 'cluster', instances: 2, unstableRestarts: 0, exitCode: null, + ...overrides, + }; +} + +function app(release: string | null, overrides: Partial = {}): AppSnapshot { + return { + app: 'api', appType: 'backend', processes: [proc()], + currentRelease: release === null ? null : `/var/www/api/releases/${release}`, + releases: [], ...overrides, + }; +} + +function server(name: string, apps: AppSnapshot[], extra: Partial = {}): ServerSnapshot { + return { + server: name, timestamp: '2026-10-02T10:00:00Z', system: parseSystemStats(''), + deployLock: null, apps, ...extra, + }; +} + +describe('buildFleetRows', () => { + it('gives one row per app per server, in fleet order', () => { + const fleets = pivotByApp([server('a', [app('r2')]), server('b', [app('r2')])]); + const rows = buildFleetRows(fleets); + expect(rows.map((r) => [r.app, r.server, r.release])).toEqual([['api', 'a', 'r2'], ['api', 'b', 'r2']]); + expect(rows.every((r) => r.replicated)).toBe(true); + }); + + it('marks the replica that is not on the newest release as behind while the fleet is split', () => { + const rows = buildFleetRows(pivotByApp([server('a', [app('r2')]), server('b', [app('r1')])])); + expect(rows.map((r) => r.releaseState)).toEqual(['current', 'behind']); + }); + + it('reads converged replicas as current', () => { + const rows = buildFleetRows(pivotByApp([server('a', [app('r1')]), server('b', [app('r1')])])); + expect(rows.map((r) => r.releaseState)).toEqual(['current', 'current']); + }); + + it('keeps a row for a replica that could not be reached', () => { + const down = server('b', [], { error: 'timeout', plannedApps: [{ app: 'api', appType: 'backend' }] }); + const rows = buildFleetRows(pivotByApp([server('a', [app('r1')]), down])); + expect(rows.map((r) => [r.server, r.reachable])).toEqual([['a', true], ['b', false]]); + expect(rows[1].releaseState).toBe('unknown'); + }); + + it('sums process counts, memory and restarts per replica', () => { + const snapshot = app('r1', { processes: [proc({ restarts: 2 }), proc({ status: 'errored', restarts: 1, memory: 50 })] }); + const [row] = buildFleetRows(pivotByApp([server('a', [snapshot])])); + expect([row.online, row.total, row.memoryMb, row.restarts]).toEqual([1, 2, 150, 3]); + expect(row.replicated).toBe(false); + }); + + it('keeps a stable key per app and server', () => { + const rows = buildFleetRows(pivotByApp([server('a', [app('r1')])])); + expect(rows[0].key).toBe(buildFleetRows(pivotByApp([server('a', [app('r9')])]))[0].key); + }); +}); + +describe('describeFleet', () => { + it('names unreachable replicas first, then a split roll, then a converged fleet', () => { + const down = server('b', [], { error: 'x', plannedApps: [{ app: 'api', appType: 'backend' }] }); + expect(describeFleet(pivotByApp([server('a', [app('r1')]), down])[0]).tone).toBe('bad'); + expect(describeFleet(pivotByApp([server('a', [app('r2')]), server('b', [app('r1')])])[0]).text).toMatch(/split across 2 releases/); + expect(describeFleet(pivotByApp([server('a', [app('r1')]), server('b', [app('r1')])])[0])).toEqual({ + tone: 'ok', text: 'converged on r1', + }); + }); + + it('warns about a replica with no release', () => { + expect(describeFleet(pivotByApp([server('a', [app('r1')]), server('b', [app(null)])])[0]).tone).toBe('warn'); + }); +}); + +describe('toMetricsSnapshot', () => { + it('flattens host facts and app facts into the shape the panels take', () => { + const s = server('a', [app('r1')], { deployLock: { lockedAt: 'now', ageSeconds: 5 } }); + const flat = toMetricsSnapshot(s, s.apps[0]); + expect(flat.deployLock?.ageSeconds).toBe(5); + expect(flat.processes).toHaveLength(1); + expect(flat.currentRelease).toContain('r1'); + }); +}); + +describe('describeEvent', () => { + it('names the server so a fleet event is attributable', () => { + expect(describeEvent({ kind: 'health-failing', at: '', server: 'b', app: 'api', streak: 3 }).text).toContain('api on b'); + expect(describeEvent({ kind: 'server-unreachable', at: '', server: 'b', message: 'timeout' }).tone).toBe('bad'); + expect(describeEvent({ kind: 'notice', at: '', message: 'hi' })).toEqual({ tone: 'info', text: 'hi' }); + }); +}); + +// ── Log view reducer ────────────────────────────────────────────── + +const facets = { servers: ['a', 'b'], apps: ['api', 'site'], processes: ['api-web', 'api-worker'] }; +const liveLines = [{ id: 1 }, { id: 2 }] as LogLine[]; + +describe('logViewReducer', () => { + const reduce = (state: LogViewState, ...actions: Parameters[1][]) => + actions.reduce(logViewReducer, state); + + it('resets app and process when the server changes, and process when the app changes', () => { + let state = reduce(INITIAL_LOG_VIEW, { type: 'cycle', dimension: 'app', direction: 1, facets }); + state = reduce(state, { type: 'cycle', dimension: 'process', direction: 1, facets }); + expect(state.filter).toMatchObject({ app: 'api', process: 'api-web' }); + + const app = reduce(state, { type: 'cycle', dimension: 'app', direction: 1, facets }); + expect(app.filter).toMatchObject({ app: 'site', process: null }); + + const srv = reduce(state, { type: 'cycle', dimension: 'server', direction: 1, facets }); + expect(srv.filter).toMatchObject({ server: 'a', app: null, process: null }); + }); + + it('builds a query while typing and drops it on cancel but keeps it on commit', () => { + let state = reduce(INITIAL_LOG_VIEW, { type: 'start-search' }, { type: 'type', text: 'ti' }, { type: 'type', text: 'me' }); + expect(state).toMatchObject({ typing: true, filter: { query: 'time' } }); + state = reduce(state, { type: 'backspace' }); + expect(state.filter.query).toBe('tim'); + expect(reduce(state, { type: 'commit-search' })).toMatchObject({ typing: false, filter: { query: 'tim' } }); + expect(reduce(state, { type: 'cancel-search' })).toMatchObject({ typing: false, filter: { query: '' } }); + }); + + it('clears filters but keeps the chosen mode', () => { + const state = reduce(INITIAL_LOG_VIEW, { type: 'toggle-mode' }, { type: 'cycle-level' }, { type: 'cycle', dimension: 'server', direction: 1, facets }); + const cleared = reduce(state, { type: 'clear-filters' }); + expect(cleared.filter).toMatchObject({ server: null, minLevel: null, mode: 'dim' }); + }); + + it('freezes on the first scroll up and clamps the offset', () => { + let state = reduce(INITIAL_LOG_VIEW, { type: 'scroll', delta: 1, live: liveLines, max: 3 }); + expect(state.frozen).toBe(liveLines); + expect(state.offset).toBe(1); + state = reduce(state, { type: 'scroll', delta: 10, live: liveLines, max: 3 }); + expect(state.offset).toBe(3); + state = reduce(state, { type: 'scroll', delta: -10, live: liveLines, max: 3 }); + expect(state.offset).toBe(0); + expect(state.frozen).toBe(liveLines); + }); + + it('ignores scrolling down while following', () => { + expect(reduce(INITIAL_LOG_VIEW, { type: 'scroll', delta: -1, live: liveLines, max: 3 })).toBe(INITIAL_LOG_VIEW); + }); + + it('toggles pause and resumes at the tail', () => { + const paused = reduce(INITIAL_LOG_VIEW, { type: 'toggle-pause', live: liveLines }); + expect(paused.frozen).toBe(liveLines); + expect(reduce(paused, { type: 'toggle-pause', live: liveLines }).frozen).toBeNull(); + expect(reduce({ ...paused, offset: 2 }, { type: 'resume' })).toMatchObject({ frozen: null, offset: 0 }); + }); + + it('enters scoped to what was being looked at', () => { + const state = reduce(INITIAL_LOG_VIEW, { type: 'start-search' }, { type: 'type', text: 'x' }, { type: 'enter', server: 'b', app: 'api' }); + expect(state).toMatchObject({ typing: false, frozen: null, filter: { server: 'b', app: 'api', query: '' } }); + }); +}); + +describe('sourceLabel', () => { + const line = { server: 'b', app: 'api', process: 'api-web' } as LogLine; + + it('shows only the parts that vary on screen', () => { + expect(sourceLabel(line, true, true)).toBe('b api:api-web'); + expect(sourceLabel(line, true, false)).toBe('b:api-web'); + expect(sourceLabel(line, false, false)).toBe('api-web'); + }); + + it('omits a process that is just the app, or unknown', () => { + expect(sourceLabel({ ...line, process: 'api' } as LogLine, true, true)).toBe('b api'); + expect(sourceLabel({ ...line, process: null } as LogLine, true, true)).toBe('b api'); + expect(sourceLabel({ ...line, process: null } as LogLine, false, false)).toBe(''); + }); +}); diff --git a/tests/unit/monitor.test.ts b/tests/unit/monitor.test.ts index 65694af..9d9a3fb 100644 --- a/tests/unit/monitor.test.ts +++ b/tests/unit/monitor.test.ts @@ -5,9 +5,9 @@ import { parsePm2Jlist, parseSystemStats, parseReleaseRecords, parseDeployLock, import { collectMetrics, collectLogs, collectCaddyLogs } from '../../src/cli/monitor/poller.js'; import { appSectionName } from '../../src/domain/observe/script.js'; import { restartProcess, rollbackToRelease } from '../../src/cli/monitor/actions.js'; -import { logLineColor } from '../../src/cli/monitor/panels/LogPanel.js'; +import { logLineColor } from '../../src/cli/monitor/log-color.js'; import { assembleConfig } from '../../src/config/assembly.js'; -import { getAccessoriesForMonitorTarget, getAppsForMonitorTarget, resolveMonitorSession } from '../../src/cli/monitor/monitor-session.js'; +import { getAccessoriesForMonitorTarget, getAppsForMonitorTarget } from '../../src/cli/monitor/monitor-session.js'; // ── Charts ──────────────────────────────────────────────────────── @@ -691,45 +691,6 @@ describe('collectCaddyLogs', () => { }); describe('monitor session', () => { - it('resolves the selected app server target', () => { - const config = assembleConfig({ - servers: { - app: { host: '1.1.1.1', user: 'deploy', port: 22 }, - data: { host: '2.2.2.2', user: 'deploy', port: 22 }, - }, - remotePath: '/var/www/app', - apps: [ - { name: 'api', appType: 'backend', on: 'app', healthCheck: { enabled: true } }, - { name: 'worker', appType: 'backend', on: 'data', healthCheck: { enabled: true } }, - ], - }); - - const session = resolveMonitorSession(config, 'worker'); - - expect(session.isOk()).toBe(true); - if (session.isOk()) { - expect(session.value.target.name).toBe('data'); - expect(session.value.target.ssh.host).toBe('2.2.2.2'); - } - }); - - it('picks a fleet replica with --on', () => { - const config = assembleConfig({ - servers: { - a: { host: '1.1.1.1', user: 'deploy', port: 22 }, - b: { host: '2.2.2.2', user: 'deploy', port: 22 }, - }, - remotePath: '/var/www/app', - apps: [ - { name: 'api', appType: 'backend', on: ['a', 'b'], healthCheck: { enabled: true } }, - ], - }); - - const session = resolveMonitorSession(config, 'api', 'b'); - expect(session.isOk()).toBe(true); - if (session.isOk()) expect(session.value.target.name).toBe('b'); - }); - it('limits selectable apps to the connected server target', () => { const config = assembleConfig({ servers: { @@ -886,14 +847,14 @@ describe('Ink components', () => { expect(typeof mod.ReleasePanel).toBe('function'); }); - it('LogPanel is a function', async () => { - const mod = await import('../../src/cli/monitor/panels/LogPanel.js'); - expect(typeof mod.LogPanel).toBe('function'); + it('LogViewer is a function', async () => { + const mod = await import('../../src/cli/monitor/panels/LogViewer.js'); + expect(typeof mod.LogViewer).toBe('function'); }); - it('AppSelector is a function', async () => { - const mod = await import('../../src/cli/monitor/app-selector.js'); - expect(typeof mod.AppSelector).toBe('function'); + it('FleetPanel is a function', async () => { + const mod = await import('../../src/cli/monitor/panels/FleetPanel.js'); + expect(typeof mod.FleetPanel).toBe('function'); }); it('App is a function', async () => { diff --git a/tests/unit/observe-logs.test.ts b/tests/unit/observe-logs.test.ts new file mode 100644 index 0000000..ee551f8 --- /dev/null +++ b/tests/unit/observe-logs.test.ts @@ -0,0 +1,556 @@ +import { describe, it, expect } from 'vitest'; +import { + detectLevel, + isContinuation, + LineAssembler, + parsePm2Line, + ReplayGuard, + type LogLine, +} from '../../src/domain/observe/log-line.js'; +import { + applyLogFilter, + compileQuery, + countLevels, + cycleMinLevel, + cycleOption, + EMPTY_LOG_FILTER, + isFiltering, + logFacets, + type LogFilter, +} from '../../src/domain/observe/log-filter.js'; +import { + caddyFollowCommand, + journalFollowCommand, + planLogSources, + pm2FollowCommand, +} from '../../src/domain/observe/log-source.js'; +import { LogStream, type LogBinding } from '../../src/services/observe/log-stream.js'; +import { RemoteExecutor, type ExecOptions } from '../../src/domain/remote/executor.js'; +import type { ExecResult, ShipnodeApp } from '../../src/shared/types.js'; +import { FakeRemoteExecutor } from '../testing/fake-executor.js'; + +function line(partial: Partial & { id: number }): LogLine { + return { + at: partial.id, + server: 'a', + app: 'api', + process: 'web', + levelKnown: true, + level: 'info', + text: `line ${partial.id}`, + ...partial, + }; +} + +// ── Classification and parsing ──────────────────────────────────── + +describe('detectLevel', () => { + it('reads structured and plain levels', () => { + expect(detectLevel('{"level":"error","msg":"x"}')).toBe('error'); + expect(detectLevel('{"level":40,"msg":"x"}')).toBe('warn'); + expect(detectLevel('{"level":30,"msg":"x"}')).toBe('info'); + expect(detectLevel('{"level":20,"msg":"x"}')).toBe('debug'); + expect(detectLevel('[ERROR] connection refused')).toBe('error'); + expect(detectLevel('WARN memory high')).toBe('warn'); + }); + + it('returns undefined for lines that announce nothing', () => { + expect(detectLevel('GET /health 200 12ms')).toBeUndefined(); + expect(detectLevel('reflection off the mirror')).toBeUndefined(); + }); + + it('grades a Caddy access log line by its status', () => { + const access = (status: number) => JSON.stringify({ level: 'info', status, request: { uri: '/' } }); + expect(detectLevel(access(200))).toBe('info'); + expect(detectLevel(access(404))).toBe('warn'); + expect(detectLevel(access(502))).toBe('error'); + }); + + it('treats stack frames as continuations', () => { + expect(isContinuation(' at Object. (/app/x.js:1:1)')).toBe(true); + expect(isContinuation('GET /at 200')).toBe(false); + }); +}); + +describe('parsePm2Line', () => { + it('splits the process prefix from the message', () => { + expect(parsePm2Line('0|api-web | listening on :3000')).toEqual({ process: 'api-web', text: 'listening on :3000' }); + }); + + it('strips colour codes', () => { + expect(parsePm2Line('1|worker | \u001b[31mboom\u001b[0m')).toEqual({ process: 'worker', text: 'boom' }); + }); + + it('drops the per-file banner and keeps unprefixed notices', () => { + expect(parsePm2Line('/home/deploy/.pm2/logs/api-web-out.log last 20 lines:')).toBeNull(); + expect(parsePm2Line('[TAILING] Tailing last 20 lines')).toEqual({ + process: null, + text: '[TAILING] Tailing last 20 lines', + }); + }); +}); + +describe('LineAssembler', () => { + it('holds a partial line until its newline arrives', () => { + const assembler = new LineAssembler(); + expect(assembler.push('hel')).toEqual([]); + expect(assembler.push('lo\nwor')).toEqual(['hello']); + expect(assembler.push('ld\n')).toEqual(['world']); + }); + + it('drops CR from pty line endings and blank lines, and flushes the tail', () => { + const assembler = new LineAssembler(); + expect(assembler.push('a\r\n\r\nb\r\nc')).toEqual(['a', 'b']); + expect(assembler.flush()).toEqual(['c']); + expect(assembler.flush()).toEqual([]); + }); +}); + +describe('ReplayGuard', () => { + it('shows everything until armed', () => { + const guard = new ReplayGuard(); + expect(['a', 'b'].map((t) => guard.accept(t))).toEqual([true, true]); + }); + + it('swallows a replayed backlog and passes what follows it', () => { + const guard = new ReplayGuard(); + ['a', 'b', 'c'].forEach((t) => guard.accept(t)); + guard.arm(); + // The server replays its last 2 lines, then something new happens. + expect(['b', 'c', 'd', 'e'].map((t) => guard.accept(t))).toEqual([false, false, true, true]); + }); + + it('shows everything when the replay shares nothing with what was seen', () => { + const guard = new ReplayGuard(); + guard.accept('a'); + guard.arm(); + expect(['x', 'y'].map((t) => guard.accept(t))).toEqual([true, true]); + }); + + it('does not swallow a genuine repeat after the replay ends', () => { + const guard = new ReplayGuard(); + ['a', 'b'].forEach((t) => guard.accept(t)); + guard.arm(); + expect(['b', 'new', 'b'].map((t) => guard.accept(t))).toEqual([false, true, true]); + }); +}); + +// ── Filtering ───────────────────────────────────────────────────── + +describe('compileQuery', () => { + it('matches plain text case-insensitively', () => { + const { matcher } = compileQuery('Timeout'); + expect(matcher?.('request TIMEOUT after 5s')).toBe(true); + expect(matcher?.('all good')).toBe(false); + }); + + it('treats /x/ as a regex, with an optional i flag', () => { + expect(compileQuery('/^GET \\/api/').matcher?.('GET /api/users')).toBe(true); + expect(compileQuery('/^GET \\/api/').matcher?.('get /api/users')).toBe(false); + expect(compileQuery('/^GET \\/api/i').matcher?.('get /api/users')).toBe(true); + }); + + it('negates with a leading !', () => { + const { matcher } = compileQuery('!health'); + expect(matcher?.('GET /health')).toBe(false); + expect(matcher?.('GET /users')).toBe(true); + }); + + it('reports a broken regex instead of throwing', () => { + const result = compileQuery('/(unclosed/'); + expect(result.matcher).toBeNull(); + expect(result.error).toBeDefined(); + }); + + it('means "no query" for an empty string or a lone !', () => { + expect(compileQuery('').matcher).toBeNull(); + expect(compileQuery('!').matcher?.('!')).toBe(true); + }); +}); + +describe('applyLogFilter', () => { + const lines = [ + line({ id: 1, server: 'a', app: 'api', process: 'web', level: 'info', text: 'ready' }), + line({ id: 2, server: 'b', app: 'api', process: 'web', level: 'error', text: 'db timeout' }), + line({ id: 3, server: 'b', app: 'site', process: null, level: 'warn', text: 'slow response' }), + line({ id: 4, server: 'a', app: 'api', process: 'worker', level: 'error', text: 'job failed' }), + ]; + const ids = (filter: Partial) => + applyLogFilter(lines, { ...EMPTY_LOG_FILTER, ...filter }).lines.map((l) => l.id); + + it('keeps everything with no filter', () => { + expect(ids({})).toEqual([1, 2, 3, 4]); + }); + + it('narrows by server, app and process', () => { + expect(ids({ server: 'b' })).toEqual([2, 3]); + expect(ids({ app: 'api' })).toEqual([1, 2, 4]); + expect(ids({ server: 'a', app: 'api', process: 'worker' })).toEqual([4]); + }); + + it('shows the chosen level and above', () => { + expect(ids({ minLevel: 'warn' })).toEqual([2, 3, 4]); + expect(ids({ minLevel: 'error' })).toEqual([2, 4]); + }); + + it('combines dimensions', () => { + expect(ids({ server: 'b', minLevel: 'error', query: 'timeout' })).toEqual([2]); + }); + + it('hides non-matching lines in hide mode and keeps them dimmed in dim mode', () => { + expect(ids({ query: 'fail' })).toEqual([4]); + const dim = applyLogFilter(lines, { ...EMPTY_LOG_FILTER, query: 'fail', mode: 'dim' }); + expect(dim.lines.map((l) => l.id)).toEqual([1, 2, 3, 4]); + expect([...dim.matched]).toEqual([4]); + }); + + it('surfaces a query error and ignores the query while it is broken', () => { + const result = applyLogFilter(lines, { ...EMPTY_LOG_FILTER, query: '/(/' }); + expect(result.queryError).toBeDefined(); + expect(result.lines).toHaveLength(4); + }); +}); + +describe('countLevels', () => { + it('counts under every filter but the level, so switching level shows what to expect', () => { + const lines = [ + line({ id: 1, server: 'a', level: 'error' }), + line({ id: 2, server: 'a', level: 'warn' }), + line({ id: 3, server: 'b', level: 'error' }), + ]; + const counts = countLevels(lines, { ...EMPTY_LOG_FILTER, server: 'a', minLevel: 'error' }); + expect(counts).toEqual({ debug: 0, info: 0, warn: 1, error: 1 }); + }); +}); + +describe('countLevels and stack frames', () => { + it('counts an exception once, not once per inherited frame', () => { + const lines = [ + line({ id: 1, level: 'error', levelKnown: true }), + line({ id: 2, level: 'error', levelKnown: false }), + line({ id: 3, level: 'error', levelKnown: false }), + ]; + expect(countLevels(lines, EMPTY_LOG_FILTER).error).toBe(1); + }); +}); + +describe('cycling', () => { + it('walks all -> each -> all in both directions', () => { + expect(cycleOption(['a', 'b'], null, 1)).toBe('a'); + expect(cycleOption(['a', 'b'], 'b', 1)).toBeNull(); + expect(cycleOption(['a', 'b'], null, -1)).toBe('b'); + }); + + it('restarts from "all" when the current value is gone', () => { + expect(cycleOption(['a'], 'gone', 1)).toBe('a'); + }); + + it('cycles the level all -> warn -> error -> all', () => { + expect(cycleMinLevel(null)).toBe('warn'); + expect(cycleMinLevel('warn')).toBe('error'); + expect(cycleMinLevel('error')).toBeNull(); + }); + + it('reports whether anything narrows the view', () => { + expect(isFiltering(EMPTY_LOG_FILTER)).toBe(false); + expect(isFiltering({ ...EMPTY_LOG_FILTER, query: 'x' })).toBe(true); + expect(isFiltering({ ...EMPTY_LOG_FILTER, mode: 'dim' })).toBe(false); + }); +}); + +describe('logFacets', () => { + const lines = [ + line({ id: 1, server: 'a', app: 'api', process: 'web' }), + line({ id: 2, server: 'a', app: 'api', process: 'worker' }), + line({ id: 3, server: 'b', app: 'site', process: null }), + ]; + + it('offers everything seen when nothing is chosen', () => { + expect(logFacets(lines, { server: null, app: null })).toEqual({ + servers: ['a', 'b'], + apps: ['api', 'site'], + processes: ['web', 'worker'], + }); + }); + + it('narrows apps and processes to the chosen server and app', () => { + expect(logFacets(lines, { server: 'b', app: null })).toEqual({ + servers: ['a', 'b'], + apps: ['site'], + processes: [], + }); + expect(logFacets(lines, { server: 'a', app: 'api' }).processes).toEqual(['web', 'worker']); + }); +}); + +// ── Sources ─────────────────────────────────────────────────────── + +describe('follow commands', () => { + it('follows PM2 with a backlog and no colour, quoting the namespace', () => { + const command = pm2FollowCommand("api'x", 20); + expect(command).toContain('pm2 logs'); + expect(command).toContain('--lines 20'); + expect(command).toContain('--no-color'); + expect(command).not.toContain("api'x "); + }); + + it('follows a systemd unit through journalctl -f', () => { + const command = journalFollowCommand('shipnode-api-web', 20); + expect(command).toContain('journalctl -u'); + expect(command).toContain('-f'); + expect(command).toContain('-n 20'); + }); + + it('follows Caddy across log rotation', () => { + expect(caddyFollowCommand('site', 20)).toContain('-F'); + expect(caddyFollowCommand('site', 20)).toContain('/var/log/caddy/site.log'); + }); +}); + +describe('planLogSources', () => { + const backend = { + name: 'api', + appType: 'backend', + pm2: { apps: [{ name: 'api' }, { name: 'worker' }] }, + } as unknown as ShipnodeApp; + + it('gives a PM2 app one stream that tags lines with the server and app', async () => { + const sources = await planLogSources(new FakeRemoteExecutor(), 'a', '/var/www', backend); + expect(sources).toHaveLength(1); + expect(sources[0]).toMatchObject({ server: 'a', app: 'api', kind: 'pm2' }); + expect(sources[0].command(20)).toContain("'api'"); + }); + + it('follows the access log of a frontend app', async () => { + const site = { name: 'site', appType: 'frontend' } as unknown as ShipnodeApp; + const [source] = await planLogSources(new FakeRemoteExecutor(), 'a', '/var/www', site); + expect(source.kind).toBe('caddy'); + expect(source.parse('{"x":1}')).toEqual({ process: null, text: '{"x":1}' }); + }); + + it('gives a watt app one stream per unit', async () => { + const watt = { + name: 'api', + appType: 'backend', + runtime: 'watt', + pm2: { apps: [{ name: 'api' }, { name: 'worker' }] }, + } as unknown as ShipnodeApp; + const sources = await planLogSources(new FakeRemoteExecutor(), 'a', '/var/www', watt); + expect(sources.map((s) => s.kind)).toEqual(['systemd', 'systemd']); + expect(sources[0].parse('hello')?.process).toBe('api'); + expect(sources[1].parse('hello')?.process).toBe('api-worker'); + }); + + it('has nothing to follow for a backend with no processes', async () => { + const empty = { name: 'x', appType: 'backend' } as unknown as ShipnodeApp; + expect(await planLogSources(new FakeRemoteExecutor(), 'a', '/var/www', empty)).toEqual([]); + }); +}); + +// ── Stream ──────────────────────────────────────────────────────── + +/** An executor whose commands the test drives by hand, like a remote that streams. */ +class StreamingExecutor extends RemoteExecutor { + commands: string[] = []; + options: Array = []; + private sessions: Array<{ emit: (chunk: string) => void; end: (exitCode: number) => void; fail: (e: Error) => void }> = []; + + async exec(command: string, options?: ExecOptions): Promise { + this.commands.push(command); + this.options.push(options); + return new Promise((resolve, reject) => { + const session = { + emit: (chunk: string) => options?.onData?.(chunk, 'stdout'), + end: (exitCode: number) => resolve({ stdout: '', stderr: '', exitCode }), + fail: (error: Error) => reject(error), + }; + this.sessions.push(session); + options?.signal?.addEventListener('abort', () => resolve({ stdout: '', stderr: '', exitCode: 143 }), { once: true }); + }); + } + + session(n: number) { + return this.sessions[n]; + } +} + +const tick = () => new Promise((resolve) => setTimeout(resolve, 5)); + +function makeStream(executors: Record, extra: Partial[0]> = {}) { + const bindings: LogBinding[] = Object.entries(executors).map(([server, executor]) => ({ + executor, + source: { + server, + app: 'api', + kind: 'pm2', + command: (n) => `follow ${server} ${n}`, + parse: parsePm2Line, + }, + })); + return new LogStream({ + bindings, + flushMs: 1, + liveAfterMs: 1, + backoffMs: () => 1, + ...extra, + }); +} + +describe('LogStream', () => { + it('merges servers into one buffer, tagging each line with its server', async () => { + const a = new StreamingExecutor(); + const b = new StreamingExecutor(); + const stream = makeStream({ a, b }); + stream.start(); + await tick(); + + a.session(0).emit('0|web | hello from a\n'); + b.session(0).emit('0|web | hello from b\n'); + await tick(); + + expect(stream.lines().map((l) => [l.server, l.process, l.text])).toEqual([ + ['a', 'web', 'hello from a'], + ['b', 'web', 'hello from b'], + ]); + await stream.stop(); + }); + + it('follows over a pty and asks for backlog on connect', async () => { + const a = new StreamingExecutor(); + const stream = makeStream({ a }, { backlog: 7 }); + stream.start(); + await tick(); + expect(a.commands[0]).toBe('follow a 7'); + expect(a.options[0]?.pty).toBe(true); + await stream.stop(); + }); + + it('reassembles a line split across chunks', async () => { + const a = new StreamingExecutor(); + const stream = makeStream({ a }); + stream.start(); + await tick(); + a.session(0).emit('0|web | split '); + a.session(0).emit('line\n'); + await tick(); + expect(stream.lines().map((l) => l.text)).toEqual(['split line']); + await stream.stop(); + }); + + it('classifies levels and lets a stack frame inherit the error above it', async () => { + const a = new StreamingExecutor(); + const stream = makeStream({ a }); + stream.start(); + await tick(); + a.session(0).emit('0|web | TypeError: boom\n0|web | at run (/app/x.js:1:1)\n0|web | ready\n'); + await tick(); + expect(stream.lines().map((l) => l.level)).toEqual(['error', 'error', 'info']); + expect(stream.lines().map((l) => l.levelKnown)).toEqual([true, false, false]); + await stream.stop(); + }); + + it('reconnects after the stream ends, without replaying lines already shown', async () => { + const a = new StreamingExecutor(); + const stream = makeStream({ a }); + stream.start(); + await tick(); + + a.session(0).emit('0|web | one\n0|web | two\n'); + a.session(0).end(255); + await tick(); + await tick(); + + expect(a.commands).toHaveLength(2); + a.session(1).emit('0|web | one\n0|web | two\n0|web | three\n'); + await tick(); + + expect(stream.lines().map((l) => l.text)).toEqual(['one', 'two', 'three']); + await stream.stop(); + }); + + it('reports a failing source as reconnecting with the reason, then live again', async () => { + const a = new StreamingExecutor(); + const stream = makeStream({ a }, { liveAfterMs: 1000 }); + stream.start(); + await tick(); + + a.session(0).fail(new Error('connection reset')); + await tick(); + expect(stream.health()[0]).toMatchObject({ state: 'reconnecting', error: 'connection reset' }); + + await tick(); + a.session(1).emit('0|web | back\n'); + await tick(); + expect(stream.health()[0]).toMatchObject({ state: 'live' }); + expect(stream.health()[0].error).toBeUndefined(); + await stream.stop(); + }); + + it('keeps one source failing from silencing the others', async () => { + const a = new StreamingExecutor(); + const b = new StreamingExecutor(); + const stream = makeStream({ a, b }); + stream.start(); + await tick(); + a.session(0).fail(new Error('down')); + b.session(0).emit('0|web | still here\n'); + await tick(); + expect(stream.lines().map((l) => l.text)).toEqual(['still here']); + await stream.stop(); + }); + + it('bounds the buffer, dropping the oldest', async () => { + const a = new StreamingExecutor(); + const stream = makeStream({ a }, { capacity: 3 }); + stream.start(); + await tick(); + a.session(0).emit([1, 2, 3, 4, 5].map((n) => `0|web | m${n}\n`).join('')); + await tick(); + expect(stream.lines().map((l) => l.text)).toEqual(['m3', 'm4', 'm5']); + await stream.stop(); + }); + + it('stops by aborting every channel and stays stopped', async () => { + const a = new StreamingExecutor(); + const stream = makeStream({ a }); + stream.start(); + await tick(); + await stream.stop(); + expect(a.options[0]?.signal?.aborted).toBe(true); + expect(stream.isRunning).toBe(false); + await tick(); + expect(a.commands).toHaveLength(1); + }); + + it('clear drops the buffer but keeps following', async () => { + const a = new StreamingExecutor(); + const stream = makeStream({ a }); + stream.start(); + await tick(); + a.session(0).emit('0|web | old\n'); + await tick(); + stream.clear(); + expect(stream.lines()).toEqual([]); + a.session(0).emit('0|web | new\n'); + await tick(); + expect(stream.lines().map((l) => l.text)).toEqual(['new']); + await stream.stop(); + }); + + it('notifies subscribers in batches rather than per line', async () => { + const a = new StreamingExecutor(); + const stream = makeStream({ a }, { flushMs: 20 }); + let notifications = 0; + stream.subscribe(() => { + notifications += 1; + }); + stream.start(); + await tick(); + const before = notifications; + a.session(0).emit(Array.from({ length: 50 }, (_, i) => `0|web | m${i}\n`).join('')); + await new Promise((resolve) => setTimeout(resolve, 40)); + expect(notifications - before).toBeLessThanOrEqual(2); + expect(stream.lines()).toHaveLength(50); + await stream.stop(); + }); +}); diff --git a/website/src/components/Commands.astro b/website/src/components/Commands.astro index 5372041..0e1d702 100644 --- a/website/src/components/Commands.astro +++ b/website/src/components/Commands.astro @@ -21,7 +21,8 @@ const groups = [ label: "Operate", commands: [ ["shipnode status", "Inspect PM2 and release state"], - ["shipnode logs", "Stream process logs"], + ["shipnode logs --follow", "Stream and filter logs from every server"], + ["shipnode monitor", "Live dashboard across every server"], ["shipnode metrics", "Open the PM2 monitoring dashboard"], ["shipnode restart", "Restart the application"], ["shipnode stop", "Stop the application"], diff --git a/website/src/content/docs/docs/commands/logs.md b/website/src/content/docs/docs/commands/logs.md index 5f062f4..ca42303 100644 --- a/website/src/content/docs/docs/commands/logs.md +++ b/website/src/content/docs/docs/commands/logs.md @@ -1,18 +1,39 @@ --- title: shipnode logs -description: Stream PM2 logs from the server. +description: Show or stream application logs from every server, with filters. --- ```bash -npx shipnode logs +npx shipnode logs # last 100 lines per app, per server npx shipnode logs --lines 500 -npx shipnode logs --process mailer +npx shipnode logs --process mailer --app api +npx shipnode logs --follow # stream every server live, merged +npx shipnode logs -f --level error # only errors, from the whole fleet +npx shipnode logs -f --grep '/timeout|ECONN/i' --app api --on web-2 ``` +Without `--follow` the command prints recent history and exits. With it, every +server's logs are streamed over the SSH connections shipnode already opens and +merged into one stream until you press Ctrl-C. Lines are prefixed with the +server and app they came from (only the parts that vary), so a fleet's output +stays attributable. Log lines go to stdout and status to stderr, so +`shipnode logs -f | grep …` stays clean. + +A dropped connection is retried with backoff, and the history the server replays +on reconnect is not printed twice. + ## Options | Flag | Purpose | |---|---| -| `--lines ` | Number of historical lines to fetch (default `100`). | -| `--process ` | Restrict to a single worker (use the short name from your config). | +| `--lines ` | Lines of history to fetch (default `100`). With `--follow`, the backlog shown on connect. | +| `-f, --follow` | Stream new lines from every server until Ctrl-C. | +| `--level ` | Only this level and above: `warn` or `error`. Levels are read from JSON logs (`level`, `severity`), `[ERROR]`-style tags and plain `ERROR`/`WARN` words. | +| `--grep ` | Only matching lines. Plain text is a case-insensitive substring; `/regex/` (optionally `/regex/i`) is a regular expression; a leading `!` excludes. | +| `--process ` | Restrict to a single worker (use the short name from your config). Needs `--app`. | +| `--app ` | Restrict to one app. | +| `--on ` | Restrict to one server. | | `--config ` | Use a non-default config file. | + +Frontend apps stream the Caddy access log; an access-log line counts as `warn` +for a 4xx status and `error` for 5xx. diff --git a/website/src/content/docs/docs/commands/monitor.md b/website/src/content/docs/docs/commands/monitor.md new file mode 100644 index 0000000..d5e0a2a --- /dev/null +++ b/website/src/content/docs/docs/commands/monitor.md @@ -0,0 +1,65 @@ +--- +title: shipnode monitor +description: Live terminal dashboard across every server, with filterable streaming logs. +--- + +```bash +npx shipnode monitor # the whole workspace +npx shipnode monitor --app api # one app, on every server it runs on +npx shipnode monitor --on web-2 # one server +npx shipnode monitor --once # one snapshot, then exit +npx shipnode monitor --json # the same snapshot as JSON +``` + +The monitor holds one SSH connection per server and shows three views. + +**Fleet overview** lists every app on every server it runs on: the release each +replica serves, whether processes are up, CPU and memory, the health check, and +one sentence per app saying whether the fleet agrees. A half-finished rolling +deploy shows as `split across 2 releases`, the replica that missed it is marked +`behind`, and an unreachable server keeps its row. Below it, each server's CPU, +memory and disk, and any deploy lock. `↑/↓` selects, `Enter` opens a replica. + +**Replica detail** is the per-process view: PM2 or systemd processes, system +gauges, accessories and releases. `Enter` on a process restarts it. `Enter` on a +release rolls back, except for an app that runs on several servers: rolling back +one replica would split the fleet, so use `shipnode rollback` for those. + +**Logs** (`f`) streams every server's logs live into one merged view. Opening it +from the overview scopes it to the selected app; from a replica, to that app on +that server. Filters apply to what is already in the buffer, so changing them +never reconnects and never loses a line that was only hidden. + +| Key | Action | +|---|---| +| `s` / `S` | Filter by server (next / previous), cycling through *all* | +| `a` / `A` | Filter by app | +| `p` / `P` | Filter by process | +| `v` | Level: all → warn and above → error only | +| `/` | Search: plain text, `/regex/`, or `!text` to exclude | +| `m` | Search mode: hide non-matching lines, or keep them dimmed for context | +| `c` | Clear filters | +| `space` | Pause and resume | +| `↑/↓`, `PgUp/PgDn` | Scroll back (pauses); `G` returns to the newest line | +| `C` | Clear the buffer | +| `Esc` / `f` | Back | + +The filter bar shows live `ERR` and `WARN` counts under the current server, app +and search filters, so you can see what a level filter would show before you +switch to it. A source that stops delivering is retried with backoff and flagged +in the view. `l` toggles a small log strip under the overview and detail views. + +The buffer keeps the newest 2,000 lines across all servers. Lines are ordered +by when they arrived here, not by the servers' clocks, which are not trusted to +agree. + +## Options + +| Flag | Purpose | +|---|---| +| `--interval ` | Polling interval (default `2`). | +| `--app ` | Watch one app. | +| `--on ` | Watch one server instead of the whole fleet. | +| `--once` | Collect one snapshot and exit. | +| `--json` | Print one snapshot as JSON (implies `--once`). | +| `--config ` | Use a non-default config file. | From 81b13c596e9d9eb0495e4addb0ca9b167260fef4 Mon Sep 17 00:00:00 2001 From: Alade YESSOUFOU Date: Fri, 2 Oct 2026 20:13:16 +0200 Subject: [PATCH 3/4] feat(monitor): redesign the TUI around a small visual system - theme tokens: neutral chrome, colour only for state, one accent; no background fills so light terminals work - Panel with the title set into the border; aligned tables with headings - responsive columns: tables drop low-priority columns to fit instead of overflowing into the next panel - breadcrumb header with freshness (live/stale) and stream health, a dedicated alert line, and a key-hint footer that flashes action results - fleet: Apps table grouped by app with every problem in the verdict, Servers table with meters; detail: process table with per-process details and trends, compact Host panel, Releases table - logs: timestamps, short source labels, filter chips and counts, clear empty states; two-column help and centred confirm dialog --- CHANGELOG.md | 1 + src/cli/monitor/App.tsx | 252 +++++++++------- src/cli/monitor/components/ConfirmDialog.tsx | 36 +-- src/cli/monitor/components/HelpOverlay.tsx | 90 +++--- src/cli/monitor/components/KeyHints.tsx | 19 ++ src/cli/monitor/components/Panel.tsx | 70 +++++ src/cli/monitor/components/Table.tsx | 77 +++++ src/cli/monitor/components/charts.tsx | 20 +- src/cli/monitor/fleet-model.ts | 22 +- src/cli/monitor/hooks/use-now.ts | 11 + src/cli/monitor/layout/HeaderBar.tsx | 33 --- src/cli/monitor/layout/MonitorFrame.tsx | 111 +++++-- src/cli/monitor/layout/StatusBar.tsx | 27 -- src/cli/monitor/panels/AccessoriesPanel.tsx | 54 ++-- src/cli/monitor/panels/EventsPanel.tsx | 33 ++- src/cli/monitor/panels/FleetPanel.tsx | 272 +++++++++++------- src/cli/monitor/panels/LogViewer.tsx | 165 +++++++---- src/cli/monitor/panels/Pm2Panel.tsx | 185 ++++++------ src/cli/monitor/panels/ReleasePanel.tsx | 90 +++--- .../monitor/panels/StaticFrontendPanel.tsx | 93 +++--- src/cli/monitor/panels/SystemPanel.tsx | 110 +++---- src/cli/monitor/theme.ts | 45 +++ tests/unit/monitor-fleet.test.ts | 42 ++- tests/unit/monitor.test.ts | 10 +- 24 files changed, 1154 insertions(+), 714 deletions(-) create mode 100644 src/cli/monitor/components/KeyHints.tsx create mode 100644 src/cli/monitor/components/Panel.tsx create mode 100644 src/cli/monitor/components/Table.tsx create mode 100644 src/cli/monitor/hooks/use-now.ts delete mode 100644 src/cli/monitor/layout/HeaderBar.tsx delete mode 100644 src/cli/monitor/layout/StatusBar.tsx create mode 100644 src/cli/monitor/theme.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b7414b..865f27a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to `@devalade/shipnode` will be documented here. ### Added - **`monitor` watches the whole fleet.** With no `--on` it connects to every server and opens a fleet overview: each app on each server it runs on, the release each replica serves, process and health state, and a one-line verdict per app (a half-finished roll reads `split across 2 releases` and marks the replica that is `behind`; an unreachable server keeps its row). `Enter` opens the per-replica view the old single-server monitor showed. `--on` and `--app` still narrow it. Rollback from the monitor is refused for an app that runs on several servers, since it would split the fleet; use `shipnode rollback`. - **Live log streaming with filters.** `f` opens a merged, streaming view of every server's logs (PM2, systemd and Caddy access logs) over the existing SSH connections, replacing the 2-second `pm2 logs --nostream` poll and its duplicated or dropped lines. Filter by server (`s`), app (`a`), process (`p`), level (`v`: all, warn+, error) and text, `/regex/` or `!exclude` search (`/`), in hide or dim mode (`m`), with live `ERR`/`WARN` counts. Pause, scroll back, and clear are supported. A dropped connection is retried with backoff without replaying lines already shown. +- **The monitor has a consistent visual design.** Panels carry their title in the border, data sits in aligned tables with column headings, and colour is reserved for state (green ok, amber degraded, red failing) with one accent for titles and selection. A breadcrumb header shows where you are and how fresh the data is (`live`, `stale` once polls stop arriving), alerts get their own line, and the footer lists the keys for the current view and briefly confirms actions. Tables drop their least important columns on narrow terminals instead of overflowing, and nothing paints a background, so light terminal themes work. - **`shipnode logs --follow`** streams the same merged logs to the terminal, with `--level`, `--grep`, `--app`, `--on` and `--process`. `--level` and `--grep` also filter the one-shot `logs` output. ### Changed diff --git a/src/cli/monitor/App.tsx b/src/cli/monitor/App.tsx index f8d6b12..86e8bdb 100644 --- a/src/cli/monitor/App.tsx +++ b/src/cli/monitor/App.tsx @@ -1,4 +1,4 @@ -import { Box, useApp, useInput, useStdout } from 'ink'; +import { Box, Text, useApp, useInput, useStdout } from 'ink'; import { useEffect, useReducer, useRef, useState } from 'react'; import { HEALTH_ALERT_THRESHOLD } from '../../services/observe/session.js'; import type { ObserveEvent } from '../../services/observe/events.js'; @@ -6,17 +6,19 @@ import { applyLogFilter, countLevels, logFacets } from '../../domain/observe/log import type { FleetConnection } from '../observe.js'; import { Pm2Panel } from './panels/Pm2Panel.js'; import { SystemPanel } from './panels/SystemPanel.js'; -import { ReleasePanel } from './panels/ReleasePanel.js'; +import { ReleasePanel, releasePanelHeight } from './panels/ReleasePanel.js'; import { EventsPanel } from './panels/EventsPanel.js'; -import { FleetPanel } from './panels/FleetPanel.js'; -import { LogViewer } from './panels/LogViewer.js'; +import { AppsPanel, ServersPanel, serversPanelHeight } from './panels/FleetPanel.js'; +import { LOG_VIEWER_CHROME, LogViewer } from './panels/LogViewer.js'; import { StaticFrontendPanel } from './panels/StaticFrontendPanel.js'; import { AccessoriesPanel } from './panels/AccessoriesPanel.js'; import { HelpOverlay } from './components/HelpOverlay.js'; import { ConfirmDialog } from './components/ConfirmDialog.js'; import { restartProcess, rollbackToRelease } from './actions.js'; -import { MonitorFrame, WaitingPanel } from './layout/MonitorFrame.js'; -import type { HeaderAlert } from './layout/HeaderBar.js'; +import { MonitorFrame, type Flash, type HeaderAlert } from './layout/MonitorFrame.js'; +import { Panel } from './components/Panel.js'; +import type { Hint } from './components/KeyHints.js'; +import type { Tone } from './theme.js'; import { buildFleetRows, toMetricsSnapshot } from './fleet-model.js'; import { INITIAL_LOG_VIEW, logViewReducer } from './log-view-state.js'; import { useFleet } from './hooks/use-fleet.js'; @@ -25,10 +27,12 @@ import { useLogStream } from './hooks/use-log-stream.js'; type View = 'fleet' | 'detail' | 'logs'; type Overlay = 'none' | 'help' | 'confirmRestart' | 'confirmRollback'; -/** Rows the logs view spends on everything but log lines: header, status, border, title, filter bar, source warning. */ -const LOGS_CHROME_ROWS = 9; +/** Lines of log or activity in the strip under the fleet and replica views. */ const STRIP_ROWS = 5; +/** The strip's panel: its lines plus top and bottom edges. */ +const BOTTOM_HEIGHT = STRIP_ROWS + 2; const MAX_ALERTS = 3; +const FLASH_MS = 4000; interface AppProps { fleet: FleetConnection; @@ -49,10 +53,29 @@ export function App({ fleet: connection, interval, focusApp }: AppProps) { const [detailKey, setDetailKey] = useState(null); const [detailRow, setDetailRow] = useState(0); const [logView, dispatchLog] = useReducer(logViewReducer, INITIAL_LOG_VIEW); + const [flash, setFlash] = useState(null); + const flashTimer = useRef | null>(null); + + /** Say something in the footer for a moment. */ + const showFlash = (next: Flash): void => { + setFlash(next); + if (flashTimer.current !== null) clearTimeout(flashTimer.current); + flashTimer.current = setTimeout(() => setFlash(null), FLASH_MS); + }; + + /** Flash it and keep it in the activity log. */ + const notify = (tone: Tone, text: string): void => { + session.notice(text); + showFlash({ tone, text }); + }; + useEffect(() => () => { + if (flashTimer.current !== null) clearTimeout(flashTimer.current); + }, []); const rows = buildFleetRows(state.fleets); const logs = useLogStream(connection.hosts, stripOn || view === 'logs'); const terminalRows = stdout?.rows ?? 24; + const terminalCols = stdout?.columns ?? 100; // ── Fleet selection ────────────────────────────────────────────── const selectedAt = Math.max(0, rows.findIndex((row) => row.key === selectedKey)); @@ -89,7 +112,6 @@ export function App({ fleet: connection, interval, focusApp }: AppProps) { const selectedProcess = detailIndex < processes.length ? processes[detailIndex] : undefined; const selectedRelease = detailIndex >= processes.length ? releaseRows[detailIndex - processes.length] : undefined; const currentTimestamp = detailSnapshot?.currentRelease?.split('/').pop() ?? null; - const releaseBoxExtra = Math.max(0, Math.min(6, terminalRows - 30)); // ── Logs ───────────────────────────────────────────────────────── const source = logView.frozen ?? logs.lines; @@ -100,22 +122,26 @@ export function App({ fleet: connection, interval, focusApp }: AppProps) { const showServer = serverNames.length > 1; const showApp = new Set(logs.lines.map((line) => line.app)).size > 1; const troubled = logs.health.filter((entry) => entry.state !== 'live'); - const logHeight = Math.max(3, terminalRows - LOGS_CHROME_ROWS); - const maxScroll = Math.max(0, filtered.lines.length - logHeight); // ── Alerts ─────────────────────────────────────────────────────── const alerts: HeaderAlert[] = []; for (const server of state.servers) { - if (server.error !== undefined) alerts.push({ text: `${server.server} DOWN` }); - else if (server.deployLock != null) alerts.push({ text: `DEPLOY LOCK ${server.server} (${server.deployLock.ageSeconds}s)` }); + if (server.error !== undefined) alerts.push({ text: `${server.server} unreachable` }); + else if (server.deployLock != null) alerts.push({ text: `${server.server} deploy lock ${server.deployLock.ageSeconds}s` }); } for (const row of state.fleets.flatMap((f) => f.replicas.map((r) => ({ app: f.app, server: r.server })))) { const streak = session.healthFailStreak(row.server, row.app); - if (streak >= HEALTH_ALERT_THRESHOLD) alerts.push({ text: `HEALTH ${row.app}@${row.server} ×${streak}` }); + if (streak >= HEALTH_ALERT_THRESHOLD) alerts.push({ text: `${row.app} on ${row.server} health failing ×${streak}` }); } const shownAlerts = alerts.slice(0, MAX_ALERTS); if (alerts.length > MAX_ALERTS) shownAlerts.push({ text: `+${alerts.length - MAX_ALERTS} more` }); + // ── Height budget ──────────────────────────────────────────────── + // Every view gets the terminal minus the header, the alert bar when shown, and the footer. + const bodyHeight = Math.max(8, terminalRows - 2 - (shownAlerts.length > 0 ? 1 : 0)); + const logHeight = Math.max(3, bodyHeight - LOG_VIEWER_CHROME); + const maxScroll = Math.max(0, filtered.lines.length - logHeight); + // Ring the terminal bell once when a health check starts failing. const lastEvent = useRef(undefined); useEffect(() => { @@ -129,18 +155,20 @@ export function App({ fleet: connection, interval, focusApp }: AppProps) { const confirmRestart = async (): Promise => { setOverlay('none'); if (selectedProcess === undefined || detailHost?.executor == null) return; - session.notice(`Restarting ${selectedProcess.pm2Name} on ${detailHost.name}…`); + notify('info', `Restarting ${selectedProcess.pm2Name} on ${detailHost.name}…`); const result = await restartProcess(detailHost.executor, selectedProcess.pm2Name, selectedProcess.supervisor); - session.notice(result.isOk() ? `Restarted ${selectedProcess.pm2Name} on ${detailHost.name}` : result.error.message); + if (result.isOk()) notify('ok', `Restarted ${selectedProcess.pm2Name} on ${detailHost.name}`); + else notify('bad', result.error.message); void refresh(); }; const confirmRollback = async (): Promise => { setOverlay('none'); if (selectedRelease === undefined || detailHost?.executor == null || detailApp === undefined) return; - session.notice(`Rolling back ${detailApp.name} on ${detailHost.name} to ${selectedRelease.timestamp}…`); + notify('info', `Rolling back ${detailApp.name} on ${detailHost.name} to ${selectedRelease.timestamp}…`); const result = await rollbackToRelease(detailHost.executor, detailHost.config, detailApp, selectedRelease.timestamp); - session.notice(result.isOk() ? `Rolled back to ${selectedRelease.timestamp}` : result.error.message); + if (result.isOk()) notify('ok', `Rolled back ${detailApp.name} to ${selectedRelease.timestamp}`); + else notify('bad', result.error.message); void refresh(); }; @@ -184,7 +212,7 @@ export function App({ fleet: connection, interval, focusApp }: AppProps) { } if (input === 'r' || input === 'R') { void refresh(); - session.notice('Refresh triggered'); + showFlash({ tone: 'muted', text: 'refreshing…' }); return; } @@ -237,21 +265,21 @@ export function App({ fleet: connection, interval, focusApp }: AppProps) { if (key.downArrow) return setDetailRow(Math.min(Math.max(totalDetailRows - 1, 0), detailIndex + 1)); if (key.return || input === 'x' || input === 'X') { if (detailSnapshot?.deployLock != null) { - session.notice(`Blocked: a deploy is in progress on ${detail?.server} (lock held)`); + notify('bad', `A deploy holds the lock on ${detail?.server} - try again when it finishes`); return; } if (selectedProcess !== undefined) return setOverlay('confirmRestart'); if (selectedRelease !== undefined) { if (detail?.replicated === true) { - session.notice(`Rollback on one replica of ${detail.app} would split the fleet - use 'shipnode rollback'`); + notify('warn', `Rolling back one replica would split ${detail.app} - use 'shipnode rollback'`); return; } if (selectedRelease.status !== 'success') { - session.notice('Cannot roll back to a failed release'); + notify('warn', 'That release failed to deploy - pick a successful one'); return; } if (selectedRelease.timestamp === currentTimestamp) { - session.notice(`${selectedRelease.timestamp} is already the current release`); + notify('muted', `${selectedRelease.timestamp} is already current`); return; } setOverlay('confirmRollback'); @@ -294,107 +322,115 @@ export function App({ fleet: connection, interval, focusApp }: AppProps) { } // ── Views ──────────────────────────────────────────────────────── - const unreachable = state.servers.filter((server) => server.error !== undefined).length; const frame = { interval, - liveMode: stripOn || view === 'logs', - alerts: shownAlerts, - lastUpdate: state.lastUpdate, polling: state.polling, + lastUpdate: state.lastUpdate, + streaming: logs.health.length > 0 ? { live: logs.health.length - troubled.length, total: logs.health.length } : null, + alerts: shownAlerts, + flash, + }; + + const viewerProps = { + matched: filtered.matched, + filter: logView.filter, + total: source.length, + servers: serverNames, + showServer, + showApp, + frozen: logView.frozen !== null, + health: logs.health, }; if (view === 'logs') { - const scope = [logView.filter.server, logView.filter.app].filter((part) => part !== null).join(' / '); + const scope = [logView.filter.server, logView.filter.app].filter((part): part is string => part !== null); + const hints: Hint[] = logView.typing + ? [['⏎', 'keep search'], ['esc', 'drop search']] + : [['s a p', 'server·app·proc'], ['v', 'level'], ['/', 'search'], ['space', logView.frozen ? 'resume' : 'pause'], ['c', 'clear'], ['esc', 'back']]; return ( - 0 ? `no logs from ${logs.skipped.join(', ')}` : null} - > - - - + 0 ? scope : ['everything'])]} hints={hints}> + ); } const bottom = stripOn ? ( - + ) : ( - + ); if (view === 'fleet') { + const serversHeight = serversPanelHeight(connection.hosts.length); + // The apps panel takes what is left; its body loses the two edges and the column header. + const appsBody = bodyHeight - serversHeight - BOTTOM_HEIGHT - 3; return ( 0 ? ` · ${unreachable} down` : ''}`} - hints="↑/↓ select Enter open f logs l strip" + crumbs={['fleet']} + hints={[['↑↓', 'select'], ['⏎', 'open'], ['f', 'logs'], ['l', stripOn ? 'activity' : 'log strip'], ['r', 'refresh']]} > - - + + + - {bottom} + {bottom} ); } // detail - const unavailable = detail?.reachable === false ? (detail.error ?? 'server unreachable') : null; + const crumbs = detail === undefined ? ['fleet', 'replica'] : ['fleet', detail.app, detail.server]; + const accessories = detailHost?.accessoryNames ?? []; + const rollbackNote = detail?.replicated === true ? 'roll back fleets with shipnode rollback' : undefined; + const hints: Hint[] = [ + ['↑↓', 'select'], + ['⏎', selectedRelease !== undefined ? 'roll back' : 'restart'], + ['f', 'logs'], + ['l', stripOn ? 'activity' : 'log strip'], + ['esc', 'fleet'], + ]; + + if (detail !== undefined && !detail.reachable) { + return ( + + + {detail.error ?? 'The last poll could not reach this server.'} + The overview keeps polling; this view fills in once the server answers. + + + ); + } + + // The host column is fixed; the process table takes the rest and drops columns to fit it. + const sideWidth = terminalCols >= 120 ? 46 : terminalCols >= 100 ? 40 : 34; + const mainWidth = terminalCols - sideWidth; + + const waiting = ( + + Waiting for the first poll… + + ); + return ( - + - + {detailApp?.appType === 'frontend' ? ( ) : detailSnapshot && detailHistory ? ( ) : ( - - )} - - - {detailSnapshot && detailHistory ? ( - <> - - {(detailHost?.accessoryNames.length ?? 0) > 0 && ( - - )} - - ) : ( - + waiting )} + {detailSnapshot && detail !== undefined && ( + + + + + {accessories.length > 0 && ( + + + + )} + + )} {detailSnapshot && ( - 0 ? 6 : 7) + releaseBoxExtra}> + )} - {bottom} + {bottom} ); } diff --git a/src/cli/monitor/components/ConfirmDialog.tsx b/src/cli/monitor/components/ConfirmDialog.tsx index 361d361..b7745d8 100644 --- a/src/cli/monitor/components/ConfirmDialog.tsx +++ b/src/cli/monitor/components/ConfirmDialog.tsx @@ -1,7 +1,7 @@ import { Box, Text, useInput } from 'ink'; - -const BG = '#0d1117'; -const ACCENT = '#d6a85d'; +import { Panel } from './Panel.js'; +import { KeyHints } from './KeyHints.js'; +import { color } from '../theme.js'; interface ConfirmDialogProps { title: string; @@ -20,26 +20,16 @@ export function ConfirmDialog({ title, lines, onConfirm, onCancel }: ConfirmDial }); return ( - - {title} - {lines.map((line, i) => ( - {line} - ))} - - - y/Enter - confirm · - n/Esc - cancel - - + + + {title} + {lines.map((line, i) => ( + {line} + ))} + + + + ); } diff --git a/src/cli/monitor/components/HelpOverlay.tsx b/src/cli/monitor/components/HelpOverlay.tsx index 93e194a..cf0c6e8 100644 --- a/src/cli/monitor/components/HelpOverlay.tsx +++ b/src/cli/monitor/components/HelpOverlay.tsx @@ -1,80 +1,84 @@ import { Box, Text } from 'ink'; - -const BG = '#0d1117'; -const ACCENT = '#d6a85d'; +import { Panel } from './Panel.js'; +import { color } from '../theme.js'; type Section = { title: string; bindings: Array<[string, string]> }; -const SECTIONS: Section[] = [ +const LEFT: Section[] = [ { title: 'Everywhere', bindings: [ - ['q', 'quit'], + ['f', 'logs, scoped to the selection'], + ['l', 'log strip under the view'], ['r', 'refresh now'], - ['f', 'full-screen logs (scoped to what is selected)'], - ['l', 'toggle the live log strip'], ['?', 'this help'], + ['q', 'quit'], ], }, { - title: 'Fleet overview', + title: 'Fleet', bindings: [ - ['↑/↓', 'select an app on a server'], - ['Enter', 'open that replica'], + ['↑ ↓', 'select a replica'], + ['⏎', 'open it'], ], }, { - title: 'Replica detail', + title: 'Replica', bindings: [ - ['Esc', 'back to the fleet'], - ['↑/↓', 'select process or release'], - ['Enter/x', 'restart process / roll back (single-server apps)'], + ['↑ ↓', 'select process or release'], + ['⏎ x', 'restart / roll back'], + ['esc', 'back to the fleet'], ], }, +]; + +const RIGHT: Section[] = [ { title: 'Logs', bindings: [ - ['s / S', 'filter by server (next / previous)'], - ['a / A', 'filter by app'], - ['p / P', 'filter by process'], - ['v', 'level: all → warn+ → error'], + ['s S', 'server: next / previous'], + ['a A', 'app'], + ['p P', 'process'], + ['v', 'level: all, ≥warn, error'], ['/', 'search: text, /regex/, !not'], - ['m', 'search mode: hide others / dim others'], + ['m', 'hide or dim non-matches'], ['c', 'clear filters'], - ['space', 'pause / resume'], - ['↑/↓ PgUp/PgDn', 'scroll back (pauses)'], - ['G', 'jump to newest and follow'], - ['C', 'clear the buffer'], - ['Esc / f', 'back'], + ['space', 'pause'], + ['↑ ↓ pgup pgdn', 'scroll back'], + ['G', 'follow newest'], + ['C', 'clear buffer'], + ['esc f', 'back'], ], }, ]; -export function HelpOverlay() { +function Column({ sections }: { sections: Section[] }) { return ( - - Keybindings - {SECTIONS.map((section) => ( - - {section.title} + + {sections.map((section) => ( + + {section.title} {section.bindings.map(([keys, description]) => ( - - {keys.padEnd(15)} - {description} + + {keys} + {description} ))} ))} - - press any key to close - + + ); +} + +export function HelpOverlay() { + return ( + + any key closes} width={96} focused> + + + + + ); } diff --git a/src/cli/monitor/components/KeyHints.tsx b/src/cli/monitor/components/KeyHints.tsx new file mode 100644 index 0000000..8d65956 --- /dev/null +++ b/src/cli/monitor/components/KeyHints.tsx @@ -0,0 +1,19 @@ +import { Text } from 'ink'; +import { Fragment } from 'react'; + +export type Hint = readonly [keys: string, label: string]; + +/** `⏎ open ↑↓ select f logs` - keys bright, labels dim, the k9s / lazygit footer. */ +export function KeyHints({ hints }: { hints: readonly Hint[] }) { + return ( + + {hints.map(([keys, label], index) => ( + + {index > 0 && {' '}} + {keys} + {label} + + ))} + + ); +} diff --git a/src/cli/monitor/components/Panel.tsx b/src/cli/monitor/components/Panel.tsx new file mode 100644 index 0000000..d6ad15c --- /dev/null +++ b/src/cli/monitor/components/Panel.tsx @@ -0,0 +1,70 @@ +import { Box, Text } from 'ink'; +import type { ReactNode } from 'react'; +import { color } from '../theme.js'; + +/** Long enough for any terminal; clipped to the panel's width by overflow. */ +const RULE = '─'.repeat(512); + +interface PanelProps { + title: string; + /** Short, dim context after the title: "2 servers", "health 200 · 12ms". */ + subtitle?: ReactNode; + /** Right-aligned in the top border. */ + right?: ReactNode; + /** The panel the keys act on gets an accent border. */ + focused?: boolean; + width?: number | string; + height?: number; + flexGrow?: number; + children?: ReactNode; +} + +/** Columns a panel's border and padding take from its width. */ +export const PANEL_INSET = 4; + +/** + * A rounded box with its title set into the top border, the way a tiling + * terminal app labels panes. Ink has no titled border, so the top edge is drawn + * by hand: a rule clipped by overflow to whatever width the layout grants. + */ +export function Panel({ title, subtitle, right, focused = false, width, height, flexGrow, children }: PanelProps) { + const edge = focused ? color.accent : color.border; + return ( + + + + ╭─ + {title} + + {/* When the edge is crowded the subtitle gives way first, then the rule. */} + {subtitle !== undefined && ( + + {subtitle} + + )} + + + {RULE} + + {right !== undefined && ( + + {right} + + )} + + ─╮ + + + + {children} + + + ); +} diff --git a/src/cli/monitor/components/Table.tsx b/src/cli/monitor/components/Table.tsx new file mode 100644 index 0000000..35a8723 --- /dev/null +++ b/src/cli/monitor/components/Table.tsx @@ -0,0 +1,77 @@ +import { Box, Text } from 'ink'; +import type { ReactNode } from 'react'; + +export interface Column { + key: string; + header: string; + width: number; + align?: 'left' | 'right'; + /** + * Narrow terminals drop columns rather than overflow: higher numbers go + * first. Columns without a priority are the ones the table is for and stay. + */ + drop?: number; +} + +/** + * The columns that fit in `available` cells after `reserved` (gutters). + * + * Ink does not reliably clip text that overruns its box - it bleeds into the + * neighbouring panel - so a table must never be laid out wider than it is. + */ +export function fitColumns(columns: readonly Column[], available: number, reserved: number = 0): Column[] { + let kept = [...columns]; + const total = (): number => reserved + kept.reduce((sum, column) => sum + column.width, 0); + const droppable = columns + .filter((column) => column.drop !== undefined) + .sort((a, b) => (b.drop ?? 0) - (a.drop ?? 0)); + for (const column of droppable) { + if (total() <= available) break; + kept = kept.filter((candidate) => candidate !== column); + } + // Still too wide with only the essentials: the first column (a name) gives up the difference. + const over = total() - available; + if (over > 0 && kept.length > 0) { + kept[0] = { ...kept[0], width: Math.max(MIN_FIRST_COLUMN, kept[0].width - over) }; + } + return kept; +} + +const MIN_FIRST_COLUMN = 8; + +/** One fixed-width cell. Content that does not fit is cut, never wrapped, so rows stay one line. */ +export function Cell({ width, align = 'left', children }: { width: number; align?: 'left' | 'right'; children: ReactNode }) { + return ( + + {children} + + ); +} + +/** Dim uppercase column headings aligned to the cells below. `lead` matches the rows' marker gutter. */ +export function TableHeader({ columns, lead = 0 }: { columns: readonly Column[]; lead?: number }) { + return ( + + {lead > 0 && } + {columns.map((column) => ( + + {column.header} + + ))} + + ); +} + +/** A row laid out by `columns`, taking each cell's content from `cells` by key. */ +export function TableRow({ columns, cells, lead }: { columns: readonly Column[]; cells: Record; lead?: ReactNode }) { + return ( + + {lead} + {columns.map((column) => ( + + {cells[column.key]} + + ))} + + ); +} diff --git a/src/cli/monitor/components/charts.tsx b/src/cli/monitor/components/charts.tsx index 27d8b37..b648161 100644 --- a/src/cli/monitor/components/charts.tsx +++ b/src/cli/monitor/components/charts.tsx @@ -1,9 +1,16 @@ import { Text } from 'ink'; import { buildGauge, buildSparkline } from '../charts.js'; +/** Filled part in its threshold colour, the remainder dim, so the eye reads the fill. */ export function Gauge({ percent, width }: { percent: number; width: number }) { const { bar, color } = buildGauge(percent, width); - return {bar}; + const filled = bar.replace(/░/g, ''); + return ( + + {filled} + {bar.slice(filled.length)} + + ); } export function Sparkline({ values, width }: { values: number[]; width: number }) { @@ -20,3 +27,14 @@ export function Sparkline({ values, width }: { values: number[]; width: number } ); } + +/** A gauge with its percentage, right-aligned to a fixed width so columns line up. */ +export function Meter({ percent, width = 10 }: { percent: number; width?: number }) { + const clamped = Math.max(0, Math.min(1, percent)); + return ( + + + {`${Math.round(clamped * 100)}%`.padStart(4)} + + ); +} diff --git a/src/cli/monitor/fleet-model.ts b/src/cli/monitor/fleet-model.ts index 55a33d4..9067a21 100644 --- a/src/cli/monitor/fleet-model.ts +++ b/src/cli/monitor/fleet-model.ts @@ -96,18 +96,22 @@ export interface FleetHeadline { text: string; } -/** One sentence per app: is the fleet agreed on a release, and can every replica be seen? */ +/** + * One line per app: is the fleet agreed on a release, and can every replica be + * seen? Every problem is named, worst first, because an unreachable replica can + * hide a split roll behind it. + */ export function describeFleet(fleet: FleetView): FleetHeadline { const { convergence, unreachable, replicas } = fleet; const total = replicas.length + unreachable.length; - if (unreachable.length > 0) { - return { tone: 'bad', text: `${unreachable.length}/${total} unreachable: ${unreachable.join(', ')}` }; - } - if (convergence.releases.length > 1) { - return { tone: 'bad', text: `split across ${convergence.releases.length} releases - a roll stopped partway` }; - } - if (convergence.undeployed.length > 0) { - return { tone: 'warn', text: `no release on ${convergence.undeployed.join(', ')}` }; + const bad: string[] = []; + const warn: string[] = []; + if (unreachable.length > 0) bad.push(`${unreachable.length}/${total} unreachable: ${unreachable.join(', ')}`); + if (convergence.releases.length > 1) bad.push(`split across ${convergence.releases.length} releases - a roll stopped partway`); + if (convergence.undeployed.length > 0) warn.push(`no release on ${convergence.undeployed.join(', ')}`); + + if (bad.length + warn.length > 0) { + return { tone: bad.length > 0 ? 'bad' : 'warn', text: [...bad, ...warn].join(' · ') }; } return { tone: 'ok', text: total > 1 ? `converged on ${convergence.releases[0] ?? 'no release'}` : 'ok' }; } diff --git a/src/cli/monitor/hooks/use-now.ts b/src/cli/monitor/hooks/use-now.ts new file mode 100644 index 0000000..d2fabdc --- /dev/null +++ b/src/cli/monitor/hooks/use-now.ts @@ -0,0 +1,11 @@ +import { useEffect, useState } from 'react'; + +/** The current time, re-rendering every `ms` - for "updated 3s ago" and flash expiry. */ +export function useNow(ms: number = 1000): number { + const [now, setNow] = useState(() => Date.now()); + useEffect(() => { + const timer = setInterval(() => setNow(Date.now()), ms); + return () => clearInterval(timer); + }, [ms]); + return now; +} diff --git a/src/cli/monitor/layout/HeaderBar.tsx b/src/cli/monitor/layout/HeaderBar.tsx deleted file mode 100644 index d9f12f8..0000000 --- a/src/cli/monitor/layout/HeaderBar.tsx +++ /dev/null @@ -1,33 +0,0 @@ -import { Box, Text } from 'ink'; - -const ACCENT = '#d6a85d'; -const BG_HEADER = '#161b22'; - -export interface HeaderAlert { - text: string; -} - -interface HeaderBarProps { - /** What is being watched: "fleet · 3 servers", or "api on b". */ - scope: string; - interval: number; - liveMode: boolean; - alerts: readonly HeaderAlert[]; -} - -export function HeaderBar({ scope, interval, liveMode, alerts }: HeaderBarProps) { - return ( - - - ShipNode Monitor - — {scope} - {' │'} interval: {interval}s - {liveMode ? {' │'} logs: ON : │ logs: off} - {alerts.map((alert) => ( - {' │ '}{alert.text} - ))} - {' │'} [?] help - - - ); -} diff --git a/src/cli/monitor/layout/MonitorFrame.tsx b/src/cli/monitor/layout/MonitorFrame.tsx index 4562870..ae99bec 100644 --- a/src/cli/monitor/layout/MonitorFrame.tsx +++ b/src/cli/monitor/layout/MonitorFrame.tsx @@ -1,38 +1,109 @@ import { Box, Text } from 'ink'; -import type { ReactNode } from 'react'; -import { HeaderBar, type HeaderAlert } from './HeaderBar.js'; -import { StatusBar } from './StatusBar.js'; +import { Fragment, type ReactNode } from 'react'; +import { KeyHints, type Hint } from '../components/KeyHints.js'; +import { color, glyph, toneColor, type Tone } from '../theme.js'; +import { useNow } from '../hooks/use-now.js'; -const BG = '#0d1117'; -const BORDER = '#30363d'; +export interface HeaderAlert { + text: string; +} + +export interface Flash { + tone: Tone; + text: string; +} interface MonitorFrameProps { - scope: string; + /** Where the user is: ['fleet'], ['fleet', 'api', 'b'], ['logs', 'api']. */ + crumbs: readonly string[]; interval: number; - liveMode: boolean; - alerts: readonly HeaderAlert[]; - lastUpdate: string | null; polling: boolean; - summary: string; - hints?: string; - error?: string | null; + lastUpdate: string | null; + /** Live log streaming state, shown only once streaming has started. */ + streaming?: { live: number; total: number } | null; + alerts: readonly HeaderAlert[]; + hints: readonly Hint[]; + /** A short-lived message from the last action, shown in the footer. */ + flash?: Flash | null; children: ReactNode; } -export function MonitorFrame({ scope, interval, liveMode, alerts, lastUpdate, polling, summary, hints, error, children }: MonitorFrameProps) { +function ago(iso: string | null, now: number): string { + if (iso === null) return 'waiting'; + const seconds = Math.max(0, Math.round((now - Date.parse(iso)) / 1000)); + return seconds < 2 ? 'just now' : `${seconds}s ago`; +} + +function Header({ crumbs, interval, polling, lastUpdate, streaming }: Pick) { + const now = useNow(); + const stale = lastUpdate !== null && now - Date.parse(lastUpdate) > interval * 3000; + const pulse = lastUpdate === null || polling ? glyph.unknown : glyph.ok; + const pulseColor = stale ? color.bad : lastUpdate === null ? undefined : color.ok; + return ( + + + {glyph.brand} shipnode + {' '} + {crumbs.map((crumb, index) => ( + + {index > 0 && {glyph.crumb} } + {crumb} + + ))} + + + {streaming != null && ( + + ≋ + {streaming.live}/{streaming.total} streams + + )} + {pulse} + {stale ? 'stale' : 'live'} {glyph.sep} every {interval}s {glyph.sep} {ago(lastUpdate, now)} + + + ); +} + +function AlertBar({ alerts }: { alerts: readonly HeaderAlert[] }) { + return ( + + + {alerts.map((alert, index) => ( + + {index > 0 ? ' ' : ''}{glyph.alert} {alert.text} + + ))} + + + ); +} + +function Footer({ hints, flash }: { hints: readonly Hint[]; flash?: Flash | null }) { return ( - - - {children} - + + + + + + {flash != null ? ( + {flash.text} + ) : ( + + )} + ); } -export function WaitingPanel() { +/** Header, optional alert bar, the view, and the key-hint footer. */ +export function MonitorFrame({ alerts, hints, flash, children, ...header }: MonitorFrameProps) { return ( - - Waiting for data... + +
+ {alerts.length > 0 && } + {children} +