From 3127446502e49bd9f58035ecb6103b2dac848c52 Mon Sep 17 00:00:00 2001 From: Alade YESSOUFOU Date: Thu, 1 Oct 2026 18:15:58 +0200 Subject: [PATCH 1/2] feat(blue-green): stop the old colour by default and let rollback start it again Keeping the previous colour running made every app cost twice its memory, and blueGreenRetention 'none' removed that cost by also removing rollback. Add a 'warm' mode, now the default, modelled on Kamal: after the Caddy flip the old colour drains for 10 seconds and is stopped, its release stays on disk, and shipnode rollback points current at that release, starts the colour from it, health-checks it, flips traffic and stops the colour that was serving. A failed start restores current and removes the half-started colour. deploy-state.json records the release each colour runs. A reaped watt unit is now disabled as well as stopped so it does not return on reboot. --- CHANGELOG.md | 10 ++ README.md | 10 +- docs/adr/0005-blue-green-zero-downtime.md | 2 +- docs/adr/0010-warm-blue-green-retention.md | 28 +++ src/cli/commands/rollback.ts | 166 ++++++++++++++---- src/config/builder.ts | 12 +- src/config/schema.ts | 2 +- src/domain/deploy/backend-strategy.ts | 38 ++-- src/domain/deploy/blue-green.ts | 34 +++- src/domain/deploy/orchestrator.ts | 21 ++- src/domain/deploy/retention.ts | 56 ++++++ src/domain/runtime/watt.ts | 14 ++ src/shared/types.ts | 2 +- tests/unit/backend-strategy.test.ts | 28 +++ tests/unit/blue-green-orchestrator.test.ts | 18 ++ tests/unit/rollback-warm.test.ts | 195 +++++++++++++++++++++ tests/unit/schema.test.ts | 16 +- tests/unit/watt-runtime.test.ts | 20 ++- 18 files changed, 593 insertions(+), 79 deletions(-) create mode 100644 docs/adr/0010-warm-blue-green-retention.md create mode 100644 src/domain/deploy/retention.ts create mode 100644 tests/unit/rollback-warm.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 98fffe3..15a1fcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ All notable changes to `@devalade/shipnode` will be documented here. ## [Unreleased] +### Changed +- **Blue-green now stops the old colour by default (`blueGreenRetention: 'warm'`).** After Caddy switches traffic, the previous colour is stopped following a 10-second drain instead of staying in memory, so each app holds one copy. Its release stays on disk. This changes behaviour for configs that never set the option, which used to keep both colours running; set `'rollback'` to keep that. See [ADR-0010](docs/adr/0010-warm-blue-green-retention.md). + +### Added +- **`shipnode rollback` can start a stopped colour.** When the previous colour is not running it points `current` at the release that colour ran, starts it from there, waits for its health check, flips traffic, and stops the colour that was serving. A failed start leaves the app as it was. `deploy-state.json` now records `blueRelease` / `greenRelease` to make this possible; an app deployed before this change can roll back this way after its second deploy with it, because the colour it would go back to has no recorded release until then. +- **A drain before the old colour stops**, so a request already in flight when Caddy flips can finish. + +### Fixed +- **A reaped `watt` colour came back after a reboot.** Its unit stayed enabled, so it restarted on whatever `current` pointed at and held memory. A stopped colour's unit is now disabled as well, and re-enabled when it is started again. + ## [3.2.0-beta.2] - 2026-10-01 ### Added diff --git a/README.md b/README.md index ca3f938..c548b5d 100644 --- a/README.md +++ b/README.md @@ -327,7 +327,7 @@ See [ADR-0007](docs/adr/0007-fleet-replication.md) and [ADR-0008](docs/adr/0008- | `.installCommand(cmd)` | derived from pkg manager | Override the install command run on the server (e.g. `'npm ci --legacy-peer-deps'`). Equivalent to `pkgManager(pm, { installCommand: cmd })` | | `.buildDir(dir)` | auto-detected | Frontend build output dir | | `.zeroDowntime(altPort?)` | automatic for Caddy backends | Force blue-green releases and optionally choose the green port | -| `.blueGreenRetention('rollback' \| 'none')` | `'rollback'` | Keep the old web process for instant rollback, or reclaim it after the switch | +| `.blueGreenRetention('warm' \| 'rollback' \| 'none')` | `'warm'` | What happens to the old colour after the switch: stop it but keep its release so `rollback` can start it again (`warm`), keep it running for an instant flip (`rollback`), or stop it with no rollback (`none`) | | `.noZeroDowntime()` | — | Opt out and recreate PM2 processes during deploy | | `.healthCheck(path, opts?)` | `/health`, 30s, 3 retries | Post-deploy health check | | `.noHealthCheck()` | — | Skip health check | @@ -575,16 +575,16 @@ export default shipnode Backends with a domain and a PM2 web port use blue-green releases automatically. Shipnode starts the new web process on the idle port, checks it, and reloads Caddy only after it is healthy. Apps without a domain and worker-only apps keep the PM2 recreate path. Use `.noZeroDowntime()` to opt out explicitly; raw configuration may set `zeroDowntime: false`. -Blue-green keeps the previous and current web processes resident, so budget about **2× the web process memory**. On an eligible Caddy backend, `.zeroDowntime(altPort?)` remains available to force the mode and choose the alternate port. By default, Shipnode uses a less commonly occupied port offset by 10,000 (`3000 → 13000`); near the top of the TCP range it subtracts 10,000 instead. +By default (`warm`) the old colour is stopped a few seconds after Caddy switches traffic, so only one copy of the web app is in memory. Its release stays on disk, and `shipnode rollback` starts it again, waits for its health check, then flips traffic — seconds, not a redeploy. This is the model Kamal uses. On an eligible Caddy backend, `.zeroDowntime(altPort?)` remains available to force the mode and choose the alternate port. By default, Shipnode uses a less commonly occupied port offset by 10,000 (`3000 → 13000`); near the top of the TCP range it subtracts 10,000 instead. -For memory-constrained apps, retain both processes only while the new release starts and passes its health check: +To trade memory for an instant rollback, keep the old colour running: ```ts .zeroDowntime() - .blueGreenRetention('none') + .blueGreenRetention('rollback') ``` -After Caddy switches traffic, Shipnode stops the old colour immediately. This disables instant `shipnode rollback`; redeploy the desired release instead. +Budget about **2× the web process memory** with `rollback`, since both colours stay resident. `'none'` behaves like `warm` but disables `shipnode rollback` entirely. See [ADR-0010](docs/adr/0010-warm-blue-green-retention.md). Every app still uses a Capistrano-style release structure: diff --git a/docs/adr/0005-blue-green-zero-downtime.md b/docs/adr/0005-blue-green-zero-downtime.md index 5559b83..6284617 100644 --- a/docs/adr/0005-blue-green-zero-downtime.md +++ b/docs/adr/0005-blue-green-zero-downtime.md @@ -25,7 +25,7 @@ Because the previous colour is still running, rollback is an instant Caddy flip ## Trade-offs -- **~2× memory for the web app** — both colours are resident between deploys. Documented; the price of instant rollback. +- **~2× memory for the web app** — with `blueGreenRetention: 'rollback'` both colours are resident between deploys. Documented; the price of instant rollback. The default is now `warm`, which stops the old colour and avoids this cost ([ADR-0010](0010-warm-blue-green-retention.md)). - **The port pair is fixed at the first deploy** and persisted. Later changes to the web port or `altPort` in config are ignored until the state file is cleared, so a running colour is never silently re-homed. - **The first migration targets green.** A pre-existing uncoloured process can continue serving on the configured blue port through health and the Caddy reload; it is cleaned up only after the flip succeeds. - **Default on for Caddy backends.** Backends without a domain and worker-only apps keep recreate semantics. `.noZeroDowntime()` is the explicit builder opt-out. diff --git a/docs/adr/0010-warm-blue-green-retention.md b/docs/adr/0010-warm-blue-green-retention.md new file mode 100644 index 0000000..d2fab4b --- /dev/null +++ b/docs/adr/0010-warm-blue-green-retention.md @@ -0,0 +1,28 @@ +# Warm blue-green retention is the default + +[ADR-0005](0005-blue-green-zero-downtime.md) kept the previous colour running after every flip so a rollback was an instant Caddy flip. The cost is a second copy of every web app in memory, permanently. On a shared server with no swap that is the difference between fitting and being killed: running 40 small apps this way held roughly 3–4 GB that did nothing until someone rolled back. + +`blueGreenRetention: 'none'` removed the cost but also removed rollback — `shipnode rollback` refused, and the only way back was a full redeploy. Nothing sat between the two. + +## Decision + +`warm` is the default, modelled on how Kamal handles old containers: stop the old version after a short drain, keep it on disk, and start it again on rollback. + +- **After the flip.** Caddy has stopped sending the old colour new requests. Shipnode waits `DRAIN_SECONDS` (10) so requests already in flight can finish, then stops it: a PM2 process is deleted by exact name, a watt unit is stopped **and disabled** so it does not return on a reboot and hold memory for nothing. Its release directory is untouched (`keepReleases` still applies). +- **Recording what ran.** `deploy-state.json` gains `blueRelease` / `greenRelease`, the release each colour was last booted from. State written before this change has neither field, and rollback says so instead of guessing. +- **Rollback.** If the previous colour is running (`rollback` retention) it is still an instant flip. Otherwise shipnode points `current` at that colour's release — its launcher files resolve through `current` (ADR-0001), so starting it from anywhere else would run the wrong code — starts it from that release's own ecosystem file or unit, waits for its health check, flips Caddy, and then stops the colour that was serving. If it cannot start, the half-started colour is removed and `current` is restored, so a failed rollback leaves the app as it was. + +## Modes + +| | After the flip | `shipnode rollback` | Memory | +|---|---|---|---| +| `warm` (default) | stopped after a drain | boots it again, seconds | 1× | +| `rollback` | left running | instant flip | 2× | +| `none` | stopped after a drain | refused; redeploy | 1× | + +## Trade-offs + +- **Rollback is no longer instant by default.** It takes the colour's boot time plus its health check. An app that needs the instant flip sets `'rollback'`. +- **Changing the default changes behaviour for existing configs** that never set the option: their previous colour is now stopped after a flip. That is the intent, but it is a behaviour change and is recorded in the changelog. +- **Workers are not rolled back.** There is a single worker set, restarted against the new release in `afterHealthy`; a rollback moves web traffic only, as the instant flip always did. +- **The drain is a fixed wait,** not a check that connections have closed. It covers quick requests; a long-lived connection (WebSocket, streaming) can still be cut when the old colour stops. diff --git a/src/cli/commands/rollback.ts b/src/cli/commands/rollback.ts index 95743e0..fc35f4c 100644 --- a/src/cli/commands/rollback.ts +++ b/src/cli/commands/rollback.ts @@ -14,12 +14,15 @@ import { otherColor, portFor, coloredWebName, + releaseFor, + type DeployColor, } from '../../domain/deploy/blue-green.js'; +import { reapColourCommand } from '../../domain/deploy/retention.js'; import { rollFleet, type FleetEvent } from '../../domain/deploy/fleet.js'; import { isFleet } from '../../domain/servers.js'; import type { RemoteExecutor } from '../../domain/remote/executor.js'; -import type { ShipnodeConfig, ShipnodeApp } from '../../shared/types.js'; -import { isActiveCommand, isWatt, resolveWattUnits, restartUnitCommand, wattUnitName } from '../../domain/runtime/watt.js'; +import type { ShipnodeConfig, ShipnodeApp, Pm2App } from '../../shared/types.js'; +import { enableAndStartUnitCommand, isActiveCommand, isWatt, resolveWattUnits, restartUnitCommand, wattUnitName } from '../../domain/runtime/watt.js'; import { configForAppResult, configForServer, getServerTargets } from '../../domain/servers.js'; /** @@ -224,10 +227,17 @@ async function rollbackReplica( ui.success(`Rolled back to ${target.timestamp}`); } +const MISE = 'export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:$PATH"'; + /** - * Instant blue-green rollback: the previous colour is still resident, so we - * flip Caddy's upstream back to it and swap the persisted active colour. No - * process restart, no dropped requests. + * Blue-green rollback: send traffic back to the previous colour. + * + * With retention `rollback` that colour is still running, so this is an instant + * flip. With `warm` (the default) it was stopped after the last flip, so it is + * booted again from the release it ran — `current` is pointed back at that + * release first, because the colour's launcher files resolve through `current` — + * health-checked, and only then does traffic move. The colour that was serving + * is stopped afterwards, so memory stays at one copy. * * Only one step back is possible — older colours were reaped by later deploys. * For anything deeper, redeploy the desired release instead. @@ -242,14 +252,14 @@ async function rollbackBlueGreen( ): Promise { if (app.blueGreenRetention === 'none') { throw new Error( - 'Instant blue-green rollback is disabled because blueGreenRetention is "none". ' + - 'Redeploy the desired release instead.', + 'Blue-green rollback is disabled because blueGreenRetention is "none". ' + + 'Use "warm" to keep rollback without holding the old colour in memory, or redeploy the desired release.', ); } if (stepsBack !== 1) { throw new Error( - `Blue-green rollback only supports one step (the live previous colour). ` + + `Blue-green rollback only supports one step (the previous colour). ` + `To go further back, redeploy the desired release.`, ); } @@ -268,37 +278,44 @@ async function rollbackBlueGreen( const previousPort = portFor(previous, state); const previousName = coloredWebName(namespace, webApp.name, previous); - // The previous colour must still be online to serve traffic after the flip. - // Parse pm2's JSON in-process rather than relying on `node` being on the - // remote PATH at rollback time. - let online = false; - if (isWatt(app)) { - const unit = wattUnitName(namespace, webApp.name, previous); - online = (await executor.exec(isActiveCommand(unit))).exitCode === 0; - } else { - const mise = `export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:$PATH"`; - const jlist = await executor.exec(`${mise} && mise exec -- pm2 jlist`); - try { - const entries = JSON.parse(jlist.stdout.trim()) as Array<{ name: string; pm2_env?: { status?: string } }>; - online = entries.some((e) => e.name === previousName && e.pm2_env?.status === 'online'); - } catch { - online = false; - } - } - if (!online) { - throw new Error( - `Previous colour "${previousName}" is not running — cannot instant-rollback. ` + - `Redeploy the desired release instead.`, - ); - } + const online = await isColourOnline(executor, app, namespace, webApp.name, previous); ui.warn(`Active colour: ${state.activeColor} (port ${portFor(state.activeColor, state)})`); ui.warn(`Rollback target: ${previous} (port ${previousPort})`); - const ok = await ask('Flip traffic back to the previous colour?'); - if (!ok) { - ui.info('Rollback cancelled.'); - return; + let bootFrom: string | undefined; + if (online) { + if (!(await ask('Flip traffic back to the previous colour?'))) { + ui.info('Rollback cancelled.'); + return; + } + } else { + const release = releaseFor(state, previous); + if (release === undefined) { + throw new Error( + `Previous colour "${previousName}" is stopped and the server did not record which release it ran ` + + `(its state predates warm rollback). Redeploy the desired release instead.`, + ); + } + const onDisk = await executor.exec(`test -d "${appPath}/releases/${release}"`); + if (onDisk.exitCode !== 0) { + throw new Error( + `Release ${release} is no longer on the server (older releases are cleaned up after keepReleases). ` + + `Redeploy the desired release instead.`, + ); + } + ui.warn(`"${previousName}" is stopped; it will be started again from release ${release}.`); + if (!(await ask(`Start ${previous} from release ${release} and flip traffic to it?`))) { + ui.info('Rollback cancelled.'); + return; + } + bootFrom = release; + } + + if (bootFrom !== undefined) { + await bootColourFromRelease({ + executor, config, app, appPath, namespace, webApp, color: previous, port: previousPort, release: bootFrom, + }); } const caddy = new CaddyService(executor, config); @@ -306,5 +323,84 @@ async function rollbackBlueGreen( await caddy.reload(); await writeDeployState(executor, appPath, { ...state, activeColor: previous }); + // The colour that was serving is no longer needed: keep one copy in memory. + if (app.blueGreenRetention === 'warm') { + await executor.execOrThrow(reapColourCommand(app, namespace, webApp.name, state.activeColor)); + } + ui.success(`Rolled back — traffic now on ${previous} (port ${previousPort})`); } + +/** Whether the web process of one colour is running right now. */ +async function isColourOnline( + executor: RemoteExecutor, + app: ShipnodeApp, + namespace: string, + webName: string, + color: DeployColor, +): Promise { + if (isWatt(app)) { + return (await executor.exec(isActiveCommand(wattUnitName(namespace, webName, color)))).exitCode === 0; + } + // Parse pm2's JSON in-process rather than relying on `node` being on the + // remote PATH at rollback time. + const name = coloredWebName(namespace, webName, color); + const jlist = await executor.exec(`${MISE} && mise exec -- pm2 jlist`); + try { + const entries = JSON.parse(jlist.stdout.trim()) as Array<{ name: string; pm2_env?: { status?: string } }>; + return entries.some((e) => e.name === name && e.pm2_env?.status === 'online'); + } catch { + return false; + } +} + +interface BootColourInput { + executor: RemoteExecutor; + config: ShipnodeConfig; + app: ShipnodeApp; + appPath: string; + namespace: string; + webApp: Pm2App; + color: DeployColor; + port: number; + release: string; +} + +/** + * Start a stopped colour from the release it ran and wait until it is healthy. + * + * `current` is pointed at that release for the start and left there on success, + * since the colour's launcher resolves its files through `current`. On failure + * the colour is stopped again and `current` is put back, so a rollback that + * cannot start leaves the app exactly as it was. + */ +async function bootColourFromRelease(input: BootColourInput): Promise { + const { executor, config, app, appPath, namespace, webApp, color, port, release } = input; + const releases = new ReleaseManager(executor, appPath, app.keepReleases); + const before = (await executor.exec(`readlink "${appPath}/current"`)).stdout.trim(); + + await releases.switchSymlink(`${appPath}/releases/${release}`); + try { + if (isWatt(app)) { + await executor.execOrThrow(enableAndStartUnitCommand(wattUnitName(namespace, webApp.name, color))); + } else { + await executor.execOrThrow( + `cd "${appPath}/current" && ${MISE} && ` + + `mise exec -- pm2 start "${appPath}/current/ecosystem.web.config.cjs" --update-env && ` + + `mise exec -- pm2 save`, + ); + } + if (app.healthCheck.enabled) { + ui.info(`Waiting for ${color} to pass its health check...`); + await new HealthCheckService(executor, config).perform(app, { + httpPort: port, + pm2Apps: [webApp], + resolvePm2Name: (a: Pm2App) => coloredWebName(namespace, a.name, color), + }); + } + } catch (error) { + await executor.exec(reapColourCommand(app, namespace, webApp.name, color)); + if (before !== '') await releases.switchSymlink(before); + throw error; + } +} diff --git a/src/config/builder.ts b/src/config/builder.ts index 3602e92..5c5f397 100644 --- a/src/config/builder.ts +++ b/src/config/builder.ts @@ -182,7 +182,11 @@ export class ShipnodeBuilder { return this; } - /** Reclaim the inactive web process after a successful blue-green switch. */ + /** + * What happens to the old colour after a successful blue-green switch: `warm` + * (default) stops it but keeps its release so `rollback` can start it again, + * `rollback` keeps it running for an instant flip, `none` stops it with no rollback. + */ blueGreenRetention(retention: ShipnodeApp['blueGreenRetention']): this { this.config.blueGreenRetention = retention; return this; @@ -443,7 +447,11 @@ export class ShipnodeAppBuilder { return this; } - /** Reclaim the inactive web process after a successful blue-green switch. */ + /** + * What happens to the old colour after a successful blue-green switch: `warm` + * (default) stops it but keeps its release so `rollback` can start it again, + * `rollback` keeps it running for an instant flip, `none` stops it with no rollback. + */ blueGreenRetention(retention: ShipnodeApp['blueGreenRetention']): this { this.state.blueGreenRetention = retention; return this; diff --git a/src/config/schema.ts b/src/config/schema.ts index 51a3f07..1211dd5 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -276,7 +276,7 @@ export const ShipnodeAppSchema = z.object({ envFile: z.string().default('.env'), keepReleases: z.number().int().min(1).default(5), zeroDowntime: z.boolean().optional(), - blueGreenRetention: z.enum(['rollback', 'none']).default('rollback'), + blueGreenRetention: z.enum(['warm', 'rollback', 'none']).default('warm'), altPort: z.number().int().positive().optional(), sharedDirs: z.array(z.string()).optional(), sharedFiles: z.array(z.string()).optional(), diff --git a/src/domain/deploy/backend-strategy.ts b/src/domain/deploy/backend-strategy.ts index c02f744..6a84fc1 100644 --- a/src/domain/deploy/backend-strategy.ts +++ b/src/domain/deploy/backend-strategy.ts @@ -5,6 +5,7 @@ import { resolve } from 'path'; import type { ShipnodeConfig, ShipnodeApp, Pm2App, PkgManager } from '../../shared/types.js'; import { getPm2Name } from '../pm2/apps.js'; import { coloredWebName } from './blue-green.js'; +import { drainCommand, pm2DeleteExactCommand, reapColourCommand, reapsPreviousColour } from './retention.js'; import { getInstallCommand, getRunCommand, detectPkgManager } from '../framework/detector.js'; import { RSYNC_DEFAULT_EXCLUDES } from '../../shared/constants.js'; import { DeployError } from '../../shared/errors.js'; @@ -308,27 +309,19 @@ export class BackendStrategy implements DeploymentStrategy { if (!webApp) return; const mise = `export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:$PATH"`; - // First deploy: drop a pre-blue-green uncoloured process (name === namespace). - // Later deploys with retention "none": drop the idle coloured sibling. - const previousName = ctx.deployTarget.previousColor === null - ? getPm2Name(namespace, webApp.name) - : this.app.blueGreenRetention === 'none' - ? coloredWebName(namespace, webApp.name, ctx.deployTarget.previousColor) - : undefined; - if (previousName === undefined) return; - - // `pm2 delete ` also matches namespace, so `pm2 delete hub` would kill - // `hub-green`. Resolve to pm_id by exact process name instead (jq is part of setup). - const deleteExact = - `id=$(mise exec -- pm2 jlist 2>/dev/null | jq -r --arg n "${previousName}" ` + - `'.[] | select(.name == $n) | .pm_id' | head -n1) && ` + - `{ [ -n "$id" ] && mise exec -- pm2 delete "$id" || true; }`; + const { previousColor } = ctx.deployTarget; - await ctx.executor.execOrThrow( - `${mise} && ` + - `{ ${deleteExact}; } && ` + - `mise exec -- pm2 save`, - ); + // First deploy: drop a pre-blue-green uncoloured process (name === namespace). + // Later deploys: stop the colour that just stopped serving, unless retention + // keeps it running for an instant rollback. Either way it finishes in-flight + // requests first; Caddy has already stopped sending it new ones. + if (previousColor === null) { + await ctx.executor.execOrThrow( + `${drainCommand} && ${pm2DeleteExactCommand(mise, getPm2Name(namespace, webApp.name))}`, + ); + } else if (reapsPreviousColour(this.app.blueGreenRetention)) { + await ctx.executor.execOrThrow(reapColourCommand(this.app, namespace, webApp.name, previousColor)); + } } // ------------------------------------------------------------------------- @@ -467,12 +460,13 @@ export class BackendStrategy implements DeploymentStrategy { // First blue-green deploy: retire the pre-blue-green process — an // uncoloured watt unit, or a PM2 process when migrating from PM2. const mise = `export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:$PATH"`; + await ctx.executor.execOrThrow(drainCommand); await ctx.executor.execOrThrow(removeUnitCommand(wattUnitName(namespace, web.name))); await ctx.executor.execOrThrow(`${mise} && { mise exec -- pm2 delete "${namespace}" 2>/dev/null || true; }`); return; } - if (this.app.blueGreenRetention === 'none') { - await ctx.executor.execOrThrow(stopUnitCommand(wattUnitName(namespace, web.name, target.previousColor))); + if (reapsPreviousColour(this.app.blueGreenRetention)) { + await ctx.executor.execOrThrow(reapColourCommand(this.app, namespace, web.name, target.previousColor)); } } diff --git a/src/domain/deploy/blue-green.ts b/src/domain/deploy/blue-green.ts index 1731050..3ddd6c7 100644 --- a/src/domain/deploy/blue-green.ts +++ b/src/domain/deploy/blue-green.ts @@ -20,6 +20,23 @@ export interface DeployState { activeColor: DeployColor; bluePort: number; greenPort: number; + /** + * The release each colour was last booted from. Warm rollback needs it: the + * idle colour is stopped after a flip, and bringing it back means starting it + * from its own release. Absent in state written before warm rollback existed. + */ + blueRelease?: string; + greenRelease?: string; +} + +/** The release a colour was booted from, if the state recorded it. */ +export function releaseFor(state: DeployState, color: DeployColor): string | undefined { + return color === 'blue' ? state.blueRelease : state.greenRelease; +} + +/** `state` with `color` now running `release`. */ +export function withRelease(state: DeployState, color: DeployColor, release: string): DeployState { + return color === 'blue' ? { ...state, blueRelease: release } : { ...state, greenRelease: release }; } export interface DeployTarget { @@ -34,6 +51,9 @@ export interface DeployTarget { /** Port pair to persist once the flip succeeds. */ bluePort: number; greenPort: number; + /** Releases already recorded for each colour, carried forward when the flip is persisted. */ + blueRelease?: string; + greenRelease?: string; } export function otherColor(color: DeployColor): DeployColor { @@ -88,7 +108,17 @@ export async function readDeployState( typeof parsed.bluePort === 'number' && typeof parsed.greenPort === 'number' ) { - return { activeColor: parsed.activeColor, bluePort: parsed.bluePort, greenPort: parsed.greenPort }; + const recorded = (key: 'blueRelease' | 'greenRelease'): { [k in typeof key]?: string } => { + const value = (parsed as Record)[key]; + return typeof value === 'string' && value !== '' ? { [key]: value } : {}; + }; + return { + activeColor: parsed.activeColor, + bluePort: parsed.bluePort, + greenPort: parsed.greenPort, + ...recorded('blueRelease'), + ...recorded('greenRelease'), + }; } } catch { // Corrupt/legacy state is treated as "no state" — the next deploy behaves @@ -139,5 +169,7 @@ export function resolveTarget( previousPort: portFor(state.activeColor, state), bluePort: state.bluePort, greenPort: state.greenPort, + blueRelease: state.blueRelease, + greenRelease: state.greenRelease, }; } diff --git a/src/domain/deploy/orchestrator.ts b/src/domain/deploy/orchestrator.ts index d9d801f..1f5f685 100644 --- a/src/domain/deploy/orchestrator.ts +++ b/src/domain/deploy/orchestrator.ts @@ -11,6 +11,7 @@ import { readDeployState, writeDeployState, resolveTarget, + withRelease, resolveAltPort, coloredWebName, type DeployTarget, @@ -182,11 +183,21 @@ export class DeployOrchestrator { await this.caddy.configureBackend(app, target.port); await this.caddy.reload(); trafficSwitched = true; - await writeDeployState(this.executor, appPath, { - activeColor: target.color, - bluePort: target.bluePort, - greenPort: target.greenPort, - }); + await writeDeployState( + this.executor, + appPath, + withRelease( + { + activeColor: target.color, + bluePort: target.bluePort, + greenPort: target.greenPort, + blueRelease: target.blueRelease, + greenRelease: target.greenRelease, + }, + target.color, + timestamp, + ), + ); if (strategy.afterTrafficSwitch) { await strategy.afterTrafficSwitch(startCtx); } diff --git a/src/domain/deploy/retention.ts b/src/domain/deploy/retention.ts new file mode 100644 index 0000000..5603cdc --- /dev/null +++ b/src/domain/deploy/retention.ts @@ -0,0 +1,56 @@ +import type { BlueGreenRetention, ShipnodeApp } from '../../shared/types.js'; +import { coloredWebName, type DeployColor } from './blue-green.js'; +import { isWatt, parkUnitCommand, wattUnitName } from '../runtime/watt.js'; + +/** + * What happens to the colour that stopped serving after a blue-green flip. + * + * - `warm` (default): stopped after a short drain, kept on disk as a release. + * `rollback` boots it again from that release — seconds, and no memory held + * in between. This is the model Kamal uses. + * - `rollback`: left running, so a rollback is an instant flip at the cost of a + * second copy of the app in memory. + * - `none`: stopped like `warm`, and rollback is refused; redeploy instead. + */ +export function reapsPreviousColour(retention: BlueGreenRetention): boolean { + return retention !== 'rollback'; +} + +/** + * Seconds the old colour keeps running after Caddy flips away from it, so a + * request already in flight finishes before the process is stopped. Caddy has + * stopped sending it new ones by then. + */ +export const DRAIN_SECONDS = 10; + +/** Shell prefix that waits out the drain; chain the reap after it with `&&`. */ +export const drainCommand = `sleep ${DRAIN_SECONDS}`; + +/** + * Delete one PM2 process by exact name. `pm2 delete ` also matches a + * namespace, so `pm2 delete hub` would take `hub-green` with it; resolving the + * pm_id from the exact process name avoids that. `mise` is the PATH export. + */ +export function pm2DeleteExactCommand(mise: string, name: string): string { + return ( + `${mise} && ` + + `{ id=$(mise exec -- pm2 jlist 2>/dev/null | jq -r --arg n "${name}" ` + + `'.[] | select(.name == $n) | .pm_id' | head -n1) && ` + + `{ [ -n "$id" ] && mise exec -- pm2 delete "$id" || true; }; } && ` + + `mise exec -- pm2 save` + ); +} + +const MISE = 'export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:$PATH"'; + +/** + * Stop the web process of one colour after the drain: a watt unit is stopped + * and disabled, a PM2 process is deleted by exact name. The release it ran stays on disk, so + * `rollback` can boot it again. + */ +export function reapColourCommand(app: ShipnodeApp, namespace: string, webName: string, color: DeployColor): string { + const stop = isWatt(app) + ? parkUnitCommand(wattUnitName(namespace, webName, color)) + : pm2DeleteExactCommand(MISE, coloredWebName(namespace, webName, color)); + return `${drainCommand} && ${stop}`; +} diff --git a/src/domain/runtime/watt.ts b/src/domain/runtime/watt.ts index cf7a8fd..dc1ac68 100644 --- a/src/domain/runtime/watt.ts +++ b/src/domain/runtime/watt.ts @@ -180,6 +180,20 @@ export function stopUnitCommand(unit: string): string { return `${SUDO}; $S systemctl stop ${unit} 2>/dev/null || true`; } +/** + * Stop a unit and keep it from starting at boot, leaving the unit file in place. + * A colour parked after a blue-green flip must not come back on a reboot — it + * would run whatever `current` points at and hold memory for nothing. + */ +export function parkUnitCommand(unit: string): string { + return `${SUDO}; $S systemctl disable --now ${unit} 2>/dev/null || true`; +} + +/** Re-enable a parked unit and start it, so it also survives a reboot. */ +export function enableAndStartUnitCommand(unit: string): string { + return `${SUDO}; $S systemctl enable ${unit} && $S systemctl restart ${unit}`; +} + /** Stop, disable and delete a unit; a missing unit is not an error. */ export function removeUnitCommand(unit: string): string { return `${SUDO}; $S systemctl disable --now ${unit} 2>/dev/null || true; ` + diff --git a/src/shared/types.ts b/src/shared/types.ts index 64d3c38..6a939fa 100644 --- a/src/shared/types.ts +++ b/src/shared/types.ts @@ -1,5 +1,5 @@ export type AppType = 'backend' | 'frontend'; -export type BlueGreenRetention = 'rollback' | 'none'; +export type BlueGreenRetention = 'warm' | 'rollback' | 'none'; export type PkgManager = 'npm' | 'yarn' | 'pnpm' | 'bun'; diff --git a/tests/unit/backend-strategy.test.ts b/tests/unit/backend-strategy.test.ts index 57a6d7c..31ef487 100644 --- a/tests/unit/backend-strategy.test.ts +++ b/tests/unit/backend-strategy.test.ts @@ -643,6 +643,34 @@ describe('BackendStrategy.startApp — blue-green', () => { expect(cleanup).toContain('pm2 save'); }); + it('drains, then reclaims the inactive colour by default (warm)', async () => { + const strategy = makeStrategy(makeConfig({ zeroDowntime: true, domain: 'api.example.com' }), '/local/project'); + const executor = new FakeRemoteExecutor(); + const ctx = makeCtx(executor, { deployTarget: bgTarget({ color: 'green', previousColor: 'blue' }) }); + + await strategy.afterTrafficSwitch!(ctx); + + const cleanup = executor.getLastCommand()?.command; + // Caddy has stopped sending requests to the old colour; give in-flight ones time to finish. + expect(cleanup).toMatch(/^sleep 10 && /); + expect(cleanup).toContain('--arg n "myapp-blue"'); + expect(cleanup).toContain('pm2 delete "$id"'); + }); + + it('leaves the previous colour running when retention is rollback', async () => { + const strategy = makeStrategy(makeConfig({ + zeroDowntime: true, + domain: 'api.example.com', + blueGreenRetention: 'rollback', + }), '/local/project'); + const executor = new FakeRemoteExecutor(); + const ctx = makeCtx(executor, { deployTarget: bgTarget({ color: 'green', previousColor: 'blue' }) }); + + await strategy.afterTrafficSwitch!(ctx); + + expect(executor.getHistory()).toHaveLength(0); + }); + it('keeps workers in a single set written at start, reloaded only in afterHealthy', async () => { const config = assembleConfig({ app: 'backend', diff --git a/tests/unit/blue-green-orchestrator.test.ts b/tests/unit/blue-green-orchestrator.test.ts index 6033e1c..b3584e4 100644 --- a/tests/unit/blue-green-orchestrator.test.ts +++ b/tests/unit/blue-green-orchestrator.test.ts @@ -126,6 +126,24 @@ describe('blue-green deploy (orchestrator)', () => { expect(stateIdx).toBeGreaterThan(caddyIdx); }); + it('records which release each colour runs so a stopped colour can be started again', async () => { + const executor = new FakeRemoteExecutor(); + const state: DeployState = { activeColor: 'blue', bluePort: 3000, greenPort: 3001, blueRelease: 'REL-OLD' }; + baseStubs(executor) + .when((cmd) => cmd.includes('deploy-state.json') && cmd.includes('cat'), { stdout: JSON.stringify(state), stderr: '', exitCode: 0 }) + .when((cmd) => cmd.includes('date') && cmd.includes('curl'), { stdout: '200 12', stderr: '', exitCode: 0 }); + const orchestrator = await buildOrchestrator(executor, bgConfig()); + + await orchestrator.deploy({ cwd: '/test', skipBuild: false, releaseId: 'REL-NEW' }); + + const write = executor.getHistory().map((h) => h.command) + .find((c) => c.includes('deploy-state.json') && c.includes('base64 -d'))!; + const written = JSON.parse(Buffer.from(/printf '%s' '([^']+)'/.exec(write)![1], 'base64').toString()); + expect(written).toEqual({ + activeColor: 'green', bluePort: 3000, greenPort: 3001, blueRelease: 'REL-OLD', greenRelease: 'REL-NEW', + }); + }); + it('second switch (green active) reuses blue only after the first migration cleaned it', async () => { const executor = new FakeRemoteExecutor(); const state: DeployState = { activeColor: 'green', bluePort: 3000, greenPort: 3001 }; diff --git a/tests/unit/rollback-warm.test.ts b/tests/unit/rollback-warm.test.ts new file mode 100644 index 0000000..6f6fa33 --- /dev/null +++ b/tests/unit/rollback-warm.test.ts @@ -0,0 +1,195 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { FakeRemoteExecutor } from '../testing/fake-executor.js'; +import type { ShipnodeConfig } from '../../src/shared/types.js'; + +let executor: FakeRemoteExecutor; + +vi.mock('../../src/config/loader.js', () => ({ loadConfig: vi.fn() })); +vi.mock('../../src/cli/prompt.js', () => ({ confirm: vi.fn() })); +vi.mock('../../src/cli/ui.js', () => ({ + ui: { banner: vi.fn(), step: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn(), success: vi.fn(), note: vi.fn() }, +})); +vi.mock('../../src/infrastructure/ssh/connection.js', () => ({ + SshConnection: class { + async connect(): Promise {} + disconnect(): void {} + exec(command: string, options?: { timeout?: number }): Promise { + return executor.exec(command, options); + } + execOrThrow(command: string, options?: { timeout?: number }): Promise { + return executor.execOrThrow(command, options); + } + }, +})); + +const { cmdRollback } = await import('../../src/cli/commands/rollback.js'); +const { loadConfig } = await import('../../src/config/loader.js'); +const { confirm } = await import('../../src/cli/prompt.js'); +const { ui } = await import('../../src/cli/ui.js'); + +type Retention = 'warm' | 'rollback' | 'none'; + +function singleServer(overrides: { retention?: Retention; runtime?: 'watt' } = {}): ShipnodeConfig { + return { + ssh: { host: '10.0.0.11', user: 'deploy', port: 22 }, + servers: { main: { host: '10.0.0.11', user: 'deploy', port: 22 } }, + remotePath: '/var/www/app', + nodeVersion: '22', + apps: [{ + name: 'api', + appType: 'backend', + on: 'main', + zeroDowntime: true, + blueGreenRetention: overrides.retention ?? 'warm', + ...(overrides.runtime ? { runtime: overrides.runtime, watt: { main: 'dist/server.js' } } : {}), + domain: 'api.example.com', + pm2: { apps: [{ name: 'api', port: 3000 }] }, + healthCheck: { enabled: false, path: '/health', timeout: 30, retries: 3, startupDelay: 0 }, + envFile: '.env', + keepReleases: 5, + }], + } as unknown as ShipnodeConfig; +} + +const STATE = { activeColor: 'blue', bluePort: 3000, greenPort: 13000, blueRelease: 'R2', greenRelease: 'R1' }; +const pm2Online = (...names: string[]) => + JSON.stringify(names.map((name) => ({ name, pm2_env: { status: 'online' } }))); + +/** A host where `blue` serves, `green` is stopped, and both releases are on disk. */ +function host(state: Record = STATE): FakeRemoteExecutor { + return new FakeRemoteExecutor() + .when((c) => c.startsWith('cat ') && c.includes('deploy-state.json'), { + stdout: JSON.stringify(state), stderr: '', exitCode: 0, + }) + .when((c) => c.includes('pm2 jlist') && !c.includes('jq'), { stdout: pm2Online('api-blue'), stderr: '', exitCode: 0 }) + .when((c) => c.startsWith('readlink'), { stdout: '/var/www/app/api/releases/R2\n', stderr: '', exitCode: 0 }); +} + +const history = () => executor.getHistory().map((entry) => entry.command); +const indexOf = (needle: string) => history().findIndex((c) => c.includes(needle)); + +beforeEach(() => { + vi.mocked(loadConfig).mockResolvedValue(singleServer()); + vi.mocked(confirm).mockReset().mockResolvedValue(true); + vi.mocked(ui.error).mockReset(); + vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`exit:${code}`); + }) as never); +}); + +async function rollback(): Promise { + try { + await cmdRollback('/project', { app: 'api' }); + return undefined; + } catch (error) { + // Errors reach the user through ui.error + exit(1); anything thrown before + // that point is a test-setup problem and must not pass for a clean run. + return (vi.mocked(ui.error).mock.calls[0]?.[0] as string | undefined) ?? `UNEXPECTED: ${error instanceof Error ? (error.stack ?? '').split('\n').slice(0, 4).join(' | ') : String(error)}`; + } +} + +describe('warm blue-green rollback (PM2)', () => { + it('starts the stopped colour from its own release, then flips, then stops the one that was serving', async () => { + executor = host(); + + expect(await rollback()).toBeUndefined(); + + + const link = indexOf('releases/R1'); + const start = indexOf('pm2 start'); + const stateWrite = history().findIndex((c) => c.includes('deploy-state.json') && c.includes('base64 -d')); + const reap = indexOf('sleep 10'); + expect(link).toBeGreaterThanOrEqual(0); + expect(history()[start]).toContain('ecosystem.web.config.cjs'); + // `current` moves before the start, the flip comes after it, and the old colour is stopped last. + expect(link).toBeLessThan(start); + expect(start).toBeLessThan(stateWrite); + expect(stateWrite).toBeLessThan(reap); + expect(history()[reap]).toContain('api-blue'); + expect(vi.mocked(confirm).mock.calls[0][0]).toContain('R1'); + }); + + it('records the new active colour and leaves both release records in place', async () => { + executor = host(); + + await rollback(); + + const written = history().find((c) => c.includes('deploy-state.json') && c.includes('base64 -d'))!; + const b64 = /printf '%s' '([^']+)'/.exec(written)![1]; + expect(JSON.parse(Buffer.from(b64, 'base64').toString())).toEqual({ ...STATE, activeColor: 'green' }); + }); + + it('flips without starting anything when the previous colour is still running', async () => { + vi.mocked(loadConfig).mockResolvedValue(singleServer({ retention: 'rollback' })); + // Both colours are running, so the previous one is still there to flip to. + executor = new FakeRemoteExecutor() + .when((c) => c.startsWith('cat ') && c.includes('deploy-state.json'), { stdout: JSON.stringify(STATE), stderr: '', exitCode: 0 }) + .when((c) => c.includes('pm2 jlist'), { stdout: pm2Online('api-blue', 'api-green'), stderr: '', exitCode: 0 }); + + expect(await rollback()).toBeUndefined(); + + expect(indexOf('pm2 start')).toBe(-1); + expect(indexOf('sleep 10')).toBe(-1); + expect(history().some((c) => c.includes('deploy-state.json') && c.includes('base64 -d'))).toBe(true); + }); + + it('puts everything back when the colour cannot start', async () => { + executor = host().when((c) => c.includes('pm2 start'), { stdout: '', stderr: 'boom', exitCode: 1 }); + + expect(await rollback()).toBeDefined(); + + // `current` is pointed at R1 for the start and restored to R2 afterwards, + // the half-started colour is removed, and traffic never moved. + const links = history().filter((c) => c.includes('current.tmp') && c.includes('ln -sfn')); + expect(links[0]).toContain('releases/R1'); + expect(links[links.length - 1]).toContain('releases/R2'); + expect(history().some((c) => c.includes('api-green') && c.includes('pm2 delete'))).toBe(true); + expect(history().some((c) => c.includes('deploy-state.json') && c.includes('base64 -d'))).toBe(false); + }); + + it('declines cleanly and touches nothing', async () => { + executor = host(); + vi.mocked(confirm).mockResolvedValue(false); + + expect(await rollback()).toBeUndefined(); + + expect(indexOf('pm2 start')).toBe(-1); + expect(indexOf('ln -sfn')).toBe(-1); + }); + + it('refuses when the server never recorded which release the colour ran', async () => { + executor = host({ activeColor: 'blue', bluePort: 3000, greenPort: 13000 }); + + expect(await rollback()).toMatch(/did not record which release/); + expect(indexOf('pm2 start')).toBe(-1); + }); + + it('refuses when that release has been cleaned up', async () => { + executor = host().when((c) => c.startsWith('test -d'), { stdout: '', stderr: '', exitCode: 1 }); + + expect(await rollback()).toMatch(/no longer on the server/); + expect(indexOf('pm2 start')).toBe(-1); + }); + + it('is refused under none, which keeps nothing to roll back to', async () => { + vi.mocked(loadConfig).mockResolvedValue(singleServer({ retention: 'none' })); + executor = host(); + + expect(await rollback()).toMatch(/disabled because blueGreenRetention is "none"/); + }); +}); + +describe('warm blue-green rollback (watt)', () => { + it('re-enables and starts the parked unit, then parks the one that was serving', async () => { + vi.mocked(loadConfig).mockResolvedValue(singleServer({ runtime: 'watt' })); + executor = host().when((c) => c.includes('systemctl is-active'), { stdout: '', stderr: '', exitCode: 3 }); + + expect(await rollback()).toBeUndefined(); + + const start = indexOf('systemctl enable shipnode-api-green'); + expect(history()[start]).toContain('systemctl restart shipnode-api-green'); + const park = indexOf('disable --now shipnode-api-blue'); + expect(start).toBeGreaterThanOrEqual(0); + expect(park).toBeGreaterThan(start); + }); +}); diff --git a/tests/unit/schema.test.ts b/tests/unit/schema.test.ts index 2421cba..57f1e98 100644 --- a/tests/unit/schema.test.ts +++ b/tests/unit/schema.test.ts @@ -137,10 +137,24 @@ describe('ShipnodeConfigSchema', () => { expect(result.success).toBe(true); if (result.success) { expect(result.data.apps[0].zeroDowntime).toBe(true); - expect(result.data.apps[0].blueGreenRetention).toBe('rollback'); + expect(result.data.apps[0].blueGreenRetention).toBe('warm'); } }); + it.each(['warm', 'rollback'] as const)('accepts %s blue-green retention', (retention) => { + const result = ShipnodeConfigSchema.safeParse({ + app: 'backend', + ssh: { host: '1.2.3.4', user: 'deploy' }, + remotePath: '/var/www/app', + domain: 'api.example.com', + pm2: { apps: [{ name: 'api', port: 3000 }] }, + blueGreenRetention: retention, + }); + + expect(result.success).toBe(true); + if (result.success) expect(result.data.apps[0].blueGreenRetention).toBe(retention); + }); + it('accepts memory-saving blue-green retention', () => { const result = ShipnodeConfigSchema.safeParse({ app: 'backend', diff --git a/tests/unit/watt-runtime.test.ts b/tests/unit/watt-runtime.test.ts index 9d89ee0..080ec4d 100644 --- a/tests/unit/watt-runtime.test.ts +++ b/tests/unit/watt-runtime.test.ts @@ -138,18 +138,28 @@ describe('BackendStrategy watt — blue-green', () => { expect(cmds(e).some((c) => c.includes('systemctl restart shipnode-api-mailer'))).toBe(true); }); - it('keeps the previous colour when retention is rollback', async () => { - const cfg = bg(); + it('keeps the previous colour running when retention is rollback', async () => { + const cfg = config({ domain: 'api.example.com', zeroDowntime: true, blueGreenRetention: 'rollback' }); const e = new FakeRemoteExecutor(); await strategy(cfg).afterTrafficSwitch!(ctx(e, cfg, { deployTarget: target() })); expect(cmds(e)).toHaveLength(0); }); - it('stops the previous colour when retention is none', async () => { - const cfg = config({ domain: 'api.example.com', zeroDowntime: true, blueGreenRetention: 'none' }); + it.each(['warm', 'none'] as const)('drains, then parks the previous colour when retention is %s', async (retention) => { + const cfg = config({ domain: 'api.example.com', zeroDowntime: true, blueGreenRetention: retention }); + const e = new FakeRemoteExecutor(); + await strategy(cfg).afterTrafficSwitch!(ctx(e, cfg, { deployTarget: target() })); + const reap = cmds(e).find((c) => c.includes('disable --now shipnode-api-blue')); + // Disabled as well as stopped, so it does not come back on a reboot. + expect(reap).toBeDefined(); + expect(reap).toMatch(/^sleep 10 && /); + }); + + it('parks the previous colour by default', async () => { + const cfg = bg(); const e = new FakeRemoteExecutor(); await strategy(cfg).afterTrafficSwitch!(ctx(e, cfg, { deployTarget: target() })); - expect(cmds(e).some((c) => c.includes('systemctl stop shipnode-api-blue'))).toBe(true); + expect(cmds(e).some((c) => c.includes('disable --now shipnode-api-blue'))).toBe(true); }); it('retires the pre-blue-green unit and any PM2 process on the first flip', async () => { From 5a69ca4043f30fc5bcd0d78f4b9e71f4b0b99f57 Mon Sep 17 00:00:00 2001 From: Alade YESSOUFOU Date: Thu, 1 Oct 2026 22:03:35 +0200 Subject: [PATCH 2/2] fix(rollback): keep the original error when restoring current also fails A failed warm rollback restores current in its catch block; if that restore threw, the user saw only the restore error. Warn with where current is left and the command to put it back, then rethrow the original start or health error. --- src/cli/commands/rollback.ts | 13 ++++++++++++- tests/unit/rollback-warm.test.ts | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/cli/commands/rollback.ts b/src/cli/commands/rollback.ts index fc35f4c..b26a22f 100644 --- a/src/cli/commands/rollback.ts +++ b/src/cli/commands/rollback.ts @@ -400,7 +400,18 @@ async function bootColourFromRelease(input: BootColourInput): Promise { } } catch (error) { await executor.exec(reapColourCommand(app, namespace, webApp.name, color)); - if (before !== '') await releases.switchSymlink(before); + if (before !== '') { + try { + await releases.switchSymlink(before); + } catch { + // Don't let a failed restore hide why the start failed: say where + // `current` is left and how to put it back, then rethrow the original. + ui.warn( + `Could not point current back at its previous release. It still points at ${appPath}/releases/${release}; ` + + `restore it with: ln -sfn "${before}" "${appPath}/current"`, + ); + } + } throw error; } } diff --git a/tests/unit/rollback-warm.test.ts b/tests/unit/rollback-warm.test.ts index 6f6fa33..b98035a 100644 --- a/tests/unit/rollback-warm.test.ts +++ b/tests/unit/rollback-warm.test.ts @@ -147,6 +147,20 @@ describe('warm blue-green rollback (PM2)', () => { expect(history().some((c) => c.includes('deploy-state.json') && c.includes('base64 -d'))).toBe(false); }); + it('reports the original failure, and how to recover, when restoring current also fails', async () => { + executor = host() + .when((c) => c.includes('pm2 start'), { stdout: '', stderr: 'boom', exitCode: 1 }) + .when((c) => c.includes('ln -sfn') && c.includes('releases/R2'), { stdout: '', stderr: 'disk error', exitCode: 1 }); + + const reported = await rollback(); + + expect(reported).toContain('boom'); + expect(reported).not.toContain('disk error'); + const warning = vi.mocked(ui.warn).mock.calls.map((c) => String(c[0])).find((m) => m.includes('restore it with')); + expect(warning).toContain('releases/R1'); + expect(warning).toContain('ln -sfn "/var/www/app/api/releases/R2"'); + }); + it('declines cleanly and touches nothing', async () => { executor = host(); vi.mocked(confirm).mockResolvedValue(false);