fix: migrate prod db off neon free-tier to railway postgres + resilience hardening

CLAUDE.md

@@ -7,15 +7,15 @@ Distro TV is an ambient **channel surface** that runs in the developer's termina
 ## Architecture
 
 - **CLI + Daemon** — `@distrotv/cli` distributed via GitHub Releases + `curl ... | sh` install script (NOT npm). Binary: `distro`, alias `dtv`. Hooks into Claude Code via settings.json (PreToolUse, Stop, UserPromptSubmit). Daemon on Unix socket manages slot display, key capture, local ledger (SQLite).
-- **Backend API** — Express + Drizzle ORM + Neon PostgreSQL + Upstash Redis. Auth, device registration, channels, watchlists, alerts, slot impression ingestion.
+- **Backend API** — Express + Drizzle ORM + Railway Postgres + Upstash Redis. Auth, device registration, channels, watchlists, alerts, slot impression ingestion.
 - **Dashboard** — Next.js 14, App Router, Tailwind. Reading list, watchlist management, preferences.
 - **Payments** — deferred post-M1. Base Sepolia testnet targeted for M6+.
 
 ## Tech Stack
 
 - **everything TypeScript** — monorepo via Turborepo + pnpm workspaces
 - packages: `cli`, `api`, `dashboard`, `shared`
-- Express, Drizzle, Neon, Upstash, better-sqlite3, commander, tsup
+- Express, Drizzle, Railway Postgres, Upstash, better-sqlite3, commander, tsup
 - deploy: Railway GitHub autodeploy (API), Vercel (frontend + landing at distrotv.xyz), GitHub Releases (CLI tarball — tag `cli-v*` triggers `.github/workflows/release-cli.yml`) + GitHub Pages at `get.distrotv.xyz` for `install.sh` (`.github/workflows/deploy-install.yml`)
 
 ## Milestones

gitbook-docs/architecture/monorepo.md

@@ -84,7 +84,7 @@ These dispatch through Turbo. Package-level behavior lives inside each workspace
 
 ## `packages/api/.env.example`
 
-- local and Neon Postgres URLs
+- local and Railway Postgres URLs
 - JWT secret
 - Resend API key (magic-link emails)
 - Upstash Redis credentials

gitbook-docs/architecture/overview.md

@@ -12,7 +12,7 @@ flowchart LR
   C[CLI / Daemon] --> A[packages/api]
   D[Dashboard] --> A
 
-  A --> P[(Neon Postgres)]
+  A --> P[(Railway Postgres)]
   A --> R[(Upstash Redis)]
 
   S[packages/shared] --> A
@@ -88,7 +88,7 @@ claude hooks fire (PreToolUse / UserPromptSubmit)
 
 - install script lives in `frontend/public/install.sh` (single source of truth); served from GitHub Pages at `get.distrotv.xyz` (off Vercel's firewall) — Vercel keeps a fallback copy at `distrotv.xyz/install.sh`
 - waitlist intake is gone post-M1; the landing page has no form submission
-- API runtime uses Neon Postgres + Upstash Redis
+- API runtime uses Railway Postgres (co-located, private networking) + Upstash Redis
 - daemon is a per-user singleton over a Unix socket
 - local ledger (SQLite at `~/.distro/ledger.db`) is ground truth for slot impressions — backend can be down
 - hooks always exit 0 — never block the tool runner

gitbook-docs/engineering/dev-workflow.md

@@ -38,16 +38,16 @@ bash ~/.superset/worktrees/devdrip/setup-worktree.sh
 
 ### Env matrix
 
-| Variable                     | development (default)                                      | test                   | staging/production                 |
-| ---------------------------- | ---------------------------------------------------------- | ---------------------- | ---------------------------------- |
-| `NODE_ENV`                   | `development`                                              | `test` (set by vitest) | `production`                       |
-| `DISTRO_ENV`                 | `local`                                                    | `local`                | `staging` / `prod`                 |
-| `DB_TARGET`                  | `local`                                                    | unset (tests mock DB)  | `neon`                             |
-| `DATABASE_URL_LOCAL*`        | `postgres://distrotv:distrotv@localhost:5432/distrotv_dev` | —                      | —                                  |
-| `DATABASE_URL*`              | commented in `.env.shared`                                 | —                      | Railway env vars                   |
-| `GITHUB_CLIENT_*`            | n/a (GitHub OAuth removed post-M1)                         | n/a                    | n/a                                |
-| `UPSTASH_REDIS_REST_*`       | unset → in-memory `TestRedis` fallback                     | unset → `TestRedis`    | real Upstash creds                 |
-| `DISTROTV_ALLOW_NEON_IN_DEV` | unset                                                      | —                      | — (guard not active in production) |
+| Variable                     | development (default)                                      | test                   | staging/production                                                                          |
+| ---------------------------- | ---------------------------------------------------------- | ---------------------- | ------------------------------------------------------------------------------------------- |
+| `NODE_ENV`                   | `development`                                              | `test` (set by vitest) | `production`                                                                                |
+| `DISTRO_ENV`                 | `local`                                                    | `local`                | `staging` / `prod`                                                                          |
+| `DB_TARGET`                  | `local`                                                    | unset (tests mock DB)  | `railway`                                                                                   |
+| `DATABASE_URL_LOCAL*`        | `postgres://distrotv:distrotv@localhost:5432/distrotv_dev` | —                      | —                                                                                           |
+| `DATABASE_URL*`              | commented in `.env.shared`                                 | —                      | Railway env vars (points to in-project `Postgres` service via `${{Postgres.DATABASE_URL}}`) |
+| `GITHUB_CLIENT_*`            | n/a (GitHub OAuth removed post-M1)                         | n/a                    | n/a                                                                                         |
+| `UPSTASH_REDIS_REST_*`       | unset → in-memory `TestRedis` fallback                     | unset → `TestRedis`    | real Upstash creds                                                                          |
+| `DISTROTV_ALLOW_NEON_IN_DEV` | unset                                                      | —                      | — (guard not active in production)                                                          |
 
 ### `DISTRO_ENV` — single source of truth for URLs
 
@@ -135,7 +135,7 @@ Copy values from `packages/api/.env.example`.
 Important toggles:
 
 - `DB_TARGET=local` (default in dev) uses `DATABASE_URL_LOCAL` → Docker Postgres
-- `DB_TARGET=neon` uses `DATABASE_URL` → Neon (deployed envs; requires `DISTROTV_ALLOW_NEON_IN_DEV=1` in dev)
+- `DB_TARGET=railway` (prod) uses `DATABASE_URL` → Railway Postgres; `neon` is still accepted for backward compat. Any non-`local` DB_TARGET in dev requires `DISTROTV_ALLOW_NEON_IN_DEV=1`.
 
 Important runtime vars:
 

gitbook-docs/engineering/incidents.md

@@ -0,0 +1,17 @@
+# Incidents
+
+## 2026-06-23 — prod DB offline (Neon free-tier quota exhausted)
+
+**Symptom:** all CLI users saw `offline demo (backend not reachable)`; no channel content.
+
+**Root cause:** Neon free-tier compute-time quota exhausted (HTTP 402) → `devdrip`
+compute suspended (Jun 19) + branch archived (Jun 20) → API DB connections timed out →
+`/me/content/next` 500 → CLI demo fallback. Contributing: uncached `/health` `SELECT 1`
+pinned the compute active 24/7; no alerting, so the outage was silent for ~4 days.
+
+**Fix:** migrated prod Postgres to Railway Postgres (co-located, private networking),
+fresh DB (no data migration). Hardened: railway db target, cached db health probe,
+explicit pool options.
+
+**Follow-up (deferred):** alerting on `/health` degraded (external monitor or GH Actions
+cron → webhook) so a future DB outage pages instead of going silent.

packages/api/src/__tests__/migrate-url.test.ts

@@ -0,0 +1,19 @@
+import { describe, expect, it } from "vitest"
+import { resolveMigrateUrl } from "../migrate.js"
+
+describe("resolveMigrateUrl", () => {
+  it("railway → remote unpooled url", () => {
+    const r = resolveMigrateUrl({ DB_TARGET: "railway", DATABASE_URL_UNPOOLED: "postgres://r/db" })
+    expect(r).toEqual({ target: "remote", url: "postgres://r/db" })
+  })
+
+  it("neon → remote (backward compat)", () => {
+    const r = resolveMigrateUrl({ DB_TARGET: "neon", DATABASE_URL: "postgres://n/db" })
+    expect(r).toEqual({ target: "remote", url: "postgres://n/db" })
+  })
+
+  it("unset DB_TARGET defaults to local", () => {
+    const r = resolveMigrateUrl({ DATABASE_URL_LOCAL: "postgres://localhost/dev" })
+    expect(r).toEqual({ target: "local", url: "postgres://localhost/dev" })
+  })
+})

packages/api/src/config/__tests__/env.test.ts

@@ -0,0 +1,51 @@
+import { afterEach, describe, expect, it, vi } from "vitest"
+
+describe("env.dbTarget", () => {
+  const prev = process.env["DB_TARGET"]
+  afterEach(() => {
+    if (prev === undefined) delete process.env["DB_TARGET"]
+    else process.env["DB_TARGET"] = prev
+  })
+
+  it("accepts railway", async () => {
+    process.env["DB_TARGET"] = "railway"
+    const { env } = await import("../env.js")
+    expect(env.dbTarget).toBe("railway")
+  })
+
+  it("accepts neon (backward compat)", async () => {
+    process.env["DB_TARGET"] = "neon"
+    const { env } = await import("../env.js")
+    expect(env.dbTarget).toBe("neon")
+  })
+
+  it("rejects an unknown target", async () => {
+    process.env["DB_TARGET"] = "bogus"
+    const { env } = await import("../env.js")
+    expect(() => env.dbTarget).toThrow(/DB_TARGET/)
+  })
+})
+
+describe("assertEnvSafe", () => {
+  it("refuses dev + railway without the escape hatch", async () => {
+    vi.resetModules()
+    process.env["NODE_ENV"] = "development"
+    process.env["DB_TARGET"] = "railway"
+    delete process.env["DISTROTV_ALLOW_NEON_IN_DEV"]
+    const { assertEnvSafe } = await import("../env.js")
+    expect(() => assertEnvSafe()).toThrow(/refusing to start/)
+    process.env["NODE_ENV"] = "test"
+    process.env["DB_TARGET"] = "local"
+    vi.resetModules()
+  })
+
+  it("allows dev + local", async () => {
+    vi.resetModules()
+    process.env["NODE_ENV"] = "development"
+    process.env["DB_TARGET"] = "local"
+    const { assertEnvSafe } = await import("../env.js")
+    expect(() => assertEnvSafe()).not.toThrow()
+    process.env["NODE_ENV"] = "test"
+    vi.resetModules()
+  })
+})

packages/api/src/config/env.ts

@@ -26,10 +26,10 @@ export const env = {
   distroEnv: bundle.env,
   apiUrl: bundle.apiUrl,
   webUrl: bundle.webUrl,
-  get dbTarget(): "local" | "neon" {
+  get dbTarget(): "local" | "neon" | "railway" {
     const val = optionalEnv("DB_TARGET", "local")
-    if (val !== "local" && val !== "neon")
-      throw new Error(`DB_TARGET must be "local" or "neon", got "${val}"`)
+    if (val !== "local" && val !== "neon" && val !== "railway")
+      throw new Error(`DB_TARGET must be "local", "neon", or "railway", got "${val}"`)
     return val
   },
 
@@ -95,20 +95,15 @@ export const env = {
 
 // PostHog deferred.
 
-/**
- * Refuses to boot if we'd be pointing a dev process at the deployed Neon DB.
- * Escape hatch: DISTROTV_ALLOW_NEON_IN_DEV=1 for deliberate integration testing
- * against Neon from a dev machine.
- */
 export function assertEnvSafe(): void {
   if (env.nodeEnv !== "development") return
-  if (env.dbTarget !== "neon") return
+  if (env.dbTarget === "local") return
   if (process.env["DISTROTV_ALLOW_NEON_IN_DEV"] === "1") return
 
   throw new Error(
-    "refusing to start: NODE_ENV=development with DB_TARGET=neon. " +
+    `refusing to start: NODE_ENV=development with DB_TARGET=${env.dbTarget}. ` +
       "switch to local (set DB_TARGET=local and run `docker compose up -d postgres` " +
-      "from the repo root), or, to deliberately test against neon, re-run with " +
+      "from the repo root), or, to deliberately test against a remote DB, re-run with " +
       "DISTROTV_ALLOW_NEON_IN_DEV=1"
   )
 }

packages/api/src/db/__tests__/db-url.test.ts

@@ -0,0 +1,35 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest"
+import { dbUrlForTarget } from "../index.js"
+
+describe("dbUrlForTarget", () => {
+  const snapshot = { ...process.env }
+  beforeEach(() => {
+    delete process.env["DATABASE_URL"]
+    delete process.env["DATABASE_URL_UNPOOLED"]
+    delete process.env["DATABASE_URL_LOCAL"]
+  })
+  afterEach(() => {
+    process.env = { ...snapshot }
+  })
+
+  it("railway → unpooled url, falling back to DATABASE_URL", () => {
+    process.env["DATABASE_URL"] = "postgres://pooled/db"
+    expect(dbUrlForTarget("railway")).toBe("postgres://pooled/db")
+    process.env["DATABASE_URL_UNPOOLED"] = "postgres://direct/db"
+    expect(dbUrlForTarget("railway")).toBe("postgres://direct/db")
+  })
+
+  it("neon resolves the same remote contract", () => {
+    process.env["DATABASE_URL_UNPOOLED"] = "postgres://neon/db"
+    expect(dbUrlForTarget("neon")).toBe("postgres://neon/db")
+  })
+
+  it("local → DATABASE_URL_LOCAL", () => {
+    process.env["DATABASE_URL_LOCAL"] = "postgres://localhost/dev"
+    expect(dbUrlForTarget("local")).toBe("postgres://localhost/dev")
+  })
+
+  it("throws when the url is missing", () => {
+    expect(() => dbUrlForTarget("railway")).toThrow(/database url required/)
+  })
+})

packages/api/src/db/index.ts

@@ -4,22 +4,29 @@ import * as schema from "./schema/index.js"
 import { env } from "../config/env.js"
 
 // the api runs as a persistent server (railway), so we use the postgres-js
-// (tcp) driver for every target. the neon-http driver can't do transactions
-// (setSubscriptions / setWatchlists need them); the unpooled neon endpoint is
-// a real postgres connection that supports them. local dev already used this.
+// (tcp) driver for every target. any non-local target (neon | railway) is a
+// real tcp postgres reached via the unpooled url, falling back to the pooled one.
 type DbInstance = ReturnType<typeof drizzlePg<typeof schema>>
 
 let _db: DbInstance | undefined
 
+export function dbUrlForTarget(target: "local" | "neon" | "railway"): string {
+  const url =
+    target === "local"
+      ? process.env["DATABASE_URL_LOCAL"]
+      : (process.env["DATABASE_URL_UNPOOLED"] ?? process.env["DATABASE_URL"])
+  if (!url) throw new Error(`database url required for DB_TARGET=${target}`)
+  return url
+}
+
 export function getDb() {
   if (!_db) {
-    const target = env.dbTarget
-    const url =
-      target === "local"
-        ? process.env["DATABASE_URL_LOCAL"]
-        : (process.env["DATABASE_URL_UNPOOLED"] ?? process.env["DATABASE_URL"])
-    if (!url) throw new Error(`database url required for DB_TARGET=${target}`)
-    _db = drizzlePg(postgres(url), { schema })
+    const url = dbUrlForTarget(env.dbTarget)
+    // explicit pool options: bound connections, fail fast on connect, and let
+    // idle connections close so the db can scale down between bursts.
+    _db = drizzlePg(postgres(url, { max: 10, connect_timeout: 10, idle_timeout: 30 }), {
+      schema,
+    })
   }
   return _db
 }

packages/api/src/lib/__tests__/probes.test.ts

@@ -0,0 +1,44 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"
+
+const execute = vi.fn()
+vi.mock("../../db/index.js", () => ({ getDb: () => ({ execute }) }))
+vi.mock("../redis.js", () => ({ getRedis: () => ({ ping: vi.fn().mockResolvedValue("PONG") }) }))
+
+describe("probeDb caching", () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+    vi.setSystemTime(new Date("2026-06-23T00:00:00Z"))
+    execute.mockReset()
+    execute.mockResolvedValue([{ ok: 1 }])
+  })
+  afterEach(() => {
+    vi.useRealTimers()
+    vi.resetModules()
+  })
+
+  it("caches a successful probe within the TTL", async () => {
+    vi.resetModules()
+    const { probeDb } = await import("../probes.js")
+    await probeDb()
+    await probeDb()
+    expect(execute).toHaveBeenCalledTimes(1)
+  })
+
+  it("re-probes after the TTL expires", async () => {
+    vi.resetModules()
+    const { probeDb } = await import("../probes.js")
+    await probeDb()
+    vi.setSystemTime(new Date("2026-06-23T00:00:11Z"))
+    await probeDb()
+    expect(execute).toHaveBeenCalledTimes(2)
+  })
+
+  it("does not cache a failed probe", async () => {
+    vi.resetModules()
+    const { probeDb } = await import("../probes.js")
+    execute.mockRejectedValueOnce(new Error("db down"))
+    await expect(probeDb()).rejects.toThrow()
+    await probeDb()
+    expect(execute).toHaveBeenCalledTimes(2)
+  })
+})

packages/api/src/lib/probes.ts

@@ -13,8 +13,16 @@ function withTimeout(promise: Promise<unknown>, label: string): Promise<void> {
   ])
 }
 
+// monitoring can poll /health every few seconds; cache a successful db probe
+// for 10s so we don't run SELECT 1 (and keep the db compute pinned) on every
+// poll. only success is cached — a failure always re-probes and surfaces fast.
+const DB_PING_CACHE_MS = 10_000
+let lastDbOkAt = 0
+
 export async function probeDb(): Promise<void> {
+  if (Date.now() - lastDbOkAt < DB_PING_CACHE_MS) return
   await withTimeout(getDb().execute(sql`SELECT 1`), "db")
+  lastDbOkAt = Date.now()
 }
 
 // platform/monitoring can poll /health every few seconds; cache a successful

packages/api/src/migrate.ts

@@ -16,14 +16,21 @@ const migrationsFolder = [join(here, "migrations"), join(here, "../src/db/migrat
   existsSync(join(p, "meta/_journal.json"))
 )
 
-const target = process.env["DB_TARGET"] === "neon" ? "neon" : "local"
-const url =
-  target === "neon"
-    ? (process.env["DATABASE_URL_UNPOOLED"] ?? process.env["DATABASE_URL"])
-    : (process.env["DATABASE_URL_LOCAL_UNPOOLED"] ?? process.env["DATABASE_URL_LOCAL"])
+export function resolveMigrateUrl(e: NodeJS.ProcessEnv = process.env): {
+  target: "local" | "remote"
+  url: string | undefined
+} {
+  const target = (e["DB_TARGET"] ?? "local") === "local" ? "local" : "remote"
+  const url =
+    target === "remote"
+      ? (e["DATABASE_URL_UNPOOLED"] ?? e["DATABASE_URL"])
+      : (e["DATABASE_URL_LOCAL_UNPOOLED"] ?? e["DATABASE_URL_LOCAL"])
+  return { target, url }
+}
 
 async function main(): Promise<void> {
   if (!migrationsFolder) throw new Error("migrate: migrations folder not found")
+  const { target, url } = resolveMigrateUrl()
   if (!url) throw new Error(`migrate: database url required for DB_TARGET=${target}`)
   const sql = postgres(url, { max: 1 })
   try {
@@ -34,7 +41,11 @@ async function main(): Promise<void> {
   }
 }
 
-main().catch((err) => {
-  console.error("[migrate] failed:", err)
-  process.exit(1)
-})
+// only auto-run when invoked directly (node dist/migrate.js); importing for
+// tests must not trigger a live migration.
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+  main().catch((err) => {
+    console.error("[migrate] failed:", err)
+    process.exit(1)
+  })
+}