fix(health): limit each dependency check to 3 seconds and log the failed check

src/routes/api.health.ts

@@ -1,38 +1,23 @@
 import { createFileRoute } from "@tanstack/react-router"
 
 import { checkDatabase } from "#/server/db/index.ts"
-import { jsonError } from "#/server/http.ts"
+import { runHealthCheck } from "#/server/health.ts"
 import { checkObjectStore } from "#/server/object-store.ts"
 
 export const Route = createFileRoute("/api/health")({
   server: {
     handlers: {
       GET: async () => {
-        const checks = {
-          database: { status: "unknown" as "unknown" | "ok" | "error" },
-          objectStore: { status: "unknown" as "unknown" | "ok" | "error" },
-        }
-        try {
-          await checkDatabase()
-          checks.database.status = "ok"
-        } catch {
-          checks.database.status = "error"
-        }
-        try {
-          await checkObjectStore()
-          checks.objectStore.status = "ok"
-        } catch {
-          checks.objectStore.status = "error"
-        }
-        const healthy = Object.values(checks).every((check) => check.status === "ok")
-        try {
-          return Response.json(
-            { status: healthy ? "ok" : "degraded", checks },
-            { status: healthy ? 200 : 503 }
-          )
-        } catch (error) {
-          return jsonError(error)
-        }
+        const [database, objectStore] = await Promise.all([
+          runHealthCheck("database", checkDatabase),
+          runHealthCheck("objectStore", checkObjectStore),
+        ])
+        const checks = { database: { status: database }, objectStore: { status: objectStore } }
+        const healthy = database === "ok" && objectStore === "ok"
+        return Response.json(
+          { status: healthy ? "ok" : "degraded", checks },
+          { status: healthy ? 200 : 503 }
+        )
       },
     },
   },

src/server/health.test.ts

@@ -0,0 +1,74 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"
+
+import { HEALTH_CHECK_TIMEOUT_MS, runHealthCheck } from "./health.ts"
+import { initializeLogging, shutdownLogging } from "./logging.ts"
+
+const lines: string[] = []
+
+beforeEach(() => {
+  vi.stubEnv("NODE_ENV", "production")
+  vi.stubEnv("POSTHOG_LOGS_ENABLED", "false")
+  lines.length = 0
+  initializeLogging()
+  vi.spyOn(process.stdout, "write").mockImplementation((chunk) => {
+    lines.push(String(chunk))
+    return true
+  })
+})
+
+afterEach(async () => {
+  vi.useRealTimers()
+  await shutdownLogging()
+  vi.restoreAllMocks()
+  vi.unstubAllEnvs()
+})
+
+function records(): Record<string, unknown>[] {
+  return lines.map((line) => JSON.parse(line) as Record<string, unknown>)
+}
+
+describe("health checks", () => {
+  it("passes when the dependency answers", async () => {
+    await expect(runHealthCheck("database", async () => {})).resolves.toBe("ok")
+    expect(lines).toEqual([])
+  })
+
+  it("names the dependency that failed", async () => {
+    const status = await runHealthCheck("objectStore", async () => {
+      throw new Error("Access Denied")
+    })
+
+    expect(status).toBe("error")
+    expect(records()).toContainEqual(
+      expect.objectContaining({
+        level: "WARN",
+        message: "Health check failed",
+        check: "objectStore",
+        "exception.message": "Access Denied",
+      })
+    )
+  })
+
+  it("fails and aborts a dependency that does not answer in time", async () => {
+    vi.useFakeTimers()
+    let aborted: unknown
+    const status = runHealthCheck(
+      "objectStore",
+      (signal) =>
+        new Promise<void>(() => {
+          signal.addEventListener("abort", () => (aborted = signal.reason))
+        })
+    )
+
+    await vi.advanceTimersByTimeAsync(HEALTH_CHECK_TIMEOUT_MS)
+
+    await expect(status).resolves.toBe("error")
+    expect(aborted).toBeInstanceOf(Error)
+    expect(records()).toContainEqual(
+      expect.objectContaining({
+        check: "objectStore",
+        "exception.message": `objectStore did not answer within ${HEALTH_CHECK_TIMEOUT_MS} ms`,
+      })
+    )
+  })
+})

src/server/health.ts

@@ -0,0 +1,36 @@
+import Logger from "./logger.ts"
+
+const logger = new Logger("health")
+
+/**
+ * Docker and Coolify give the health check 5 seconds. Without a limit, a dependency that stops
+ * answering held each check open for minutes after its caller had given up, and the response
+ * gave no sign of which dependency was at fault.
+ */
+export const HEALTH_CHECK_TIMEOUT_MS = 3_000
+
+type CheckStatus = "ok" | "error"
+
+export async function runHealthCheck(
+  name: string,
+  check: (signal: AbortSignal) => Promise<void>
+): Promise<CheckStatus> {
+  const controller = new AbortController()
+  let timer: ReturnType<typeof setTimeout> | undefined
+  const timeout = new Promise<never>((_, reject) => {
+    timer = setTimeout(() => {
+      const error = new Error(`${name} did not answer within ${HEALTH_CHECK_TIMEOUT_MS} ms`)
+      controller.abort(error)
+      reject(error)
+    }, HEALTH_CHECK_TIMEOUT_MS)
+  })
+  try {
+    await Promise.race([check(controller.signal), timeout])
+    return "ok"
+  } catch (error) {
+    logger.warn("Health check failed", { check: name, error })
+    return "error"
+  } finally {
+    clearTimeout(timer)
+  }
+}

src/server/object-store.ts

@@ -52,9 +52,9 @@ export async function ensureBucket(): Promise<void> {
   }
 }
 
-export async function checkObjectStore(): Promise<void> {
+export async function checkObjectStore(abortSignal?: AbortSignal): Promise<void> {
   await ensureBucket()
-  await s3.send(new HeadBucketCommand({ Bucket: configuration.S3_BUCKET }))
+  await s3.send(new HeadBucketCommand({ Bucket: configuration.S3_BUCKET }), { abortSignal })
 }
 
 export async function putObject(input: {