diff --git a/src/routes/api.health.ts b/src/routes/api.health.ts index 2c38c14..ed3cf74 100644 --- a/src/routes/api.health.ts +++ b/src/routes/api.health.ts @@ -1,38 +1,23 @@ import { createFileRoute } from "@tanstack/react-router" import { checkDatabase } from "#/server/db/index.ts" -import { jsonError } from "#/server/http.ts" +import { runHealthCheck } from "#/server/health.ts" import { checkObjectStore } from "#/server/object-store.ts" export const Route = createFileRoute("/api/health")({ server: { handlers: { GET: async () => { - const checks = { - database: { status: "unknown" as "unknown" | "ok" | "error" }, - objectStore: { status: "unknown" as "unknown" | "ok" | "error" }, - } - try { - await checkDatabase() - checks.database.status = "ok" - } catch { - checks.database.status = "error" - } - try { - await checkObjectStore() - checks.objectStore.status = "ok" - } catch { - checks.objectStore.status = "error" - } - const healthy = Object.values(checks).every((check) => check.status === "ok") - try { - return Response.json( - { status: healthy ? "ok" : "degraded", checks }, - { status: healthy ? 200 : 503 } - ) - } catch (error) { - return jsonError(error) - } + const [database, objectStore] = await Promise.all([ + runHealthCheck("database", checkDatabase), + runHealthCheck("objectStore", checkObjectStore), + ]) + const checks = { database: { status: database }, objectStore: { status: objectStore } } + const healthy = database === "ok" && objectStore === "ok" + return Response.json( + { status: healthy ? "ok" : "degraded", checks }, + { status: healthy ? 200 : 503 } + ) }, }, }, diff --git a/src/server/health.test.ts b/src/server/health.test.ts new file mode 100644 index 0000000..0c7158c --- /dev/null +++ b/src/server/health.test.ts @@ -0,0 +1,74 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" + +import { HEALTH_CHECK_TIMEOUT_MS, runHealthCheck } from "./health.ts" +import { initializeLogging, shutdownLogging } from "./logging.ts" + +const lines: string[] = [] + +beforeEach(() => { + vi.stubEnv("NODE_ENV", "production") + vi.stubEnv("POSTHOG_LOGS_ENABLED", "false") + lines.length = 0 + initializeLogging() + vi.spyOn(process.stdout, "write").mockImplementation((chunk) => { + lines.push(String(chunk)) + return true + }) +}) + +afterEach(async () => { + vi.useRealTimers() + await shutdownLogging() + vi.restoreAllMocks() + vi.unstubAllEnvs() +}) + +function records(): Record[] { + return lines.map((line) => JSON.parse(line) as Record) +} + +describe("health checks", () => { + it("passes when the dependency answers", async () => { + await expect(runHealthCheck("database", async () => {})).resolves.toBe("ok") + expect(lines).toEqual([]) + }) + + it("names the dependency that failed", async () => { + const status = await runHealthCheck("objectStore", async () => { + throw new Error("Access Denied") + }) + + expect(status).toBe("error") + expect(records()).toContainEqual( + expect.objectContaining({ + level: "WARN", + message: "Health check failed", + check: "objectStore", + "exception.message": "Access Denied", + }) + ) + }) + + it("fails and aborts a dependency that does not answer in time", async () => { + vi.useFakeTimers() + let aborted: unknown + const status = runHealthCheck( + "objectStore", + (signal) => + new Promise(() => { + signal.addEventListener("abort", () => (aborted = signal.reason)) + }) + ) + + await vi.advanceTimersByTimeAsync(HEALTH_CHECK_TIMEOUT_MS) + + await expect(status).resolves.toBe("error") + expect(aborted).toBeInstanceOf(Error) + expect(records()).toContainEqual( + expect.objectContaining({ + check: "objectStore", + "exception.message": `objectStore did not answer within ${HEALTH_CHECK_TIMEOUT_MS} ms`, + }) + ) + }) +}) diff --git a/src/server/health.ts b/src/server/health.ts new file mode 100644 index 0000000..4122d85 --- /dev/null +++ b/src/server/health.ts @@ -0,0 +1,36 @@ +import Logger from "./logger.ts" + +const logger = new Logger("health") + +/** + * Docker and Coolify give the health check 5 seconds. Without a limit, a dependency that stops + * answering held each check open for minutes after its caller had given up, and the response + * gave no sign of which dependency was at fault. + */ +export const HEALTH_CHECK_TIMEOUT_MS = 3_000 + +type CheckStatus = "ok" | "error" + +export async function runHealthCheck( + name: string, + check: (signal: AbortSignal) => Promise +): Promise { + const controller = new AbortController() + let timer: ReturnType | undefined + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => { + const error = new Error(`${name} did not answer within ${HEALTH_CHECK_TIMEOUT_MS} ms`) + controller.abort(error) + reject(error) + }, HEALTH_CHECK_TIMEOUT_MS) + }) + try { + await Promise.race([check(controller.signal), timeout]) + return "ok" + } catch (error) { + logger.warn("Health check failed", { check: name, error }) + return "error" + } finally { + clearTimeout(timer) + } +} diff --git a/src/server/object-store.ts b/src/server/object-store.ts index b43f6e7..ddef293 100644 --- a/src/server/object-store.ts +++ b/src/server/object-store.ts @@ -52,9 +52,9 @@ export async function ensureBucket(): Promise { } } -export async function checkObjectStore(): Promise { +export async function checkObjectStore(abortSignal?: AbortSignal): Promise { await ensureBucket() - await s3.send(new HeadBucketCommand({ Bucket: configuration.S3_BUCKET })) + await s3.send(new HeadBucketCommand({ Bucket: configuration.S3_BUCKET }), { abortSignal }) } export async function putObject(input: {