From a694b071520dea6b4d1fe060c80bacbe5e18e732 Mon Sep 17 00:00:00 2001 From: "posthog-eu[bot]" <226701856+posthog-eu[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 05:48:51 +0000 Subject: [PATCH 1/2] fix(health): limit each dependency check to 3 seconds and log the failed check The health route waited on Postgres and object storage with no time limit and discarded the error. When a dependency stopped answering, each check stayed open for one to three minutes after Docker had stopped waiting, and the logs did not show which dependency failed. Each check now runs in parallel with a 3-second limit. The limit aborts the S3 request. The route logs a warning with the check name and the error. Co-Authored-By: Claude Opus 5.5 Generated-By: PostHog Desktop Task-Id: 440fbfb3-5c8e-435c-9aac-8d27e27e647d --- src/routes/api.health.ts | 37 ++++++------------- src/server/health.test.ts | 74 ++++++++++++++++++++++++++++++++++++++ src/server/health.ts | 36 +++++++++++++++++++ src/server/object-store.ts | 4 +-- 4 files changed, 123 insertions(+), 28 deletions(-) create mode 100644 src/server/health.test.ts create mode 100644 src/server/health.ts diff --git a/src/routes/api.health.ts b/src/routes/api.health.ts index 2c38c14..ec944c1 100644 --- a/src/routes/api.health.ts +++ b/src/routes/api.health.ts @@ -1,38 +1,23 @@ import { createFileRoute } from "@tanstack/react-router" import { checkDatabase } from "#/server/db/index.ts" -import { jsonError } from "#/server/http.ts" +import { runHealthCheck } from "#/server/health.ts" import { checkObjectStore } from "#/server/object-store.ts" export const Route = createFileRoute("/api/health")({ server: { handlers: { GET: async () => { - const checks = { - database: { status: "unknown" as "unknown" | "ok" | "error" }, - objectStore: { status: "unknown" as "unknown" | "ok" | "error" }, - } - try { - await checkDatabase() - checks.database.status = "ok" - } catch { - checks.database.status = "error" - } - try { - await checkObjectStore() - checks.objectStore.status = "ok" - } catch { - checks.objectStore.status = "error" - } - const healthy = Object.values(checks).every((check) => check.status === "ok") - try { - return Response.json( - { status: healthy ? "ok" : "degraded", checks }, - { status: healthy ? 200 : 503 } - ) - } catch (error) { - return jsonError(error) - } + const [database, objectStore] = await Promise.all([ + runHealthCheck("database", () => checkDatabase()), + runHealthCheck("objectStore", (signal) => checkObjectStore(signal)), + ]) + const checks = { database: { status: database }, objectStore: { status: objectStore } } + const healthy = database === "ok" && objectStore === "ok" + return Response.json( + { status: healthy ? "ok" : "degraded", checks }, + { status: healthy ? 200 : 503 } + ) }, }, }, diff --git a/src/server/health.test.ts b/src/server/health.test.ts new file mode 100644 index 0000000..0c7158c --- /dev/null +++ b/src/server/health.test.ts @@ -0,0 +1,74 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" + +import { HEALTH_CHECK_TIMEOUT_MS, runHealthCheck } from "./health.ts" +import { initializeLogging, shutdownLogging } from "./logging.ts" + +const lines: string[] = [] + +beforeEach(() => { + vi.stubEnv("NODE_ENV", "production") + vi.stubEnv("POSTHOG_LOGS_ENABLED", "false") + lines.length = 0 + initializeLogging() + vi.spyOn(process.stdout, "write").mockImplementation((chunk) => { + lines.push(String(chunk)) + return true + }) +}) + +afterEach(async () => { + vi.useRealTimers() + await shutdownLogging() + vi.restoreAllMocks() + vi.unstubAllEnvs() +}) + +function records(): Record[] { + return lines.map((line) => JSON.parse(line) as Record) +} + +describe("health checks", () => { + it("passes when the dependency answers", async () => { + await expect(runHealthCheck("database", async () => {})).resolves.toBe("ok") + expect(lines).toEqual([]) + }) + + it("names the dependency that failed", async () => { + const status = await runHealthCheck("objectStore", async () => { + throw new Error("Access Denied") + }) + + expect(status).toBe("error") + expect(records()).toContainEqual( + expect.objectContaining({ + level: "WARN", + message: "Health check failed", + check: "objectStore", + "exception.message": "Access Denied", + }) + ) + }) + + it("fails and aborts a dependency that does not answer in time", async () => { + vi.useFakeTimers() + let aborted: unknown + const status = runHealthCheck( + "objectStore", + (signal) => + new Promise(() => { + signal.addEventListener("abort", () => (aborted = signal.reason)) + }) + ) + + await vi.advanceTimersByTimeAsync(HEALTH_CHECK_TIMEOUT_MS) + + await expect(status).resolves.toBe("error") + expect(aborted).toBeInstanceOf(Error) + expect(records()).toContainEqual( + expect.objectContaining({ + check: "objectStore", + "exception.message": `objectStore did not answer within ${HEALTH_CHECK_TIMEOUT_MS} ms`, + }) + ) + }) +}) diff --git a/src/server/health.ts b/src/server/health.ts new file mode 100644 index 0000000..4de891a --- /dev/null +++ b/src/server/health.ts @@ -0,0 +1,36 @@ +import Logger from "./logger.ts" + +const logger = new Logger("health") + +/** + * Docker and Coolify give the health check 5 seconds. Without a limit, a dependency that stops + * answering held each check open for minutes after its caller had given up, and the response + * gave no sign of which dependency was at fault. + */ +export const HEALTH_CHECK_TIMEOUT_MS = 3_000 + +export type CheckStatus = "ok" | "error" + +export async function runHealthCheck( + name: string, + check: (signal: AbortSignal) => Promise +): Promise { + const controller = new AbortController() + let timer: ReturnType | undefined + const timeout = new Promise((_, reject) => { + timer = setTimeout(() => { + const error = new Error(`${name} did not answer within ${HEALTH_CHECK_TIMEOUT_MS} ms`) + controller.abort(error) + reject(error) + }, HEALTH_CHECK_TIMEOUT_MS) + }) + try { + await Promise.race([check(controller.signal), timeout]) + return "ok" + } catch (error) { + logger.warn("Health check failed", { check: name, error }) + return "error" + } finally { + clearTimeout(timer) + } +} diff --git a/src/server/object-store.ts b/src/server/object-store.ts index b43f6e7..ddef293 100644 --- a/src/server/object-store.ts +++ b/src/server/object-store.ts @@ -52,9 +52,9 @@ export async function ensureBucket(): Promise { } } -export async function checkObjectStore(): Promise { +export async function checkObjectStore(abortSignal?: AbortSignal): Promise { await ensureBucket() - await s3.send(new HeadBucketCommand({ Bucket: configuration.S3_BUCKET })) + await s3.send(new HeadBucketCommand({ Bucket: configuration.S3_BUCKET }), { abortSignal }) } export async function putObject(input: { From 9160f5f9fdda5b96ccd7e2a8873943ccc9ef49ab Mon Sep 17 00:00:00 2001 From: "posthog-eu[bot]" <226701856+posthog-eu[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 05:49:30 +0000 Subject: [PATCH 2/2] refactor(health): pass the check functions directly Co-Authored-By: Claude Opus 5.5 Generated-By: PostHog Desktop Task-Id: 440fbfb3-5c8e-435c-9aac-8d27e27e647d --- src/routes/api.health.ts | 4 ++-- src/server/health.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/routes/api.health.ts b/src/routes/api.health.ts index ec944c1..ed3cf74 100644 --- a/src/routes/api.health.ts +++ b/src/routes/api.health.ts @@ -9,8 +9,8 @@ export const Route = createFileRoute("/api/health")({ handlers: { GET: async () => { const [database, objectStore] = await Promise.all([ - runHealthCheck("database", () => checkDatabase()), - runHealthCheck("objectStore", (signal) => checkObjectStore(signal)), + runHealthCheck("database", checkDatabase), + runHealthCheck("objectStore", checkObjectStore), ]) const checks = { database: { status: database }, objectStore: { status: objectStore } } const healthy = database === "ok" && objectStore === "ok" diff --git a/src/server/health.ts b/src/server/health.ts index 4de891a..4122d85 100644 --- a/src/server/health.ts +++ b/src/server/health.ts @@ -9,7 +9,7 @@ const logger = new Logger("health") */ export const HEALTH_CHECK_TIMEOUT_MS = 3_000 -export type CheckStatus = "ok" | "error" +type CheckStatus = "ok" | "error" export async function runHealthCheck( name: string,