From 1d139c751857b7ad0a674d544db4f0694bb792e7 Mon Sep 17 00:00:00 2001 From: NinjaLikesCheez Date: Sun, 18 Oct 2026 12:42:00 +0200 Subject: [PATCH] Skip the Sentry heartbeat when PIXEL_ENV=local A laptop running `just dev` with a DSN checked in to a shared `pixel-local` cron monitor, so stopping it opened an issue about ten minutes later. `planHeartbeat` now decides whether to run it (off without a DSN, with PIXEL_HEARTBEAT_MINUTES=0, or for local), and startup logs `heartbeat.off` with the reason. Co-Authored-By: Claude Opus 5.5 --- .env.example | 2 +- CHANGELOG.md | 1 + docs/architecture.md | 4 ++-- docs/runbook.md | 2 +- src/index.ts | 13 ++++++----- src/observability/heartbeat.test.ts | 35 ++++++++++++++++++++++++++++- src/observability/heartbeat.ts | 21 +++++++++++++++++ 7 files changed, 68 insertions(+), 10 deletions(-) diff --git a/.env.example b/.env.example index 36d367d..e7cfe52 100644 --- a/.env.example +++ b/.env.example @@ -87,7 +87,7 @@ SENTRY_DSN= # How many of those traces get a CPU profile (0–1). Default 1. 0 turns profiling off. # SENTRY_PROFILE_SESSION_SAMPLE_RATE=1 # With Sentry set up, Pixel checks in with a Sentry cron monitor (pixel-) this often, so Sentry can -# alert when it goes quiet. 0 turns it off. Default 5. +# alert when it goes quiet. 0 turns it off. Default 5. Never runs with PIXEL_ENV=local. # PIXEL_HEARTBEAT_MINUTES=5 # PIXEL_ADMINS_FILE=config/admins.yaml # PIXEL_MEMBERS_FILE=config/members.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 423cd5b..006f7d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,7 @@ Until 1.0.0, releases stay on 0.x: `Added` bumps minor, `Fixed`/`Changed` (and t - `/schedule`: `when` no longer guesses. Four digits after a date are read the same with or without a month name: a time (`14 nov 1900` is 19:00, `14 nov 2030` is 20:30, as `14-11 2030` already was), and a year only beside another time (`14 nov 2030 19:00`) or when no clock reads them (`14 nov 2099`). Months must be English names or their short forms (`sep`, `sept`, `september`), so words like `junk`, `decent` or `mars` are refused instead of read as June, December or March. - CD on `main` no longer loses an image: runs queue instead of cancelling each other, and each one publishes when the image inputs differ from the last run that published, not just from the previous push. Before, a docs-only merge right after a code merge cancelled that merge's publish and then skipped its own, so the code merge never got a GHCR image or a Sentry release. - The Discord DM sent when a capability is granted or revoked now really has mentions turned off. It was sent with `allowedMentions`, a field the Discord API ignores, instead of `allowed_mentions`. +- The Sentry heartbeat no longer runs when `PIXEL_ENV=local`. A laptop running `just dev` with a DSN used to check in to a shared `pixel-local` cron monitor, which opened an issue about ten minutes after anyone stopped it. Startup now logs `heartbeat.off` with the reason. `pixel-dev` and `pixel-prod` are unchanged. ## [0.17.0] - 2026-10-15 diff --git a/docs/architecture.md b/docs/architecture.md index 10287d6..dca1794 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -412,7 +412,7 @@ The message **body** is a file (`PIXEL_CLOSING_TIME_FILE`, default `data/closing - **Sentry Logs:** Sentry's pino integration (set up in `observability/sentry-options.ts`) sends `info` and above as Sentry Logs, whatever `LOG_LEVEL` is (debug stays in the console and the file), with each line's fields as attributes. They aren't turned into error events: real failures still reach Sentry as events through the `ErrorReporter`. Nothing is sent when no DSN is set. - **Secrets:** pino redacts `token` and authorization headers by name, and every line written (console and file) has anything shaped like a bot or Home Assistant token masked on the way out. Sentry's copy is scrubbed again in `beforeSendLog`. - **Failures nobody caught:** an unhandled promise rejection and an uncaught exception are logged too (`process.unhandled_rejection`, `process.uncaught_exception`), so they reach the file and Sentry as well as the console. Neither changes whether the process exits: a rejection is logged and the bot carries on (like Sentry's default), and an exception still ends the process. -- **Heartbeat** (`observability/heartbeat.ts`): a dead bot can't report its own death, and Discord doesn't health-check bots, so Pixel checks in with a **Sentry cron monitor** (`pixel-`, one per environment) every `PIXEL_HEARTBEAT_MINUTES` (default 5), and when Discord connects. A check-in is `ok` while the Discord gateway is connected and `error` while the process is up but disconnected. Sentry creates and updates the monitor from the first check-in (interval schedule, five minutes' margin so a deploy or restart doesn't count, one missed or failed check-in opens an issue, one good one resolves it). So a crash, a hang, a lost Discord connection or the host going down becomes a Sentry issue within about ten minutes. Off without a DSN or with `PIXEL_HEARTBEAT_MINUTES=0`. A failing check-in is logged and never affects the bot. +- **Heartbeat** (`observability/heartbeat.ts`): a dead bot can't report its own death, and Discord doesn't health-check bots, so Pixel checks in with a **Sentry cron monitor** (`pixel-`, one per environment) every `PIXEL_HEARTBEAT_MINUTES` (default 5), and when Discord connects. A check-in is `ok` while the Discord gateway is connected and `error` while the process is up but disconnected. Sentry creates and updates the monitor from the first check-in (interval schedule, five minutes' margin so a deploy or restart doesn't count, one missed or failed check-in opens an issue, one good one resolves it). So a crash, a hang, a lost Discord connection or the host going down becomes a Sentry issue within about ten minutes. Off without a DSN, with `PIXEL_HEARTBEAT_MINUTES=0`, or when `PIXEL_ENV=local` (`planHeartbeat` decides, and startup logs `heartbeat.off` with the reason): laptops would otherwise share one `pixel-local` monitor that opens an issue every time someone stops `just dev`. A failing check-in is logged and never affects the bot. - **Health:** a small HTTP server exposes `/healthz` (process alive) and `/readyz` (Discord gateway connected). ## Configuration @@ -455,7 +455,7 @@ Environment variables are validated by `config.ts` (zod). Nothing else reads `pr | `SENTRY_TRACES_SAMPLE_RATE` | | Default `1`. How many traces Sentry keeps (`0`–`1`). Needs `SENTRY_DSN`. Errors are always kept. | | `SENTRY_PROFILE_SESSION_SAMPLE_RATE` | | Default `1`. How many of those traces get a CPU profile (`0`–`1`). Needs `SENTRY_DSN`. | | `LOG_LEVEL` | | Default `info` | -| `PIXEL_HEARTBEAT_MINUTES` | | Default `5`. How often Pixel checks in with its Sentry cron monitor (`pixel-`), so Sentry alerts when it goes quiet. 0 turns it off. Needs `SENTRY_DSN` | +| `PIXEL_HEARTBEAT_MINUTES` | | Default `5`. How often Pixel checks in with its Sentry cron monitor (`pixel-`), so Sentry alerts when it goes quiet. 0 turns it off. Needs `SENTRY_DSN`; never runs with `PIXEL_ENV=local` | | `PIXEL_LOG_DIR` | | Default `data/logs`. A rotating JSON log file is written here (about two weeks; owner-only on a normal filesystem, not on the Azure Files share). Empty turns the file off; the console and Sentry Logs still get every line | | `HEALTH_PORT` | | Default `8080` | | `SPACEAPI_URL` | | Default `https://spaceapi.pixelbar.nl/`; http(s) only | diff --git a/docs/runbook.md b/docs/runbook.md index 8f1ce06..7114d07 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -597,7 +597,7 @@ Sentry gets errors (with the tags `command`, `feature`, `platform`, `tier` and t Errors are always sent (`sampleRate` 1). Traces use `SENTRY_TRACES_SAMPLE_RATE` (default 1: keep them all). Profiles use `SENTRY_PROFILE_SESSION_SAMPLE_RATE` (default 1). `/healthz` and `/readyz` are not traced. - **Where alerts go:** 🚧 not decided. Set up alert rules for new issues in prod, and write here who is notified and where (email, Discord channel). -- **Pixel going quiet:** Pixel checks in with a Sentry cron monitor, `pixel-prod` or `pixel-dev` (Sentry → **Crons**), every 5 minutes while it's connected to Discord. If check-ins stop (a crash, a hang, the host down) or say it's disconnected, the monitor opens an issue within about ten minutes, and closes it by itself when Pixel is back. In the monitor's settings, set who is alerted. 🚧 Record here who that is. +- **Pixel going quiet:** Pixel checks in with a Sentry cron monitor, `pixel-prod` or `pixel-dev` (Sentry → **Crons**), every 5 minutes while it's connected to Discord. If check-ins stop (a crash, a hang, the host down) or say it's disconnected, the monitor opens an issue within about ten minutes, and closes it by itself when Pixel is back. A laptop `just dev` (`PIXEL_ENV=local`) never checks in, even with a DSN, so stopping it doesn't alert anyone; if Sentry still lists an old `pixel-local` monitor, delete it. In the monitor's settings, set who is alerted. 🚧 Record here who that is. - **Planned downtime:** a restart or deploy fits in the five-minute margin. For anything longer, mute the monitor in Sentry first (Crons → the monitor → Mute) and unmute it after, or you'll be alerted. - **Triage an error:** 1. Open the issue and read the stack, the tags and the user. The user is their ID, so you can find what they did in the logs (see [Moderation](#moderation)). diff --git a/src/index.ts b/src/index.ts index cd75f3b..f811860 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,7 +11,7 @@ import { loadBuildInfo } from "./observability/build-info.ts"; import { startHealthServer } from "./observability/health.ts"; import { type Heartbeat, - monitorSlug, + planHeartbeat, sentryCheckIn, startHeartbeat, } from "./observability/heartbeat.ts"; @@ -126,14 +126,17 @@ async function main(): Promise { startMastodonPublisher({ mastodon: config.mastodon, announcer, logger }); const health = startHealthServer(config.healthPort, () => discord.isReady()); // Check in with Sentry while healthy, so it can alert when Pixel goes quiet (a crash, a - // hang, the host going down). Only with Sentry set up. - if (config.sentryDsn && config.heartbeatMinutes > 0) { + // hang, the host going down). Only with Sentry set up, and never from a laptop. + const heartbeatPlan = planHeartbeat(config); + if ("monitorSlug" in heartbeatPlan) { heartbeat = startHeartbeat({ - intervalMs: config.heartbeatMinutes * 60_000, + intervalMs: heartbeatPlan.intervalMinutes * 60_000, isHealthy: () => discord.isReady(), - checkIn: sentryCheckIn(monitorSlug(config.env), config.heartbeatMinutes), + checkIn: sentryCheckIn(heartbeatPlan.monitorSlug, heartbeatPlan.intervalMinutes), logger, }); + } else { + logger.info({ event: "heartbeat.off", reason: heartbeatPlan.off }, "heartbeat is off"); } spaceStatus.start(); diff --git a/src/observability/heartbeat.test.ts b/src/observability/heartbeat.test.ts index 3510fe0..b73acac 100644 --- a/src/observability/heartbeat.test.ts +++ b/src/observability/heartbeat.test.ts @@ -1,7 +1,7 @@ import * as Sentry from "@sentry/node"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import { silentLogger } from "../core/logger.ts"; -import { monitorSlug, sentryCheckIn, startHeartbeat } from "./heartbeat.ts"; +import { monitorSlug, planHeartbeat, sentryCheckIn, startHeartbeat } from "./heartbeat.ts"; import { sentryOptions } from "./sentry-options.ts"; describe("startHeartbeat", () => { @@ -90,6 +90,39 @@ describe("monitorSlug", () => { }); }); +describe("planHeartbeat", () => { + const dsn = "https://public@sentry.example.com/1"; + + it("checks in to the environment's monitor in dev and prod", () => { + expect(planHeartbeat({ env: "dev", sentryDsn: dsn, heartbeatMinutes: 5 })).toEqual({ + monitorSlug: "pixel-dev", + intervalMinutes: 5, + }); + expect(planHeartbeat({ env: "prod", sentryDsn: dsn, heartbeatMinutes: 10 })).toEqual({ + monitorSlug: "pixel-prod", + intervalMinutes: 10, + }); + }); + + it("is off on a laptop, even with a DSN, so developers don't share a monitor that alerts when they stop", () => { + expect(planHeartbeat({ env: "local", sentryDsn: dsn, heartbeatMinutes: 5 })).toEqual({ + off: "local", + }); + }); + + it("is off without Sentry or when turned off", () => { + expect(planHeartbeat({ env: "prod", sentryDsn: undefined, heartbeatMinutes: 5 })).toEqual({ + off: "no-sentry", + }); + expect(planHeartbeat({ env: "prod", sentryDsn: "", heartbeatMinutes: 5 })).toEqual({ + off: "no-sentry", + }); + expect(planHeartbeat({ env: "prod", sentryDsn: dsn, heartbeatMinutes: 0 })).toEqual({ + off: "disabled", + }); + }); +}); + describe("sentryCheckIn, with a real Sentry client", () => { const envelopes: unknown[][] = []; beforeAll(() => { diff --git a/src/observability/heartbeat.ts b/src/observability/heartbeat.ts index cee3859..b3dea8d 100644 --- a/src/observability/heartbeat.ts +++ b/src/observability/heartbeat.ts @@ -69,3 +69,24 @@ export function sentryCheckIn(monitorSlug: string, intervalMinutes: number): Che /** The monitor's name in Sentry, one per environment, so dev doesn't page about prod or the other way round. */ export const monitorSlug = (env: string): string => `pixel-${env}`; + +/** The monitor the heartbeat checks in to, or why it doesn't run. */ +export type HeartbeatPlan = + | { monitorSlug: string; intervalMinutes: number } + | { off: "no-sentry" | "disabled" | "local" }; + +/** + * Whether to run the heartbeat. Only a deployed environment gets one: laptops running + * `just dev` with a DSN would all share one `pixel-local` monitor, and every time someone + * stopped theirs the monitor would open an issue. + */ +export function planHeartbeat(config: { + env: string; + sentryDsn: string | undefined; + heartbeatMinutes: number; +}): HeartbeatPlan { + if (!config.sentryDsn) return { off: "no-sentry" }; + if (config.heartbeatMinutes <= 0) return { off: "disabled" }; + if (config.env === "local") return { off: "local" }; + return { monitorSlug: monitorSlug(config.env), intervalMinutes: config.heartbeatMinutes }; +}