From c86e9b6d99f40a89073220cd2ccc6ccedb1809f7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 17:45:37 +0000 Subject: [PATCH 1/2] Add a signed space state webhook so open/closed is noticed at once The space state service can POST /hooks/space-state when the space opens or closes. Pixel checks the HMAC-SHA256 signature (X-Spacestate-Timestamp, X-Spacestate-Signature, 5-minute window) and then only re-reads SpaceAPI: nothing in the body is trusted, logged or echoed. SpaceStatus.recheck() never reuses a check already under way, and a change found that way is confirmed after 5 s instead of a whole poll, so announcements go out in about 5 s rather than 30-60. Polling is unchanged and stays the fallback. The webhook listens on its own port (WEBHOOK_PORT, 8081) only when SPACESTATE_WEBHOOK_SECRET is set, so the health probes stay private. Terraform gains spacestate_webhook_enabled (external HTTPS ingress to 8081 only, plus the Key Vault secret spacestate-webhook-secret), optional webhook_allowed_ip_ranges and the spacestate_webhook_url output. Sentry now traces only incoming /hooks/ requests. See ADR 0018. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q6YEJeThAVV24LzGijYXTB --- .env.example | 8 + AGENTS.md | 2 + CHANGELOG.md | 4 + Dockerfile | 3 +- docs/.vitepress/config.ts | 4 + docs/adr/0004-azure-container-apps.md | 2 +- docs/adr/0009-container-apps-dev.md | 2 +- docs/adr/0018-space-state-webhook.md | 38 ++++ docs/architecture.md | 28 ++- docs/infra.md | 27 ++- docs/maintainers/index.md | 1 + docs/runbook.md | 35 +++- infra/envs/dev/README.md | 4 +- infra/envs/dev/main.tf | 2 + infra/envs/dev/outputs.tf | 5 + infra/envs/dev/terraform.example.tfvars | 5 + infra/envs/dev/variables.tf | 12 ++ infra/envs/prod/README.md | 2 +- infra/envs/prod/main.tf | 2 + infra/envs/prod/outputs.tf | 5 + infra/envs/prod/terraform.example.tfvars | 5 + infra/envs/prod/variables.tf | 12 ++ infra/modules/pixel/container_app.tf | 53 +++++ infra/modules/pixel/locals.tf | 6 + infra/modules/pixel/outputs.tf | 9 +- infra/modules/pixel/variables.tf | 17 ++ infra/secrets.manifest.yaml | 8 + src/adapters/webhook/server.test.ts | 236 +++++++++++++++++++++++ src/adapters/webhook/server.ts | 113 +++++++++++ src/adapters/webhook/signature.test.ts | 80 ++++++++ src/adapters/webhook/signature.ts | 57 ++++++ src/app.test.ts | 1 + src/config.test.ts | 42 ++++ src/config.ts | 20 ++ src/features/features.test.ts | 1 + src/features/status/announce.test.ts | 34 +++- src/features/status/announce.ts | 12 +- src/index.ts | 13 ++ src/observability/sentry-options.test.ts | 16 +- src/observability/sentry-options.ts | 17 +- src/services/space-status.test.ts | 86 ++++++++- src/services/space-status.ts | 45 ++++- vitest.config.ts | 7 + 43 files changed, 1027 insertions(+), 54 deletions(-) create mode 100644 docs/adr/0018-space-state-webhook.md create mode 100644 src/adapters/webhook/server.test.ts create mode 100644 src/adapters/webhook/server.ts create mode 100644 src/adapters/webhook/signature.test.ts create mode 100644 src/adapters/webhook/signature.ts diff --git a/.env.example b/.env.example index e7cfe52..7a7e5c9 100644 --- a/.env.example +++ b/.env.example @@ -67,6 +67,14 @@ DISCORD_ROLE_FRIEND= # with that call, as a JSON object of names to values. # SPACESTATE_ADDITIONAL_HEADERS={"X-Example": "value"} +# Optional: the space state service calls Pixel's webhook when the space opens or closes, so the +# announcement goes out in seconds instead of at the next SpaceAPI poll (polling carries on as the +# fallback). At least 32 characters (openssl rand -hex 32), shared with that service, which signs each +# call with it. A secret (Key Vault spacestate-webhook-secret when hosted). Unset: no webhook, polling only. +# SPACESTATE_WEBHOOK_SECRET= +# The webhook's port: the only one meant to be reachable from outside. Must differ from HEALTH_PORT. +# WEBHOOK_PORT=8081 + # Optional: control Home Assistant. Set both or neither. Use the address Pixel can reach it at (for # example your Nabu Casa cloud URL) and a long-lived access token made by a NON-ADMIN Home Assistant # user: Home Assistant can't limit a token to certain devices, so a non-admin user is the best fence. diff --git a/AGENTS.md b/AGENTS.md index 3302f96..212637b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,6 +67,7 @@ src/ services/ # access-config, space-status, info-content adapters/discord/ # interactive + publishers + calendar source adapters/mastodon/ # outbound publisher only (toots); HTTP client stays here + adapters/webhook/ # the one inbound HTTP endpoint: the signed space state webhook observability/ # logger, Sentry helpers, secret scrubbing, health config/ # *.example.yaml committed; real admins.yaml / members.yaml gitignored data/ # runtime state; gitignored. Safe to delete EXCEPT schedules.yaml (the scheduled posts) @@ -109,6 +110,7 @@ infra/ # Terraform. bootstrap/ is #8; modules/pixel and envs/de - **User-set posts:** scheduled messages and polls, the closing-time file, and any later user-authored announcement go through the core interpolator (`core/interpolate.ts`) at **post time**. Features call `interpolator.interpolate` right before send (for `/schedule`, pass the scheduled occurrence, not a late send). Date tokens accept a whole-day offset (`{{date+2}}`). Add a token by registering a provider on that interpolator (see `dateTokens`); do not grow an if/else in each feature. Unknown tokens are left as written (`{{name}}`). Don't put tokens in Pixel's own canned replies. After interpolating, `escapeMarkdown` any user-authored text that is embedded in a reply or a portable announcement field — interpolate first so date values are not escaped twice. Never log the template or the interpolated body. - **Announcements:** features hand an announcement (a typed kind from `core/announcement.ts`) to the `Announcer` and never call a publisher or a platform directly. Publishers decide how each kind looks on their platform. Never hard-code platform accounts, handles or channels: they come from config. Announcement text must be safe to show anywhere, Discord posts must keep mentions disabled, and Mastodon toots must not mention anyone. Which channels get open/closed announcements, and in which style (live, timeline, single), is set at runtime by admins through the core `AnnouncementRoutes` (`/admin announcements`, saved in `data/announcement-routes.yaml`); the Discord adapter runs each pairing as its own publisher. Don't hard-code channels, and don't add a style without the same three things: a render in `announce-render.ts`, a publisher, and tests. - **Mastodon:** outbound only. Features never import it. The HTTP client lives only in `src/adapters/mastodon/`. `MASTODON_ACCESS_TOKEN` is a secret (never log it, put it in a message or a test fixture that looks real). Both `MASTODON_INSTANCE` and the token or neither: missing config leaves the publisher off and Discord still works. Tests map kinds to toot text with plain objects and mock HTTP at the client boundary, never a live instance. +- **Webhook (inbound HTTP):** the only public endpoint is `POST /hooks/space-state` in `src/adapters/webhook/` ([ADR 0018](docs/adr/0018-space-state-webhook.md)). It is a nudge, never a source of truth: an accepted call only runs `spaceStatus.recheck()`, which reads SpaceAPI again. Never act on, log or echo anything in the body, and never let it set or announce a state. Every request is checked against `SPACESTATE_WEBHOOK_SECRET` (a secret) before anything else, refusals say only `unauthorized` (the log has the reason), and it never replaces polling. It listens on `WEBHOOK_PORT`, never on `HEALTH_PORT`: Azure ingress exposes only the webhook port, so don't add routes to the health server or point ingress at it. Adding another inbound route needs its own signature check, tests and an ADR. - **Calendar:** features read events from the core `Calendar` (neutral `CalendarEvent`s) and never touch a platform. An adapter plugs in a `CalendarSource` when it's ready. **Text written by other people** (event titles, locations and so on) must go through `escapeMarkdown` before it's put in a reply. - **Content:** `content/info/*.md` is committed to a **public** repository, so never put secrets (wifi passwords, door codes, tokens) or personal data in it, and don't add member-only topics. Keep topics short and link to the canonical page on pixelbar.nl rather than copying it. Don't invent facts: every claim should come from Pixelbar's own pages or from the board. Adding, removing or renaming a topic needs the commands re-registered (`just register dev` or `just register prod`), and CI loads the real content, so keep it valid. - **Background work:** a feature that needs it declares `start()`, which returns a stop function. Don't start timers or listeners at import time or in a factory. Anything that runs from a timer must catch its own failures so none escapes as an unhandled rejection. diff --git a/CHANGELOG.md b/CHANGELOG.md index a95af2f..e6218c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Until 1.0.0, releases stay on 0.x: `Added` bumps minor, `Fixed`/`Changed` (and t ## [Unreleased] +### Added + +- Space state webhook: the space state service can call `POST /hooks/space-state` when the space opens or closes, and Pixel reads SpaceAPI straight away instead of waiting for the next 30-second poll. A change found this way is confirmed after 5 seconds, so open/closed announcements go out in about 5 seconds rather than 30–60. Polling carries on unchanged as the fallback. Requests must be signed with HMAC-SHA256 (`X-Spacestate-Timestamp`, `X-Spacestate-Signature`) using the new secret `SPACESTATE_WEBHOOK_SECRET` (Key Vault `spacestate-webhook-secret`), and an accepted one only makes Pixel read SpaceAPI again: nothing in it is trusted or logged. Pixel listens on its own port, `WEBHOOK_PORT` (default 8081), only when the secret is set. On Azure, the new `spacestate_webhook_enabled` Terraform variable turns on HTTPS ingress to that port alone (the health probes stay private), `webhook_allowed_ip_ranges` can limit who reaches it, and the `spacestate_webhook_url` output is the address to give the service. Sentry now traces only incoming webhook requests, so probes and internet scanners don't use the trace quota. See [ADR 0018](docs/adr/0018-space-state-webhook.md) and the runbook's "Instant updates: the space state webhook". + ## [0.18.0] - 2026-10-19 ### Added diff --git a/Dockerfile b/Dockerfile index 7fa905d..c1193e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,8 @@ COPY content ./content # files whose loss only costs details like the "open for 2h" time. RUN mkdir -p /app/data && chown node:node /app/data USER node -EXPOSE 8080 +# 8080: health probes (private). 8081: the space state webhook, when SPACESTATE_WEBHOOK_SECRET is set. +EXPOSE 8080 8081 # No --enable-source-maps: Node would rewrite stack frames to src/*.ts (not in the # image) before Sentry sees them, so the maps uploaded for /app/dist would never # match. Sentry applies those maps; the logs show dist/*.js frames. diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 324069a..fa72e05 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -171,6 +171,10 @@ export default defineConfig({ text: "0017 · One writer for app secrets", link: "/maintainers/adr/0017-one-writer-for-app-secrets", }, + { + text: "0018 · Space state webhook", + link: "/maintainers/adr/0018-space-state-webhook", + }, ], }, ], diff --git a/docs/adr/0004-azure-container-apps.md b/docs/adr/0004-azure-container-apps.md index 117616e..a87e1f3 100644 --- a/docs/adr/0004-azure-container-apps.md +++ b/docs/adr/0004-azure-container-apps.md @@ -1,6 +1,6 @@ # 0004. Azure Container Apps, single replica, GHCR, two environments -* Status: accepted +* Status: accepted ("no public ingress" amended by [0018](0018-space-state-webhook.md): optional ingress for the space state webhook only) * Date: 2026-10-03 ## Context diff --git a/docs/adr/0009-container-apps-dev.md b/docs/adr/0009-container-apps-dev.md index 9ee89ba..42448bb 100644 --- a/docs/adr/0009-container-apps-dev.md +++ b/docs/adr/0009-container-apps-dev.md @@ -1,6 +1,6 @@ # 0009. Container Apps module and the `dev` environment -* Status: accepted +* Status: accepted ("no public ingress" amended by [0018](0018-space-state-webhook.md): optional ingress for the space state webhook only) * Date: 2026-10-08 * Issue: #9 diff --git a/docs/adr/0018-space-state-webhook.md b/docs/adr/0018-space-state-webhook.md new file mode 100644 index 0000000..37758d4 --- /dev/null +++ b/docs/adr/0018-space-state-webhook.md @@ -0,0 +1,38 @@ +# 0018. A signed webhook from the space state service, as a nudge to read SpaceAPI + +* Status: accepted (amends "no public ingress" in [ADR 0004](0004-azure-container-apps.md) and [ADR 0009](0009-container-apps-dev.md)) +* Date: 2026-10-20 + +## Context + +Pixel learns that the space opened or closed by polling SpaceAPI every 30 seconds, then confirms the change on the next poll before announcing it (`READINGS_TO_CONFIRM`). So an announcement goes out 30 to 60 seconds after the switch is flipped. The space state service ([`pixelbar/spacestate`](https://github.com/pixelbar/spacestate)) knows the moment the state changes, and can tell Pixel. + +Until now Pixel had no public ingress at all: everything it does is outbound. Taking a webhook means opening an endpoint to the internet for the first time. + +## Decision + +- **The webhook is a nudge, not a source of truth.** An accepted call makes Pixel read SpaceAPI again at once (`SpaceStatus.recheck()`). Nothing in the body is used, logged or echoed. The state, and when it changed, still come only from SpaceAPI, so a forged or replayed call can do no more than cause one extra read, and the webhook and SpaceAPI can never disagree about what Pixel announces. +- **`recheck()` never reuses a read that was already under way**, because that read may have started before the change. It waits for it and starts a fresh one. Rechecks that arrive during the same read share the fresh one. +- **A change the webhook pointed out is confirmed after 5 seconds** (`PUSH_CONFIRM_MS`), with a second SpaceAPI read, instead of after a whole poll interval. A switch flicked back sends its own webhook, which resets the wait, so the flicker protection still holds. A change Pixel finds by polling is confirmed as before. +- **Polling stays exactly as it was.** If a webhook is lost, the next poll catches the change and it is announced the usual way, 30 to 60 seconds late. +- **Every request is signed.** The sender and Pixel share `SPACESTATE_WEBHOOK_SECRET` (at least 32 characters; Key Vault `spacestate-webhook-secret`, app group). The contract: + + ``` + POST /hooks/space-state + X-Spacestate-Timestamp: + X-Spacestate-Signature: sha256=." with the secret> + Content-Type: application/json (the body is not read; send {"state":"open"} or similar) + ``` + + Pixel answers `202` when it accepts the call, `401` for a missing, malformed, stale (more than 5 minutes either way) or wrong signature, `404` for any other path, `405` for anything but POST, and `413` for a body over 4 KiB. The answer never says why a call was refused; Pixel's log does (`webhook.rejected` with a `reason`). The sender should send once, with a short timeout, and not retry: polling is the retry. A test vector is in `src/adapters/webhook/signature.test.ts`. +- **Its own port.** The webhook listens on `WEBHOOK_PORT` (8081), and only when the secret is set. Azure ingress targets only that port, so `/healthz` and `/readyz` (8080) stay private. Config refuses to put both on the same port. +- **Ingress is opt-in per stack.** `spacestate_webhook_enabled` turns on external HTTPS ingress (`allow_insecure_connections = false`, Azure terminates TLS) and wires the secret. `webhook_allowed_ip_ranges` can limit it to the sender's addresses. The URL is the `spacestate_webhook_url` output. Ingress does not change scaling: `min_replicas = max_replicas = 1` still holds. +- **Only `/hooks/` is traced in Sentry.** Probes and whatever else the internet sends to the public port are not, so scanners don't spend the trace quota. + +## Consequences + +- Pixel has a public address when the webhook is on. The code behind it is small (`src/adapters/webhook/`), checks the signature before doing anything, caps the body, times out slow senders, and has strict coverage floors. +- Rotating the secret means changing it in both places at once. Until both agree, webhooks are refused with `401` and Pixel falls back to polling; nothing breaks. +- The space state service needs to send the webhook. Until it does, or while SpaceAPI is still served by something other than that service, nothing changes: Pixel polls. +- During a deploy, a webhook can reach the old or the new revision, or neither while they swap. Any of those is fine: the next poll catches up. +- Home Assistant still has no inbound webhooks ([ADR 0011](0011-home-assistant.md)). That decision was about acting on devices on the say-so of a URL; this webhook acts on nothing. diff --git a/docs/architecture.md b/docs/architecture.md index 4fb45f1..a2ed97b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -242,7 +242,7 @@ Pixel itself never writes the devices file, and it keeps no list of everything H | Service / port | Purpose | Implementation | | ---------------- | ------------------------------------ | ----------------------------------------------------------------- | | `accessConfig` | Admin and member lists | Loads and validates the two YAML files at startup (`ConfigTierSource`). Capability grant/revoke also DMs the person via `DirectMessenger` (`core/ports/direct-message.ts`) | -| `spaceStatus` | Is the space open? | `services/space-status.ts`. `checkNow()` asks `SPACEAPI_URL` (SpaceAPI v0.13) live, with a 5 s timeout, a size cap and validation. Overlapping checks share one request. Background polling every 30 s tracks when the state changed, and `onChange` fires on open↔closed flips, with how long the previous state lasted. The last state and its time are saved to `space.state` and restored on startup (below). After 10 consecutive failures (about 5 minutes) it reports once, and it logs when SpaceAPI recovers | +| `spaceStatus` | Is the space open? | `services/space-status.ts`. `checkNow()` asks `SPACEAPI_URL` (SpaceAPI v0.13) live, with a 5 s timeout, a size cap and validation. Overlapping checks share one request. Background polling every 30 s tracks when the state changed, and `onChange` fires on open↔closed flips, with how long the previous state lasted and whether a poll or the webhook found it (`via`). `recheck()` is what the space state webhook calls: it never reuses a check already under way (see below). The last state and its time are saved to `space.state` and restored on startup (below). After 10 consecutive failures (about 5 minutes) it reports once, and it logs when SpaceAPI recovers | | `calendar` | Upcoming events | `core/calendar.ts`. A neutral `CalendarEvent` and a `CalendarSource` that an adapter plugs in once it's ready (like announcement publishers), so before that `/events` says the calendar isn't available. **Nothing is cached**: every call asks the source, so a renamed or rescheduled event shows up straight away. If the source fails that's an error (`CalendarUnavailableError`), never an out-of-date list; it's logged, and reported to Sentry once per outage rather than on every command | | `infoContent` | `/info` topics | `services/info-content.ts`. Loads and validates the markdown files in `content/info/` at startup (see below). Invalid content stops startup, and CI loads the real content too | @@ -256,6 +256,16 @@ On startup the saved state is trusted only once a live reading agrees with it: The [spaceapi.io directory](https://api.spaceapi.io/openapi.json) was considered as a source of "last changed", but its `lastSeen` is when the directory last *reached* the endpoint (about every minute), not when the state changed. It also keeps no history. +### The space state webhook + +Polling finds a change up to 30 s late. The space state service ([`pixelbar/spacestate`](https://github.com/pixelbar/spacestate)) knows at once, so it can call Pixel's webhook, `POST /hooks/space-state` ([ADR 0018](adr/0018-space-state-webhook.md)): + +- **A nudge, not the truth.** An accepted call makes Pixel read SpaceAPI straight away (`spaceStatus.recheck()`). Nothing in the body is used, logged or echoed: the state and when it changed still come only from SpaceAPI. So a replayed or forged call can cause one extra read and nothing else. +- **Never a stale read.** If a check is already under way when the webhook arrives, it may have read the state from before the change, so `recheck()` waits for it and then asks again. Rechecks that arrive during the same check share the next one. +- **Signed.** `adapters/webhook/signature.ts`: `X-Spacestate-Timestamp` (unix seconds) and `X-Spacestate-Signature: sha256=.">` with `SPACESTATE_WEBHOOK_SECRET`, compared in constant time. More than 5 minutes either way is refused. Every refusal is a bare `401`; the log says why (`webhook.rejected`, `reason` `missing`, `malformed`, `stale` or `bad-signature`) and never has the body or the signature. +- **Small and private by default.** `adapters/webhook/server.ts` listens on `WEBHOOK_PORT` (8081) only when the secret is set, answers `202` once it accepts, `404` for any other path, `405` for anything but POST and `413` past 4 KiB, and drops slow senders (5 s for headers, 10 s per request). It's a different port from `HEALTH_PORT`, so exposing it doesn't expose the probes. +- **Polling stays.** A webhook that never arrives (the service is down, the network drops it, the secret is wrong) just means the next poll finds the change, 30–60 s later as before. + ## Phase 1 features | Feature | Commands / jobs | Tier | Built | Notes | @@ -383,6 +393,7 @@ The text of the answer, in markdown… When the space opens or closes, the `status` feature announces it through the announcer (`features/status/announce.ts`): - **Never on startup.** The status service only reports changes it saw, so starting up, or a change while Pixel was down, announces nothing. - **Only once it holds.** A change is announced once the new state has been seen on 2 polls in a row (`READINGS_TO_CONFIRM`), so about 30–60 s after it happens. If it flips back before that, nothing is posted, so flicking the switch doesn't flood the channel. +- **Sooner when the webhook says so.** A change the webhook pointed out is read again after 5 s (`PUSH_CONFIRM_MS`) instead of a whole interval, so it's announced about 5 s after it happens. A switch flicked back sends its own webhook, which resets the wait, so nothing is posted for a flicker then either. - **If SpaceAPI is unreachable while confirming,** Pixel keeps trying each interval for up to about 5 minutes, then gives up rather than guessing. - **At most once.** A change is never announced twice, even if every publisher failed. - **At startup, stale posts are corrected, not re-announced.** Each publisher gets the current state (`Publisher.reconcile`) once SpaceAPI answers. The live style uses it to turn a leftover "open" post into "closed" (without a closing time, since Pixel didn't see it) when the space closed while Pixel was down. It never posts anything new. Mastodon does not post on reconcile. @@ -398,7 +409,7 @@ The message **body** is a file (`PIXEL_CLOSING_TIME_FILE`, default `data/closing - **Sentry** (`@sentry/node`) is initialised in `src/instrument.ts`, which is loaded with `--import` before the app. If no DSN is set, it does nothing. - Each command scope carries the tags `command`, `feature`, `platform` and `tier`. The Sentry user is the stable platform ID (`discord:`), so a user's issues can be traced over time, with their Discord handle as `username` and display name as `name` so people can recognise them. - **The Discord ID is on every error that can be traced to someone.** The dispatcher runs each command (and each autocomplete) inside `ErrorReporter.withContext`, which sets the user and tags on Sentry's isolation scope for the whole run, starts a `command` span, and counts a `pixel.command` metric. So an error reported anywhere beneath it, however deep (Home Assistant unreachable, the role mirror, a background report) carries who asked, as do breadcrumbs, traces and Sentry Logs from that moment. Two commands at once stay separate. A failure handling an interaction in the Discord adapter names the person too (`captureBackground(error, source, actor)`). Errors with no person behind them (SpaceAPI polling, startup, an uncaught exception outside a command) have no user. `dataCollection.userInfo: false` doesn't affect this: it only stops Sentry inferring IP addresses and the like. A test with a real Sentry client checks the ID is on the sent events. - - **Tracing** is on (`tracesSampleRate` from `SENTRY_TRACES_SAMPLE_RATE`, default `1`). Discord commands are not HTTP, so the command span is started in `withContext`. Outgoing HTTP (SpaceAPI, Home Assistant, Discord's API, Mastodon) is traced by the default HTTP and fetch integrations. Incoming `/healthz` and `/readyz` probes are ignored, and they are **not** counted as request sessions: Pixel's session is the process (`processSessionIntegration`, a default), so crash-free rate tracks the bot staying up, not Azure poking it. + - **Tracing** is on (`tracesSampleRate` from `SENTRY_TRACES_SAMPLE_RATE`, default `1`). Discord commands are not HTTP, so the command span is started in `withContext`. Outgoing HTTP (SpaceAPI, Home Assistant, Discord's API, Mastodon) is traced by the default HTTP and fetch integrations. Of incoming requests only the webhooks (`/hooks/...`) are traced: the `/healthz` and `/readyz` probes, and whatever the internet sends to the public webhook port, are ignored (`isUntracedRequest`), and none are counted as request sessions: Pixel's session is the process (`processSessionIntegration`, a default), so crash-free rate tracks the bot staying up, not Azure poking it. - **Runtime metrics** (`nodeRuntimeMetricsIntegration`) send CPU, memory, event-loop delay and uptime every 30 seconds. - **Profiling** (`@sentry/profiling-node`) attaches a CPU profile to sampled traces (`profileLifecycle: "trace"`). `profileSessionSampleRate` comes from `SENTRY_PROFILE_SESSION_SAMPLE_RATE` (default `1`). The native integration is left out when the rate is `0`. - Errors are not sampled (`sampleRate` is `1`). @@ -413,7 +424,7 @@ The message **body** is a file (`PIXEL_CLOSING_TIME_FILE`, default `data/closing - **Secrets:** pino redacts `token` and authorization headers by name, and every line written (console and file) has anything shaped like a bot or Home Assistant token masked on the way out. Sentry's copy is scrubbed again in `beforeSendLog`. - **Failures nobody caught:** an unhandled promise rejection and an uncaught exception are logged too (`process.unhandled_rejection`, `process.uncaught_exception`), so they reach the file and Sentry as well as the console. Neither changes whether the process exits: a rejection is logged and the bot carries on (like Sentry's default), and an exception still ends the process. - **Heartbeat** (`observability/heartbeat.ts`): a dead bot can't report its own death, and Discord doesn't health-check bots, so Pixel checks in with a **Sentry cron monitor** (`pixel-`, one per environment) every `PIXEL_HEARTBEAT_MINUTES` (default 5), and when Discord connects. A check-in is `ok` while the Discord gateway is connected and `error` while the process is up but disconnected. Sentry creates and updates the monitor from the first check-in (interval schedule, five minutes' margin so a deploy or restart doesn't count, one missed or failed check-in opens an issue, one good one resolves it). So a crash, a hang, a lost Discord connection or the host going down becomes a Sentry issue within about ten minutes. Off without a DSN, with `PIXEL_HEARTBEAT_MINUTES=0`, or when `PIXEL_ENV=local` (`planHeartbeat` decides, and startup logs `heartbeat.off` with the reason): laptops would otherwise share one `pixel-local` monitor that opens an issue every time someone stops `just dev`. A failing check-in is logged and never affects the bot. -- **Health:** a small HTTP server exposes `/healthz` (process alive) and `/readyz` (Discord gateway connected). +- **Health:** a small HTTP server exposes `/healthz` (process alive) and `/readyz` (Discord gateway connected) on `HEALTH_PORT`. It is never exposed to the internet; the webhook has its own port. ## Configuration @@ -450,6 +461,7 @@ Environment variables are validated by `config.ts` (zod). Nothing else reads `pr | `SPACESTATE_USER` | yes | Optional, with the address and token. Sent as `user` to that API. Key Vault `spacestate-user` when hosted | | `SPACESTATE_TOKEN` | yes | Optional, with the address and user. Sent as `token` to that API. A secret. All three unset means `/spacestate` says it isn't set up | | `SPACESTATE_ADDITIONAL_HEADERS` | yes | Optional. A JSON object of extra headers for that call, such as `{"X-Example": "value"}`. A secret. Key Vault `spacestate-additional-headers` when hosted | +| `SPACESTATE_WEBHOOK_SECRET` | yes | Optional. At least 32 characters, shared with the space state service, which signs its webhook with it. Set, Pixel listens for the webhook on `WEBHOOK_PORT`; unset, it only polls. A secret. Key Vault `spacestate-webhook-secret` when hosted | | `PIXEL_HOME_ASSISTANT_DIR` | | Default `config/home-assistant`. Holds `devices.yaml`, the allow-list of devices. Required when Home Assistant is set up | | `SENTRY_DSN` | yes | Optional. Operator-only (not in `config.ts`): `SENTRY_AUTH_TOKEN`, `SENTRY_ORG`, `SENTRY_PROJECT` for `just sentry-release` / CD | | `SENTRY_TRACES_SAMPLE_RATE` | | Default `1`. How many traces Sentry keeps (`0`–`1`). Needs `SENTRY_DSN`. Errors are always kept. | @@ -457,7 +469,8 @@ Environment variables are validated by `config.ts` (zod). Nothing else reads `pr | `LOG_LEVEL` | | Default `info` | | `PIXEL_HEARTBEAT_MINUTES` | | Default `5`. How often Pixel checks in with its Sentry cron monitor (`pixel-`), so Sentry alerts when it goes quiet. 0 turns it off. Needs `SENTRY_DSN`; never runs with `PIXEL_ENV=local` | | `PIXEL_LOG_DIR` | | Default `data/logs`. A rotating JSON log file is written here (about two weeks; owner-only on a normal filesystem, not on the Azure Files share). Empty turns the file off; the console and Sentry Logs still get every line | -| `HEALTH_PORT` | | Default `8080` | +| `HEALTH_PORT` | | Default `8080`. Private: probes only | +| `WEBHOOK_PORT` | | Default `8081`. The space state webhook, the only port exposed to the internet. Must differ from `HEALTH_PORT`. Unused without `SPACESTATE_WEBHOOK_SECRET` | | `SPACEAPI_URL` | | Default `https://spaceapi.pixelbar.nl/`; http(s) only | If neither announcement channel is set, nothing is announced on Discord. Both can be the same channel. Mastodon is a separate publisher: unset `MASTODON_INSTANCE` and `MASTODON_ACCESS_TOKEN` leave it off. No account or instance is hard-coded. @@ -472,7 +485,7 @@ How the hosted pieces fit together (diagrams of Azure, GitHub, Discord, Home Ass There is **no database**. Access, schedules and (planned) account linking are files. Do not add Postgres because an old issue said so (#15, #18). -- **Platform:** Azure Container Apps, **exactly one replica**, no ingress, with a managed identity. A Discord gateway connection needs an always-on process. Two replicas would both connect and answer every command twice. Max replicas = 1 (not a variable). Deploys **stop the old instance before starting the new one** (#11). This stack only pins min=max=1 and `revision_mode = Single`. A database lock is not a plan; there is no database. +- **Platform:** Azure Container Apps, **exactly one replica**, with a managed identity. No ingress, unless `spacestate_webhook_enabled` turns on HTTPS ingress to the webhook port alone ([ADR 0018](adr/0018-space-state-webhook.md)). A Discord gateway connection needs an always-on process. Two replicas would both connect and answer every command twice. Max replicas = 1 (not a variable). Deploys **stop the old instance before starting the new one** (#11). This stack only pins min=max=1 and `revision_mode = Single`. A database lock is not a plan; there is no database. - **Images:** local `just deploy dev` pushes `ghcr.io/pixelbar/pixel:` (or a `dev-dirty-*` tag) and moving `:dev` for Azure `dev`. Merges to `main` push the SHA and moving `:main` via [`.github/workflows/cd.yml`](../.github/workflows/cd.yml). A Release rebuilds from the bump commit and also tags `:vX.Y.Z`. Builds are **linux/amd64** (Azure Container Apps); the Mini is arm64, so `just deploy dev` cross-builds. The images contain no secrets and no access lists. The GHCR package is **public** so Container Apps can pull without a registry password. If it is ever private again, `dev` can set `container_registry_server` / `container_registry_username` and Key Vault `ghcr-pull-token`. The running app pins an explicit tag, never `latest` alone. After first create, **dev** Terraform ignores that pin so apply cannot roll it back (#73). Prod apply will set `container_image`. CI still builds with `push: false` as a smoke check. - **Secrets:** Key Vault. Discord token, Home Assistant token, Sentry DSN, Tailscale auth key, `admins.yaml`, and optional `ghcr-pull-token` (private GHCR only) reach the container as secrets, never as image layers or Terraform state. The app ones are written only by `just secrets-push dev|prod` ([ADR 0016](adr/0016-secrets-sync.md), [ADR 0017](adr/0017-one-writer-for-app-secrets.md)); Terraform writes only the Tailscale key and `ghcr-pull-token`, from ephemeral write-only inputs (`write_secrets`). Prod must not receive the Mini local / Pixel Dev token (that bot lives in `pixel-dev-kv`). - **Access files:** `admins.yaml` is a **read-only** Key Vault secret mount. `members.yaml` is rewritten by admin commands, so it **cannot** be a read-only secret mount. It lives on a **writable Azure Files volume with snapshots**. `config/home-assistant/` is on the same volume: `devices.yaml` is a human allow-list (changed with `just ha-devices`). There is no block-list (#6). @@ -571,10 +584,10 @@ Record significant decisions as short ADRs in `docs/adr/NNNN-title.md`. Numbers | 0001 | TypeScript, ports-and-adapters core, interactive vs publisher adapters | accepted | | 0002 | Phase 1 tiers from gitignored YAML files; admins in a separate file; fail closed | accepted | | 0003 | No database: access, schedules and planned linking are files | accepted | -| 0004 | Azure Container Apps, single replica; GHCR; dev + prod | accepted | +| 0004 | Azure Container Apps, single replica; GHCR; dev + prod | accepted (ingress amended by 0018) | | 0005 | Sentry for errors (no PII), pino to stdout, a rotating file and Sentry Logs | accepted | | 0008 | Terraform bootstrap: remote state (Azure AD, no shared keys) and GitHub OIDC (UAMI per env) | accepted | -| 0009 | Container Apps module + `dev`: one replica, required Azure Files volume, Key Vault, optional Tailscale sidecar | accepted | +| 0009 | Container Apps module + `dev`: one replica, required Azure Files volume, Key Vault, optional Tailscale sidecar | accepted (ingress amended by 0018) | | 0010 | Temporary GHCR CD: local `just deploy dev` → Azure `dev`; `main` publishes GHCR only (prod CD off until #12) | accepted (prod-off superseded by 0012) | | 0011 | Home Assistant: official WebSocket client, non-admin token, no inbound webhooks, allow-list is the fence | accepted | | 0012 | Prod environment (`infra/envs/prod`) and approved CD to `pixel-prod` (same digest, no rebuild) | accepted (merge promotion superseded by 0013, registration by 0014) | @@ -583,3 +596,4 @@ Record significant decisions as short ADRs in `docs/adr/NNNN-title.md`. Numbers | 0015 | Secrets manifest (`infra/secrets.manifest.yaml`) is the one list of Key Vault secrets; Terraform reads it and a test fails on drift | accepted | | 0016 | `just secrets-pull` / `just secrets-push` sync the manifest's app secrets between Key Vault and `.env.`, diffing against the live vault before a push; `just pull` / `just push` add the shared `terraform-tfvars` and read-only share mirrors | accepted | | 0017 | `just secrets-push` / `just push` are the only writers of the app secrets; Terraform's `write_secrets` writes only the infra group, and `removed` blocks drop the old app secrets from state without deleting them | accepted | +| 0018 | A signed webhook from the space state service makes Pixel read SpaceAPI at once; nothing in it is trusted, polling stays as the fallback, and opt-in ingress exposes only the webhook port | accepted | diff --git a/docs/infra.md b/docs/infra.md index dddcd30..368f845 100644 --- a/docs/infra.md +++ b/docs/infra.md @@ -1,8 +1,8 @@ # Pixel infrastructure -How Pixel is hosted. Terraform lives in [`infra/`](../infra/). Day-to-day steps are in the [runbook](runbook.md). Design decisions: [ADR 0008](adr/0008-terraform-bootstrap.md) (bootstrap + OIDC), [ADR 0009](adr/0009-container-apps-dev.md) (Container Apps `dev`), [ADR 0010](adr/0010-ghcr-cd.md) (temporary CD routing for `dev`), [ADR 0012](adr/0012-prod-environment.md) (prod environment), [ADR 0013](adr/0013-release-workflow.md) (releases promote prod). +How Pixel is hosted. Terraform lives in [`infra/`](../infra/). Day-to-day steps are in the [runbook](runbook.md). Design decisions: [ADR 0008](adr/0008-terraform-bootstrap.md) (bootstrap + OIDC), [ADR 0009](adr/0009-container-apps-dev.md) (Container Apps `dev`), [ADR 0010](adr/0010-ghcr-cd.md) (temporary CD routing for `dev`), [ADR 0012](adr/0012-prod-environment.md) (prod environment), [ADR 0013](adr/0013-release-workflow.md) (releases promote prod), [ADR 0018](adr/0018-space-state-webhook.md) (the space state webhook and its ingress). -There is **no database**. Discord users never connect to Azure: the replica has **no public ingress** and **exactly one replica**. `prod` is [`infra/envs/prod`](../infra/envs/prod/) ([#12](https://github.com/pixelbar/pixel/issues/12)). +There is **no database**. Discord users never connect to Azure: the replica has **exactly one replica** and **no public ingress**, except the optional space state webhook (HTTPS to one port and one path, signed; `spacestate_webhook_enabled`). `prod` is [`infra/envs/prod`](../infra/envs/prod/) ([#12](https://github.com/pixelbar/pixel/issues/12)). ## How the pieces are set @@ -21,7 +21,7 @@ West Europe, subscription in bootstrap / env variables. Live **`pixel-dev`** is | Backup | `pixel-dev-rsv` / `pixel-prod-rsv` | Daily share backup, 14-day keep (plus 14-day share soft-delete) | | Logs | `pixel-dev-logs` / `pixel-prod-logs` | Container Apps diagnostics, 30 days | | CAE | `pixel-dev-cae` / `pixel-prod-cae` | Consumption, **no VNet** | -| App | `pixel-dev` / `pixel-prod` | `min = max` replicas **1**, `revision_mode = Single`, ingress **none** | +| App | `pixel-dev` / `pixel-prod` | `min = max` replicas **1**, `revision_mode = Single`, ingress **none** unless `spacestate_webhook_enabled`: then external HTTPS to port 8081 only (the webhook), optionally limited to `webhook_allowed_ip_ranges` | | Image | `ghcr.io/pixelbar/pixel:` (public) | No `.env`, no access YAML. Azure `dev` is rolled by `just deploy dev`; `prod` by the Release workflow after approval | **CD:** `just deploy dev` on a laptop builds linux/amd64, pushes GHCR, `az containerapp update` on `pixel-dev` with min=max=1, then `just register dev`. Merges to `main` publish GHCR (`:` and `:main`) and Sentry source maps. They do **not** roll `pixel-prod`. A Release rebuilds from the bumped commit (`:vX.Y.Z`), publishes the docs site, then Environment `prod` approval rolls `pixel-prod` to **that digest** (no Terraform apply). Production slash commands: the Release's `register-prod` job, after a second approval (fallback `just register prod`). @@ -63,7 +63,7 @@ flowchart LR FILES["Azure Files pixeldevdata / pixel"] RSV["Backup pixel-dev-rsv"] LAW["Log Analytics 30d"] - CAE["CAE pixel-dev-cae
Consumption, no VNet, no ingress"] + CAE["CAE pixel-dev-cae
Consumption, no VNet
ingress only for the webhook, when on"] APP["Container App pixel-dev
1 replica: pixel + init prepare-volume"] end subgraph prod["RG pixel-prod"] @@ -95,6 +95,7 @@ flowchart LR DISCORD["Discord gateway + REST
Pixel Dev / test guild"] HA["Home Assistant
Nabu Casa today"] SPACE["SpaceAPI spaceapi.pixelbar.nl"] + SSVC["Space state service
pixelbar/spacestate"] SENTRY["Sentry org/project"] TS["Pixelbar tailnet
sidecar stub off"] end @@ -102,6 +103,7 @@ flowchart LR APP -->|outbound only| DISCORD APP -->|HTTPS + WS| HA APP --> SPACE + SSVC -.->|"signed webhook, HTTPS :443 → :8081
only when spacestate_webhook_enabled"| APP APP --> SENTRY TS -.->|not wired| APP ``` @@ -112,7 +114,7 @@ One replica. Containers share a network namespace; they do **not** share filesys flowchart TB subgraph replica["pixel-dev replica min=max=1"] INIT["init prepare-volume
busybox:1.37.0
mkdir + chmod persist"] - PIXEL["container pixel
0.5 vCPU / 1Gi
probes /healthz /readyz :8080"] + PIXEL["container pixel
0.5 vCPU / 1Gi
probes /healthz /readyz :8080 (private)
webhook /hooks/space-state :8081"] TSSTUB["tailscale sidecar
Terraform stub, live OFF"] INIT --> PIXEL end @@ -122,6 +124,7 @@ flowchart TB FILEVOL --> INIT FILEVOL --> PIXEL PIXEL -->|Discord, SpaceAPI, Sentry, KV, GHCR| EGRESS["Azure egress"] + INGRESS["Azure ingress, HTTPS only
when spacestate_webhook_enabled"] -->|:8081 only| PIXEL PIXEL -->|HA token + Nabu Casa URL| NABU["*.ui.nabu.casa"] ``` @@ -129,7 +132,7 @@ flowchart TB 1. **Bootstrap** (hand-applied) created the three resource groups, state storage, and per-env GitHub identities. Env stacks deploy *into* those groups; they do not recreate them. 2. **`infra/modules/pixel`** + **`infra/envs/dev`** created the vault, files share, CAE, app, backup, logs, and `pixel-dev-app`. Remote state is container `dev`. **`infra/envs/prod`** is the same module in `pixel-prod` (state container `prod`). -3. **Secrets** are Key Vault names (`discord-token`, `admins-yaml`, `home-assistant-token`, `mastodon-access-token`, `spacestate-user`, `spacestate-token`, `sentry-dsn`). The Container App references **URIs + UAMI**. Only `just secrets-push` writes these app values ([ADR 0017](adr/0017-one-writer-for-app-secrets.md)); Terraform's ephemeral `write_secrets` inputs cover just the infra secrets `tailscale-auth-key` and `ghcr-pull-token`. Discord / HA / Sentry values do **not** belong in state. The Azure Files **account key** does (Container Apps can only mount SMB with the key). +3. **Secrets** are Key Vault names (`discord-token`, `admins-yaml`, `home-assistant-token`, `mastodon-access-token`, `spacestate-user`, `spacestate-token`, `spacestate-webhook-secret`, `sentry-dsn`). The Container App references **URIs + UAMI**. Only `just secrets-push` writes these app values ([ADR 0017](adr/0017-one-writer-for-app-secrets.md)); Terraform's ephemeral `write_secrets` inputs cover just the infra secrets `tailscale-auth-key` and `ghcr-pull-token`. Discord / HA / Sentry values do **not** belong in state. The Azure Files **account key** does (Container Apps can only mount SMB with the key). 4. **Image:** GHCR is public, so the app pulls with no registry password. `just deploy dev` updates the running **dev** image with `az containerapp update` and does **not** apply Terraform. After first create, Terraform **ignores** that **dev** image so apply cannot roll it back (#73). Prod apply sets `container_image`; a Release then rolls `pixel-prod` to the released SHA (same digest, no Terraform apply). Pin an explicit SHA (or `dev-dirty-*` on **dev** only), never `latest` alone. 5. **Discord:** outbound gateway + REST. No FQDN. `/healthz` is process up; `/readyz` is Discord connected. Home Assistant is **not** on the probes (HA down must not restart the replica). 6. **Home Assistant:** both URL and token, or neither. Live `dev` uses a Nabu Casa URL. The allow-list is `devices.yaml` on the share. A Tailscale sidecar is the preferred path ([#43](https://github.com/pixelbar/pixel/issues/43), [#74](https://github.com/pixelbar/pixel/issues/74)); the Terraform stub is **off** and is not enough to flip on as-is. Dev must not control the real doors. @@ -166,11 +169,19 @@ Do not raise replica count. Two replicas (or a laptop `just dev` plus this app) How to seed the share and vault: [dev README](../infra/envs/dev/README.md). -## One replica, no public ingress +## One replica, and ingress only for the webhook Hard constraints in the module (not variables): `min_replicas = 1`, `max_replicas = 1`. `just deploy dev` re-asserts that, and `just deploy prod` refuses (prod ships from the Release workflow). The Release prod job re-asserts the same on `pixel-prod`. -Ingress is off. Discord, SpaceAPI, Sentry, GHCR, Key Vault, and today’s Nabu Casa path all use ordinary Azure egress. Nothing in Discord connects *to* Azure. +Ingress is off by default. Discord, SpaceAPI, Sentry, GHCR, Key Vault, and today’s Nabu Casa path all use ordinary Azure egress. Nothing in Discord connects *to* Azure. + +The one exception is the space state webhook ([ADR 0018](adr/0018-space-state-webhook.md)), off until a stack sets `spacestate_webhook_enabled = true`. Then: + +- **External HTTPS ingress, to port 8081 only.** Azure terminates TLS and refuses plain HTTP (`allow_insecure_connections = false`). The health probes are on 8080, which ingress never targets, so `/healthz` and `/readyz` stay private. On 8081 Pixel answers only `POST /hooks/space-state`; everything else is a 404. +- **Every request must be signed** with Key Vault `spacestate-webhook-secret` (`SPACESTATE_WEBHOOK_SECRET`), and an accepted one only makes Pixel read SpaceAPI again. Nothing in it is trusted. +- **Optionally limited by address:** `webhook_allowed_ip_ranges` (CIDRs) adds Allow rules, and everyone else gets a 403 from Azure before reaching Pixel. The space state service also runs on Container Apps Consumption, whose outbound addresses can change, so leaving it empty and relying on the signature is fine. +- **The URL** is the `spacestate_webhook_url` output: `https://./hooks/space-state`. It stays the same across revisions and deploys. +- **No effect on scaling or deploys.** min=max=1 still holds; ingress just routes to whichever revision is ready. A webhook lost during a revision swap is caught by the next poll. ## What is not in Azure diff --git a/docs/maintainers/index.md b/docs/maintainers/index.md index 30c55b1..50da53b 100644 --- a/docs/maintainers/index.md +++ b/docs/maintainers/index.md @@ -39,6 +39,7 @@ Architecture decision records, newest last. | [0015](./adr/0015-secrets-manifest) | One manifest lists every Key Vault secret. | | [0016](./adr/0016-secrets-sync) | `just pull` / `just push` (and `secrets-pull` / `secrets-push`) sync tfvars and app secrets between Key Vault and the laptop. | | [0017](./adr/0017-one-writer-for-app-secrets) | Only the sync tool writes app secrets; Terraform writes just the infra group. | +| [0018](./adr/0018-space-state-webhook) | A signed webhook from the space state service makes Pixel read SpaceAPI straight away. | ## Working on the site diff --git a/docs/runbook.md b/docs/runbook.md index 9c8ca0b..9ed1cc2 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -24,7 +24,7 @@ Before you change anything on **prod**, read "Rules of thumb" below. - [Azure Container App (`dev`)](#azure-container-app-dev) - [Closing-time](#closing-time) - [Mastodon](#mastodon) -- [Space state](#space-state) +- [Space state](#space-state) ([the webhook](#instant-updates-the-space-state-webhook)) - [Secrets](#secrets) ([sync with Key Vault](#sync-secrets-with-key-vault), [pull and push an environment](#pull-and-push-an-environment)) - [Deploys](#deploys) - [CI and CD](#ci-and-cd) @@ -49,7 +49,7 @@ Before you change anything on **prod**, read "Rules of thumb" below. | --- | --- | | **Environments** | `dev` (Pixel Dev bot, test guild) and `prod` (production Pixel bot, production guild). Separate bots, tokens and secrets. Never copy the Dev token into prod. | | **Who is on the hook when the bot is down** | 🚧 Not decided. Put a named contact (and a backup) here, and where to reach them. | -| **Where Pixel runs** | Azure `pixel-dev` (one replica, no public ingress) plus `pixel-prod` after a maintainer apply, plus any laptop `just dev`. Subscription `d150e252-e2f0-47fb-8a4a-c3f29e9aebd4`, West Europe. Resource groups `pixel-bootstrap`, `pixel-dev`, `pixel-prod`. `just deploy dev` rolls `pixel-dev`. **Do not** also run `just dev` on the Pixel Dev token while that app is up. A Release rolls `pixel-prod` after Environment `prod` approval. Diagrams: [`infra.md`](infra.md). | +| **Where Pixel runs** | Azure `pixel-dev` (one replica, no public ingress unless the space state webhook is on) plus `pixel-prod` after a maintainer apply, plus any laptop `just dev`. Subscription `d150e252-e2f0-47fb-8a4a-c3f29e9aebd4`, West Europe. Resource groups `pixel-bootstrap`, `pixel-dev`, `pixel-prod`. `just deploy dev` rolls `pixel-dev`. **Do not** also run `just dev` on the Pixel Dev token while that app is up. A Release rolls `pixel-prod` after Environment `prod` approval. Diagrams: [`infra.md`](infra.md). | | **Where the code and CI are** | GitHub, `pixelbar/pixel`. CI runs `just check` and/or a Docker smoke build only when the files those jobs depend on have not already passed. Docs PRs run Biome + the site build. CD on `main` publishes `ghcr.io/pixelbar/pixel:` when the image inputs differ from the last publish. | | **Access you may need** | The Discord Developer Portal (bot token), a Discord role that can manage the server, the Sentry project, the GitHub repo, the Home Assistant admin account, Owner on the Pixel Azure subscription (to apply bootstrap), and the host. | @@ -322,6 +322,32 @@ Members with the `spacestate` capability can set the space open or closed with ` The user and token are in the URL's query string, so treat anything that could hold the whole URL (a proxy log, a copy of the settings) as a secret. +### Instant updates: the space state webhook + +Without it, Pixel notices an open or close by polling SpaceAPI every 30 s and announces it 30–60 s later. With it, the space state service ([`pixelbar/spacestate`](https://github.com/pixelbar/spacestate)) calls Pixel when the state changes, Pixel reads SpaceAPI straight away, and the announcement goes out about 5 s later. Polling carries on regardless, so if the webhook fails nothing breaks: announcements are just back to 30–60 s. Design: [ADR 0018](adr/0018-space-state-webhook.md). + +**Turn it on** (Azure): + +1. Make a secret of at least 32 characters, for example `openssl rand -hex 32`. Put it in `.env.` as `SPACESTATE_WEBHOOK_SECRET` and `just secrets-push `, which creates Key Vault `spacestate-webhook-secret`. +2. In the environment's `terraform.tfvars` (`just pull ` first), set `spacestate_webhook_enabled = true`, and optionally `webhook_allowed_ip_ranges` to the space state service's outbound addresses. `just push `, then `just tf-plan `. The plan should show only an `ingress` block added to the Container App, the `WEBHOOK_PORT` setting and the `spacestate-webhook-secret` reference. A maintainer applies it (`just tf-apply dev`; prod needs `PIXEL_APPLY_PROD=1` and is never applied by an agent). +3. Read the URL: `terraform -chdir=infra/envs/ output spacestate_webhook_url` (`https://pixel-..westeurope.azurecontainerapps.io/hooks/space-state`). +4. Give the space state service that URL and the same secret, so it signs each call as ADR 0018 describes. +5. Check: Pixel's log says `webhook.received` on the next open or close, followed by `spaceapi.changed` with `via: "push"`. Startup logs `webhook.off` when the secret isn't set. + +**Locally:** set `SPACESTATE_WEBHOOK_SECRET` in `.env` and Pixel listens on `WEBHOOK_PORT` (default 8081). Try it with a signed call: + +```sh +ts=$(date +%s); body='{"state":"open"}' +sig=$(printf '%s' "$ts.$body" | openssl dgst -sha256 -hmac "$SPACESTATE_WEBHOOK_SECRET" | awk '{print $2}') +curl -i -X POST localhost:8081/hooks/space-state -H "X-Spacestate-Timestamp: $ts" -H "X-Spacestate-Signature: sha256=$sig" -d "$body" +``` + +`202` means it was accepted (Pixel then reads SpaceAPI; the body itself is ignored). `401` means the signature, the secret or the clock is off: the log line `webhook.rejected` has the `reason` (`missing`, `malformed`, `stale` (more than 5 minutes off) or `bad-signature`). + +**Turn it off:** set `spacestate_webhook_enabled = false` and apply. Ingress goes away and Pixel goes back to polling only. In a hurry, clearing the secret on the service side does the same in practice: Pixel refuses every call and keeps polling. + +**If it seems not to work:** announcements still arriving 30–60 s late means the webhook isn't getting through. Look for `webhook.received` / `webhook.rejected` in the logs. No lines at all: the service isn't calling, or the address is wrong, or `webhook_allowed_ip_ranges` blocks it (Azure answers 403 itself). `bad-signature`: the two secrets differ. `stale`: a clock is wrong. If SpaceAPI is still served by the old setup rather than the new service, a webhook can arrive before SpaceAPI shows the change; the next poll catches it. + ## Secrets Every secret lives in the secret store for its environment (a local `.env` for development, Key Vault `pixel-dev-kv` in Azure `dev`) and nowhere else. Pixel never logs them, and scrubs anything shaped like a Discord or Home Assistant token from logs and Sentry, but don't rely on that. After rotating, **restart Pixel**: secrets are read once at startup. @@ -332,6 +358,7 @@ Every secret lives in the secret store for its environment (a local `.env` for d | `HOME_ASSISTANT_TOKEN` | A long-lived token in HA; Key Vault `home-assistant-token` | It may have leaked, or the Pixel user changes | | `MASTODON_ACCESS_TOKEN` | Instance Development app (`write:statuses`); Key Vault `mastodon-access-token` when hosted | It may have leaked, the app is rotated, or you move instance | | `SPACESTATE_USER`, `SPACESTATE_TOKEN` | Whoever runs the space state API; Key Vault `spacestate-user` and `spacestate-token` when hosted | They may have leaked, or the API's credentials change | +| `SPACESTATE_WEBHOOK_SECRET` | Made by you (`openssl rand -hex 32`), shared with the space state service; Key Vault `spacestate-webhook-secret` when hosted | It may have leaked, or someone with access to either side leaves | | `SENTRY_DSN` | Sentry project settings; Key Vault `sentry-dsn` | It's being abused, or you move projects | | Tailscale auth key | Tailscale admin; Key Vault `tailscale-auth-key` | It may have leaked, or you rotate tagged keys | | `admins.yaml` | Hand-edited; Key Vault `admins-yaml` (not a token) | When admins change | @@ -363,6 +390,8 @@ Every secret lives in the secret store for its environment (a local `.env` for d 2. Put them in the secret store as `SPACESTATE_USER` and `SPACESTATE_TOKEN`, and restart Pixel. 3. Run `/spacestate set` with the state the space is already in. If Sentry shows `spacestate` with HTTP 401 or 403, the credentials are wrong. +**Rotate the webhook secret:** make a new one, `just secrets-push ` it (and restart), and set the same value on the space state service. Between the two, webhooks get `401` and Pixel polls as usual, so the order doesn't matter and nothing is lost but speed. Check for `webhook.received` on the next change. + **Rotate the Sentry DSN:** Sentry project → Settings → Client Keys: create a new key, disable the old one, update `SENTRY_DSN`, restart. Check that a test event arrives. **If a secret leaked:** rotate it first, then work out where it leaked from. Check the logs and Sentry for the time window. A leaked Discord token lets someone act as the bot in your server, so treat it as urgent. @@ -596,7 +625,7 @@ Sentry gets errors (with the tags `command`, `feature`, `platform`, `tier` and t **Retention:** the [privacy notice](privacy.md) documents **30 days** for Sentry (same as Azure Log Analytics). 🚧 Apply that in the Sentry project (Settings → Security & Privacy / Data Retention). Terraform does not set Sentry retention. -Errors are always sent (`sampleRate` 1). Traces use `SENTRY_TRACES_SAMPLE_RATE` (default 1: keep them all). Profiles use `SENTRY_PROFILE_SESSION_SAMPLE_RATE` (default 1). `/healthz` and `/readyz` are not traced. +Errors are always sent (`sampleRate` 1). Traces use `SENTRY_TRACES_SAMPLE_RATE` (default 1: keep them all). Profiles use `SENTRY_PROFILE_SESSION_SAMPLE_RATE` (default 1). Only the webhook (`/hooks/...`) is traced among incoming requests: `/healthz`, `/readyz` and anything else that reaches the public webhook port are not. - **Where alerts go:** 🚧 not decided. Set up alert rules for new issues in prod, and write here who is notified and where (email, Discord channel). - **Pixel going quiet:** Pixel checks in with a Sentry cron monitor, `pixel-prod` or `pixel-dev` (Sentry → **Crons**), every 5 minutes while it's connected to Discord. If check-ins stop (a crash, a hang, the host down) or say it's disconnected, the monitor opens an issue within about ten minutes, and closes it by itself when Pixel is back. A laptop `just dev` (`PIXEL_ENV=local`) never checks in, even with a DSN, so stopping it doesn't alert anyone; if Sentry still lists an old `pixel-local` monitor, delete it. In the monitor's settings, set who is alerted. 🚧 Record here who that is. diff --git a/infra/envs/dev/README.md b/infra/envs/dev/README.md index 1b13ee1..fcf17ed 100644 --- a/infra/envs/dev/README.md +++ b/infra/envs/dev/README.md @@ -16,7 +16,7 @@ Inside `pixel-dev` (already exists; not recreated): | Key Vault (RBAC, no secret values in state) | `pixel-dev-kv` | | Log Analytics (30-day retention; see [`docs/privacy.md`](../../../docs/privacy.md)) | `pixel-dev-logs` | | Container Apps environment | `pixel-dev-cae` | -| Container App, min=max replicas **1**, no public ingress | `pixel-dev` | +| Container App, min=max replicas **1**, no public ingress (unless `spacestate_webhook_enabled`: HTTPS to the webhook port only) | `pixel-dev` | | Azure Files account + share `pixel` | `pixeldevdata` | | Daily file-share backup, 14-day keep | `pixel-dev-rsv` | @@ -57,7 +57,7 @@ Do not run Pixel locally against the same Discord token while this app is up. ## Secrets (Key Vault) -**App secrets** (`discord-token`, `admins-yaml`, `home-assistant-token`, `mastodon-access-token`, `spacestate-user`, `spacestate-token`, `sentry-dsn`): `just secrets-push dev` is their only writer ([ADR 0017](../../../docs/adr/0017-one-writer-for-app-secrets.md)). Terraform has no inputs for them. Push them **before** the apply that turns their feature on; the apply only references them. +**App secrets** (`discord-token`, `admins-yaml`, `home-assistant-token`, `mastodon-access-token`, `spacestate-user`, `spacestate-token`, `spacestate-webhook-secret`, `sentry-dsn`): `just secrets-push dev` is their only writer ([ADR 0017](../../../docs/adr/0017-one-writer-for-app-secrets.md)). Terraform has no inputs for them. Push them **before** the apply that turns their feature on; the apply only references them. **Infra secrets** (`tailscale-auth-key`, `ghcr-pull-token`): Terraform writes them when `write_secrets = true`, from gitignored `secrets.auto.tfvars` (or `TF_VAR_*`), never the shared `terraform.tfvars`. Values are write-only and are not stored in Terraform state. diff --git a/infra/envs/dev/main.tf b/infra/envs/dev/main.tf index d9fa5c5..075b451 100644 --- a/infra/envs/dev/main.tf +++ b/infra/envs/dev/main.tf @@ -20,6 +20,8 @@ module "pixel" { mastodon_instance = var.mastodon_instance spacestate_api_url = var.spacestate_api_url spacestate_additional_headers_enabled = var.spacestate_additional_headers_enabled + spacestate_webhook_enabled = var.spacestate_webhook_enabled + webhook_allowed_ip_ranges = var.webhook_allowed_ip_ranges sentry_enabled = var.sentry_enabled tailscale_enabled = var.tailscale_enabled tailscale_image = var.tailscale_image diff --git a/infra/envs/dev/outputs.tf b/infra/envs/dev/outputs.tf index 909d072..e45dc7a 100644 --- a/infra/envs/dev/outputs.tf +++ b/infra/envs/dev/outputs.tf @@ -30,3 +30,8 @@ output "log_analytics_workspace_name" { output "tailscale_enabled" { value = module.pixel.tailscale_enabled } + +output "spacestate_webhook_url" { + value = module.pixel.spacestate_webhook_url + description = "Where the space state service sends its webhook, or null when it is off." +} diff --git a/infra/envs/dev/terraform.example.tfvars b/infra/envs/dev/terraform.example.tfvars index 78a206a..b50c19d 100644 --- a/infra/envs/dev/terraform.example.tfvars +++ b/infra/envs/dev/terraform.example.tfvars @@ -39,6 +39,11 @@ discord_guild_id = "100000000000000002" # spacestate_api_url = "https://space.example/api/state" # spacestate_additional_headers_enabled = true # also needs Key Vault secret spacestate-additional-headers +# The space state webhook: public HTTPS ingress to port 8081 only (ADR 0018). Push Key +# Vault secret spacestate-webhook-secret first. The URL is the spacestate_webhook_url output. +# spacestate_webhook_enabled = true +# webhook_allowed_ip_ranges = ["203.0.113.7/32"] # optional; empty lets anyone try (signed requests only) + sentry_enabled = false tailscale_enabled = false diff --git a/infra/envs/dev/variables.tf b/infra/envs/dev/variables.tf index 711b803..1f3edc1 100644 --- a/infra/envs/dev/variables.tf +++ b/infra/envs/dev/variables.tf @@ -134,6 +134,18 @@ variable "spacestate_additional_headers_enabled" { description = "If true (and spacestate_api_url is set), the app reads the extra headers from Key Vault secret spacestate-additional-headers." } +variable "spacestate_webhook_enabled" { + type = bool + default = false + description = "If true, open public HTTPS ingress to the space state webhook and read its secret from Key Vault secret spacestate-webhook-secret (push it first)." +} + +variable "webhook_allowed_ip_ranges" { + type = list(string) + default = [] + description = "Optional. CIDR ranges allowed to reach the webhook. Empty allows anyone; requests must still be signed." +} + variable "tailscale_auth_key" { type = string ephemeral = true diff --git a/infra/envs/prod/README.md b/infra/envs/prod/README.md index 3c81374..f84f023 100644 --- a/infra/envs/prod/README.md +++ b/infra/envs/prod/README.md @@ -18,7 +18,7 @@ Inside `pixel-prod` (already exists; not recreated): | Key Vault (RBAC, purge protection **on**) | `pixel-prod-kv` | | Log Analytics (30-day retention, pending #7) | `pixel-prod-logs` | | Container Apps environment | `pixel-prod-cae` | -| Container App, min=max replicas **1**, no public ingress | `pixel-prod` | +| Container App, min=max replicas **1**, no public ingress (unless `spacestate_webhook_enabled`: HTTPS to the webhook port only) | `pixel-prod` | | Azure Files account + share `pixel` | `pixelproddata` | | Daily file-share backup, 14-day keep | `pixel-prod-rsv` | diff --git a/infra/envs/prod/main.tf b/infra/envs/prod/main.tf index 185f008..7641a56 100644 --- a/infra/envs/prod/main.tf +++ b/infra/envs/prod/main.tf @@ -20,6 +20,8 @@ module "pixel" { mastodon_instance = var.mastodon_instance spacestate_api_url = var.spacestate_api_url spacestate_additional_headers_enabled = var.spacestate_additional_headers_enabled + spacestate_webhook_enabled = var.spacestate_webhook_enabled + webhook_allowed_ip_ranges = var.webhook_allowed_ip_ranges sentry_enabled = var.sentry_enabled tailscale_enabled = var.tailscale_enabled tailscale_image = var.tailscale_image diff --git a/infra/envs/prod/outputs.tf b/infra/envs/prod/outputs.tf index 909d072..e45dc7a 100644 --- a/infra/envs/prod/outputs.tf +++ b/infra/envs/prod/outputs.tf @@ -30,3 +30,8 @@ output "log_analytics_workspace_name" { output "tailscale_enabled" { value = module.pixel.tailscale_enabled } + +output "spacestate_webhook_url" { + value = module.pixel.spacestate_webhook_url + description = "Where the space state service sends its webhook, or null when it is off." +} diff --git a/infra/envs/prod/terraform.example.tfvars b/infra/envs/prod/terraform.example.tfvars index 038ef0d..06fe429 100644 --- a/infra/envs/prod/terraform.example.tfvars +++ b/infra/envs/prod/terraform.example.tfvars @@ -29,6 +29,11 @@ discord_guild_id = "100000000000000002" # spacestate_api_url = "https://space.example/api/state" # spacestate_additional_headers_enabled = true # also needs Key Vault secret spacestate-additional-headers +# The space state webhook: public HTTPS ingress to port 8081 only (ADR 0018). Push Key +# Vault secret spacestate-webhook-secret first. The URL is the spacestate_webhook_url output. +# spacestate_webhook_enabled = true +# webhook_allowed_ip_ranges = ["203.0.113.7/32"] # optional; empty lets anyone try (signed requests only) + # Lets the Release workflow read discord-token (only) and register slash # commands (ADR 0014). az identity show -g pixel-bootstrap -n pixel-github-prod --query principalId -o tsv # github_register_principal_id = "00000000-0000-0000-0000-000000000000" diff --git a/infra/envs/prod/variables.tf b/infra/envs/prod/variables.tf index ef3d056..1e6987d 100644 --- a/infra/envs/prod/variables.tf +++ b/infra/envs/prod/variables.tf @@ -134,6 +134,18 @@ variable "spacestate_additional_headers_enabled" { description = "If true (and spacestate_api_url is set), the app reads the extra headers from Key Vault secret spacestate-additional-headers." } +variable "spacestate_webhook_enabled" { + type = bool + default = false + description = "If true, open public HTTPS ingress to the space state webhook and read its secret from Key Vault secret spacestate-webhook-secret (push it first)." +} + +variable "webhook_allowed_ip_ranges" { + type = list(string) + default = [] + description = "Optional. CIDR ranges allowed to reach the webhook. Empty allows anyone; requests must still be signed." +} + variable "tailscale_auth_key" { type = string ephemeral = true diff --git a/infra/modules/pixel/container_app.tf b/infra/modules/pixel/container_app.tf index 2ada129..50c7e54 100644 --- a/infra/modules/pixel/container_app.tf +++ b/infra/modules/pixel/container_app.tf @@ -64,6 +64,34 @@ resource "azurerm_container_app" "this" { } } + + # Off unless the space state webhook is on. Then it exposes only WEBHOOK_PORT over HTTPS + # (Azure terminates TLS; plain HTTP is refused), never the health port. Pixel checks the + # signature on every request (ADR 0018). It doesn't change scaling: min=max=1 still holds. + dynamic "ingress" { + for_each = var.spacestate_webhook_enabled ? [1] : [] + content { + external_enabled = true + target_port = local.webhook_port + transport = "http" + allow_insecure_connections = false + + traffic_weight { + latest_revision = true + percentage = 100 + } + + dynamic "ip_security_restriction" { + for_each = var.webhook_allowed_ip_ranges + content { + name = "webhook-sender-${ip_security_restriction.key}" + ip_address_range = ip_security_restriction.value + action = "Allow" + } + } + } + } + template { # Discord connects a long-lived gateway. Two replicas answer every command # twice. min=max=1 is a hard constraint, not a default. Overlap during a @@ -262,6 +290,31 @@ resource "azurerm_container_app" "managed" { } } + + dynamic "ingress" { + for_each = var.spacestate_webhook_enabled ? [1] : [] + content { + external_enabled = true + target_port = local.webhook_port + transport = "http" + allow_insecure_connections = false + + traffic_weight { + latest_revision = true + percentage = 100 + } + + dynamic "ip_security_restriction" { + for_each = var.webhook_allowed_ip_ranges + content { + name = "webhook-sender-${ip_security_restriction.key}" + ip_address_range = ip_security_restriction.value + action = "Allow" + } + } + } + } + template { min_replicas = 1 max_replicas = 1 diff --git a/infra/modules/pixel/locals.tf b/infra/modules/pixel/locals.tf index 9c15d88..40b6667 100644 --- a/infra/modules/pixel/locals.tf +++ b/infra/modules/pixel/locals.tf @@ -11,6 +11,10 @@ locals { persist_mount = "/app/persist" secrets_mount = "/app/secrets" + # The space state webhook listens here (WEBHOOK_PORT). It is the only port ingress + # exposes; the probes use HEALTH_PORT (8080), which stays private. + webhook_port = 8081 + # node:24-slim runs as uid/gid 1000. Azure Files replaces /app/data from the # image, so the mount must make that user the owner. azure_file_mount_options = "uid=1000,gid=1000,dir_mode=0775,file_mode=0664,nobrl,mfsymlinks,cache=none" @@ -27,6 +31,7 @@ locals { mastodon = var.mastodon_instance != null spacestate = var.spacestate_api_url != null spacestate-headers = var.spacestate_api_url != null && var.spacestate_additional_headers_enabled + spacestate-webhook = var.spacestate_webhook_enabled sentry = var.sentry_enabled tailscale = var.tailscale_enabled container-registry = var.container_registry_server != null @@ -72,6 +77,7 @@ locals { var.home_assistant_url != null ? { HOME_ASSISTANT_URL = var.home_assistant_url } : {}, var.mastodon_instance != null ? { MASTODON_INSTANCE = var.mastodon_instance } : {}, var.spacestate_api_url != null ? { SPACESTATE_API_URL = var.spacestate_api_url } : {}, + var.spacestate_webhook_enabled ? { WEBHOOK_PORT = tostring(local.webhook_port) } : {}, ) # Environment variable => Key Vault secret name, for what the Pixel container reads from its diff --git a/infra/modules/pixel/outputs.tf b/infra/modules/pixel/outputs.tf index eb21b1c..8dec687 100644 --- a/infra/modules/pixel/outputs.tf +++ b/infra/modules/pixel/outputs.tf @@ -4,7 +4,7 @@ output "resource_group_name" { output "container_app_name" { value = local.name_prefix - description = "Container App name (local.name_prefix). One replica, no public ingress. Not taken from the app resource object — that object is sensitive because of secret blocks." + description = "Container App name (local.name_prefix). One replica. Public ingress only when spacestate_webhook_enabled. Not taken from the app resource object — that object is sensitive because of secret blocks." } output "container_app_environment_name" { @@ -45,3 +45,10 @@ output "managed_identity_client_id" { output "tailscale_enabled" { value = var.tailscale_enabled } + +output "spacestate_webhook_url" { + # Built from the environment's domain, like container_app_name, rather than read from + # the sensitive app object. External ingress is always .. + value = var.spacestate_webhook_enabled ? "https://${local.name_prefix}.${azurerm_container_app_environment.this.default_domain}/hooks/space-state" : null + description = "Where the space state service sends its webhook, or null when it is off. Give it this URL and the spacestate-webhook-secret value." +} diff --git a/infra/modules/pixel/variables.tf b/infra/modules/pixel/variables.tf index 1134387..3d3d038 100644 --- a/infra/modules/pixel/variables.tf +++ b/infra/modules/pixel/variables.tf @@ -192,6 +192,23 @@ variable "spacestate_additional_headers_enabled" { description = "If true (and spacestate_api_url is set), the app reads SPACESTATE_ADDITIONAL_HEADERS from Key Vault secret spacestate-additional-headers." } +variable "spacestate_webhook_enabled" { + type = bool + default = false + description = "If true, turn on public HTTPS ingress to the space state webhook (port 8081, /hooks/space-state) and read SPACESTATE_WEBHOOK_SECRET from Key Vault secret spacestate-webhook-secret. Push that secret first. The health probes stay private either way." +} + +variable "webhook_allowed_ip_ranges" { + type = list(string) + default = [] + description = "Optional. CIDR ranges allowed to reach the webhook ingress, such as the space state service's outbound addresses. Empty allows anyone: every request must still be signed with the webhook secret." + + validation { + condition = alltrue([for r in var.webhook_allowed_ip_ranges : can(cidrhost(r, 0))]) + error_message = "webhook_allowed_ip_ranges must be CIDR ranges like 203.0.113.7/32." + } +} + variable "tailscale_auth_key" { type = string ephemeral = true diff --git a/infra/secrets.manifest.yaml b/infra/secrets.manifest.yaml index 94e2451..e920b33 100644 --- a/infra/secrets.manifest.yaml +++ b/infra/secrets.manifest.yaml @@ -22,6 +22,7 @@ features: mastodon: Mastodon announcements (MASTODON_INSTANCE is set) spacestate: /spacestate (SPACESTATE_API_URL is set) spacestate-headers: Extra headers for the /spacestate call (spacestate_additional_headers_enabled) + spacestate-webhook: The space state webhook and its public ingress (spacestate_webhook_enabled) sentry: Sentry error reporting (sentry_enabled) tailscale: Tailscale sidecar (tailscale_enabled) container-registry: Private image registry (container_registry_server is set) @@ -87,6 +88,13 @@ secrets: feature: spacestate-headers description: Extra headers for the space state API, as a JSON object (for example the address the call is made from). + - vault: spacestate-webhook-secret + group: app + delivery: env + env: SPACESTATE_WEBHOOK_SECRET + feature: spacestate-webhook + description: Shared with the space state service, which signs its webhook with it (HMAC-SHA256). At least 32 characters. + - vault: sentry-dsn group: app delivery: env diff --git a/src/adapters/webhook/server.test.ts b/src/adapters/webhook/server.test.ts new file mode 100644 index 0000000..a3ac498 --- /dev/null +++ b/src/adapters/webhook/server.test.ts @@ -0,0 +1,236 @@ +import { once } from "node:events"; +import { request } from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { Logger } from "../../core/logger.ts"; +import { MAX_BODY_BYTES, SPACE_STATE_WEBHOOK_PATH, startWebhookServer } from "./server.ts"; +import { SIGNATURE_HEADER, sign, TIMESTAMP_HEADER } from "./signature.ts"; + +const SECRET = "test-webhook-secret-not-a-real-one-0000"; +const NOW = new Date("2026-10-20T12:00:00Z"); +const SENT_AT = NOW.getTime() / 1000; +const BODY = '{"state":"open","changedAt":"2026-10-20T12:00:00.000Z"}'; + +function recordingLogger() { + const lines: { level: string; obj: object }[] = []; + const logger: Logger = { + debug: (obj) => lines.push({ level: "debug", obj }), + info: (obj) => lines.push({ level: "info", obj }), + warn: (obj) => lines.push({ level: "warn", obj }), + error: (obj) => lines.push({ level: "error", obj }), + child: () => logger, + }; + return { logger, lines }; +} + +const signed = (body = BODY, secret = SECRET) => ({ + [TIMESTAMP_HEADER]: String(SENT_AT), + [SIGNATURE_HEADER]: sign(secret, SENT_AT, body), +}); + +describe("startWebhookServer", () => { + let close: (() => void) | undefined; + afterEach(() => close?.()); + + async function start( + onSpaceStateChanged: () => void = vi.fn(), + clock: "fixed" | "real" = "fixed", + ) { + const { logger, lines } = recordingLogger(); + const server = startWebhookServer({ + port: 0, + secret: SECRET, + logger, + onSpaceStateChanged, + ...(clock === "fixed" ? { now: () => NOW } : {}), + }); + close = () => server.close(); + await once(server, "listening"); + const { port } = server.address() as AddressInfo; + const send = async ( + path: string, + init: { method?: string; body?: string; headers?: Record } = {}, + ) => { + const response = await fetch(`http://127.0.0.1:${port}${path}`, { + method: init.method ?? "POST", + ...(init.body === undefined ? {} : { body: init.body }), + headers: init.headers ?? {}, + }); + return { status: response.status, text: await response.text(), headers: response.headers }; + }; + return { send, lines, port }; + } + + it("accepts a signed request and asks for a fresh reading, after replying", async () => { + const onChange = vi.fn(); + const { send, lines } = await start(onChange); + const response = await send(SPACE_STATE_WEBHOOK_PATH, { body: BODY, headers: signed() }); + expect(response).toMatchObject({ status: 202, text: "accepted" }); + expect(onChange).toHaveBeenCalledOnce(); + expect(lines).toContainEqual({ + level: "info", + obj: { event: "webhook.received", hook: "space-state" }, + }); + }); + + it("ignores a query string on the path", async () => { + const { send } = await start(); + const response = await send(`${SPACE_STATE_WEBHOOK_PATH}?source=test`, { + body: BODY, + headers: signed(), + }); + expect(response.status).toBe(202); + }); + + it("refuses an unsigned or badly signed request, says nothing more, and logs why", async () => { + const onChange = vi.fn(); + const { send, lines } = await start(onChange); + const unsigned = await send(SPACE_STATE_WEBHOOK_PATH, { body: BODY }); + const forged = await send(SPACE_STATE_WEBHOOK_PATH, { + body: BODY, + headers: signed(BODY, "a-guessed-secret"), + }); + expect(unsigned).toMatchObject({ status: 401, text: "unauthorized" }); + expect(forged).toMatchObject({ status: 401, text: "unauthorized" }); + expect(onChange).not.toHaveBeenCalled(); + expect(lines.filter((l) => l.level === "warn").map((l) => l.obj)).toEqual([ + { event: "webhook.rejected", hook: "space-state", reason: "missing" }, + { event: "webhook.rejected", hook: "space-state", reason: "bad-signature" }, + ]); + }); + + it("never logs the body or the signature", async () => { + const { send, lines } = await start(); + await send(SPACE_STATE_WEBHOOK_PATH, { body: BODY, headers: signed() }); + await send(SPACE_STATE_WEBHOOK_PATH, { body: BODY, headers: signed(BODY, "wrong") }); + const logged = JSON.stringify(lines); + expect(logged).not.toContain("changedAt"); + expect(logged).not.toContain("sha256="); + expect(logged).not.toContain(SECRET); + }); + + it("returns 404 for any other path, without logging", async () => { + const { send, lines } = await start(); + expect((await send("/", { method: "GET" })).status).toBe(404); + expect((await send("/healthz", { method: "GET" })).status).toBe(404); + expect((await send("/hooks/other", { body: BODY, headers: signed() })).status).toBe(404); + expect(lines).toEqual([]); + }); + + it("only takes POST", async () => { + const { send } = await start(); + const response = await send(SPACE_STATE_WEBHOOK_PATH, { method: "GET" }); + expect(response.status).toBe(405); + expect(response.headers.get("allow")).toBe("POST"); + }); + + it("refuses a body that says it is too large, without reading it", async () => { + const onChange = vi.fn(); + const { send } = await start(onChange); + const body = "x".repeat(MAX_BODY_BYTES + 1); + const response = await send(SPACE_STATE_WEBHOOK_PATH, { body, headers: signed(body) }); + expect(response.status).toBe(413); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("drops the connection when a body sent without a length grows too large", async () => { + const onChange = vi.fn(); + const { port } = await start(onChange); + const outcome = await new Promise((resolve) => { + const req = request({ + host: "127.0.0.1", + port, + path: SPACE_STATE_WEBHOOK_PATH, + method: "POST", + headers: { "transfer-encoding": "chunked", ...signed() }, + }); + req.on("response", (res) => resolve(`status ${res.statusCode}`)); + req.on("error", () => resolve("dropped")); + req.write("x".repeat(MAX_BODY_BYTES)); + req.write("x".repeat(MAX_BODY_BYTES)); + req.end(); + }); + expect(outcome).toBe("dropped"); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("ignores repeated signature headers", async () => { + const onChange = vi.fn(); + const { port } = await start(onChange); + const status = await new Promise((resolve, reject) => { + const signature = sign(SECRET, SENT_AT, BODY); + const req = request({ + host: "127.0.0.1", + port, + path: SPACE_STATE_WEBHOOK_PATH, + method: "POST", + // Node joins most repeated headers with ", ", which no longer matches. + headers: { + [TIMESTAMP_HEADER]: String(SENT_AT), + [SIGNATURE_HEADER]: [signature, signature], + }, + }); + req.on("response", (res) => { + res.resume(); + resolve(res.statusCode); + }); + req.on("error", reject); + req.end(BODY); + }); + expect(status).toBe(401); + expect(onChange).not.toHaveBeenCalled(); + }); + + it("still answers when the handler throws, and logs it", async () => { + const { send, lines } = await start(() => { + throw new Error("boom"); + }); + const response = await send(SPACE_STATE_WEBHOOK_PATH, { body: BODY, headers: signed() }); + expect(response.status).toBe(202); + expect(lines.some((l) => l.level === "error")).toBe(true); + }); + + it("uses the real clock by default", async () => { + const onChange = vi.fn(); + const { send } = await start(onChange, "real"); + const sentAt = Math.floor(Date.now() / 1000); + const response = await send(SPACE_STATE_WEBHOOK_PATH, { + body: BODY, + headers: { + [TIMESTAMP_HEADER]: String(sentAt), + [SIGNATURE_HEADER]: sign(SECRET, sentAt, BODY), + }, + }); + expect(response.status).toBe(202); + expect(onChange).toHaveBeenCalledOnce(); + }); + + it("shrugs off a sender that hangs up mid-request, and keeps serving", async () => { + const onChange = vi.fn(); + const { port, send, lines } = await start(onChange); + const req = request({ + host: "127.0.0.1", + port, + path: SPACE_STATE_WEBHOOK_PATH, + method: "POST", + headers: { "transfer-encoding": "chunked", ...signed() }, + }); + req.on("error", () => {}); + req.write("{"); + await vi.waitFor(() => expect(req.socket).toBeTruthy()); + await new Promise((resolve) => setTimeout(resolve, 20)); + req.destroy(); + await vi.waitFor(() => + expect(lines).toContainEqual( + expect.objectContaining({ + level: "debug", + obj: expect.objectContaining({ event: "webhook.aborted" }), + }), + ), + ); + expect(onChange).not.toHaveBeenCalled(); + expect((await send(SPACE_STATE_WEBHOOK_PATH, { body: BODY, headers: signed() })).status).toBe( + 202, + ); + }); +}); diff --git a/src/adapters/webhook/server.ts b/src/adapters/webhook/server.ts new file mode 100644 index 0000000..e90dd8c --- /dev/null +++ b/src/adapters/webhook/server.ts @@ -0,0 +1,113 @@ +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import type { Logger } from "../../core/logger.ts"; +import { SIGNATURE_HEADER, TIMESTAMP_HEADER, verify } from "./signature.ts"; + +/** + * The one public HTTP endpoint: the space state service calls it when the space opens or + * closes, so Pixel notices straight away instead of at the next SpaceAPI poll. + * + * It listens on its own port, which is the only one Azure's ingress exposes, so the health + * probes stay private. Every request must be signed (`signature.ts`). An accepted one only + * asks Pixel to read SpaceAPI again: the body is never trusted, logged or echoed, so the + * worst a replayed or forged request can do is cause one extra read. Polling carries on as + * before, so a webhook that never arrives costs only the delay. + */ + +export const SPACE_STATE_WEBHOOK_PATH = "/hooks/space-state"; + +/** The sender's body is small; anything bigger is refused before it's read. */ +export const MAX_BODY_BYTES = 4 * 1024; + +export type WebhookServerOptions = { + port: number; + /** Shared with the space state service. A secret. */ + secret: string; + logger: Logger; + /** Called once per accepted webhook, after the reply has gone. Must not throw. */ + onSpaceStateChanged: () => void; + now?: () => Date; +}; + +export function startWebhookServer(options: WebhookServerOptions): Server { + const logger = options.logger.child({ service: "webhook" }); + const now = options.now ?? (() => new Date()); + + const handle = async (req: IncomingMessage, res: ServerResponse): Promise => { + const path = (req.url ?? "").split("?")[0]; + if (path !== SPACE_STATE_WEBHOOK_PATH) return reply(res, 404, "not found"); + if (req.method !== "POST") return reply(res, 405, "method not allowed", { allow: "POST" }); + if (Number(req.headers["content-length"]) > MAX_BODY_BYTES) { + return reply(res, 413, "too large"); + } + + const body = await readBody(req); + if (body === undefined) { + // Sent without a length and too big: drop the connection rather than read on. + res.destroy(); + return; + } + + const check = verify({ + secret: options.secret, + timestamp: header(req, TIMESTAMP_HEADER), + signature: header(req, SIGNATURE_HEADER), + body, + now: now(), + }); + if (!check.ok) { + logger.warn( + { event: "webhook.rejected", hook: "space-state", reason: check.reason }, + "refused a space state webhook", + ); + return reply(res, 401, "unauthorized"); + } + + logger.info({ event: "webhook.received", hook: "space-state" }, "space state webhook"); + reply(res, 202, "accepted"); + try { + options.onSpaceStateChanged(); + } catch (error) { + logger.error({ event: "webhook.failed", err: error }, "space state webhook handler failed"); + } + }; + + const server = createServer((req, res) => { + handle(req, res).catch((error: unknown) => { + // The client went away mid-request, most likely. There's nobody left to tell. + logger.debug({ event: "webhook.aborted", err: error }, "webhook request failed"); + if (!res.headersSent) reply(res, 400, "bad request"); + }); + }); + // A slow sender can't hold a connection open for long. + server.headersTimeout = 5_000; + server.requestTimeout = 10_000; + server.listen(options.port); + return server; +} + +function reply( + res: ServerResponse, + status: number, + text: string, + headers: Record = {}, +): void { + res.writeHead(status, { "content-type": "text/plain", ...headers }).end(text); +} + +function header(req: IncomingMessage, name: string): string | undefined { + const value = req.headers[name]; + return Array.isArray(value) ? undefined : value; +} + +/** The body as text, or undefined once it passes {@link MAX_BODY_BYTES}. */ +async function readBody(req: IncomingMessage): Promise { + const chunks: Buffer[] = []; + let size = 0; + for await (const chunk of req) { + const buffer = chunk as Buffer; + size += buffer.length; + if (size > MAX_BODY_BYTES) return undefined; + chunks.push(buffer); + } + return Buffer.concat(chunks).toString("utf8"); +} diff --git a/src/adapters/webhook/signature.test.ts b/src/adapters/webhook/signature.test.ts new file mode 100644 index 0000000..e0b9c31 --- /dev/null +++ b/src/adapters/webhook/signature.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { SIGNATURE_TOLERANCE_SECONDS, sign, verify } from "./signature.ts"; + +const SECRET = "test-webhook-secret-not-a-real-one-0000"; +const NOW = new Date("2026-10-20T12:00:00Z"); +const SENT_AT = NOW.getTime() / 1000; +const BODY = '{"state":"open"}'; + +function check(overrides: Partial[0]> = {}) { + return verify({ + secret: SECRET, + timestamp: String(SENT_AT), + signature: sign(SECRET, SENT_AT, BODY), + body: BODY, + now: NOW, + ...overrides, + }); +} + +describe("sign", () => { + it("is the hex HMAC-SHA256 of timestamp.body, with a sha256= prefix", () => { + // A fixed vector (from openssl), so the sender can check its own implementation: + // printf '%s' '1700000000.{"state":"open"}' | openssl dgst -sha256 -hmac secret + expect(sign("secret", 1_700_000_000, '{"state":"open"}')).toBe( + "sha256=3b952390cdeffcd2f0e89af5b7edfa771f4e5c5770b143d542d4a1c624a9898f", + ); + }); +}); + +describe("verify", () => { + it("accepts a request signed with the shared secret", () => { + expect(check()).toEqual({ ok: true }); + }); + + it("accepts an upper-case signature and whitespace around the headers", () => { + const signature = ` ${sign(SECRET, SENT_AT, BODY).toUpperCase().replace("SHA256=", "sha256=")} `; + expect(check({ signature, timestamp: ` ${SENT_AT} ` })).toEqual({ ok: true }); + }); + + it.each([ + ["no timestamp", { timestamp: undefined }], + ["no signature", { signature: undefined }], + ])("refuses a request with %s", (_name, overrides) => { + expect(check(overrides)).toEqual({ ok: false, reason: "missing" }); + }); + + it.each([ + ["a timestamp that isn't a number", { timestamp: "yesterday" }], + ["a signature without the prefix", { signature: "a".repeat(64) }], + ["a signature of the wrong length", { signature: `sha256=${"a".repeat(63)}` }], + ["a signature that isn't hex", { signature: `sha256=${"z".repeat(64)}` }], + ])("refuses %s", (_name, overrides) => { + expect(check(overrides)).toEqual({ ok: false, reason: "malformed" }); + }); + + it("refuses a request sent too long ago, or too far ahead", () => { + for (const offset of [SIGNATURE_TOLERANCE_SECONDS + 1, -(SIGNATURE_TOLERANCE_SECONDS + 1)]) { + const sentAt = SENT_AT - offset; + const signature = sign(SECRET, sentAt, BODY); + expect(check({ timestamp: String(sentAt), signature })).toEqual({ + ok: false, + reason: "stale", + }); + } + }); + + it("allows a clock that is a little off", () => { + const sentAt = SENT_AT - SIGNATURE_TOLERANCE_SECONDS; + const signature = sign(SECRET, sentAt, BODY); + expect(check({ timestamp: String(sentAt), signature })).toEqual({ ok: true }); + }); + + it.each([ + ["another secret", { signature: sign("some-other-secret", SENT_AT, BODY) }], + ["a changed body", { body: '{"state":"closed"}' }], + ["a changed timestamp", { timestamp: String(SENT_AT + 1) }], + ])("refuses a signature made with %s", (_name, overrides) => { + expect(check(overrides)).toEqual({ ok: false, reason: "bad-signature" }); + }); +}); diff --git a/src/adapters/webhook/signature.ts b/src/adapters/webhook/signature.ts new file mode 100644 index 0000000..b6dc03d --- /dev/null +++ b/src/adapters/webhook/signature.ts @@ -0,0 +1,57 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; + +/** + * How the space state service signs the webhook it sends when the space opens or closes. + * + * X-Spacestate-Timestamp: + * X-Spacestate-Signature: sha256=." with the shared secret> + * + * The timestamp is inside what is signed, so an old request can't be sent again with a new + * time, and one older than {@link SIGNATURE_TOLERANCE_SECONDS} is refused. Even a valid one + * only makes Pixel read SpaceAPI again: nothing in the body is trusted. + */ + +export const TIMESTAMP_HEADER = "x-spacestate-timestamp"; +export const SIGNATURE_HEADER = "x-spacestate-signature"; + +/** How far the sender's clock may be from Pixel's, either way. */ +export const SIGNATURE_TOLERANCE_SECONDS = 300; + +const SIGNATURE = /^sha256=([0-9a-f]{64})$/; +const TIMESTAMP = /^\d{1,12}$/; + +export type SignatureCheck = + | { ok: true } + /** Why it was refused. For the log only: the caller is told nothing more than "unauthorized". */ + | { ok: false; reason: "missing" | "malformed" | "stale" | "bad-signature" }; + +/** The signature header value for a body sent at `timestamp` (unix seconds). */ +export function sign(secret: string, timestamp: number, body: string): string { + return `sha256=${hmac(secret, `${timestamp}.${body}`).toString("hex")}`; +} + +export function verify(options: { + secret: string; + timestamp: string | undefined; + signature: string | undefined; + body: string; + now: Date; +}): SignatureCheck { + const { secret, timestamp, signature, body, now } = options; + if (timestamp === undefined || signature === undefined) return { ok: false, reason: "missing" }; + const match = SIGNATURE.exec(signature.trim().toLowerCase()); + const sentAt = timestamp.trim(); + if (!match?.[1] || !TIMESTAMP.test(sentAt)) return { ok: false, reason: "malformed" }; + + const skew = Math.abs(now.getTime() / 1000 - Number(sentAt)); + if (skew > SIGNATURE_TOLERANCE_SECONDS) return { ok: false, reason: "stale" }; + + const expected = hmac(secret, `${sentAt}.${body}`); + const given = Buffer.from(match[1], "hex"); + // Both are 32 bytes: the pattern above fixes the length, so this never throws. + return timingSafeEqual(expected, given) ? { ok: true } : { ok: false, reason: "bad-signature" }; +} + +function hmac(secret: string, data: string): Buffer { + return createHmac("sha256", secret).update(data).digest(); +} diff --git a/src/app.test.ts b/src/app.test.ts index 22b93a2..6acd6ab 100644 --- a/src/app.test.ts +++ b/src/app.test.ts @@ -46,6 +46,7 @@ describe("buildCore", () => { logDir: undefined, timezone: "Europe/Amsterdam", healthPort: 0, + spaceStateWebhook: undefined, sentryDsn: undefined, sentryTracesSampleRate: 1, sentryProfileSessionSampleRate: 1, diff --git a/src/config.test.ts b/src/config.test.ts index f1fdb1e..83b6e84 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -431,6 +431,48 @@ describe("space state API settings", () => { }); }); +describe("the space state webhook", () => { + const SECRET = "test-webhook-secret-not-a-real-one-0000"; + + it("is off without a secret", () => { + expect(loadConfig(VALID).spaceStateWebhook).toBeUndefined(); + expect( + loadConfig({ ...VALID, SPACESTATE_WEBHOOK_SECRET: "" }).spaceStateWebhook, + ).toBeUndefined(); + }); + + it("listens on 8081 unless told otherwise", () => { + expect(loadConfig({ ...VALID, SPACESTATE_WEBHOOK_SECRET: SECRET }).spaceStateWebhook).toEqual({ + port: 8081, + secret: SECRET, + }); + expect( + loadConfig({ ...VALID, SPACESTATE_WEBHOOK_SECRET: SECRET, WEBHOOK_PORT: "9000" }) + .spaceStateWebhook?.port, + ).toBe(9000); + }); + + it("refuses a short secret without echoing it", () => { + let message = ""; + try { + loadConfig({ ...VALID, SPACESTATE_WEBHOOK_SECRET: "too-short-secret" }); + } catch (error) { + message = (error as Error).message; + } + expect(message).toMatch(/SPACESTATE_WEBHOOK_SECRET: must be at least 32 characters/); + expect(message).not.toContain("too-short-secret"); + }); + + it("won't share the health port, which must stay private", () => { + expect(() => loadConfig({ ...VALID, WEBHOOK_PORT: "8080" })).toThrow(/WEBHOOK_PORT/); + expect(() => loadConfig({ ...VALID, WEBHOOK_PORT: "9000", HEALTH_PORT: "9000" })).toThrow( + /WEBHOOK_PORT/, + ); + // 0 picks a free port each time, so both can be 0 (tests do this). + expect(() => loadConfig({ ...VALID, WEBHOOK_PORT: "0", HEALTH_PORT: "0" })).not.toThrow(); + }); +}); + describe("Mastodon settings", () => { const INSTANCE = "https://mastodon.example"; const TOKEN = "mastodon-test-token"; diff --git a/src/config.ts b/src/config.ts index d5661e7..54e5dc9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -81,6 +81,8 @@ const envSchema = z .default("Europe/Amsterdam") .refine(isValidTimeZone, { error: "must be a time zone name like Europe/Amsterdam" }), HEALTH_PORT: z.coerce.number().int().min(0).max(65535).default(8080), + // The public port: the space state webhook. Only listened on when the secret below is set. + WEBHOOK_PORT: z.coerce.number().int().min(0).max(65535).default(8081), SENTRY_DSN: optional(z.url()), // 0 drops every trace, 1 keeps them all. Empty uses the default (1). SENTRY_TRACES_SAMPLE_RATE: sampleRate, @@ -93,6 +95,10 @@ const envSchema = z SPACESTATE_TOKEN: optional(z.string().trim().min(1)), // Optional, and only used when the three above are set. SPACESTATE_ADDITIONAL_HEADERS: optional(extraHeaders), + // Unset means no webhook: Pixel only polls. Shared with the space state service, which signs with it. + SPACESTATE_WEBHOOK_SECRET: optional( + z.string().trim().min(32, { error: "must be at least 32 characters" }).max(512), + ), DISCORD_TOKEN: z.string().min(1), DISCORD_APP_ID: snowflake, @@ -139,6 +145,10 @@ const envSchema = z path: ["MASTODON_INSTANCE"], error: "set both MASTODON_INSTANCE and MASTODON_ACCESS_TOKEN, or neither", }) + .refine((e) => e.WEBHOOK_PORT === 0 || e.WEBHOOK_PORT !== e.HEALTH_PORT, { + path: ["WEBHOOK_PORT"], + error: "must differ from HEALTH_PORT: the webhook port is public, the health port is not", + }) .refine( (e) => [e.SPACESTATE_API_URL, e.SPACESTATE_USER, e.SPACESTATE_TOKEN].every((v) => v === undefined) || @@ -173,6 +183,12 @@ export type Config = { /** The time zone times are shown in, and that `{{date}}` tokens use. E.g. "Europe/Amsterdam". */ timezone: string; healthPort: number; + /** + * The space state webhook (`adapters/webhook`): the port it listens on, which is the only + * one exposed to the internet, and the secret requests are signed with. A secret. Undefined + * means off: nothing listens and Pixel only polls SpaceAPI. + */ + spaceStateWebhook: { port: number; secret: string } | undefined; sentryDsn: string | undefined; /** How many traces Sentry keeps (0–1). Default 1. Unused without a DSN. */ sentryTracesSampleRate: number; @@ -267,6 +283,10 @@ export function loadConfig(env: NodeJS.ProcessEnv = process.env): Config { logDir: e.PIXEL_LOG_DIR === "" ? undefined : e.PIXEL_LOG_DIR, timezone: e.PIXEL_TIMEZONE, healthPort: e.HEALTH_PORT, + spaceStateWebhook: + e.SPACESTATE_WEBHOOK_SECRET !== undefined + ? { port: e.WEBHOOK_PORT, secret: e.SPACESTATE_WEBHOOK_SECRET } + : undefined, sentryDsn: e.SENTRY_DSN, sentryTracesSampleRate: e.SENTRY_TRACES_SAMPLE_RATE ?? 1, sentryProfileSessionSampleRate: e.SENTRY_PROFILE_SESSION_SAMPLE_RATE ?? 1, diff --git a/src/features/features.test.ts b/src/features/features.test.ts index 01d04a5..503b393 100644 --- a/src/features/features.test.ts +++ b/src/features/features.test.ts @@ -40,6 +40,7 @@ const access: AccessStore = { const spaceStatus: SpaceStatus = { pollIntervalMs: 30_000, checkNow: async () => ({ state: "open", since: null, checkedAt: new Date() }), + recheck: async () => ({ state: "open", since: null, checkedAt: new Date() }), start() {}, stop() {}, onChange: () => () => {}, diff --git a/src/features/status/announce.test.ts b/src/features/status/announce.test.ts index cc082e8..6e0b83c 100644 --- a/src/features/status/announce.test.ts +++ b/src/features/status/announce.test.ts @@ -2,7 +2,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { Announcement, SpaceSnapshot } from "../../core/announcement.ts"; import { type Logger, silentLogger } from "../../core/logger.ts"; import type { SpaceChange, SpaceReading } from "../../services/space-status.ts"; -import { READINGS_TO_CONFIRM, startSpaceAnnouncements } from "./announce.ts"; +import { PUSH_CONFIRM_MS, READINGS_TO_CONFIRM, startSpaceAnnouncements } from "./announce.ts"; const INTERVAL = 30_000; const T0 = new Date("2026-10-03T12:00:00Z"); @@ -61,7 +61,8 @@ function setup( await vi.advanceTimersByTimeAsync(0); }, emit: (change: Partial & Pick) => { - for (const listener of listeners) listener({ at: T0, previousSince: null, ...change }); + for (const listener of listeners) + listener({ at: T0, previousSince: null, via: "poll", ...change }); }, advance: (ms: number) => vi.advanceTimersByTimeAsync(ms), }; @@ -123,6 +124,35 @@ describe("announcing changes", () => { }); }); + it("confirms a change the webhook pointed out after a few seconds, not a whole interval", async () => { + expect(PUSH_CONFIRM_MS).toBeLessThan(INTERVAL); + const { announcer, set, emit, advance } = setup(reading("closed")); + await advance(0); + + set(reading("open", T0)); + emit({ from: "closed", to: "open", at: T0, via: "push" }); + await advance(PUSH_CONFIRM_MS - 1); + expect(announcer.announce).not.toHaveBeenCalled(); + + await advance(1); + expect(announcer.announce).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ state: "open", at: T0 }), + ); + }); + + it("announces nothing when the webhook says it flipped straight back", async () => { + const { announcer, set, emit, advance } = setup(reading("closed")); + await advance(0); + + set(reading("open", T0)); + emit({ from: "closed", to: "open", via: "push" }); + await advance(PUSH_CONFIRM_MS / 2); + set(reading("closed")); + emit({ from: "open", to: "closed", via: "push" }); + await advance(INTERVAL * 2); + expect(announcer.announce).not.toHaveBeenCalled(); + }); + it("says how long the space had been open when it closes", async () => { const openedAt = minutesBefore(T0, 200); const { announcer, set, emit, advance } = setup(reading("open", openedAt)); diff --git a/src/features/status/announce.ts b/src/features/status/announce.ts index 57550bd..2ee89ec 100644 --- a/src/features/status/announce.ts +++ b/src/features/status/announce.ts @@ -7,6 +7,13 @@ import type { SpaceChange, SpaceReading, SpaceStatus } from "../../services/spac /** A change is announced once the new state has been seen on this many polls in a row. */ export const READINGS_TO_CONFIRM = 2; +/** + * How long to wait before confirming a change the webhook told us about. A webhook comes + * for every flip, so a switch flicked back sends another one, which resets the wait; + * there's no need to wait for a whole poll to rule that out. + */ +export const PUSH_CONFIRM_MS = 5_000; + /** How many extra intervals to keep trying to confirm a change while SpaceAPI is unreachable. */ const MAX_CONFIRM_RETRIES = 10; @@ -29,6 +36,8 @@ export type SpaceAnnouncementsDeps = { * - **Only once it holds.** A change is announced after the new state has been * seen on {@link READINGS_TO_CONFIRM} polls in a row. If it flips back before * that, nothing is posted, so flicking the switch doesn't flood the channel. + * A change the webhook pointed out is read again after {@link PUSH_CONFIRM_MS} + * instead of a whole poll interval. * - **At most once.** A change is never announced twice, even if every * publisher fails. * - At startup, publishers get the current state to correct stale posts @@ -126,7 +135,8 @@ export function startSpaceAnnouncements({ lastChange = change; confirmRetries = 0; clearTimeout(settleTimer); - settleTimer = setTimeout(() => run("settle", settle), (READINGS_TO_CONFIRM - 1) * intervalMs); + const wait = change.via === "push" ? PUSH_CONFIRM_MS : (READINGS_TO_CONFIRM - 1) * intervalMs; + settleTimer = setTimeout(() => run("settle", settle), wait); }); run("reconcile", reconcile); diff --git a/src/index.ts b/src/index.ts index f811860..b266abb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,7 @@ import * as Sentry from "@sentry/node"; import { createDiscordAdapter } from "./adapters/discord/index.ts"; import { HomeAssistantBackend } from "./adapters/home-assistant/backend.ts"; import { startMastodonPublisher } from "./adapters/mastodon/index.ts"; +import { startWebhookServer } from "./adapters/webhook/server.ts"; import { buildCore } from "./app.ts"; import { loadConfig } from "./config.ts"; import { type Stop, startFeatures } from "./core/feature.ts"; @@ -125,6 +126,17 @@ async function main(): Promise { // blocks Discord. Registered before Discord connects so it exists when features start. startMastodonPublisher({ mastodon: config.mastodon, announcer, logger }); const health = startHealthServer(config.healthPort, () => discord.isReady()); + // The space state service calls this when the space opens or closes, so Pixel reads SpaceAPI + // straight away. Polling carries on either way: a webhook that never arrives only costs time. + const webhook = config.spaceStateWebhook + ? startWebhookServer({ + ...config.spaceStateWebhook, + logger, + // Failures are already logged and counted by the status service. + onSpaceStateChanged: () => void spaceStatus.recheck().catch(() => {}), + }) + : undefined; + if (!webhook) logger.info({ event: "webhook.off" }, "space state webhook is off: polling only"); // Check in with Sentry while healthy, so it can alert when Pixel goes quiet (a crash, a // hang, the host going down). Only with Sentry set up, and never from a laptop. const heartbeatPlan = planHeartbeat(config); @@ -152,6 +164,7 @@ async function main(): Promise { spaceStatus.stop(); homeAssistant?.close(); await discord.stop(); + webhook?.close(); health.close(); await Sentry.flush(2000); process.exit(0); diff --git a/src/observability/sentry-options.test.ts b/src/observability/sentry-options.test.ts index bea0dcc..3b9ebe5 100644 --- a/src/observability/sentry-options.test.ts +++ b/src/observability/sentry-options.test.ts @@ -3,7 +3,7 @@ import * as Sentry from "@sentry/node"; import { pino } from "pino"; import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; import { REDACT_PATHS } from "./logger.ts"; -import { isHealthProbe, SENTRY_LOG_LEVELS, sentryOptions } from "./sentry-options.ts"; +import { isUntracedRequest, SENTRY_LOG_LEVELS, sentryOptions } from "./sentry-options.ts"; const sentryCfg = { dsn: "https://key@o0.ingest.sentry.io/1", @@ -73,12 +73,14 @@ describe("sentryOptions", () => { ]); }); - it("ignores health probes, including ones with a query string", () => { - expect(isHealthProbe("/healthz")).toBe(true); - expect(isHealthProbe("/readyz")).toBe(true); - expect(isHealthProbe("/healthz?ready=1")).toBe(true); - expect(isHealthProbe("/")).toBe(false); - expect(isHealthProbe("/api")).toBe(false); + it("traces only the webhooks: not health probes, nor anything else that turns up", () => { + expect(isUntracedRequest("/healthz")).toBe(true); + expect(isUntracedRequest("/readyz")).toBe(true); + expect(isUntracedRequest("/healthz?ready=1")).toBe(true); + expect(isUntracedRequest("/")).toBe(true); + expect(isUntracedRequest("/wp-login.php")).toBe(true); + expect(isUntracedRequest("/hooks/space-state")).toBe(false); + expect(isUntracedRequest("/hooks/space-state?x=1")).toBe(false); }); it("scrubs secrets from a log before it is sent: the message and every attribute", () => { diff --git a/src/observability/sentry-options.ts b/src/observability/sentry-options.ts index bcb2ddb..5e57eae 100644 --- a/src/observability/sentry-options.ts +++ b/src/observability/sentry-options.ts @@ -11,13 +11,14 @@ import { scrubDeep } from "./scrub.ts"; export const SENTRY_LOG_LEVELS = ["info", "warn", "error", "fatal"] as const; /** - * Azure (and local) probes hit these. They aren't worth a Sentry trace, and they - * must not be counted as request sessions: Pixel's session is the process, not - * an HTTP request. + * Incoming requests that aren't worth a Sentry trace: everything but the webhooks + * (`/hooks/...`). That covers Azure's `/healthz` and `/readyz` probes, and whatever the + * internet tries against the public webhook port. None of them count as request + * sessions either: Pixel's session is the process, not an HTTP request. */ -export function isHealthProbe(urlPath: string): boolean { +export function isUntracedRequest(urlPath: string): boolean { const path = urlPath.split("?")[0] ?? urlPath; - return path === "/healthz" || path === "/readyz"; + return !path.startsWith("/hooks/"); } /** @@ -27,8 +28,8 @@ export function isHealthProbe(urlPath: string): boolean { * secrets have been scrubbed from it. * * Tracing, process sessions, runtime metrics and CPU profiling are on: this is - * a long-running Discord bot, not a request-scoped web app. Incoming `/healthz` - * and `/readyz` probes are ignored. A missing DSN never reaches here. + * a long-running Discord bot, not a request-scoped web app. Of incoming requests, + * only the webhooks are traced. A missing DSN never reaches here. */ export function sentryOptions(sentry: SentryConfig): Sentry.NodeOptions { return { @@ -53,7 +54,7 @@ export function sentryOptions(sentry: SentryConfig): Sentry.NodeOptions { profileLifecycle: "trace", integrations: [ Sentry.httpIntegration({ - ignoreIncomingRequests: isHealthProbe, + ignoreIncomingRequests: isUntracedRequest, // Health probes would otherwise be one "session" each. Crash-free // rate comes from processSessionIntegration (a default), not HTTP. sessions: false, diff --git a/src/services/space-status.test.ts b/src/services/space-status.test.ts index 911b75e..56abc69 100644 --- a/src/services/space-status.test.ts +++ b/src/services/space-status.test.ts @@ -52,6 +52,18 @@ function fakeFetch() { queue.push(async () => Promise.reject(error)); return this; }, + /** A response that arrives only when `release` is called. */ + held(body: unknown) { + let release: () => void = () => {}; + const arrived = new Promise((resolve) => { + release = resolve; + }); + queue.push(async () => { + await arrived; + return new Response(JSON.stringify(body)); + }); + return release; + }, }; } @@ -84,6 +96,70 @@ function setup(options: { failureThreshold?: number; store?: SpaceStateStore } = }; } +describe("SpaceApiStatus.recheck", () => { + it("checks straight away when nothing is under way, and says the change came from a push", async () => { + const { service, fetch, advance } = setup(); + const changes: SpaceChange[] = []; + service.onChange((c) => changes.push(c)); + fetch.json(open(false)).json(open(true)); + await service.checkNow(); + const at = advance(1000); + expect((await service.recheck()).state).toBe("open"); + expect(changes).toEqual([{ from: "closed", to: "open", at, previousSince: null, via: "push" }]); + }); + + it("never reuses a check that started before it: it waits, then asks again", async () => { + const { service, fetch } = setup(); + const changes: SpaceChange[] = []; + service.onChange((c) => changes.push(c)); + fetch.json(open(false)); + await service.checkNow(); + + // A poll is under way and will read the state from before the change. + const release = fetch.held(open(false)); + fetch.json(open(true)); + const poll = service.checkNow(); + const recheck = service.recheck(); + release(); + + expect((await poll).state).toBe("closed"); + expect((await recheck).state).toBe("open"); + expect(fetch.fn).toHaveBeenCalledTimes(3); + expect(changes.map((c) => c.via)).toEqual(["push"]); + }); + + it("shares one follow-up check between rechecks that arrive during the same check", async () => { + const { service, fetch } = setup(); + const release = fetch.held(open(false)); + fetch.json(open(true)); + const poll = service.checkNow(); + const first = service.recheck(); + const second = service.recheck(); + release(); + await poll; + const [a, b] = await Promise.all([first, second]); + expect(a).toBe(b); + expect(fetch.fn).toHaveBeenCalledTimes(2); + }); + + it("still asks again when the check it waited for failed", async () => { + const { service, fetch } = setup(); + fetch.fail(new Error("network down")).json(open(true)); + const poll = service.checkNow(); + const recheck = service.recheck(); + await expect(poll).rejects.toBeInstanceOf(SpaceApiError); + expect((await recheck).state).toBe("open"); + }); + + it("lets other checks share the recheck's request", async () => { + const { service, fetch } = setup(); + fetch.json(open(true)); + const [a, b] = await Promise.all([service.recheck(), service.checkNow()]); + expect(a).toBe(b); + expect(fetch.fn).toHaveBeenCalledTimes(1); + }); +}); + describe("SpaceApiStatus.checkNow", () => { it("reads open and closed from state.open", async () => { const { service, fetch } = setup(); @@ -215,7 +291,7 @@ describe("tracking changes", () => { await service.checkNow(); const at = advance(1000); await service.checkNow(); - expect(changes).toEqual([{ from: "closed", to: "open", at, previousSince: null }]); + expect(changes).toEqual([{ from: "closed", to: "open", at, previousSince: null, via: "poll" }]); }); it("tells listeners when the state it changed from began, if it saw that", async () => { @@ -229,8 +305,8 @@ describe("tracking changes", () => { const closedAt = advance(3 * 3_600_000); await service.checkNow(); expect(changes).toEqual([ - { from: "closed", to: "open", at: openedAt, previousSince: null }, - { from: "open", to: "closed", at: closedAt, previousSince: openedAt }, + { from: "closed", to: "open", at: openedAt, previousSince: null, via: "poll" }, + { from: "open", to: "closed", at: closedAt, previousSince: openedAt, via: "poll" }, ]); }); @@ -417,7 +493,9 @@ describe("persistence", () => { await service.checkNow(); const at = advance(60_000); await service.checkNow(); - expect(changes).toEqual([{ from: "open", to: "closed", at, previousSince: earlier }]); + expect(changes).toEqual([ + { from: "open", to: "closed", at, previousSince: earlier, via: "poll" }, + ]); expect(store.saves).toEqual([{ state: "closed", since: at }]); }); diff --git a/src/services/space-status.ts b/src/services/space-status.ts index 20e90c3..e3a1ae8 100644 --- a/src/services/space-status.ts +++ b/src/services/space-status.ts @@ -9,6 +9,10 @@ import type { PersistedSpaceState, SpaceStateStore } from "./space-state-store.t * - Background polling (`start()`) keeps checking so Pixel notices changes even * when nobody asks — that's how it knows "since when". SpaceAPI itself has no * change timestamp, so "since" is only known for changes Pixel observed. + * - `recheck()` is for when something says the state just changed (the space state + * webhook): it never reuses a check that was already under way, because that one may + * have read the state from before the change. It's only a nudge: the state still comes + * from SpaceAPI, never from whoever asked. * - `onChange` fires on real open↔closed flips (used by announcements, #3). * - With a `store`, the last known state and "since" survive restarts. On * startup the saved state is only trusted once a live reading agrees with it: @@ -35,12 +39,19 @@ export type SpaceChange = { at: Date; /** When the state it changed *from* began, if Pixel knew (e.g. how long it had been open). */ previousSince: Date | null; + /** `push` when a `recheck()` (the webhook) found it, `poll` for any other check. */ + via: CheckTrigger; }; +/** What started a check: a webhook saying the state changed, or anything else. */ +export type CheckTrigger = "poll" | "push"; + export type SpaceStatus = { /** How often the background poll checks SpaceAPI. */ readonly pollIntervalMs: number; checkNow(): Promise; + /** Checks again because the state is said to have changed. Never reuses an earlier check. */ + recheck(): Promise; start(): void; stop(): void; onChange(listener: (change: SpaceChange) => void): () => void; @@ -89,6 +100,8 @@ export class SpaceApiStatus implements SpaceStatus { #restored = false; #saveFailureReported = false; #inFlight: Promise | undefined; + /** A recheck waiting for the check under way to finish, so it can start a fresh one. */ + #queuedRecheck: Promise | undefined; #consecutiveFailures = 0; #timer: ReturnType | undefined; @@ -112,10 +125,28 @@ export class SpaceApiStatus implements SpaceStatus { } checkNow(): Promise { - this.#inFlight ??= this.#check().finally(() => { + return this.#inFlight ?? this.#startCheck("poll"); + } + + recheck(): Promise { + const underWay = this.#inFlight; + if (!underWay) return this.#startCheck("push"); + // Rechecks that arrive during the same check share the one that follows it. + this.#queuedRecheck ??= underWay + .catch(() => {}) + .then(() => { + this.#queuedRecheck = undefined; + return this.#inFlight ?? this.#startCheck("push"); + }); + return this.#queuedRecheck; + } + + #startCheck(via: CheckTrigger): Promise { + const check = this.#check(via).finally(() => { this.#inFlight = undefined; }); - return this.#inFlight; + this.#inFlight = check; + return check; } start(): void { @@ -140,7 +171,7 @@ export class SpaceApiStatus implements SpaceStatus { return () => this.#listeners.delete(listener); } - async #check(): Promise { + async #check(via: CheckTrigger): Promise { let state: SpaceState; try { state = await this.#fetchState(); @@ -151,7 +182,7 @@ export class SpaceApiStatus implements SpaceStatus { : new SpaceApiError("SpaceAPI request failed", { cause: error }); } this.#recordSuccess(); - return this.#record(state); + return this.#record(state, via); } async #fetchState(): Promise { @@ -208,7 +239,7 @@ export class SpaceApiStatus implements SpaceStatus { ); } - #record(state: SpaceState): SpaceReading { + #record(state: SpaceState, via: CheckTrigger): SpaceReading { const now = this.#now(); if (state === "unknown") return { state, since: null, checkedAt: now }; @@ -228,10 +259,10 @@ export class SpaceApiStatus implements SpaceStatus { const previousSince = previous.since; this.#remember({ state, since: now }); this.#logger.info( - { event: "spaceapi.changed", from: previous.state, to: state }, + { event: "spaceapi.changed", from: previous.state, to: state, via }, "space state changed", ); - this.#emit({ from: previous.state, to: state, at: now, previousSince }); + this.#emit({ from: previous.state, to: state, at: now, previousSince, via }); } } this.#restored = false; diff --git a/vitest.config.ts b/vitest.config.ts index 5b8e1bd..6ccfc5e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -34,6 +34,13 @@ export default defineConfig({ functions: 98, branches: 85, }, + // The one public endpoint: who may nudge Pixel is decided here. + "src/adapters/webhook/{signature,server}.ts": { + lines: 98, + statements: 98, + functions: 98, + branches: 85, + }, "src/observability/{scrub,sentry-reporter,health}.ts": { lines: 98, statements: 98, From 5eb479547d276b1050a29d4d35003ba425c49bbd Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 20 Oct 2026 17:57:19 +0000 Subject: [PATCH 2/2] Bind a custom hostname (pixel.pixelbar.nl) to the webhook ingress custom_domain and bind_custom_domain put the webhook ingress on a hostname of our own with an Azure-managed certificate. pixelbar.nl's DNS is at TransIP, so it takes two applies: the first only prints the asuid TXT and CNAME records (custom_domain_dns_records); the second issues the certificate (CNAME validation) and binds the hostname once they resolve. spacestate_webhook_url then uses the hostname. Both variables are validated: a domain needs the webhook ingress, binding needs a domain. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Q6YEJeThAVV24LzGijYXTB --- CHANGELOG.md | 2 +- docs/adr/0018-space-state-webhook.md | 1 + docs/infra.md | 5 +-- docs/runbook.md | 12 ++++++++ infra/envs/dev/README.md | 2 +- infra/envs/dev/main.tf | 2 ++ infra/envs/dev/outputs.tf | 5 +++ infra/envs/dev/terraform.example.tfvars | 4 +++ infra/envs/dev/variables.tf | 12 ++++++++ infra/envs/prod/README.md | 2 +- infra/envs/prod/main.tf | 2 ++ infra/envs/prod/outputs.tf | 5 +++ infra/envs/prod/terraform.example.tfvars | 4 +++ infra/envs/prod/variables.tf | 12 ++++++++ infra/modules/pixel/domain.tf | 39 ++++++++++++++++++++++++ infra/modules/pixel/outputs.tf | 19 +++++++++++- infra/modules/pixel/variables.tf | 27 ++++++++++++++++ 17 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 infra/modules/pixel/domain.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index e6218c4..1d9443d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ Until 1.0.0, releases stay on 0.x: `Added` bumps minor, `Fixed`/`Changed` (and t ### Added -- Space state webhook: the space state service can call `POST /hooks/space-state` when the space opens or closes, and Pixel reads SpaceAPI straight away instead of waiting for the next 30-second poll. A change found this way is confirmed after 5 seconds, so open/closed announcements go out in about 5 seconds rather than 30–60. Polling carries on unchanged as the fallback. Requests must be signed with HMAC-SHA256 (`X-Spacestate-Timestamp`, `X-Spacestate-Signature`) using the new secret `SPACESTATE_WEBHOOK_SECRET` (Key Vault `spacestate-webhook-secret`), and an accepted one only makes Pixel read SpaceAPI again: nothing in it is trusted or logged. Pixel listens on its own port, `WEBHOOK_PORT` (default 8081), only when the secret is set. On Azure, the new `spacestate_webhook_enabled` Terraform variable turns on HTTPS ingress to that port alone (the health probes stay private), `webhook_allowed_ip_ranges` can limit who reaches it, and the `spacestate_webhook_url` output is the address to give the service. Sentry now traces only incoming webhook requests, so probes and internet scanners don't use the trace quota. See [ADR 0018](docs/adr/0018-space-state-webhook.md) and the runbook's "Instant updates: the space state webhook". +- Space state webhook: the space state service can call `POST /hooks/space-state` when the space opens or closes, and Pixel reads SpaceAPI straight away instead of waiting for the next 30-second poll. A change found this way is confirmed after 5 seconds, so open/closed announcements go out in about 5 seconds rather than 30–60. Polling carries on unchanged as the fallback. Requests must be signed with HMAC-SHA256 (`X-Spacestate-Timestamp`, `X-Spacestate-Signature`) using the new secret `SPACESTATE_WEBHOOK_SECRET` (Key Vault `spacestate-webhook-secret`), and an accepted one only makes Pixel read SpaceAPI again: nothing in it is trusted or logged. Pixel listens on its own port, `WEBHOOK_PORT` (default 8081), only when the secret is set. On Azure, the new `spacestate_webhook_enabled` Terraform variable turns on HTTPS ingress to that port alone (the health probes stay private), `webhook_allowed_ip_ranges` can limit who reaches it, and the `spacestate_webhook_url` output is the address to give the service. `custom_domain` and `bind_custom_domain` put it on a hostname of our own, `pixel.pixelbar.nl` for prod, with an Azure-managed certificate: the first apply prints the TXT and CNAME records to create at TransIP (`custom_domain_dns_records`), the second binds the hostname once they resolve. Sentry now traces only incoming webhook requests, so probes and internet scanners don't use the trace quota. See [ADR 0018](docs/adr/0018-space-state-webhook.md) and the runbook's "Instant updates: the space state webhook". ## [0.18.0] - 2026-10-19 diff --git a/docs/adr/0018-space-state-webhook.md b/docs/adr/0018-space-state-webhook.md index 37758d4..1130d03 100644 --- a/docs/adr/0018-space-state-webhook.md +++ b/docs/adr/0018-space-state-webhook.md @@ -27,6 +27,7 @@ Until now Pixel had no public ingress at all: everything it does is outbound. Ta Pixel answers `202` when it accepts the call, `401` for a missing, malformed, stale (more than 5 minutes either way) or wrong signature, `404` for any other path, `405` for anything but POST, and `413` for a body over 4 KiB. The answer never says why a call was refused; Pixel's log does (`webhook.rejected` with a `reason`). The sender should send once, with a short timeout, and not retry: polling is the retry. A test vector is in `src/adapters/webhook/signature.test.ts`. - **Its own port.** The webhook listens on `WEBHOOK_PORT` (8081), and only when the secret is set. Azure ingress targets only that port, so `/healthz` and `/readyz` (8080) stay private. Config refuses to put both on the same port. - **Ingress is opt-in per stack.** `spacestate_webhook_enabled` turns on external HTTPS ingress (`allow_insecure_connections = false`, Azure terminates TLS) and wires the secret. `webhook_allowed_ip_ranges` can limit it to the sender's addresses. The URL is the `spacestate_webhook_url` output. Ingress does not change scaling: `min_replicas = max_replicas = 1` still holds. +- **A custom hostname, optionally.** `custom_domain` (prod: `pixel.pixelbar.nl`) binds a hostname to the same ingress with an Azure-managed certificate (`infra/modules/pixel/domain.tf`). pixelbar.nl's DNS is at TransIP, outside Azure, so it takes two applies: the first only prints the `asuid.` TXT and CNAME records (`custom_domain_dns_records`); `bind_custom_domain = true` issues the certificate (CNAME validation, the same record the binding needs) and binds it once they resolve. Then `spacestate_webhook_url` uses the hostname, which survives the app or its environment being recreated, unlike the generated `*.azurecontainerapps.io` address. The generated address keeps working too. - **Only `/hooks/` is traced in Sentry.** Probes and whatever else the internet sends to the public port are not, so scanners don't spend the trace quota. ## Consequences diff --git a/docs/infra.md b/docs/infra.md index 368f845..dabea61 100644 --- a/docs/infra.md +++ b/docs/infra.md @@ -21,7 +21,7 @@ West Europe, subscription in bootstrap / env variables. Live **`pixel-dev`** is | Backup | `pixel-dev-rsv` / `pixel-prod-rsv` | Daily share backup, 14-day keep (plus 14-day share soft-delete) | | Logs | `pixel-dev-logs` / `pixel-prod-logs` | Container Apps diagnostics, 30 days | | CAE | `pixel-dev-cae` / `pixel-prod-cae` | Consumption, **no VNet** | -| App | `pixel-dev` / `pixel-prod` | `min = max` replicas **1**, `revision_mode = Single`, ingress **none** unless `spacestate_webhook_enabled`: then external HTTPS to port 8081 only (the webhook), optionally limited to `webhook_allowed_ip_ranges` | +| App | `pixel-dev` / `pixel-prod` | `min = max` replicas **1**, `revision_mode = Single`, ingress **none** unless `spacestate_webhook_enabled`: then external HTTPS to port 8081 only (the webhook), optionally limited to `webhook_allowed_ip_ranges`, optionally on a custom hostname (`pixel.pixelbar.nl` for prod) with a managed certificate | | Image | `ghcr.io/pixelbar/pixel:` (public) | No `.env`, no access YAML. Azure `dev` is rolled by `just deploy dev`; `prod` by the Release workflow after approval | **CD:** `just deploy dev` on a laptop builds linux/amd64, pushes GHCR, `az containerapp update` on `pixel-dev` with min=max=1, then `just register dev`. Merges to `main` publish GHCR (`:` and `:main`) and Sentry source maps. They do **not** roll `pixel-prod`. A Release rebuilds from the bumped commit (`:vX.Y.Z`), publishes the docs site, then Environment `prod` approval rolls `pixel-prod` to **that digest** (no Terraform apply). Production slash commands: the Release's `register-prod` job, after a second approval (fallback `just register prod`). @@ -180,7 +180,8 @@ The one exception is the space state webhook ([ADR 0018](adr/0018-space-state-we - **External HTTPS ingress, to port 8081 only.** Azure terminates TLS and refuses plain HTTP (`allow_insecure_connections = false`). The health probes are on 8080, which ingress never targets, so `/healthz` and `/readyz` stay private. On 8081 Pixel answers only `POST /hooks/space-state`; everything else is a 404. - **Every request must be signed** with Key Vault `spacestate-webhook-secret` (`SPACESTATE_WEBHOOK_SECRET`), and an accepted one only makes Pixel read SpaceAPI again. Nothing in it is trusted. - **Optionally limited by address:** `webhook_allowed_ip_ranges` (CIDRs) adds Allow rules, and everyone else gets a 403 from Azure before reaching Pixel. The space state service also runs on Container Apps Consumption, whose outbound addresses can change, so leaving it empty and relying on the signature is fine. -- **The URL** is the `spacestate_webhook_url` output: `https://./hooks/space-state`. It stays the same across revisions and deploys. +- **The URL** is the `spacestate_webhook_url` output: `https://./hooks/space-state`, or `https://pixel.pixelbar.nl/hooks/space-state` once the custom domain is bound. Both stay the same across revisions and deploys; only the custom domain survives the environment being recreated. +- **Custom domain** (`infra/modules/pixel/domain.tf`): `custom_domain` (prod `pixel.pixelbar.nl`, dev e.g. `pixel-dev.pixelbar.nl`) binds a hostname to the same ingress with a free Azure-managed certificate that Azure renews itself. pixelbar.nl's DNS is at **TransIP**, not Azure, so it takes two applies: the first prints the records (`custom_domain_dns_records`: TXT `asuid.` with the environment's verification ID, CNAME `` to the app's own address), and once they resolve `bind_custom_domain = true` issues the certificate (CNAME validation) and binds it. It needs `spacestate_webhook_enabled`. Steps: runbook, "Give the webhook a hostname". - **No effect on scaling or deploys.** min=max=1 still holds; ingress just routes to whichever revision is ready. A webhook lost during a revision swap is caught by the next poll. ## What is not in Azure diff --git a/docs/runbook.md b/docs/runbook.md index 9ed1cc2..c73c438 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -334,6 +334,18 @@ Without it, Pixel notices an open or close by polling SpaceAPI every 30 s and an 4. Give the space state service that URL and the same secret, so it signs each call as ADR 0018 describes. 5. Check: Pixel's log says `webhook.received` on the next open or close, followed by `spaceapi.changed` with `via: "push"`. Startup logs `webhook.off` when the secret isn't set. +**Give the webhook a hostname** (optional; prod `pixel.pixelbar.nl`, dev e.g. `pixel-dev.pixelbar.nl`). Not required for it to work, but the hostname is easier to hand out and survives the Container App environment being recreated, which would change the generated address. It needs the webhook turned on (above). pixelbar.nl's DNS is at TransIP, so it takes two applies: + +1. Set `custom_domain = "pixel.pixelbar.nl"` in the environment's `terraform.tfvars`, `just push `, plan and apply. Nothing is bound yet. +2. `terraform -chdir=infra/envs/ output custom_domain_dns_records` prints two records. At TransIP (pixelbar.nl → DNS), add them, with the part before `.pixelbar.nl` as the name: + - **TXT** `asuid.pixel` with the `txt.value` (the environment's verification ID; it isn't a secret). + - **CNAME** `pixel` with the `cname.value` and a trailing dot (`pixel-prod..westeurope.azurecontainerapps.io.`). +3. Wait until both resolve, for example `dig +short TXT asuid.pixel.pixelbar.nl` and `dig +short CNAME pixel.pixelbar.nl`. Applying before then fails, harmlessly. +4. Set `bind_custom_domain = true`, `just push `, plan and apply. The plan should add only a managed certificate and a custom domain. Issuing the certificate takes a few minutes. +5. `spacestate_webhook_url` now says `https://pixel.pixelbar.nl/hooks/space-state`. Give the space state service that address. `curl -i https://pixel.pixelbar.nl/hooks/space-state` should answer `405` (only POST is allowed), with a valid certificate. The old address keeps working, so nothing breaks while the service changes over. + +Azure renews the certificate itself as long as the CNAME stays in place. Don't delete the TXT or CNAME record while the domain is bound. To remove the hostname, set `bind_custom_domain = false` and apply, then delete the records. + **Locally:** set `SPACESTATE_WEBHOOK_SECRET` in `.env` and Pixel listens on `WEBHOOK_PORT` (default 8081). Try it with a signed call: ```sh diff --git a/infra/envs/dev/README.md b/infra/envs/dev/README.md index fcf17ed..446c46d 100644 --- a/infra/envs/dev/README.md +++ b/infra/envs/dev/README.md @@ -16,7 +16,7 @@ Inside `pixel-dev` (already exists; not recreated): | Key Vault (RBAC, no secret values in state) | `pixel-dev-kv` | | Log Analytics (30-day retention; see [`docs/privacy.md`](../../../docs/privacy.md)) | `pixel-dev-logs` | | Container Apps environment | `pixel-dev-cae` | -| Container App, min=max replicas **1**, no public ingress (unless `spacestate_webhook_enabled`: HTTPS to the webhook port only) | `pixel-dev` | +| Container App, min=max replicas **1**, no public ingress (unless `spacestate_webhook_enabled`: HTTPS to the webhook port only, optionally on `custom_domain`) | `pixel-dev` | | Azure Files account + share `pixel` | `pixeldevdata` | | Daily file-share backup, 14-day keep | `pixel-dev-rsv` | diff --git a/infra/envs/dev/main.tf b/infra/envs/dev/main.tf index 075b451..f8e676a 100644 --- a/infra/envs/dev/main.tf +++ b/infra/envs/dev/main.tf @@ -22,6 +22,8 @@ module "pixel" { spacestate_additional_headers_enabled = var.spacestate_additional_headers_enabled spacestate_webhook_enabled = var.spacestate_webhook_enabled webhook_allowed_ip_ranges = var.webhook_allowed_ip_ranges + custom_domain = var.custom_domain + bind_custom_domain = var.bind_custom_domain sentry_enabled = var.sentry_enabled tailscale_enabled = var.tailscale_enabled tailscale_image = var.tailscale_image diff --git a/infra/envs/dev/outputs.tf b/infra/envs/dev/outputs.tf index e45dc7a..08e63e9 100644 --- a/infra/envs/dev/outputs.tf +++ b/infra/envs/dev/outputs.tf @@ -35,3 +35,8 @@ output "spacestate_webhook_url" { value = module.pixel.spacestate_webhook_url description = "Where the space state service sends its webhook, or null when it is off." } + +output "custom_domain_dns_records" { + value = module.pixel.custom_domain_dns_records + description = "The TXT and CNAME records to create at the DNS host for custom_domain, or null when it isn't set." +} diff --git a/infra/envs/dev/terraform.example.tfvars b/infra/envs/dev/terraform.example.tfvars index b50c19d..272de33 100644 --- a/infra/envs/dev/terraform.example.tfvars +++ b/infra/envs/dev/terraform.example.tfvars @@ -43,6 +43,10 @@ discord_guild_id = "100000000000000002" # Vault secret spacestate-webhook-secret first. The URL is the spacestate_webhook_url output. # spacestate_webhook_enabled = true # webhook_allowed_ip_ranges = ["203.0.113.7/32"] # optional; empty lets anyone try (signed requests only) +# A hostname for it, in two applies: set custom_domain and apply, create the records from +# the custom_domain_dns_records output at TransIP, then set bind_custom_domain and apply. +# custom_domain = "pixel-dev.pixelbar.nl" +# bind_custom_domain = true sentry_enabled = false tailscale_enabled = false diff --git a/infra/envs/dev/variables.tf b/infra/envs/dev/variables.tf index 1f3edc1..07f664d 100644 --- a/infra/envs/dev/variables.tf +++ b/infra/envs/dev/variables.tf @@ -146,6 +146,18 @@ variable "webhook_allowed_ip_ranges" { description = "Optional. CIDR ranges allowed to reach the webhook. Empty allows anyone; requests must still be signed." } +variable "custom_domain" { + type = string + default = null + description = "Optional. A hostname for the webhook ingress, such as pixel.pixelbar.nl. Needs spacestate_webhook_enabled. Set it, apply, create the records from custom_domain_dns_records, then set bind_custom_domain." +} + +variable "bind_custom_domain" { + type = bool + default = false + description = "Bind custom_domain with a managed certificate. Only once its DNS records resolve." +} + variable "tailscale_auth_key" { type = string ephemeral = true diff --git a/infra/envs/prod/README.md b/infra/envs/prod/README.md index f84f023..d5c9dca 100644 --- a/infra/envs/prod/README.md +++ b/infra/envs/prod/README.md @@ -18,7 +18,7 @@ Inside `pixel-prod` (already exists; not recreated): | Key Vault (RBAC, purge protection **on**) | `pixel-prod-kv` | | Log Analytics (30-day retention, pending #7) | `pixel-prod-logs` | | Container Apps environment | `pixel-prod-cae` | -| Container App, min=max replicas **1**, no public ingress (unless `spacestate_webhook_enabled`: HTTPS to the webhook port only) | `pixel-prod` | +| Container App, min=max replicas **1**, no public ingress (unless `spacestate_webhook_enabled`: HTTPS to the webhook port only, optionally on `custom_domain`) | `pixel-prod` | | Azure Files account + share `pixel` | `pixelproddata` | | Daily file-share backup, 14-day keep | `pixel-prod-rsv` | diff --git a/infra/envs/prod/main.tf b/infra/envs/prod/main.tf index 7641a56..5439eca 100644 --- a/infra/envs/prod/main.tf +++ b/infra/envs/prod/main.tf @@ -22,6 +22,8 @@ module "pixel" { spacestate_additional_headers_enabled = var.spacestate_additional_headers_enabled spacestate_webhook_enabled = var.spacestate_webhook_enabled webhook_allowed_ip_ranges = var.webhook_allowed_ip_ranges + custom_domain = var.custom_domain + bind_custom_domain = var.bind_custom_domain sentry_enabled = var.sentry_enabled tailscale_enabled = var.tailscale_enabled tailscale_image = var.tailscale_image diff --git a/infra/envs/prod/outputs.tf b/infra/envs/prod/outputs.tf index e45dc7a..08e63e9 100644 --- a/infra/envs/prod/outputs.tf +++ b/infra/envs/prod/outputs.tf @@ -35,3 +35,8 @@ output "spacestate_webhook_url" { value = module.pixel.spacestate_webhook_url description = "Where the space state service sends its webhook, or null when it is off." } + +output "custom_domain_dns_records" { + value = module.pixel.custom_domain_dns_records + description = "The TXT and CNAME records to create at the DNS host for custom_domain, or null when it isn't set." +} diff --git a/infra/envs/prod/terraform.example.tfvars b/infra/envs/prod/terraform.example.tfvars index 06fe429..02a9226 100644 --- a/infra/envs/prod/terraform.example.tfvars +++ b/infra/envs/prod/terraform.example.tfvars @@ -33,6 +33,10 @@ discord_guild_id = "100000000000000002" # Vault secret spacestate-webhook-secret first. The URL is the spacestate_webhook_url output. # spacestate_webhook_enabled = true # webhook_allowed_ip_ranges = ["203.0.113.7/32"] # optional; empty lets anyone try (signed requests only) +# A hostname for it, in two applies: set custom_domain and apply, create the records from +# the custom_domain_dns_records output at TransIP, then set bind_custom_domain and apply. +# custom_domain = "pixel.pixelbar.nl" +# bind_custom_domain = true # Lets the Release workflow read discord-token (only) and register slash # commands (ADR 0014). az identity show -g pixel-bootstrap -n pixel-github-prod --query principalId -o tsv diff --git a/infra/envs/prod/variables.tf b/infra/envs/prod/variables.tf index 1e6987d..c08a84e 100644 --- a/infra/envs/prod/variables.tf +++ b/infra/envs/prod/variables.tf @@ -146,6 +146,18 @@ variable "webhook_allowed_ip_ranges" { description = "Optional. CIDR ranges allowed to reach the webhook. Empty allows anyone; requests must still be signed." } +variable "custom_domain" { + type = string + default = null + description = "Optional. A hostname for the webhook ingress, such as pixel.pixelbar.nl. Needs spacestate_webhook_enabled. Set it, apply, create the records from custom_domain_dns_records, then set bind_custom_domain." +} + +variable "bind_custom_domain" { + type = bool + default = false + description = "Bind custom_domain with a managed certificate. Only once its DNS records resolve." +} + variable "tailscale_auth_key" { type = string ephemeral = true diff --git a/infra/modules/pixel/domain.tf b/infra/modules/pixel/domain.tf new file mode 100644 index 0000000..36062ba --- /dev/null +++ b/infra/modules/pixel/domain.tf @@ -0,0 +1,39 @@ +# A custom hostname for the webhook ingress (pixel.pixelbar.nl for prod), with an +# Azure-managed certificate. pixelbar.nl's DNS is at TransIP, outside Azure, so it takes +# two applies (ADR 0018): +# 1. custom_domain set, bind_custom_domain = false: read custom_domain_dns_records and +# create the TXT and CNAME records at TransIP. +# 2. Once they resolve, bind_custom_domain = true: Azure issues the certificate (CNAME +# validation, the same record the binding needs) and binds the hostname. +# The default *.azurecontainerapps.io address keeps working throughout. + +locals { + # Whichever of the two app resources this stack has (dev ignores the image, prod doesn't). + container_app_id = var.environment == "dev" ? azurerm_container_app.this[0].id : azurerm_container_app.managed[0].id + + # The app's own address: the CNAME target, and the webhook URL until the domain is bound. + default_fqdn = "${local.name_prefix}.${azurerm_container_app_environment.this.default_domain}" +} + +resource "azurerm_container_app_environment_managed_certificate" "custom_domain" { + count = var.bind_custom_domain ? 1 : 0 + + name = replace(var.custom_domain, ".", "-") + container_app_environment_id = azurerm_container_app_environment.this.id + subject_name = var.custom_domain + domain_control_validation = "CNAME" + tags = local.tags +} + +resource "azurerm_container_app_custom_domain" "this" { + count = var.bind_custom_domain ? 1 : 0 + + name = var.custom_domain + container_app_id = local.container_app_id + container_app_environment_certificate_id = azurerm_container_app_environment_managed_certificate.custom_domain[0].id + certificate_binding_type = "SniEnabled" + + # The certificate reference already orders these, but issuance has been seen to race + # the binding (pixelbar/spacestate hit it), so say it explicitly. + depends_on = [azurerm_container_app_environment_managed_certificate.custom_domain] +} diff --git a/infra/modules/pixel/outputs.tf b/infra/modules/pixel/outputs.tf index 8dec687..7cc93f9 100644 --- a/infra/modules/pixel/outputs.tf +++ b/infra/modules/pixel/outputs.tf @@ -49,6 +49,23 @@ output "tailscale_enabled" { output "spacestate_webhook_url" { # Built from the environment's domain, like container_app_name, rather than read from # the sensitive app object. External ingress is always .. - value = var.spacestate_webhook_enabled ? "https://${local.name_prefix}.${azurerm_container_app_environment.this.default_domain}/hooks/space-state" : null + # Once a custom domain is bound, that hostname; until then the app's own address. + value = var.spacestate_webhook_enabled ? "https://${var.bind_custom_domain ? var.custom_domain : local.default_fqdn}/hooks/space-state" : null description = "Where the space state service sends its webhook, or null when it is off. Give it this URL and the spacestate-webhook-secret value." } + +output "custom_domain_dns_records" { + # The verification ID isn't a secret: it ends up in public DNS. + value = var.custom_domain == null ? null : { + txt = { + name = "asuid.${var.custom_domain}" + value = azurerm_container_app_environment.this.custom_domain_verification_id + } + cname = { + name = var.custom_domain + value = local.default_fqdn + } + bound = var.bind_custom_domain + } + description = "The records to create at the DNS host (TransIP for pixelbar.nl) before setting bind_custom_domain, or null without custom_domain. A subdomain needs the CNAME; an apex domain can't have one." +} diff --git a/infra/modules/pixel/variables.tf b/infra/modules/pixel/variables.tf index 3d3d038..235a9e3 100644 --- a/infra/modules/pixel/variables.tf +++ b/infra/modules/pixel/variables.tf @@ -209,6 +209,33 @@ variable "webhook_allowed_ip_ranges" { } } +variable "custom_domain" { + type = string + default = null + description = "Optional. A hostname for the webhook ingress, such as pixel.pixelbar.nl (prod). Needs spacestate_webhook_enabled. Setting it only prints the DNS records to create (custom_domain_dns_records); bind_custom_domain binds it." + + validation { + condition = var.custom_domain == null || can(regex("^([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z]{2,}$", var.custom_domain)) + error_message = "custom_domain must be a lower-case hostname like pixel.pixelbar.nl, without a scheme or path." + } + + validation { + condition = var.custom_domain == null || var.spacestate_webhook_enabled + error_message = "custom_domain needs spacestate_webhook_enabled: without ingress there is nothing to bind it to." + } +} + +variable "bind_custom_domain" { + type = bool + default = false + description = "Second apply of the custom domain: issue the managed certificate and bind custom_domain. Turn on only once the TXT and CNAME records from custom_domain_dns_records resolve, or the apply fails." + + validation { + condition = !var.bind_custom_domain || var.custom_domain != null + error_message = "bind_custom_domain needs custom_domain." + } +} + variable "tailscale_auth_key" { type = string ephemeral = true